def tokenize(self, string):
"""Tokenize incoming string."""
#punkt = WhitespaceTokenizer()
punkt= PunktLanguageVars()
generic_tokens = punkt.word_tokenize(string)
generic_tokens = [x for item in generic_tokens for x in ([item] if item != 'nec' else ['c', 'ne'])] # Handle 'nec' as a special case.
specific_tokens = []
for generic_token in generic_tokens:
is_enclitic = False
if generic_token not in self.exceptions:
for enclitic in self.enclitics:
if generic_token.endswith(enclitic):
if enclitic == 'cum':
if generic_token in self.inclusions:
specific_tokens += [enclitic] + [generic_token[:-len(enclitic)]]
else:
specific_tokens += [generic_token]
elif enclitic == 'st':
if generic_token.endswith('ust'):
specific_tokens += [generic_token[:-len(enclitic)+1]] + ['est']
else:
# Does not handle 'similist', 'qualist', etc. correctly
specific_tokens += [generic_token[:-len(enclitic)]] + ['est']
else:
specific_tokens += [enclitic] + [generic_token[:-len(enclitic)]]
is_enclitic = True
break
if not is_enclitic:
specific_tokens.append(generic_token)
#return iter(specific_tokens) #change this one into an iterator.
startPoint=0 #this is to accumulate the start point.
for item in specific_tokens:
itemLength=len(item)
yield item, startPoint, startPoint+itemLength
startPoint=startPoint+itemLength+1
def _tokenize(self, text):
"""
Use NLTK's standard tokenizer, rm punctuation.
:param text: pre-processed text
:return: tokenized text
:rtype : list
"""
sentence_tokenizer = TokenizeSentence('latin')
sentences = sentence_tokenizer.tokenize_sentences(text.lower())
sent_words = []
punkt = PunktLanguageVars()
for sentence in sentences:
words = punkt.word_tokenize(sentence)
assert isinstance(words, list)
words_new = []
for word in words:
if word not in self.punctuation or self.abbreviations or self.numbers or self.abbreviations: # pylint: disable=line-too-long
words_new.append(word)
# rm all numbers here with: re.compose(r'[09]')
sent_words.append(words_new)
return sent_words
def _tokenize(self, text):
"""
Use NLTK's standard tokenizer, rm punctuation.
:param text: pre-processed text
:return: tokenized text
:rtype : list
"""
sentence_tokenizer = TokenizeSentence('latin')
sentences = sentence_tokenizer.tokenize_sentences(text.lower())
sent_words = []
punkt = PunktLanguageVars()
for sentence in sentences:
words = punkt.word_tokenize(sentence)
assert isinstance(words, list)
words_new = []
for word in words:
if word not in self.punctuation or self.abbreviations or self.numbers or self.abbreviations: # pylint: disable=line-too-long
words_new.append(word)
# rm all numbers here with: re.compose(r'[09]')
sent_words.append(words_new)
return sent_words
def tag_ner(lang, input_text, output_type=list):
"""Run NER for chosen language.
"""
_check_latest_data(lang)
assert lang in NER_DICT.keys(), "Invalid language. Choose from: {}".format(
", ".join(NER_DICT.keys()))
types = [str, list]
assert type(input_text) in types, "Input must be: {}.".format(
", ".join(types))
assert output_type in types, "Output must be a {}.".format(
", ".join(types))
if type(input_text) == str:
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(input_text)
new_tokens = []
for word in tokens:
if word.endswith("."):
new_tokens.append(word[:-1])
new_tokens.append(".")
else:
new_tokens.append(word)
input_text = new_tokens
ner_file_path = os.path.expanduser(NER_DICT[lang])
with open(ner_file_path) as file_open:
ner_str = file_open.read()
ner_list = ner_str.split("\n")
ner_tuple_list = []
for count, word_token in enumerate(input_text):
match = False
for ner_word in ner_list:
# the replacer slows things down, but is necessary
if word_token == ner_word:
ner_tuple = (word_token, "Entity")
ner_tuple_list.append(ner_tuple)
match = True
break
if not match:
ner_tuple_list.append((word_token, ))
if output_type is str:
string = ""
for tup in ner_tuple_list:
start_space = " "
final_space = ""
# this is some mediocre string reconstitution
# maybe not worth the effort
if tup[0] in [",", ".", ";", ":", "?", "!"]:
start_space = ""
if len(tup) == 2:
string += start_space + tup[0] + "/" + tup[1] + final_space
else:
string += start_space + tup[0] + final_space
return string
return ner_tuple_list
def nltk_tokenize_words(string, attached_period=False, language=None):
"""Wrap NLTK's tokenizer PunktLanguageVars(), but make final period
its own token.
>>> nltk_punkt("Sentence 1. Sentence 2.")
>>> ['Sentence', 'one', '.', 'Sentence', 'two', '.']
Optionally keep the NLTK's output:
>>> nltk_punkt("Sentence 1. Sentence 2.", attached_period=True)
>>> ['Sentence', 'one.', 'Sentence', 'two.']
TODO: Run some tests to determine whether there is a large penalty for
re-calling PunktLanguageVars() for each use of this function. If so, this
will need to become a class, perhaps inheriting from the PunktLanguageVars
object. Maybe integrate with WordTokenizer.
"""
assert isinstance(string, str), "Incoming string must be type str."
if language=='sanskrit':
periods = ['.', 'ред','рее']
else:
periods = ['.']
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(string)
if attached_period:
return tokens
new_tokens = []
for word in tokens:
for char in periods:
if word.endswith(char):
new_tokens.append(word[:-1])
new_tokens.append(char)
break
else:
new_tokens.append(word)
return new_tokens
def tokenize(self, string):
"""Tokenize incoming string."""
punkt = PunktLanguageVars()
generic_tokens = punkt.word_tokenize(string)
# Rewrite as an if-else block for exceptions rather than separate list comprehensions
generic_tokens = [x for item in generic_tokens for x in ([item] if item != 'nec' else ['c', 'ne'])] # Handle 'nec' as a special case.
generic_tokens = [x for item in generic_tokens for x in ([item] if item != 'sodes' else ['si', 'audes'])] # Handle 'sodes' as a special case.
generic_tokens = [x for item in generic_tokens for x in ([item] if item != 'sultis' else ['si', 'vultis'])] # Handle 'sultis' as a special case.
specific_tokens = []
for generic_token in generic_tokens:
is_enclitic = False
if generic_token not in self.exceptions:
for enclitic in self.enclitics:
if generic_token.endswith(enclitic):
if enclitic == 'cum':
if generic_token in self.inclusions:
specific_tokens += [enclitic] + [generic_token[:-len(enclitic)]]
else:
specific_tokens += [generic_token]
elif enclitic == 'st':
if generic_token.endswith('ust'):
specific_tokens += [generic_token[:-len(enclitic)+1]] + ['est']
else:
# Does not handle 'similist', 'qualist', etc. correctly
specific_tokens += [generic_token[:-len(enclitic)]] + ['est']
else:
specific_tokens += [enclitic] + [generic_token[:-len(enclitic)]]
is_enclitic = True
break
if not is_enclitic:
specific_tokens.append(generic_token)
return specific_tokens
def rm_stopwords(self, stoplist=[]):
"""Removes words or phrases from the text.
Given a list of words or phrases, gives new text with those phrases
removed.
Args:
stoplist (:obj:`list`) List of words or phrases to filter from text
Returns:
:obj:`self.__class__` New version of text, with stop words/phrases removed
Example:
>>> stopwords = ['ipsum', 'sit']
>>> text = EnglishText('Lorem ipsum dolor sit amet...')
>>> text.rm_stopwords(stoplist=stopwords)
>>> print(modified_text)
'Lorem dolor amet...'
""" # noqa
filtered_words = []
# converts text to list of words with NLTK tokenizer
tokenizer = PunktLanguageVars()
tokens = tokenizer.word_tokenize(str(self.data))
# loop through each word, if not in stoplist, append
for word in tokens:
not_found = True
for stopword in stoplist:
if str(word).strip().lower() == str(stopword).strip().lower():
not_found = False
if not_found:
filtered_words.append(word)
# return rejoined word
return self.__class__(" ".join(filtered_words), self.options)
def nltk_tokenize_words(string, attached_period=False, language=None):
"""Wrap NLTK's tokenizer PunktLanguageVars(), but make final period
its own token.
>>> nltk_punkt("Sentence 1. Sentence 2.")
>>> ['Sentence', 'one', '.', 'Sentence', 'two', '.']
Optionally keep the NLTK's output:
>>> nltk_punkt("Sentence 1. Sentence 2.", attached_period=True)
>>> ['Sentence', 'one.', 'Sentence', 'two.']
TODO: Run some tests to determine whether there is a large penalty for
re-calling PunktLanguageVars() for each use of this function. If so, this
will need to become a class, perhaps inheriting from the PunktLanguageVars
object. Maybe integrate with WordTokenizer.
"""
assert isinstance(string, str), "Incoming string must be type str."
if language=='sanskrit':
periods = ['.', 'ред','рее']
else:
periods = ['.']
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(string)
if attached_period:
return tokens
new_tokens = []
for word in tokens:
for char in periods:
if word.endswith(char):
new_tokens.append(word[:-1])
new_tokens.append(char)
break
else:
new_tokens.append(word)
return new_tokens
def tokenize(self, string):
"""Tokenize incoming string."""
punkt = PunktLanguageVars()
generic_tokens = punkt.word_tokenize(string)
generic_tokens = [x for item in generic_tokens for x in ([item] if item != 'nec' else ['c', 'ne'])] # Handle 'nec' as a special case.
specific_tokens = []
for generic_token in generic_tokens:
is_enclitic = False
if generic_token not in self.exceptions:
for enclitic in self.enclitics:
if generic_token.endswith(enclitic):
if enclitic == 'mst':
specific_tokens += [generic_token[:-len(enclitic)+1]] + ['e'+ generic_token[-len(enclitic)+1:]]
elif enclitic == 'cum':
if generic_token in self.inclusions:
specific_tokens += [enclitic] + [generic_token[:-len(enclitic)]]
else:
specific_tokens += [generic_token]
else:
specific_tokens += [enclitic] + [generic_token[:-len(enclitic)]]
is_enclitic = True
break
if not is_enclitic:
specific_tokens.append(generic_token)
return specific_tokens
def __init__(self):
"""Language taken as argument, necessary used when saving word frequencies to
``cltk_data/user_data``."""
self.punkt = PunktLanguageVars()
self.punctuation = [
',', '.', ';', ':', '"', "'", '?', '-', '!', '*', '[', ']', '{',
'}'
]
def tokenize(desc):
'''
INPUT: List of cleaned descriptions
OUTPUT: Tokenized and stemmed list of words from the descriptions
'''
plv = PunktLanguageVars()
snowball = SnowballStemmer('english')
return [snowball.stem(word) for word in plv.word_tokenize(desc.lower())]
def tokenize(doc):
'''
INPUT: Document
OUTPUT: Tokenized and stemmed list of words from the document
'''
plv = PunktLanguageVars()
snowball = SnowballStemmer('english')
return [snowball.stem(word) for word in plv.word_tokenize(doc.lower())]
def tokenize(desc):
'''
INPUT: List of cleaned descriptions
OUTPUT: Tokenized and stemmed list of words from the descriptions
'''
plv = PunktLanguageVars()
snowball = SnowballStemmer('english')
return [snowball.stem(word) for word in plv.word_tokenize(desc.lower())]
def tag_ner(lang, input_text, output_type=list):
"""Run NER for chosen language.
"""
_check_latest_data(lang)
assert lang in NER_DICT.keys(), \
'Invalid language. Choose from: {}'.format(', '.join(NER_DICT.keys()))
types = [str, list]
assert type(input_text) in types, 'Input must be: {}.'.format(', '.join(types))
assert output_type in types, 'Output must be a {}.'.format(', '.join(types))
if type(input_text) == str:
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(input_text)
new_tokens = []
for word in tokens:
if word.endswith('.'):
new_tokens.append(word[:-1])
new_tokens.append('.')
else:
new_tokens.append(word)
input_text = new_tokens
ner_file_path = os.path.expanduser(NER_DICT[lang])
with open(ner_file_path) as file_open:
ner_str = file_open.read()
ner_list = ner_str.split('\n')
ner_tuple_list = []
for count, word_token in enumerate(input_text):
match = False
for ner_word in ner_list:
# the replacer slows things down, but is necessary
if word_token == ner_word:
ner_tuple = (word_token, 'Entity')
ner_tuple_list.append(ner_tuple)
match = True
break
if not match:
ner_tuple_list.append((word_token,))
if output_type is str:
string = ''
for tup in ner_tuple_list:
start_space = ' '
final_space = ''
# this is some mediocre string reconstitution
# maybe not worth the effort
if tup[0] in [',', '.', ';', ':', '?', '!']:
start_space = ''
if len(tup) == 2:
string += start_space + tup[0] + '/' + tup[1] + final_space
else:
string += start_space + tup[0] + final_space
return string
return ner_tuple_list
def test_french_stopwords(self):
##test filtering French stopwords
sentence = "En pensé ai e en talant que d ’ Yonec vus die avant dunt il fu nez, e de sun pere cum il vint primes a sa mere ."
lowered = sentence.lower()
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(lowered)
no_stops = [w for w in tokens if w not in FRENCH_STOPS]
target_list = ['pensé', 'talant', 'd', '’', 'yonec', 'die', 'avant', 'dunt', 'nez', ',', 'pere', 'cum', 'primes',
'mere','.']
self.assertEqual(no_stops, target_list)
def test_latin_stopwords(self):
"""Test filtering Latin stopwords."""
sentence = 'Quo usque tandem abutere, Catilina, patientia nostra?'
lowered = sentence.lower()
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(lowered)
no_stops = [w for w in tokens if w not in LATIN_STOPS]
target_list = ['usque', 'tandem', 'abutere', ',', 'catilina', ',',
'patientia', 'nostra', '?']
self.assertEqual(no_stops, target_list)
def test_latin_stopwords(self):
"""Test filtering Latin stopwords."""
sentence = 'Quo usque tandem abutere, Catilina, patientia nostra?'
lowered = sentence.lower()
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(lowered)
no_stops = [w for w in tokens if w not in LATIN_STOPS]
target_list = ['usque', 'tandem', 'abutere', ',', 'catilina', ',',
'patientia', 'nostra', '?']
self.assertEqual(no_stops, target_list)
def test_french_stopwords(self):
##test filtering French stopwords
sentence = "En pensé ai e en talant que d ’ Yonec vus die avant dunt il fu nez, e de sun pere cum il vint primes a sa mere ."
lowered = sentence.lower()
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(lowered)
no_stops = [w for w in tokens if w not in FRENCH_STOPS]
target_list = ['pensé', 'talant', 'd', '’', 'yonec', 'die', 'avant', 'dunt', 'nez', ',', 'pere', 'cum', 'primes',
'mere','.']
self.assertEqual(no_stops, target_list)
def preprocess(self, text):
if self.token:
lemma = self.lemmatizer.lemmatize(text)
else:
plv = PunktLanguageVars()
unigrams = plv.word_tokenize(text)
lemma = self.lemmatizer.lemmatize(unigrams)
lemma = [t[0] if t[1] == "punc" else t[1] for t in lemma]
return " ".join(lemma)
def test_old_norse_stopwords(self):
"""
Test filtering Old Norse stopwords
Sentence extracted from Eiríks saga rauða (http://www.heimskringla.no/wiki/Eir%C3%ADks_saga_rau%C3%B0a)
"""
sentence = 'Þat var einn morgin, er þeir Karlsefni sá fyrir ofan rjóðrit flekk nökkurn, sem glitraði við þeim'
lowered = sentence.lower()
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(lowered)
no_stops = [w for w in tokens if w not in OLD_NORSE_STOPS]
target_list = ['var', 'einn', 'morgin', ',', 'karlsefni', 'rjóðrit', 'flekk', 'nökkurn', ',', 'glitraði']
self.assertEqual(no_stops, target_list)
def test_old_norse_stopwords(self):
"""
Test filtering Old Norse stopwords
Sentence extracted from Eiríks saga rauða (http://www.heimskringla.no/wiki/Eir%C3%ADks_saga_rau%C3%B0a)
"""
sentence = 'Þat var einn morgin, er þeir Karlsefni sá fyrir ofan rjóðrit flekk nökkurn, sem glitraði við þeim'
lowered = sentence.lower()
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(lowered)
no_stops = [w for w in tokens if w not in OLD_NORSE_STOPS]
target_list = ['var', 'einn', 'morgin', ',', 'karlsefni', 'rjóðrit', 'flekk', 'nökkurn', ',', 'glitraði']
self.assertEqual(no_stops, target_list)
def test_greek_stopwords(self):
"""Test filtering Greek stopwords."""
sentence = 'Ἅρπαγος δὲ καταστρεψάμενος Ἰωνίην ἐποιέετο στρατηίην \
ἐπὶ Κᾶρας καὶ Καυνίους καὶ Λυκίους, ἅμα ἀγόμενος καὶ Ἴωνας καὶ \
Αἰολέας.'
lowered = sentence.lower()
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(lowered)
no_stops = [w for w in tokens if w not in GREEK_STOPS]
target_list = ['ἅρπαγος', 'καταστρεψάμενος', 'ἰωνίην', 'ἐποιέετο',
'στρατηίην', 'κᾶρας', 'καυνίους', 'λυκίους', ',',
'ἅμα', 'ἀγόμενος', 'ἴωνας', 'αἰολέας.']
self.assertEqual(no_stops, target_list)
def test_greek_stopwords(self):
"""Test filtering Greek stopwords."""
sentence = 'Ἅρπαγος δὲ καταστρεψάμενος Ἰωνίην ἐποιέετο στρατηίην \
ἐπὶ Κᾶρας καὶ Καυνίους καὶ Λυκίους, ἅμα ἀγόμενος καὶ Ἴωνας καὶ \
Αἰολέας.'
lowered = sentence.lower()
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(lowered)
no_stops = [w for w in tokens if w not in GREEK_STOPS]
target_list = ['ἅρπαγος', 'καταστρεψάμενος', 'ἰωνίην', 'ἐποιέετο',
'στρατηίην', 'κᾶρας', 'καυνίους', 'λυκίους', ',',
'ἅμα', 'ἀγόμενος', 'ἴωνας', 'αἰολέας.']
self.assertEqual(no_stops, target_list)
def lemmatize(self, input_text, return_raw=False, return_string=False):
"""Take incoming string or list of tokens. Lookup done against a
key-value list of lemmata-headword. If a string, tokenize with
``PunktLanguageVars()``. If a final period appears on a token, remove
it, then re-add once replacement done.
TODO: rm check for final period, change PunktLanguageVars()
"""
assert type(input_text) in [
list, str
], logger.error("Input must be a list or string.")
if type(input_text) is str:
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(input_text)
else:
tokens = input_text
lemmatized_tokens = []
for token in tokens:
# check for final period
final_period = False
if token[-1] == ".":
final_period = True
token = token[:-1]
# look for token in lemma dict keys
if token.lower() in self.lemmata.keys():
headword = self.lemmata[token.lower()]
# re-add final period if rm'd
if final_period:
headword += "."
# append to return list
if not return_raw:
lemmatized_tokens.append(headword)
else:
lemmatized_tokens.append(token + "/" + headword)
# if token not found in lemma-headword list
else:
# re-add final period if rm'd
if final_period:
token += "."
if not return_raw:
lemmatized_tokens.append(token)
else:
lemmatized_tokens.append(token + "/" + token)
if not return_string:
return lemmatized_tokens
elif return_string:
return " ".join(lemmatized_tokens)
def test_akkadian_stopwords(self):
"""
Test filtering Akkadian stopwrods
Sentence extracted from the law code of Hammurabi, law 3 (Martha Roth 2nd Edition 1997, Law Collections from
Mesopotamia and Asia Minor).
"""
sentence = "šumma awīlum ina dīnim ana šībūt sarrātim ūṣiamma awat iqbû la uktīn šumma dīnum šû dīn napištim awīlum šû iddâk"
lowered = sentence.lower()
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(lowered)
no_stops = [w for w in tokens if w not in AKKADIAN_STOPS]
target_list = ['awīlum', 'dīnim', 'šībūt', 'sarrātim', 'ūṣiamma', 'awat', 'iqbû', 'uktīn', 'dīnum',
'dīn', 'napištim', 'awīlum', 'iddâk']
self.assertEqual(no_stops, target_list)
def preprocess(self, text):
if self.path:
with open(self.path, "r") as f:
sw = set(f.read().split("\n")[:-1])
else:
sw = set(stopwords.words(self.lang))
if self.token:
unigrams = text
else:
plv = PunktLanguageVars()
unigrams = plv.word_tokenize(text.lower())
return " ".join([unigram for unigram in unigrams if not unigram in sw])
def lemmatize(self, input_text, return_raw=False, return_string=False):
"""Take incoming string or list of tokens. Lookup done against a
key-value list of lemmata-headword. If a string, tokenize with
``PunktLanguageVars()``. If a final period appears on a token, remove
it, then re-add once replacement done.
TODO: rm check for final period, change PunktLanguageVars() to nltk_tokenize_words()
"""
assert type(input_text) in [list, str], \
logger.error('Input must be a list or string.')
if type(input_text) is str:
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(input_text)
else:
tokens = input_text
lemmatized_tokens = []
for token in tokens:
# check for final period
final_period = False
if token[-1] == '.':
final_period = True
token = token[:-1]
# look for token in lemma dict keys
if token in self.lemmata.keys():
headword = self.lemmata[token.lower()]
# re-add final period if rm'd
if final_period:
headword += '.'
# append to return list
if not return_raw:
lemmatized_tokens.append(headword)
else:
lemmatized_tokens.append(token + '/' + headword)
# if token not found in lemma-headword list
else:
# re-add final period if rm'd
if final_period:
token += '.'
if not return_raw:
lemmatized_tokens.append(token)
else:
lemmatized_tokens.append(token + '/' + token)
if not return_string:
return lemmatized_tokens
elif return_string:
return ' '.join(lemmatized_tokens)
def test_akkadian_stopwords(self):
"""
Test filtering Akkadian stopwrods
Sentence extracted from the law code of Hammurabi, law 3 (Martha Roth 2nd Edition 1997, Law Collections from
Mesopotamia and Asia Minor).
"""
sentence = "šumma awīlum ina dīnim ana šībūt sarrātim ūṣiamma awat iqbû la uktīn šumma dīnum šû dīn napištim awīlum šû iddâk"
lowered = sentence.lower()
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(lowered)
no_stops = [w for w in tokens if w not in AKKADIAN_STOPS]
target_list = ['awīlum', 'dīnim', 'šībūt', 'sarrātim', 'ūṣiamma', 'awat', 'iqbû', 'uktīn', 'dīnum',
'dīn', 'napištim', 'awīlum', 'iddâk']
self.assertEqual(no_stops, target_list)
def rm_stopwords(self, stoplist=[]):
filtered_words = []
# converts text to list of words with NLTK tokenizer
tokenizer = PunktLanguageVars()
tokens = tokenizer.word_tokenize(str(self.data))
# loop through each word, if not in stoplist, append
for word in tokens:
not_found = True
for stopword in stoplist:
if str(word).strip().lower() == str(stopword).strip().lower():
not_found = False
if not_found:
filtered_words.append(word)
# return rejoined word
return self.__class__(" ".join(filtered_words), self.metadata)
def _build_concordance(self, text_str):
"""
Inherit or mimic the logic of ConcordanceIndex() at http://www.nltk.org/_modules/nltk/text.html
and/or ConcordanceSearchView() & SearchCorpus() at https://github.com/nltk/nltk/blob/develop/nltk/app/concordance_app.py
:param text_string: Text to be turned into a concordance
:type text_string: str
:return: list
"""
p = PunktLanguageVars()
orig_tokens = p.word_tokenize(text_str)
c = ConcordanceIndex(orig_tokens)
#! rm dupes after index, before loop
tokens = set(orig_tokens)
tokens = [x for x in tokens if x not in [',', '.', ';', ':', '"', "'", '[', ']']] # this needs to be changed or rm'ed
return c.return_concordance_all(tokens)
def tokenize(self, string):
"""Tokenize incoming string."""
punkt = PunktLanguageVars()
generic_tokens = punkt.word_tokenize(string)
specific_tokens = []
for generic_token in generic_tokens:
is_enclitic = False
if generic_token not in self.exceptions:
for enclitic in self.enclitics:
if generic_token.endswith(enclitic):
new_tokens = [generic_token[:-len(enclitic)]] + ['-' + enclitic]
specific_tokens += new_tokens
is_enclitic = True
break
if not is_enclitic:
specific_tokens.append(generic_token)
return specific_tokens
def tokenize(self, string):
"""Tokenize incoming string."""
punkt = PunktLanguageVars()
generic_tokens = punkt.word_tokenize(string)
specific_tokens = []
for generic_token in generic_tokens:
is_enclitic = False
if generic_token not in self.exceptions:
for enclitic in self.enclitics:
if generic_token.endswith(enclitic):
new_tokens = [generic_token[:-len(enclitic)]
] + ['-' + enclitic]
specific_tokens += new_tokens
is_enclitic = True
break
if not is_enclitic:
specific_tokens.append(generic_token)
return specific_tokens
def build_concordance(text_str: str) -> List[List[str]]:
"""
Inherit or mimic the logic of ConcordanceIndex() at:
http://www.nltk.org/_modules/nltk/text.html
and/or ConcordanceSearchView() & SearchCorpus() at:
https://github.com/nltk/nltk/blob/develop/nltk/app/concordance_app.py
:param text_string: Text to be turned into a concordance
:type text_string: str
:return: list
"""
punkt_vars = PunktLanguageVars() # type: PunktLanguageVars
orig_tokens = punkt_vars.word_tokenize(text_str) # type: List[str]
concordance_index = ConcordanceIndex(orig_tokens) # type: Any
#! rm dupes after index, before loop
tokens_set = set(orig_tokens) # type: Set[str]
punct_list = [',', '.', ';', ':', '"', "'", '[', ']'] # type: List[str]
# this needs to be changed or rm'ed
tokens = [x for x in tokens_set if x not in punct_list] # List[str]
index = concordance_index.return_concordance_all(tokens) # List[List[str]]
return index
def _build_concordance(self, text_str):
"""
Inherit or mimic the logic of ConcordanceIndex() at http://www.nltk.org/_modules/nltk/text.html
and/or ConcordanceSearchView() & SearchCorpus() at https://github.com/nltk/nltk/blob/develop/nltk/app/concordance_app.py
:param text_string: Text to be turned into a concordance
:type text_string: str
:return: list
"""
p = PunktLanguageVars()
orig_tokens = p.word_tokenize(text_str)
c = ConcordanceIndex(orig_tokens)
#! rm dupes after index, before loop
tokens = set(orig_tokens)
tokens = [
x for x in tokens
if x not in [',', '.', ';', ':', '"', "'", '[', ']']
] # this needs to be changed or rm'ed
return c.return_concordance_all(tokens)
def tokenize(self, string):
"""Tokenize incoming string."""
punkt = PunktLanguageVars()
generic_tokens = punkt.word_tokenize(string)
# Rewrite as an if-else block for exceptions rather than separate list comprehensions
generic_tokens = [x for item in generic_tokens for x in ([item] if item.lower() != 'nec' else ['c', item[:-1]])] # Handle 'nec' as a special case.
generic_tokens = [x for item in generic_tokens for x in ([item] if item.lower() != 'sodes' else [item[0]+'i', 'audes'])] # Handle 'sodes' as a special case.
generic_tokens = [x for item in generic_tokens for x in ([item] if item.lower() != 'sultis' else [item[0]+'i', 'vultis'])] # Handle 'sultis' as a special case.
generic_tokens = [x for item in generic_tokens for x in ([item] if item.lower() != 'satin' else [item[:-1] + 's', 'ne'])] # Handle 'satin' as a special case.
generic_tokens = [x for item in generic_tokens for x in ([item] if item.lower() != 'scin' else [item[:-1] + 's', 'ne'])] # Handle 'scin' as a special case.
specific_tokens = []
for generic_token in generic_tokens:
is_enclitic = False
if generic_token not in self.exceptions:
for enclitic in self.enclitics:
if generic_token.endswith(enclitic):
if enclitic == 'cum':
if generic_token.lower() in self.inclusions:
specific_tokens += [enclitic] + [generic_token[:-len(enclitic)]]
else:
specific_tokens += [generic_token]
elif enclitic == 'n':
specific_tokens += [generic_token[:-len(enclitic)]] + ['ne']
elif enclitic == 'st':
if generic_token.endswith('ust'):
specific_tokens += [generic_token[:-len(enclitic)+1]] + ['est']
else:
# Does not handle 'similist', 'qualist', etc. correctly
specific_tokens += [generic_token[:-len(enclitic)]] + ['est']
else:
specific_tokens += [enclitic] + [generic_token[:-len(enclitic)]]
is_enclitic = True
break
if not is_enclitic:
specific_tokens.append(generic_token)
return specific_tokens
def setup_tokenizers(terminal_punctuation):
PunktLanguageVars.sent_end_chars = terminal_punctuation
PunktLanguageVars.re_boundary_realignment = re.compile(r'[›»》’”\'\"）\)\]\}\>]+?(?:\s+|(?=--)|$)', re.MULTILINE)
global word_tokenizer
global sentence_tokenizers
#Accessing private variables of PunktLanguageVars because nltk has a faulty design pattern that necessitates it.
#Issue reported here: https://github.com/nltk/nltk/issues/2068
word_tokenizer = PunktLanguageVars()
word_tokenizer._re_word_tokenizer = re.compile(PunktLanguageVars._word_tokenize_fmt % {
'NonWord': r"(?:[\d\.\?¿؟\!¡！‽…⋯᠁ฯ,،，､、。°※··᛫~\:;;\\\/⧸⁄（）\(\)\[\]\{\}\<\>\'\"‘’“”‹›«»《》\|‖\=\-\‐\‒\–\—\―_\+\*\^\$£€§%#@&†‡])",
'MultiChar': PunktLanguageVars._re_multi_char_punct,
'WordStart': r"[^\d\.\?¿؟\!¡！‽…⋯᠁ฯ,،，､、。°※··᛫~\:;;\\\/⧸⁄（）\(\)\[\]\{\}\<\>\'\"‘’“”‹›«»《》\|‖\=\-\‐\‒\–\—\―_\+\*\^\$£€§%#@&†‡]",
}, re.UNICODE | re.VERBOSE)
word_tokenizer._re_period_context = re.compile(PunktLanguageVars._period_context_fmt % {
'NonWord': r"(?:[\d\.\?¿؟\!¡！‽…⋯᠁ฯ,،，､、。°※··᛫~\:;;\\\/⧸⁄（）\(\)\[\]\{\}\<\>\'\"‘’“”‹›«»《》\|‖\=\-\‐\‒\–\—\―_\+\*\^\$£€§%#@&†‡])",
'SentEndChars': word_tokenizer._re_sent_end_chars,
}, re.UNICODE | re.VERBOSE)
x = PunktLanguageVars()
x._re_word_tokenizer = re.compile(PunktLanguageVars._word_tokenize_fmt % {
'NonWord': r"(?:[\?¿؟\!¡！‽…⋯᠁ฯ,،，､、。°※··᛫~\:;;\\\/⧸⁄（）\(\)\[\]\{\}\<\>\'\"‘’“”‹›«»《》\|‖\=\-\‐\‒\–\—\―_\+\*\^\$£€§%#@&†‡])",
'MultiChar': PunktLanguageVars._re_multi_char_punct,
'WordStart': r"[^\?¿؟\!¡！‽…⋯᠁ฯ,،，､、。°※··᛫~\:;;\\\/⧸⁄（）\(\)\[\]\{\}\<\>\'\"‘’“”‹›«»《》\|‖\=\-\‐\‒\–\—\―_\+\*\^\$£€§%#@&†‡]",
}, re.UNICODE | re.VERBOSE)
x._re_period_context = re.compile(PunktLanguageVars._period_context_fmt % {
'NonWord': r"(?:[\?¿؟\!¡！‽…⋯᠁ฯ,،，､、。°※··᛫~\:;;\\\/⧸⁄（）\(\)\[\]\{\}\<\>\'\"‘’“”‹›«»《》\|‖\=\-\‐\‒\–\—\―_\+\*\^\$£€§%#@&†‡])",
'SentEndChars': x._re_sent_end_chars,
}, re.UNICODE | re.VERBOSE)
#Read tokenizers from pickle files (also include an untrained tokenizer). Mapping from language name to tokenizer
sentence_tokenizers = dict({None: PunktSentenceTokenizer(lang_vars=PunktLanguageVars())}, **{
current_file_name[:current_file_name.index('.')]: pickle.load(open(join(current_path, current_file_name), mode='rb'))
for current_path, current_dir_names, current_file_names in os.walk(sentence_tokenizer_dir)
for current_file_name in current_file_names if current_file_name.endswith('.pickle')
})
for s in sentence_tokenizers.values():
s._lang_vars._re_period_context = x._re_period_context
s._lang_vars._re_word_tokenizer = x._re_word_tokenizer
class Frequency:
"""Methods for making word frequency lists."""
def __init__(self):
"""Language taken as argument, necessary used when saving word frequencies to
``cltk_data/user_data``."""
self.punkt = PunktLanguageVars()
self.punctuation = [
',', '.', ';', ':', '"', "'", '?', '-', '!', '*', '[', ']', '{',
'}'
]
def counter_from_str(self, string):
"""Build word frequency list from incoming string."""
string_list = [
chars for chars in string if chars not in self.punctuation
]
string_joined = ''.join(string_list)
tokens = self.punkt.word_tokenize(string_joined)
return Counter(tokens)
def counter_from_corpus(self, corpus):
"""Build word frequency list from one of several available corpora.
TODO: Make this count iteratively, not all at once
"""
assert corpus in ['phi5', 'tlg'], \
"Corpus '{0}' not available. Choose from 'phi5' or 'tlg'.".format(corpus)
all_strings = self._assemble_corpus_string(corpus=corpus)
return self.counter_from_str(all_strings)
def _assemble_corpus_string(self, corpus):
"""Takes a list of filepaths, returns a string containing contents of
all files."""
if corpus == 'phi5':
filepaths = assemble_phi5_author_filepaths()
file_cleaner = phi5_plaintext_cleanup
elif corpus == 'tlg':
filepaths = assemble_tlg_author_filepaths()
file_cleaner = tlg_plaintext_cleanup
for filepath in filepaths:
with open(filepath) as file_open:
file_read = file_open.read().lower()
file_clean = file_cleaner(file_read)
yield file_clean
class Frequency:
"""Methods for making word frequency lists."""
def __init__(self):
"""Language taken as argument, necessary used when saving word frequencies to
``cltk_data/user_data``."""
self.punkt = PunktLanguageVars()
self.punctuation = [',', '.', ';', ':', '"', "'", '?', '-', '!', '*', '[', ']', '{', '}']
def counter_from_str(self, string):
"""Build word frequency list from incoming string."""
string_list = [chars for chars in string if chars not in self.punctuation]
string_joined = ''.join(string_list)
tokens = self.punkt.word_tokenize(string_joined)
return Counter(tokens)
def counter_from_corpus(self, corpus):
"""Build word frequency list from one of several available corpora.
TODO: Make this count iteratively, not all at once
"""
assert corpus in ['phi5', 'tlg'], \
"Corpus '{0}' not available. Choose from 'phi5' or 'tlg'.".format(corpus)
all_strings = self._assemble_corpus_string(corpus=corpus)
return self.counter_from_str(all_strings)
def _assemble_corpus_string(self, corpus):
"""Takes a list of filepaths, returns a string containing contents of
all files."""
if corpus == 'phi5':
filepaths = assemble_phi5_author_filepaths()
file_cleaner = phi5_plaintext_cleanup
elif corpus == 'tlg':
filepaths = assemble_tlg_author_filepaths()
file_cleaner = tlg_plaintext_cleanup
for filepath in filepaths:
with open(filepath) as file_open:
file_read = file_open.read().lower()
file_clean = file_cleaner(file_read)
yield file_clean
#!/usr/bin/python3
import concurrent.futures
from PIL import Image
import os
import re
import pickle
import docx
import logging
from pathlib import Path
import pytesseract
from nltk.tokenize.punkt import PunktLanguageVars
from string import punctuation
from multiprocessing import Lock
lemmatizer = PunktLanguageVars()
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
format_log = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
log_path = Path(Path.cwd() / 'ocr.log')
fileHandler = logging.FileHandler(log_path)
fileHandler.setFormatter(format_log)
logger.addHandler(fileHandler)
stream = logging.StreamHandler()
stream.setFormatter(format_log)
logger.addHandler(stream)
lock = Lock()
new_paragraphs = []
paragraphs = []
def tag_ner(lang, input_text, output_type=list):
"""Run NER for chosen language.
Choosing output_type=list, returns a list of tuples:
>>> tag_ner('latin', input_text=text_str, output_type=list)
>>> [('ut',), ('Venus', 'Entity'), (',',), ('ut',), ('Sirius', 'Entity'),
(',',), ('ut',), ('Spica', 'Entity')]
"""
_check_latest_data(lang)
assert lang in NER_DICT.keys(), \
'Invalid language. Choose from: {}'.format(', '.join(NER_DICT.keys()))
types = [str, list]
assert type(input_text) in types, 'Input must be: {}.'.format(
', '.join(types))
assert output_type in types, 'Output must be a {}.'.format(
', '.join(types))
if type(input_text) == str:
punkt = PunktLanguageVars()
tokens = punkt.word_tokenize(input_text)
new_tokens = []
for word in tokens:
if word.endswith('.'):
new_tokens.append(word[:-1])
new_tokens.append('.')
else:
new_tokens.append(word)
input_text = new_tokens
ner_file_path = os.path.expanduser(NER_DICT[lang])
with open(ner_file_path, encoding='utf-8') as file_open:
ner_str = file_open.read()
ner_list = ner_str.split('\n')
ner_tuple_list = []
for count, word_token in enumerate(input_text):
match = False
for ner_word in ner_list:
# the replacer slows things down, but is necessary
if word_token == ner_word:
ner_tuple = (word_token, 'Entity')
ner_tuple_list.append(ner_tuple)
match = True
break
if not match:
ner_tuple_list.append((word_token, ))
if output_type is str:
string = ''
for tup in ner_tuple_list:
start_space = ' '
final_space = ''
# this is some mediocre string reconstitution
# maybe not worth the effort
if tup[0] in [',', '.', ';', ':', '?', '!']:
start_space = ''
if len(tup) == 2:
string += start_space + tup[0] + '/' + tup[1] + final_space
else:
string += start_space + tup[0] + final_space
return string
return ner_tuple_list
def tokenize_latin_words(string):
from cltk.tokenize.latin_exceptions import latin_exceptions
assert isinstance(string, str), "Incoming string must be type str."
def matchcase(word):
# From Python Cookbook
def replace(m):
text = m.group()
if text.isupper():
return word.upper()
elif text.islower():
return word.lower()
elif text[0].isupper():
return word.capitalize()
else:
return word
return replace
replacements = [(r'mecum', 'cum me'),
(r'tecum', 'cum te'),
(r'secum', 'cum se'),
(r'nobiscum', 'cum nobis'),
(r'vobiscum', 'cum vobis'),
(r'quocum', 'cum quo'),
(r'quacum', 'cum qua'),
(r'quicum', 'cum qui'),
(r'quibuscum', 'cum quibus'),
(r'sodes', 'si audes'),
(r'satin', 'satis ne'),
(r'scin', 'scis ne'),
(r'sultis', 'si vultis'),
(r'similist', 'similis est'),
(r'qualist', 'qualis est')
]
for replacement in replacements:
string = re.sub(replacement[0], matchcase(replacement[1]), string, flags=re.IGNORECASE)
punkt_param = PunktParameters()
abbreviations = ['c', 'l', 'm', 'p', 'q', 't', 'ti', 'sex', 'a', 'd', 'cn', 'sp', "m'", 'ser', 'ap', 'n', 'v', 'k', 'mam', 'post', 'f', 'oct', 'opet', 'paul', 'pro', 'sert', 'st', 'sta', 'v', 'vol', 'vop']
punkt_param.abbrev_types = set(abbreviations)
sent_tokenizer = PunktSentenceTokenizer(punkt_param)
word_tokenizer = PunktLanguageVars()
sents = sent_tokenizer.tokenize(string)
enclitics = ['que', 'n', 'ue', 've', 'st']
exceptions = enclitics
exceptions = list(set(exceptions + latin_exceptions))
tokens = []
for sent in sents:
temp_tokens = word_tokenizer.word_tokenize(sent)
if temp_tokens[0].endswith('ne'):
if temp_tokens[0].lower() not in exceptions:
temp = [temp_tokens[0][:-2], '-ne']
temp_tokens = temp + temp_tokens[1:]
if temp_tokens[-1].endswith('.'):
final_word = temp_tokens[-1][:-1]
del temp_tokens[-1]
temp_tokens += [final_word, '.']
for token in temp_tokens:
tokens.append(token)
# Break enclitic handling into own function?
specific_tokens = []
for token in tokens:
is_enclitic = False
if token.lower() not in exceptions:
for enclitic in enclitics:
if token.endswith(enclitic):
if enclitic == 'n':
specific_tokens += [token[:-len(enclitic)]] + ['-ne']
elif enclitic == 'st':
if token.endswith('ust'):
specific_tokens += [token[:-len(enclitic) + 1]] + ['est']
else:
specific_tokens += [token[:-len(enclitic)]] + ['est']
else:
specific_tokens += [token[:-len(enclitic)]] + ['-' + enclitic]
is_enclitic = True
break
if not is_enclitic:
specific_tokens.append(token)
return specific_tokens
def tokenize(self, string):
"""Tokenize incoming string."""
def matchcase(word):
# From Python Cookbook
def replace(m):
text = m.group()
if text.isupper():
return word.upper()
elif text.islower():
return word.lower()
elif text[0].isupper():
return word.capitalize()
else:
return word
return replace
replacements = [(r'mecum', 'cum me'),
(r'tecum', 'cum te'),
(r'secum', 'cum se'),
(r'nobiscum', 'cum nobis'),
(r'vobiscum', 'cum vobis'),
(r'quocum', 'cum quo'),
(r'quacum', 'cum qua'),
(r'quicum', 'cum qui'),
(r'quibuscum', 'cum quibus'),
(r'sodes', 'si audes'),
(r'satin', 'satis ne'),
(r'scin', 'scis ne'),
(r'sultis', 'si vultis'),
(r'similist', 'similis est'),
(r'qualist', 'qualis est')
]
for replacement in replacements:
string = re.sub(replacement[0], matchcase(replacement[1]), string, flags=re.IGNORECASE)
print(string)
punkt = PunktLanguageVars()
generic_tokens = punkt.word_tokenize(string)
specific_tokens = []
for generic_token in generic_tokens:
is_enclitic = False
if generic_token.lower() not in self.exceptions:
for enclitic in self.enclitics:
if generic_token.endswith(enclitic):
if enclitic == 'n':
specific_tokens += [generic_token[:-len(enclitic)]] + ['-ne']
elif enclitic == 'st':
if generic_token.endswith('ust'):
specific_tokens += [generic_token[:-len(enclitic)+1]] + ['est']
else:
specific_tokens += [generic_token[:-len(enclitic)]] + ['est']
else:
specific_tokens += [generic_token[:-len(enclitic)]] + ['-' + enclitic]
is_enclitic = True
break
if not is_enclitic:
specific_tokens.append(generic_token)
return specific_tokens
class LatinLanguageVars(PunktLanguageVars):
_re_non_word_chars = PunktLanguageVars()._re_non_word_chars.replace(
"'", "")
def tokenize_latin_words(string):
"""
Tokenizer divides the string into a list of substrings
>>> from cltk.corpus.utils.formatter import remove_non_ascii
>>> text = 'Dices ἐστιν ἐμός pulchrum esse inimicos ulcisci.'
>>> tokenize_latin_words(text)
['Dices', 'ἐστιν', 'ἐμός', 'pulchrum', 'esse', 'inimicos', 'ulcisci', '.']
:param string: This accepts the string value that needs to be tokenized
:returns: A list of substrings extracted from the string
"""
from cltk.tokenize.latin_exceptions import latin_exceptions
assert isinstance(string, str), "Incoming string must be type str."
def matchcase(word):
# From Python Cookbook
def replace(m):
text = m.group()
if text.isupper():
return word.upper()
elif text.islower():
return word.lower()
elif text[0].isupper():
return word.capitalize()
else:
return word
return replace
replacements = [(r'\bmecum\b', 'cum me'), (r'\btecum\b', 'cum te'),
(r'\bsecum\b', 'cum se'), (r'\bnobiscum\b', 'cum nobis'),
(r'\bvobiscum\b', 'cum vobis'), (r'\bquocum\b', 'cum quo'),
(r'\bquacum\b', 'cum qua'), (r'\bquicum\b', 'cum qui'),
(r'\bquibuscum\b', 'cum quibus'),
(r'\bsodes\b', 'si audes'), (r'\bsatin\b', 'satis ne'),
(r'\bscin\b', 'scis ne'), (r'\bsultis\b', 'si vultis'),
(r'\bsimilist\b', 'similis est'),
(r'\bqualist\b', 'qualis est')]
for replacement in replacements:
string = re.sub(replacement[0],
matchcase(replacement[1]),
string,
flags=re.IGNORECASE)
punkt_param = PunktParameters()
abbreviations = [
'c', 'l', 'm', 'p', 'q', 't', 'ti', 'sex', 'a', 'd', 'cn', 'sp', "m'",
'ser', 'ap', 'n', 'v', 'k', 'mam', 'post', 'f', 'oct', 'opet', 'paul',
'pro', 'sert', 'st', 'sta', 'v', 'vol', 'vop'
]
punkt_param.abbrev_types = set(abbreviations)
sent_tokenizer = PunktSentenceTokenizer(punkt_param)
word_tokenizer = PunktLanguageVars()
sents = sent_tokenizer.tokenize(string)
enclitics = ['que', 'n', 'ue', 've', 'st']
exceptions = enclitics
exceptions = list(set(exceptions + latin_exceptions))
tokens = []
for sent in sents:
temp_tokens = word_tokenizer.word_tokenize(sent)
# Need to check that tokens exist before handling them;
# needed to make stream.readlines work in PlaintextCorpusReader
if temp_tokens:
if temp_tokens[0].endswith('ne') and len(temp_tokens[0]) > 2:
if temp_tokens[0].lower() not in exceptions:
temp = [temp_tokens[0][:-2], '-ne']
temp_tokens = temp + temp_tokens[1:]
if temp_tokens[-1].endswith('.'):
final_word = temp_tokens[-1][:-1]
del temp_tokens[-1]
temp_tokens += [final_word, '.']
for token in temp_tokens:
tokens.append(token)
# Break enclitic handling into own function?
specific_tokens = []
for token in tokens:
is_enclitic = False
if token.lower() not in exceptions:
for enclitic in enclitics:
if token.endswith(enclitic):
if enclitic == 'n':
specific_tokens += [token[:-len(enclitic)]] + ['-ne']
elif enclitic == 'st':
if token.endswith('ust'):
specific_tokens += [token[:-len(enclitic) + 1]
] + ['est']
else:
specific_tokens += [token[:-len(enclitic)]
] + ['est']
else:
specific_tokens += [token[:-len(enclitic)]
] + ['-' + enclitic]
is_enclitic = True
break
if not is_enclitic:
specific_tokens.append(token)
return specific_tokens
import re
import nltk
import os
from bs4 import BeautifulSoup
from nltk.tokenize.punkt import PunktLanguageVars
from cltk.stop.middle_english.stops import STOPS_LIST
import json
STOPS_LIST += ['!', ',' ,'.' ,'?', ';', "'", ':', '--', '[', ']']
with open('./stopwords.txt') as f:
lines = f.readlines()
for line in lines:
STOPS_LIST.append(line.strip('\n'))
p = PunktLanguageVars()
root = "./"
plays = os.listdir(root)
for play in plays:
if play == 'data' or re.match('\..*', play):
continue
playdir = root + play + '/'
if not os.path.isdir(playdir):
continue
print(play + " ...")
scenes = os.listdir(playdir)
role = dict()
speechbuf = []
name = ""
for file in scenes:
scene = playdir + '/' + file
def __init__(self):
"""Language taken as argument, necessary used when saving word frequencies to
``cltk_data/user_data``."""
self.punkt = PunktLanguageVars()
self.punctuation = [',', '.', ';', ':', '"', "'", '?', '-', '!', '*', '[', ']', '{', '}']
def tokenize_latin_words(string):
"""
Tokenizer divides the string into a list of substrings
>>> from cltk.corpus.utils.formatter import remove_non_ascii
>>> text = 'Dices ἐστιν ἐμός pulchrum esse inimicos ulcisci.'
>>> remove_non_ascii(text)
>>> 'Dices pulchrum esse inimicos ulcisci.
:param string: This accepts the string value that needs to be tokenized
:returns: A list of substrings extracted from the string
"""
from cltk.tokenize.latin_exceptions import latin_exceptions
assert isinstance(string, str), "Incoming string must be type str."
def matchcase(word):
# From Python Cookbook
def replace(m):
text = m.group()
if text.isupper():
return word.upper()
elif text.islower():
return word.lower()
elif text[0].isupper():
return word.capitalize()
else:
return word
return replace
replacements = [(r'mecum', 'cum me'),
(r'tecum', 'cum te'),
(r'secum', 'cum se'),
(r'nobiscum', 'cum nobis'),
(r'vobiscum', 'cum vobis'),
(r'quocum', 'cum quo'),
(r'quacum', 'cum qua'),
(r'quicum', 'cum qui'),
(r'quibuscum', 'cum quibus'),
(r'sodes', 'si audes'),
(r'satin', 'satis ne'),
(r'scin', 'scis ne'),
(r'sultis', 'si vultis'),
(r'similist', 'similis est'),
(r'qualist', 'qualis est')
]
for replacement in replacements:
string = re.sub(replacement[0], matchcase(replacement[1]), string, flags=re.IGNORECASE)
punkt_param = PunktParameters()
abbreviations = ['c', 'l', 'm', 'p', 'q', 't', 'ti', 'sex', 'a', 'd', 'cn', 'sp', "m'", 'ser', 'ap', 'n', 'v', 'k', 'mam', 'post', 'f', 'oct', 'opet', 'paul', 'pro', 'sert', 'st', 'sta', 'v', 'vol', 'vop']
punkt_param.abbrev_types = set(abbreviations)
sent_tokenizer = PunktSentenceTokenizer(punkt_param)
word_tokenizer = PunktLanguageVars()
sents = sent_tokenizer.tokenize(string)
enclitics = ['que', 'n', 'ue', 've', 'st']
exceptions = enclitics
exceptions = list(set(exceptions + latin_exceptions))
tokens = []
for sent in sents:
temp_tokens = word_tokenizer.word_tokenize(sent)
# Need to check that tokens exist before handling them; needed to make stream.readlines work in PlaintextCorpusReader
if temp_tokens:
if temp_tokens[0].endswith('ne'):
if temp_tokens[0].lower() not in exceptions:
temp = [temp_tokens[0][:-2], '-ne']
temp_tokens = temp + temp_tokens[1:]
if temp_tokens[-1].endswith('.'):
final_word = temp_tokens[-1][:-1]
del temp_tokens[-1]
temp_tokens += [final_word, '.']
for token in temp_tokens:
tokens.append(token)
# Break enclitic handling into own function?
specific_tokens = []
for token in tokens:
is_enclitic = False
if token.lower() not in exceptions:
for enclitic in enclitics:
if token.endswith(enclitic):
if enclitic == 'n':
specific_tokens += [token[:-len(enclitic)]] + ['-ne']
elif enclitic == 'st':
if token.endswith('ust'):
specific_tokens += [token[:-len(enclitic) + 1]] + ['est']
else:
specific_tokens += [token[:-len(enclitic)]] + ['est']
else:
specific_tokens += [token[:-len(enclitic)]] + ['-' + enclitic]
is_enclitic = True
break
if not is_enclitic:
specific_tokens.append(token)
return specific_tokens
# -*- coding: utf-8 -*-
#pylint: disable = missing-docstring, blacklisted-name, unused-argument, invalid-name, line-too-long, protected-access
import unittest
import re
from nltk.tokenize.punkt import PunktSentenceTokenizer, PunktLanguageVars
import context #pylint: disable=unused-import
from qcrit import textual_feature
#[^\s\d’”\'\"）\)\]\}\.,:;]
#[“‘—\-†&vâ\*\^（α-ωΑ-Ὠ`̔]
#΄´´``′″‴
textual_feature.setup_tokenizers(terminal_punctuation=('.', ';', ';'))
p = PunktLanguageVars()
#TODO don't mess with the PunktLanguageVars instance variables, mess with the class variables
p._re_word_tokenizer = re.compile(
PunktLanguageVars._word_tokenize_fmt % {
'NonWord':
r"(?:[\d\.\?¿؟\!¡！‽…⋯᠁ฯ,،，､、。°※··᛫~\:;;\\\/⧸⁄（）\(\)\[\]\{\}\<\>\'\"‘’“”‹›«»《》\|‖\=\-\‐\‒\–\—\―_\+\*\^\$£€§%#@&†‡])",
'MultiChar':
PunktLanguageVars._re_multi_char_punct,
'WordStart':
r"[^\d\.\?¿؟\!¡！‽…⋯᠁ฯ,،，､、。°※··᛫~\:;;\\\/⧸⁄（）\(\)\[\]\{\}\<\>\'\"‘’“”‹›«»《》\|‖\=\-\‐\‒\–\—\―_\+\*\^\$£€§%#@&†‡]",
}, re.UNICODE | re.VERBOSE)
p._re_period_context = re.compile(
PunktLanguageVars._period_context_fmt % {
'NonWord':
r"(?:[\d\.\?¿؟\!¡！‽…⋯᠁ฯ,،，､、。°※··᛫~\:;;\\\/⧸⁄（）\(\)\[\]\{\}\<\>\'\"‘’“”‹›«»《》\|‖\=\-\‐\‒\–\—\―_\+\*\^\$£€§%#@&†‡])",
'SentEndChars': p._re_sent_end_chars,
}, re.UNICODE | re.VERBOSE)
else:
lemmatized_tokens.append(token + '/' + headword)
# if token not found in lemma-headword list
else:
# re-add final period if rm'd
if final_period:
token += '.'
if not return_raw:
lemmatized_tokens.append(token)
else:
lemmatized_tokens.append(token + '/' + token)
if not return_string:
return lemmatized_tokens
elif return_string:
return ' '.join(lemmatized_tokens)
if __name__ == '__main__':
REPLACER = LemmaReplacer('latin')
PUNKT = PunktLanguageVars()
#STRING = 'Est interdum praestare mercaturis rem quaerere, nisi tam periculosum sit, et item foenerari, si tam honestum. Maiores nostri sic habuerunt et ita in legibus posiuerunt: furem dupli condemnari, foeneratorem quadrupli. Quanto peiorem ciuem existimarint foeneratorem quam furem, hinc licet existimare. Et uirum bonum quom laudabant, ita laudabant: bonum agricolam bonumque colonum; amplissime laudari existimabatur qui ita laudabatur. Mercatorem autem strenuum studiosumque rei quaerendae existimo, uerum, ut supra dixi, periculosum et calamitosum. At ex agricolis et uiri fortissimi et milites strenuissimi gignuntur, maximeque pius quaestus stabilissimusque consequitur minimeque inuidiosus, minimeque male cogitantes sunt qui in eo studio occupati sunt. Nunc, ut ad rem redeam, quod promisi institutum principium hoc erit.' # pylint: disable=line-too-long
STRING = 'hominum divomque voluptas'
#EX_TOKENS = PUNKT.word_tokenize(UMLEMMATIZED)
# UMLEMMATIZED = ['τὴν', 'διάγνωσιν', 'αὐτῶν', 'ἔρχεσθαι']
# LEMMATIZED = REPLACER.lemmatize(UMLEMMATIZED, return_raw=False, return_string=True)
LEMMATIZED = REPLACER.lemmatize(STRING.split(),
return_raw=False,
return_string=True)
print(LEMMATIZED)
for f in docs:
#leemos cada documento
exec("file = codecs.open(datapath+'{0}.txt','r','utf-8')".format(f))
content = file.read()
file.close()
#convertimos a minusculas
content = content.lower()
#quitamos numeros y signos de puntuacion para bag of words, bigramas y trigramas
toker = RegexpTokenizer(r'\W+|(,.;)+|[0-9]+', gaps=True)
nc = toker.tokenize(content)
#dejamos solo puntuacion para representacion de signos de puntuacion
tokerPunct = RegexpTokenizer(r'[^,.;!?]+', gaps=True)
ncPunct = tokerPunct.tokenize(content)
p = PunktLanguageVars()
ncGreek = p.word_tokenize(content)
#quitamos palabras funcionales
if language=='english':
filtered_words = [w for w in nc if not w in stopwords.words('english')]
elif language=='greek':
filtered_words = [w for w in ncGreek if not w in STOPS_LIST]
#creamos un diccionario y contamos los elementos mas comunes para bag of words, bigramas y trigramas
contador = Counter(filtered_words)
#obtenemos palabras mas comunes
exec("{0}_mc = contador.most_common(num_common)".format(f))
exec("{0}_str_bow = []".format(f))
exec("{0}_num_bow = []".format(f))
def tokenize_latin_words(string):
from cltk.tokenize.latin_exceptions import latin_exceptions
assert isinstance(string, str), "Incoming string must be type str."
def matchcase(word):
# From Python Cookbook
def replace(m):
text = m.group()
if text.isupper():
return word.upper()
elif text.islower():
return word.lower()
elif text[0].isupper():
return word.capitalize()
else:
return word
return replace
replacements = [(r'mecum', 'cum me'),
(r'tecum', 'cum te'),
(r'secum', 'cum se'),
(r'nobiscum', 'cum nobis'),
(r'vobiscum', 'cum vobis'),
(r'quocum', 'cum quo'),
(r'quacum', 'cum qua'),
(r'quicum', 'cum qui'),
(r'quibuscum', 'cum quibus'),
(r'sodes', 'si audes'),
(r'satin', 'satis ne'),
(r'scin', 'scis ne'),
(r'sultis', 'si vultis'),
(r'similist', 'similis est'),
(r'qualist', 'qualis est')
]
for replacement in replacements:
string = re.sub(replacement[0], matchcase(replacement[1]), string, flags=re.IGNORECASE)
punkt_param = PunktParameters()
abbreviations = ['c', 'l', 'm', 'p', 'q', 't', 'ti', 'sex', 'a', 'd', 'cn', 'sp', "m'", 'ser', 'ap', 'n', 'v', 'k', 'mam', 'post', 'f', 'oct', 'opet', 'paul', 'pro', 'sert', 'st', 'sta', 'v', 'vol', 'vop']
punkt_param.abbrev_types = set(abbreviations)
sent_tokenizer = PunktSentenceTokenizer(punkt_param)
word_tokenizer = PunktLanguageVars()
sents = sent_tokenizer.tokenize(string)
enclitics = ['que', 'n', 'ue', 've', 'st']
exceptions = enclitics
exceptions = list(set(exceptions + latin_exceptions))
tokens = []
for sent in sents:
temp_tokens = word_tokenizer.word_tokenize(sent)
# Need to check that tokens exist before handling them; needed to make stream.readlines work in PlaintextCorpusReader
if temp_tokens:
if temp_tokens[0].endswith('ne'):
if temp_tokens[0].lower() not in exceptions:
temp = [temp_tokens[0][:-2], '-ne']
temp_tokens = temp + temp_tokens[1:]
if temp_tokens[-1].endswith('.'):
final_word = temp_tokens[-1][:-1]
del temp_tokens[-1]
temp_tokens += [final_word, '.']
for token in temp_tokens:
tokens.append(token)
# Break enclitic handling into own function?
specific_tokens = []
for token in tokens:
is_enclitic = False
if token.lower() not in exceptions:
for enclitic in enclitics:
if token.endswith(enclitic):
if enclitic == 'n':
specific_tokens += [token[:-len(enclitic)]] + ['-ne']
elif enclitic == 'st':
if token.endswith('ust'):
specific_tokens += [token[:-len(enclitic) + 1]] + ['est']
else:
specific_tokens += [token[:-len(enclitic)]] + ['est']
else:
specific_tokens += [token[:-len(enclitic)]] + ['-' + enclitic]
is_enclitic = True
break
if not is_enclitic:
specific_tokens.append(token)
return specific_tokens
def featurize(tweet):
tweet = tweet.lower()
tweet = url_away(tweet)
tokens = PunktLanguageVars.word_tokenize(tweet)
tokens = filter(lambda x: len(x) > 2, tokens)
return tokens