def cope_lookups():
lookups = Lookups()
lookups.add_table("lemma_lookup", {"cope": "cope", "coped": "cope"})
lookups.add_table("lemma_index", {"verb": ("cope", "cop")})
lookups.add_table("lemma_exc", {"verb": {"coping": ("cope", )}})
lookups.add_table("lemma_rules", {"verb": [["ing", ""]]})
return lookups
def __init__(
self,
vocab: Vocab,
name: str = "morphologizer",
*,
overwrite_lemma: bool = False,
) -> None:
super().__init__()
self.name = name
self.vocab = vocab
self.voikko = libvoikko.Voikko("fi")
self.lookups = Lookups()
self.overwrite_lemma = overwrite_lemma
self.aux_labels = [vocab.strings.add(x) for x in ["aux", "aux:pass"]]
self.cop_labels = [vocab.strings.add(x) for x in ["cop", "cop:own"]]
self.nsubj_labels = [
vocab.strings.add(x) for x in ["nsubj", "nsubj:cop"]
]
self.ccomp_labels = [
vocab.strings.add(x)
for x in ["csubj", "csubj:cop", "xcomp", "xcomp:ds"]
]
self.relcl_labels = [
vocab.strings.add(x) for x in ["acl:relcl", "ccomp"]
]
self.foreign_tag = vocab.strings.add('Foreign')
def concept_sets(self, value):
"""
Sets concepts_sets and the attributes derived from it.
Args:
value (list of list of str): A list of lists of strings; each string being a concept,
each set in the larger list corresponding to a document which has the tags seen in the set.
"""
self._concept_sets = value
LOG.debug("Extracting raw keywords as concepts.")
all_concepts = [
concept
for concept_set in tqdm(self._concept_sets)
for concept in concept_set
if concept.strip() != ""
]
raw_concepts = set(all_concepts)
LOG.debug("Lemmatizing {} raw concepts.".format(len(raw_concepts)))
concepts = [c.lower() for c in raw_concepts]
self.raw2lemma = {rc: c for rc, c in zip(raw_concepts, concepts)}
lookups = Lookups()
lookups.add_table("lemma_lookup", self.raw2lemma)
self.lemmatizer = Lemmatizer(lookups)
self.lemma2raw = {v: k for k, v in self.raw2lemma.items()}
lemma_concepts = [
self.lemmatizer(concept, "NOUN")[0] for concept in all_concepts
]
self.concepts_frequencies = Counter(lemma_concepts)
self.concepts = set(lemma_concepts)
self._fit_concept_indices()
def create_lookups_from_json_reader(path: Path) -> Lookups:
lookups = Lookups()
for p in path.glob("*.json"):
table_name = p.stem
data = srsly.read_json(p)
lookups.add_table(table_name, data)
return lookups
def test_lookups_api():
table_name = "test"
data = {"foo": "bar", "hello": "world"}
lookups = Lookups()
lookups.add_table(table_name, data)
assert len(lookups) == 1
assert table_name in lookups
assert lookups.has_table(table_name)
table = lookups.get_table(table_name)
assert table.name == table_name
assert len(table) == 2
assert table["hello"] == "world"
table["a"] = "b"
assert table["a"] == "b"
table = lookups.get_table(table_name)
assert len(table) == 3
with pytest.raises(KeyError):
lookups.get_table("xyz")
with pytest.raises(ValueError):
lookups.add_table(table_name)
table = lookups.remove_table(table_name)
assert table.name == table_name
assert len(lookups) == 0
assert table_name not in lookups
with pytest.raises(KeyError):
lookups.get_table(table_name)
def lemmatizer():
lookups = Lookups()
lookups.add_table("lemma_lookup", {
"dogs": "dog",
"boxen": "box",
"mice": "mouse"
})
return Lemmatizer(lookups)
def initialize(
self,
get_examples: Callable[[], Iterable[Example]],
*,
nlp: Language = None
) -> None:
lookups = Lookups()
self._lookups = lookups.from_disk(path=self.source)
def test_lookups_to_from_bytes():
lookups = Lookups()
lookups.add_table("table1", {"foo": "bar", "hello": "world"})
lookups.add_table("table2", {"a": 1, "b": 2, "c": 3})
lookups_bytes = lookups.to_bytes()
new_lookups = Lookups()
new_lookups.from_bytes(lookups_bytes)
assert len(new_lookups) == 2
assert "table1" in new_lookups
assert "table2" in new_lookups
table1 = new_lookups.get_table("table1")
assert len(table1) == 2
assert table1["foo"] == "bar"
table2 = new_lookups.get_table("table2")
assert len(table2) == 3
assert table2["b"] == 2
assert new_lookups.to_bytes() == lookups_bytes
def test_lookups_to_from_disk():
lookups = Lookups()
lookups.add_table("table1", {"foo": "bar", "hello": "world"})
lookups.add_table("table2", {"a": 1, "b": 2, "c": 3})
with make_tempdir() as tmpdir:
lookups.to_disk(tmpdir)
new_lookups = Lookups()
new_lookups.from_disk(tmpdir)
assert len(new_lookups) == 2
assert "table1" in new_lookups
assert "table2" in new_lookups
table1 = new_lookups.get_table("table1")
assert len(table1) == 2
assert table1["foo"] == "bar"
table2 = new_lookups.get_table("table2")
assert len(table2) == 3
assert table2["b"] == 2
def test_issue595():
"""Test lemmatization of base forms"""
words = ["Do", "n't", "feed", "the", "dog"]
lookups = Lookups()
lookups.add_table("lemma_rules", {"verb": [["ed", "e"]]})
lookups.add_table("lemma_index", {"verb": {}})
lookups.add_table("lemma_exc", {"verb": {}})
vocab = Vocab()
doc = Doc(vocab, words=words)
doc[2].tag_ = "VB"
assert doc[2].text == "feed"
assert doc[2].lemma_ == "feed"
def test_issue1387():
tag_map = {"VBG": {POS: VERB, VerbForm_part: True}}
lookups = Lookups()
lookups.add_table("lemma_index", {"verb": ("cope", "cop")})
lookups.add_table("lemma_exc", {"verb": {"coping": ("cope", )}})
lookups.add_table("lemma_rules", {"verb": [["ing", ""]]})
lemmatizer = Lemmatizer(lookups)
vocab = Vocab(lemmatizer=lemmatizer, tag_map=tag_map)
doc = Doc(vocab, words=["coping"])
doc[0].tag_ = "VBG"
assert doc[0].text == "coping"
assert doc[0].lemma_ == "cope"
def __init__(self):
self.entities = []
self.columns = []
self.relationships = []
self.synonyms_col = []
self.synonyms_tab = []
self.entity_graph = []
self.loaded_entities = []
self.config = Configuration()
self.conn = pyodbc.connect(self.config.get_sql_connection_string())
lookups = Lookups()
self.lemmatizer = Lemmatizer(lookups)
self.load_db_model()
def test_ner_warns_no_lookups():
nlp = Language()
nlp.vocab.lookups = Lookups()
assert not len(nlp.vocab.lookups)
ner = nlp.create_pipe("ner")
nlp.add_pipe(ner)
with pytest.warns(UserWarning):
nlp.begin_training()
nlp.vocab.lookups.add_table("lexeme_norm")
nlp.vocab.lookups.get_table("lexeme_norm")["a"] = "A"
with pytest.warns(None) as record:
nlp.begin_training()
assert not record.list
def test_tagger_warns_no_lemma_lookups():
nlp = Language()
nlp.vocab.lookups = Lookups()
assert not len(nlp.vocab.lookups)
tagger = nlp.create_pipe("tagger")
with pytest.warns(UserWarning):
tagger.begin_training()
nlp.add_pipe(tagger)
with pytest.warns(UserWarning):
nlp.begin_training()
nlp.vocab.lookups.add_table("lemma_lookup")
with pytest.warns(None) as record:
nlp.begin_training()
assert not record.list
def test_issue595():
"""Test lemmatization of base forms"""
words = ["Do", "n't", "feed", "the", "dog"]
tag_map = {"VB": {POS: VERB, VerbForm_inf: True}}
lookups = Lookups()
lookups.add_table("lemma_rules", {"verb": [["ed", "e"]]})
lookups.add_table("lemma_index", {"verb": {}})
lookups.add_table("lemma_exc", {"verb": {}})
lemmatizer = Lemmatizer(lookups)
vocab = Vocab(lemmatizer=lemmatizer, tag_map=tag_map)
doc = Doc(vocab, words=words)
doc[2].tag_ = "VB"
assert doc[2].text == "feed"
assert doc[2].lemma_ == "feed"
def test_ner_warns_no_lookups(caplog):
nlp = English()
assert nlp.lang in util.LEXEME_NORM_LANGS
nlp.vocab.lookups = Lookups()
assert not len(nlp.vocab.lookups)
nlp.add_pipe("ner")
with caplog.at_level(logging.DEBUG):
nlp.initialize()
assert "W033" in caplog.text
caplog.clear()
nlp.vocab.lookups.add_table("lexeme_norm")
nlp.vocab.lookups.get_table("lexeme_norm")["a"] = "A"
with caplog.at_level(logging.DEBUG):
nlp.initialize()
assert "W033" not in caplog.text
def test_lemmatizer_init(nlp):
lemmatizer = nlp.add_pipe("lemmatizer", config={"mode": "lookup"})
assert isinstance(lemmatizer.lookups, Lookups)
assert not lemmatizer.lookups.tables
assert lemmatizer.mode == "lookup"
with pytest.raises(ValueError):
nlp("test")
nlp.initialize()
assert lemmatizer.lookups.tables
assert nlp("cope")[0].lemma_ == "cope"
assert nlp("coped")[0].lemma_ == "cope"
# replace any tables from spacy-lookups-data
lemmatizer.lookups = Lookups()
# lookup with no tables sets text as lemma
assert nlp("cope")[0].lemma_ == "cope"
assert nlp("coped")[0].lemma_ == "coped"
nlp.remove_pipe("lemmatizer")
lemmatizer = nlp.add_pipe("lemmatizer", config={"mode": "lookup"})
with pytest.raises(ValueError):
# Can't initialize without required tables
lemmatizer.initialize(lookups=Lookups())
lookups = Lookups()
lookups.add_table("lemma_lookup", {})
lemmatizer.initialize(lookups=lookups)
def make_lookups_bin(self,
lookup_name_pattern='lemma_lookup_{}',
filename_pattern='it_lemma_lookup_{}.json'):
lookups = Lookups()
lookup_keys = list(self.tag_map.keys())
for lookup_pos in lookup_keys:
lookup_name = lookup_name_pattern.format(lookup_pos.lower())
filename = filename_pattern.format(lookup_pos.lower())
with open(os.path.join(self.out_path, filename)) as json_file:
lookup_dict = json.load(json_file)
lookups.add_table(lookup_name, lookup_dict)
with open(os.path.join(self.out_path,
'it_lemma_lookup.json')) as json_file:
lookup_dict = json.load(json_file)
lookups.add_table('lemma_lookup', lookup_dict)
lookups.to_disk(self.out_path, 'lookups.bin')
def lemmatize(self, tokens, toke=False):
lookups = Lookups()
lookups.add_table('lemma_index', lemma_index)
lookups.add_table('lemma_exc', lemma_exc)
lookups.add_table('lemma_rules', lemma_rules)
lemmatizer = Lemmatizer(lookups)
lemmas = []
for t in tokens:
lemmas.append(lemmatizer(token.text, token.tag_))
if toke:
return lemmas
return " ".join(lemmas)
def test_lemmatizer_without_is_base_form_implementation():
# Norwegian example from #5658
lookups = Lookups()
lookups.add_table("lemma_rules", {"noun": []})
lookups.add_table("lemma_index", {"noun": {}})
lookups.add_table("lemma_exc",
{"noun": {
"formuesskatten": ["formuesskatt"]
}})
lemmatizer = Lemmatizer(lookups, is_base_form=None)
assert lemmatizer("Formuesskatten", "noun", {
'Definite': 'def',
'Gender': 'masc',
'Number': 'sing'
}) == ["formuesskatt"]
def test_lookups_api():
table_name = "test"
data = {"foo": "bar", "hello": "world"}
lookups = Lookups()
lookups.add_table(table_name, data)
assert table_name in lookups
assert lookups.has_table(table_name)
table = lookups.get_table(table_name)
assert table.name == table_name
assert len(table) == 2
assert table.get("hello") == "world"
table.set("a", "b")
assert table.get("a") == "b"
table = lookups.get_table(table_name)
assert len(table) == 3
with pytest.raises(KeyError):
lookups.get_table("xyz")
def from_jsons(
self, in_indices, in_raw2lemma
): # a little strange because it does not fill in all attributes
"""
Load index and raw2lemma dictionaries into empty ConceptExtractor
Args:
in_indices ():
in_raw2lemma ():
"""
with open(in_indices, "r") as f0:
self.concept_index_mapping = json.load(f0)
with open(in_raw2lemma, "r") as f0:
self.raw2lemma = json.load(f0)
lookups = Lookups()
lookups.add_table("lemma_lookup", self.raw2lemma)
self.lemmatizer = Lemmatizer(lookups)
self.lemma2raw = {v: k for k, v in self.raw2lemma.items()}
self.concepts = self.concept_index_mapping.keys()
tmp_frequencies = {
concept: len(index) for concept, index in self.concept_index_mapping.items()
}
self.concepts_frequencies = Counter(tmp_frequencies)
def lemmatize():
""""""
lookups = Lookups()
lookups.add_table("lemma_rules", {"noun": [["s", ""]]})
lemmatizer = Lemmatizer(lookups)
return lemmatizer
def from_disk(self, path, exclude=tuple()) -> "LookupLemmatizer":
path: Path = ensure_path(path)
lookups = Lookups()
self._lookups = lookups.from_disk(path=path)
return self
def create_lemmatizer():
lookups = Lookups()
with open("lookups/fi_lemma_exc.json") as f:
lookups.add_table("lemma_exc", json.load(f))
return FinnishLemmatizer(lookups)
def P(T):
import pandas as pd
import emoji #checking if a character is an emojis
from collections import Counter
#remove the formating of source
T['source'] = T['source'].str.lower()
T['source'] = T['source'].str.findall('>([^<]+?)<').apply(
lambda x: x[0] if len(x) >= 1 else '')
#import location dictionary and generate country
T['location'] = [
T.loc[k, 'place']['country_code']
if not pd.isnull(T.loc[k, 'place']) else i['location']
for k, i in enumerate(T['user'])
]
Trans = pd.read_csv(
'/Users/livi/Documents/2020 Fall/data mining/Proposal/Tweepy related files/transloc.csv',
index_col=0)
Trans['googlemap'] = Trans['googlemap'].apply(eval)
Trans.set_index('UserInfo', inplace=True)
locdict = Trans.T.to_dict('records')
locdict = locdict[0]
kys = list(locdict.keys())
for k in kys:
if locdict[k] == None:
del locdict[k]
elif len(locdict[k]) != 0:
if 'address_components' in locdict[k][0]:
for ii in locdict[k][0]['address_components']:
if 'country' in ii['types']:
locdict[k] = ii['long_name']
else:
del locdict[k]
elif len(locdict[k]) > 1:
if 'address_components' in locdict[k][1]:
for ii in locdict[k][1]['address_components']:
if 'country' in ii['types']:
locdict[k] = ii['long_name']
else:
del locdict[k]
else:
del locdict[k]
else:
del locdict[k]
## Generate the column
l = []
for i in T['location']:
try:
l.append(locdict[i])
except:
l.append(float('nan'))
T['CountryCode'] = l
print('Finish Generate Country Code')
#Generate Extended tweets and SDGs
for i in range(len(T)):
quote = None
comment = None
#prepare quote part
if not pd.isnull(T.loc[i, 'quoted_status']):
try:
quote = T.loc[i,
'quoted_status']['extended_tweet']['full_text']
except:
quote = T.loc[i, 'quoted_status']['text']
#print('no extended_tweet for quote',i)
#prepare comment part
if pd.isnull(T.loc[i, 'extended_tweet']):
if pd.isnull(T.loc[i, 'retweeted_status']):
try:
comment = T.loc[i, 'text']
except:
print('no text', i)
else:
try:
comment = T.loc[
i, 'retweeted_status']['extended_tweet']['full_text']
except:
comment = T.loc[i, 'retweeted_status']['text']
#print('no extended_tweet for retweeted status',i)
else:
try:
comment = T.loc[i, 'extended_tweet']['full_text']
except:
print('no extended_tweet', i)
#combine quote and comments
if pd.isnull(quote):
T.loc[i, 'extended_tweet'] = comment
else:
T.loc[i, 'extended_tweet'] = '\"' + comment + ' \" ' + quote
## remove some useless information
T['extended_tweet'] = T['extended_tweet'].str.replace("http\S+", "")
#T['extended_tweet']=T['extended_tweet'].str.replace("@\S+","")
T['extended_tweet'] = T['extended_tweet'].str.replace("&", "")
print('Finish Generate Extended Tweets')
T = T.reset_index(drop=True)
T['extended_tweet'] = T['extended_tweet'].str.lower()
T['SDG'] = T['extended_tweet'].str.upper()
T['SDG'] = T['SDG'].str.findall('(SDG\d+)')
print('Finish Generate SDGs')
# Generate User Information and hashtags
T['id'] = [i['id'] for i in T['user']]
#T['name']=[i['name']for i in T['user']]
T['screen_name'] = [i['screen_name'] for i in T['user']]
T['url'] = [i['url'] for i in T['user']]
T['friends_count'] = T['user'].apply(lambda x: x['friends_count'])
T['followers_count'] = T['user'].apply(lambda x: x['followers_count'])
T['hashtags'] = T['extended_tweet'].str.findall('#\S+')
print('Finish Generate UserInfo and Hashtags')
# Prepare lemmatized analysis and tokenized extended tweets
def char_is_emoji(character):
return character in emoji.UNICODE_EMOJI #does the text contain an emoji?
def text_has_emoji(text):
for character in text:
if character in emoji.UNICODE_EMOJI:
return True
return False #remove the emoji
def deEmojify(inputString):
return inputString.encode('ascii', 'ignore').decode('ascii')
T['extended_tweet'] = T['extended_tweet'].apply(lambda x: deEmojify(x))
import spacy
from spacy.lemmatizer import Lemmatizer
from spacy.lookups import Lookups
sp = spacy.load('en')
lookups = Lookups()
lemm = Lemmatizer(lookups)
def lemma_function(text):
dummy = []
#this is just a test to see if it works
for word in sp(text):
dummy.append(word.lemma_)
return ' '.join(dummy)
T['extended_tweet_lemmatized'] = T['extended_tweet'].apply(
lambda x: lemma_function(x))
T['extended_tweet_lemmatized'] = T['extended_tweet_lemmatized'].apply(
lambda x: x.replace('-PRON-', ''))
print('Finish deemoji and lemmatization')
# Generate Sentiment Scores
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
analyser = SentimentIntensityAnalyzer()
def sentiment_analyzer_scores(sentence):
score = analyser.polarity_scores(sentence)
print("{:-<40} {}".format(sentence, str(score)))
T['neg'] = T['extended_tweet_lemmatized'].apply(
lambda x: analyser.polarity_scores(x)['neg'])
T['neu'] = T['extended_tweet_lemmatized'].apply(
lambda x: analyser.polarity_scores(x)['neu'])
T['pos'] = T['extended_tweet_lemmatized'].apply(
lambda x: analyser.polarity_scores(x)['pos'])
T['compound'] = T['extended_tweet_lemmatized'].apply(
lambda x: analyser.polarity_scores(x)['compound'])
print('Finish Generate Sentiment Score')
return T
docx = ""
returnedSearch = HeadingSearch(filename)
#If we didnt find anything with the heading search, just use the whole document.
if (returnedSearch == False):
document1 = docx2txt.process(filename)
docx = nlp(document1)
#Otherwise, send the headingsearch result through nlp
else:
docx = nlp(returnedSearch)
word_frequencies = {} # how many times each word occurs int the document
words = [
] # a list of every word in the document stores in the same index of the frequency array
#spacy lemmatizer to get root words
lookups = Lookups()
lemmatizer = Lemmatizer(lookups)
for word in docx: # go through every word in document
if word.text not in stopwords: # as long as the word isnt a stop word
if lemmatizer.lookup(word.text) not in word_frequencies.keys(
): # if we havent come across the word yet
word_frequencies[lemmatizer.lookup(
word.text)] = 1 # its frequency is one
words.append(lemmatizer.lookup(word.text)) # add it to words
else:
word_frequencies[lemmatizer.lookup(
word.text
)] += 1 # otherwise it is already in the list, so increment it
#Sort through the array by bubble sort
def morphology():
lemmatizer = Lemmatizer(Lookups())
return Morphology(StringStore(), {}, lemmatizer)