def analyse(self, **params):
lang = params.get("language", "auto")
res = requests.post(
"http://www.sentiment140.com/api/bulkClassifyJson",
json.dumps({
"language": lang,
"data": [{
"text": params["input"]
}]
}))
p = params.get("prefix", None)
response = Results(prefix=p)
polarity_value = self.maxPolarityValue * int(
res.json()["data"][0]["polarity"]) * 0.25
polarity = "marl:Neutral"
neutral_value = self.maxPolarityValue / 2.0
if polarity_value > neutral_value:
polarity = "marl:Positive"
elif polarity_value < neutral_value:
polarity = "marl:Negative"
entry = Entry(id="Entry0", nif__isString=params["input"])
sentiment = Sentiment(id="Sentiment0",
prefix=p,
marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
sentiment.prov__wasGeneratedBy = self.id
entry.sentiments = []
entry.sentiments.append(sentiment)
entry.language = lang
response.entries.append(entry)
return response
def analyse_entry(self, entry, params):
lang = params["language"]
res = requests.post(
ENDPOINT,
json.dumps({
"language": lang,
"data": [{
"text": entry['nif:isString']
}]
}))
p = params.get("prefix", None)
polarity_value = self.maxPolarityValue * int(
res.json()["data"][0]["polarity"]) * 0.25
polarity = "marl:Neutral"
neutral_value = self.maxPolarityValue / 2.0
if polarity_value > neutral_value:
polarity = "marl:Positive"
elif polarity_value < neutral_value:
polarity = "marl:Negative"
sentiment = Sentiment(prefix=p,
marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
sentiment.prov__wasGeneratedBy = self.id
entry.sentiments.append(sentiment)
entry.language = lang
yield entry
def analyse(self, **params):
lang = params.get("language", "auto")
res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
json.dumps({"language": lang,
"data": [{"text": params["input"]}]
}
)
)
p = params.get("prefix", None)
response = Results(prefix=p)
polarity_value = self.maxPolarityValue*int(res.json()["data"][0]
["polarity"]) * 0.25
polarity = "marl:Neutral"
neutral_value = self.maxPolarityValue / 2.0
if polarity_value > neutral_value:
polarity = "marl:Positive"
elif polarity_value < neutral_value:
polarity = "marl:Negative"
entry = Entry(id="Entry0",
nif__isString=params["input"])
sentiment = Sentiment(id="Sentiment0",
prefix=p,
marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
sentiment.prov__wasGeneratedBy = self.id
entry.sentiments = []
entry.sentiments.append(sentiment)
entry.language = lang
response.entries.append(entry)
return response
def analyse_entry(self, entry, params):
lang = params.get("language", "auto")
res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
json.dumps({
"language": lang,
"data": [{
"text": entry.text
}]
}))
p = params.get("prefix", None)
polarity_value = self.maxPolarityValue * int(
res.json()["data"][0]["polarity"]) * 0.25
polarity = "marl:Neutral"
neutral_value = self.maxPolarityValue / 2.0
if polarity_value > neutral_value:
polarity = "marl:Positive"
elif polarity_value < neutral_value:
polarity = "marl:Negative"
sentiment = Sentiment(
prefix=p,
marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
sentiment.prov__wasGeneratedBy = self.id
entry.sentiments = []
entry.sentiments.append(sentiment)
entry.language = lang
yield entry
def analyse_entry(self, entry, params):
lang = params.get("language", "auto")
res = requests.post(
"http://www.sentiment140.com/api/bulkClassifyJson",
json.dumps({
"language": lang,
"data": [{
"text": entry.text
}]
}))
p = params.get("prefix", None)
polarity_value = self.maxPolarityValue * int(
res.json()["data"][0]["polarity"]) * 0.25
polarity = "marl:Neutral"
neutral_value = self.maxPolarityValue / 2.0
if polarity_value > neutral_value:
polarity = "marl:Positive"
elif polarity_value < neutral_value:
polarity = "marl:Negative"
sentiment = Sentiment(prefix=p,
marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
sentiment.prov__wasGeneratedBy = self.id
entry.sentiments = []
entry.sentiments.append(sentiment)
entry.language = lang
yield entry
def analyse_entry(self, entry, params):
txt = entry.get("text", None)
model = "general" # general_es / general_es / general_fr
api = 'http://api.meaningcloud.com/sentiment-2.1'
lang = params.get("language")
key = params["apiKey"]
parameters = {
'key': key,
'model': model,
'lang': lang,
'of': 'json',
'txt': txt,
'src': 'its-not-a-real-python-sdk'
}
try:
r = requests.post(api, params=parameters, timeout=3)
except requests.exceptions.Timeout:
raise Error("Meaning Cloud API does not response")
api_response = r.json()
if not api_response.get('score_tag'):
raise Error(r.json())
logger.info(api_response)
response = Results()
agg_polarity, agg_polarityValue = self._polarity(
api_response.get('score_tag', None))
agg_opinion = Sentiment(id="Opinion0",
marl__hasPolarity=agg_polarity,
marl__polarityValue=agg_polarityValue,
marl__opinionCount=len(
api_response['sentence_list']))
entry.sentiments.append(agg_opinion)
logger.info(api_response['sentence_list'])
count = 1
for sentence in api_response['sentence_list']:
for nopinion in sentence['segment_list']:
logger.info(nopinion)
polarity, polarityValue = self._polarity(
nopinion.get('score_tag', None))
opinion = Sentiment(
id="Opinion{}".format(count),
marl__hasPolarity=polarity,
marl__polarityValue=polarityValue,
marl__aggregatesOpinion=agg_opinion.get('id'),
nif__anchorOf=nopinion.get('text', None),
nif__beginIndex=nopinion.get('inip', None),
nif__endIndex=nopinion.get('endp', None))
count += 1
entry.sentiments.append(opinion)
yield entry
def test_template(self):
r = Results()
e = Entry()
e.nif__isString = 'testing the template'
sent = Sentiment()
sent.polarity = 'marl:Positive'
r.entries.append(e)
e.sentiments.append(sent)
template = ('{% for entry in entries %}'
'{{ entry["nif:isString"] | upper }}'
',{{entry.sentiments[0]["marl:hasPolarity"].split(":")[1]}}'
'{% endfor %}')
res = r.serialize(template=template)
assert res == 'TESTING THE TEMPLATE,Positive'
def test_template(self):
r = Results()
e = Entry()
e.nif__isString = 'testing the template'
sent = Sentiment()
sent.polarity = 'marl:Positive'
r.entries.append(e)
e.sentiments.append(sent)
template = (
'{% for entry in entries %}'
'{{ entry["nif:isString"] | upper }}'
',{{entry.sentiments[0]["marl:hasPolarity"].split(":")[1]}}'
'{% endfor %}')
res = r.serialize(template=template)
assert res == 'TESTING THE TEMPLATE,Positive'
def analyse_gov_daf(self, tweet_text):
logger.debug(
"SentimentAnalysisDL Analysing with params {}".format(tweet_text))
text_input = tweet_text #params.get("input", None)
# st = datetime.now()
text = self.cleanTweet(text_input)
# logger.info("{} {}".format(datetime.now() - st, "tweet cleaned"))
X_test = self.convert_text_to_vector([text], self._tokenizer)
y_pred = self.classify(X_test)
#response = Results()
response_gov_daf = {}
response_gov_daf["entries"] = []
#entry = Entry()
entry_gov_daf = {}
entry_gov_daf["sentiments"] = []
_mapping_labels = {0: 'positive', 1: 'negative', 2: 'neutral'}
_mapping_values = {0: "1", 1: "-1", 2: "0"}
for sentence, y_i in zip([text], y_pred):
sentiment = Sentiment()
sentiment['marl:hasPolarity'] = _mapping_labels[y_i]
sentiment["marl:polarityValue"] = _mapping_values[y_i]
entry_gov_daf["sentiments"].append(sentiment)
entry_gov_daf["nif__isString"] = text_input
response_gov_daf["entries"].append(entry_gov_daf)
return response_gov_daf
def analyse_entry(self, entry, params):
lang = params.get("language", "auto")
polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
polarity = "marl:Neutral"
if polarity_value > 0:
polarity = "marl:Positive"
elif polarity_value < 0:
polarity = "marl:Negative"
sentiment = Sentiment({
"marl:hasPolarity": polarity,
"marl:polarityValue": polarity_value
})
sentiment["prov:wasGeneratedBy"] = self.id
entry.sentiments.append(sentiment)
entry.language = lang
yield entry
def analyse(self, **params):
lang = params.get("language", "auto")
response = Results()
polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
polarity = "marl:Neutral"
if polarity_value > 0:
polarity = "marl:Positive"
elif polarity_value < 0:
polarity = "marl:Negative"
entry = Entry({"id": ":Entry0", "nif:isString": params["input"]})
sentiment = Sentiment({
"id": ":Sentiment0",
"marl:hasPolarity": polarity,
"marl:polarityValue": polarity_value
})
sentiment["prov:wasGeneratedBy"] = self.id
entry.sentiments = []
entry.sentiments.append(sentiment)
entry.language = lang
response.entries.append(entry)
return response
def analyse_entry(self, entry, params):
text = entry.get("text", None)
features = {
word.lower(): (word in word_tokenize(text.lower()))
for word in self._all_words
}
result = self._NaiveBayesClassifier.classify(features)
polarity = "marl:Neutral"
polarity_value = 0
if result == 'pos':
polarity = "marl:Positive"
polarity_value = self.maxPolarityValue
elif result == 'neg':
polarity = "marl:Negative"
polarity_value = self.minPolarityValue
sentiment = Sentiment({
"marl:hasPolarity": polarity,
"marl:polarityValue": polarity_value
})
entry.sentiments.append(sentiment)
yield entry
def test_sentiment(self):
s = Sentiment()
self.assertRaises(jsonschema.ValidationError, s.validate)
s.nif__anchorOf = "so much testing"
s.prov__wasGeneratedBy = ""
s.validate()
def test_sentiment(self):
s = Sentiment()
self.assertRaises(jsonschema.ValidationError, s.validate)
s.nif__anchorOf = "so much testing"
s.prov__wasGeneratedBy = ""
s.validate()
def analyse_entry(self, entry, activity):
params = activity.params
txt = entry['nif:isString']
api = 'http://api.meaningcloud.com/'
lang = params.get("language")
model = "general"
key = params["apikey"]
parameters = {
'key': key,
'model': model,
'lang': lang,
'of': 'json',
'txt': txt,
'tt': 'a'
}
try:
r = requests.post(api + "sentiment-2.1",
params=parameters,
timeout=3)
parameters['lang'] = r.json()['model'].split('_')[1]
lang = parameters['lang']
r2 = requests.post(api + "topics-2.0",
params=parameters,
timeout=3)
except requests.exceptions.Timeout:
raise Error("Meaning Cloud API does not response")
api_response = r.json()
api_response_topics = r2.json()
if not api_response.get('score_tag'):
raise Error(r.json())
entry['language_detected'] = lang
self.log.debug(api_response)
agg_polarity, agg_polarityValue = self._polarity(
api_response.get('score_tag', None))
agg_opinion = Sentiment(id="Opinion0",
marl__hasPolarity=agg_polarity,
marl__polarityValue=agg_polarityValue,
marl__opinionCount=len(
api_response['sentence_list']))
agg_opinion.prov(self)
entry.sentiments.append(agg_opinion)
self.log.debug(api_response['sentence_list'])
count = 1
for sentence in api_response['sentence_list']:
for nopinion in sentence['segment_list']:
self.log.debug(nopinion)
polarity, polarityValue = self._polarity(
nopinion.get('score_tag', None))
opinion = Sentiment(
id="Opinion{}".format(count),
marl__hasPolarity=polarity,
marl__polarityValue=polarityValue,
marl__aggregatesOpinion=agg_opinion.get('id'),
nif__anchorOf=nopinion.get('text', None),
nif__beginIndex=int(nopinion.get('inip', None)),
nif__endIndex=int(nopinion.get('endp', None)))
count += 1
opinion.prov(self)
entry.sentiments.append(opinion)
mapper = {
'es': 'es.',
'en': '',
'ca': 'es.',
'it': 'it.',
'fr': 'fr.',
'pt': 'pt.'
}
for sent_entity in api_response_topics['entity_list']:
resource = "_".join(sent_entity.get('form', None).split())
entity = Entity(
id="Entity{}".format(sent_entity.get('id')),
itsrdf__taIdentRef="http://{}dbpedia.org/resource/{}".format(
mapper[lang], resource),
nif__anchorOf=sent_entity.get('form', None),
nif__beginIndex=int(sent_entity['variant_list'][0].get(
'inip', None)),
nif__endIndex=int(sent_entity['variant_list'][0].get(
'endp', None)))
sementity = sent_entity['sementity'].get('type',
None).split(">")[-1]
entity['@type'] = "ODENTITY_{}".format(sementity)
entity.prov(self)
if 'senpy:hasEntity' not in entry:
entry['senpy:hasEntity'] = []
entry['senpy:hasEntity'].append(entity)
for topic in api_response_topics['concept_list']:
if 'semtheme_list' in topic:
for theme in topic['semtheme_list']:
concept = Topic()
concept.id = "Topic{}".format(topic.get('id'))
concept['@type'] = "ODTHEME_{}".format(
theme['type'].split(">")[-1])
concept[
'fam:topic-reference'] = "http://dbpedia.org/resource/{}".format(
theme['type'].split('>')[-1])
entry.prov(self)
if 'senpy:hasTopic' not in entry:
entry['senpy:hasTopic'] = []
entry['senpy:hasTopic'].append(concept)
yield entry
def analyse_entry(self, entry, params):
language = params.get("language")
text = entry.get("text", None)
tokens = self._tokenize(text)
tokens = self._pos(tokens)
sufixes = {'es': 'spa', 'en': 'eng', 'it': 'ita', 'fr': 'fra'}
for i in tokens:
tokens[i]['lemmas'] = {}
for w in tokens[i]['tokens']:
lemmas = wn.lemmas(w[0], lang=sufixes[language])
if len(lemmas) == 0:
continue
tokens[i]['lemmas'][w[0]] = lemmas
if language == "en":
trans = TextBlob(unicode(text))
else:
trans = TextBlob(unicode(text)).translate(from_lang=language,
to='en')
useful_synsets = {}
for s_i, t_s in enumerate(trans.sentences):
useful_synsets[s_i] = {}
for w_i, t_w in enumerate(trans.sentences[s_i].words):
synsets = wn.synsets(trans.sentences[s_i].words[w_i])
if len(synsets) == 0:
continue
eq_synset = self._compare_synsets(synsets, tokens, s_i)
useful_synsets[s_i][t_w] = eq_synset
scores = {}
for i in tokens:
scores[i] = {}
if useful_synsets != None:
for word in useful_synsets[i]:
if useful_synsets[i][word] is None:
continue
temp_scores = self._swn.get_score(
useful_synsets[i][word].name().split('.')[0].replace(
' ', ' '))
for score in temp_scores:
if score['synset'] == useful_synsets[i][word]:
t_score = score['pos'] - score['neg']
f_score = 'neu'
if t_score > 0:
f_score = 'pos'
elif t_score < 0:
f_score = 'neg'
score['score'] = f_score
scores[i][word] = score
break
p = params.get("prefix", None)
for i in scores:
n_pos = 0.0
n_neg = 0.0
for w in scores[i]:
if scores[i][w]['score'] == 'pos':
n_pos += 1.0
elif scores[i][w]['score'] == 'neg':
n_neg += 1.0
inter = interp1d([-1.0, 1.0], [0.0, 1.0])
try:
g_score = (n_pos - n_neg) / (n_pos + n_neg)
g_score = float(inter(g_score))
except:
if n_pos == 0 and n_neg == 0:
g_score = 0.5
polarity = 'marl:Neutral'
polarity_value = 0
if g_score > 0.5:
polarity = 'marl:Positive'
polarity_value = 1
elif g_score < 0.5:
polarity = 'marl:Negative'
polarity_value = -1
opinion = Sentiment(id="Opinion0" + '_' + str(i),
marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
entry.sentiments.append(opinion)
yield entry
