def __init__(self, info):
if 'centroids' not in info:
raise Error('Centroid conversion plugins should provide '
'the centroids in their senpy file')
if 'onyx:doesConversion' not in info:
if 'centroids_direction' not in info:
raise Error('Please, provide centroids direction')
cf, ct = info['centroids_direction']
info['onyx:doesConversion'] = [{
'onyx:conversionFrom': cf,
'onyx:conversionTo': ct
}, {
'onyx:conversionFrom': ct,
'onyx:conversionTo': cf
}]
if 'aliases' in info:
aliases = info['aliases']
ncentroids = {}
for k1, v1 in info['centroids'].items():
nv1 = {}
for k2, v2 in v1.items():
nv1[aliases.get(k2, k2)] = v2
ncentroids[aliases.get(k1, k1)] = nv1
info['centroids'] = ncentroids
super(CentroidConversion, self).__init__(info)
def analyse_entry(self, entry, params):
txt = entry.get("text", None)
model = "general" # general_es / general_es / general_fr
api = 'http://api.meaningcloud.com/sentiment-2.1'
lang = params.get("language")
key = params["apiKey"]
parameters = {
'key': key,
'model': model,
'lang': lang,
'of': 'json',
'txt': txt,
'src': 'its-not-a-real-python-sdk'
}
try:
r = requests.post(api, params=parameters, timeout=3)
except requests.exceptions.Timeout:
raise Error("Meaning Cloud API does not response")
api_response = r.json()
if not api_response.get('score_tag'):
raise Error(r.json())
logger.info(api_response)
response = Results()
agg_polarity, agg_polarityValue = self._polarity(
api_response.get('score_tag', None))
agg_opinion = Sentiment(id="Opinion0",
marl__hasPolarity=agg_polarity,
marl__polarityValue=agg_polarityValue,
marl__opinionCount=len(
api_response['sentence_list']))
entry.sentiments.append(agg_opinion)
logger.info(api_response['sentence_list'])
count = 1
for sentence in api_response['sentence_list']:
for nopinion in sentence['segment_list']:
logger.info(nopinion)
polarity, polarityValue = self._polarity(
nopinion.get('score_tag', None))
opinion = Sentiment(
id="Opinion{}".format(count),
marl__hasPolarity=polarity,
marl__polarityValue=polarityValue,
marl__aggregatesOpinion=agg_opinion.get('id'),
nif__anchorOf=nopinion.get('text', None),
nif__beginIndex=nopinion.get('inip', None),
nif__endIndex=nopinion.get('endp', None))
count += 1
entry.sentiments.append(opinion)
yield entry
class ErrorPlugin(plugins.Analyser):
author = 'nobody'
version = 0
ex = Error()
def process(self, *args, **kwargs):
raise self.ex
def test_analyse_error(self):
class ErrorPlugin(plugins.Analyser):
author = 'nobody'
version = 0
ex = Error()
def process(self, *args, **kwargs):
raise self.ex
m = ErrorPlugin(ex=Error('error in analysis', status=500))
self.senpy.add_plugin(m)
try:
analyse(self.senpy, input='nothing', algorithm='ErrorPlugin')
assert False
except Error as ex:
assert 'error in analysis' in ex['message']
assert ex['status'] == 500
m.ex = Exception('generic exception on analysis')
try:
analyse(self.senpy, input='nothing', algorithm='ErrorPlugin')
assert False
except Exception as ex:
assert 'generic exception on analysis' in str(ex)
def test_client(self):
endpoint = 'http://dummy/'
client = Client(endpoint)
with patch_requests('http://dummy/', Results()):
resp = client.analyse('hello')
assert isinstance(resp, Results)
with patch_requests('http://dummy/', Error('Nothing')):
try:
client.analyse(input='hello', algorithm='NONEXISTENT')
raise Exception(
'Exceptions should be raised. This is not golang')
except Error:
pass
def analyse(self, **kwargs):
params = dict(kwargs)
txt = params["input"]
logger.info('TXT:%s' % txt)
endpoint = params["endpoint"]
lang = params.get("language")
key = params["apiKey"]
sentiplug = params["sentiments-plugin"]
s_params = params.copy()
s_params.update({'algo':sentiplug,'language':lang, 'meaningCloud-key':key})
senti_response = requests.get(endpoint, params=s_params).json()
logger.info('SENTIPARAMS: %s' % s_params)
logger.info('SENTIRESPONSE: %s' % senti_response)
if 'entries' not in senti_response:
raise Error(senti_response)
senti_response = Results(senti_response)
logger.info('SENTI: %s' % senti_response)
logger.info(senti_response)
emoplug = params["emotions-plugin"]
e_params = params.copy()
e_params.update({'algo':emoplug,'language':lang})
emo_response = requests.get(endpoint, params=e_params).json()
if 'entries' not in emo_response:
raise Error(emo_response)
emo_response = Results(emo_response)
logger.info('EMO: %s' % emo_response)
logger.info(emo_response)
#Senpy Response
response = Results()
response.analysis = [senti_response.analysis, emo_response.analysis]
unified = senti_response.entries[0]
unified["emotions"] = emo_response.entries[0]["emotions"]
response.entries.append(unified)
return response
def convert(self, emotionSet, fromModel, toModel, params):
cf, ct = self.centroids_direction
logger.debug('{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel,
params))
e = EmotionSet()
if fromModel == cf and toModel == ct:
e.onyx__hasEmotion.append(self._forward_conversion(emotionSet))
elif fromModel == ct and toModel == cf:
for i in emotionSet.onyx__hasEmotion:
e.onyx__hasEmotion.append(self._backwards_conversion(i))
else:
raise Error('EMOTION MODEL NOT KNOWN')
yield e
def __init__(self, info):
if 'centroids' not in info:
raise Error('Centroid conversion plugins should provide '
'the centroids in their senpy file')
if 'onyx:doesConversion' not in info:
if 'centroids_direction' not in info:
raise Error('Please, provide centroids direction')
cf, ct = info['centroids_direction']
info['onyx:doesConversion'] = [{
'onyx:conversionFrom': cf,
'onyx:conversionTo': ct
}, {
'onyx:conversionFrom': ct,
'onyx:conversionTo': cf
}]
if 'aliases' in info:
aliases = info['aliases']
ncentroids = {}
for k1, v1 in info['centroids'].items():
nv1 = {}
for k2, v2 in v1.items():
nv1[aliases.get(k2, k2)] = v2
ncentroids[aliases.get(k1, k1)] = nv1
info['centroids'] = ncentroids
super(CentroidConversion, self).__init__(info)
self.dimensions = set()
for c in self.centroids.values():
self.dimensions.update(c.keys())
self.neutralPoints = self.get("neutralPoints", dict())
if not self.neutralPoints:
for i in self.dimensions:
self.neutralPoints[i] = self.get("neutralValue", 0)
def analyse(self, **params):
logger.debug("emotionService with params {}".format(params))
filename = params.get("i", None)
## FILE MANIPULATIONS ------------------------------- \
if validators.url(filename):
filename = self._download_file(saveFolder = self._storage_path, url = filename)
else:
filename = os.path.join(self._storage_path,filename)
logger.info("{} {}".format(datetime.now(), filename))
if not os.path.isfile(filename):
raise Error("File %s does not exist" % filename)
## EXTRACTING FEATURES ------------------------------- \
feature_set = self._extract_features(filename, convert=True)
# self._remove_file(filename)
## GENERATING OUTPUT --------------------------------- \
response = Results()
entry = Entry()
entry['filename'] = os.path.basename(filename)
emotionSet = EmotionSet()
emotionSet.id = "Emotions"
emotion1 = Emotion()
for dimension in self._dimensions:
emotion1[ self._centroid_mappings[dimension] ] = 5*(1+feature_set[dimension])
emotionSet.onyx__hasEmotion.append(emotion1)
entry.emotions = [emotionSet,]
response.entries.append(entry)
return response
def test_client(self):
endpoint = 'http://dummy/'
client = Client(endpoint)
with patch_requests(Results()) as (request, response):
resp = client.analyse('hello')
assert isinstance(resp, Results)
request.assert_called_with(
url=endpoint + '/', method='GET', params={'input': 'hello'})
with patch_requests(Error('Nothing')) as (request, response):
try:
client.analyse(input='hello', algorithm='NONEXISTENT')
raise Exception('Exceptions should be raised. This is not golang')
except Error:
pass
request.assert_called_with(
url=endpoint + '/',
method='GET',
params={'input': 'hello',
'algorithm': 'NONEXISTENT'})
def test_client(self):
endpoint = 'http://dummy/'
client = Client(endpoint)
success = Call(Results())
with patch('requests.request', return_value=success) as patched:
resp = client.analyse('hello')
assert isinstance(resp, Results)
patched.assert_called_with(
url=endpoint + '/', method='GET', params={'input': 'hello'})
error = Call(Error('Nothing'))
with patch('requests.request', return_value=error) as patched:
try:
client.analyse(input='hello', algorithm='NONEXISTENT')
raise Exception('Exceptions should be raised. This is not golang')
except Error:
pass
patched.assert_called_with(
url=endpoint + '/',
method='GET',
params={'input': 'hello',
'algorithm': 'NONEXISTENT'})
def test_analyse_error(self):
mm = mock.MagicMock()
mm.id = 'magic_mock'
mm.analyse_entries.side_effect = Error('error on analysis', status=500)
self.senpy.plugins['MOCK'] = mm
try:
self.senpy.analyse(input='nothing', algorithm='MOCK')
assert False
except Error as ex:
assert ex['message'] == 'error on analysis'
assert ex['status'] == 500
mm.analyse.side_effect = Exception('generic exception on analysis')
mm.analyse_entries.side_effect = Exception(
'generic exception on analysis')
try:
self.senpy.analyse(input='nothing', algorithm='MOCK')
assert False
except Error as ex:
assert ex['message'] == 'generic exception on analysis'
assert ex['status'] == 500
def test_analyse_error(self):
mm = mock.MagicMock()
mm.id = 'magic_mock'
mm.name = 'mock'
mm.is_activated = True
mm.analyse_entries.side_effect = Error('error in analysis', status=500)
self.senpy.add_plugin(mm)
try:
analyse(self.senpy, input='nothing', algorithm='MOCK')
assert False
except Error as ex:
assert 'error in analysis' in ex['message']
assert ex['status'] == 500
ex = Exception('generic exception on analysis')
mm.analyse.side_effect = ex
mm.analyse_entries.side_effect = ex
try:
analyse(self.senpy, input='nothing', algorithm='MOCK')
assert False
except Exception as ex:
assert 'generic exception on analysis' in str(ex)
def analyse_entry(self, entry, activity):
params = activity.params
txt = entry['nif:isString']
api = 'http://api.meaningcloud.com/'
lang = params.get("language")
model = "general"
key = params["apikey"]
parameters = {
'key': key,
'model': model,
'lang': lang,
'of': 'json',
'txt': txt,
'tt': 'a'
}
try:
r = requests.post(api + "sentiment-2.1",
params=parameters,
timeout=3)
parameters['lang'] = r.json()['model'].split('_')[1]
lang = parameters['lang']
r2 = requests.post(api + "topics-2.0",
params=parameters,
timeout=3)
except requests.exceptions.Timeout:
raise Error("Meaning Cloud API does not response")
api_response = r.json()
api_response_topics = r2.json()
if not api_response.get('score_tag'):
raise Error(r.json())
entry['language_detected'] = lang
self.log.debug(api_response)
agg_polarity, agg_polarityValue = self._polarity(
api_response.get('score_tag', None))
agg_opinion = Sentiment(id="Opinion0",
marl__hasPolarity=agg_polarity,
marl__polarityValue=agg_polarityValue,
marl__opinionCount=len(
api_response['sentence_list']))
agg_opinion.prov(self)
entry.sentiments.append(agg_opinion)
self.log.debug(api_response['sentence_list'])
count = 1
for sentence in api_response['sentence_list']:
for nopinion in sentence['segment_list']:
self.log.debug(nopinion)
polarity, polarityValue = self._polarity(
nopinion.get('score_tag', None))
opinion = Sentiment(
id="Opinion{}".format(count),
marl__hasPolarity=polarity,
marl__polarityValue=polarityValue,
marl__aggregatesOpinion=agg_opinion.get('id'),
nif__anchorOf=nopinion.get('text', None),
nif__beginIndex=int(nopinion.get('inip', None)),
nif__endIndex=int(nopinion.get('endp', None)))
count += 1
opinion.prov(self)
entry.sentiments.append(opinion)
mapper = {
'es': 'es.',
'en': '',
'ca': 'es.',
'it': 'it.',
'fr': 'fr.',
'pt': 'pt.'
}
for sent_entity in api_response_topics['entity_list']:
resource = "_".join(sent_entity.get('form', None).split())
entity = Entity(
id="Entity{}".format(sent_entity.get('id')),
itsrdf__taIdentRef="http://{}dbpedia.org/resource/{}".format(
mapper[lang], resource),
nif__anchorOf=sent_entity.get('form', None),
nif__beginIndex=int(sent_entity['variant_list'][0].get(
'inip', None)),
nif__endIndex=int(sent_entity['variant_list'][0].get(
'endp', None)))
sementity = sent_entity['sementity'].get('type',
None).split(">")[-1]
entity['@type'] = "ODENTITY_{}".format(sementity)
entity.prov(self)
if 'senpy:hasEntity' not in entry:
entry['senpy:hasEntity'] = []
entry['senpy:hasEntity'].append(entity)
for topic in api_response_topics['concept_list']:
if 'semtheme_list' in topic:
for theme in topic['semtheme_list']:
concept = Topic()
concept.id = "Topic{}".format(topic.get('id'))
concept['@type'] = "ODTHEME_{}".format(
theme['type'].split(">")[-1])
concept[
'fam:topic-reference'] = "http://dbpedia.org/resource/{}".format(
theme['type'].split('>')[-1])
entry.prov(self)
if 'senpy:hasTopic' not in entry:
entry['senpy:hasTopic'] = []
entry['senpy:hasTopic'].append(concept)
yield entry
def predict_one(self, features, activity):
language = activity.param("language")
text = features[0]
tokens = self._tokenize(text)
tokens = self._pos(tokens)
sufixes = {'es':'spa','en':'eng','it':'ita','fr':'fra'}
tokens['lemmas'] = {}
for w in tokens['tokens']:
lemmas = wn.lemmas(w[0], lang=sufixes[language])
if len(lemmas) == 0:
continue
tokens['lemmas'][w[0]] = lemmas
if language == "en":
trans = TextBlob(unicode(text))
else:
try:
trans = TextBlob(unicode(text)).translate(from_lang=language,to='en')
except Exception as ex:
raise Error('Could not translate the text from "{}" to "{}": {}'.format(language,
'en',
str(ex)))
useful_synsets = {}
for w_i, t_w in enumerate(trans.sentences[0].words):
synsets = wn.synsets(trans.sentences[0].words[w_i])
if len(synsets) == 0:
continue
eq_synset = self._compare_synsets(synsets, tokens)
useful_synsets[t_w] = eq_synset
scores = {}
scores = {}
if useful_synsets != None:
for word in useful_synsets:
if useful_synsets[word] is None:
continue
temp_scores = self._swn.get_score(useful_synsets[word].name().split('.')[0].replace(' ',' '))
for score in temp_scores:
if score['synset'] == useful_synsets[word]:
t_score = score['pos'] - score['neg']
f_score = 'neu'
if t_score > 0:
f_score = 'pos'
elif t_score < 0:
f_score = 'neg'
score['score'] = f_score
scores[word] = score
break
g_score = 0.5
for i in scores:
n_pos = 0.0
n_neg = 0.0
for w in scores:
if scores[w]['score'] == 'pos':
n_pos += 1.0
elif scores[w]['score'] == 'neg':
n_neg += 1.0
inter = interp1d([-1.0, 1.0], [0.0, 1.0])
try:
g_score = (n_pos - n_neg) / (n_pos + n_neg)
g_score = float(inter(g_score))
except:
if n_pos == 0 and n_neg == 0:
g_score = 0.5
if g_score > 0.5: # Positive
return [1, 0, 0]
elif g_score < 0.5: # Negative
return [0, 0, 1]
else:
return [0, 1, 0]
