def __init__(self, info): if 'centroids' not in info: raise Error('Centroid conversion plugins should provide ' 'the centroids in their senpy file') if 'onyx:doesConversion' not in info: if 'centroids_direction' not in info: raise Error('Please, provide centroids direction') cf, ct = info['centroids_direction'] info['onyx:doesConversion'] = [{ 'onyx:conversionFrom': cf, 'onyx:conversionTo': ct }, { 'onyx:conversionFrom': ct, 'onyx:conversionTo': cf }] if 'aliases' in info: aliases = info['aliases'] ncentroids = {} for k1, v1 in info['centroids'].items(): nv1 = {} for k2, v2 in v1.items(): nv1[aliases.get(k2, k2)] = v2 ncentroids[aliases.get(k1, k1)] = nv1 info['centroids'] = ncentroids super(CentroidConversion, self).__init__(info)
def analyse_entry(self, entry, params): txt = entry.get("text", None) model = "general" # general_es / general_es / general_fr api = 'http://api.meaningcloud.com/sentiment-2.1' lang = params.get("language") key = params["apiKey"] parameters = { 'key': key, 'model': model, 'lang': lang, 'of': 'json', 'txt': txt, 'src': 'its-not-a-real-python-sdk' } try: r = requests.post(api, params=parameters, timeout=3) except requests.exceptions.Timeout: raise Error("Meaning Cloud API does not response") api_response = r.json() if not api_response.get('score_tag'): raise Error(r.json()) logger.info(api_response) response = Results() agg_polarity, agg_polarityValue = self._polarity( api_response.get('score_tag', None)) agg_opinion = Sentiment(id="Opinion0", marl__hasPolarity=agg_polarity, marl__polarityValue=agg_polarityValue, marl__opinionCount=len( api_response['sentence_list'])) entry.sentiments.append(agg_opinion) logger.info(api_response['sentence_list']) count = 1 for sentence in api_response['sentence_list']: for nopinion in sentence['segment_list']: logger.info(nopinion) polarity, polarityValue = self._polarity( nopinion.get('score_tag', None)) opinion = Sentiment( id="Opinion{}".format(count), marl__hasPolarity=polarity, marl__polarityValue=polarityValue, marl__aggregatesOpinion=agg_opinion.get('id'), nif__anchorOf=nopinion.get('text', None), nif__beginIndex=nopinion.get('inip', None), nif__endIndex=nopinion.get('endp', None)) count += 1 entry.sentiments.append(opinion) yield entry
class ErrorPlugin(plugins.Analyser): author = 'nobody' version = 0 ex = Error() def process(self, *args, **kwargs): raise self.ex
def test_analyse_error(self): class ErrorPlugin(plugins.Analyser): author = 'nobody' version = 0 ex = Error() def process(self, *args, **kwargs): raise self.ex m = ErrorPlugin(ex=Error('error in analysis', status=500)) self.senpy.add_plugin(m) try: analyse(self.senpy, input='nothing', algorithm='ErrorPlugin') assert False except Error as ex: assert 'error in analysis' in ex['message'] assert ex['status'] == 500 m.ex = Exception('generic exception on analysis') try: analyse(self.senpy, input='nothing', algorithm='ErrorPlugin') assert False except Exception as ex: assert 'generic exception on analysis' in str(ex)
def test_client(self): endpoint = 'http://dummy/' client = Client(endpoint) with patch_requests('http://dummy/', Results()): resp = client.analyse('hello') assert isinstance(resp, Results) with patch_requests('http://dummy/', Error('Nothing')): try: client.analyse(input='hello', algorithm='NONEXISTENT') raise Exception( 'Exceptions should be raised. This is not golang') except Error: pass
def analyse(self, **kwargs): params = dict(kwargs) txt = params["input"] logger.info('TXT:%s' % txt) endpoint = params["endpoint"] lang = params.get("language") key = params["apiKey"] sentiplug = params["sentiments-plugin"] s_params = params.copy() s_params.update({'algo':sentiplug,'language':lang, 'meaningCloud-key':key}) senti_response = requests.get(endpoint, params=s_params).json() logger.info('SENTIPARAMS: %s' % s_params) logger.info('SENTIRESPONSE: %s' % senti_response) if 'entries' not in senti_response: raise Error(senti_response) senti_response = Results(senti_response) logger.info('SENTI: %s' % senti_response) logger.info(senti_response) emoplug = params["emotions-plugin"] e_params = params.copy() e_params.update({'algo':emoplug,'language':lang}) emo_response = requests.get(endpoint, params=e_params).json() if 'entries' not in emo_response: raise Error(emo_response) emo_response = Results(emo_response) logger.info('EMO: %s' % emo_response) logger.info(emo_response) #Senpy Response response = Results() response.analysis = [senti_response.analysis, emo_response.analysis] unified = senti_response.entries[0] unified["emotions"] = emo_response.entries[0]["emotions"] response.entries.append(unified) return response
def convert(self, emotionSet, fromModel, toModel, params): cf, ct = self.centroids_direction logger.debug('{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params)) e = EmotionSet() if fromModel == cf and toModel == ct: e.onyx__hasEmotion.append(self._forward_conversion(emotionSet)) elif fromModel == ct and toModel == cf: for i in emotionSet.onyx__hasEmotion: e.onyx__hasEmotion.append(self._backwards_conversion(i)) else: raise Error('EMOTION MODEL NOT KNOWN') yield e
def __init__(self, info): if 'centroids' not in info: raise Error('Centroid conversion plugins should provide ' 'the centroids in their senpy file') if 'onyx:doesConversion' not in info: if 'centroids_direction' not in info: raise Error('Please, provide centroids direction') cf, ct = info['centroids_direction'] info['onyx:doesConversion'] = [{ 'onyx:conversionFrom': cf, 'onyx:conversionTo': ct }, { 'onyx:conversionFrom': ct, 'onyx:conversionTo': cf }] if 'aliases' in info: aliases = info['aliases'] ncentroids = {} for k1, v1 in info['centroids'].items(): nv1 = {} for k2, v2 in v1.items(): nv1[aliases.get(k2, k2)] = v2 ncentroids[aliases.get(k1, k1)] = nv1 info['centroids'] = ncentroids super(CentroidConversion, self).__init__(info) self.dimensions = set() for c in self.centroids.values(): self.dimensions.update(c.keys()) self.neutralPoints = self.get("neutralPoints", dict()) if not self.neutralPoints: for i in self.dimensions: self.neutralPoints[i] = self.get("neutralValue", 0)
def analyse(self, **params): logger.debug("emotionService with params {}".format(params)) filename = params.get("i", None) ## FILE MANIPULATIONS ------------------------------- \ if validators.url(filename): filename = self._download_file(saveFolder = self._storage_path, url = filename) else: filename = os.path.join(self._storage_path,filename) logger.info("{} {}".format(datetime.now(), filename)) if not os.path.isfile(filename): raise Error("File %s does not exist" % filename) ## EXTRACTING FEATURES ------------------------------- \ feature_set = self._extract_features(filename, convert=True) # self._remove_file(filename) ## GENERATING OUTPUT --------------------------------- \ response = Results() entry = Entry() entry['filename'] = os.path.basename(filename) emotionSet = EmotionSet() emotionSet.id = "Emotions" emotion1 = Emotion() for dimension in self._dimensions: emotion1[ self._centroid_mappings[dimension] ] = 5*(1+feature_set[dimension]) emotionSet.onyx__hasEmotion.append(emotion1) entry.emotions = [emotionSet,] response.entries.append(entry) return response
def test_client(self): endpoint = 'http://dummy/' client = Client(endpoint) with patch_requests(Results()) as (request, response): resp = client.analyse('hello') assert isinstance(resp, Results) request.assert_called_with( url=endpoint + '/', method='GET', params={'input': 'hello'}) with patch_requests(Error('Nothing')) as (request, response): try: client.analyse(input='hello', algorithm='NONEXISTENT') raise Exception('Exceptions should be raised. This is not golang') except Error: pass request.assert_called_with( url=endpoint + '/', method='GET', params={'input': 'hello', 'algorithm': 'NONEXISTENT'})
def test_client(self): endpoint = 'http://dummy/' client = Client(endpoint) success = Call(Results()) with patch('requests.request', return_value=success) as patched: resp = client.analyse('hello') assert isinstance(resp, Results) patched.assert_called_with( url=endpoint + '/', method='GET', params={'input': 'hello'}) error = Call(Error('Nothing')) with patch('requests.request', return_value=error) as patched: try: client.analyse(input='hello', algorithm='NONEXISTENT') raise Exception('Exceptions should be raised. This is not golang') except Error: pass patched.assert_called_with( url=endpoint + '/', method='GET', params={'input': 'hello', 'algorithm': 'NONEXISTENT'})
def test_analyse_error(self): mm = mock.MagicMock() mm.id = 'magic_mock' mm.analyse_entries.side_effect = Error('error on analysis', status=500) self.senpy.plugins['MOCK'] = mm try: self.senpy.analyse(input='nothing', algorithm='MOCK') assert False except Error as ex: assert ex['message'] == 'error on analysis' assert ex['status'] == 500 mm.analyse.side_effect = Exception('generic exception on analysis') mm.analyse_entries.side_effect = Exception( 'generic exception on analysis') try: self.senpy.analyse(input='nothing', algorithm='MOCK') assert False except Error as ex: assert ex['message'] == 'generic exception on analysis' assert ex['status'] == 500
def test_analyse_error(self): mm = mock.MagicMock() mm.id = 'magic_mock' mm.name = 'mock' mm.is_activated = True mm.analyse_entries.side_effect = Error('error in analysis', status=500) self.senpy.add_plugin(mm) try: analyse(self.senpy, input='nothing', algorithm='MOCK') assert False except Error as ex: assert 'error in analysis' in ex['message'] assert ex['status'] == 500 ex = Exception('generic exception on analysis') mm.analyse.side_effect = ex mm.analyse_entries.side_effect = ex try: analyse(self.senpy, input='nothing', algorithm='MOCK') assert False except Exception as ex: assert 'generic exception on analysis' in str(ex)
def analyse_entry(self, entry, activity): params = activity.params txt = entry['nif:isString'] api = 'http://api.meaningcloud.com/' lang = params.get("language") model = "general" key = params["apikey"] parameters = { 'key': key, 'model': model, 'lang': lang, 'of': 'json', 'txt': txt, 'tt': 'a' } try: r = requests.post(api + "sentiment-2.1", params=parameters, timeout=3) parameters['lang'] = r.json()['model'].split('_')[1] lang = parameters['lang'] r2 = requests.post(api + "topics-2.0", params=parameters, timeout=3) except requests.exceptions.Timeout: raise Error("Meaning Cloud API does not response") api_response = r.json() api_response_topics = r2.json() if not api_response.get('score_tag'): raise Error(r.json()) entry['language_detected'] = lang self.log.debug(api_response) agg_polarity, agg_polarityValue = self._polarity( api_response.get('score_tag', None)) agg_opinion = Sentiment(id="Opinion0", marl__hasPolarity=agg_polarity, marl__polarityValue=agg_polarityValue, marl__opinionCount=len( api_response['sentence_list'])) agg_opinion.prov(self) entry.sentiments.append(agg_opinion) self.log.debug(api_response['sentence_list']) count = 1 for sentence in api_response['sentence_list']: for nopinion in sentence['segment_list']: self.log.debug(nopinion) polarity, polarityValue = self._polarity( nopinion.get('score_tag', None)) opinion = Sentiment( id="Opinion{}".format(count), marl__hasPolarity=polarity, marl__polarityValue=polarityValue, marl__aggregatesOpinion=agg_opinion.get('id'), nif__anchorOf=nopinion.get('text', None), nif__beginIndex=int(nopinion.get('inip', None)), nif__endIndex=int(nopinion.get('endp', None))) count += 1 opinion.prov(self) entry.sentiments.append(opinion) mapper = { 'es': 'es.', 'en': '', 'ca': 'es.', 'it': 'it.', 'fr': 'fr.', 'pt': 'pt.' } for sent_entity in api_response_topics['entity_list']: resource = "_".join(sent_entity.get('form', None).split()) entity = Entity( id="Entity{}".format(sent_entity.get('id')), itsrdf__taIdentRef="http://{}dbpedia.org/resource/{}".format( mapper[lang], resource), nif__anchorOf=sent_entity.get('form', None), nif__beginIndex=int(sent_entity['variant_list'][0].get( 'inip', None)), nif__endIndex=int(sent_entity['variant_list'][0].get( 'endp', None))) sementity = sent_entity['sementity'].get('type', None).split(">")[-1] entity['@type'] = "ODENTITY_{}".format(sementity) entity.prov(self) if 'senpy:hasEntity' not in entry: entry['senpy:hasEntity'] = [] entry['senpy:hasEntity'].append(entity) for topic in api_response_topics['concept_list']: if 'semtheme_list' in topic: for theme in topic['semtheme_list']: concept = Topic() concept.id = "Topic{}".format(topic.get('id')) concept['@type'] = "ODTHEME_{}".format( theme['type'].split(">")[-1]) concept[ 'fam:topic-reference'] = "http://dbpedia.org/resource/{}".format( theme['type'].split('>')[-1]) entry.prov(self) if 'senpy:hasTopic' not in entry: entry['senpy:hasTopic'] = [] entry['senpy:hasTopic'].append(concept) yield entry
def predict_one(self, features, activity): language = activity.param("language") text = features[0] tokens = self._tokenize(text) tokens = self._pos(tokens) sufixes = {'es':'spa','en':'eng','it':'ita','fr':'fra'} tokens['lemmas'] = {} for w in tokens['tokens']: lemmas = wn.lemmas(w[0], lang=sufixes[language]) if len(lemmas) == 0: continue tokens['lemmas'][w[0]] = lemmas if language == "en": trans = TextBlob(unicode(text)) else: try: trans = TextBlob(unicode(text)).translate(from_lang=language,to='en') except Exception as ex: raise Error('Could not translate the text from "{}" to "{}": {}'.format(language, 'en', str(ex))) useful_synsets = {} for w_i, t_w in enumerate(trans.sentences[0].words): synsets = wn.synsets(trans.sentences[0].words[w_i]) if len(synsets) == 0: continue eq_synset = self._compare_synsets(synsets, tokens) useful_synsets[t_w] = eq_synset scores = {} scores = {} if useful_synsets != None: for word in useful_synsets: if useful_synsets[word] is None: continue temp_scores = self._swn.get_score(useful_synsets[word].name().split('.')[0].replace(' ',' ')) for score in temp_scores: if score['synset'] == useful_synsets[word]: t_score = score['pos'] - score['neg'] f_score = 'neu' if t_score > 0: f_score = 'pos' elif t_score < 0: f_score = 'neg' score['score'] = f_score scores[word] = score break g_score = 0.5 for i in scores: n_pos = 0.0 n_neg = 0.0 for w in scores: if scores[w]['score'] == 'pos': n_pos += 1.0 elif scores[w]['score'] == 'neg': n_neg += 1.0 inter = interp1d([-1.0, 1.0], [0.0, 1.0]) try: g_score = (n_pos - n_neg) / (n_pos + n_neg) g_score = float(inter(g_score)) except: if n_pos == 0 and n_neg == 0: g_score = 0.5 if g_score > 0.5: # Positive return [1, 0, 0] elif g_score < 0.5: # Negative return [0, 0, 1] else: return [0, 1, 0]