def analyse_entry(self, entry, params): txt = entry.get("text", None) model = "general" # general_es / general_es / general_fr api = 'http://api.meaningcloud.com/sentiment-2.1' lang = params.get("language") key = params["apiKey"] parameters = { 'key': key, 'model': model, 'lang': lang, 'of': 'json', 'txt': txt, 'src': 'its-not-a-real-python-sdk' } try: r = requests.post(api, params=parameters, timeout=3) except requests.exceptions.Timeout: raise Error("Meaning Cloud API does not response") api_response = r.json() if not api_response.get('score_tag'): raise Error(r.json()) logger.info(api_response) response = Results() agg_polarity, agg_polarityValue = self._polarity( api_response.get('score_tag', None)) agg_opinion = Sentiment(id="Opinion0", marl__hasPolarity=agg_polarity, marl__polarityValue=agg_polarityValue, marl__opinionCount=len( api_response['sentence_list'])) entry.sentiments.append(agg_opinion) logger.info(api_response['sentence_list']) count = 1 for sentence in api_response['sentence_list']: for nopinion in sentence['segment_list']: logger.info(nopinion) polarity, polarityValue = self._polarity( nopinion.get('score_tag', None)) opinion = Sentiment( id="Opinion{}".format(count), marl__hasPolarity=polarity, marl__polarityValue=polarityValue, marl__aggregatesOpinion=agg_opinion.get('id'), nif__anchorOf=nopinion.get('text', None), nif__beginIndex=nopinion.get('inip', None), nif__endIndex=nopinion.get('endp', None)) count += 1 entry.sentiments.append(opinion) yield entry
def analyse_entry(self, entry, params): lang = params["language"] res = requests.post( ENDPOINT, json.dumps({ "language": lang, "data": [{ "text": entry['nif:isString'] }] })) p = params.get("prefix", None) polarity_value = self.maxPolarityValue * int( res.json()["data"][0]["polarity"]) * 0.25 polarity = "marl:Neutral" neutral_value = self.maxPolarityValue / 2.0 if polarity_value > neutral_value: polarity = "marl:Positive" elif polarity_value < neutral_value: polarity = "marl:Negative" sentiment = Sentiment(prefix=p, marl__hasPolarity=polarity, marl__polarityValue=polarity_value) sentiment.prov__wasGeneratedBy = self.id entry.sentiments.append(sentiment) entry.language = lang yield entry
def analyse_gov_daf(self, tweet_text): logger.debug( "SentimentAnalysisDL Analysing with params {}".format(tweet_text)) text_input = tweet_text #params.get("input", None) # st = datetime.now() text = self.cleanTweet(text_input) # logger.info("{} {}".format(datetime.now() - st, "tweet cleaned")) X_test = self.convert_text_to_vector([text], self._tokenizer) y_pred = self.classify(X_test) #response = Results() response_gov_daf = {} response_gov_daf["entries"] = [] #entry = Entry() entry_gov_daf = {} entry_gov_daf["sentiments"] = [] _mapping_labels = {0: 'positive', 1: 'negative', 2: 'neutral'} _mapping_values = {0: "1", 1: "-1", 2: "0"} for sentence, y_i in zip([text], y_pred): sentiment = Sentiment() sentiment['marl:hasPolarity'] = _mapping_labels[y_i] sentiment["marl:polarityValue"] = _mapping_values[y_i] entry_gov_daf["sentiments"].append(sentiment) entry_gov_daf["nif__isString"] = text_input response_gov_daf["entries"].append(entry_gov_daf) return response_gov_daf
def analyse(self, **params): lang = params.get("language", "auto") res = requests.post( "http://www.sentiment140.com/api/bulkClassifyJson", json.dumps({ "language": lang, "data": [{ "text": params["input"] }] })) p = params.get("prefix", None) response = Results(prefix=p) polarity_value = self.maxPolarityValue * int( res.json()["data"][0]["polarity"]) * 0.25 polarity = "marl:Neutral" neutral_value = self.maxPolarityValue / 2.0 if polarity_value > neutral_value: polarity = "marl:Positive" elif polarity_value < neutral_value: polarity = "marl:Negative" entry = Entry(id="Entry0", nif__isString=params["input"]) sentiment = Sentiment(id="Sentiment0", prefix=p, marl__hasPolarity=polarity, marl__polarityValue=polarity_value) sentiment.prov__wasGeneratedBy = self.id entry.sentiments = [] entry.sentiments.append(sentiment) entry.language = lang response.entries.append(entry) return response
def analyse_entry(self,entry,params): lang = params.get("language", "auto") res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson", json.dumps({"language": lang, "data": [{"text": entry.get("text",None)}] } ) ) p = params.get("prefix", None) polarity_value = self.maxPolarityValue*int(res.json()["data"][0] ["polarity"]) * 0.25 polarity = "marl:Neutral" neutral_value = 0 if polarity_value > neutral_value: polarity = "marl:Positive" elif polarity_value < neutral_value: polarity = "marl:Negative" sentiment = Sentiment(id="Sentiment0", prefix=p, marl__hasPolarity=polarity, marl__polarityValue=polarity_value) entry.sentiments.append(sentiment) yield entry
def test_template(self): r = Results() e = Entry() e.nif__isString = 'testing the template' sent = Sentiment() sent.polarity = 'marl:Positive' r.entries.append(e) e.sentiments.append(sent) template = ( '{% for entry in entries %}' '{{ entry["nif:isString"] | upper }}' ',{{entry.sentiments[0]["marl:hasPolarity"].split(":")[1]}}' '{% endfor %}') res = r.serialize(template=template) assert res == 'TESTING THE TEMPLATE,Positive'
def analyse_entry(self, entry, params): lang = params.get("language", "auto") polarity_value = max(-1, min(1, random.gauss(0.2, 0.2))) polarity = "marl:Neutral" if polarity_value > 0: polarity = "marl:Positive" elif polarity_value < 0: polarity = "marl:Negative" sentiment = Sentiment({ "marl:hasPolarity": polarity, "marl:polarityValue": polarity_value }) sentiment["prov:wasGeneratedBy"] = self.id entry.sentiments.append(sentiment) entry.language = lang yield entry
def analyse(self, **params): lang = params.get("language", "auto") response = Results() polarity_value = max(-1, min(1, random.gauss(0.2, 0.2))) polarity = "marl:Neutral" if polarity_value > 0: polarity = "marl:Positive" elif polarity_value < 0: polarity = "marl:Negative" entry = Entry({"id": ":Entry0", "nif:isString": params["input"]}) sentiment = Sentiment({ "id": ":Sentiment0", "marl:hasPolarity": polarity, "marl:polarityValue": polarity_value }) sentiment["prov:wasGeneratedBy"] = self.id entry.sentiments = [] entry.sentiments.append(sentiment) entry.language = lang response.entries.append(entry) return response
def analyse_entry(self, entry, params): text = entry.get("text", None) features = { word.lower(): (word in word_tokenize(text.lower())) for word in self._all_words } result = self._NaiveBayesClassifier.classify(features) polarity = "marl:Neutral" polarity_value = 0 if result == 'pos': polarity = "marl:Positive" polarity_value = self.maxPolarityValue elif result == 'neg': polarity = "marl:Negative" polarity_value = self.minPolarityValue sentiment = Sentiment({ "marl:hasPolarity": polarity, "marl:polarityValue": polarity_value }) entry.sentiments.append(sentiment) yield entry
def test_sentiment(self): s = Sentiment() self.assertRaises(jsonschema.ValidationError, s.validate) s.nif__anchorOf = "so much testing" s.prov__wasGeneratedBy = "" s.validate()
def analyse_entry(self, entry, activity): params = activity.params txt = entry['nif:isString'] api = 'http://api.meaningcloud.com/' lang = params.get("language") model = "general" key = params["apikey"] parameters = { 'key': key, 'model': model, 'lang': lang, 'of': 'json', 'txt': txt, 'tt': 'a' } try: r = requests.post(api + "sentiment-2.1", params=parameters, timeout=3) parameters['lang'] = r.json()['model'].split('_')[1] lang = parameters['lang'] r2 = requests.post(api + "topics-2.0", params=parameters, timeout=3) except requests.exceptions.Timeout: raise Error("Meaning Cloud API does not response") api_response = r.json() api_response_topics = r2.json() if not api_response.get('score_tag'): raise Error(r.json()) entry['language_detected'] = lang self.log.debug(api_response) agg_polarity, agg_polarityValue = self._polarity( api_response.get('score_tag', None)) agg_opinion = Sentiment(id="Opinion0", marl__hasPolarity=agg_polarity, marl__polarityValue=agg_polarityValue, marl__opinionCount=len( api_response['sentence_list'])) agg_opinion.prov(self) entry.sentiments.append(agg_opinion) self.log.debug(api_response['sentence_list']) count = 1 for sentence in api_response['sentence_list']: for nopinion in sentence['segment_list']: self.log.debug(nopinion) polarity, polarityValue = self._polarity( nopinion.get('score_tag', None)) opinion = Sentiment( id="Opinion{}".format(count), marl__hasPolarity=polarity, marl__polarityValue=polarityValue, marl__aggregatesOpinion=agg_opinion.get('id'), nif__anchorOf=nopinion.get('text', None), nif__beginIndex=int(nopinion.get('inip', None)), nif__endIndex=int(nopinion.get('endp', None))) count += 1 opinion.prov(self) entry.sentiments.append(opinion) mapper = { 'es': 'es.', 'en': '', 'ca': 'es.', 'it': 'it.', 'fr': 'fr.', 'pt': 'pt.' } for sent_entity in api_response_topics['entity_list']: resource = "_".join(sent_entity.get('form', None).split()) entity = Entity( id="Entity{}".format(sent_entity.get('id')), itsrdf__taIdentRef="http://{}dbpedia.org/resource/{}".format( mapper[lang], resource), nif__anchorOf=sent_entity.get('form', None), nif__beginIndex=int(sent_entity['variant_list'][0].get( 'inip', None)), nif__endIndex=int(sent_entity['variant_list'][0].get( 'endp', None))) sementity = sent_entity['sementity'].get('type', None).split(">")[-1] entity['@type'] = "ODENTITY_{}".format(sementity) entity.prov(self) if 'senpy:hasEntity' not in entry: entry['senpy:hasEntity'] = [] entry['senpy:hasEntity'].append(entity) for topic in api_response_topics['concept_list']: if 'semtheme_list' in topic: for theme in topic['semtheme_list']: concept = Topic() concept.id = "Topic{}".format(topic.get('id')) concept['@type'] = "ODTHEME_{}".format( theme['type'].split(">")[-1]) concept[ 'fam:topic-reference'] = "http://dbpedia.org/resource/{}".format( theme['type'].split('>')[-1]) entry.prov(self) if 'senpy:hasTopic' not in entry: entry['senpy:hasTopic'] = [] entry['senpy:hasTopic'].append(concept) yield entry
def analyse_entry(self, entry, params): language = params.get("language") text = entry.get("text", None) tokens = self._tokenize(text) tokens = self._pos(tokens) sufixes = {'es': 'spa', 'en': 'eng', 'it': 'ita', 'fr': 'fra'} for i in tokens: tokens[i]['lemmas'] = {} for w in tokens[i]['tokens']: lemmas = wn.lemmas(w[0], lang=sufixes[language]) if len(lemmas) == 0: continue tokens[i]['lemmas'][w[0]] = lemmas if language == "en": trans = TextBlob(unicode(text)) else: trans = TextBlob(unicode(text)).translate(from_lang=language, to='en') useful_synsets = {} for s_i, t_s in enumerate(trans.sentences): useful_synsets[s_i] = {} for w_i, t_w in enumerate(trans.sentences[s_i].words): synsets = wn.synsets(trans.sentences[s_i].words[w_i]) if len(synsets) == 0: continue eq_synset = self._compare_synsets(synsets, tokens, s_i) useful_synsets[s_i][t_w] = eq_synset scores = {} for i in tokens: scores[i] = {} if useful_synsets != None: for word in useful_synsets[i]: if useful_synsets[i][word] is None: continue temp_scores = self._swn.get_score( useful_synsets[i][word].name().split('.')[0].replace( ' ', ' ')) for score in temp_scores: if score['synset'] == useful_synsets[i][word]: t_score = score['pos'] - score['neg'] f_score = 'neu' if t_score > 0: f_score = 'pos' elif t_score < 0: f_score = 'neg' score['score'] = f_score scores[i][word] = score break p = params.get("prefix", None) for i in scores: n_pos = 0.0 n_neg = 0.0 for w in scores[i]: if scores[i][w]['score'] == 'pos': n_pos += 1.0 elif scores[i][w]['score'] == 'neg': n_neg += 1.0 inter = interp1d([-1.0, 1.0], [0.0, 1.0]) try: g_score = (n_pos - n_neg) / (n_pos + n_neg) g_score = float(inter(g_score)) except: if n_pos == 0 and n_neg == 0: g_score = 0.5 polarity = 'marl:Neutral' polarity_value = 0 if g_score > 0.5: polarity = 'marl:Positive' polarity_value = 1 elif g_score < 0.5: polarity = 'marl:Negative' polarity_value = -1 opinion = Sentiment(id="Opinion0" + '_' + str(i), marl__hasPolarity=polarity, marl__polarityValue=polarity_value) entry.sentiments.append(opinion) yield entry