def getSentimentAnalysis(url): """ Get sentiment analysis of a given URL :param: url: URL to get sentiment analysis :returns: Sentiment magnitude and emotion analysis in JSON format """ html = urllib.request.urlopen(url).read() soup = BeautifulSoup(html, 'html.parser') text = textFromHtml(html) natural_language_understanding = nl.enableWatsonNatLang() response = natural_language_understanding.analyze( text= text, features=[ # Get general sentiment of text Features.Sentiment( document=True ), # Get emotion towards relevant entities (max:3) Features.Entities( emotion=True, limit = 3 ) ] ) return json.dumps(response, indent=2)
def on_data(self, data): try: #json_data = status._json tweet = json.loads(data) textdata = tweet['text'] # print textdata if tweet['coordinates']: print tweet['coordinates'] #response = queue.send_message(MessageBody=tweet['text']) # print (tweet) producer = KafkaProducer( value_serializer=lambda v: json.dumps(v).encode('utf-8')) producer.send('fizzbuzz', textdata) consumer = KafkaConsumer(bootstrap_servers='localhost:9092', auto_offset_reset='earliest') consumer.subscribe(['fizzbuzz']) for message in consumer: print(message) nlu = watson_developer_cloud.NaturalLanguageUnderstandingV1( version='2017-02-27', username='', password='') nlu.analyze(text=textdata, features=[features.Sentiment()]) except Exception as e: #print("exception: "+e) pass def on_error(self, status): print(status) return True
def run_watson_nlu(): files = glob.glob('work/bug-*.json') (user, passwd) = get_auth() for fname in files: with open(fname) as f: LOG.debug("Processing %s" % fname) bug = json.loads(f.read()) num = bug["link"].split("/")[-1] with open("work/res-%s.json" % num, "w") as out: nlu = watson_developer_cloud.NaturalLanguageUnderstandingV1( version='2017-02-27', username=user, password=passwd) res = nlu.analyze(text=bug["comments"], features=[ features.Concepts(), features.Keywords(), features.Emotion(), features.Sentiment(), ]) output = { "link": bug["link"], "tags": bug["tags"], "importance": bug["importance"], "length": len(bug["comments"]), "results": res } out.write(json.dumps(output, indent=4))
def worker_main(queue): while True: messages = queue.receive_messages(MessageAttributeNames=['Id', 'Tweet', 'Latitude', 'Longitude']) print("Received messages") if len(messages)>0: for message in messages: # Get the custom author message attribute if it was set if message.message_attributes is not None: id = message.message_attributes.get('Id').get('StringValue') tweet = message.message_attributes.get('Tweet').get('StringValue') lat = message.message_attributes.get('Latitude').get('StringValue') lng = message.message_attributes.get('Longitude').get('StringValue') try: response = natural_language_understanding.analyze( text=tweet, features=[features.Sentiment()]) senti = response["sentiment"]["document"]["label"] print senti # senti = response.get('docSentiment').get('type') except Exception as e: print("ERROR: "+str(e)) senti = "neutral" # Using SNS sns_message = {"id":id, "tweet":tweet, "lat":lat, "lng": lng, "sentiment":senti} print("SNS messsage: "+str(sns_message)) sns.publish(TargetArn=arn, Message=json.dumps({'default':json.dumps(sns_message)})) # print('Id: {0}; Tweet: {1}; Latitude: {2}; Longitude: {3}; sentiment: {4}'.format(id,tweet,lat,lng,senti)) # Let the queue know that the message is processed message.delete() else: time.sleep(1)
def analyse(queue, sns_topic): messages = queue.receive_messages(MessageAttributeNames=['All'], VisibilityTimeout=30, MaxNumberOfMessages=10) for message in messages: if message.body is not None and message.message_attributes is not None: #print(message.body) sns_message = {} nlp = NaturalLanguageUnderstandingV1(version='2017-02-27',\ url = "https://gateway.watsonplatform.net/natural-language-understanding/api",\ username = configure.ibm_username,\ password = configure.ibm_password) response = nlp.analyze(text=message.body, features=[features.Sentiment()]) print(response["sentiment"]["document"]["label"]) if response["sentiment"]["document"]["label"] is not None: username = message.message_attributes["username"] sentiment = response["sentiment"]["document"]["label"] lat = message.message_attributes["lat"] long = message.message_attributes["long"] timestamp_ms = message.message_attributes["timestamp_ms"] sns_message["username"] = username sns_message["content"] = message.body sns_message["lat"] = lat sns_message["long"] = long sns_message["sentiment"] = sentiment sns_message["timestamp_ms"] = timestamp_ms message_for_send = json.dumps(sns_message) response1 = sns_topic.publish(Message=message_for_send) print(response1) else: print("sentiment analyse error!")
def on_status(self, status): natural_language_understanding = NaturalLanguageUnderstandingV1( version='2017-02-27', username='******', password='******') twitts = status.text coordinates = status.coordinates language = status.user.lang if status.place and language == 'en': if coordinates is not None and len(coordinates) > 0: coordinates = status.coordinates['coordinates'] print 'coordinates: ', coordinates print 'twitts: ', twitts try: response = natural_language_understanding.analyze( text=twitts, features=[features.Sentiment()]) sentiment = response['sentiment']['document']['label'] except Exception as e: sentiment = "neutral" print sentiment upload_data = { "twitts": twitts, "coordinates": coordinates, "sentiment": sentiment } print requests.post( 'https://search-trends-pnoxxtizp4zrbmwnvgsifem74y.us-east-1.es.amazonaws.com/twittmap/data', json=upload_data) return True
def emotionRank(url): response = natural_language_understanding.analyze( url=url, features=[ Features.Sentiment() ]) return response
def get_sentiment(content): try: resp = nlp.analyze(text=content, features=[features.Sentiment()]) response = resp["sentiment"]["document"]["score"] print str(response) + ": " + content return response except: print "Error getting sentiment from Watson NLU" return None
def sentiment(tips): # Helper function to return text sentiment analysis # Load Watson credentials username=os.environ.get('NLU_USERNAME') password = os.environ.get('NLU_PASSWORD') nlu = watson_developer_cloud.NaturalLanguageUnderstandingV1(version='2017-02-27', username=username, password=password) output = nlu.analyze(text=tips, features=[features.Sentiment()]) return output['sentiment']['document']['score']
def main(): load_dotenv(find_dotenv()) nlu_username = os.environ.get('NLU_USERNAME') nlu_password = os.environ.get('NLU_PASSWORD') nlu = NLU(username=nlu_username, password=nlu_password, version='2017-02-27') result = nlu.analyze(text='I hate galvanize', features=[features.Sentiment()])['sentiment']['document'] print(result['label'], result['score'])
def test_analyze_throws(self): nlu = NaturalLanguageUnderstandingV1(version='2016-01-23', url='http://bogus.com', username='******', password='******') with pytest.raises(ValueError): nlu.analyze([features.Sentiment()]) with pytest.raises(ValueError): nlu.analyze([], text="this will not work")
def nl_processing(reqd_text): response = natural_language_understanding.analyze(text=reqd_text, features=[ features.Entities(), features.Keywords(), features.Emotion(), features.Concepts(), features.Sentiment() ]) return response
def analyze(s): response = natural_language_understanding.analyze(text=s, features=[ Features.Keywords( emotion=True, sentiment=True, limit=2), Features.Sentiment() ]) return response
def getData(user, pw, doc): natural_language_understanding = NaturalLanguageUnderstandingV1( version='2017-02-27', username=user, password=pw) url = 'https://gateway.watsonplatform.net/natural-language-understanding/api' response = natural_language_understanding.analyze( text=doc, features=[features.Sentiment(), features.Emotion()]) return json.dumps(response)
def analyse_emotions(self, tweet): # print('Analysis Text:',tweet) response = natural_language_understanding.analyze( text=tweet, features=[Features.Emotion(), Features.Sentiment()]) list_emotion = list( response['emotion']['document']['emotion'].values()) list_emotion.append(response['sentiment']['document']['score']) print('List emotion is : ', list_emotion) return list_emotion
def analyze_sentiment(self, answer): result = self.nlu.analyze( text=answer, features=[features.Keywords(), features.Sentiment()]) if result['keywords']: keywords = result['keywords'][0] keyword = keywords['text'] else: keyword = None sentiment = result['sentiment']['document']['score'] return sentiment, keyword
def test_text_analyze(self): nlu_url = "http://bogus.com/v1/analyze" responses.add(responses.POST, nlu_url, body="{\"resulting_key\": true}", status=200, content_type='application/json') nlu = NaturalLanguageUnderstandingV1(version='2016-01-23', url='http://bogus.com', username='******', password='******') nlu.analyze([features.Sentiment()], text="hello this is a test") assert len(responses.calls) == 1
def worker(): global q print 'Worker Initialized' attributes = ['id', 'text', 'time', 'lat', 'lon'] while True: responses = q.receive_messages(MessageAttributeNames=attributes) if len(responses) != 0: for response in responses: if response.message_attributes is None: response.delete() continue id = response.message_attributes.get('id').get('StringValue') text = response.message_attributes.get('text').get( 'StringValue') time = response.message_attributes.get('time').get( 'StringValue') lat = response.message_attributes.get('lat').get('StringValue') lon = response.message_attributes.get('lon').get('StringValue') try: natural_language_understanding = NaturalLanguageUnderstandingV1(\ version='2017-02-27',\ username=nlu_creds['username'],\ password=nlu_creds['password']\ ) nlu_response = natural_language_understanding.analyze(\ text=text,\ features=[features.Entities(), features.Keywords(), features.Sentiment()]\ ) sentiment = nlu_response['sentiment']['document']['label'] except Exception as e: print 'Error:', e sentiment = 'neutral' # Send to AWS SNS notification = { 'id': id, 'text': text, 'time': time, 'lat': lat, 'lon': lon, 'sentiment': sentiment } try: print notification sns.publish(TargetArn=sns_arn, Message=json.dumps( {'default': json.dumps(notification)})) response.delete() except Exception as e: print 'Error:', e sleep(2)
def analyze(self, content, keyword): response = self.natural_language_understanding.analyze( text=content, features=[ Features.Sentiment( # Emotion options targets=[keyword] ) ] ) score = response["sentiment"]["document"]["score"] return score
def test_url_analyze(self): nlu_url = "http://bogus.com/v1/analyze" responses.add(responses.POST, nlu_url, body="{\"resulting_key\": true}", status=200, content_type='application/json') nlu = NaturalLanguageUnderstandingV1(version='2016-01-23', url='http://bogus.com', username='******', password='******') nlu.analyze([features.Sentiment(), features.Emotion(document=False)], url="http://cnn.com", xpath="/bogus/xpath", language="en") assert len(responses.calls) == 1
def open_request(): data = request.get_json() text = data['transcription'] patient_id = data['patient_id'] if text not in watson_cache: try: watson_cache[text] = natural_language_understanding.analyze( text=text, features=[features.Keywords(), features.Sentiment()]) except WatsonException as err: print err enqueue(patient_id, text, watson_cache[text]) return jsonify({'result': watson_cache[text]})
def extract_sentiment_ner_trec_full(src, dst_dir): """ Extracts tweet overall sentiment, sentiment per NER, NERs, keywords, sentiment per keyword for the full dataset that's read from a .txt file. Parameters ---------- src: str - path to dataset. dst_dir: - directory in which results will be stored. """ tweets = read_txt(src) # Since tweets are ordered according to topic, label them in a # random order keys = tweets.keys() random.shuffle(keys) for idx, tid in enumerate(keys): fname = "{}.json".format(tid) dst = os.path.join(dst_dir, fname) # If file already exists, data was extracted before and due to # rate-limiting the rest couldn't be extracted if not os.path.isfile(dst): try: # Extract features for a tweet via Watson response = natural_language_understanding.analyze( text=tweets[tid]["text"], # Get entities and their features=[ # Overall tweet sentiment Features.Sentiment(), # NER detection and sentiment per NER Features.Entities(sentiment=False), Features.Keywords(sentiment=False), ]) # Store results in UTF-8 encoding fname = "{}.json".format(tid) dst = os.path.join(dst_dir, fname) with codecs.open(dst, "w", encoding="utf-8") as f: # https://stackoverflow.com/questions/18337407/saving-utf-8-texts-in-json-dumps-as-utf8-not-as-u-escape-sequence data = json.dumps(response, ensure_ascii=False, encoding='utf8') f.write(unicode(data)) # Illegal language except watson_developer_cloud.watson_developer_cloud_service.\ WatsonException: pass print "Finished extraction for {} tweets".format(idx + 1)
def map_feature(name): feature_name_mappings = { 'keywords': features.Keywords(), 'entities': features.Entities(), 'concepts': features.Concepts(), 'categories': features.Categories(), 'sentiment': features.Sentiment(), 'emotion': features.Emotion() } if name in feature_name_mappings: return feature_name_mappings[name] else: print("Invalid feature name") return None
def get_text_data(text,language): username = os.environ.get("BLUEMIX-NLU-USERNAME") password = os.environ.get("BLUEMIX-NLU-PASSWORD") natural_language_understanding = NaturalLanguageUnderstanding( version = "2017-02-27", username=username, password=password ) return natural_language_understanding.analyze( text = text, features = [features.Emotion(), features.Sentiment(), features.Keywords()], language = language )
def nlu(text): response = n.analyze(text=text, features=[ features.Emotion(), features.Concepts(), features.Categories(), features.Entities(), features.Keywords(), features.SemanticRoles(), features.Relations(), features.Sentiment() ], language='en') return json.dumps(response, indent=2)
def featureList(self, tags): f_list = [] for tag in tags: if tag == "sentiment": f_list.append(features.Sentiment()) elif tag == "categories": f_list.append(features.Categories()) elif tag == "concepts": f_list.append(features.Concepts()) elif tag == "emotion": f_list.append(features.Emotion()) elif tag == "entities": f_list.append(features.Entities()) return f_list
def worker_task(): while True: msgs = queue.receive_messages( MessageAttributeNames=['Id', 'Tweet', 'Latitude', 'Longitude']) sns.subscribe( TopicArn=arn, Protocol='http', Endpoint= 'http://twitter-senti-env.2mvfwajxvd.us-east-1.elasticbeanstalk.com/sns_test_handler/' ) if len(msgs) > 0: for message in msgs: id = message.message_attributes.get('Id').get('StringValue') tweet = message.message_attributes.get('Tweet').get( 'StringValue') lat = message.message_attributes.get('Latitude').get( 'StringValue') lon = message.message_attributes.get('Longitude').get( 'StringValue') #print(tweet) try: response = nlu.analyze(text=tweet, features=[features.Sentiment()]) print(response) s = json.dumps(response) r = json.loads(s) sentiment = r['sentiment']['document']['label'] senti_score = r['sentiment']['document']['score'] #sentiment = "pos" #senti_score = 0.5 sns_message = { 'id': id, 'tweet': tweet, 'lat': lat, 'lon': lon, 'senti': sentiment, 'score': senti_score } print("SNS message:" + str(sns_message)) sns.publish(TopicArn=arn, Message=json.dumps( {'default': json.dumps(sns_message)})) except Exception as e: print("ERROR:" + str(e)) else: time.sleep(1)
def IBMWatsonSentimentAnalyzer( self, text ): # IBM Watson Sentiment Analysis. Returns score and label (float, str pair) try: natural_language_understanding = NaturalLanguageUnderstandingV1( username=WATSON_USERNAME, password=WATSON_PASSWORD, version=WATSON_VERSION, url=WATSON_APIURL) response = natural_language_understanding.analyze( text=text, features=[Features.Sentiment()]) return float(response["sentiment"]["document"]["score"]), str( response["sentiment"]["document"]["label"]) except: return -1000, "neutral"
def _addAnalysis(text, sentimentArray, emotionsDict): try: result = natural_language_understanding.analyze( text=text, features=[Features.Sentiment(), Features.Emotion()]) result['emotion'] except: return sentiment_score = result['sentiment']['document']['score'] sentimentArray.append(sentiment_score) emotions = result['emotion']['document']['emotion'] for emo in emotions: if emo not in emotionsDict: emotionsDict[emo] = [] emotionsDict[emo].append(emotions[emo])
def sentimentMining(): score = 0 totalCount = 0 negativeCount = 0 positiveCount = 0 inputText = '' natural_language_understanding = NaturalLanguageUnderstandingV1( version='2017-04-14', username='******', password='******') # the input file to be used for sentiment analysis inputFile = '' with open(inputFile, 'r') as f: reader = csv.reader(f) cnt = 0 for row in reader: # skipping the first row which is the heading if cnt == 0: cnt += 1 else: if len(row) == 0: pass # if empty review else: inputText = row[0] cnt += 1 totalCount += 1 # sentiment analysis of every review response = natural_language_understanding.analyze( text=inputText, features=[features.Sentiment(), features.Emotion()], language='en') print(totalCount) # counting the number of positive and negative review with the score if response["sentiment"]["document"][ "label"] == 'positive': positiveCount += 1 score += response["sentiment"]["document"]["score"] elif response["sentiment"]["document"][ "label"] == 'negative': negativeCount += 1 score += response["sentiment"]["document"]["score"] return positiveCount, negativeCount, score, totalCount