def getsentimentfromUrl(url):
global errorcounter
try:
response = natural_language_understanding.analyze(
url=url,
language="de",
return_analyzed_text=True,
features=Features(
entities=EntitiesOptions(
sentiment=True)))
errorcounter = max(errorcounter - 0.2, 0)
return response, False, errorcounter
except:
print("WATSON API has internal server error, waiting for 3 seconds and retry")
time.sleep(3)
try:
response = natural_language_understanding.analyze(
url=url,
language="de",
return_analyzed_text=True,
features=Features(
entities=EntitiesOptions(
sentiment=True)))
return response, False, errorcounter
except:
errorcounter += 1
print("WATSON API has internal server error")
print(errorcounter)
return None, True, errorcounter
def get_nlu_data(samples):
"""Query IBM NLU to get keyword data for each sample."""
data = {}
nlu = NaturalLanguageUnderstandingV1(
version='2018-03-16',
username='******',
password='******')
for s in samples:
response = nlu.analyze(
text=s,
language='en',
features=Features(
keywords=KeywordsOptions(
emotion=True,
limit=5),
entities=EntitiesOptions(
emotion=True,
limit=5)
))
data[s] = {'key' : {}, 'ent' : {}}
for kwd_data in response.result['keywords']:
if ('relevance' not in kwd_data or 'emotion' not in kwd_data):
continue # skip this one, it doesn't have full data?
data[s]['key'][kwd_data['text']] = kwd_data
for ent_data in response.result['entities']:
if ('relevance' not in ent_data or 'emotion' not in ent_data):
continue #yuh yeet
data[s]['ent'][ent_data['text']] = ent_data
return data
def understanding(self):
if not self.transcription:
self.transcript()
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2017-02-27',
username=os.environ['UNDERSTANDING_USERNAME'],
password=os.environ['UNDERSTANDING_PASSWORD'])
self.analysis = natural_language_understanding.analyze(
text=self.transcription['results'][0]['alternatives'][0]
['transcript'],
features=Features(categories=CategoriesOptions(),
concepts=ConceptsOptions(),
emotion=EmotionOptions(),
entities=EntitiesOptions(emotion=True,
sentiment=True,
mentions=True),
keywords=KeywordsOptions(emotion=True,
sentiment=True),
relations=RelationsOptions(),
sentiment=SentimentOptions()))
logger.info('Completed analysis of recorded file')
return self.analysis
def watson_nlp_analysis(text):
if text == '': return text
max_limit_one = 10
max_limit_two = 30
naturalLanguageUnderstanding = NaturalLanguageUnderstandingV1(
version='2018-11-16',
iam_apikey=os.environ['WATSON'],
url=
'https://gateway.watsonplatform.net/natural-language-understanding/api'
)
response = naturalLanguageUnderstanding.analyze(
text=text,
features=Features(concepts=ConceptsOptions(limit=max_limit_one),
categories=CategoriesOptions(limit=max_limit_one),
sentiment=SentimentOptions(document=True),
emotion=EmotionOptions(document=True),
entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=max_limit_two),
keywords=KeywordsOptions(
emotion=True,
sentiment=True,
limit=max_limit_two))).get_result()
return response
def process(key, text):
naturalLanguageUnderstanding = NaturalLanguageUnderstandingV1(
version='2018-11-16',
iam_apikey=key,
url=
'https://gateway.watsonplatform.net/natural-language-understanding/api'
)
# t = 'IBM is an American multinational technology company '
# 'headquartered in Armonk, New York, United States, '
# 'with operations in over 170 countries.'
t = text
try:
response = naturalLanguageUnderstanding.analyze(
text=t,
features=Features(entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=2),
keywords=KeywordsOptions(emotion=True,
sentiment=True,
limit=2))).get_result()
except:
return False
print(json.dumps(response, indent=2))
return (json.dumps(response, indent=2))
def analyze_audio():
print(request.json['texto'])
''' Parte para programar y traducir al inglés '''
language_translator = LanguageTranslator(
version='2018-03-16',
iam_api_key='XmyHrVcLnTgWC3Ou33zGB989tcrOxocykZeZDUJxdlP6',
url='https://gateway.watsonplatform.net/language-translator/api')
translation = language_translator.translate(
text=request.json['texto'],
model_id='es-en')
''' Parte para sacar insights del texto '''
natural_language_understanding = NaturalLanguageUnderstandingV1(
username='******',
password='******',
version='2018-03-16')
response = natural_language_understanding.analyze(
text=json.loads(json.dumps(translation, indent=2, ensure_ascii=False))["translations"][0]["translation"],
features=Features(
entities=EntitiesOptions(
emotion=True,
sentiment=True,
limit=2),
keywords=KeywordsOptions(
emotion=True,
sentiment=True,
limit=2)))
return jsonify(json.dumps(response, indent=2))
def main(dict):
try:
model_id=""
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2018-04-05',
iam_apikey="",
url='')
resposta = natural_language_understanding.analyze(
text=dict['texto'],
features=Features(
entities=EntitiesOptions(emotion=True, sentiment=True, model=model_id),
sentiment=SentimentOptions()),
).get_result()
retorno = {}
# Salvar as entidades no retorno
if 'entities' in resposta:
for i in range(len(resposta['entities'])):
retorno[resposta['entities'][i]['type']] = resposta['entities'][i]['text']
# Salvar o sentimento no retorno
if 'sentiment' in resposta:
retorno['sentiment'] = resposta['sentiment']['document']['label']
dict['err'] = False
dict['resposta'] = retorno
return dict
except:
dict['err'] = True
dict['resposta'] = "Erro na chamada ao NLU."
return dict
def posting_generator():
jobs = []
#job_title = "software engineer"
#job_location = "Pittsburgh, PA"
search_url = 'https://www.careerbuilder.com/jobs-software-engineer-in-pittsburgh,pa?keywords=Software+Engineer&location=Pittsburgh%2C+PA'
base_url = 'https://www.careerbuilder.com'
next_page = urllib.request.urlopen(search_url, None, None)
nlu = NLU(
_apikey='BU11gy3frJMRMKz4XQ_sPJ_HGF3p-qEr74xUlEVTWvsY',
version='2018-03-19'
)
def nextPage(soup):
print("BREAK 1")
next_link = soup.find("a", class_="Next Page")
if next_link is not None:
print("BREAK 2")
next_url = next_link.find_parent("a")['href']
next_page = next_url
return next_page
else:
print("BREAK 3")
return 0
while True:
soup = BeautifulSoup(next_page, 'html.parser')
#next_page = nextPage(soup)
for job in soup.find_all('h2'):
if job.get('class') == 'job-title show-for-medium-up':
url = 'https://www.careerbuilder.com' + job.a['href']
response = nlu.analyze(
url=url,
features=Features(
entities=EntitiesOptions(
limit=1000
),
keywords=KeywordsOptions(
limit=1000
),
)
).get_result()
jobs.append(response)
# jsonprinter(response)
yield response
next_url = nextPage(soup)
if next_url == 0:
break
else:
next_page = urllib.request.urlopen(next_url, None, None)
print("END OF PROGRAM!")
def IntelligentCrawlUrl(URL):
"""
This Function uses IBM Watson's Natural Language Understanding API to crawl the links and get company or person names based on a knowledge graph it already has.
This Function also return Company/Person names based on relevance score by IBM Natural Language Cognitive API.
"""
ListOfEntityOutput = []
try:
response = NaturalLanguageUnderstanding.analyze(
url=URL,
features=Features(entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=250),
sentiment=SentimentOptions(),
keywords=KeywordsOptions(emotion=True,
sentiment=True,
limit=250)))
except Exception as e:
response = {}
if response:
for EveryEntity in response["entities"]:
if EveryEntity["type"] == "Company":
if EveryEntity["relevance"] > 0.25:
ListOfEntityOutput.append(EveryEntity["text"])
print(ListOfEntityOutput)
return ListOfEntityOutput
def concepts(self):
IBM_dict = {}
IBM_response = self.naturalLanguageUnderstanding.analyze(
text=self.text,
features=Features(
entities=EntitiesOptions(emotion=True, sentiment=True, limit=10),
keywords=KeywordsOptions(emotion=True, sentiment=True,limit=10),
sentiment=SentimentOptions(),
categories=CategoriesOptions()
)).get_result()
sent_dict = {'sentiment': IBM_response['sentiment']['document']['score']}
IBM_dict['sentiment'] = sent_dict
ent_result = []
ents = IBM_response['entities']
for e in ents:
ent_result.append(e['text'].lower())
ent_result.sort()
IBM_dict['entities'] = ent_result
kws = []
for keyword in IBM_response['keywords']:
kws.append(keyword['text'].lower())
kws.sort()
IBM_dict['keywords'] = kws
cats = []
for category in IBM_response['categories']:
cats.append(category['label'])
IBM_dict['categories'] = cats
return IBM_dict
def nlu_data():
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2017-02-27',
username='******',
password='******')
r = open("C:\\Users\\kishan.sampat\\Desktop\\user_input.txt", "r")
extracted = r.read().splitlines()
#print(extracted)
mainArray = []
for elements in extracted:
array = []
if len(elements) > 0:
#print(elements)
response = natural_language_understanding.analyze(
text=elements,
features=Features(entities=EntitiesOptions(sentiment=True,
limit=3),
keywords=KeywordsOptions()))
with open('C:\\Users\\kishan.sampat\\Desktop\\user_input.csv',
'w',
newline='') as outfile:
for each in response['keywords']:
tex = each['text']
lemet = porter_stemmer.stem(tex)
array.append(lemet)
mainArray.append(array)
csv.writer(outfile).writerows(mainArray)
else:
break
def watson_analyze_text_understanding(text):
"""
Input:
text to be analyzed
Output:
response from the watson API
Taken from the watson API docs.
"""
natural_language_understanding = NaturalLanguageUnderstandingV1(
username=settings.WATSON_UNDERSTANDING_USERNAME,
password=settings.WATSON_UNDERSTANDING_PASSWORD,
version="2017-02-27",
)
response = {}
try:
response = natural_language_understanding.analyze(
text=text,
features=Features(
entities=EntitiesOptions(emotion=True, sentiment=True,
limit=2),
keywords=KeywordsOptions(emotion=True, sentiment=True,
limit=2),
),
)
except Exception as e:
print e
print "Proceeding without the watson data"
return response
def get_sentiment(target_text):
from watson_developer_cloud import NaturalLanguageUnderstandingV1
from watson_developer_cloud.natural_language_understanding_v1 \
import Features, EntitiesOptions, KeywordsOptions
natural_language_understanding = NaturalLanguageUnderstandingV1(
username="******",
password="******",
version="2018-03-16")
""" Opening the file and reading the text contained """
file = open("messages_to_be_analyzed.txt",'r')
target_text = file.read()
response = natural_language_understanding.analyze(
text= target_text,
features=Features(
entities=EntitiesOptions(
emotion=True,
sentiment=True,
limit=2),
keywords=KeywordsOptions(
emotion=True,
sentiment=True,
limit=2)))
return [response.result.get("keywords")[0].get("emotion").get(emotion) for emotion in
[entry for entry in response.result.get("keywords")[0].get("emotion")]]
def identifyKeyworkdsAndEntities(self, data):
self.app.logger.info(
'Preparing to invoke Natural Language Understanding service')
txt = data.encode("utf-8", "replace")
nlu = self.getNLUService()
results = nlu.analyze(text=data,
return_analyzed_text=True,
features=Features(entities=EntitiesOptions(),
keywords=KeywordsOptions()))
self.app.logger.info(json.dumps(results, indent=2))
primeEntity = None
primeKeyword = None
if 'entities' in results:
entities = results['entities']
if 0 < len(entities):
primeEntity = entities[0].get('text', None)
if 'keywords' in results:
keywords = results['keywords']
if 0 < len(keywords):
primeKeyword = keywords[0].get('text', None)
retData = {"prime_entity": primeEntity, "prime_keyword": primeKeyword}
return retData
def nlu_fact():
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2017-02-27',
username='******',
password='******')
r = open("C:\\Users\\kishan.sampat\\Desktop\\fact.txt","r")
extracted = r.read()
response = natural_language_understanding.analyze(
text= extracted,
features=Features(entities=EntitiesOptions(sentiment=True,limit=3), keywords=KeywordsOptions()))
#print(response)
# print(json.dumps(response, indent=5))
#json_parsed = json.dumps(response)
#print(json_parsed)
with open('C:\\Users\\kishan.sampat\\Desktop\\fact_data.csv', 'w') as outfile:
#json.dump(response , outfile)
for each in response['keywords']:
tex = each['text']
lemet = porter_stemmer.stem(tex)
#print(lemet)
json.dump(lemet , outfile)
outfile.write('\n')
for ent in response['entities']:
ents = ent['text']
lemet_ent = porter_stemmer.stem(ents)
#print(lemet_ent)
json.dump(lemet_ent , outfile)
outfile.write('\n')
def requestWatsonSentence(sentences):
nlu = NaturalLanguageUnderstandingV1(version='2017-02-27',
username=credentials.nluKey,
password=credentials.nluId)
keywords = dict()
entities = dict()
for i in range(len(sentences)):
tmp = nlu.analyze(text=sentences[i],
features=Features(
entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=2),
keywords=KeywordsOptions(emotion=True,
sentiment=True,
limit=2)))
keywords[len(keywords)] = [
item.get('text', {}) for item in tmp.get('keywords', {})
if item.get("relevance") > 0.1
]
entities[len(entities)] = [
item.get('text', {}) for item in tmp.get('entities', {})
if item.get("relevance") > 0.75
]
# return keywords, tones #'rtype': dictionary
return entities, keywords
def makeWikiSection(self, sectiontitle):
print("Accessing IBM Watson for NLP understanding on " + sectiontitle +
" (subtopic of " + self._topic + ")")
response = self.watsonobj.analyze(
text=self._page.section(sectiontitle),
features=Features(concepts=ConceptsOptions(limit=3),
entities=EntitiesOptions(limit=3),
keywords=KeywordsOptions(limit=5),
relations=RelationsOptions(),
semantic_roles=SemanticRolesOptions(limit=3)))
if sectiontitle in wikipedia.search(
sectiontitle) and sectiontitle is not "See also":
return Node("Section",
title=sectiontitle,
content=self._page.section(sectiontitle),
concepts=json.dumps(response["concepts"]),
entities=json.dumps(response["entities"]),
keywords=json.dumps(response["keywords"]),
relations=json.dumps(response["relations"]),
semantic_roles=json.dumps(response["semantic_roles"]),
mainarticleurl=wikipedia.page(self._topic).url)
return Node("Section",
title=sectiontitle,
content=self._page.section(sectiontitle),
concepts=json.dumps(response["concepts"]),
entities=json.dumps(response["entities"]),
keywords=json.dumps(response["keywords"]),
relations=json.dumps(response["relations"]),
semantic_roles=json.dumps(response["semantic_roles"]))
def getAnalysis(review):
natural_language_understanding = NaturalLanguageUnderstandingV1(
username='******',
password='******',
version='2018-03-16')
response = natural_language_understanding.analyze(
text=review,
features=Features(entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=2),
keywords=KeywordsOptions(emotion=True,
sentiment=True,
limit=2))).get_result()
keywords = response["keywords"]
numKeywords = len(keywords)
if numKeywords == 0:
print("no keywords")
return np.zeros(6)
sentiments = np.array([
keyword["sentiment"]["score"] * keyword["relevance"]
for keyword in keywords
])
totalSentiment = np.sum(sentiments) / numKeywords
emotionNames = ['sadness', 'joy', 'fear', 'disgust', 'anger']
emotions = np.array([
np.array([keyword["emotion"][name] for name in emotionNames])
for keyword in keywords
])
totalEmotions = np.sum(emotions, 0) / numKeywords
features = np.insert(totalEmotions, 0, totalSentiment, axis=0)
return features
def get_keywords(content):
response = service.analyze(text=content,
features=Features(
entities=EntitiesOptions(),
keywords=KeywordsOptions())).get_result()
kword_list = [keyword["text"] for keyword in response["keywords"]]
return json.dumps(kword_list)
def fun_natural_language_understanding(text):
natural_language_understanding = NaturalLanguageUnderstandingV1(
version="2018-09-28", iam_apikey=' ', url='')
features = Features(entities=EntitiesOptions(sentiment=True, model=""))
response = natural_language_understanding.analyze(
text=text, features=features).get_result()
return response
def tweetEmotion(tweet):
# *** WATSON ANALYSIS ***
#returns emotion: joy, anger, disgust, sadness, fear
# ----- WATSON CLOUD LOGIN -----
natural_language_understanding = NaturalLanguageUnderstandingV1(
username="******",
password="******",
version="2017-02-27")
try:
response = natural_language_understanding.analyze(
text=tweet,
features=Features(entities=EntitiesOptions(
emotion=True, sentiment=True,limit=1),
keywords=KeywordsOptions(
emotion=True, sentiment=True,limit=1
))
)
except watson_developer_cloud.watson_service.WatsonApiException as e:
response = []
# print "response", response
json_data = json.dumps(response, indent=2) #loads into objects
json_data = json.loads(json_data) #converts into dictionary
# print "jason_data", json_data
if len(json_data) > 0:
# check where emotions located: keywords or entities
# print len(json_data["keywords"])
# print len(json_data["entities"])
#check if emotion data exists
if (len(json_data["keywords"]) > 0 and "emotion" in json_data["keywords"][0]):
result = json_data["keywords"][0]["emotion"]
print result
elif (len(json_data["entities"]) > 0 and "emotion" in json_data["entities"][0]):
result = json_data["entities"][0]["emotion"]
print result
else:
result = "undef"
print result
# return result
else:
result = "undef"
# print result
# return result
if result == "undef":
return "undef"
else:
maxEmotion = max(result, key=result.get)
print maxEmotion
def analyze(self, sentence):
response_places = []
response = self.natural_language_understanding.analyze(
text=sentence['input'],
features=Features(entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=1),
keywords=KeywordsOptions(emotion=True,
sentiment=True,
limit=1)))
# print(json.dumps(response, indent=2))
'''
places
'''
if len(response['entities']) > 0 and str(
response['entities'][0]['type']).lower() in place_entity_type:
new_input = {}
destination_name = response['keywords'][0]['text']
new_input['current_lat'] = sentence['current_lat']
new_input['current_lng'] = sentence['current_lng']
new_input['destination_name'] = destination_name
location_request = requests.get(
url=config.GOOGLE_GET_LOCATION_GEO.format(
destination_name, config.GOOGLE_API_KEY))
location_geo = json.loads(
location_request.content)['results'][0]['geometry']['location']
new_input['destination_lat'] = location_geo['lat']
new_input['destination_lng'] = location_geo['lng']
return get_recommended_trip(new_input)
else:
keyword = response['keywords'][0]['text']
message_res = WatsonWatsonAssistantAdapter().message(keyword)
places = message_res['output']['text']
for place in places:
google_places_response = requests.get(
url=config.GOOGLE_PLACE_URL.format(place,
config.GOOGLE_API_KEY))
google_place = json.loads(
google_places_response.content)['candidates'][0]
place_element = {}
place_element['name'] = google_place['name']
place_element['image'] = config.GOOGLE_IMAGE_URL.format(
google_place['photos'][0]['photo_reference'],
config.GOOGLE_API_KEY)
place_element['rating'] = google_place['rating']
place_element['lat'] = google_place['geometry']['location'][
'lat']
place_element['lng'] = google_place['geometry']['location'][
'lng']
response_places.append(place_element)
print response_places
response_dict = {}
response_dict['places'] = response_places
response_dict['trips'] = {}
return response_dict
def analizarTexto(USER):
#Open the file with the name of the user.
resultfile = open("result_"+USER+".json", "w+")
targets =[
'vida', 'Guatemala', 'amor', 'sexo', 'politico',
'poliltica','Yo', 'sonrisa', 'pais','novio','novia',
'enojo', 'hermano', 'hermana','mama','papa','familia',
'deporte', 'relacion'
]
#Get the configuration file
with open('config.json', 'r') as f:
config = json.load(f)
#reading the data file
datafile = open("data_" +USER+".txt","r")
data = datafile.read()
#print the data... remove this..
print (data)
#Authentication
natural_language_understanding = NaturalLanguageUnderstandingV1(
version=config["version"],
username=config["username"],
password=config["password"]
)
response = natural_language_understanding.analyze(
text=data,
features=Features(
entities=EntitiesOptions(
emotion=True,
sentiment=True,
limit=2),
keywords=KeywordsOptions(
emotion=True,
sentiment=True,
limit=2),
sentiment=SentimentOptions(
targets=targets
)
#Doesn't support spanish language yet
# ,
# emotion=EmotionOptions(
# targets=targets
# )
)
)
result = str(response)
print (result)
resultfile.write(result + "")
resultfile.close()
f.close()
datafile.close()
def get_entities(self, text=None, url=None):
if text == None and url == None:
return "Error"
response = naturalLanguageUnderstanding.analyze(
text=text,
url=url,
features=Features(entities=EntitiesOptions(
limit=50))).get_result()
return response
def nluRun(txt):
response = natural_language_understanding.analyze(
text=txt,
features=Features(entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=3),
keywords=KeywordsOptions(emotion=True,
sentiment=True,
limit=3)))
return response
def __init__(self, nlu_details: dict) -> None:
self.version = nlu_details["version"]
self.url = nlu_details["url"]
self.apikey = nlu_details["apikey"]
self.nlu = NaturalLanguageUnderstandingV1(
version=self.version, url=self.url, iam_apikey=self.apikey)
self.features = Features(categories=CategoriesOptions(), entities=EntitiesOptions(
emotion=True, sentiment=True), keywords=KeywordsOptions(emotion=True, sentiment=True))
def analyze_article(url):
response = nlu.analyze(url=url,
features=Features(
entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=2),
keywords=KeywordsOptions(emotion=True,
sentiment=True,
limit=2)))
return response
def get_response(text):
response = natural_language_understanding.analyze(
text=text,
features=Features(entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=2),
keywords=KeywordsOptions(emotion=True,
sentiment=True,
limit=2)))
return response['keywords']
def main(args):
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2017-02-27',
username='******',
password='******')
try:
response = natural_language_understanding.analyze(
text=args.get("text", None),
url=args.get("url", None),
html=args.get("html", None),
features=Features(entities=EntitiesOptions(),
emotion=EmotionOptions()))
except WatsonException as ex:
return {"NULL": "NULL"}
if (args.get("type", "Emotion") == "Emotion"):
result = emotion2result(response)
return result
itemlist = dic2item(response)
wiki_query = "http://en.wikipedia.org/w/api.php?action=query&" \
"prop=extracts&format=json&exintro=&titles="
count = 0
index = 0
extractlist = {}
while (count < 3 and index < len(itemlist)):
temp = itemlist[index][0].encode("utf8")
item = temp.split(" ")
string = ""
for i in item:
string += i + "+"
string = string[:len(string) - 1]
res = try_url(wiki_query + string)
# print res
res_json = json.loads(res)
extract = res_json["query"]["pages"]
pagenum = extract.keys()[0]
if (pagenum != "-1"):
count += 1
extract = extract[pagenum]
extract = extract["extract"]
extract = extract.encode("utf8")
slist = extract.split(". ")
if (slist is not None):
extract = slist[0] + "."
extract = clean(extract)
extractlist[itemlist[index][0]] = extract
index += 1
if (extractlist == {}):
return {"NULL": "NULL"}
return extractlist
def analyse_text(text):
print("Texto: %s" % text)
response = natural_language_understanding.analyze(
text=text,
features=Features(
entities=EntitiesOptions(emotion=True, sentiment=True),
keywords=KeywordsOptions(emotion=True, sentiment=True),
)).get_result()
return response
