def replace_words(self, newpoem, feelings):
try:
feelings = str(feelings)
newpoem = str(newpoem)
poem_nla = \
self.natural_language_understanding.analyze(text=newpoem,
features=[features.Keywords()])
feelings_nla = \
self.natural_language_understanding.analyze(text=feelings,
features=[features.Keywords()])
fw = []
for keyword in feelings_nla["keywords"]:
fw.append(keyword['text'].encode('utf-8'))
pw = []
for keyword in poem_nla["keywords"]:
pw.append(keyword['text'].encode('utf-8'))
x = randint(0, len(pw) - 1)
y = randint(0, len(fw) - 1)
return newpoem.replace(pw[x], fw[y])
except:
return None
def eval_default():
response = nlu.analyze(
text='Bruce Banner is the Hulk and Bruce Wayne is BATMAN! '
'Superman fears not Banner, but Wayne.',
features=[features.Entities(),
features.Keywords()])
return jsonify(response)
def parse_foods(path, format='mp3'):
"""
Parses the audio file & returns the foods that are contained in the speech
:param path: path to the audio file
:return: the list of foods
"""
with open(path, 'rb') as audio:
txt = json.dumps(SpeechAnalysis.speech_to_text(
'6d736cd4-1e01-4d6c-9c61-78ce7f803024',
'SBE03B3o5PTP').recognize(audio,
content_type='audio/' + format,
timestamps=True,
word_confidence=True),
indent=2)
json_text = json.loads(txt)
text = json_text['results'][0]['alternatives'][0]['transcript']
keywords = SpeechAnalysis.nlp_parse(
'991ee8cd-25f9-4b5a-a1f8-cfe5834dfcb9',
'Jtera8xCDVup').analyze(text=text,
features=[features.Keywords()])
s = pd.Series(keywords)
keyword_list = []
for k in s.keywords:
keyword_list.append(k['text'])
#food = wn.synset('food.n.02')
food = pickle.load(open("./nltk_food2.p", "rb"))
foods = [k for k in keyword_list if k in food]
return json.dumps(foods)
def run_watson_nlu():
files = glob.glob('work/bug-*.json')
(user, passwd) = get_auth()
for fname in files:
with open(fname) as f:
LOG.debug("Processing %s" % fname)
bug = json.loads(f.read())
num = bug["link"].split("/")[-1]
with open("work/res-%s.json" % num, "w") as out:
nlu = watson_developer_cloud.NaturalLanguageUnderstandingV1(
version='2017-02-27', username=user, password=passwd)
res = nlu.analyze(text=bug["comments"],
features=[
features.Concepts(),
features.Keywords(),
features.Emotion(),
features.Sentiment(),
])
output = {
"link": bug["link"],
"tags": bug["tags"],
"importance": bug["importance"],
"length": len(bug["comments"]),
"results": res
}
out.write(json.dumps(output, indent=4))
def get_keywords(text):
natural_language_understanding = NaturalLanguageUnderstandingV1(
username="******",
password="******",
version="2017-02-27")
response = natural_language_understanding.analyze(
text=text,
features=[
Features.Entities(emotion=True, sentiment=True, limit=10),
Features.Keywords(emotion=True, sentiment=True, limit=10)
])
# print(json.dumps(response, indent=2))
to_ret = {}
for a in response["keywords"]:
to_ret.update({a['text']: '#'})
for a in response["entities"]:
try:
to_ret.update({a['text']: a['disambiguation']['dbpedia_resource']})
except:
to_ret.update({a['text']: '#'})
return to_ret
def watsonanalyze(stringinput):
response = natural_language_understanding.analyze(
text=stringinput,
features=[
Features.Entities(emotion=True, sentiment=True, limit=2),
Features.Keywords(emotion=True, sentiment=True, limit=20)
])
return json.loads(json.dumps(response, indent=2))
def alexa_analyze():
if (request.method == 'POST'):
content = request.get_json(force=True)
name = content["request"]["intent"]["name"]
if (name == "IsRecyclable"):
query = content["request"]["intent"]["slots"]["recycle"]["value"]
elif (name == "IsCompostable"):
query = content["request"]["intent"]["slots"]["compost"]["value"]
if (query):
NLU_response = NLU.analyze(text=query,
features=[features.Keywords()])
desc = str(NLU_response)
for i in NLU_response["keywords"]:
payload = "key={}&txt={}&model={}".format(
MC_KEY, i["text"], MC_MODEL)
MC_response = requests.request("POST",
url=MC_URL,
data=payload,
headers=MC_HEADERS)
MC_response = json.loads(MC_response.text)
desc = ("No, you cannot recycle a " + query) if (
name == "IsRecyclable") else ("No, you cannot compost a " +
query)
if (len(MC_response["category_list"]) > 0):
if (MC_response["category_list"][0]["label"]
== "Compostable" and name == "IsCompostable"):
desc = "Yes, you can compost a {}".format(query)
break
elif (MC_response["category_list"][0]["label"]
== "Recyclable" and name == "IsRecyclable"):
desc = "Yes, you can recycle a {}".format(query)
break
alexa_response = {
"version": "1.0",
"response": {
"outputSpeech": {
"type": "PlainText",
"text": desc,
}
}
}
return jsonify(alexa_response)
else:
desc = ("No, you cannot recycle a " + query) if (
name == "IsRecyclable") else ("No, you cannot compost a " +
query)
alexa_response = {
"version": "1.0",
"response": {
"outputSpeech": {
"type": "PlainText",
"text": desc,
}
}
}
return jsonify(alexa_response)
def nl_processing(reqd_text):
response = natural_language_understanding.analyze(text=reqd_text,
features=[
features.Entities(),
features.Keywords(),
features.Emotion(),
features.Concepts(),
features.Sentiment()
])
return response
def nlp(input_stuff):
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2017-02-27',
username="******",
password="******")
response = natural_language_understanding.analyze(
text=input_stuff, features=[features.Entities(),
features.Keywords()])
return (response["entities"])
def analyze(s):
response = natural_language_understanding.analyze(text=s,
features=[
Features.Keywords(
emotion=True,
sentiment=True,
limit=2),
Features.Sentiment()
])
return response
def nlp(input_stuff):
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2017-02-27',
username='******',
password='******')
response = natural_language_understanding.analyze(
text=input_stuff, features=[features.Entities(),
features.Keywords()])
return (response["entities"])
def _execute_request(text):
natural_language_understanding = NaturalLanguageUnderstandingV1(
username=constants.WATSON_USER,
password=constants.WATSON_PASS,
version="2017-02-27")
response = natural_language_understanding.analyze(
text=text, features=[features.Keywords()])
return response
def analyze_sentiment(self, answer):
result = self.nlu.analyze(
text=answer, features=[features.Keywords(),
features.Sentiment()])
if result['keywords']:
keywords = result['keywords'][0]
keyword = keywords['text']
else:
keyword = None
sentiment = result['sentiment']['document']['score']
return sentiment, keyword
def main():
with open("compost.txt", "r") as f:
strings = f.read().split()
while (strings):
response = NLU.analyze(text=strings.pop(0),
features=[
features.Keywords(),
features.Categories(),
features.Concepts()
])
print str(response.keywords[0].text)
def analyze_url(url):
try:
response = natural_language_understanding.analyze(
url=url, features=[features.Keywords(),
features.Categories()])
except WatsonException as e:
print("analyze_url: WatsonException")
if 'Error: limit exceeded for free plan, Code: 403' in e.args:
raise e
return None
return response
def worker():
global q
print 'Worker Initialized'
attributes = ['id', 'text', 'time', 'lat', 'lon']
while True:
responses = q.receive_messages(MessageAttributeNames=attributes)
if len(responses) != 0:
for response in responses:
if response.message_attributes is None:
response.delete()
continue
id = response.message_attributes.get('id').get('StringValue')
text = response.message_attributes.get('text').get(
'StringValue')
time = response.message_attributes.get('time').get(
'StringValue')
lat = response.message_attributes.get('lat').get('StringValue')
lon = response.message_attributes.get('lon').get('StringValue')
try:
natural_language_understanding = NaturalLanguageUnderstandingV1(\
version='2017-02-27',\
username=nlu_creds['username'],\
password=nlu_creds['password']\
)
nlu_response = natural_language_understanding.analyze(\
text=text,\
features=[features.Entities(), features.Keywords(), features.Sentiment()]\
)
sentiment = nlu_response['sentiment']['document']['label']
except Exception as e:
print 'Error:', e
sentiment = 'neutral'
# Send to AWS SNS
notification = {
'id': id,
'text': text,
'time': time,
'lat': lat,
'lon': lon,
'sentiment': sentiment
}
try:
print notification
sns.publish(TargetArn=sns_arn,
Message=json.dumps(
{'default': json.dumps(notification)}))
response.delete()
except Exception as e:
print 'Error:', e
sleep(2)
def get_keywords(text):
try:
response = natural_language_understanding.analyze(
text=text, features=[features.Keywords()])
l = list(map(lambda k: k['text'], response['keywords']))
except Exception as e:
print(e)
return []
return l
def open_request():
data = request.get_json()
text = data['transcription']
patient_id = data['patient_id']
if text not in watson_cache:
try:
watson_cache[text] = natural_language_understanding.analyze(
text=text,
features=[features.Keywords(), features.Sentiment()])
except WatsonException as err:
print err
enqueue(patient_id, text, watson_cache[text])
return jsonify({'result': watson_cache[text]})
def extract_sentiment_ner_trec_full(src, dst_dir):
"""
Extracts tweet overall sentiment, sentiment per NER, NERs,
keywords, sentiment per keyword for the full dataset that's read from a
.txt file.
Parameters
----------
src: str - path to dataset.
dst_dir: - directory in which results will be stored.
"""
tweets = read_txt(src)
# Since tweets are ordered according to topic, label them in a
# random order
keys = tweets.keys()
random.shuffle(keys)
for idx, tid in enumerate(keys):
fname = "{}.json".format(tid)
dst = os.path.join(dst_dir, fname)
# If file already exists, data was extracted before and due to
# rate-limiting the rest couldn't be extracted
if not os.path.isfile(dst):
try:
# Extract features for a tweet via Watson
response = natural_language_understanding.analyze(
text=tweets[tid]["text"],
# Get entities and their
features=[
# Overall tweet sentiment
Features.Sentiment(),
# NER detection and sentiment per NER
Features.Entities(sentiment=False),
Features.Keywords(sentiment=False),
])
# Store results in UTF-8 encoding
fname = "{}.json".format(tid)
dst = os.path.join(dst_dir, fname)
with codecs.open(dst, "w", encoding="utf-8") as f:
# https://stackoverflow.com/questions/18337407/saving-utf-8-texts-in-json-dumps-as-utf8-not-as-u-escape-sequence
data = json.dumps(response,
ensure_ascii=False,
encoding='utf8')
f.write(unicode(data))
# Illegal language
except watson_developer_cloud.watson_developer_cloud_service.\
WatsonException:
pass
print "Finished extraction for {} tweets".format(idx + 1)
def get_text_data(text,language):
username = os.environ.get("BLUEMIX-NLU-USERNAME")
password = os.environ.get("BLUEMIX-NLU-PASSWORD")
natural_language_understanding = NaturalLanguageUnderstanding(
version = "2017-02-27",
username=username,
password=password
)
return natural_language_understanding.analyze(
text = text,
features = [features.Emotion(), features.Sentiment(), features.Keywords()],
language = language
)
def map_feature(name):
feature_name_mappings = {
'keywords': features.Keywords(),
'entities': features.Entities(),
'concepts': features.Concepts(),
'categories': features.Categories(),
'sentiment': features.Sentiment(),
'emotion': features.Emotion()
}
if name in feature_name_mappings:
return feature_name_mappings[name]
else:
print("Invalid feature name")
return None
def watson(user_url):
natural_language_understanding = NaturalLanguageUnderstandingV1(
username="******",
password="******",
version="2017-02-27")
response = natural_language_understanding.analyze(
url=user_url,
features=[Features.Keywords(emotion=False, sentiment=False, limit=15)])
keywords = []
for keyword in response['keywords']:
if keyword['relevance'] > 0.80 and len(keywords) < 8:
keywords.append(keyword['text'].encode('utf-8'))
return keywords
def nlu(text):
response = n.analyze(text=text,
features=[
features.Emotion(),
features.Concepts(),
features.Categories(),
features.Entities(),
features.Keywords(),
features.SemanticRoles(),
features.Relations(),
features.Sentiment()
],
language='en')
return json.dumps(response, indent=2)
def extract_data(text):
# Use Watson's NLU API to extract the keywords, entities and concepts from a text
bm_username = "******"
bm_password = "******"
nlu = watson_developer_cloud.NaturalLanguageUnderstandingV1(
version='2017-02-27', username=bm_username, password=bm_password)
ents = nlu.analyze(text=text,
features=[
features.Entities(),
features.Keywords(),
features.Concepts()
])
ents["tweet"] = text
return ents
def test_model():
model = request.forms.get('model')
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2017-02-27',
username="******",
password='******')
response = natural_language_understanding.analyze(text=model,
features=[
features.Entities(),
features.Keywords(),
features.Concepts()
])
return (json.dumps(response, indent=2))
def extract_sentiment_ner_twitter(cleaned, min_annos, dst_dir):
"""
Extracts tweet overall sentiment, sentiment per NER, NERs,
keywords, sentiment per keyword.
Parameters
----------
cleaned: bool - True if only cleaned data should be used (i.e. any
additional labels (and their annotation times) assigned to tweets considered
"Irrelevant" are ignored)
min_annos: int - minimum number of annotators who must've labeled a tweet
for it to be considered.
dst_dir: - directory in which results will be stored.
"""
tweets = read_twitter(cleaned, min_annos)
for idx, tid in enumerate(tweets):
tweet = tweets[tid]
fname = "{}.json".format(tid)
dst = os.path.join(dst_dir, fname)
print tweet["text"]
# If file already exists, data was extracted before and due to
# rate-limiting the rest couldn't be extracted
if not os.path.isfile(dst):
# Extract features for a tweet via Watson
response = natural_language_understanding.analyze(
text=tweet["text"],
# Get entities and their
features=[
# Overall tweet sentiment
Features.Sentiment(),
# NER detection and sentiment per NER
Features.Entities(sentiment=True),
Features.Keywords(sentiment=True),
])
# Store results in UTF-8 encoding
fname = "{}.json".format(tid)
dst = os.path.join(dst_dir, fname)
with codecs.open(dst, "w", encoding="utf-8") as f:
# https://stackoverflow.com/questions/18337407/saving-utf-8-texts-in-json-dumps-as-utf8-not-as-u-escape-sequence
data = json.dumps(response,
ensure_ascii=False,
encoding='utf8')
f.write(unicode(data))
print "Finished extraction for {} tweets".format(idx + 1)
def respond_chat(request):
print("respond_chat got called")
global response
global inp
# Getting the last context for the concerned user
last_context = Mess.objects.filter(user=request.user).last().context
# print(last_context.last().created)
# Sending the message to the bot and fetching a response
print("INP------------")
print(inp)
nlu_response = nlu.analyze(
text=inp, features=[features.Entities(),
features.Keywords()])
if (last_context == ""):
response = conversation.message(workspace_id=workspace_id,
message_input={'text': inp})
else:
response = conversation.message(workspace_id=workspace_id,
message_input={'text': inp},
context=eval(last_context))
action = identifier(response, nlu_response)
print("ACTION REQUESTED")
print(action)
# CHECK WHAT THE ACTION IS
# print(type(response['intents']))
# if(response['intents']['intent'][0] == "create_playlist"):
# print("USER WANTS A PLAYLIST MAN")
new_mess = Mess(text=response['output']['text'][0],
user=request.user,
created=timezone.now(),
reality_coefficient=False,
context=repr(response['context']))
new_mess.save()
mess = Mess.objects.all()
response_text = serializers.serialize('json', Mess.objects.all())
return HttpResponse(response_text, content_type='application/json')
def clasificarDescripcion(pDescripcion):
# if 'VCAP_SERVICES' in os.environ:
# services = json.loads(os.getenv('VCAP_SERVICES'))
with open('AutosClasificados\core\config.json') as json_data_file:
vConfig = json.load(json_data_file)
vAPIUserNLU= vConfig["watsonNLU"]["vAPIUser"]
vAPIPassNLU = vConfig["watsonNLU"]["vAPIPass"]
vAPIVersionNLU = vConfig["watsonNLU"]["vAPIVersion"]
vUmbralMinScore_WNLU = vConfig["watsonNLU"]["vUmbralMinScore_WNLU"]
vUmbralMinDescripcion = vConfig["otros"]["vUmbralMinDescripcion"]
vResultado_NLU = ''
vWatson_NLU = NaturalLanguageUnderstandingV1(username=vAPIUserNLU, password=vAPIPassNLU, version=vAPIVersionNLU)
vListaKeywords = list()
try:
if len(pDescripcion) > vUmbralMinDescripcion:
vResultado_NLU = vWatson_NLU.analyze(
text=pDescripcion,
features=[
Features.Entities(
emotion=True,
sentiment=True,
limit=6
),
Features.Keywords(
emotion=True,
sentiment=True,
limit=6
)
],
language="en"
)
vResultado_NLU = json.loads(json.dumps(vResultado_NLU, indent=2))
if vResultado_NLU['keywords']:
for entitien in vResultado_NLU['entities']:
print(entitien)
for vResultado in vResultado_NLU['keywords']:
print(vResultado)
if vResultado['relevance'] > vUmbralMinScore_WNLU:
vListaKeywords.append(vResultado['text'])
return vListaKeywords
except:
vListaKeywords.append('No hay Keywords disponibles')
return vListaKeywords
def get_sentiment(txt):
response = natural_language_understanding.analyze(
text=txt,
features=[
Features.Entities(
emotion=True,
sentiment=True,
limit=2
),
Features.Keywords(
emotion=True,
sentiment=True,
limit=2
)
]
)
print(json.dumps(response, indent=2))
def report(self, text):
"""
Returns the Watson Data for a specific text.
"""
# Real Call
payload = self.natural_language_understanding.analyze(
text=text,
features=[
features.Entities(),
features.Keywords(),
features.Emotion()
])
# Fake Call, since we only have limited access to IBM
# payload = self.mock_watson(text)
return payload