def understanding(self):
if not self.transcription:
self.transcript()
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2017-02-27',
username=os.environ['UNDERSTANDING_USERNAME'],
password=os.environ['UNDERSTANDING_PASSWORD'])
self.analysis = natural_language_understanding.analyze(
text=self.transcription['results'][0]['alternatives'][0]
['transcript'],
features=Features(categories=CategoriesOptions(),
concepts=ConceptsOptions(),
emotion=EmotionOptions(),
entities=EntitiesOptions(emotion=True,
sentiment=True,
mentions=True),
keywords=KeywordsOptions(emotion=True,
sentiment=True),
relations=RelationsOptions(),
sentiment=SentimentOptions()))
logger.info('Completed analysis of recorded file')
return self.analysis
def topic_ibm(content):
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2018-11-16',
iam_apikey="MpIV_8-gUahq8R3WzNh7C0g1JgDZGueVrUzq8441rQVb",
url=
"https://gateway-lon.watsonplatform.net/natural-language-understanding/api"
)
response = natural_language_understanding.analyze(
text=content,
features=Features(categories=CategoriesOptions(limit=3))).get_result()
fichier = json.dumps(response, indent=2)
print(fichier)
data_dict_02 = json.loads(fichier)
if len(data_dict_02["categories"]) > 2:
category1 = data_dict_02["categories"][0]["label"]
category2 = data_dict_02["categories"][1]["label"]
resultat1 = category1.split("/")
resultat2 = category2.split("/")
print('resultat1 ', resultat1)
print('resultat2 ', resultat2)
resultat = []
resultat.append(resultat1[1])
resultat.append(resultat2[1])
print(resultat)
return resultat
def ibmContent(text):
response1 = natural_language_understanding.analyze(
text=text,
features=Features(keywords=KeywordsOptions(limit=10))).get_result()
jData = json.loads(json.dumps(response1, indent=2,
ensure_ascii=False)) #keyword 추출
response2 = natural_language_understanding.analyze(
text=text,
features=Features(categories=CategoriesOptions(limit=3))).get_result()
jData2 = json.loads(json.dumps(response2, indent=2,
ensure_ascii=False)) #카테고리 추출
keywords = ''
relevance = ''
categories = ''
for i in jData['keywords']:
keywords += i['text']
keywords += '#'
for i in jData['keywords']:
relevance += str(i['relevance'])
relevance += '#'
for i in jData2['categories']:
categories += i['label']
categories += '#'
return keywords, relevance, categories
def watson_nlp_analysis(text):
if text == '': return text
max_limit_one = 10
max_limit_two = 30
naturalLanguageUnderstanding = NaturalLanguageUnderstandingV1(
version='2018-11-16',
iam_apikey=os.environ['WATSON'],
url=
'https://gateway.watsonplatform.net/natural-language-understanding/api'
)
response = naturalLanguageUnderstanding.analyze(
text=text,
features=Features(concepts=ConceptsOptions(limit=max_limit_one),
categories=CategoriesOptions(limit=max_limit_one),
sentiment=SentimentOptions(document=True),
emotion=EmotionOptions(document=True),
entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=max_limit_two),
keywords=KeywordsOptions(
emotion=True,
sentiment=True,
limit=max_limit_two))).get_result()
return response
def process_text(self, conv):
unicodedata.normalize('NFKD', conv).encode('ascii', 'ignore')
nlp = self.nlp_api.analyze(
text=conv,
language='es',
features=Features(keywords=KeywordsOptions(emotion=True,
sentiment=True),
categories=CategoriesOptions(),
sentiment=SentimentOptions(targets=self.flags)))
if self.log_file != None:
print('INFO: logging NLP to %s\n' % self.log_file)
with open(self.log_file, 'w') as outfile:
json.dump(nlp, outfile)
doc_score = 0
target_score = []
try:
doc_score = nlp['sentiment']['document']['score']
for target in nlp['sentiment']['targets']:
target_score.append({
'text': target['text'],
'score': target['score']
})
except KeyError:
print('INFO: no target found')
return doc_score, target_score
def nlu():
f = open(november_tweets, 'r', encoding='UTF8')
g = open(november_gt, 'a')
e = open(november_opinion, 'a')
num_lines = sum(1 for line in open(november_tweets, encoding='UTF8'))
while num_lines != 0:
nline = f.readline()
newline = str(nline)
if len(newline) > 15:
response2 = natural_language_understanding.analyze(
text=newline,
language='en',
features=Features(keywords=KeywordsOptions(emotion=True,
sentiment=True,
limit=2),
categories=CategoriesOptions()))
aaa = (json.dumps(response2, indent=2))
print(aaa)
bbb = json.loads(aaa)
single_thing = (str(bbb['categories']))
single_sentiment = (str(bbb['keywords']))
g.write(single_thing + '\n')
e.write(single_sentiment + '\n')
else:
g.write("Not enough data" + '\n')
e.write("Not enough data" + '\n')
num_lines -= 1
return num_lines
def concepts(self):
IBM_dict = {}
IBM_response = self.naturalLanguageUnderstanding.analyze(
text=self.text,
features=Features(
entities=EntitiesOptions(emotion=True, sentiment=True, limit=10),
keywords=KeywordsOptions(emotion=True, sentiment=True,limit=10),
sentiment=SentimentOptions(),
categories=CategoriesOptions()
)).get_result()
sent_dict = {'sentiment': IBM_response['sentiment']['document']['score']}
IBM_dict['sentiment'] = sent_dict
ent_result = []
ents = IBM_response['entities']
for e in ents:
ent_result.append(e['text'].lower())
ent_result.sort()
IBM_dict['entities'] = ent_result
kws = []
for keyword in IBM_response['keywords']:
kws.append(keyword['text'].lower())
kws.sort()
IBM_dict['keywords'] = kws
cats = []
for category in IBM_response['categories']:
cats.append(category['label'])
IBM_dict['categories'] = cats
return IBM_dict
def get_keywords_from_url(url):
text = None
if 'reddit' in url:
submission = _reddit.submission(url=url)
if submission.selftext is not None:
text = submission.selftext
url = None
else:
return []
res = _watson_nlu.analyze(
url=url,
text=text,
features=Features(
categories=CategoriesOptions(),
concepts=ConceptsOptions(limit=5)
)).get_result()
kwds = set()
if 'categories' in res:
categories = res['categories']
categories = sorted(categories, key=extract_relevancy, reverse=True)[:10]
for category in categories:
labels = re.split(',| |/', category['label'])
for label in labels:
kwds.add(label)
if 'concepts' in res:
for concept in res['concepts']:
kwds.add(concept['text'])
for stopword in _stopwords:
if stopword in kwds:
kwds.remove(stopword)
return list(kwds)
def getres(file_loc):
with open(file_loc, 'r+') as f:
head = f.readline()
content = f.read()
req = '<html><body><h2>{0}</h2>{1}</body></html>'.format(head, content)
text = head + content
tone_res = tone_analyzer.tone(req, content_type='text/html').get_result()
res = natural_language_understanding.analyze(
html=req,
features=Features(
categories=CategoriesOptions(limit=1),
concepts=ConceptsOptions(limit=5),
keywords=KeywordsOptions(limit=5, sentiment=True, emotion=True),
sentiment=SentimentOptions(),
# entities=EntitiesOptions(limit=5, mentions=True, sentiment=True, emotion=True),
),
).get_result()
sentiment = res["sentiment"]["document"]["score"]
concepts = [(concepts["text"], concepts["relevance"])
for concepts in res["concepts"]]
categories = (res["categories"][0]["label"].split("/"),
res["categories"][0]["score"])
keywords = [(keywords["text"], keywords["relevance"])
for keywords in res["keywords"]]
tones = [(tone["tone_id"], tone["score"])
for tone in tone_res["document_tone"]["tones"]]
return (sentiment, concepts, keywords, tones, text)
def getAPIResponse(textToConvert): response = understandingObj.analyze( text = str(textToConvert), features=Features( categories=CategoriesOptions() ) ) return response
def __init__(self, nlu_details: dict) -> None:
self.version = nlu_details["version"]
self.url = nlu_details["url"]
self.apikey = nlu_details["apikey"]
self.nlu = NaturalLanguageUnderstandingV1(
version=self.version, url=self.url, iam_apikey=self.apikey)
self.features = Features(categories=CategoriesOptions(), entities=EntitiesOptions(
emotion=True, sentiment=True), keywords=KeywordsOptions(emotion=True, sentiment=True))
def get_news_sentiment(request):
try:
response = natural_language_understanding.analyze(
url=request.GET.get('url'),
features=Features(sentiment=SentimentOptions(),
emotion=EmotionOptions(),
concepts=ConceptsOptions(limit=5),
categories=CategoriesOptions()))
return Response(response)
except:
return Response({"error": 'problem retrieving'})
def get_categories(text):
try:
response = natural_language_understanding.analyze(
text=text, features=Features(categories=CategoriesOptions()))
categories = []
for category in response["categories"]:
categories.append(category["label"])
return categories
except:
return []
def queryWatson(headline, story):
return nlu.analyze(text=headline + "; " + story,
language="en",
features=Features(
entities=EntitiesOptions(emotion=False,
sentiment=False,
limit=60),
keywords=KeywordsOptions(emotion=False,
sentiment=False,
limit=60),
categories=CategoriesOptions(limit=60),
concepts=ConceptsOptions(limit=50)))
def call_api(self, content):
try:
response = self.client.analyze(
text=content,
features=Features(entities=EntitiesOptions(),
keywords=KeywordsOptions(),
categories=CategoriesOptions(),
concepts=ConceptsOptions())).get_result()
except WatsonException as exception:
print(exception)
response = {"ibm_exception": str(exception)}
return json.dumps(response, sort_keys=True, indent=4)
def analyze_using_NLU(analysistext):
res = dict()
response = natural_language_understanding.analyze(
text=analysistext,
features=Features(sentiment=SentimentOptions(),
entities=EntitiesOptions(),
keywords=KeywordsOptions(),
emotion=EmotionOptions(),
concepts=ConceptsOptions(),
categories=CategoriesOptions(),
semantic_roles=SemanticRolesOptions()))
res['results'] = response
return json.dumps(res)
def get_analysis(self, text):
"""
Returns a dictionary containing categories and entities
"""
response = self.user.analyze(
text=text,
clean=True,
language="pt",
features=Features(categories=CategoriesOptions(),
entities=EntitiesOptions()),
)
return response
def get_keywords(self, sentence):
response = self.natural_language_understanding.analyze(
text=sentence,
return_analyzed_text='True',
features=Features(concepts=ConceptsOptions(),
categories=CategoriesOptions(),
relations=RelationsOptions(),
semantic_roles=SemanticRolesOptions(),
sentiment=SentimentOptions(),
entities=EntitiesOptions(),
keywords=KeywordsOptions())).get_result()
keywords = map(lambda x: (x['text'], x['type']), response['entities'])
return keywords
def analyze(links):
for link in links:
response = natural_language_understanding.analyze(
url=link,
features=Features(
entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=15),
emotion=EmotionOptions(targets=['keyword1', 'keyword2']),
keywords=KeywordsOptions(emotion=True, sentiment=True,
limit=2),
concepts=ConceptsOptions(limit=5),
sentiment=SentimentOptions(targets=['stocks']),
categories=CategoriesOptions()))
def analyze_using_NLU(analysistext):
""" Extract results from Watson Natural Language Understanding for each news item
"""
res = dict()
response = natural_language_understanding.analyze(
text=analysistext,
features=Features(
sentiment=SentimentOptions(),
entities=EntitiesOptions(),
keywords=KeywordsOptions(),
emotion=EmotionOptions(),
concepts=ConceptsOptions(),
categories=CategoriesOptions(),
))
res['results'] = response
return res
def __init__(self, nlu_vcap):
self.version = nlu_vcap["version"]
self.url = nlu_vcap["url"]
self.username = nlu_vcap["username"]
self.password = nlu_vcap["password"]
self.nlu = NaturalLanguageUnderstandingV1(version=self.version,
url=self.url,
username=self.username,
password=self.password)
self.features = Features(categories=CategoriesOptions(),
entities=EntitiesOptions(emotion=True,
sentiment=True),
keywords=KeywordsOptions(emotion=True,
sentiment=True))
def interface(Purl):
response = naturalLanguageUnderstanding.analyze(
url=Purl,
features=Features(sentiment=SentimentOptions(),
categories=CategoriesOptions(limit=1),
concepts=ConceptsOptions(limit=1),
entities=EntitiesOptions(limit=1),
metadata=MetadataOptions())).get_result()
result = json.dumps(response, indent=2)
#Sentiment
vLabel = response['sentiment']['document']['label']
score = response['sentiment']['document']['score']
mdTitle = response['metadata']['title']
mdDate = response['metadata']['publication_date']
vScore = math.ceil(float(score) * 100)
#category
try:
for category in response['categories']:
categoryLabel = category['label']
for concept in response['concepts']:
conceptText = concept['text']
conceptRelevance = str(concept['relevance'])
conceptSource = concept['dbpedia_resource']
for entity in response['entities']:
entityText = entity['text']
entityType = entity['type']
except:
vScore = "ERROR"
vLabel = "ERROR"
categoryLabel = "ERROR"
conceptText = "ERROR"
conceptRelevance = "ERROR"
conceptSource = "ERROR"
entityText = "ERROR"
entityType = "ERROR"
mdTitle = "ERROR"
mdDate = "ERROR"
print(response)
return vLabel, vScore, categoryLabel, conceptText, conceptRelevance, conceptSource, entityText, entityType, mdTitle, mdDate
def analyze_category_test(input_folder_path, user_id, output_folder_path):
error_list = list()
file_path = input_folder_path + user_id + ".txt"
with open(file_path, 'r', encoding='utf-8') as myfile:
data = myfile.read()
try:
response = natural_language_understanding.analyze(
text=data, features=Features(categories=CategoriesOptions()))
except:
print(user_id)
response = {'categories': 'unknown'}
#output
output_path = output_folder_path + user_id + ".json"
if not os.path.exists(output_folder_path):
os.makedirs(output_folder_path)
with open(output_path, 'w') as outfile:
json.dump(response, outfile)
def process_batch(self, collection):
target_score = []
doc_score = 0
found = 0
for conv in collection:
conv = conv['sentence']
unicodedata.normalize('NFKD', conv).encode('ascii', 'ignore')
nlp = self.nlp_api.analyze(
text=conv,
language='es',
features=Features(
keywords=KeywordsOptions(emotion=True, sentiment=True),
categories=CategoriesOptions(),
sentiment=SentimentOptions(targets=self.flags)))
try:
doc_score += nlp['sentiment']['document']['score']
for target in nlp['sentiment']['targets']:
target_score.append({
'text': target['text'],
'score': target['score']
})
found += 1
except KeyError:
print('INFO: no target found in sentence')
if found == 0:
doc_score = 0
else:
doc_score /= found
json_out = {
'sentiment': {
'document': {
'score': doc_score
},
'targets': target_score
}
}
if self.log_file != None:
print('INFO: logging NLP to %s\n' % self.log_file)
with open(self.log_file, 'w') as outfile:
json.dump(json_out, outfile)
return doc_score, target_score
def insights(text):
natural_language_understanding = NaturalLanguageUnderstandingV1(
username='******',
password='******',
version='2018-03-16')
response = natural_language_understanding.analyze(
text=text,
features=Features(entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=2),
categories=CategoriesOptions(),
relations=RelationsOptions(),
keywords=KeywordsOptions(emotion=True,
sentiment=True,
limit=2)))
return response
def generate(self, duvida=8):
natural_language_understanding = NaturalLanguageUnderstandingV1(
username='******', password='******', version='2018-03-16')
vr_text = duvida
response = natural_language_understanding.analyze(
text=vr_text,
language='pt',
features=Features(categories=CategoriesOptions(),
entities=EntitiesOptions(model='xxxxxxxxxxxx')))
resposta = ''
for i in response['entities']:
resposta.join(i['text'])
return ''.join(resposta)
def resultjson():
if request.method == 'POST':
#text1 = request.form
text1 = request.get_json(force=True)
user = natural_language_understanding.analyze(
text=str(text1),
features=Features(entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=2),
relations=RelationsOptions(),
categories=CategoriesOptions(),
semantic_roles=SemanticRolesOptions(),
concepts=ConceptsOptions(limit=3),
keywords=KeywordsOptions(emotion=True,
sentiment=True,
limit=2)))
return jsonify(user)
def understand_transcript(transcription):
nlp_client = NaturalLanguageUnderstandingV1(
version="2017-02-27",
username=os.environ["WATSON_UNDERSTANDING_USERNAME"],
password=os.environ["WATSON_UNDERSTANDING_PASSWORD"],
)
return nlp_client.analyze(
text=transcription["results"][0]["alternatives"][0]["transcript"],
features=Features(
categories=CategoriesOptions(),
concepts=ConceptsOptions(),
emotion=EmotionOptions(),
entities=EntitiesOptions(),
keywords=KeywordsOptions(),
sentiment=SentimentOptions(),
),
)
def analyseVideo(text):
response = natural_language_understanding.analyze(
text = text,
features=Features(
categories=CategoriesOptions(),
entities=EntitiesOptions(
emotion=True,
sentiment=True,
limit=3),
keywords=KeywordsOptions(
emotion=True,
sentiment=True,
limit=3)))
#json.dumps(response, indent=2)
print("Classification: ")
for cat in response["categories"]:
print(str(cat["label"]) + "(" + str(round(cat["score"]*100)) + "%)")
print("\n")
def watson_nlc():
import json
from watson_developer_cloud import NaturalLanguageUnderstandingV1
from watson_developer_cloud.natural_language_understanding_v1 import Features, CategoriesOptions
from ConfigParser import SafeConfigParser
config = SafeConfigParser()
config.read('/etc/watson_cfg.ini')
natural_language_understanding = NaturalLanguageUnderstandingV1(
username=config.get('watson', 'username'),
password=config.get('watson', 'password'),
version=config.get('watson', 'version'))
response = natural_language_understanding.analyze(
url='https://fr.wikipedia.org/wiki/Gen%C3%A8ve',
features=Features(categories=CategoriesOptions()))
return (json.dumps(response))
