def makeWikiSection(self, sectiontitle):
print("Accessing IBM Watson for NLP understanding on " + sectiontitle +
" (subtopic of " + self._topic + ")")
response = self.watsonobj.analyze(
text=self._page.section(sectiontitle),
features=Features(concepts=ConceptsOptions(limit=3),
entities=EntitiesOptions(limit=3),
keywords=KeywordsOptions(limit=5),
relations=RelationsOptions(),
semantic_roles=SemanticRolesOptions(limit=3)))
if sectiontitle in wikipedia.search(
sectiontitle) and sectiontitle is not "See also":
return Node("Section",
title=sectiontitle,
content=self._page.section(sectiontitle),
concepts=json.dumps(response["concepts"]),
entities=json.dumps(response["entities"]),
keywords=json.dumps(response["keywords"]),
relations=json.dumps(response["relations"]),
semantic_roles=json.dumps(response["semantic_roles"]),
mainarticleurl=wikipedia.page(self._topic).url)
return Node("Section",
title=sectiontitle,
content=self._page.section(sectiontitle),
concepts=json.dumps(response["concepts"]),
entities=json.dumps(response["entities"]),
keywords=json.dumps(response["keywords"]),
relations=json.dumps(response["relations"]),
semantic_roles=json.dumps(response["semantic_roles"]))
def extractKeywords(text):
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2017-02-27',
username='******',
password='******')
response = natural_language_understanding.analyze(
text=text,
features=Features(keywords=KeywordsOptions(),
semantic_roles=SemanticRolesOptions()))
print(json.dumps(response, indent=2))
def get_keywords(self, sentence):
response = self.natural_language_understanding.analyze(
text=sentence,
return_analyzed_text='True',
features=Features(concepts=ConceptsOptions(),
categories=CategoriesOptions(),
relations=RelationsOptions(),
semantic_roles=SemanticRolesOptions(),
sentiment=SentimentOptions(),
entities=EntitiesOptions(),
keywords=KeywordsOptions())).get_result()
keywords = map(lambda x: (x['text'], x['type']), response['entities'])
return keywords
def analyze_using_NLU(analysistext):
res = dict()
response = natural_language_understanding.analyze(
text=analysistext,
features=Features(sentiment=SentimentOptions(),
entities=EntitiesOptions(),
keywords=KeywordsOptions(),
emotion=EmotionOptions(),
concepts=ConceptsOptions(),
categories=CategoriesOptions(),
semantic_roles=SemanticRolesOptions()))
res['results'] = response
return json.dumps(res)
def sem_role():
os.system('clear')
for text in texts_pos:
response = natural_language_understanding.analyze(
text=text,
features=Features(semantic_roles=SemanticRolesOptions(
entities=True, keywords=True))).get_result()
print('\nText: ' + text + "\nActions:-")
for x in response["semantic_roles"]:
if x["subject"]["text"].lower() == 'i':
print(colored(x["action"]["text"], color="red"))
input("\n\nPress Enter to continue...")
os.system('clear')
menu()
def resultjson():
if request.method == 'POST':
#text1 = request.form
text1 = request.get_json(force=True)
user = natural_language_understanding.analyze(
text=str(text1),
features=Features(entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=2),
relations=RelationsOptions(),
categories=CategoriesOptions(),
semantic_roles=SemanticRolesOptions(),
concepts=ConceptsOptions(limit=3),
keywords=KeywordsOptions(emotion=True,
sentiment=True,
limit=2)))
return jsonify(user)
def ibm_nlu(data, filename):
clean = [clean_tweet(tweet) for tweet in data['Text']]
cleaned = [x for x in clean if x not in stopwords]
# print(cleaned)
response = natural_language_understanding.analyze(
text=str(cleaned),
features=Features(entities=EntitiesOptions(emotion=True,
sentiment=True,
limit=100),
categories=CategoriesOptions(),
keywords=KeywordsOptions(sentiment=True,
emotion=True,
limit=100),
relations=RelationsOptions(),
semantic_roles=SemanticRolesOptions(),
sentiment=SentimentOptions()))
# print(json.dumps(response, indent=2))
res = json.dumps(response, indent=2)
filename = filename + '.json'
file = open(filename, 'w')
file.write(res)
file.close
def summarize_case(search_results):
summary = ''
for index, each_result in enumerate(search_results):
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2018-11-16',
iam_apikey='fhfAp9h12U5DSbZ0AzPomZ-suKwnboNAg3EdorPHsB5e',
url=
'https://gateway-lon.watsonplatform.net/natural-language-understanding/api'
)
search_results[index][
'analytics'] = natural_language_understanding.analyze(
text=each_result['case_body'],
language='en',
features=Features(
categories=CategoriesOptions(limit=3),
semantic_roles=SemanticRolesOptions())).get_result()
for role in search_results[index]['analytics']['semantic_roles']:
if role['sentence'] not in summary:
summary += ' ' + role['sentence']
search_results[index]['analytics']['summary'] = summary
return search_results
def fact_extraction(doc): #Type: Str
facts=[]
doc=doc.translate(translator)
response = natural_language_understanding.analyze(
text=doc,
features=Features(
semantic_roles=SemanticRolesOptions(entities=True)
)).get_result()
for i in response['semantic_roles']:
if('subject' in i.keys() and 'object' in i.keys()):
if('entities' in i['subject'].keys() and 'entities' in i['object'].keys()):
d=i['subject']['entities']
e=i['object']['entities']
if(d==[] or e==[]):
continue
sentence=i['sentence']
print(sentence)
for add in sentence.split("."):
if(add!=''):
out=re.sub(r'[^\w\d\s\.]+', '', add)
facts.append(out.strip()+".")
return facts
def triplet_return(text):
global df
if(len(text.split())<4):
return None
tup_negated_list=[]
try:
response = natural_language_understanding.analyze(
text=text,language="English", features=Features(semantic_roles=SemanticRolesOptions(limit=10, entities=True, keywords=True))).get_result()
except:
return tup_negated_list
# tup: 3 entity tuple with (subject, object, action)
semantic_roles_list = response["semantic_roles"]
for semantic_roles in semantic_roles_list:
try:
subject_str=""
object_str=""
if("keywords" in semantic_roles["subject"]):
for i in semantic_roles["subject"]["keywords"]:
subject_str=subject_str+i["text"]+" "
else:
subject_str=semantic_roles["subject"]["text"]
if("keywords" in semantic_roles["object"]):
for i in semantic_roles["object"]["keywords"]:
object_str=object_str+i["text"]+" "
else:
object_str=semantic_roles["object"]["text"]
tup = (subject_str, object_str, semantic_roles["action"]["text"])
negated = False
if("negated" in semantic_roles["action"]["verb"].keys()):
negated=True
print((tup, negated)) #Uncomment to view file.
tup_negated_list.append((tup,negated))
except:
pass
return tup_negated_list
def main():
# Check for service credentials and transcript files location
if not hasattr(settings, 'TONE_ANALYZER_USER') or not hasattr(settings, 'TONE_ANALYZER_PASSWORD') or not hasattr(settings, 'NLU_USER') or not hasattr(settings,'NLU_PASSWORD') or not hasattr(settings, 'TEST_DATA_DIR'):
print("Error: Service credentials and/or test data dir missing. Terminating ...")
sys.exit(1)
else:
tone_analyzer_user = settings.TONE_ANALYZER_USER
tone_analyzer_password = settings.TONE_ANALYZER_PASSWORD
nlu_user = settings.NLU_USER
nlu_password = settings.NLU_PASSWORD
test_data_dir = settings.TEST_DATA_DIR
# Create service clients
tone_analyzer = ToneAnalyzerV3(username= tone_analyzer_user, password= tone_analyzer_password, version='2017-09-26')
natural_language_understanding = NaturalLanguageUnderstandingV1(version='2017-02-27', username=nlu_user, password=nlu_password)
# Loop through all call transcript files
test_files = glob.glob(test_data_dir + '/**/*.txt', recursive=True)
print('Analyzing %d earnings call transcripts ...' % (len(test_files)))
for filename in test_files:
print("Analyzing transcript file name " + filename)
with open(filename, 'r') as transcript:
tone = tone_analyzer.tone(tone_input=transcript.read(), content_type="text/plain")
# Get joy and sort by descending score
sentences_with_joy = []
for each_sentence in tone['sentences_tone']:
for each_tone in each_sentence['tones']:
if each_tone['tone_id'] == 'joy':
sentences_with_joy.append({'sentence_id': each_sentence['sentence_id'], 'text': each_sentence['text'], 'score': each_tone['score']})
break
sentences_with_joy = sorted(sentences_with_joy, key=itemgetter('score'), reverse=True)
# Only top 5 are being selected
if len(sentences_with_joy) > 5:
sentences_with_joy = sentences_with_joy[:5]
index = 1
print('\nMost positive statements from earnings call:\n')
# Go through top positive sentences and use NLU to get keywords and
# Semantic Roles
for sentence in sentences_with_joy:
print(str(index) + ') ' + sentence['text'])
nlu_analysis = natural_language_understanding.analyze(text = sentence['text'], features=Features(keywords=KeywordsOptions(), semantic_roles=SemanticRolesOptions(keywords=True)))
first_keyword = True
for each_item in nlu_analysis['keywords']:
if each_item['relevance'] > 0.5:
if first_keyword:
print('')
print('NLU Analysis:')
print('keywords: ' + each_item['text'], end='')
first_keyword = False
else:
print(', ' + each_item['text'], end='')
print('')
first_semantic_role = True
for each_item in nlu_analysis['semantic_roles']:
if first_semantic_role:
print('semantic_roles:')
first_semantic_role = False
subject_dict = each_item.get('subject')
if subject_dict is None:
print('subject: N/A ', end='')
else:
print('subject: ' + subject_dict['text'], end=' ')
action_dict = each_item.get('action')
if action_dict is None:
print('action: N/A ', end='')
else:
print('action: ' + action_dict['text'], end=' ')
object_dict = each_item.get('object')
if object_dict is None:
print('object: N/A', end='')
else:
print('object: ' + object_dict['text'], end='')
print()
index = index + 1
print('\n')
print('Processing complete. Exiting ...')
# In[4]:
natural_language_understanding = NaturalLanguageUnderstandingV1(
username=username, password=password, version='2018-03-16')
response = natural_language_understanding.analyze(
url=
'https://www.nytimes.com/2018/07/16/opinion/trump-and-putin-vs-america.html?action=click&pgtype=Homepage&clickSource=story-heading&module=opinion-c-col-left-region®ion=opinion-c-col-left-region&WT.nav=opinion-c-col-left-region',
language='en',
features=Features(keywords=KeywordsOptions(sentiment=False,
emotion=False,
limit=20),
entities=EntitiesOptions(sentiment=False,
emotion=False,
limit=50),
semantic_roles=SemanticRolesOptions()))
entities = response['entities']
keywords = response['keywords']
semantic = response['semantic_roles']
# In[5]:
# Question 1
# Extract keywords and entities
# define type of words
# create questions
def Q1(x):
return {'Person': "Who is ", 'Location': "Where is "}.get(x, "What is ")
def test_semantic_roles(self):
s = Features(semantic_roles=SemanticRolesOptions())
assert s._to_dict() == {'semantic_roles': {}}
version='2017-02-27')
text = u'Using generative design, virtual reality, 3D printing, and a cloud-based supply chain, Hackrod is challenging the traditional approach to automobile manufacturing. The company is developing a methodology that will enable the rapid prototyping of bespoke vehicle solutions and will place the consumer as co-creator in the automotive space. Beginning with the film “Autonomo,” where Hackrod aims to showcase its vehicles and then offer the designs as templates to customers, Hackrod’s design and manufacturing engine will feature engineering analysis and performance simulation and well as supply chain management, ordering, and hardware integration. Hackrod currently has a partnership with AutoDesk to create virtual car prototypes using the Autodesk VRED 3D visualization and virtual prototyping software.'
response = natural_language_understanding.analyze(
text=text,
features=Features(
keywords=KeywordsOptions(sentiment=True, emotion=True, limit=2)))
print(json.dumps(response, indent=2))
response = natural_language_understanding.analyze(
text=text,
features=Features(entities=EntitiesOptions(sentiment=True, limit=1)))
print(json.dumps(response, indent=2))
response = natural_language_understanding.analyze(
text=text, features=Features(semantic_roles=SemanticRolesOptions()))
print(json.dumps(response, indent=2))
response = natural_language_understanding.analyze(
url='www.ibm.com', features=Features(concepts=ConceptsOptions(limit=3)))
print(json.dumps(response, indent=2))
response = natural_language_understanding.analyze(
url='www.ibm.com', features=Features(categories=CategoriesOptions()))
print(json.dumps(response, indent=2))
except:
continue
# Pass WDS values to NLU for analysis
import json
from watson_developer_cloud import NaturalLanguageUnderstandingV1
from watson_developer_cloud.natural_language_understanding_v1 \
import Features, EntitiesOptions, KeywordsOptions,ConceptsOptions,CategoriesOptions,RelationsOptions, SemanticRolesOptions
try:
naturalLanguageUnderstanding = NaturalLanguageUnderstandingV1(
version='2018-11-16',
iam_apikey='E87U7OBeyXoAyNG1nLVzvT7EMr6Es-812AHaLlvtE0-R',
url='https://gateway.watsonplatform.net/natural-language-understanding/api')
response = naturalLanguageUnderstanding.analyze(
text=wdsWatsonValues,
features=Features(
entities=EntitiesOptions(emotion=False, sentiment=False, mentions=True, limit=50),
keywords=KeywordsOptions(emotion=False, sentiment=False,limit=20),
concepts=ConceptsOptions(limit=2),
categories=CategoriesOptions(limit=2),
relations=RelationsOptions(),
semantic_roles=SemanticRolesOptions())).get_result()
with open(nluOutFileName,'w') as outfile:
json.dump(response, outfile)
except:
print('No line headers found. No values in WDS. Watson cannot provide any entities')