def test_concepts(self):
c = features.Concepts()
assert(c.name() == 'concepts')
assert(c.toDict() == {})
c = features.Concepts(limit=10)
assert(c.name() == 'concepts')
assert(c.toDict() == {'limit': 10})
def analyze():
inputType = request.form['button']
if inputType == "Text":
input = request.form['text-analysis']
concepts = nlu.analyze(text=input, features=[features.Concepts()])
# categories = nlu.analyze(text=input, features=[features.Categories()])
else:
input = request.form['url-analysis']
concepts = nlu.analyze(url=input.strip(),
features=[features.Concepts()])
# categories = nlu.analyze(url=input.strip(), features=[features.Categories()])
return jsonify({'concepts': concepts["concepts"]})
def concept_recog(path='./test.txt'):
#with open(path, 'rt', encoding='utf-8') as f:
# inputs = f.read()
with open(path, 'rb') as f:
inputs = f.read().decode("UTF-8")
natural_language_understanding = nlu(
url=("https://gateway.aibril-watson.kr/" +
"natural-language-understanding/api"),
username="******",
password="******",
version="2017-02-27")
response = natural_language_understanding.analyze(
text=inputs,
features=[
Features.Concepts(
# Concepts options
limit=3)
])
# print(json.dumps(response))
texts = response['concepts']
text_list = []
for text in texts:
text_list.append(text['text'])
for text in text_list:
print(text)
return ' '.join(text_list)
def run_watson_nlu():
files = glob.glob('work/bug-*.json')
(user, passwd) = get_auth()
for fname in files:
with open(fname) as f:
LOG.debug("Processing %s" % fname)
bug = json.loads(f.read())
num = bug["link"].split("/")[-1]
with open("work/res-%s.json" % num, "w") as out:
nlu = watson_developer_cloud.NaturalLanguageUnderstandingV1(
version='2017-02-27', username=user, password=passwd)
res = nlu.analyze(text=bug["comments"],
features=[
features.Concepts(),
features.Keywords(),
features.Emotion(),
features.Sentiment(),
])
output = {
"link": bug["link"],
"tags": bug["tags"],
"importance": bug["importance"],
"length": len(bug["comments"]),
"results": res
}
out.write(json.dumps(output, indent=4))
def processText(fname):
print('fname', fname)
in_text = readfile(fname)
in_text = str(in_text)
natural_language_understanding = NaturalLanguageUnderstandingV1(
username="******",
password="******",
version="2017-02-27")
response = natural_language_understanding.analyze(
text=in_text,
features=[
Features.Concepts(
# Concepts options
limit=50),
# Features.Keywords(
# # Keywords options
# # sentiment=True,
# # emotion=True,
# limit=10
# ),
# Features.Entities(
# )
])
# return jsonify(response)
return json.dumps(response, indent=2)
def analyze(html):
response = nlu.analyze(
html=html,
features=[
# features.Keywords(limit=32)
features.Concepts(limit=32)
# features.Entities(limit=32)
],
)
return response
def nl_processing(reqd_text):
response = natural_language_understanding.analyze(text=reqd_text,
features=[
features.Entities(),
features.Keywords(),
features.Emotion(),
features.Concepts(),
features.Sentiment()
])
return response
def main():
with open("compost.txt", "r") as f:
strings = f.read().split()
while (strings):
response = NLU.analyze(text=strings.pop(0),
features=[
features.Keywords(),
features.Categories(),
features.Concepts()
])
print str(response.keywords[0].text)
def map_feature(name):
feature_name_mappings = {
'keywords': features.Keywords(),
'entities': features.Entities(),
'concepts': features.Concepts(),
'categories': features.Categories(),
'sentiment': features.Sentiment(),
'emotion': features.Emotion()
}
if name in feature_name_mappings:
return feature_name_mappings[name]
else:
print("Invalid feature name")
return None
def nlu(text):
response = n.analyze(text=text,
features=[
features.Emotion(),
features.Concepts(),
features.Categories(),
features.Entities(),
features.Keywords(),
features.SemanticRoles(),
features.Relations(),
features.Sentiment()
],
language='en')
return json.dumps(response, indent=2)
def featureList(self, tags):
f_list = []
for tag in tags:
if tag == "sentiment":
f_list.append(features.Sentiment())
elif tag == "categories":
f_list.append(features.Categories())
elif tag == "concepts":
f_list.append(features.Concepts())
elif tag == "emotion":
f_list.append(features.Emotion())
elif tag == "entities":
f_list.append(features.Entities())
return f_list
def extract_data(text):
# Use Watson's NLU API to extract the keywords, entities and concepts from a text
bm_username = "******"
bm_password = "******"
nlu = watson_developer_cloud.NaturalLanguageUnderstandingV1(
version='2017-02-27', username=bm_username, password=bm_password)
ents = nlu.analyze(text=text,
features=[
features.Entities(),
features.Keywords(),
features.Concepts()
])
ents["tweet"] = text
return ents
def test_model():
model = request.forms.get('model')
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2017-02-27',
username="******",
password='******')
response = natural_language_understanding.analyze(text=model,
features=[
features.Entities(),
features.Keywords(),
features.Concepts()
])
return (json.dumps(response, indent=2))
def calc_concepts(text_input):
import watson_developer_cloud.natural_language_understanding.features.v1 as features
NLU = NaturalLanguageUnderstandingV1(
username=os.environ.get("NLU_USERNAME"),
password=os.environ.get("NLU_PASSWORD"),
version='2016-05-19')
#features = ['concepts', 'keywords']
#tips = 'The IBM Watson™ AlchemyLanguage service is a collection of text analysis functions that derive semantic information from your content. You can input text, HTML, or a public URL and leverage sophisticated natural language processing techniques to get a quick high-level understanding of your content and obtain detailed insights such as sentiment for detected entities and keywords. See a video overview of the service here.'
r = NLU.analyze(text=text_input, features=[features.Concepts()])
concepts = r['concepts']
dict_of_concepts = dict()
for c in concepts:
dict_of_concepts[c['text']] = c['relevance']
#print(dict_of_concepts)
return dict_of_concepts
def analyze(text, threshold=0.5):
text = text.encode('ascii', errors='ignore')
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2017-02-27',
username='******',
password='******')
response = natural_language_understanding.analyze(text=text,
features=[
features.Entities(),
features.Keywords(),
features.Concepts(),
features.Sentiment()
])
decoder = json.JSONDecoder()
decoded_response = decoder.decode(json.dumps(response, indent=2))
language = decoded_response["language"]
keywords = decoded_response["keywords"]
entities = decoded_response["entities"]
concepts = decoded_response["concepts"]
sentiment = decoded_response["sentiment"]
keywords = sorted(keywords, key=lambda x: -x['relevance'])
keywords = [
keyword for keyword in keywords if keyword['relevance'] >= threshold
]
keywords = [keyword['text'] for keyword in keywords]
entities = sorted(entities, key=lambda x: -x['relevance'])
entities = [
entity for entity in entities if entity['relevance'] >= threshold
]
entities = [(entity['type'], entity['text']) for entity in entities]
concepts = sorted(concepts, key=lambda x: -x['relevance'])
concepts = [
concept for concept in concepts if concept['relevance'] >= threshold
]
concepts = [concept['text'] for concept in concepts]
sentiment = (sentiment['document']['label'],
sentiment['document']['score'])
def main(params):
natural_language_understanding = NaturalLanguageUnderstandingV1(
username=params["username"],
password=params["password"],
version=params["version"])
response = natural_language_understanding.analyze(
url=params["url"],
features=[
Features.Concepts(limit=1),
Features.Entities(limit=1),
Features.Keywords(limit=1),
Features.Categories(),
Features.Emotion(),
Features.Sentiment(),
Features.MetaData(),
Features.Relations(),
Features.SemanticRoles(limit=1)
])
return response
def wikiGetConcepts(page):
wikitext = page.summary
response = natural_language_understanding.analyze(
text=wikitext, features=[Features.Concepts()])
queryTerms = {}
for concept in response['concepts']:
if concept['relevance'] > 0.5:
queryTerms[concept['text'].encode('ascii',
'ignore')] = concept['relevance']
sortedTerms = sorted(queryTerms, key=queryTerms.get, reverse=True)
charlen = 0
for i, term in enumerate(sortedTerms):
charlen += len(term)
if charlen >= 350:
index = i
break
else:
index = i
return sortedTerms[:index]
def db_watson_query():
client = Cloudant(user_name, password, url = url)
client.connect()
databaseName = "catalog"
myDatabase = client[databaseName]
if(myDatabase.exists()):
print("Successfully created a database {}".format(databaseName))
result_collection = Result(myDatabase.all_docs, include_docs=True)
print ("Retrieved full document:\n{0}\n".format(result_collection[0]))
end_point = '{0}/{1}'.format(url, databaseName + "/_all_docs")
params = {'include_docs': 'true'}
response = client.r_session.get(end_point, params=params)
#connect to Watson NLU
natural_language_understanding = NaturalLanguageUnderstandingV1(
username=user_name_watson,
password=password_watson,
version="2017-02-27")
#feed data to watson api
for i in range(0, 300):
tmp = result_collection[i][0]['doc']
response = natural_language_understanding.analyze(
text = tmp['description'],
features=[
Features.Concepts(
# Concepts options
limit=5
)
]
)
jss = json.dumps(response, indent=2)
parses = parseJson(response)
mydocument = myDatabase[tmp['_id']]
mydocument['keywords'] = parses
mydocument.save()
client.disconnect()
def execute_watson_request(text):
natural_language_understanding = NaturalLanguageUnderstandingV1(
username=constants.WATSON_USER,
password=constants.WATSON_PASS,
version="2017-02-27")
try:
response = natural_language_understanding.analyze(
text=text,
features=[
features.Concepts(),
features.Categories(),
features.Emotion(),
features.Entities(emotion=True, sentiment=True),
features.Keywords(emotion=True, sentiment=True),
features.Sentiment()
])
return response
except WatsonException as error:
return str(error)
def get_features(html):
natural_language_understanding = NaturalLanguageUnderstandingV1(
version=BLUEMIX_VERSION,
username=BLUEMIX_USER_NAME,
password=BLUEMIX_USER_PASS)
feature_dict = natural_language_understanding.analyze(
html=html,
features=[
features.Concepts(), # keep
# features.Entities(emotion=True, sentiment=True),
# features.Keywords(emotion=True, sentiment=True),
# features.Categories(),
# features.Emotion(document=True),
# features.MetaData(),
# features.SemanticRoles(entities=True, keywords=True),
# features.Relations(),
# features.Sentiment(document=True)
])
return feature_dict
def nlp(input_stuff):
# Calls NaturalLanguageUnderstandingV1 API
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2017-02-27',
username="******", # API Key
password="******") # Replace with personal API
response = natural_language_understanding.analyze(
text=input_stuff,
features=[features.Concepts(), features.Entities(), features.Keywords(), features.Categories(), features.Emotion(), features.Sentiment(), features.Relations(), features.SemanticRoles()])
nlu_data = {
'sentiment': response["sentiment"],
'semanticRoles': response["semantic_roles"],
'concepts': response["concepts"],
'entities': response["entities"],
'relations': response["relations"],
'concepts': response["concepts"],
'categoreis': response["categories"]
}
nlu_data = [nlu_data]
# print(nlu_data)
return(nlu_data)
def understand_text(self):
natural_language_understanding = NaturalLanguageUnderstandingV1(
username=nlu_settings.get("username"),
password=nlu_settings.get("password"),
version="2017-02-27")
self.nl_understanding = natural_language_understanding.analyze(
text=self.converted_text,
features=[
Features.Entities(emotion=True, sentiment=True, limit=100),
Features.Keywords(emotion=True, sentiment=True, limit=100),
Features.Categories(),
Features.Concepts(),
Features.Sentiment(),
Features.Emotion(),
# Features.Feature(),
# Features.MetaData(),
Features.Relations(),
Features.SemanticRoles(),
])
return self.nl_understanding
def get_features_prof():
list_profs = glob.glob(
"C:\\Users\\Mukund\\Downloads\\findMyAdvisor\\findMyAdvisor\\prof_pages\\*"
)
dict_prof_bow = {}
dict_prof_concept = {}
for profs in list_profs:
with open(profs, 'r') as myfile:
data = myfile.read()
try:
response = natural_language_understanding.analyze(
text=data,
features=[
Features.Entities(),
Features.Keywords(),
Features.Concepts()
])
keywords = response["keywords"]
entities = response["entities"]
concepts = response["concepts"]
bag_of_words = set()
concept_set = set()
for type in [keywords, entities, concepts]:
for item in type:
bag_of_words.add(item["text"].lower())
for type in [concepts]:
for item in type:
concept_set.add(item["text"].lower())
dict_prof_bow[profs.split('/')[-1]] = ' '.join(
list(bag_of_words))
dict_prof_concept[profs.split('/')[-1]] = ' '.join(
list(concept_set))
except:
continue
return dict_prof_bow, dict_prof_concept
def callNLU(text):
'''
Checks what features are enabled, then makes a call to NLU and returns JSON.
:param text The string containing the information you want to analyse.
'''
if text == None or text.strip() == '':
return {}
f = []
if c.getboolean('nlu_feature', 'concepts'): f.append(features.Concepts())
if c.getboolean('nlu_feature', 'entities'): f.append(features.Entities())
if c.getboolean('nlu_feature', 'keywords'): f.append(features.Keywords())
if c.getboolean('nlu_feature', 'categories'):
f.append(features.Categories())
if c.getboolean('nlu_feature', 'emotion'): f.append(features.Emotion())
if c.getboolean('nlu_feature', 'semanticroles'):
f.append(features.SemanticRoles())
if c.getboolean('nlu_feature', 'relations'): f.append(features.Relations())
if c.getboolean('nlu_feature', 'sentiment'): f.append(features.Sentiment())
r = nlu.analyze(text=text, features=f)
return r
def get_features_resume(text):
data = text
try:
response = natural_language_understanding.analyze(
text=data,
features=[
Features.Entities(),
Features.Keywords(),
Features.Concepts()
])
keywords = response["keywords"]
entities = response["entities"]
concepts = response["concepts"]
bag_of_words = set()
for type in [keywords, entities, concepts]:
for item in type:
bag_of_words.add(item["text"])
return ' '.join(list(bag_of_words)).lower()
except:
return 'No Resume Found'
def get_data_from_bluemix(target_url):
nl_understanding = cache_get(target_url)
if not nl_understanding:
natural_language_understanding = NaturalLanguageUnderstandingV1(
username=nlu_settings.get("username"),
password=nlu_settings.get("password"),
version="2017-02-27")
features = [
Features.Entities(limit=100,emotion=True,sentiment=True),
Features.Keywords(limit=100,emotion=True,sentiment=True),
Features.Categories(),
Features.Concepts(),
Features.Sentiment(),
Features.Emotion(),
# Features.Feature(),
# Features.MetaData(),
Features.Relations(),
Features.SemanticRoles(),
]
nl_understanding = None
for i in range(NUMBEROFTRIES):
try:
nl_understanding = natural_language_understanding.analyze(
url=target_url,
features=features
)
except:
pass
if nl_understanding:
break
cache_put(target_url, nl_understanding)
return nl_understanding
if q in line.lower():
questionToInsert = line.rstrip()
questions[questionToInsert] = counter
break
#When the end of the grouping has been reached, run the APIs
if (counter % lineCount == 0):
natural_language_understanding = NaturalLanguageUnderstandingV1(
username="******",
password="******",
version="2017-02-27")
response = natural_language_understanding.analyze(
text=dialogue,
features=[
Features.Concepts(),
#Features.Keywords(),
Features.Categories()
])
currConcepts = []
currKeywords = []
currCategories = []
for concept in response['concepts']:
if (concept['text'] not in concepts):
currConcepts.append(concept['text'])
concepts[concept['text']] = [counter]
else:
concepts[concept['text']].append(counter)
'''for keyword in response['keywords']:
if(keyword['text'] not in keywords):
print("[SCRAPER] The Star stories downloaded successfully(?).")
feed = r.json()
articles = []
watson_failed = True
for article in feed:
tags = []
tags_weighted = []
if not watson_failed:
try:
response = nlu.analyze(text=article['body'],
features=[features.Concepts()])
for concept in response['concepts'][:5]:
tags.append(concept['text'])
tags_weighted.append({concept['text']: concept['relevance']})
except Exception as e:
print(e)
watson_failed = True
article['sorted_tags'] = list(set(tags))
article['theme'] = tags_weighted
articles.append({'node': article})
data = {'nodes': articles, 'tags': []}
def handle_message(conversation_client, slack_client, workspace_id, context,
message, channel, user):
"""Handler for messages coming from Watson Conversation using context.
Fields in context will trigger various actions in this application.
:param str message: text from UI
:param SlackSender sender: used for send_message, hard-coded as Slack
:returns: True if UI input is required, False if we want app
processing and no input
:rtype: Bool
"""
global gv_nlu, gv_cortical_client, gv_bot_deafault_channel_name, gv_bot_deafault_channel_id, gv_ai
url_list = []
response = ""
cortical_response_text = ""
nlu_analyzed_text = ""
nlu_responce_text = ""
nlu_keyword = None
nlu_entities = None
context = None
# extract URLs from the message of the post
url_list = get_urls(slack_client, message)
if url_list is not None:
# send the message to user indicating that teh process of analysis started
slack_client.api_call("chat.postMessage",
channel=channel,
text="analyzing . . . ",
as_user=True)
for i in range(len(url_list)):
try:
# Analyze the URL article using WATSON Natural Language Understanding
nlu_response = gv_nlu.analyze(url=url_list[i],
return_analyzed_text=True,
features=[
features.Categories(),
features.Concepts(),
features.Emotion(),
features.Entities(),
features.Keywords(),
features.MetaData(),
features.Relations(),
features.Sentiment()
])
# get information from JSON format resulted by NLU
nlu_responce_text, nlu_sentiment, nlu_categoties, nlu_entities, nlu_keyword, nlu_concepts, nlu_analyzed_text = convert_nlujson(
url_list[i], nlu_response)
except WatsonException:
# print(json.dumps(nlu_response, indent=2))
nlu_responce_text = "Sentiments can not be retrieved from the URL"
# performs CORTICAL SEMANTIC analysis and returns results as a response text
cortical_response_text = cortical_analyze(nlu_analyzed_text,
nlu_keyword,
nlu_entities)
# build response text
title = "\n\n\n ===== Watson Sentiment Analysis =====\n"
response = response + title + nlu_responce_text + cortical_response_text ## Uncomment to add URL to the response text <+ url_list[i]>
i = i + 1
else:
response = "No valid URL found !!!"
return response
def exe_api(text):
natural_language_understanding = NaturalLanguageUnderstandingV1(
version='2017-02-27',
username='******',
password='******')
response = natural_language_understanding.analyze(
text=text,
features=[features.Entities(), features.Keywords(), features.Concepts(), features.Sentiment(), features.Emotion()])
name = ''
location = ''
priority = 'LOW'
for entity in response['entities']:
if entity['type'] == 'Person':
name = entity['text']
elif entity['type'] == 'Location':
location = entity['text']
fear = response['emotion']['document']['emotion']['fear']
anger = response['emotion']['document']['emotion']['anger']
if fear >= 0.4 or anger >= 0.4:
priority = 'HIGH'
elif fear >= 0.3 or anger >= 0.3:
priority = 'MEDIUM'
deportation_count = 0
visa_count = 0
greencard_count = 0
words = [w for w in text.split(' ') if not w in stopwords.words("english")]
base_words = []
for word in words:
word = WordNetLemmatizer().lemmatize(word, 'v')
base_words.append(word)
if word in deportation:
deportation_count += 1
elif word in visa:
visa_count += 1
elif word in greencard:
greencard_count += 1
stage1 = {'deportation':deportation_count, 'visa':visa_count, 'greencard':greencard_count}
stage1_ans = max(stage1, key=stage1.get)
about = '''{
"concern": "'''+stage1_ans+'''",
'''
if stage1_ans == 'deportation':
for word in base_words:
if word in arrest:
# deportation -> arrested -> offense -> information
offense = ''
for w in base_words:
if w.lower() in offenses:
offense = w
break
if offense == '':
keywords = [w['text'] for w in response['keywords']]
tags = nltk.pos_tag(keywords)
for tag in tags:
if 'NN' in tag:
offense = tag[0]
break
information = ''
url = '['
count = 0
try:
results = search('deportation because of '+offense)['webPages']['value']
for result in results:
url += '''{"link": "'''+result['url']+'''", "name": "'''+result['name']+'''"},'''
url = url[:-1]
url += ']'
while len(information) < 70:
u = results[count]['url']
information = exe_summarizer(u)
count += 1
except Exception:
information = '''
Among other things, the person will become ineligible to. receive asylum, as described in Bars to Receiving Asylum or Refugee Status. He or she may also lose eligibility for a U.S. visa or green card, as described. in Crimes. That Make U.S. Visa or Green Card Applicants Inadmissible. If the person is already in the U.S. with a visa or green card, he or she will likely be ordered removed, as described in Crimes. That Will Make an Immigrant Deportable. And if the person somehow gets as far as submitting an application for U.S. citizenship, the aggravated felony conviction will result in not only denial of that application and permanently barred from U.S. citizenship, but in his or her being placed in removal proceedings. There’s a sort of mismatch, in which state crimes that may sound minor to most people, did not involve violence, and may not even be called felonies are nevertheless viewed as aggravated felonies by federal immigration authorities.
'''
pass
about += '''
"reason": "arrest",
"offense": "'''+offense+'''",
"information": "'''+information+'''",
"url": '''+url+'''
}
'''
break
elif word in overstay:
# deported - > overstay -> visa type
visa_type = ''
for word in base_words:
if word in type:
visa_type = word
about += '''
"reason": "overstay",
"type": "'''+visa_type+'''"
}
'''
break
elif stage1_ans == 'visa':
for word in base_words:
if word in visa:
visa_type = ''
for word in base_words:
if word in type:
visa_type = word
information = ''
url = '['
count = 0
try:
results = search(' '.join(text.split(' ')[-4:]))['webPages']['value']
for result in results:
url += '''{"link": "''' + result['url'] + '''", "name": "''' + result['name'] + '''"},'''
url = url[:-1]
url += ']'
while len(information) < 90:
u = results[count]['url']
information = exe_summarizer(u)
count += 1
except Exception:
information = '''
There are various types of nonimmigrant visas for temporary visitors to travel to the U.S., if you are not a U.S. citizen or U.S. lawful permanent resident. It's important to have information about the type of nonimmigrant visa you will need for travel, and the steps required to apply for the visa at a U.S. Embassy or Consulate abroad.
'''
pass
about += '''
"type": "''' + visa_type + '''",
"information": "''' + information + '''",
"url": ''' + url + '''
}
'''
break
elif stage1_ans == 'greencard':
pass
else:
about = '''
{
"concern": "general"
}
'''
built_json = ''
try:
built_json = json.dumps(json.loads('''
{
"name": "'''+name.title()+'''",
"location": "'''+location+'''",
"priority": "'''+priority+'''",
"transcript": "'''+text+'''",
"about": '''+about+'''
}
'''), indent=4)
except Exception:
print name
print location
print priority
print about
return built_json
