def ExtractTaxonomy(text):
alchemyapi = AlchemyAPI()
response = alchemyapi.taxonomy('text', text)
results = []
if response['status'] == 'OK':
for category in response['taxonomy']:
results.append(category['label'] + ' : ' + category['score'])
else:
print('Error in taxonomy call: ', response['statusInfo'])
return results
class NLP:
def __init__(self):
self.alchemyapi = AlchemyAPI()
def get_categories(self, text):
response = self.alchemyapi.taxonomy('text', text)
if response['status'] == 'OK' and len(response['taxonomy']) > 0:
taxonomy = response['taxonomy'][0]
tokens = taxonomy['label'].split('/')
return tokens[1]
def get_keywords(self, text):
response = self.alchemyapi.keywords('text', text)
if response['status'] == 'OK' and len(response['keywords']) > 0:
return [x['text'] for x in response['keywords']]
def entity_topic_extraction(self, myText):
alchemyApi = AlchemyAPI()
# put all entities in a list
entity_list = []
response = alchemyApi.entities('text', myText, {'sentiment': 1})
if response['status'] == 'OK':
for entity in response['entities']:
#entity_list.append((entity['text'].encode('utf-8'),entity['type']))
entity_list.append((entity['text'].encode('utf-8'), entity['type'].encode('utf-8')))
# put all taxonomy in a list
response = alchemyApi.taxonomy('text', myText)
# put all taxonomy in a list
taxonomy_list = []
if response['status'] == 'OK':
for category in response['taxonomy']:
taxonomy_list.append(category['label'].encode('utf-8'))
return entity_list, taxonomy_list
wait = raw_input('press enter to continue')
print('')
print('')
print('')
print('############################################')
print('# Taxonomy Example #')
print('############################################')
print('')
print('')
print('Processing text: ', demo_text)
print('')
response = alchemyapi.taxonomy('text', demo_text)
if response['status'] == 'OK':
print('## Response Object ##')
print(json.dumps(response, indent=4))
print('')
print('## Categories ##')
for category in response['taxonomy']:
print(category['label'], ' : ', category['score'])
print('')
else:
print('Error in taxonomy call: ', response['statusInfo'])
print('')
return k
url = "http://quora-api.herokuapp.com/users/" + sys.argv[1] + "/activity"
data = requests.get(url).json()
data = data["activity"]
payload = {}
# count=0
# getDocCount()
for activities in data:
title = activities["title"]
summary = activities["summary"]
print title
document["title"] = title
document["summary"] = summary
labels = al.taxonomy("text", title)
entities = al.entities("html", summary)
keywords = al.keywords("html", summary)
sentiment = al.sentiment("html", summary)
# print labels['taxonomy']
# count+=1
payload["entities"] = {}
payload["keywords"] = []
payload["sentiment"] = {}
docNode = createDocNode(document)
try:
print "Yo"
labels = labels["taxonomy"][0]["label"]
print "Yo1"
print labels
labels = func(labels)
for microformat in response['microformats']:
print('Field: ', microformat['field'].encode('utf-8'))
print('Data: ', microformat['data'])
print('')
else:
print('Error in microformats parsing call: ', response['statusInfo'])
print('')
print('')
print('# Taxonomy Example #')
response = alchemyapi.taxonomy('text', demo_text)
if response['status'] == 'OK':
print('## Response Object ##')
with open('taxonomy.js', 'w') as outfile:
json.dump(response,outfile, indent =4)
print('')
print('## Categories ##')
for category in response['taxonomy']:
print(category['label'], ' : ', category['score'])
print('')
else:
print('Error in taxonomy call: ', response['statusInfo'])
# combined
print('Checking combined . . . ')
response = alchemyapi.combined('text', test_text)
assert(response['status'] == 'OK')
response = alchemyapi.combined('html', test_html)
assert(response['status'] == 'ERROR')
response = alchemyapi.combined('url', test_url)
assert(response['status'] == 'OK')
print('Combined tests complete!')
print('')
print('')
# taxonomy
print('Checking taxonomy . . . ')
response = alchemyapi.taxonomy('text', test_text)
assert(response['status'] == 'OK')
response = alchemyapi.taxonomy('html', test_html, {'url': 'test'})
assert(response['status'] == 'OK')
response = alchemyapi.taxonomy('url', test_url)
assert(response['status'] == 'OK')
print('Taxonomy tests complete!')
print('')
print('')
# image
print('Checking image extraction . . . ')
response = alchemyapi.imageExtraction('text', test_text)
assert(response['status'] == 'ERROR')
response = alchemyapi.imageExtraction('html', test_html)
assert(response['status'] == 'ERROR')
def findTaxonomy(text):
alchemyapi = AlchemyAPI()
response = alchemyapi.taxonomy('text', text)
return response
return k
url = "http://quora-api.herokuapp.com/users/" + sys.argv[1] + "/activity"
data = requests.get(url).json()
data = data['activity']
payload = {}
#count=0
#getDocCount()
for activities in data:
title = activities['title']
summary = activities['summary']
print title
document['title'] = title
document['summary'] = summary
labels = al.taxonomy("text", title)
entities = al.entities("html", summary)
keywords = al.keywords("html", summary)
sentiment = al.sentiment("html", summary)
#print labels['taxonomy']
#count+=1
payload['entities'] = {}
payload['keywords'] = []
payload['sentiment'] = {}
docNode = createDocNode(document)
try:
print "Yo"
labels = labels['taxonomy'][0]['label']
print "Yo1"
print labels
labels = func(labels)
print("")
print("")
print("")
print("")
print("############################################")
print("# Taxonomy Example #")
print("############################################")
print("")
print("")
print("Processing text: ", demo_text)
print("")
response = alchemyapi.taxonomy("text", demo_text)
if response["status"] == "OK":
print("## Response Object ##")
print(json.dumps(response, indent=4))
print("")
print("## Categories ##")
for category in response["taxonomy"]:
print(category["label"], " : ", category["score"])
print("")
else:
print("Error in taxonomy call: ", response["statusInfo"])
print("")
# combined
print('Checking combined . . . ')
response = alchemyapi.combined('text', test_text)
assert (response['status'] == 'OK')
response = alchemyapi.combined('html', test_html)
assert (response['status'] == 'ERROR')
response = alchemyapi.combined('url', test_url)
assert (response['status'] == 'OK')
print('Combined tests complete!')
print('')
print('')
# taxonomy
print('Checking taxonomy . . . ')
response = alchemyapi.taxonomy('text', test_text)
assert (response['status'] == 'OK')
response = alchemyapi.taxonomy('html', test_html, {'url': 'test'})
assert (response['status'] == 'OK')
response = alchemyapi.taxonomy('url', test_url)
assert (response['status'] == 'OK')
print('Taxonomy tests complete!')
print('')
print('')
# image
print('Checking image extraction . . . ')
response = alchemyapi.imageExtraction('text', test_text)
assert (response['status'] == 'ERROR')
response = alchemyapi.imageExtraction('html', test_html)
assert (response['status'] == 'ERROR')
class AlchemyPost:
def __init__(self, post_tumblr, post_id, consumer_key, consumer_secret, oauth_token, oauth_secret):
self.post_tumblr = post_tumblr
self.post_id = post_id
self._init_tumblr(consumer_key, consumer_secret, oauth_token, oauth_secret)
self._init_alchemy()
def _init_tumblr(self, consumer_key, consumer_secret, oauth_token, oauth_secret):
self._client = pytumblr.TumblrRestClient(consumer_key, consumer_secret, oauth_token, oauth_secret)
def _init_alchemy(self):
self.alchemyapi = AlchemyAPI()
self.content = {}
def analyze_post(self):
self.post = self._get_content_post()
self._alchemy_entities()
self._alchemy_keywords()
self._alchemy_concepts()
self._alchemy_sentiment()
self._alchemy_relations()
self._alchemy_category()
self._alchemy_feeds()
self._alchemy_taxonomy()
def print_content(self):
print(json.dumps(self.content, indent=4))
def _get_content_post(self):
print "*",
infos = self._get_infos_post()
self.title = ''
self.tags = []
if 'tags' in infos:
self.tags = infos['tags']
if infos['type'] == 'text':
return self._get_content_text(infos)
if infos['type'] == 'quote':
return self._get_content_quote(infos)
return ''
def _get_infos_post(self):
infos = self._client.posts(self.post_tumblr, id=self.post_id)
if 'posts' in infos and len(infos['posts'])>0:
return infos['posts'][0]
return {}
def _get_content_text(self, infos):
content = "<h1>" + str(infos['title']) + "</h1>"
content += " <br>" + str(infos['body'])
content += " <br>" + " ".join(infos['tags'])
return content
def _get_content_quote(self, infos):
content = str(infos['text'])
content += " <br>" + str(infos['source'])
content += " <br>" + " ".join(infos['tags'])
return content
def _alchemy_entities(self):
print ".",
response = self.alchemyapi.entities('html', self.post)
if response['status'] != 'OK':
return False
self.content['entities'] = response['entities']
return True
def _alchemy_keywords(self):
print ".",
response = self.alchemyapi.keywords('html', self.post)
if response['status'] != 'OK':
return False
self.content['keywords'] = response['keywords']
return True
def _alchemy_concepts(self):
print ".",
response = self.alchemyapi.concepts('html', self.post)
if response['status'] != 'OK':
return False
self.content['concepts'] = response['concepts']
return True
def _alchemy_sentiment(self):
print ".",
response = self.alchemyapi.sentiment('html', self.post)
if response['status'] != 'OK':
return False
self.content['sentiment'] = response['docSentiment']
return True
def _alchemy_relations(self):
print ".",
response = self.alchemyapi.relations('html', self.post)
if response['status'] != 'OK':
return False
self.content['relations'] = response['relations']
return True
def _alchemy_category(self):
print ".",
response = self.alchemyapi.category('html', self.post)
if response['status'] != 'OK':
return False
self.content['category'] = response['category']
self.content['score'] = response['score']
return True
def _alchemy_feeds(self):
print ".",
response = self.alchemyapi.feeds('html', self.post)
if response['status'] != 'OK':
return False
self.content['feeds'] = response['feeds']
return True
def _alchemy_taxonomy(self):
print ".",
response = self.alchemyapi.taxonomy('html', self.post)
if response['status'] != 'OK':
return False
self.content['taxonomy'] = response['taxonomy']
return True
def user_analysis_sentiments(request):
if request.method == 'GET':
print request.GET.get('user', '')
user = request.GET.get('user', '')
messages = []
message = Message.objects.filter(user_send=user.decode("utf8"))
for m in message:
messages.append(m.message_text)
text = ",".join(messages)
alchemyapi = AlchemyAPI()
#keywords
response = alchemyapi.keywords('text', text, {'sentiment': 1})
if response['status'] == 'OK':
keywords = []
for keyword in response['keywords']:
keyword_text = keyword['text'].encode('utf-8')
keyword_relevance = keyword['relevance']
keyword_sentiment = keyword['sentiment']['type']
key_word = {'keyword_text': keyword_text, 'keyword_relevance': keyword_relevance,
'keyword_sentiment': keyword_sentiment}
keywords.append(key_word)
else:
print('Error in keyword extaction call: ', response['statusInfo'])
response = alchemyapi.concepts('text', text)
if response['status'] == 'OK':
concepts = []
for concept in response['concepts']:
concept_text = concept['text']
concept_relevance = concept['relevance']
concept_entity = {'concept_text': concept_text, 'concept_relevance': concept_relevance}
concepts.append(concept_entity)
else:
print('Error in concept tagging call: ', response['statusInfo'])
response = alchemyapi.language('text', text)
if response['status'] == 'OK':
print(response['wikipedia'])
language = response['language']
iso_639_1 = response['iso-639-1']
native_speakers = response['native-speakers']
wikipedia = response['wikipedia']
language_id = {'language': language, 'iso_639_1': iso_639_1, 'native_speakers': native_speakers, 'wikipedia': wikipedia}
else:
print('Error in language detection call: ', response['statusInfo'])
response = alchemyapi.relations('text', text)
if response['status'] == 'OK':
relations = []
for relation in response['relations']:
if 'subject' in relation:
relation_subject_text = relation['subject']['text'].encode('utf-8')
if 'action' in relation:
relation_action_text = relation['action']['text'].encode('utf-8')
if 'object' in relation:
relation_object_text = relation['object']['text'].encode('utf-8')
relation_entity = {'relation_subject_text': relation_subject_text,
'relation_action_text': relation_action_text,
'relation_object_text': relation_object_text}
relations.append(relation_entity)
else:
print('Error in relation extaction call: ', response['statusInfo'])
response = alchemyapi.category('text', text)
if response['status'] == 'OK':
print('text: ', response['category'])
category = response['category']
print('score: ', response['score'])
score = response['score']
categories = {'category': category, 'score': score}
else:
print('Error in text categorization call: ', response['statusInfo'])
response = alchemyapi.taxonomy('text', text)
if response['status'] == 'OK':
taxonomies = []
for category in response['taxonomy']:
taxonomy_label = category['label']
taxonomy_score = category['score']
taxonomy = {'taxonomy_label': taxonomy_label, 'taxonomy_score': taxonomy_score}
taxonomies.append(taxonomy)
else:
print('Error in taxonomy call: ', response['statusInfo'])
response = alchemyapi.combined('text', text)
if response['status'] == 'OK':
print('## Response Object ##')
print(json.dumps(response, indent=4))
print('')
user = {'user_name': 'LOL', 'keywords': keywords, 'concepts': concepts, 'language_id': language_id,
'relations': relations, 'categories': categories, 'taxonomies': taxonomies}
return HttpResponse(json.dumps(user), content_type="application/json")
