def main():
alchemyapi = AlchemyAPI()
try:
filename = sys.argv[1]
except IndexError:
print "Give a filename as the second argument!"
sys.exit(1)
text = pdf_to_str(filename)
if len(text) >= LENGTH_LIMIT:
print "PDF content is longer ({} characters) than the maximum \
of {}, skipping remainder".format(len(text), LENGTH_LIMIT)
text = text[:LENGTH_LIMIT]
print "KEYWORDS"
response = alchemyapi.keywords('text', text)
for keyword in response['keywords']:
print ' - {}'.format(keyword['text'])
print
print "CONCEPTS"
response = alchemyapi.concepts('text', text)
for concept in response['concepts']:
print ' - {}'.format(concept['text'])
def getKeywordPerPost():
reader = open('output_sony_posts.txt')
all_json_posts = reader.read().splitlines()
alchemyapi = AlchemyAPI()
counter = 0
for p in all_json_posts:
print str(counter)
if counter < 1000:
counter = counter + 1
continue
#elif counter > 2000:
# break
else:
counter = counter + 1
content = json.loads(p)["cleanContent"]
response = alchemyapi.keywords('text',content.encode("UTF-8"))
if response['status'] == 'OK':
keywords = []
posts = ""
for keyword in response['keywords']:
keywords.append(keyword['text'])
posts = posts + keyword['text'] + ","
posts = posts[:-1] + "\n"
if posts <> "\n":
with codecs.open("keyPerPost.txt", "a") as f:
f.write(posts.encode("UTF-8"))
else:
print "error" + str(counter)
def extractKeywordsFromUrl(self,url):
"""method for extracting keywords from given text"""
#creating AlchemyAPI object
alchemyapi = AlchemyAPI()
#requesting json response from AlchemyAPI server
response = alchemyapi.keywords('url', url)
if response['status'] == 'OK':
for keywords in response['keywords']:
#concept object for storing the extracted concept
keyword = AlchemyStructure.Keyword()
#extracting the keyword
keyword.setText(keywords['text'])
#extracting the relevance of keyword
keyword.setRelevance(keywords['relevance'])
#append the concept into the list of retrieved concepts
self.keywordsFromUrl.append(keyword)
else:
print('Error in keyword tagging call: ', response['statusInfo'])
def getKeywords(uID, inputText):
alchemyapi = AlchemyAPI()
#alchemyapi.loadAPIKey("api_key.txt")
response = alchemyapi.keywords('text',inputText)
print inputText
if response['status'] == 'OK':
#print('## Response Object ##')
#print(json.dumps(response, indent=4))
#print('')
#print('## Keywords ##')
keywords = []
posts = uID + " : "
for keyword in response['keywords']:
keywords.append(keyword['text'])
posts = posts + keyword['text'] + "|"
userKeywords[uID] = keywords
posts = posts + "\n"
with codecs.open("outNew.txt", "a") as f:
f.write(posts.encode("UTF-8"))
return True
else:
print('idError: ', uID)
with codecs.open("keywordOut2.txt", "a") as f:
text = uID + "\n"
f.write(text.encode("UTF-8"))
return False
def get_sentiment(company_id, text):
alchemyapi = AlchemyAPI()
key_phrases = []
for apikey in engine.get_random_alchemy_credentials():
alchemyapi.apikey = apikey
response = alchemyapi.keywords('text', text, {'sentiment': 1})
if response['status'] == 'OK':
if len(response['keywords']) == 0:
return 0
# related_words = models.RelatedWord.query.filter_by(company_id=company_id).all()
for keyword in response["keywords"]:
if 'sentiment' in keyword:
if keyword['sentiment'].has_key('score'):
key_phrases.append(float(keyword['sentiment']['score']))
elif keyword['sentiment']['type'] == 'neutral':
key_phrases.append(0)
if len(key_phrases) == 0:
return 0
else:
return float("{0:.2f}".format(sum(key_phrases)/len(key_phrases)))
elif response['status'] == 'ERROR' and response['statusInfo'] != 'unsupported-text-language':
print "ERROR: getting sentiment " + response['statusInfo']
# Skip onto the next api key
continue
else:
print "None of the above " + response['statusInfo']
return 0
#Return none when all api keys are exhausted
return None
def sentiment_alchemy(url):
alchemyapi = AlchemyAPI()
response = alchemyapi.sentiment('url', url)
response['usage'] = None
if response['status'] == 'OK':
print('## Response Object ##')
print(json.dumps(response, indent=4))
print('')
print('## Author ##')
print('author: ', response.get('author', ''))
print('')
else:
print('Error in author extraction call: ', response['statusInfo'])
response = alchemyapi.keywords('url', url)
del (response['usage'])
if response['status'] == 'OK':
print('## Response Object ##')
print(json.dumps(response, indent=4))
print('')
print('## Keywords ##')
for keyword in response['keywords']:
print('text: ', keyword['text'].encode('utf-8'))
print('relevance: ', keyword['relevance'])
print('sentiment: ', keyword.get('sentiment', {}).get('type', ''))
if 'score' in keyword.get('sentiment', {}):
print('sentiment score: ' + keyword['sentiment']['score'])
print('')
else:
print('Error in keyword extaction call: ', response.get('statusInfo', ''))
def nlp_process(ids, ids_hash):
#instantiate an elasticsearch client
es = Elasticsearch()
#instantiate an alchemy client
alchemyapi = AlchemyAPI()
for item in ids:
data = ' '.join(ids_hash[item])
lowers = data.lower()
alchem_data = []
response = alchemyapi.keywords('text', lowers, {'sentiment': 1})
if response['status'] == 'OK':
print('#Success#')
for keyword in response['keywords']:
al_temp = defaultdict()
al_temp['text'] = keyword['text'].encode('utf-8')
al_temp['relevance'] = keyword['relevance']
al_temp['sentiment'] = keyword['sentiment']['type']
if 'score' in keyword['sentiment']:
al_temp['score'] = keyword['sentiment']['score']
alchem_data.append(al_temp)
else:
print('Error in keyword extaction call: ', response['statusInfo'])
print len(alchem_data)
# prepare body for insertion
doc = {"business_id": item, "word_freq": alchem_data}
exit()
template = {"create": {"_index": "alchem", "_type": "doc"}}
res = es.index(index="alchem", doc_type='doc', body=doc)
def use_api(self, input):
keyword_array=[]
alchemyapi = AlchemyAPI()
response = alchemyapi.keywords("text", input)
for keyword in response["keywords"]:
#print keyword["text"].encode('utf=8')
keyword_array.append(keyword["text"].encode('utf=8'))
return keyword_array
def fetchkeywords(demo_text):
alchemyapi = AlchemyAPI()
response = alchemyapi.keywords('text',demo_text, { 'sentiment':1 })
listoftopwords = []
if response['status'] == 'OK':
for keyword in response['keywords']:
listoftopwords.append(keyword['text'])
return listoftopwords
def performKeywordExtraction(text):
keywordText=[]
alchemyapi = AlchemyAPI()
response = alchemyapi.keywords("text", text)
if response['status'] == 'OK':
keywords = response['keywords']
for keyword in keywords:
if (float(keyword['relevance'])>0.1):
keywordText.append(keyword['text'])
return keywordText
def connect_alchemy(url):
# to connect with alchemy and tag the content
from alchemyapi import AlchemyAPI
alchemyapi = AlchemyAPI()
resp = alchemyapi.text('url', url)
response = alchemyapi.keywords("text", resp['text'])
keywors = response["keywords"]
def connect_alchemy(url):
# to connect with alchemy and tag the content
from alchemyapi import AlchemyAPI
alchemyapi = AlchemyAPI()
resp = alchemyapi.text('url', url)
response = alchemyapi.keywords("text", resp['text'])
keywors = response["keywords"]
def solve(passage, question, choices):
relevant_text = get_relevant_text(passage, question)
alchemyapi = AlchemyAPI()
passage_sentiment = {}
response = alchemyapi.keywords('text', relevant_text, {'sentiment': 1})
if response['status'] == 'OK':
for keyword in response['keywords']:
if 'sentiment' in keyword.keys() and 'score' in keyword['sentiment']:
passage_sentiment[keyword["text"]] = keyword['sentiment']
choices_sentiment = {}
for choice in choices:
response = alchemyapi.keywords('text', "the" + choices[choice], {'sentiment': 1})
if response['status'] == 'OK':
for keyword in response['keywords']:
if 'sentiment' in keyword.keys() and 'score' in keyword['sentiment']:
choices_sentiment[choice] = keyword['sentiment']
return rank_choices_sentiment(passage_sentiment, choices_sentiment)
def getAlcData(arrOfObj):
alchemyapi = AlchemyAPI()
#for x in range(0,len(arrOfObj)):
for x in range(0, 10):
asc = unicodedata.normalize('NFKD', arrOfObj[x].text).encode('ascii','ignore')
print x
print asc
arrOfObj[x].responseEntities = alchemyapi.entities('text',asc, { 'sentiment':1 })
arrOfObj[x].responseKeywords = alchemyapi.keywords('text',asc, { 'sentiment':1 })
class App:
def __init__(self):
self.alchemyapi = AlchemyAPI()
self.raw_text = ''
self.concepts = None
self.keywords = None
def parse_url(self, url=None):
text_response = self.alchemyapi.text('url', url)
if text_response['status'] == 'OK':
self.raw_text = text_response['text'].encode('utf-8')
else:
print('Error in text extraction call: ', text_response['statusInfo'])
def extract_concepts(self):
concept_response = self.alchemyapi.concepts('text', self.raw_text)
if concept_response['status'] == 'OK':
self.concepts = concept_response['concepts']
# print('## Concepts ##')
# for concept in self.concepts:
# print('text: ', concept['text'])
# print('relevance: ', concept['relevance'])
# print('')
else:
print('Error in concept tagging call: ', concept_response['statusInfo'])
def extract_keywords(self):
keyword_response = self.alchemyapi.keywords('text', self.raw_text, {'sentiment': 1})
if keyword_response['status'] == 'OK':
self.keywords = keyword_response['keywords']
# print('')
# print('## Keywords ##')
# for keyword in self.keywords:
# print('text: ', keyword['text'].encode('utf-8'))
# print('relevance: ', keyword['relevance'])
# print('sentiment: ', keyword['sentiment']['type'])
# if 'score' in keyword['sentiment']:
# print('sentiment score: ' + keyword['sentiment']['score'])
# print('')
else:
print('Error in keyword extraction call: ', keyword_response['statusInfo'])
def define_concepts(self):
for concept in self.concepts:
definition = duckduckgo.get_zci(concept['text'])
print('%s -> %s' % (concept['text'], definition))
print('')
def define_keywords(self):
for keyword in self.keywords:
definition = duckduckgo.get_zci(keyword['text'])
print('%s -> %s' % (keyword['text'], definition))
print('')
def checkDailyQuotaAndRunAlchemy(commentDb,cruiseLines):
with open('data/Alchemy_response_keywords.json', 'rb') as fp:
returned_keywords = json.load(fp)
with open('data/Alchemy_response_relations.json', 'rb') as fp:
returned_relations = json.load(fp)
alchemyapi = AlchemyAPI()
test="test if finished Alchemy daily quota"
response = alchemyapi.keywords('text', test, {'sentiment': 0})
if response['status'] == 'OK':
returned_keywords,returned_relations=runAlchemyApi(cruiseLines,commentDb,returned_keywords,returned_relations,alchemyapi)
else:
print 'Error in keyword extraction call: ', response['statusInfo']
return returned_keywords, returned_relations
def solve(passage, question, choices):
relevant_text = get_relevant_text(passage, question)
alchemyapi = AlchemyAPI()
passage_sentiment = {}
response = alchemyapi.keywords('text', relevant_text, {'sentiment': 1})
if response['status'] == 'OK':
for keyword in response['keywords']:
if 'sentiment' in keyword.keys(
) and 'score' in keyword['sentiment']:
passage_sentiment[keyword["text"]] = keyword['sentiment']
choices_sentiment = {}
for choice in choices:
response = alchemyapi.keywords('text', "the" + choices[choice],
{'sentiment': 1})
if response['status'] == 'OK':
for keyword in response['keywords']:
if 'sentiment' in keyword.keys(
) and 'score' in keyword['sentiment']:
choices_sentiment[choice] = keyword['sentiment']
return rank_choices_sentiment(passage_sentiment, choices_sentiment)
def ExtractKeyword(text):
alchemyapi = AlchemyAPI()
response = alchemyapi.keywords('text', text, {'sentiment': 1})
results = []
if response['status'] == 'OK':
for keyword in response['keywords']:
results.append(keyword['text'])
# print('text: ', keyword['text'].encode('utf-8'))
# print('relevance: ', keyword['relevance'])
else:
print('Error in keyword extaction call: ', response['statusInfo'])
return results
def getSoup():
sock = urllib.urlopen('https://en.wikipedia.org/wiki/Motocross')
sockRaw = sock.read()
soup = BeautifulSoup(sockRaw, "html.parser")
soupText = soup.get_text()
# use the alchemyAPI to find the keyword/phrases from the texts
alchemyapi = AlchemyAPI()
response = alchemyapi.keywords('text', soupText, {'maxRetrieve': 10})
if response['status'] == 'OK':
print "\nThe Keywords are:"
for i in response['keywords']:
print "Word: " + i["text"] + ", Relevance: " + i["relevance"]
else:
print "Something went wrong with Alchemy."
def checkDailyQuotaAndRunAlchemy(commentDb, cruiseLines):
with open('data/Alchemy_response_keywords.json', 'rb') as fp:
returned_keywords = json.load(fp)
with open('data/Alchemy_response_relations.json', 'rb') as fp:
returned_relations = json.load(fp)
alchemyapi = AlchemyAPI()
test = "test if finished Alchemy daily quota"
response = alchemyapi.keywords('text', test, {'sentiment': 0})
if response['status'] == 'OK':
returned_keywords, returned_relations = runAlchemyApi(
cruiseLines, commentDb, returned_keywords, returned_relations,
alchemyapi)
else:
print 'Error in keyword extraction call: ', response['statusInfo']
return returned_keywords, returned_relations
def getSoup():
sock = urllib.urlopen('https://en.wikipedia.org/wiki/Motocross')
sockRaw = sock.read()
soup = BeautifulSoup(sockRaw, "html.parser")
soupText = soup.get_text()
# use the alchemyAPI to find the keyword/phrases from the texts
alchemyapi = AlchemyAPI()
response = alchemyapi.keywords('text', soupText, {'maxRetrieve': 10})
if response['status'] == 'OK':
print "\nThe Keywords are:"
for i in response['keywords']:
print "Word: " + i["text"] + ", Relevance: " + i["relevance"]
else:
print "Something went wrong with Alchemy."
def extract_keyword_en(self, corpus_text, min_len=4, max_len=50):
alchemyapi = AlchemyAPI(ALCHEMY_API_KEY)
response = alchemyapi.keywords('text', corpus_text, {'sentiment': 1})
keywords = []
if response['status'] == "OK":
for keyword in response['keywords']:
phrase = keyword['text'].encode('utf8')
freq = self.phrase_frequency(phrase, corpus_text)
if freq > 0 and phrase.lower() != self.query.lower() and len(phrase) >= min_len and len(phrase) <= max_len:
score = float(keyword['relevance'].encode('utf8'))
kw = Keyword(phrase, score, freq)
keywords.append(kw)
return keywords[:min(len(keywords), self.numberofkeywords)]
def setKeywords(self):
alchemyapi = AlchemyAPI()
response = alchemyapi.keywords('text',self.content, { 'sentiment':1 })
if response['status'] == 'OK':
for keyword in response['keywords']:
self.keywords.add(keyword['text'].encode('ascii','ignore'))
else:
print('Error in concept tagging call: ', response['statusInfo'])
self.keywords = set(["Automatic keyword generation failed"])
response = alchemyapi.concepts('text',self.content, { 'sentiment':1 })
if response['status'] == 'OK':
for keyword in response['concepts']:
self.keywords.add(keyword['text'].encode('ascii','ignore'))
else:
print('Error in concept tagging call: ', response['statusInfo'])
self.keywords = set(["Automatic keyword generation failed"])
def analysecontent(content):
"""
Process/Analyse the extracted contents with Alchemy API
Assumption: api_key.txt with a valid key is available from where this program is getting executed.
"""
print ("Processing extracted text with AlchemyAPI...")
alchemyapi = AlchemyAPI()
response = alchemyapi.keywords("text", content, {"maxRetrieve": 10})
if response["status"] == "OK":
print ("---------------------------------")
print ("## Keywords ## Relevance")
for keyword in response["keywords"]:
print ("{0}: {1}".format(keyword["text"].encode("utf-8"), keyword["relevance"]))
print ("---------------------------------")
else:
print ("Error in keyword extraction call: ", response["statusInfo"])
def analysecontent(content):
"""
Process/Analyse the extracted contents with Alchemy API
Assumption: api_key.txt with a valid key is available from where this program is getting executed.
"""
print('Processing extracted text with AlchemyAPI...')
alchemyapi = AlchemyAPI()
response = alchemyapi.keywords('text', content, {'maxRetrieve': 10})
if response['status'] == 'OK':
print('---------------------------------')
print('## Keywords ## Relevance')
for keyword in response['keywords']:
print("{0}: {1}".format(keyword['text'].encode('utf-8'),
keyword['relevance']))
print('---------------------------------')
else:
print('Error in keyword extraction call: ', response['statusInfo'])
class NLP:
def __init__(self):
self.alchemyapi = AlchemyAPI()
def get_categories(self, text):
response = self.alchemyapi.taxonomy('text', text)
if response['status'] == 'OK' and len(response['taxonomy']) > 0:
taxonomy = response['taxonomy'][0]
tokens = taxonomy['label'].split('/')
return tokens[1]
def get_keywords(self, text):
response = self.alchemyapi.keywords('text', text)
if response['status'] == 'OK' and len(response['keywords']) > 0:
return [x['text'] for x in response['keywords']]
def extract_keyword_alchemy(self, corpus):
alchemyapi = AlchemyAPI()
corpus_text = "\n".join(corpus)
response = alchemyapi.keywords('text', corpus_text, {'sentiment': 1})
keywords = []
if response['status'] == "OK":
for keyword in response['keywords']:
pharse = keyword['text'].encode('utf8')
score = float(keyword['relevance'].encode('utf8'))
kw = Keyword(pharse, score)
keywords.append(kw)
sorted_keywords = sorted(keywords, key=lambda t: t.get_score() * -1)
return sorted_keywords[:min(len(sorted_keywords), self.numberofkeywords)]
def get_sentiment(text):
alchemyapi = AlchemyAPI()
alchemyapi.apikey = get_random_alchemy_credentials()
response = alchemyapi.keywords('text', text, {'sentiment': 1})
relevances = []
if 'keywords' not in response or len(response['keywords']) == 0:
return None
for keyword in response["keywords"]:
for company_word in concepts:
if company_word.lower() in text.lower() and 'sentiment' in keyword:
if 'score' in keyword['sentiment']:
relevances.append(float(keyword['sentiment']['score']))
elif keyword['sentiment']['type'] == 'neutral':
relevances.append(0.5)
if not relevances:
return 0.5
else:
return float("{0:.2f}".format(sum(relevances)/len(relevances)))
def extractKeywordsFromUrl(self, url):
"""method for extracting keywords from given text"""
# creating AlchemyAPI object
alchemyapi = AlchemyAPI()
# requesting json response from AlchemyAPI server
response = alchemyapi.keywords("url", url)
if response["status"] == "OK":
for keywords in response["keywords"]:
# concept object for storing the extracted concept
keyword = AlchemyStructure.Keyword()
# extracting the keyword
keyword.setText(keywords["text"])
# extracting the relevance of keyword
keyword.setRelevance(keywords["relevance"])
# instantiating the sentiment object
sentimentObj = AlchemyStructure.Sentiment()
print(keywords["sentiment"]["type"])
# extracting the score of the sentiment associated with the keyword
if keywords["sentiment"]["type"] == "neutral":
sentimentObj.setScore("0")
else:
sentimentObj.setScore(keywords["sentiment"]["score"])
# extracting the type of the sentiment associated with the keyword -> positive, negative or neutral
sentimentObj.setType(keywords["sentiment"]["type"])
# set the sentiment for keyword
keyword.setSentiment(sentimentObj)
# append the concept into the list of retrieved concepts
self.keywordsFromUrl.append(keyword)
else:
print("Error in keyword tagging call: ", response["statusInfo"])
def keywords_and_sentiment(contentList):
# Accepts a string of posts (or chats), and returns a list of keywords
alchemyapi = AlchemyAPI()
relevanceList = []
sentiment = 0
counter = 0
for post_message in contentList:
response = alchemyapi.keywords("text", post_message, {"sentiment": 1})
if response["status"] == "OK":
for keyword in response["keywords"]:
if "score" in keyword["sentiment"]:
sentiment += float(keyword["sentiment"]["score"])
if float(keyword["relevance"]) > 0.97:
relevanceList.append(keyword["text"])
counter += 1
if DEV_MODE:
if not counter is 0:
print float(sentiment / counter)
return relevanceList
def keywords_and_sentiment(contentList):
# Accepts a string of posts (or chats), and returns a list of keywords
alchemyapi = AlchemyAPI()
relevanceList = []
sentiment = 0
counter = 0
for post_message in contentList:
response = alchemyapi.keywords('text', post_message, {'sentiment': 1})
if response['status'] == 'OK':
for keyword in response['keywords']:
if 'score' in keyword['sentiment']:
sentiment += float(keyword['sentiment']['score'])
if float(keyword['relevance']) > .97:
relevanceList.append(keyword['text'])
counter += 1
if DEV_MODE:
if not counter is 0:
print float(sentiment / counter)
return relevanceList
def run_app():
##### part 1
page = requests.get('https://en.wikipedia.org/wiki/Michael_Jordan')
tree = html.fromstring(page.content)
h3s = tree.xpath('//h3/span/text()')
h3s_combined = ''
for next_h3 in h3s:
h3s_combined += ' '
h3s_combined += next_h3
print('***********************************************')
print('*** The h3s on this page were: ', h3s_combined)
print('***********************************************')
##### part 2
#alchemy_api_key="xxxx"
alchemyapi = AlchemyAPI()
response = alchemyapi.keywords('text', h3s_combined, {
'sentiment': 1,
'maxRetrieve': 10
})
if response['status'] == 'OK':
print('## Response Object ##')
# print(json.dumps(response, indent=4))
print('')
print('## Keywords ##')
for keyword in response['keywords']:
print('text: ', keyword['text'].encode('utf-8'))
print('relevance: ', keyword['relevance'])
print('sentiment: ', keyword['sentiment']['type'])
print('')
else:
print('Error in keyword extaction call: ', response['statusInfo'])
def alchemyExtract(cleanText,options):
"""Uses alchemyAPI to find keywords in the cleaned text.
In this case,options[1] should specify the number of keywords to be
used. Default to 4."""
try:
options[1]=int(options[1])
except IndexError as e:
options[1]=4
except ValueError as e:
print("You selected alchemy, and options[1] was not a valid integer.")
raise e
from alchemyapi import AlchemyAPI
alch = AlchemyAPI()
response = alch.keywords('text',cleanText,{'sentiment':1})
rKeywords = response['keywords']
finalKeywords = []
if options[1]<len(rKeywords):
for i in range(options[1]):
finalKeywords.append(rKeywords[i]['text'])
else:
finalKeywords = rKeywords
return "\n\nKeywords: " + ", ".join(finalKeywords) #figure out formatting for this later.
def nlp_process(ids,ids_hash):
#instantiate an elasticsearch client
es = Elasticsearch()
#instantiate an alchemy client
alchemyapi = AlchemyAPI()
for item in ids:
data = ' '.join(ids_hash[item])
lowers = data.lower()
alchem_data = []
response = alchemyapi.keywords('text', lowers, {'sentiment': 1})
if response['status'] == 'OK':
print('#Success#')
for keyword in response['keywords']:
al_temp = defaultdict()
al_temp['text'] = keyword['text'].encode('utf-8')
al_temp['relevance'] = keyword['relevance']
al_temp['sentiment'] = keyword['sentiment']['type']
if 'score' in keyword['sentiment']:
al_temp['score'] = keyword['sentiment']['score']
alchem_data.append(al_temp)
else:
print('Error in keyword extaction call: ', response['statusInfo'])
print len(alchem_data)
# prepare body for insertion
doc = {
"business_id" : item,
"word_freq": alchem_data
}
exit()
template = { "create": { "_index": "alchem", "_type": "doc"} }
res = es.index(index="alchem", doc_type='doc', body=doc)
def alchemyExtract(cleanText, options):
"""Uses alchemyAPI to find keywords in the cleaned text.
In this case,options[1] should specify the number of keywords to be
used. Default to 4."""
try:
options[1] = int(options[1])
except IndexError as e:
options[1] = 4
except ValueError as e:
print("You selected alchemy, and options[1] was not a valid integer.")
raise e
from alchemyapi import AlchemyAPI
alch = AlchemyAPI()
response = alch.keywords('text', cleanText, {'sentiment': 1})
rKeywords = response['keywords']
finalKeywords = []
if options[1] < len(rKeywords):
for i in range(options[1]):
finalKeywords.append(rKeywords[i]['text'])
else:
finalKeywords = rKeywords
return "\n\nKeywords: " + ", ".join(
finalKeywords) #figure out formatting for this later.
def run_app():
##### part 1
page = requests.get('https://en.wikipedia.org/wiki/Michael_Jordan')
tree = html.fromstring(page.content)
h3s = tree.xpath('//h3/span/text()')
h3s_combined = ''
for next_h3 in h3s:
h3s_combined += ' '
h3s_combined += next_h3
print('***********************************************')
print('*** The h3s on this page were: ', h3s_combined)
print('***********************************************')
##### part 2
#alchemy_api_key="xxxx"
alchemyapi = AlchemyAPI()
response = alchemyapi.keywords('text', h3s_combined, {'sentiment': 1, 'maxRetrieve': 10})
if response['status'] == 'OK':
print('## Response Object ##')
# print(json.dumps(response, indent=4))
print('')
print('## Keywords ##')
for keyword in response['keywords']:
print('text: ', keyword['text'].encode('utf-8'))
print('relevance: ', keyword['relevance'])
print('sentiment: ', keyword['sentiment']['type'])
print('')
else:
print('Error in keyword extaction call: ', response['statusInfo'])
def post(self):
sid = self.get_argument('sid')
abstract = self.get_argument('abstract')
print abstract
url = "https://api-us.clusterpoint.com/100629/stud/_search.json"
headers = {"Authorization":"Basic bGVvcGFuaWdyYWhpQGdtYWlsLmNvbToyM2xlbzIz"}
string = "<project><abstract>"+ abstract +"</abstract></project>"
values = dict(query=string)
r = requests.post(url,data=json.dumps(values), headers=headers)
responseDict = json.loads(r.content)
for each in responseDict['documents'][0]['project']:
if each['abstract'] == abstract:
mytext = each['abstract']
alchemyapi = AlchemyAPI()
response = alchemyapi.keywords('text', abstract, {'sentiment': 1})
finalKeys = list()
for each in response['keywords']:
try:
if float(each['relevance'])>0.5:
finalKeys.append(each['text'])
print finalKeys
self.render("tags.html", finalKeys=finalKeys)
except:
pass
def sentiment_alchemy(url):
alchemyapi = AlchemyAPI()
response = alchemyapi.sentiment('url', url)
response['usage'] = None
if response['status'] == 'OK':
print('## Response Object ##')
print(json.dumps(response, indent=4))
print('')
print('## Author ##')
print('author: ', response.get('author', ''))
print('')
else:
print('Error in author extraction call: ', response['statusInfo'])
response = alchemyapi.keywords('url', url)
del (response['usage'])
if response['status'] == 'OK':
print('## Response Object ##')
print(json.dumps(response, indent=4))
print('')
print('## Keywords ##')
for keyword in response['keywords']:
print('text: ', keyword['text'].encode('utf-8'))
print('relevance: ', keyword['relevance'])
print('sentiment: ', keyword.get('sentiment', {}).get('type', ''))
if 'score' in keyword.get('sentiment', {}):
print('sentiment score: ' + keyword['sentiment']['score'])
print('')
else:
print('Error in keyword extaction call: ',
response.get('statusInfo', ''))
print('Checking entities . . . ')
response = alchemyapi.entities('text', test_text);
assert(response['status'] == 'OK')
response = alchemyapi.entities('html', test_html);
assert(response['status'] == 'OK')
response = alchemyapi.entities('url', test_url);
assert(response['status'] == 'OK')
response = alchemyapi.entities('random', test_url);
assert(response['status'] == 'ERROR') #invalid flavor
print('Entity tests complete!')
print('')
#Keywords
print('Checking keywords . . . ')
response = alchemyapi.keywords('text', test_text);
assert(response['status'] == 'OK')
response = alchemyapi.keywords('html', test_html);
assert(response['status'] == 'OK')
response = alchemyapi.keywords('url', test_url);
assert(response['status'] == 'OK')
response = alchemyapi.keywords('random', test_url);
assert(response['status'] == 'ERROR') #invalid flavor
print('Keyword tests complete!')
print('')
#Concepts
print('Checking concepts . . . ')
import os, sys, json
from alchemyapi import AlchemyAPI
alchemyapi = AlchemyAPI()
fname = sys.argv[1]
with open(fname, 'r') as f:
Text = f.readlines()
text_file = open(fname, 'w')
for fig in range(len(Text)):
response = alchemyapi.keywords("text", Text[fig])
if response['status'] == 'OK':
for keyword in response['keywords']:
text_file.write(
str(fig) + '\t' + str(keyword['text'].encode('utf-8')) + '\t' +
str(keyword['relevance']) + '\n')
else:
print('Error in keyword extaction call: ', response['statusInfo'])
text_file.close()
else:
print('Error in sentiment analysis call: ', response['statusInfo'])
print('')
print('')
print('')
print('############################################')
print('# Keyword Extraction Example #')
print('############################################')
print('')
print('')
print('Processing text: ', demo_text)
print('')
response = alchemyapi.keywords('text', demo_text, {'sentiment': 1})
if response['status'] == 'OK':
print('## Response Object ##')
print(json.dumps(response, indent=4))
print('')
print('## Keywords ##')
for keyword in response['keywords']:
print('text: ', keyword['text'])
print('relevance: ', keyword['relevance'])
print(
'sentiment: ', keyword['sentiment']['type'] + ' (' +
keyword['sentiment']['score'] + ')')
print('')
else:
ent_rele=[]
ent_type=[]
if response['status'] == 'OK':
flag=1
for entity in response['entities']:
ent.append(entity['text'])
ent_rele.append(entity['relevance'])
ent_type.append(entity['type'])
else:
print('Error in entity extraction call: ', response['statusInfo'])
if flag==1:
#response = json.loads(json.dumps(alchemyapi.sentiment("text", trans_text)))
###### GETTING AN ERROR HERE FOR SOME REASON ######
#senti=response["docSentiment"]["type"]
response = json.loads(json.dumps(alchemyapi.keywords('text', trans_text, {'sentiment': 1})))
#size=len(response['keywords'])
keywords=[]
if response['status'] == 'OK':
for word in response['keywords']:
keywords.append(word['text'])
else:
print('Error in entity extraction call: ', response['statusInfo'])
response=json.loads(json.dumps(alchemyapi.concepts("text",trans_text)))
#size=len(response['concepts'])
concept=[]
if response['status'] == 'OK':
for con in response['concepts']:
mixed = None
alchemy_response = alchemyapi.sentiment_targeted(
'url', response['items'][i]['link'], topic)
if alchemy_response['status'] == 'OK':
sentiment = alchemy_response['docSentiment']['type']
if 'score' in alchemy_response['docSentiment']:
score = alchemy_response['docSentiment']['score']
if 'mixed' in alchemy_response['docSentiment']:
mixed = alchemy_response['docSentiment']['mixed']
else:
print j + i + 1, 'Error in targeted sentiment analysis call: ', alchemy_response[
'statusInfo']
# Keyword sentiment analysis
keyword_response = alchemyapi.keywords('url',
response['items'][i]['link'],
{'sentiment': 1})
key_sentiment = None
key_score = None
key_mixed = None
if keyword_response['status'] == 'OK':
if 'keywords' in keyword_response:
for keyword in keyword_response['keywords']:
if keyword['text'].lower() == topic.lower():
if 'sentiment' in keyword:
key_sentiment = keyword['sentiment']['type']
if 'score' in keyword['sentiment']:
key_score = keyword['sentiment']['score']
if 'mixed' in keyword['sentiment']:
key_mixed = keyword['sentiment']['mixed']
else:
print('relevance : ', concept['relevance'])
print('')
concept_temp.append(concept['text'])
con_rel_temp.append(concept['relevance'])
"""
concept_list[title_num][conc_num] = concept['text']
relevance_list[title_num][conc_num]=concept['relevance']
conc_num += 1
"""
else:
print('Error in concept tagging call: ', response['statusInfo'])
print('다시 시도 하거나 다른 주소를 입력하세요.')
response2 = alchemyapi.keywords('url',
input_url,
options={'maxRetrieve': input_maxC})
if response2['status'] == 'OK':
print('## Keywords ##')
keyword_temp = []
key_rel_temp = []
#rel_temp=[]
for keyword in response2['keywords']:
print('text: ', keyword['text'])
print('relevance: ', keyword['relevance'])
print('')
keyword_temp.append(keyword['text'])
key_rel_temp.append(keyword['relevance'])
else:
print('Error in keywords call: ', response2['statusInfo'])
print('다시 시도 하거나 다른 주소를 입력하세요.')
import bs4
from bs4 import BeautifulSoup
import urllib2
url= "http://businessoverbroadway.com/top-10-skills-in-data-science"
page = urllib2.urlopen(url)
soup = bs4.BeautifulSoup(page.read(), 'html.parser')
mytext = soup.find('p').getText()
print mytext
#Take this text and store it in the program to use in the next step.
# Part 2
from alchemyapi import AlchemyAPI
alchemyapi = AlchemyAPI()
response = alchemyapi.keywords('text', mytext)
print "Keywords" '\t' '\t' "Relevance"
for kyw in response['keywords'][1:10]:
k1= kyw['text']
r1= kyw['relevance']
print " "
print "--------------------------------------"
print k1, '\t', '\t', r1
con_rel_temp = []
key_rel_temp = []
for concept in response['concepts']:
print('text: ', concept['text'])
print('relevance : ', concept['relevance'])
print('')
concept_temp.append(concept['text'])
con_rel_temp.append(concept['relevance'])
"""
concept_list[title_num][conc_num] = concept['text']
relevance_list[title_num][conc_num]=concept['relevance']
conc_num += 1
"""
response = alchemyapi.keywords('text',
input_text,
options={'maxRetrieve': input_maxC})
print('## Keywords ##')
rel_temp = []
for keyword in response['keywords']:
print('text: ', keyword['text'])
print('relevance: ', keyword['relevance'])
print('')
keyword_temp.append(keyword['text'])
key_rel_temp.append(keyword['relevance'])
else:
print('Error in concept tagging call: ', response['statusInfo'])
if 'y' == input('이 결과를 저장하시겠습니까?(y/n)'):
title_input = input('제목을 입력하세요 : ')
concept_list.append(concept_temp)
# For this assignment, I will read in the text of a magazine article and pass it through the Alchemy API
# to rank the keywords. First, we must read the text in from the html with the help of beautifulsoup.
import urllib
import bs4
from alchemyapi import AlchemyAPI
url = "http://www.theatlantic.com/magazine/archive/2015/09/the-coddling-of-the-american-mind/399356/"
f = urllib.urlopen(url)
website = f.read()
soup = bs4.BeautifulSoup(website, 'html.parser')
story = soup.find('div', {'class': 'article-body'}).getText()
story
# Now that we have the text of the article, we will pass it to the Alchemy API and print out the top 10
# ranked words and their respective relevance.
alchemyapi = AlchemyAPI()
keywords = alchemyapi.keywords("text", story, {'maxRetrieve': 10})
i = 0
for keyword in keywords[u'keywords']:
i += 1
print 'Rank:', i, '\t Relevance:', "\t %s \t %s" % (keyword['relevance'],
keyword['text'])
def trigger(self, text):
print('Request: {0}'.format(text))
try:
alchemyapi = AlchemyAPI()
except Exception as e:
print('Could not connect to AlchemyAPI. Details: {}'.format(e))
relations = alchemyapi.relations('text', text)
action = None # Action to take. Only 'Play' implemented so far.
format = None # In pratice 'album' or 'track'.
toPlay = None # Album or track name.
if relations['status'] == 'OK':
print('## Object ##')
print(json.dumps(relations, indent=4))
for relation in relations['relations']:
if 'action' in relation:
print('Action: ', relation['action']['text'].encode('utf-8'))
if relation['action']['verb']['text'] in ('play', 'listen'):
action = 'PLAY'
else:
print('Error in relation extaction call: ', relations['statusInfo'])
# If no action found, we abort.
if not action:
print('Could not find any action to take.')
return
# Detect the artist to play
artists = []
artist = None
keywords = alchemyapi.keywords('text', text, {'sentiment': 1})
if keywords['status'] == 'OK':
print('## Response Object ##')
print(json.dumps(keywords, indent=4))
for keyword in keywords['keywords']:
artists.append((keyword['relevance'], extractArtists(keyword['text'].encode('utf-8'))))
else:
print('Error in keyword extaction call: ', keywords['statusInfo'])
print('Action: {0}'.format(action))
print('Artists extracted from the request: {0}'.format(' '.join([str(n)+'('+c+')' for c,n in artists])))
for e in artists:
if not e[1]:
continue
else:
artist = e[1][-1]
break
if not artist:
print('Could not find any artist.')
return
print('Selected artist: {0}'.format(artist))
print('To simplify I assume you are looking for an album and not a track.')
albums = AudioDB_getAlbums(artist)
if not albums:
print('Could not find any album for {0}'.format(artist))
return
album = None
for name, item in albums.iteritems():
if name in text:
album = item
break
if not album:
print('Could not find any specific album, so will try to extract a track from the full discography')
for _, album in albums.iteritems():
tracks = AudioDB_getTracks(album['id'])
for track in tracks:
if track in text:
format = 'song'
toPlay = track
break
if not toPlay:
format = 'album'
toPlay = albums.itervalues().next()['name'] # We assume the artist has at least one album
print('Could not find any album or track so I will play the album {0}'.format(toPlay))
else:
print('Selected album is {0}. Now, check the tracks.'.format(album))
tracks = AudioDB_getTracks(album['id'])
for track in tracks:
if track in text:
format = 'song'
toPlay = track
break
if not toPlay:
print("Could not find a track, so I will play the whole album.")
toPlay = album['name']
format = 'album'
else:
print('The song to play will be {0}'.format(toPlay))
print('Selected album/track: {0}'.format(toPlay))
hint = 'album' if format == 'album' else ''
args = {'q': ' '.join([toPlay, artist, hint]),
'max_results': 1}
print('Youtube query: {0}'.format(args['q']))
try:
video = get_video(args)
if video:
video_ID = video[0]
print('Youtube Video ID: {0}'.format(video_ID))
uri = 'http://www.youtube.com/watch?v={0}'.format(video_ID)
res = self.call('music.fromyoutube', uri)
else:
print("Could not find any stream.")
except HttpError, e:
print("An HTTP error %d occurred:\n%s" % (e.resp.status, e.content))
class AlchemyPost:
def __init__(self, post_tumblr, post_id, consumer_key, consumer_secret, oauth_token, oauth_secret):
self.post_tumblr = post_tumblr
self.post_id = post_id
self._init_tumblr(consumer_key, consumer_secret, oauth_token, oauth_secret)
self._init_alchemy()
def _init_tumblr(self, consumer_key, consumer_secret, oauth_token, oauth_secret):
self._client = pytumblr.TumblrRestClient(consumer_key, consumer_secret, oauth_token, oauth_secret)
def _init_alchemy(self):
self.alchemyapi = AlchemyAPI()
self.content = {}
def analyze_post(self):
self.post = self._get_content_post()
self._alchemy_entities()
self._alchemy_keywords()
self._alchemy_concepts()
self._alchemy_sentiment()
self._alchemy_relations()
self._alchemy_category()
self._alchemy_feeds()
self._alchemy_taxonomy()
def print_content(self):
print(json.dumps(self.content, indent=4))
def _get_content_post(self):
print "*",
infos = self._get_infos_post()
self.title = ''
self.tags = []
if 'tags' in infos:
self.tags = infos['tags']
if infos['type'] == 'text':
return self._get_content_text(infos)
if infos['type'] == 'quote':
return self._get_content_quote(infos)
return ''
def _get_infos_post(self):
infos = self._client.posts(self.post_tumblr, id=self.post_id)
if 'posts' in infos and len(infos['posts'])>0:
return infos['posts'][0]
return {}
def _get_content_text(self, infos):
content = "<h1>" + str(infos['title']) + "</h1>"
content += " <br>" + str(infos['body'])
content += " <br>" + " ".join(infos['tags'])
return content
def _get_content_quote(self, infos):
content = str(infos['text'])
content += " <br>" + str(infos['source'])
content += " <br>" + " ".join(infos['tags'])
return content
def _alchemy_entities(self):
print ".",
response = self.alchemyapi.entities('html', self.post)
if response['status'] != 'OK':
return False
self.content['entities'] = response['entities']
return True
def _alchemy_keywords(self):
print ".",
response = self.alchemyapi.keywords('html', self.post)
if response['status'] != 'OK':
return False
self.content['keywords'] = response['keywords']
return True
def _alchemy_concepts(self):
print ".",
response = self.alchemyapi.concepts('html', self.post)
if response['status'] != 'OK':
return False
self.content['concepts'] = response['concepts']
return True
def _alchemy_sentiment(self):
print ".",
response = self.alchemyapi.sentiment('html', self.post)
if response['status'] != 'OK':
return False
self.content['sentiment'] = response['docSentiment']
return True
def _alchemy_relations(self):
print ".",
response = self.alchemyapi.relations('html', self.post)
if response['status'] != 'OK':
return False
self.content['relations'] = response['relations']
return True
def _alchemy_category(self):
print ".",
response = self.alchemyapi.category('html', self.post)
if response['status'] != 'OK':
return False
self.content['category'] = response['category']
self.content['score'] = response['score']
return True
def _alchemy_feeds(self):
print ".",
response = self.alchemyapi.feeds('html', self.post)
if response['status'] != 'OK':
return False
self.content['feeds'] = response['feeds']
return True
def _alchemy_taxonomy(self):
print ".",
response = self.alchemyapi.taxonomy('html', self.post)
if response['status'] != 'OK':
return False
self.content['taxonomy'] = response['taxonomy']
return True
#Entities
print('Checking entities . . . ')
response = alchemyapi.entities('text', test_text)
assert (response['status'] == 'OK')
response = alchemyapi.entities('html', test_html)
assert (response['status'] == 'OK')
response = alchemyapi.entities('url', test_url)
assert (response['status'] == 'OK')
response = alchemyapi.entities('random', test_url)
assert (response['status'] == 'ERROR') #invalid flavor
print('Entity tests complete!')
print('')
#Keywords
print('Checking keywords . . . ')
response = alchemyapi.keywords('text', test_text)
assert (response['status'] == 'OK')
response = alchemyapi.keywords('html', test_html)
assert (response['status'] == 'OK')
response = alchemyapi.keywords('url', test_url)
assert (response['status'] == 'OK')
response = alchemyapi.keywords('random', test_url)
assert (response['status'] == 'ERROR') #invalid flavor
print('Keyword tests complete!')
print('')
#Concepts
print('Checking concepts . . . ')
response = alchemyapi.concepts('text', test_text)
assert (response['status'] == 'OK')
response = alchemyapi.concepts('html', test_html)
def nlp_process(ids,ids_hash, tgt_file_name):
# instantiate an elasticsearch client
#es = Elasticsearch()
outfile = open(tgt_file_name, 'w')
# instantiate an alchemy client
alchemyapi = AlchemyAPI()
# loop through each business
for business_id in ids:
# loop through reviews for each star
for star in range(1,6):
# combine review text from all reviews rated same score
print "processing business {} for review with {} star(s)".format(business_id, star)
data = '.'.join(ids_hash[(business_id, star)])
alchem_keywords = []
alchem_concepts = []
# perform analysis only if combined review length across all the reviews
# within a star is more than 100 characters
if len(data) >= 100:
# call Alchemy API combined call to determing keywords, sentiment
# and concepts (tags)
response = alchemyapi.keywords('text', data, {'sentiment': 1, 'maxRetrieve': 100})
# process response
if response['status'] == 'OK':
print('#Success#')
# process keywords
for keyword in response['keywords']:
al_temp = defaultdict()
al_temp['text'] = keyword['text'].encode('utf-8')
al_temp['relevance'] = keyword['relevance']
if 'sentiment' in keyword:
al_temp['sentiment'] = keyword['sentiment'].get('type', 'neutral')
if 'score' in keyword['sentiment']:
al_temp['score'] = keyword['sentiment']['score']
alchem_keywords.append(al_temp)
# process concepts/tags
#for keyword in response['concepts']:
# al_temp = defaultdict()
# al_temp['text'] = keyword['text'].encode('utf-8')
# al_temp['relevance'] = keyword['relevance']
# alchem_concepts.append(al_temp)
else:
print('Error in keyword extaction call: ', response['statusInfo'])
print len(alchem_keywords), len(alchem_concepts)
# prepare body for insertion
if ( len(alchem_keywords) > 0):
doc = {
"business_id" : business_id,
"stars": star,
"word_freq": alchem_keywords,
"topics": alchem_concepts
}
# write to a file
json.dump(doc, outfile)
outfile.write('\n')
print('')
print('')
print('')
print('############################################')
print('# Keyword Extraction Example #')
print('############################################')
print('')
print('')
print('Processing text: ', demo_text)
print('')
response = alchemyapi.keywords('text',demo_text, { 'sentiment':1 })
if response['status'] == 'OK':
print('## Response Object ##')
print(json.dumps(response, indent=4))
print('')
print('## Keywords ##')
for keyword in response['keywords']:
print('text: ', keyword['text'].encode('utf-8'))
print('relevance: ', keyword['relevance'])
print('sentiment: ', keyword['sentiment']['type'])
if 'score' in keyword['sentiment']:
print('sentiment score: ' + keyword['sentiment']['score'])
print('')
for p in paragraphs:
f = re.sub(r"\[.*\]", "", p.text.encode("utf-8")) # regex
# text_extracted = text_extracted + p.text.encode("utf-8")
text_extracted = text_extracted + f
web_text = text_extracted
print '\n\n'
print "URL Source: " + url
print web_text
print '\n\n'
print '#### Top 10 Keywords ####'.center(80, " ")
response = alchemyapi.keywords(
'text', web_text, {'sentiment': 1}) # From example.py of alchemyapi module
# Processing the XML
if response['status'] == 'OK':
print '\n'
print "Keyword".center(50, " ") + "Relevance".center(
10, " ") + "Sentiment".center(10, " ")
print "=" * 80 # underline
i = 0
for keyword in response[
'keywords']: # loop to specify top ten based on relevance
i += 1
if i > 10: break
if 'score' in keyword['sentiment']:
# -*- coding: utf-8 -*-
from alchemyapi import AlchemyAPI
import json
if __name__ == "__main__":
text = u'Artist tackles Chicago\'s pesky pothole problem _ by filling annoying craters with mosaics\r\n\r\nCHICAGO \u2013 The perfect pothole might not exist for many people \u2014 but for mosaic artist Jim Bachor, it\'s one with a nice oval shape. Bachor began filling those potholes a little more than a year ago, after one in front of his house became a hassle.\r\n\r\nBachor doesn\'t just fill them with cement, though. He\'s turned pothole-filling into a public art project \u2014 one with a sense of humor. He fills them with mosaics.\r\n\r\n"I just think it\'s fun to add that little bit of spark into (an) issue that people moan about," says the Chicago resident, whose work also hangs in galleries. He was first drawn to the ancient art form because of its ability to last.\r\n\r\nWith orange cones and vests displaying his last name, Bachor and his helpers look official enough to shut down a street section to work on filling a pothole.\r\n\r\nBachor uses the Chicago city flag design in his pothole art. Some versions hold phone numbers to local auto repair shops, while others simply read "POTHOLE." His most recent installment north of downtown Chicago \u2014 "#21914" \u2014 pokes fun at the huge number of potholes that exist in the city.\r\n\r\nWhile his mosaic art isn\'t a permanent solution to the city\'s pothole problem, it\'s at least a small fix, he says. The city hasn\'t shut down his project, and some community members have expressed gratitude.\r\n\r\nAfter his first project, one neighbor stopped to thank him. "And then 15 minutes later, he came back with a coffee and a Danish for me," Bachor says, "and so I thought that was really cool."\r\n\r\nGerry Shaheen, a resident of Peoria, Illinois, recently stopped to ask Bachor about his work, as the artist installed a mosaic. He says Bachor and his crew are welcome anytime to fill potholes in his city, one of many hit with an especially large number of the annoying craters after a hard winter.\r\n\r\n"I\'ll pave the way for them," Shaheen said with smirk. "No pun intended."'
#text = u'''When three Chicago financial trading firms traveled to Washington, D.C., for a crucial meeting with federal regulators, they didn\'t go alone. They brought a man schooled in the ways of the capital who had recently received $182,000 in political contributions from them: Mayor Rahm Emanuel. At stake was a proposal that would have cut into the firms\' bottom lines by making them hold large cash reserves to protect against volatility in the fledgling, high-frequency trading market they had helped pioneer. Public officials of Emanuel\'s stature rarely show up at the arcane rule-making meetings of the Commodity Futures Trading Commission. But the newly elected mayor delivered a message in the windowless conference room that day, about the important role the trading firms and others like them play in Chicago\'s economy. When regulators drafted the final rules months after the September 2011 meeting, Chicago\'s trading firms got the break they wanted. Since the Washington meeting, the three firms — DRW Trading Group, Chicago Trading Co. and Infinium Capital — have donated an additional $187,800 to Emanuel\'s campaign funds, bringing their total support of the mayor to nearly $370,000. Two other Chicago firms that stood to benefit from the rule, PEAK6 and Chopper Trading, have given an additional $334,000 to Emanuel. All the trading firms but Infinium are among an elite corps of roughly 100 donors Emanuel turned to when he first ran for mayor and is relying on again for his re-election effort. Those donors, consisting of individuals, couples, business partners and firms, are responsible for more than $14 million of the $30.5 million he has raised since fall 2010. More than half of those loyal donors have received a tangible benefit from the mayor or his administration, ranging from city contracts to help with regulators, according to a Tribune analysis. In a monthslong examination of Emanuel\'s fundraising machine, the Tribune documented his connections to investment house executives, corporate CEOs and Washington insiders who sustain him with cash even though their interests lie far beyond the policy decisions issued from the mayor\'s fifth-floor office at City Hall. Emanuel declined to be interviewed for this story. His spokeswoman said his trip to Washington in September 2011 was focused on helping the city but declined to discuss what he did on behalf of the key donors. "The mayor had a series of meetings with federal officials in Washington, as he routinely does, to advocate for Chicago priorities — including federal funding for Chicago — and regarding matters critical to our city\'s interest,\" Kelley Quinn, Emanuel\'s communications director, said in a statement. \"In these meetings he discussed federal funding to invest in housing for low-income Chicagoans, funding to invest in Chicago parkland and open space, a potential national park designation for the historic Pullman neighborhood (on) the South Side, and issues impacting Chicago\'s trading industry that employs more than 33,000 people in the city." Those Washington meetings — among more than 100 he has had during more than 30 trips as mayor — are the result of years of cultivating relationships with fellow national leaders. Emanuel\'s resume is unmatched by any mayor in the country: senior adviser to then-President Bill Clinton, top-ranking Democratic congressman from Chicago and first White House chief of staff to President Barack Obama. Add to that years as a top strategist and fundraiser for the national Democratic Party and a two-year stint as an investment banker.'''
text = text.encode('ascii', 'ignore')
alchemyapi = AlchemyAPI()
response = alchemyapi.keywords('text', text, {
'sentiment': 1,
'maxRetrieve': 20,
'keywordExtractMode': 'strict'
})
if response['status'] == "OK":
print(json.dumps(response, indent=4))
for keyword in response['keywords']:
print('Keyword text:', keyword['text'].encode('utf-8'))
print('Keyword relevance:', keyword['relevance'])
print("**********")
response = alchemyapi.concepts('text', text, {'maxRetrieve': 10})
if response['status'] == "OK":
#print(json.dumps(response, indent = 4))
keywords = [(concept['text'].encode('ascii'),
float(concept['relevance']))
url = "http://quora-api.herokuapp.com/users/" + sys.argv[1] + "/activity"
data = requests.get(url).json()
data = data['activity']
payload = {}
#count=0
#getDocCount()
for activities in data:
title = activities['title']
summary = activities['summary']
print title
document['title'] = title
document['summary'] = summary
labels = al.taxonomy("text", title)
entities = al.entities("html", summary)
keywords = al.keywords("html", summary)
sentiment = al.sentiment("html", summary)
#print labels['taxonomy']
#count+=1
payload['entities'] = {}
payload['keywords'] = []
payload['sentiment'] = {}
docNode = createDocNode(document)
try:
print "Yo"
labels = labels['taxonomy'][0]['label']
print "Yo1"
print labels
labels = func(labels)
print labels
entities = entities['entities']
def extract_keywords():
out_file = open(sys.argv[2], "w")
dir_name = sys.argv[1]
loc_hash = {}
alchemyapi = AlchemyAPI()
for file_in in glob.glob(dir_name + "/*.txt"):
word_count = {}
f_obj = open(file_in)
loc = file_in.split("/")[1].split(".")[0]
loc_hash[loc] = {}
text = f_obj.read()
text = text.replace("*******************", "")
doc_len = len(text)
print(str(doc_len))
text_words = text.split()
unigrams_freq = nltk.FreqDist(text_words)
bigrams = bgs = nltk.bigrams(text_words)
bigrams_freq = nltk.FreqDist(bigrams)
loc_hash[loc]["len"] = doc_len
print(" calling api for " + file_in + ".......")
response = alchemyapi.keywords('text', text, {'sentiment': 0})
if response['status'] == 'OK':
loc_hash[loc]["keywords"] = {}
for keyword in response['keywords']:
words = keyword['text'].split()
if len(words) < 2:
words = keyword['text'].split("/")
if (len(words) <= 2):
loc_hash[loc]["keywords"][keyword['text']] = 0
if len(words) > 1:
for i in range(len(words)):
new_word = words[i]
if new_word not in loc_hash[loc]["keywords"]:
loc_hash[loc]["keywords"][new_word] = 0
if (i != len(words) - 1):
new_word = words[i] + " " + words[i + 1]
if new_word not in loc_hash[loc]["keywords"]:
loc_hash[loc]["keywords"][new_word] = 0
else:
print('Error in keyword extaction call for file :' + file_in +
" : " + response['statusInfo'])
myRE = re.compile('^[a-zA-Z]+$')
for keyword in loc_hash[loc]["keywords"]:
words = keyword.split()
if (len(words) > 1):
if (re.match(myRE, words[0])):
if (re.match(myRE, words[1])):
t = (words[0], words[1])
freq = bigrams_freq[t]
loc_hash[loc]["keywords"][keyword] = freq
else:
if (re.match(myRE, words[0])):
freq = unigrams_freq[words[0]]
loc_hash[loc]["keywords"][keyword] = freq
json.dump(loc_hash, out_file)
for item in json_data[business]["1"]:
business_ones = business_ones + " " + item
if "2" in json_data[business]:
for item in json_data[business]["2"]:
business_twos = business_twos + " " + item
if "3" in json_data[business]:
for item in json_data[business]["3"]:
business_threes = business_threes + " " + item
if "4" in json_data[business]:
for item in json_data[business]["4"]:
business_fours = business_fours + " " + item
if "5" in json_data[business]:
for item in json_data[business]["5"]:
business_fives = business_fives + " " + item
business_response_ones = alchemyapi.keywords("text", business_ones,
{'sentiment': 1})
business_response_twos = alchemyapi.keywords("text", business_twos,
{'sentiment': 1})
business_response_threes = alchemyapi.keywords("text", business_threes,
{'sentiment': 1})
business_response_fours = alchemyapi.keywords("text", business_fours,
{'sentiment': 1})
business_response_fives = alchemyapi.keywords("text", business_fives,
{'sentiment': 1})
data = {}
data['1'] = business_response_ones
data['2'] = business_response_twos
data['3'] = business_response_threes
data['4'] = business_response_fours
data['5'] = business_response_fives
