def main(): alchemyapi = AlchemyAPI() try: filename = sys.argv[1] except IndexError: print "Give a filename as the second argument!" sys.exit(1) text = pdf_to_str(filename) if len(text) >= LENGTH_LIMIT: print "PDF content is longer ({} characters) than the maximum \ of {}, skipping remainder".format(len(text), LENGTH_LIMIT) text = text[:LENGTH_LIMIT] print "KEYWORDS" response = alchemyapi.keywords('text', text) for keyword in response['keywords']: print ' - {}'.format(keyword['text']) print print "CONCEPTS" response = alchemyapi.concepts('text', text) for concept in response['concepts']: print ' - {}'.format(concept['text'])
def getKeywordPerPost(): reader = open('output_sony_posts.txt') all_json_posts = reader.read().splitlines() alchemyapi = AlchemyAPI() counter = 0 for p in all_json_posts: print str(counter) if counter < 1000: counter = counter + 1 continue #elif counter > 2000: # break else: counter = counter + 1 content = json.loads(p)["cleanContent"] response = alchemyapi.keywords('text',content.encode("UTF-8")) if response['status'] == 'OK': keywords = [] posts = "" for keyword in response['keywords']: keywords.append(keyword['text']) posts = posts + keyword['text'] + "," posts = posts[:-1] + "\n" if posts <> "\n": with codecs.open("keyPerPost.txt", "a") as f: f.write(posts.encode("UTF-8")) else: print "error" + str(counter)
def extractKeywordsFromUrl(self,url): """method for extracting keywords from given text""" #creating AlchemyAPI object alchemyapi = AlchemyAPI() #requesting json response from AlchemyAPI server response = alchemyapi.keywords('url', url) if response['status'] == 'OK': for keywords in response['keywords']: #concept object for storing the extracted concept keyword = AlchemyStructure.Keyword() #extracting the keyword keyword.setText(keywords['text']) #extracting the relevance of keyword keyword.setRelevance(keywords['relevance']) #append the concept into the list of retrieved concepts self.keywordsFromUrl.append(keyword) else: print('Error in keyword tagging call: ', response['statusInfo'])
def getKeywords(uID, inputText): alchemyapi = AlchemyAPI() #alchemyapi.loadAPIKey("api_key.txt") response = alchemyapi.keywords('text',inputText) print inputText if response['status'] == 'OK': #print('## Response Object ##') #print(json.dumps(response, indent=4)) #print('') #print('## Keywords ##') keywords = [] posts = uID + " : " for keyword in response['keywords']: keywords.append(keyword['text']) posts = posts + keyword['text'] + "|" userKeywords[uID] = keywords posts = posts + "\n" with codecs.open("outNew.txt", "a") as f: f.write(posts.encode("UTF-8")) return True else: print('idError: ', uID) with codecs.open("keywordOut2.txt", "a") as f: text = uID + "\n" f.write(text.encode("UTF-8")) return False
def get_sentiment(company_id, text): alchemyapi = AlchemyAPI() key_phrases = [] for apikey in engine.get_random_alchemy_credentials(): alchemyapi.apikey = apikey response = alchemyapi.keywords('text', text, {'sentiment': 1}) if response['status'] == 'OK': if len(response['keywords']) == 0: return 0 # related_words = models.RelatedWord.query.filter_by(company_id=company_id).all() for keyword in response["keywords"]: if 'sentiment' in keyword: if keyword['sentiment'].has_key('score'): key_phrases.append(float(keyword['sentiment']['score'])) elif keyword['sentiment']['type'] == 'neutral': key_phrases.append(0) if len(key_phrases) == 0: return 0 else: return float("{0:.2f}".format(sum(key_phrases)/len(key_phrases))) elif response['status'] == 'ERROR' and response['statusInfo'] != 'unsupported-text-language': print "ERROR: getting sentiment " + response['statusInfo'] # Skip onto the next api key continue else: print "None of the above " + response['statusInfo'] return 0 #Return none when all api keys are exhausted return None
def sentiment_alchemy(url): alchemyapi = AlchemyAPI() response = alchemyapi.sentiment('url', url) response['usage'] = None if response['status'] == 'OK': print('## Response Object ##') print(json.dumps(response, indent=4)) print('') print('## Author ##') print('author: ', response.get('author', '')) print('') else: print('Error in author extraction call: ', response['statusInfo']) response = alchemyapi.keywords('url', url) del (response['usage']) if response['status'] == 'OK': print('## Response Object ##') print(json.dumps(response, indent=4)) print('') print('## Keywords ##') for keyword in response['keywords']: print('text: ', keyword['text'].encode('utf-8')) print('relevance: ', keyword['relevance']) print('sentiment: ', keyword.get('sentiment', {}).get('type', '')) if 'score' in keyword.get('sentiment', {}): print('sentiment score: ' + keyword['sentiment']['score']) print('') else: print('Error in keyword extaction call: ', response.get('statusInfo', ''))
def nlp_process(ids, ids_hash): #instantiate an elasticsearch client es = Elasticsearch() #instantiate an alchemy client alchemyapi = AlchemyAPI() for item in ids: data = ' '.join(ids_hash[item]) lowers = data.lower() alchem_data = [] response = alchemyapi.keywords('text', lowers, {'sentiment': 1}) if response['status'] == 'OK': print('#Success#') for keyword in response['keywords']: al_temp = defaultdict() al_temp['text'] = keyword['text'].encode('utf-8') al_temp['relevance'] = keyword['relevance'] al_temp['sentiment'] = keyword['sentiment']['type'] if 'score' in keyword['sentiment']: al_temp['score'] = keyword['sentiment']['score'] alchem_data.append(al_temp) else: print('Error in keyword extaction call: ', response['statusInfo']) print len(alchem_data) # prepare body for insertion doc = {"business_id": item, "word_freq": alchem_data} exit() template = {"create": {"_index": "alchem", "_type": "doc"}} res = es.index(index="alchem", doc_type='doc', body=doc)
def use_api(self, input): keyword_array=[] alchemyapi = AlchemyAPI() response = alchemyapi.keywords("text", input) for keyword in response["keywords"]: #print keyword["text"].encode('utf=8') keyword_array.append(keyword["text"].encode('utf=8')) return keyword_array
def fetchkeywords(demo_text): alchemyapi = AlchemyAPI() response = alchemyapi.keywords('text',demo_text, { 'sentiment':1 }) listoftopwords = [] if response['status'] == 'OK': for keyword in response['keywords']: listoftopwords.append(keyword['text']) return listoftopwords
def performKeywordExtraction(text): keywordText=[] alchemyapi = AlchemyAPI() response = alchemyapi.keywords("text", text) if response['status'] == 'OK': keywords = response['keywords'] for keyword in keywords: if (float(keyword['relevance'])>0.1): keywordText.append(keyword['text']) return keywordText
def connect_alchemy(url): # to connect with alchemy and tag the content from alchemyapi import AlchemyAPI alchemyapi = AlchemyAPI() resp = alchemyapi.text('url', url) response = alchemyapi.keywords("text", resp['text']) keywors = response["keywords"]
def solve(passage, question, choices): relevant_text = get_relevant_text(passage, question) alchemyapi = AlchemyAPI() passage_sentiment = {} response = alchemyapi.keywords('text', relevant_text, {'sentiment': 1}) if response['status'] == 'OK': for keyword in response['keywords']: if 'sentiment' in keyword.keys() and 'score' in keyword['sentiment']: passage_sentiment[keyword["text"]] = keyword['sentiment'] choices_sentiment = {} for choice in choices: response = alchemyapi.keywords('text', "the" + choices[choice], {'sentiment': 1}) if response['status'] == 'OK': for keyword in response['keywords']: if 'sentiment' in keyword.keys() and 'score' in keyword['sentiment']: choices_sentiment[choice] = keyword['sentiment'] return rank_choices_sentiment(passage_sentiment, choices_sentiment)
def getAlcData(arrOfObj): alchemyapi = AlchemyAPI() #for x in range(0,len(arrOfObj)): for x in range(0, 10): asc = unicodedata.normalize('NFKD', arrOfObj[x].text).encode('ascii','ignore') print x print asc arrOfObj[x].responseEntities = alchemyapi.entities('text',asc, { 'sentiment':1 }) arrOfObj[x].responseKeywords = alchemyapi.keywords('text',asc, { 'sentiment':1 })
class App: def __init__(self): self.alchemyapi = AlchemyAPI() self.raw_text = '' self.concepts = None self.keywords = None def parse_url(self, url=None): text_response = self.alchemyapi.text('url', url) if text_response['status'] == 'OK': self.raw_text = text_response['text'].encode('utf-8') else: print('Error in text extraction call: ', text_response['statusInfo']) def extract_concepts(self): concept_response = self.alchemyapi.concepts('text', self.raw_text) if concept_response['status'] == 'OK': self.concepts = concept_response['concepts'] # print('## Concepts ##') # for concept in self.concepts: # print('text: ', concept['text']) # print('relevance: ', concept['relevance']) # print('') else: print('Error in concept tagging call: ', concept_response['statusInfo']) def extract_keywords(self): keyword_response = self.alchemyapi.keywords('text', self.raw_text, {'sentiment': 1}) if keyword_response['status'] == 'OK': self.keywords = keyword_response['keywords'] # print('') # print('## Keywords ##') # for keyword in self.keywords: # print('text: ', keyword['text'].encode('utf-8')) # print('relevance: ', keyword['relevance']) # print('sentiment: ', keyword['sentiment']['type']) # if 'score' in keyword['sentiment']: # print('sentiment score: ' + keyword['sentiment']['score']) # print('') else: print('Error in keyword extraction call: ', keyword_response['statusInfo']) def define_concepts(self): for concept in self.concepts: definition = duckduckgo.get_zci(concept['text']) print('%s -> %s' % (concept['text'], definition)) print('') def define_keywords(self): for keyword in self.keywords: definition = duckduckgo.get_zci(keyword['text']) print('%s -> %s' % (keyword['text'], definition)) print('')
def checkDailyQuotaAndRunAlchemy(commentDb,cruiseLines): with open('data/Alchemy_response_keywords.json', 'rb') as fp: returned_keywords = json.load(fp) with open('data/Alchemy_response_relations.json', 'rb') as fp: returned_relations = json.load(fp) alchemyapi = AlchemyAPI() test="test if finished Alchemy daily quota" response = alchemyapi.keywords('text', test, {'sentiment': 0}) if response['status'] == 'OK': returned_keywords,returned_relations=runAlchemyApi(cruiseLines,commentDb,returned_keywords,returned_relations,alchemyapi) else: print 'Error in keyword extraction call: ', response['statusInfo'] return returned_keywords, returned_relations
def solve(passage, question, choices): relevant_text = get_relevant_text(passage, question) alchemyapi = AlchemyAPI() passage_sentiment = {} response = alchemyapi.keywords('text', relevant_text, {'sentiment': 1}) if response['status'] == 'OK': for keyword in response['keywords']: if 'sentiment' in keyword.keys( ) and 'score' in keyword['sentiment']: passage_sentiment[keyword["text"]] = keyword['sentiment'] choices_sentiment = {} for choice in choices: response = alchemyapi.keywords('text', "the" + choices[choice], {'sentiment': 1}) if response['status'] == 'OK': for keyword in response['keywords']: if 'sentiment' in keyword.keys( ) and 'score' in keyword['sentiment']: choices_sentiment[choice] = keyword['sentiment'] return rank_choices_sentiment(passage_sentiment, choices_sentiment)
def ExtractKeyword(text): alchemyapi = AlchemyAPI() response = alchemyapi.keywords('text', text, {'sentiment': 1}) results = [] if response['status'] == 'OK': for keyword in response['keywords']: results.append(keyword['text']) # print('text: ', keyword['text'].encode('utf-8')) # print('relevance: ', keyword['relevance']) else: print('Error in keyword extaction call: ', response['statusInfo']) return results
def getSoup(): sock = urllib.urlopen('https://en.wikipedia.org/wiki/Motocross') sockRaw = sock.read() soup = BeautifulSoup(sockRaw, "html.parser") soupText = soup.get_text() # use the alchemyAPI to find the keyword/phrases from the texts alchemyapi = AlchemyAPI() response = alchemyapi.keywords('text', soupText, {'maxRetrieve': 10}) if response['status'] == 'OK': print "\nThe Keywords are:" for i in response['keywords']: print "Word: " + i["text"] + ", Relevance: " + i["relevance"] else: print "Something went wrong with Alchemy."
def checkDailyQuotaAndRunAlchemy(commentDb, cruiseLines): with open('data/Alchemy_response_keywords.json', 'rb') as fp: returned_keywords = json.load(fp) with open('data/Alchemy_response_relations.json', 'rb') as fp: returned_relations = json.load(fp) alchemyapi = AlchemyAPI() test = "test if finished Alchemy daily quota" response = alchemyapi.keywords('text', test, {'sentiment': 0}) if response['status'] == 'OK': returned_keywords, returned_relations = runAlchemyApi( cruiseLines, commentDb, returned_keywords, returned_relations, alchemyapi) else: print 'Error in keyword extraction call: ', response['statusInfo'] return returned_keywords, returned_relations
def extract_keyword_en(self, corpus_text, min_len=4, max_len=50): alchemyapi = AlchemyAPI(ALCHEMY_API_KEY) response = alchemyapi.keywords('text', corpus_text, {'sentiment': 1}) keywords = [] if response['status'] == "OK": for keyword in response['keywords']: phrase = keyword['text'].encode('utf8') freq = self.phrase_frequency(phrase, corpus_text) if freq > 0 and phrase.lower() != self.query.lower() and len(phrase) >= min_len and len(phrase) <= max_len: score = float(keyword['relevance'].encode('utf8')) kw = Keyword(phrase, score, freq) keywords.append(kw) return keywords[:min(len(keywords), self.numberofkeywords)]
def setKeywords(self): alchemyapi = AlchemyAPI() response = alchemyapi.keywords('text',self.content, { 'sentiment':1 }) if response['status'] == 'OK': for keyword in response['keywords']: self.keywords.add(keyword['text'].encode('ascii','ignore')) else: print('Error in concept tagging call: ', response['statusInfo']) self.keywords = set(["Automatic keyword generation failed"]) response = alchemyapi.concepts('text',self.content, { 'sentiment':1 }) if response['status'] == 'OK': for keyword in response['concepts']: self.keywords.add(keyword['text'].encode('ascii','ignore')) else: print('Error in concept tagging call: ', response['statusInfo']) self.keywords = set(["Automatic keyword generation failed"])
def analysecontent(content): """ Process/Analyse the extracted contents with Alchemy API Assumption: api_key.txt with a valid key is available from where this program is getting executed. """ print ("Processing extracted text with AlchemyAPI...") alchemyapi = AlchemyAPI() response = alchemyapi.keywords("text", content, {"maxRetrieve": 10}) if response["status"] == "OK": print ("---------------------------------") print ("## Keywords ## Relevance") for keyword in response["keywords"]: print ("{0}: {1}".format(keyword["text"].encode("utf-8"), keyword["relevance"])) print ("---------------------------------") else: print ("Error in keyword extraction call: ", response["statusInfo"])
def analysecontent(content): """ Process/Analyse the extracted contents with Alchemy API Assumption: api_key.txt with a valid key is available from where this program is getting executed. """ print('Processing extracted text with AlchemyAPI...') alchemyapi = AlchemyAPI() response = alchemyapi.keywords('text', content, {'maxRetrieve': 10}) if response['status'] == 'OK': print('---------------------------------') print('## Keywords ## Relevance') for keyword in response['keywords']: print("{0}: {1}".format(keyword['text'].encode('utf-8'), keyword['relevance'])) print('---------------------------------') else: print('Error in keyword extraction call: ', response['statusInfo'])
class NLP: def __init__(self): self.alchemyapi = AlchemyAPI() def get_categories(self, text): response = self.alchemyapi.taxonomy('text', text) if response['status'] == 'OK' and len(response['taxonomy']) > 0: taxonomy = response['taxonomy'][0] tokens = taxonomy['label'].split('/') return tokens[1] def get_keywords(self, text): response = self.alchemyapi.keywords('text', text) if response['status'] == 'OK' and len(response['keywords']) > 0: return [x['text'] for x in response['keywords']]
def extract_keyword_alchemy(self, corpus): alchemyapi = AlchemyAPI() corpus_text = "\n".join(corpus) response = alchemyapi.keywords('text', corpus_text, {'sentiment': 1}) keywords = [] if response['status'] == "OK": for keyword in response['keywords']: pharse = keyword['text'].encode('utf8') score = float(keyword['relevance'].encode('utf8')) kw = Keyword(pharse, score) keywords.append(kw) sorted_keywords = sorted(keywords, key=lambda t: t.get_score() * -1) return sorted_keywords[:min(len(sorted_keywords), self.numberofkeywords)]
def get_sentiment(text): alchemyapi = AlchemyAPI() alchemyapi.apikey = get_random_alchemy_credentials() response = alchemyapi.keywords('text', text, {'sentiment': 1}) relevances = [] if 'keywords' not in response or len(response['keywords']) == 0: return None for keyword in response["keywords"]: for company_word in concepts: if company_word.lower() in text.lower() and 'sentiment' in keyword: if 'score' in keyword['sentiment']: relevances.append(float(keyword['sentiment']['score'])) elif keyword['sentiment']['type'] == 'neutral': relevances.append(0.5) if not relevances: return 0.5 else: return float("{0:.2f}".format(sum(relevances)/len(relevances)))
def extractKeywordsFromUrl(self, url): """method for extracting keywords from given text""" # creating AlchemyAPI object alchemyapi = AlchemyAPI() # requesting json response from AlchemyAPI server response = alchemyapi.keywords("url", url) if response["status"] == "OK": for keywords in response["keywords"]: # concept object for storing the extracted concept keyword = AlchemyStructure.Keyword() # extracting the keyword keyword.setText(keywords["text"]) # extracting the relevance of keyword keyword.setRelevance(keywords["relevance"]) # instantiating the sentiment object sentimentObj = AlchemyStructure.Sentiment() print(keywords["sentiment"]["type"]) # extracting the score of the sentiment associated with the keyword if keywords["sentiment"]["type"] == "neutral": sentimentObj.setScore("0") else: sentimentObj.setScore(keywords["sentiment"]["score"]) # extracting the type of the sentiment associated with the keyword -> positive, negative or neutral sentimentObj.setType(keywords["sentiment"]["type"]) # set the sentiment for keyword keyword.setSentiment(sentimentObj) # append the concept into the list of retrieved concepts self.keywordsFromUrl.append(keyword) else: print("Error in keyword tagging call: ", response["statusInfo"])
def keywords_and_sentiment(contentList): # Accepts a string of posts (or chats), and returns a list of keywords alchemyapi = AlchemyAPI() relevanceList = [] sentiment = 0 counter = 0 for post_message in contentList: response = alchemyapi.keywords("text", post_message, {"sentiment": 1}) if response["status"] == "OK": for keyword in response["keywords"]: if "score" in keyword["sentiment"]: sentiment += float(keyword["sentiment"]["score"]) if float(keyword["relevance"]) > 0.97: relevanceList.append(keyword["text"]) counter += 1 if DEV_MODE: if not counter is 0: print float(sentiment / counter) return relevanceList
def keywords_and_sentiment(contentList): # Accepts a string of posts (or chats), and returns a list of keywords alchemyapi = AlchemyAPI() relevanceList = [] sentiment = 0 counter = 0 for post_message in contentList: response = alchemyapi.keywords('text', post_message, {'sentiment': 1}) if response['status'] == 'OK': for keyword in response['keywords']: if 'score' in keyword['sentiment']: sentiment += float(keyword['sentiment']['score']) if float(keyword['relevance']) > .97: relevanceList.append(keyword['text']) counter += 1 if DEV_MODE: if not counter is 0: print float(sentiment / counter) return relevanceList
def run_app(): ##### part 1 page = requests.get('https://en.wikipedia.org/wiki/Michael_Jordan') tree = html.fromstring(page.content) h3s = tree.xpath('//h3/span/text()') h3s_combined = '' for next_h3 in h3s: h3s_combined += ' ' h3s_combined += next_h3 print('***********************************************') print('*** The h3s on this page were: ', h3s_combined) print('***********************************************') ##### part 2 #alchemy_api_key="xxxx" alchemyapi = AlchemyAPI() response = alchemyapi.keywords('text', h3s_combined, { 'sentiment': 1, 'maxRetrieve': 10 }) if response['status'] == 'OK': print('## Response Object ##') # print(json.dumps(response, indent=4)) print('') print('## Keywords ##') for keyword in response['keywords']: print('text: ', keyword['text'].encode('utf-8')) print('relevance: ', keyword['relevance']) print('sentiment: ', keyword['sentiment']['type']) print('') else: print('Error in keyword extaction call: ', response['statusInfo'])
def alchemyExtract(cleanText,options): """Uses alchemyAPI to find keywords in the cleaned text. In this case,options[1] should specify the number of keywords to be used. Default to 4.""" try: options[1]=int(options[1]) except IndexError as e: options[1]=4 except ValueError as e: print("You selected alchemy, and options[1] was not a valid integer.") raise e from alchemyapi import AlchemyAPI alch = AlchemyAPI() response = alch.keywords('text',cleanText,{'sentiment':1}) rKeywords = response['keywords'] finalKeywords = [] if options[1]<len(rKeywords): for i in range(options[1]): finalKeywords.append(rKeywords[i]['text']) else: finalKeywords = rKeywords return "\n\nKeywords: " + ", ".join(finalKeywords) #figure out formatting for this later.
def nlp_process(ids,ids_hash): #instantiate an elasticsearch client es = Elasticsearch() #instantiate an alchemy client alchemyapi = AlchemyAPI() for item in ids: data = ' '.join(ids_hash[item]) lowers = data.lower() alchem_data = [] response = alchemyapi.keywords('text', lowers, {'sentiment': 1}) if response['status'] == 'OK': print('#Success#') for keyword in response['keywords']: al_temp = defaultdict() al_temp['text'] = keyword['text'].encode('utf-8') al_temp['relevance'] = keyword['relevance'] al_temp['sentiment'] = keyword['sentiment']['type'] if 'score' in keyword['sentiment']: al_temp['score'] = keyword['sentiment']['score'] alchem_data.append(al_temp) else: print('Error in keyword extaction call: ', response['statusInfo']) print len(alchem_data) # prepare body for insertion doc = { "business_id" : item, "word_freq": alchem_data } exit() template = { "create": { "_index": "alchem", "_type": "doc"} } res = es.index(index="alchem", doc_type='doc', body=doc)
def alchemyExtract(cleanText, options): """Uses alchemyAPI to find keywords in the cleaned text. In this case,options[1] should specify the number of keywords to be used. Default to 4.""" try: options[1] = int(options[1]) except IndexError as e: options[1] = 4 except ValueError as e: print("You selected alchemy, and options[1] was not a valid integer.") raise e from alchemyapi import AlchemyAPI alch = AlchemyAPI() response = alch.keywords('text', cleanText, {'sentiment': 1}) rKeywords = response['keywords'] finalKeywords = [] if options[1] < len(rKeywords): for i in range(options[1]): finalKeywords.append(rKeywords[i]['text']) else: finalKeywords = rKeywords return "\n\nKeywords: " + ", ".join( finalKeywords) #figure out formatting for this later.
def run_app(): ##### part 1 page = requests.get('https://en.wikipedia.org/wiki/Michael_Jordan') tree = html.fromstring(page.content) h3s = tree.xpath('//h3/span/text()') h3s_combined = '' for next_h3 in h3s: h3s_combined += ' ' h3s_combined += next_h3 print('***********************************************') print('*** The h3s on this page were: ', h3s_combined) print('***********************************************') ##### part 2 #alchemy_api_key="xxxx" alchemyapi = AlchemyAPI() response = alchemyapi.keywords('text', h3s_combined, {'sentiment': 1, 'maxRetrieve': 10}) if response['status'] == 'OK': print('## Response Object ##') # print(json.dumps(response, indent=4)) print('') print('## Keywords ##') for keyword in response['keywords']: print('text: ', keyword['text'].encode('utf-8')) print('relevance: ', keyword['relevance']) print('sentiment: ', keyword['sentiment']['type']) print('') else: print('Error in keyword extaction call: ', response['statusInfo'])
def post(self): sid = self.get_argument('sid') abstract = self.get_argument('abstract') print abstract url = "https://api-us.clusterpoint.com/100629/stud/_search.json" headers = {"Authorization":"Basic bGVvcGFuaWdyYWhpQGdtYWlsLmNvbToyM2xlbzIz"} string = "<project><abstract>"+ abstract +"</abstract></project>" values = dict(query=string) r = requests.post(url,data=json.dumps(values), headers=headers) responseDict = json.loads(r.content) for each in responseDict['documents'][0]['project']: if each['abstract'] == abstract: mytext = each['abstract'] alchemyapi = AlchemyAPI() response = alchemyapi.keywords('text', abstract, {'sentiment': 1}) finalKeys = list() for each in response['keywords']: try: if float(each['relevance'])>0.5: finalKeys.append(each['text']) print finalKeys self.render("tags.html", finalKeys=finalKeys) except: pass
print('Checking entities . . . ') response = alchemyapi.entities('text', test_text); assert(response['status'] == 'OK') response = alchemyapi.entities('html', test_html); assert(response['status'] == 'OK') response = alchemyapi.entities('url', test_url); assert(response['status'] == 'OK') response = alchemyapi.entities('random', test_url); assert(response['status'] == 'ERROR') #invalid flavor print('Entity tests complete!') print('') #Keywords print('Checking keywords . . . ') response = alchemyapi.keywords('text', test_text); assert(response['status'] == 'OK') response = alchemyapi.keywords('html', test_html); assert(response['status'] == 'OK') response = alchemyapi.keywords('url', test_url); assert(response['status'] == 'OK') response = alchemyapi.keywords('random', test_url); assert(response['status'] == 'ERROR') #invalid flavor print('Keyword tests complete!') print('') #Concepts print('Checking concepts . . . ')
import os, sys, json from alchemyapi import AlchemyAPI alchemyapi = AlchemyAPI() fname = sys.argv[1] with open(fname, 'r') as f: Text = f.readlines() text_file = open(fname, 'w') for fig in range(len(Text)): response = alchemyapi.keywords("text", Text[fig]) if response['status'] == 'OK': for keyword in response['keywords']: text_file.write( str(fig) + '\t' + str(keyword['text'].encode('utf-8')) + '\t' + str(keyword['relevance']) + '\n') else: print('Error in keyword extaction call: ', response['statusInfo']) text_file.close()
else: print('Error in sentiment analysis call: ', response['statusInfo']) print('') print('') print('') print('############################################') print('# Keyword Extraction Example #') print('############################################') print('') print('') print('Processing text: ', demo_text) print('') response = alchemyapi.keywords('text', demo_text, {'sentiment': 1}) if response['status'] == 'OK': print('## Response Object ##') print(json.dumps(response, indent=4)) print('') print('## Keywords ##') for keyword in response['keywords']: print('text: ', keyword['text']) print('relevance: ', keyword['relevance']) print( 'sentiment: ', keyword['sentiment']['type'] + ' (' + keyword['sentiment']['score'] + ')') print('') else:
ent_rele=[] ent_type=[] if response['status'] == 'OK': flag=1 for entity in response['entities']: ent.append(entity['text']) ent_rele.append(entity['relevance']) ent_type.append(entity['type']) else: print('Error in entity extraction call: ', response['statusInfo']) if flag==1: #response = json.loads(json.dumps(alchemyapi.sentiment("text", trans_text))) ###### GETTING AN ERROR HERE FOR SOME REASON ###### #senti=response["docSentiment"]["type"] response = json.loads(json.dumps(alchemyapi.keywords('text', trans_text, {'sentiment': 1}))) #size=len(response['keywords']) keywords=[] if response['status'] == 'OK': for word in response['keywords']: keywords.append(word['text']) else: print('Error in entity extraction call: ', response['statusInfo']) response=json.loads(json.dumps(alchemyapi.concepts("text",trans_text))) #size=len(response['concepts']) concept=[] if response['status'] == 'OK': for con in response['concepts']:
mixed = None alchemy_response = alchemyapi.sentiment_targeted( 'url', response['items'][i]['link'], topic) if alchemy_response['status'] == 'OK': sentiment = alchemy_response['docSentiment']['type'] if 'score' in alchemy_response['docSentiment']: score = alchemy_response['docSentiment']['score'] if 'mixed' in alchemy_response['docSentiment']: mixed = alchemy_response['docSentiment']['mixed'] else: print j + i + 1, 'Error in targeted sentiment analysis call: ', alchemy_response[ 'statusInfo'] # Keyword sentiment analysis keyword_response = alchemyapi.keywords('url', response['items'][i]['link'], {'sentiment': 1}) key_sentiment = None key_score = None key_mixed = None if keyword_response['status'] == 'OK': if 'keywords' in keyword_response: for keyword in keyword_response['keywords']: if keyword['text'].lower() == topic.lower(): if 'sentiment' in keyword: key_sentiment = keyword['sentiment']['type'] if 'score' in keyword['sentiment']: key_score = keyword['sentiment']['score'] if 'mixed' in keyword['sentiment']: key_mixed = keyword['sentiment']['mixed'] else:
print('relevance : ', concept['relevance']) print('') concept_temp.append(concept['text']) con_rel_temp.append(concept['relevance']) """ concept_list[title_num][conc_num] = concept['text'] relevance_list[title_num][conc_num]=concept['relevance'] conc_num += 1 """ else: print('Error in concept tagging call: ', response['statusInfo']) print('다시 시도 하거나 다른 주소를 입력하세요.') response2 = alchemyapi.keywords('url', input_url, options={'maxRetrieve': input_maxC}) if response2['status'] == 'OK': print('## Keywords ##') keyword_temp = [] key_rel_temp = [] #rel_temp=[] for keyword in response2['keywords']: print('text: ', keyword['text']) print('relevance: ', keyword['relevance']) print('') keyword_temp.append(keyword['text']) key_rel_temp.append(keyword['relevance']) else: print('Error in keywords call: ', response2['statusInfo']) print('다시 시도 하거나 다른 주소를 입력하세요.')
import bs4 from bs4 import BeautifulSoup import urllib2 url= "http://businessoverbroadway.com/top-10-skills-in-data-science" page = urllib2.urlopen(url) soup = bs4.BeautifulSoup(page.read(), 'html.parser') mytext = soup.find('p').getText() print mytext #Take this text and store it in the program to use in the next step. # Part 2 from alchemyapi import AlchemyAPI alchemyapi = AlchemyAPI() response = alchemyapi.keywords('text', mytext) print "Keywords" '\t' '\t' "Relevance" for kyw in response['keywords'][1:10]: k1= kyw['text'] r1= kyw['relevance'] print " " print "--------------------------------------" print k1, '\t', '\t', r1
con_rel_temp = [] key_rel_temp = [] for concept in response['concepts']: print('text: ', concept['text']) print('relevance : ', concept['relevance']) print('') concept_temp.append(concept['text']) con_rel_temp.append(concept['relevance']) """ concept_list[title_num][conc_num] = concept['text'] relevance_list[title_num][conc_num]=concept['relevance'] conc_num += 1 """ response = alchemyapi.keywords('text', input_text, options={'maxRetrieve': input_maxC}) print('## Keywords ##') rel_temp = [] for keyword in response['keywords']: print('text: ', keyword['text']) print('relevance: ', keyword['relevance']) print('') keyword_temp.append(keyword['text']) key_rel_temp.append(keyword['relevance']) else: print('Error in concept tagging call: ', response['statusInfo']) if 'y' == input('이 결과를 저장하시겠습니까?(y/n)'): title_input = input('제목을 입력하세요 : ') concept_list.append(concept_temp)
# For this assignment, I will read in the text of a magazine article and pass it through the Alchemy API # to rank the keywords. First, we must read the text in from the html with the help of beautifulsoup. import urllib import bs4 from alchemyapi import AlchemyAPI url = "http://www.theatlantic.com/magazine/archive/2015/09/the-coddling-of-the-american-mind/399356/" f = urllib.urlopen(url) website = f.read() soup = bs4.BeautifulSoup(website, 'html.parser') story = soup.find('div', {'class': 'article-body'}).getText() story # Now that we have the text of the article, we will pass it to the Alchemy API and print out the top 10 # ranked words and their respective relevance. alchemyapi = AlchemyAPI() keywords = alchemyapi.keywords("text", story, {'maxRetrieve': 10}) i = 0 for keyword in keywords[u'keywords']: i += 1 print 'Rank:', i, '\t Relevance:', "\t %s \t %s" % (keyword['relevance'], keyword['text'])
def trigger(self, text): print('Request: {0}'.format(text)) try: alchemyapi = AlchemyAPI() except Exception as e: print('Could not connect to AlchemyAPI. Details: {}'.format(e)) relations = alchemyapi.relations('text', text) action = None # Action to take. Only 'Play' implemented so far. format = None # In pratice 'album' or 'track'. toPlay = None # Album or track name. if relations['status'] == 'OK': print('## Object ##') print(json.dumps(relations, indent=4)) for relation in relations['relations']: if 'action' in relation: print('Action: ', relation['action']['text'].encode('utf-8')) if relation['action']['verb']['text'] in ('play', 'listen'): action = 'PLAY' else: print('Error in relation extaction call: ', relations['statusInfo']) # If no action found, we abort. if not action: print('Could not find any action to take.') return # Detect the artist to play artists = [] artist = None keywords = alchemyapi.keywords('text', text, {'sentiment': 1}) if keywords['status'] == 'OK': print('## Response Object ##') print(json.dumps(keywords, indent=4)) for keyword in keywords['keywords']: artists.append((keyword['relevance'], extractArtists(keyword['text'].encode('utf-8')))) else: print('Error in keyword extaction call: ', keywords['statusInfo']) print('Action: {0}'.format(action)) print('Artists extracted from the request: {0}'.format(' '.join([str(n)+'('+c+')' for c,n in artists]))) for e in artists: if not e[1]: continue else: artist = e[1][-1] break if not artist: print('Could not find any artist.') return print('Selected artist: {0}'.format(artist)) print('To simplify I assume you are looking for an album and not a track.') albums = AudioDB_getAlbums(artist) if not albums: print('Could not find any album for {0}'.format(artist)) return album = None for name, item in albums.iteritems(): if name in text: album = item break if not album: print('Could not find any specific album, so will try to extract a track from the full discography') for _, album in albums.iteritems(): tracks = AudioDB_getTracks(album['id']) for track in tracks: if track in text: format = 'song' toPlay = track break if not toPlay: format = 'album' toPlay = albums.itervalues().next()['name'] # We assume the artist has at least one album print('Could not find any album or track so I will play the album {0}'.format(toPlay)) else: print('Selected album is {0}. Now, check the tracks.'.format(album)) tracks = AudioDB_getTracks(album['id']) for track in tracks: if track in text: format = 'song' toPlay = track break if not toPlay: print("Could not find a track, so I will play the whole album.") toPlay = album['name'] format = 'album' else: print('The song to play will be {0}'.format(toPlay)) print('Selected album/track: {0}'.format(toPlay)) hint = 'album' if format == 'album' else '' args = {'q': ' '.join([toPlay, artist, hint]), 'max_results': 1} print('Youtube query: {0}'.format(args['q'])) try: video = get_video(args) if video: video_ID = video[0] print('Youtube Video ID: {0}'.format(video_ID)) uri = 'http://www.youtube.com/watch?v={0}'.format(video_ID) res = self.call('music.fromyoutube', uri) else: print("Could not find any stream.") except HttpError, e: print("An HTTP error %d occurred:\n%s" % (e.resp.status, e.content))
class AlchemyPost: def __init__(self, post_tumblr, post_id, consumer_key, consumer_secret, oauth_token, oauth_secret): self.post_tumblr = post_tumblr self.post_id = post_id self._init_tumblr(consumer_key, consumer_secret, oauth_token, oauth_secret) self._init_alchemy() def _init_tumblr(self, consumer_key, consumer_secret, oauth_token, oauth_secret): self._client = pytumblr.TumblrRestClient(consumer_key, consumer_secret, oauth_token, oauth_secret) def _init_alchemy(self): self.alchemyapi = AlchemyAPI() self.content = {} def analyze_post(self): self.post = self._get_content_post() self._alchemy_entities() self._alchemy_keywords() self._alchemy_concepts() self._alchemy_sentiment() self._alchemy_relations() self._alchemy_category() self._alchemy_feeds() self._alchemy_taxonomy() def print_content(self): print(json.dumps(self.content, indent=4)) def _get_content_post(self): print "*", infos = self._get_infos_post() self.title = '' self.tags = [] if 'tags' in infos: self.tags = infos['tags'] if infos['type'] == 'text': return self._get_content_text(infos) if infos['type'] == 'quote': return self._get_content_quote(infos) return '' def _get_infos_post(self): infos = self._client.posts(self.post_tumblr, id=self.post_id) if 'posts' in infos and len(infos['posts'])>0: return infos['posts'][0] return {} def _get_content_text(self, infos): content = "<h1>" + str(infos['title']) + "</h1>" content += " <br>" + str(infos['body']) content += " <br>" + " ".join(infos['tags']) return content def _get_content_quote(self, infos): content = str(infos['text']) content += " <br>" + str(infos['source']) content += " <br>" + " ".join(infos['tags']) return content def _alchemy_entities(self): print ".", response = self.alchemyapi.entities('html', self.post) if response['status'] != 'OK': return False self.content['entities'] = response['entities'] return True def _alchemy_keywords(self): print ".", response = self.alchemyapi.keywords('html', self.post) if response['status'] != 'OK': return False self.content['keywords'] = response['keywords'] return True def _alchemy_concepts(self): print ".", response = self.alchemyapi.concepts('html', self.post) if response['status'] != 'OK': return False self.content['concepts'] = response['concepts'] return True def _alchemy_sentiment(self): print ".", response = self.alchemyapi.sentiment('html', self.post) if response['status'] != 'OK': return False self.content['sentiment'] = response['docSentiment'] return True def _alchemy_relations(self): print ".", response = self.alchemyapi.relations('html', self.post) if response['status'] != 'OK': return False self.content['relations'] = response['relations'] return True def _alchemy_category(self): print ".", response = self.alchemyapi.category('html', self.post) if response['status'] != 'OK': return False self.content['category'] = response['category'] self.content['score'] = response['score'] return True def _alchemy_feeds(self): print ".", response = self.alchemyapi.feeds('html', self.post) if response['status'] != 'OK': return False self.content['feeds'] = response['feeds'] return True def _alchemy_taxonomy(self): print ".", response = self.alchemyapi.taxonomy('html', self.post) if response['status'] != 'OK': return False self.content['taxonomy'] = response['taxonomy'] return True
#Entities print('Checking entities . . . ') response = alchemyapi.entities('text', test_text) assert (response['status'] == 'OK') response = alchemyapi.entities('html', test_html) assert (response['status'] == 'OK') response = alchemyapi.entities('url', test_url) assert (response['status'] == 'OK') response = alchemyapi.entities('random', test_url) assert (response['status'] == 'ERROR') #invalid flavor print('Entity tests complete!') print('') #Keywords print('Checking keywords . . . ') response = alchemyapi.keywords('text', test_text) assert (response['status'] == 'OK') response = alchemyapi.keywords('html', test_html) assert (response['status'] == 'OK') response = alchemyapi.keywords('url', test_url) assert (response['status'] == 'OK') response = alchemyapi.keywords('random', test_url) assert (response['status'] == 'ERROR') #invalid flavor print('Keyword tests complete!') print('') #Concepts print('Checking concepts . . . ') response = alchemyapi.concepts('text', test_text) assert (response['status'] == 'OK') response = alchemyapi.concepts('html', test_html)
def nlp_process(ids,ids_hash, tgt_file_name): # instantiate an elasticsearch client #es = Elasticsearch() outfile = open(tgt_file_name, 'w') # instantiate an alchemy client alchemyapi = AlchemyAPI() # loop through each business for business_id in ids: # loop through reviews for each star for star in range(1,6): # combine review text from all reviews rated same score print "processing business {} for review with {} star(s)".format(business_id, star) data = '.'.join(ids_hash[(business_id, star)]) alchem_keywords = [] alchem_concepts = [] # perform analysis only if combined review length across all the reviews # within a star is more than 100 characters if len(data) >= 100: # call Alchemy API combined call to determing keywords, sentiment # and concepts (tags) response = alchemyapi.keywords('text', data, {'sentiment': 1, 'maxRetrieve': 100}) # process response if response['status'] == 'OK': print('#Success#') # process keywords for keyword in response['keywords']: al_temp = defaultdict() al_temp['text'] = keyword['text'].encode('utf-8') al_temp['relevance'] = keyword['relevance'] if 'sentiment' in keyword: al_temp['sentiment'] = keyword['sentiment'].get('type', 'neutral') if 'score' in keyword['sentiment']: al_temp['score'] = keyword['sentiment']['score'] alchem_keywords.append(al_temp) # process concepts/tags #for keyword in response['concepts']: # al_temp = defaultdict() # al_temp['text'] = keyword['text'].encode('utf-8') # al_temp['relevance'] = keyword['relevance'] # alchem_concepts.append(al_temp) else: print('Error in keyword extaction call: ', response['statusInfo']) print len(alchem_keywords), len(alchem_concepts) # prepare body for insertion if ( len(alchem_keywords) > 0): doc = { "business_id" : business_id, "stars": star, "word_freq": alchem_keywords, "topics": alchem_concepts } # write to a file json.dump(doc, outfile) outfile.write('\n')
print('') print('') print('') print('############################################') print('# Keyword Extraction Example #') print('############################################') print('') print('') print('Processing text: ', demo_text) print('') response = alchemyapi.keywords('text',demo_text, { 'sentiment':1 }) if response['status'] == 'OK': print('## Response Object ##') print(json.dumps(response, indent=4)) print('') print('## Keywords ##') for keyword in response['keywords']: print('text: ', keyword['text'].encode('utf-8')) print('relevance: ', keyword['relevance']) print('sentiment: ', keyword['sentiment']['type']) if 'score' in keyword['sentiment']: print('sentiment score: ' + keyword['sentiment']['score']) print('')
for p in paragraphs: f = re.sub(r"\[.*\]", "", p.text.encode("utf-8")) # regex # text_extracted = text_extracted + p.text.encode("utf-8") text_extracted = text_extracted + f web_text = text_extracted print '\n\n' print "URL Source: " + url print web_text print '\n\n' print '#### Top 10 Keywords ####'.center(80, " ") response = alchemyapi.keywords( 'text', web_text, {'sentiment': 1}) # From example.py of alchemyapi module # Processing the XML if response['status'] == 'OK': print '\n' print "Keyword".center(50, " ") + "Relevance".center( 10, " ") + "Sentiment".center(10, " ") print "=" * 80 # underline i = 0 for keyword in response[ 'keywords']: # loop to specify top ten based on relevance i += 1 if i > 10: break if 'score' in keyword['sentiment']:
# -*- coding: utf-8 -*- from alchemyapi import AlchemyAPI import json if __name__ == "__main__": text = u'Artist tackles Chicago\'s pesky pothole problem _ by filling annoying craters with mosaics\r\n\r\nCHICAGO \u2013 The perfect pothole might not exist for many people \u2014 but for mosaic artist Jim Bachor, it\'s one with a nice oval shape. Bachor began filling those potholes a little more than a year ago, after one in front of his house became a hassle.\r\n\r\nBachor doesn\'t just fill them with cement, though. He\'s turned pothole-filling into a public art project \u2014 one with a sense of humor. He fills them with mosaics.\r\n\r\n"I just think it\'s fun to add that little bit of spark into (an) issue that people moan about," says the Chicago resident, whose work also hangs in galleries. He was first drawn to the ancient art form because of its ability to last.\r\n\r\nWith orange cones and vests displaying his last name, Bachor and his helpers look official enough to shut down a street section to work on filling a pothole.\r\n\r\nBachor uses the Chicago city flag design in his pothole art. Some versions hold phone numbers to local auto repair shops, while others simply read "POTHOLE." His most recent installment north of downtown Chicago \u2014 "#21914" \u2014 pokes fun at the huge number of potholes that exist in the city.\r\n\r\nWhile his mosaic art isn\'t a permanent solution to the city\'s pothole problem, it\'s at least a small fix, he says. The city hasn\'t shut down his project, and some community members have expressed gratitude.\r\n\r\nAfter his first project, one neighbor stopped to thank him. "And then 15 minutes later, he came back with a coffee and a Danish for me," Bachor says, "and so I thought that was really cool."\r\n\r\nGerry Shaheen, a resident of Peoria, Illinois, recently stopped to ask Bachor about his work, as the artist installed a mosaic. He says Bachor and his crew are welcome anytime to fill potholes in his city, one of many hit with an especially large number of the annoying craters after a hard winter.\r\n\r\n"I\'ll pave the way for them," Shaheen said with smirk. "No pun intended."' #text = u'''When three Chicago financial trading firms traveled to Washington, D.C., for a crucial meeting with federal regulators, they didn\'t go alone. They brought a man schooled in the ways of the capital who had recently received $182,000 in political contributions from them: Mayor Rahm Emanuel. At stake was a proposal that would have cut into the firms\' bottom lines by making them hold large cash reserves to protect against volatility in the fledgling, high-frequency trading market they had helped pioneer. Public officials of Emanuel\'s stature rarely show up at the arcane rule-making meetings of the Commodity Futures Trading Commission. But the newly elected mayor delivered a message in the windowless conference room that day, about the important role the trading firms and others like them play in Chicago\'s economy. When regulators drafted the final rules months after the September 2011 meeting, Chicago\'s trading firms got the break they wanted. Since the Washington meeting, the three firms — DRW Trading Group, Chicago Trading Co. and Infinium Capital — have donated an additional $187,800 to Emanuel\'s campaign funds, bringing their total support of the mayor to nearly $370,000. Two other Chicago firms that stood to benefit from the rule, PEAK6 and Chopper Trading, have given an additional $334,000 to Emanuel. All the trading firms but Infinium are among an elite corps of roughly 100 donors Emanuel turned to when he first ran for mayor and is relying on again for his re-election effort. Those donors, consisting of individuals, couples, business partners and firms, are responsible for more than $14 million of the $30.5 million he has raised since fall 2010. More than half of those loyal donors have received a tangible benefit from the mayor or his administration, ranging from city contracts to help with regulators, according to a Tribune analysis. In a monthslong examination of Emanuel\'s fundraising machine, the Tribune documented his connections to investment house executives, corporate CEOs and Washington insiders who sustain him with cash even though their interests lie far beyond the policy decisions issued from the mayor\'s fifth-floor office at City Hall. Emanuel declined to be interviewed for this story. His spokeswoman said his trip to Washington in September 2011 was focused on helping the city but declined to discuss what he did on behalf of the key donors. "The mayor had a series of meetings with federal officials in Washington, as he routinely does, to advocate for Chicago priorities — including federal funding for Chicago — and regarding matters critical to our city\'s interest,\" Kelley Quinn, Emanuel\'s communications director, said in a statement. \"In these meetings he discussed federal funding to invest in housing for low-income Chicagoans, funding to invest in Chicago parkland and open space, a potential national park designation for the historic Pullman neighborhood (on) the South Side, and issues impacting Chicago\'s trading industry that employs more than 33,000 people in the city." Those Washington meetings — among more than 100 he has had during more than 30 trips as mayor — are the result of years of cultivating relationships with fellow national leaders. Emanuel\'s resume is unmatched by any mayor in the country: senior adviser to then-President Bill Clinton, top-ranking Democratic congressman from Chicago and first White House chief of staff to President Barack Obama. Add to that years as a top strategist and fundraiser for the national Democratic Party and a two-year stint as an investment banker.''' text = text.encode('ascii', 'ignore') alchemyapi = AlchemyAPI() response = alchemyapi.keywords('text', text, { 'sentiment': 1, 'maxRetrieve': 20, 'keywordExtractMode': 'strict' }) if response['status'] == "OK": print(json.dumps(response, indent=4)) for keyword in response['keywords']: print('Keyword text:', keyword['text'].encode('utf-8')) print('Keyword relevance:', keyword['relevance']) print("**********") response = alchemyapi.concepts('text', text, {'maxRetrieve': 10}) if response['status'] == "OK": #print(json.dumps(response, indent = 4)) keywords = [(concept['text'].encode('ascii'), float(concept['relevance']))
url = "http://quora-api.herokuapp.com/users/" + sys.argv[1] + "/activity" data = requests.get(url).json() data = data['activity'] payload = {} #count=0 #getDocCount() for activities in data: title = activities['title'] summary = activities['summary'] print title document['title'] = title document['summary'] = summary labels = al.taxonomy("text", title) entities = al.entities("html", summary) keywords = al.keywords("html", summary) sentiment = al.sentiment("html", summary) #print labels['taxonomy'] #count+=1 payload['entities'] = {} payload['keywords'] = [] payload['sentiment'] = {} docNode = createDocNode(document) try: print "Yo" labels = labels['taxonomy'][0]['label'] print "Yo1" print labels labels = func(labels) print labels entities = entities['entities']
def extract_keywords(): out_file = open(sys.argv[2], "w") dir_name = sys.argv[1] loc_hash = {} alchemyapi = AlchemyAPI() for file_in in glob.glob(dir_name + "/*.txt"): word_count = {} f_obj = open(file_in) loc = file_in.split("/")[1].split(".")[0] loc_hash[loc] = {} text = f_obj.read() text = text.replace("*******************", "") doc_len = len(text) print(str(doc_len)) text_words = text.split() unigrams_freq = nltk.FreqDist(text_words) bigrams = bgs = nltk.bigrams(text_words) bigrams_freq = nltk.FreqDist(bigrams) loc_hash[loc]["len"] = doc_len print(" calling api for " + file_in + ".......") response = alchemyapi.keywords('text', text, {'sentiment': 0}) if response['status'] == 'OK': loc_hash[loc]["keywords"] = {} for keyword in response['keywords']: words = keyword['text'].split() if len(words) < 2: words = keyword['text'].split("/") if (len(words) <= 2): loc_hash[loc]["keywords"][keyword['text']] = 0 if len(words) > 1: for i in range(len(words)): new_word = words[i] if new_word not in loc_hash[loc]["keywords"]: loc_hash[loc]["keywords"][new_word] = 0 if (i != len(words) - 1): new_word = words[i] + " " + words[i + 1] if new_word not in loc_hash[loc]["keywords"]: loc_hash[loc]["keywords"][new_word] = 0 else: print('Error in keyword extaction call for file :' + file_in + " : " + response['statusInfo']) myRE = re.compile('^[a-zA-Z]+$') for keyword in loc_hash[loc]["keywords"]: words = keyword.split() if (len(words) > 1): if (re.match(myRE, words[0])): if (re.match(myRE, words[1])): t = (words[0], words[1]) freq = bigrams_freq[t] loc_hash[loc]["keywords"][keyword] = freq else: if (re.match(myRE, words[0])): freq = unigrams_freq[words[0]] loc_hash[loc]["keywords"][keyword] = freq json.dump(loc_hash, out_file)
for item in json_data[business]["1"]: business_ones = business_ones + " " + item if "2" in json_data[business]: for item in json_data[business]["2"]: business_twos = business_twos + " " + item if "3" in json_data[business]: for item in json_data[business]["3"]: business_threes = business_threes + " " + item if "4" in json_data[business]: for item in json_data[business]["4"]: business_fours = business_fours + " " + item if "5" in json_data[business]: for item in json_data[business]["5"]: business_fives = business_fives + " " + item business_response_ones = alchemyapi.keywords("text", business_ones, {'sentiment': 1}) business_response_twos = alchemyapi.keywords("text", business_twos, {'sentiment': 1}) business_response_threes = alchemyapi.keywords("text", business_threes, {'sentiment': 1}) business_response_fours = alchemyapi.keywords("text", business_fours, {'sentiment': 1}) business_response_fives = alchemyapi.keywords("text", business_fives, {'sentiment': 1}) data = {} data['1'] = business_response_ones data['2'] = business_response_twos data['3'] = business_response_threes data['4'] = business_response_fours data['5'] = business_response_fives