def extractConceptFromUrl(self,url): """method for extracting concepts from given url""" #creating AlchemyAPI object alchemyapi = AlchemyAPI() #requesting json response from AlchemyAPI server response = alchemyapi.concepts('url',url) if response['status'] == 'OK': for concept in response['concepts']: #concept object for storing the extracted concept concept = AlchemyStructure.Concept() #extracting the concept name concept.setText(concept['text']) #extracting the relevance of the concept concept.setRelevance(concept['relevance']) #append the concept into the list of retrieved concepts self.conceptsFromText.append(concept) else: print('Error in concept tagging call: ', response['statusInfo'])
def main(): alchemyapi = AlchemyAPI() try: filename = sys.argv[1] except IndexError: print "Give a filename as the second argument!" sys.exit(1) text = pdf_to_str(filename) if len(text) >= LENGTH_LIMIT: print "PDF content is longer ({} characters) than the maximum \ of {}, skipping remainder".format(len(text), LENGTH_LIMIT) text = text[:LENGTH_LIMIT] print "KEYWORDS" response = alchemyapi.keywords('text', text) for keyword in response['keywords']: print ' - {}'.format(keyword['text']) print print "CONCEPTS" response = alchemyapi.concepts('text', text) for concept in response['concepts']: print ' - {}'.format(concept['text'])
def extractConceptFromUrl(self, url): """method for extracting concepts from given url""" # creating AlchemyAPI object alchemyapi = AlchemyAPI() # requesting json response from AlchemyAPI server response = alchemyapi.concepts("url", url) if response["status"] == "OK": for concept in response["concepts"]: # concept object for storing the extracted concept conceptObj = AlchemyStructure.Concept() # extracting the concept name conceptObj.setText(concept["text"]) # extracting the relevance of the concept conceptObj.setRelevance(concept["relevance"]) # append the concept into the list of retrieved concepts self.conceptsFromUrl.append(conceptObj) else: print("Error in concept tagging call: ", response["statusInfo"])
def store_concepts(tweets): # Convert string array to string all_tweets_as_string = ' '.join(tweets) alchemyapi = AlchemyAPI() alchemyapi.apikey = get_random_alchemy_credentials() response = alchemyapi.concepts('text', all_tweets_as_string) if response['status'] == 'OK': for concept in response['concepts']: concepts.append(concept['text'])
def performCT(url): conceptText=[] alchemyapi = AlchemyAPI() response = alchemyapi.concepts('url', url) if response['status'] == 'OK': concepts = response['concepts'] for concept in concepts: if (float(concept['relevance'])>0.1): conceptText.append(concept['text']) return conceptText
class App: def __init__(self): self.alchemyapi = AlchemyAPI() self.raw_text = '' self.concepts = None self.keywords = None def parse_url(self, url=None): text_response = self.alchemyapi.text('url', url) if text_response['status'] == 'OK': self.raw_text = text_response['text'].encode('utf-8') else: print('Error in text extraction call: ', text_response['statusInfo']) def extract_concepts(self): concept_response = self.alchemyapi.concepts('text', self.raw_text) if concept_response['status'] == 'OK': self.concepts = concept_response['concepts'] # print('## Concepts ##') # for concept in self.concepts: # print('text: ', concept['text']) # print('relevance: ', concept['relevance']) # print('') else: print('Error in concept tagging call: ', concept_response['statusInfo']) def extract_keywords(self): keyword_response = self.alchemyapi.keywords('text', self.raw_text, {'sentiment': 1}) if keyword_response['status'] == 'OK': self.keywords = keyword_response['keywords'] # print('') # print('## Keywords ##') # for keyword in self.keywords: # print('text: ', keyword['text'].encode('utf-8')) # print('relevance: ', keyword['relevance']) # print('sentiment: ', keyword['sentiment']['type']) # if 'score' in keyword['sentiment']: # print('sentiment score: ' + keyword['sentiment']['score']) # print('') else: print('Error in keyword extraction call: ', keyword_response['statusInfo']) def define_concepts(self): for concept in self.concepts: definition = duckduckgo.get_zci(concept['text']) print('%s -> %s' % (concept['text'], definition)) print('') def define_keywords(self): for keyword in self.keywords: definition = duckduckgo.get_zci(keyword['text']) print('%s -> %s' % (keyword['text'], definition)) print('')
def findConcept(text): # Create the AlchemyAPI Object alchemyapi = AlchemyAPI() #print('############################################') #print('# Concept Tagging retrieval #') #print('############################################') #print('') print('') print('Processing text: ', text) print('') response = alchemyapi.concepts('text', text) return response
def setKeywords(self): alchemyapi = AlchemyAPI() response = alchemyapi.keywords('text',self.content, { 'sentiment':1 }) if response['status'] == 'OK': for keyword in response['keywords']: self.keywords.add(keyword['text'].encode('ascii','ignore')) else: print('Error in concept tagging call: ', response['statusInfo']) self.keywords = set(["Automatic keyword generation failed"]) response = alchemyapi.concepts('text',self.content, { 'sentiment':1 }) if response['status'] == 'OK': for keyword in response['concepts']: self.keywords.add(keyword['text'].encode('ascii','ignore')) else: print('Error in concept tagging call: ', response['statusInfo']) self.keywords = set(["Automatic keyword generation failed"])
def createTweets(source, num): #num is number of tweets to create words = createDict(source, 2) #state size of 2 allows for more combinations as tweets are small tweets = [] alchemyAPI = AlchemyAPI() for x in range(0, num): #at most 50% chance of using a hashtag if randint(0,1) == 0: tweet = generateText(words, 2, choice(range(100,140))) tweets.append(tweet) else: tweet = generateText(words, 2, choice(range(80,120))) response = alchemyAPI.concepts('text', tweet) if response['status'] == 'OK': hashtag = " #" + response['concepts'][0]['text'].replace(" ", "") if len(hashtag) <= 140 - len(tweet): tweet = tweet + hashtag tweets.append(tweet) return tweets
def generate_concepts_for_company(company_id, tweets): all_tweets_as_string = ' '.join(tweets) alchemyapi = AlchemyAPI() api_error = False for apikey in engine.get_random_alchemy_credentials(): alchemyapi.apikey = apikey response = alchemyapi.concepts('text', all_tweets_as_string) related_words = [] if response['status'] == 'OK': for concept in response['concepts']: related_words.append(concept['text']) elif response['status'] == 'ERROR' and tweets != []: print "ERROR getting concepts" + response['statusInfo'] api_error = True # Move onto the next api key continue # Return null when all api keys are exhausted if api_error and len(related_words) == 0: return None return related_words
concept_list = [] concept_rel = [] keyword_list = [] keyword_rel = [] title = [] #제목 while input_url != 'Q': #Q를 누르면 종료 #print('{}번째 분석입니다.'.format(title_num+1)) print('기사들을 세미콜론(;)으로 구분해서 입력해주세요.') input_url = input('분석할 기사 주소들을 입력하세요 > ') input_url = input_url.split(";") #기사주소들을 세미콜론을 기준으로 나누었음 if input_url != 'Q': response = alchemyapi.concepts('url', input_url, options={'maxRetrieve': input_maxC}) if response['status'] == 'OK': print('##Concepts##') #컨셉 col_num = 2 conc_num = 0 concept_temp = [] con_rel_temp = [] for concept in response['concepts']: print('text: ', concept['text']) print('relevance : ', concept['relevance']) print('') concept_temp.append(concept['text']) con_rel_temp.append(concept['relevance']) """
if flag==1: #response = json.loads(json.dumps(alchemyapi.sentiment("text", trans_text))) ###### GETTING AN ERROR HERE FOR SOME REASON ###### #senti=response["docSentiment"]["type"] response = json.loads(json.dumps(alchemyapi.keywords('text', trans_text, {'sentiment': 1}))) #size=len(response['keywords']) keywords=[] if response['status'] == 'OK': for word in response['keywords']: keywords.append(word['text']) else: print('Error in entity extraction call: ', response['statusInfo']) response=json.loads(json.dumps(alchemyapi.concepts("text",trans_text))) #size=len(response['concepts']) concept=[] if response['status'] == 'OK': for con in response['concepts']: concept.append(con['text']) else: print('Error in entity extraction call: ', response['statusInfo']) tweet_data['entities']=ent tweet_data['ent_relevance']=ent_rele tweet_data['ent_type']=ent_type tweet_data['keywords']=keywords tweet_data['concepts']=concept tweet_data['sentiment']=senti hashtagData = tweet.entities.get('hashtags')
for(dirpath, dirnames,filenames) in walk(out_txt_path): txt_name.extend(filenames) break json_data = {} entity_list = [] keywords_list = [] concept_list = [] for f in txt_name: if f[-3:] == "txt": full_text_path = out_txt_path + f with open(full_text_path, 'r') as current_txt_file: txt_data = current_txt_file.read().replace('\n','') response_entities = alchemyapi.entities('text', txt_data) response_keywords = alchemyapi.keywords('text', txt_data) response_concepts = alchemyapi.concepts('text', txt_data) if response_entities['status'] == 'OK' and response_keywords['status'] == 'OK': print "status OK" for entity in response_entities["entities"]: dict_temp = {'entity': entity['text'], 'type': entity['type'], 'relevance': entity['relevance']} entity_list.append(dict_temp) for keyword in response_keywords["keywords"]: dict_temp = {'keyword': keyword['text'], 'relevance': keyword['relevance']} keywords_list.append(dict_temp) for concept in response_concepts['concepts']: dict_temp = {'concept': concept['text'], 'relevance': concept['relevance']} concept_list.append(dict_temp)
def extract_nouns(text): alchemyapi = AlchemyAPI() response = alchemyapi.concepts("text", text) return str(response)
# CONTENT EXTRACTION FOR TRENDS trend_url = 'http://www.philly.com/philly/blogs/things_to_do/Uber-will-deliver-kittens-to-your-office-in-honor-of-National-Cat-Day--.html'; trend_article = Article(trend_url) trend_article.download() trend_article.parse() trend_title = trend_article.title trend_text = trend_article.text trend_tags = [] trend = alchemyapi.concepts('text', trend_text) if trend['status'] == 'OK': for concept in trend['concepts']: if ' ' in concept['text'].encode('utf-8'): split = concept['text'].encode('utf-8').split() for c in split: trend_tags.append(c) else: trend_tags.append(concept['text'].encode('utf-8')) else: print('Error in concept tagging call: ', trend['statusInfo']) trend_db.insert_one({'keywords': trend_tags, 'title': trend_title}) print(trend_tags) # CONTENT EXTRACTION FOR ARTICLES
CLIENT_SECRETS_FILE = "client_secret.json" with open(CLIENT_SECRETS_FILE) as json_data: d = json.load(json_data) ckey = d['ckey'] csecret = d['csecret'] atoken = d['atoken'] asecret = d['asecret'] api = TwitterAPI(ckey, csecret, atoken, asecret) SEARCH_TERM = 'Election' r = api.request('search/tweets', {'q': SEARCH_TERM, 'count': 10}) for item in r: if 'text' in item: tweet = item['text'] username = item['user']['name'] response = alchemyapi.concepts('text', tweet) if response['status'] == 'OK': for concept in response['concepts']: results = json.dumps(response, indent=4) text = concept['text'] relevance = concept['relevance'] else: text = 'None' relevance = 0 print('Error in concept tagging call: ', response['statusInfo']) print(text, relevance, tweet)
title_num = 0 #분석 텍스트 갯수 concept_list = [] concept_rel = [] keyword_list = [] keyword_rel = [] title = [] #제목 while input_text != 'Q': #Q를 누르면 종료 print('{}번째 분석입니다.'.format(title_num + 1)) input_text = input('분석할 텍스트를 입력하세요 > ') if input_text != 'Q': response = alchemyapi.concepts('text', input_text, options={'maxRetrieve': input_maxC}) if response['status'] == 'OK': print('##Concepts##') #컨셉 col_num = 2 conc_num = 0 concept_temp = [] keyword_temp = [] con_rel_temp = [] key_rel_temp = [] for concept in response['concepts']: print('text: ', concept['text']) print('relevance : ', concept['relevance']) print('') concept_temp.append(concept['text'])
# get the category response = alchemyapi.category('text',full_text) if response['status'] == 'OK': if categories.has_key(response['category']): categories[response['category']] += 1 else: categories[response['category']] = 1 print "[*] Categorized %s as %s" % (pdf_file,response['category']) # grab the concepts response = alchemyapi.concepts('text',full_text) if response['status'] == 'OK': # loop through the list of entities for concept in response['concepts']: # add each entity to our master list if concepts.has_key(concept['text']): concepts[concept['text']] += 1 else: concepts[concept['text']] = 1 print "[*] Retrieved %d concepts from %s" % (len(response['concepts']),pdf_file) else:
print('') print('') print('') print('############################################') print('# Concept Tagging Example #') print('############################################') print('') print('') print('Processing text: ', demo_text) print('') response = alchemyapi.concepts('text',demo_text) if response['status'] == 'OK': print('## Object ##') print(json.dumps(response, indent=4)) print('') print('## Concepts ##') for concept in response['concepts']: print('text: ', concept['text']) print('relevance: ', concept['relevance']) print('') else: print('Error in concept tagging call: ', response['statusInfo'])
else: print('Error in keyword extaction call: ', response['statusInfo']) print('') print('') print('') print('############################################') print('# Concept Tagging Example #') print('############################################') print('') print('') print('Processing text: ', demo_text) print('') response = alchemyapi.concepts('text', demo_text) if response['status'] == 'OK': print('## Object ##') print(json.dumps(response, indent=4)) print('') print('## Concepts ##') for concept in response['concepts']: print('text: ', concept['text']) print('relevance: ', concept['relevance']) print('') else: print('Error in concept tagging call: ', response['statusInfo']) print('')
'statusInfo'] # get the category response = alchemyapi.category('text', full_text) if response['status'] == 'OK': if categories.has_key(response['category']): categories[response['category']] += 1 else: categories[response['category']] = 1 print "[*] Categorized %s as %s" % (pdf_file, response['category']) # grab the concepts response = alchemyapi.concepts('text', full_text) if response['status'] == 'OK': # loop through the list of entities for concept in response['concepts']: # add each entity to our master list if concepts.has_key(concept['text']): concepts[concept['text']] += 1 else: concepts[concept['text']] = 1 print "[*] Retrieved %d concepts from %s" % (len( response['concepts']), pdf_file)
class AlchemyPost: def __init__(self, post_tumblr, post_id, consumer_key, consumer_secret, oauth_token, oauth_secret): self.post_tumblr = post_tumblr self.post_id = post_id self._init_tumblr(consumer_key, consumer_secret, oauth_token, oauth_secret) self._init_alchemy() def _init_tumblr(self, consumer_key, consumer_secret, oauth_token, oauth_secret): self._client = pytumblr.TumblrRestClient(consumer_key, consumer_secret, oauth_token, oauth_secret) def _init_alchemy(self): self.alchemyapi = AlchemyAPI() self.content = {} def analyze_post(self): self.post = self._get_content_post() self._alchemy_entities() self._alchemy_keywords() self._alchemy_concepts() self._alchemy_sentiment() self._alchemy_relations() self._alchemy_category() self._alchemy_feeds() self._alchemy_taxonomy() def print_content(self): print(json.dumps(self.content, indent=4)) def _get_content_post(self): print "*", infos = self._get_infos_post() self.title = '' self.tags = [] if 'tags' in infos: self.tags = infos['tags'] if infos['type'] == 'text': return self._get_content_text(infos) if infos['type'] == 'quote': return self._get_content_quote(infos) return '' def _get_infos_post(self): infos = self._client.posts(self.post_tumblr, id=self.post_id) if 'posts' in infos and len(infos['posts'])>0: return infos['posts'][0] return {} def _get_content_text(self, infos): content = "<h1>" + str(infos['title']) + "</h1>" content += " <br>" + str(infos['body']) content += " <br>" + " ".join(infos['tags']) return content def _get_content_quote(self, infos): content = str(infos['text']) content += " <br>" + str(infos['source']) content += " <br>" + " ".join(infos['tags']) return content def _alchemy_entities(self): print ".", response = self.alchemyapi.entities('html', self.post) if response['status'] != 'OK': return False self.content['entities'] = response['entities'] return True def _alchemy_keywords(self): print ".", response = self.alchemyapi.keywords('html', self.post) if response['status'] != 'OK': return False self.content['keywords'] = response['keywords'] return True def _alchemy_concepts(self): print ".", response = self.alchemyapi.concepts('html', self.post) if response['status'] != 'OK': return False self.content['concepts'] = response['concepts'] return True def _alchemy_sentiment(self): print ".", response = self.alchemyapi.sentiment('html', self.post) if response['status'] != 'OK': return False self.content['sentiment'] = response['docSentiment'] return True def _alchemy_relations(self): print ".", response = self.alchemyapi.relations('html', self.post) if response['status'] != 'OK': return False self.content['relations'] = response['relations'] return True def _alchemy_category(self): print ".", response = self.alchemyapi.category('html', self.post) if response['status'] != 'OK': return False self.content['category'] = response['category'] self.content['score'] = response['score'] return True def _alchemy_feeds(self): print ".", response = self.alchemyapi.feeds('html', self.post) if response['status'] != 'OK': return False self.content['feeds'] = response['feeds'] return True def _alchemy_taxonomy(self): print ".", response = self.alchemyapi.taxonomy('html', self.post) if response['status'] != 'OK': return False self.content['taxonomy'] = response['taxonomy'] return True
def user_analysis_sentiments(request): if request.method == 'GET': print request.GET.get('user', '') user = request.GET.get('user', '') messages = [] message = Message.objects.filter(user_send=user.decode("utf8")) for m in message: messages.append(m.message_text) text = ",".join(messages) alchemyapi = AlchemyAPI() #keywords response = alchemyapi.keywords('text', text, {'sentiment': 1}) if response['status'] == 'OK': keywords = [] for keyword in response['keywords']: keyword_text = keyword['text'].encode('utf-8') keyword_relevance = keyword['relevance'] keyword_sentiment = keyword['sentiment']['type'] key_word = {'keyword_text': keyword_text, 'keyword_relevance': keyword_relevance, 'keyword_sentiment': keyword_sentiment} keywords.append(key_word) else: print('Error in keyword extaction call: ', response['statusInfo']) response = alchemyapi.concepts('text', text) if response['status'] == 'OK': concepts = [] for concept in response['concepts']: concept_text = concept['text'] concept_relevance = concept['relevance'] concept_entity = {'concept_text': concept_text, 'concept_relevance': concept_relevance} concepts.append(concept_entity) else: print('Error in concept tagging call: ', response['statusInfo']) response = alchemyapi.language('text', text) if response['status'] == 'OK': print(response['wikipedia']) language = response['language'] iso_639_1 = response['iso-639-1'] native_speakers = response['native-speakers'] wikipedia = response['wikipedia'] language_id = {'language': language, 'iso_639_1': iso_639_1, 'native_speakers': native_speakers, 'wikipedia': wikipedia} else: print('Error in language detection call: ', response['statusInfo']) response = alchemyapi.relations('text', text) if response['status'] == 'OK': relations = [] for relation in response['relations']: if 'subject' in relation: relation_subject_text = relation['subject']['text'].encode('utf-8') if 'action' in relation: relation_action_text = relation['action']['text'].encode('utf-8') if 'object' in relation: relation_object_text = relation['object']['text'].encode('utf-8') relation_entity = {'relation_subject_text': relation_subject_text, 'relation_action_text': relation_action_text, 'relation_object_text': relation_object_text} relations.append(relation_entity) else: print('Error in relation extaction call: ', response['statusInfo']) response = alchemyapi.category('text', text) if response['status'] == 'OK': print('text: ', response['category']) category = response['category'] print('score: ', response['score']) score = response['score'] categories = {'category': category, 'score': score} else: print('Error in text categorization call: ', response['statusInfo']) response = alchemyapi.taxonomy('text', text) if response['status'] == 'OK': taxonomies = [] for category in response['taxonomy']: taxonomy_label = category['label'] taxonomy_score = category['score'] taxonomy = {'taxonomy_label': taxonomy_label, 'taxonomy_score': taxonomy_score} taxonomies.append(taxonomy) else: print('Error in taxonomy call: ', response['statusInfo']) response = alchemyapi.combined('text', text) if response['status'] == 'OK': print('## Response Object ##') print(json.dumps(response, indent=4)) print('') user = {'user_name': 'LOL', 'keywords': keywords, 'concepts': concepts, 'language_id': language_id, 'relations': relations, 'categories': categories, 'taxonomies': taxonomies} return HttpResponse(json.dumps(user), content_type="application/json")
assert(response['status'] == 'OK') response = alchemyapi.keywords('html', test_html); assert(response['status'] == 'OK') response = alchemyapi.keywords('url', test_url); assert(response['status'] == 'OK') response = alchemyapi.keywords('random', test_url); assert(response['status'] == 'ERROR') #invalid flavor print('Keyword tests complete!') print('') #Concepts print('Checking concepts . . . ') response = alchemyapi.concepts('text', test_text); assert(response['status'] == 'OK') response = alchemyapi.concepts('html', test_html); assert(response['status'] == 'OK') response = alchemyapi.concepts('url', test_url); assert(response['status'] == 'OK') response = alchemyapi.concepts('random', test_url); assert(response['status'] == 'ERROR') #invalid flavor print('Concept tests complete!') print('') #Sentiment print('Checking sentiment . . . ') response = alchemyapi.sentiment('text', test_text);
print("Error in keyword extaction call: ", response["statusInfo"]) print("") print("") print("") print("############################################") print("# Concept Tagging Example #") print("############################################") print("") print("") print("Processing text: ", demo_text) print("") response = alchemyapi.concepts("text", demo_text) if response["status"] == "OK": print("## Object ##") print(json.dumps(response, indent=4)) print("") print("## Concepts ##") for concept in response["concepts"]: print("text: ", concept["text"]) print("relevance: ", concept["relevance"]) print("") else: print("Error in concept tagging call: ", response["statusInfo"])
alchemyapi = AlchemyAPI() response = alchemyapi.keywords('text', text, { 'sentiment': 1, 'maxRetrieve': 20, 'keywordExtractMode': 'strict' }) if response['status'] == "OK": print(json.dumps(response, indent=4)) for keyword in response['keywords']: print('Keyword text:', keyword['text'].encode('utf-8')) print('Keyword relevance:', keyword['relevance']) print("**********") response = alchemyapi.concepts('text', text, {'maxRetrieve': 10}) if response['status'] == "OK": #print(json.dumps(response, indent = 4)) keywords = [(concept['text'].encode('ascii'), float(concept['relevance'])) for concept in response['concepts']] for concept in response['concepts']: print('Concept text:', concept['text'].encode('utf-8')) print('Concept relevance:', concept['relevance']) print keywords print("**********") response = alchemyapi.entities('text', text, {'maxRetrieve': 200})
#Keywords print('Checking keywords . . . ') response = alchemyapi.keywords('text', test_text) assert (response['status'] == 'OK') response = alchemyapi.keywords('html', test_html) assert (response['status'] == 'OK') response = alchemyapi.keywords('url', test_url) assert (response['status'] == 'OK') response = alchemyapi.keywords('random', test_url) assert (response['status'] == 'ERROR') #invalid flavor print('Keyword tests complete!') print('') #Concepts print('Checking concepts . . . ') response = alchemyapi.concepts('text', test_text) assert (response['status'] == 'OK') response = alchemyapi.concepts('html', test_html) assert (response['status'] == 'OK') response = alchemyapi.concepts('url', test_url) assert (response['status'] == 'OK') response = alchemyapi.concepts('random', test_url) assert (response['status'] == 'ERROR') #invalid flavor print('Concept tests complete!') print('') #Sentiment print('Checking sentiment . . . ') response = alchemyapi.sentiment('text', test_text) assert (response['status'] == 'OK') response = alchemyapi.sentiment('html', test_html)