def main():

    alchemyapi = AlchemyAPI()

    try:
        filename = sys.argv[1]
    except IndexError:
        print "Give a filename as the second argument!"
        sys.exit(1)

    text = pdf_to_str(filename)

    if len(text) >= LENGTH_LIMIT:
        print "PDF content is longer ({} characters) than the maximum \
of {}, skipping remainder".format(len(text), LENGTH_LIMIT)
        text = text[:LENGTH_LIMIT]


    print "KEYWORDS"
    response = alchemyapi.keywords('text', text)
    for keyword in response['keywords']:
        print '  - {}'.format(keyword['text'])

    print
    print "CONCEPTS"
    response = alchemyapi.concepts('text', text)
    for concept in response['concepts']:
        print '  - {}'.format(concept['text'])
def getKeywordPerPost():
    reader = open('output_sony_posts.txt')
    all_json_posts = reader.read().splitlines()
    alchemyapi = AlchemyAPI()
    counter = 0
    for p in all_json_posts:
        print str(counter)
        if counter < 1000:
            counter = counter + 1
            continue
        #elif counter > 2000:
        #    break
        else:
            counter = counter + 1
        content = json.loads(p)["cleanContent"]   
        response = alchemyapi.keywords('text',content.encode("UTF-8"))
    
        if response['status'] == 'OK':
            keywords = []
            posts = ""
            for keyword in response['keywords']:
                    keywords.append(keyword['text'])
                    posts = posts + keyword['text'] + ","
            posts = posts[:-1] + "\n"
            if posts <> "\n":
                with codecs.open("keyPerPost.txt", "a") as f:
                    f.write(posts.encode("UTF-8"))
        else:
            print "error" + str(counter)
    def extractKeywordsFromUrl(self,url):
        """method for extracting keywords from given text"""
        
        #creating AlchemyAPI object
        alchemyapi = AlchemyAPI()
              
        #requesting json response from AlchemyAPI server
        response = alchemyapi.keywords('url', url)
        
        if response['status'] == 'OK':
    

            for keywords in response['keywords']:
                
                #concept object for storing the extracted concept
                keyword = AlchemyStructure.Keyword()
                
                #extracting the keyword
                keyword.setText(keywords['text'])
                
                #extracting the relevance of keyword
                keyword.setRelevance(keywords['relevance'])
                
                #append the concept into the list of retrieved concepts
                self.keywordsFromUrl.append(keyword)

        else:
            print('Error in keyword tagging call: ', response['statusInfo'])
def getKeywords(uID, inputText):
    alchemyapi = AlchemyAPI()
    #alchemyapi.loadAPIKey("api_key.txt")
    response = alchemyapi.keywords('text',inputText)
    print inputText
    if response['status'] == 'OK':
        #print('## Response Object ##')
        #print(json.dumps(response, indent=4))


        #print('')
        #print('## Keywords ##')
        keywords = []
        posts = uID + " : "
        for keyword in response['keywords']:
                keywords.append(keyword['text'])
                posts = posts + keyword['text'] + "|"
        userKeywords[uID] = keywords
        posts = posts + "\n"
        with codecs.open("outNew.txt", "a") as f:
            f.write(posts.encode("UTF-8"))
        return True
    else:
        print('idError: ', uID)
        with codecs.open("keywordOut2.txt", "a") as f:
            text = uID + "\n"
            f.write(text.encode("UTF-8"))
        return False
Beispiel #5
0
def get_sentiment(company_id, text):
    alchemyapi = AlchemyAPI()
    key_phrases = []
    for apikey in engine.get_random_alchemy_credentials():
        alchemyapi.apikey = apikey
        response = alchemyapi.keywords('text', text, {'sentiment': 1})
        if response['status'] == 'OK':
            if len(response['keywords']) == 0:
                return 0
            # related_words = models.RelatedWord.query.filter_by(company_id=company_id).all()
            for keyword in response["keywords"]:
                if 'sentiment' in keyword:
                    if keyword['sentiment'].has_key('score'):
                        key_phrases.append(float(keyword['sentiment']['score']))
                    elif keyword['sentiment']['type'] == 'neutral':
                        key_phrases.append(0)

            if len(key_phrases) == 0:
                return 0
            else:
                return float("{0:.2f}".format(sum(key_phrases)/len(key_phrases)))
        elif response['status'] == 'ERROR' and response['statusInfo'] != 'unsupported-text-language':
            print "ERROR: getting sentiment " + response['statusInfo']
            # Skip onto the next api key
            continue
        else:
            print "None of the above " + response['statusInfo']
            return 0
    #Return none when all api keys are exhausted
    return None
Beispiel #6
0
def sentiment_alchemy(url):
    alchemyapi = AlchemyAPI()

    response = alchemyapi.sentiment('url', url)
    response['usage'] = None

    if response['status'] == 'OK':
        print('## Response Object ##')
        print(json.dumps(response, indent=4))

        print('')
        print('## Author ##')
        print('author: ', response.get('author', ''))
        print('')
    else:
        print('Error in author extraction call: ', response['statusInfo'])

    response = alchemyapi.keywords('url', url)
    del (response['usage'])

    if response['status'] == 'OK':
        print('## Response Object ##')
        print(json.dumps(response, indent=4))

        print('')
        print('## Keywords ##')
        for keyword in response['keywords']:
            print('text: ', keyword['text'].encode('utf-8'))
            print('relevance: ', keyword['relevance'])
            print('sentiment: ', keyword.get('sentiment', {}).get('type', ''))
            if 'score' in keyword.get('sentiment', {}):
                print('sentiment score: ' + keyword['sentiment']['score'])
            print('')
        else:
            print('Error in keyword extaction call: ', response.get('statusInfo', ''))
def nlp_process(ids, ids_hash):
    #instantiate an elasticsearch client
    es = Elasticsearch()

    #instantiate an alchemy client
    alchemyapi = AlchemyAPI()

    for item in ids:
        data = ' '.join(ids_hash[item])
        lowers = data.lower()
        alchem_data = []

        response = alchemyapi.keywords('text', lowers, {'sentiment': 1})

        if response['status'] == 'OK':
            print('#Success#')
            for keyword in response['keywords']:
                al_temp = defaultdict()

                al_temp['text'] = keyword['text'].encode('utf-8')
                al_temp['relevance'] = keyword['relevance']
                al_temp['sentiment'] = keyword['sentiment']['type']

                if 'score' in keyword['sentiment']:
                    al_temp['score'] = keyword['sentiment']['score']

                alchem_data.append(al_temp)
        else:
            print('Error in keyword extaction call: ', response['statusInfo'])
        print len(alchem_data)
        # prepare body for insertion
        doc = {"business_id": item, "word_freq": alchem_data}
        exit()
        template = {"create": {"_index": "alchem", "_type": "doc"}}
        res = es.index(index="alchem", doc_type='doc', body=doc)
Beispiel #8
0
 def use_api(self, input):
         keyword_array=[]
         alchemyapi = AlchemyAPI()
         response = alchemyapi.keywords("text", input)
         for keyword in response["keywords"]:
                 #print keyword["text"].encode('utf=8')
                 keyword_array.append(keyword["text"].encode('utf=8'))
         return keyword_array
Beispiel #9
0
def fetchkeywords(demo_text):
    alchemyapi = AlchemyAPI()
    response = alchemyapi.keywords('text',demo_text, { 'sentiment':1 })
    listoftopwords = []

    if response['status'] == 'OK':
        for keyword in response['keywords']:
	        listoftopwords.append(keyword['text'])

    return listoftopwords
Beispiel #10
0
def performKeywordExtraction(text):
    keywordText=[]
    alchemyapi = AlchemyAPI()
    response = alchemyapi.keywords("text", text)
    if response['status'] == 'OK':
        keywords = response['keywords']
        for keyword in keywords:
            if (float(keyword['relevance'])>0.1):
                keywordText.append(keyword['text'])
    return keywordText    
def connect_alchemy(url):
    # to connect with alchemy and tag the content
    from alchemyapi import AlchemyAPI
    alchemyapi = AlchemyAPI()

    resp = alchemyapi.text('url', url)

    response = alchemyapi.keywords("text", resp['text'])

    keywors = response["keywords"]
def connect_alchemy(url):
	# to connect with alchemy and tag the content 
	from alchemyapi import AlchemyAPI
	alchemyapi 	= AlchemyAPI()

	resp       	= alchemyapi.text('url', url)

	response 	= alchemyapi.keywords("text", resp['text'])

	keywors = response["keywords"]
def solve(passage, question, choices):
    relevant_text = get_relevant_text(passage, question)

    alchemyapi = AlchemyAPI()
    passage_sentiment = {}
    response = alchemyapi.keywords('text', relevant_text, {'sentiment': 1})
    if response['status'] == 'OK':
        for keyword in response['keywords']:
            if 'sentiment' in keyword.keys() and 'score' in keyword['sentiment']:
                passage_sentiment[keyword["text"]] = keyword['sentiment']
 
    choices_sentiment = {}
    for choice in choices:
        response = alchemyapi.keywords('text', "the" + choices[choice], {'sentiment': 1})
        if response['status'] == 'OK':
            for keyword in response['keywords']:
                if 'sentiment' in keyword.keys() and 'score' in keyword['sentiment']:
                    choices_sentiment[choice] = keyword['sentiment']
 
    return rank_choices_sentiment(passage_sentiment, choices_sentiment)
Beispiel #14
0
def getAlcData(arrOfObj):
	
	alchemyapi = AlchemyAPI()	

	
	#for x in range(0,len(arrOfObj)):
	for x in range(0, 10):
		asc = unicodedata.normalize('NFKD', arrOfObj[x].text).encode('ascii','ignore')
		print x		
		print asc 
		arrOfObj[x].responseEntities = alchemyapi.entities('text',asc, { 'sentiment':1 })
		arrOfObj[x].responseKeywords = alchemyapi.keywords('text',asc, { 'sentiment':1 })	
Beispiel #15
0
class App:
    def __init__(self):
        self.alchemyapi = AlchemyAPI()
        self.raw_text = ''
        self.concepts = None
        self.keywords = None

    def parse_url(self, url=None):
        text_response = self.alchemyapi.text('url', url)
        if text_response['status'] == 'OK':
            self.raw_text =  text_response['text'].encode('utf-8')
        else:
            print('Error in text extraction call: ', text_response['statusInfo'])

    def extract_concepts(self):
        concept_response = self.alchemyapi.concepts('text', self.raw_text)
        if concept_response['status'] == 'OK':
            self.concepts = concept_response['concepts']
            # print('## Concepts ##')
            # for concept in self.concepts:
            #     print('text: ', concept['text'])
            #     print('relevance: ', concept['relevance'])
            #     print('')
        else:
            print('Error in concept tagging call: ', concept_response['statusInfo'])

    def extract_keywords(self):
        keyword_response = self.alchemyapi.keywords('text', self.raw_text, {'sentiment': 1})
        if keyword_response['status'] == 'OK':
            self.keywords = keyword_response['keywords']
            # print('')
            # print('## Keywords ##')
            # for keyword in self.keywords:
            #     print('text: ', keyword['text'].encode('utf-8'))
            #     print('relevance: ', keyword['relevance'])
            #     print('sentiment: ', keyword['sentiment']['type'])
            #     if 'score' in keyword['sentiment']:
            #         print('sentiment score: ' + keyword['sentiment']['score'])
            #     print('')
        else:
            print('Error in keyword extraction call: ', keyword_response['statusInfo'])

    def define_concepts(self):
        for concept in self.concepts:
            definition = duckduckgo.get_zci(concept['text'])
            print('%s -> %s' % (concept['text'], definition))
            print('')

    def define_keywords(self):
        for keyword in self.keywords:
            definition = duckduckgo.get_zci(keyword['text'])
            print('%s -> %s' % (keyword['text'], definition))
            print('')
def checkDailyQuotaAndRunAlchemy(commentDb,cruiseLines):
    with open('data/Alchemy_response_keywords.json', 'rb') as fp:
            returned_keywords = json.load(fp)
    with open('data/Alchemy_response_relations.json', 'rb') as fp:
            returned_relations = json.load(fp)       
    alchemyapi = AlchemyAPI()
    test="test if finished Alchemy daily quota"
    response = alchemyapi.keywords('text', test, {'sentiment': 0})
    if response['status'] == 'OK':
        returned_keywords,returned_relations=runAlchemyApi(cruiseLines,commentDb,returned_keywords,returned_relations,alchemyapi)
    else:
        print 'Error in keyword extraction call: ', response['statusInfo']
    return returned_keywords, returned_relations 
def solve(passage, question, choices):
    relevant_text = get_relevant_text(passage, question)

    alchemyapi = AlchemyAPI()
    passage_sentiment = {}
    response = alchemyapi.keywords('text', relevant_text, {'sentiment': 1})
    if response['status'] == 'OK':
        for keyword in response['keywords']:
            if 'sentiment' in keyword.keys(
            ) and 'score' in keyword['sentiment']:
                passage_sentiment[keyword["text"]] = keyword['sentiment']

    choices_sentiment = {}
    for choice in choices:
        response = alchemyapi.keywords('text', "the" + choices[choice],
                                       {'sentiment': 1})
        if response['status'] == 'OK':
            for keyword in response['keywords']:
                if 'sentiment' in keyword.keys(
                ) and 'score' in keyword['sentiment']:
                    choices_sentiment[choice] = keyword['sentiment']

    return rank_choices_sentiment(passage_sentiment, choices_sentiment)
Beispiel #18
0
def ExtractKeyword(text):
    alchemyapi = AlchemyAPI()

    response = alchemyapi.keywords('text', text, {'sentiment': 1})
    results = []
    if response['status'] == 'OK':
        for keyword in response['keywords']:
            results.append(keyword['text'])
            # print('text: ', keyword['text'].encode('utf-8'))
            # print('relevance: ', keyword['relevance'])
    else:
        print('Error in keyword extaction call: ', response['statusInfo'])

    return results
Beispiel #19
0
def getSoup():
    sock = urllib.urlopen('https://en.wikipedia.org/wiki/Motocross')
    sockRaw = sock.read()
    soup = BeautifulSoup(sockRaw, "html.parser")
    soupText = soup.get_text()

    # use the alchemyAPI to find the keyword/phrases from the texts
    alchemyapi = AlchemyAPI()
    response = alchemyapi.keywords('text', soupText, {'maxRetrieve': 10})
    if response['status'] == 'OK':
        print "\nThe Keywords are:"
        for i in response['keywords']:
            print "Word: " + i["text"] + ", Relevance: " + i["relevance"]
    else:
        print "Something went wrong with Alchemy."
Beispiel #20
0
def checkDailyQuotaAndRunAlchemy(commentDb, cruiseLines):
    with open('data/Alchemy_response_keywords.json', 'rb') as fp:
        returned_keywords = json.load(fp)
    with open('data/Alchemy_response_relations.json', 'rb') as fp:
        returned_relations = json.load(fp)
    alchemyapi = AlchemyAPI()
    test = "test if finished Alchemy daily quota"
    response = alchemyapi.keywords('text', test, {'sentiment': 0})
    if response['status'] == 'OK':
        returned_keywords, returned_relations = runAlchemyApi(
            cruiseLines, commentDb, returned_keywords, returned_relations,
            alchemyapi)
    else:
        print 'Error in keyword extraction call: ', response['statusInfo']
    return returned_keywords, returned_relations
Beispiel #21
0
def getSoup():
    sock = urllib.urlopen('https://en.wikipedia.org/wiki/Motocross')
    sockRaw = sock.read()
    soup = BeautifulSoup(sockRaw, "html.parser")
    soupText = soup.get_text()

# use the alchemyAPI to find the keyword/phrases from the texts
    alchemyapi = AlchemyAPI()
    response = alchemyapi.keywords('text', soupText, {'maxRetrieve': 10})
    if response['status'] == 'OK':
        print "\nThe Keywords are:"
        for i in response['keywords']:
            print "Word: " + i["text"] + ", Relevance: " + i["relevance"]
    else:
        print "Something went wrong with Alchemy."
Beispiel #22
0
    def extract_keyword_en(self, corpus_text, min_len=4, max_len=50):
        alchemyapi = AlchemyAPI(ALCHEMY_API_KEY)

        response = alchemyapi.keywords('text', corpus_text, {'sentiment': 1})

        keywords = []
        if response['status'] == "OK":
            for keyword in response['keywords']:
                phrase = keyword['text'].encode('utf8')
                freq = self.phrase_frequency(phrase, corpus_text)
                if freq > 0 and phrase.lower() != self.query.lower() and len(phrase) >= min_len and len(phrase) <= max_len:
                    score = float(keyword['relevance'].encode('utf8'))
                    kw = Keyword(phrase, score, freq)
                    keywords.append(kw)

        return keywords[:min(len(keywords), self.numberofkeywords)]
Beispiel #23
0
    def setKeywords(self):
        alchemyapi = AlchemyAPI()
        response = alchemyapi.keywords('text',self.content, { 'sentiment':1 })
	if response['status'] == 'OK':
		for keyword in response['keywords']:
		    self.keywords.add(keyword['text'].encode('ascii','ignore'))
	else:
		print('Error in concept tagging call: ', response['statusInfo'])
		self.keywords = set(["Automatic keyword generation failed"])
	response = alchemyapi.concepts('text',self.content, { 'sentiment':1 })
	if response['status'] == 'OK':
		for keyword in response['concepts']:
		    self.keywords.add(keyword['text'].encode('ascii','ignore'))
	else:
		print('Error in concept tagging call: ', response['statusInfo'])
		self.keywords = set(["Automatic keyword generation failed"])
Beispiel #24
0
def analysecontent(content):
    """
    Process/Analyse the extracted contents with Alchemy API
    Assumption: api_key.txt with a valid key is available from where this program is getting executed.
    """
    print ("Processing extracted text with AlchemyAPI...")
    alchemyapi = AlchemyAPI()
    response = alchemyapi.keywords("text", content, {"maxRetrieve": 10})
    if response["status"] == "OK":
        print ("---------------------------------")
        print ("## Keywords      ## Relevance")
        for keyword in response["keywords"]:
            print ("{0}: {1}".format(keyword["text"].encode("utf-8"), keyword["relevance"]))
        print ("---------------------------------")
    else:
        print ("Error in keyword extraction call: ", response["statusInfo"])
Beispiel #25
0
def analysecontent(content):
    """
    Process/Analyse the extracted contents with Alchemy API
    Assumption: api_key.txt with a valid key is available from where this program is getting executed.
    """
    print('Processing extracted text with AlchemyAPI...')
    alchemyapi = AlchemyAPI()
    response = alchemyapi.keywords('text', content, {'maxRetrieve': 10})
    if response['status'] == 'OK':
        print('---------------------------------')
        print('## Keywords      ## Relevance')
        for keyword in response['keywords']:
            print("{0}: {1}".format(keyword['text'].encode('utf-8'),
                                    keyword['relevance']))
        print('---------------------------------')
    else:
        print('Error in keyword extraction call: ', response['statusInfo'])
Beispiel #26
0
class NLP:
    def __init__(self):
        self.alchemyapi = AlchemyAPI()

    def get_categories(self, text):
        response = self.alchemyapi.taxonomy('text', text)

        if response['status'] == 'OK' and len(response['taxonomy']) > 0:
            taxonomy = response['taxonomy'][0]
            tokens = taxonomy['label'].split('/')
            return tokens[1]

    def get_keywords(self, text):
        response = self.alchemyapi.keywords('text', text)

        if response['status'] == 'OK' and len(response['keywords']) > 0:
            return [x['text'] for x in response['keywords']]
    def extract_keyword_alchemy(self, corpus):
        alchemyapi = AlchemyAPI()

        corpus_text = "\n".join(corpus)

        response = alchemyapi.keywords('text', corpus_text, {'sentiment': 1})

        keywords = []
        if response['status'] == "OK":
            for keyword in response['keywords']:
                pharse = keyword['text'].encode('utf8')
                score = float(keyword['relevance'].encode('utf8'))
                kw = Keyword(pharse, score)
                keywords.append(kw)

        sorted_keywords = sorted(keywords, key=lambda t: t.get_score() * -1)
        
        return sorted_keywords[:min(len(sorted_keywords), self.numberofkeywords)]
Beispiel #28
0
def get_sentiment(text):
    alchemyapi = AlchemyAPI()
    alchemyapi.apikey = get_random_alchemy_credentials()
    response = alchemyapi.keywords('text', text, {'sentiment': 1})
    relevances = []
    if 'keywords' not in response or len(response['keywords']) == 0:
        return None
    for keyword in response["keywords"]:
        for company_word in concepts:
            if company_word.lower() in text.lower() and 'sentiment' in keyword:
                if 'score' in keyword['sentiment']:
                    relevances.append(float(keyword['sentiment']['score']))
                elif keyword['sentiment']['type'] == 'neutral':
                    relevances.append(0.5)
    if not relevances:
        return 0.5
    else:
        return float("{0:.2f}".format(sum(relevances)/len(relevances)))
Beispiel #29
0
    def extractKeywordsFromUrl(self, url):
        """method for extracting keywords from given text"""

        # creating AlchemyAPI object
        alchemyapi = AlchemyAPI()

        # requesting json response from AlchemyAPI server
        response = alchemyapi.keywords("url", url)

        if response["status"] == "OK":

            for keywords in response["keywords"]:

                # concept object for storing the extracted concept
                keyword = AlchemyStructure.Keyword()

                # extracting the keyword
                keyword.setText(keywords["text"])

                # extracting the relevance of keyword
                keyword.setRelevance(keywords["relevance"])

                # instantiating the sentiment object
                sentimentObj = AlchemyStructure.Sentiment()

                print(keywords["sentiment"]["type"])
                # extracting the score of the sentiment associated with the keyword
                if keywords["sentiment"]["type"] == "neutral":
                    sentimentObj.setScore("0")
                else:
                    sentimentObj.setScore(keywords["sentiment"]["score"])

                # extracting the type of the sentiment associated with the keyword -> positive, negative or neutral
                sentimentObj.setType(keywords["sentiment"]["type"])

                # set the sentiment for keyword
                keyword.setSentiment(sentimentObj)

                # append the concept into the list of retrieved concepts
                self.keywordsFromUrl.append(keyword)

        else:
            print("Error in keyword tagging call: ", response["statusInfo"])
def keywords_and_sentiment(contentList):
    # Accepts a string of posts (or chats), and returns a list of keywords
    alchemyapi = AlchemyAPI()
    relevanceList = []
    sentiment = 0
    counter = 0

    for post_message in contentList:
        response = alchemyapi.keywords("text", post_message, {"sentiment": 1})
        if response["status"] == "OK":
            for keyword in response["keywords"]:
                if "score" in keyword["sentiment"]:
                    sentiment += float(keyword["sentiment"]["score"])
                if float(keyword["relevance"]) > 0.97:
                    relevanceList.append(keyword["text"])
                counter += 1
    if DEV_MODE:
        if not counter is 0:
            print float(sentiment / counter)

    return relevanceList
def keywords_and_sentiment(contentList):
    # Accepts a string of posts (or chats), and returns a list of keywords
    alchemyapi = AlchemyAPI()
    relevanceList = []
    sentiment = 0
    counter = 0

    for post_message in contentList:
        response = alchemyapi.keywords('text', post_message, {'sentiment': 1})
        if response['status'] == 'OK':
            for keyword in response['keywords']:
                if 'score' in keyword['sentiment']:
                    sentiment += float(keyword['sentiment']['score'])
                if float(keyword['relevance']) > .97:
                    relevanceList.append(keyword['text'])
                counter += 1
    if DEV_MODE:
        if not counter is 0:
            print float(sentiment / counter)

    return relevanceList
Beispiel #32
0
def run_app():

    ##### part 1

    page = requests.get('https://en.wikipedia.org/wiki/Michael_Jordan')
    tree = html.fromstring(page.content)
    h3s = tree.xpath('//h3/span/text()')
    h3s_combined = ''

    for next_h3 in h3s:
        h3s_combined += ' '
        h3s_combined += next_h3

    print('***********************************************')
    print('*** The h3s on this page were: ', h3s_combined)
    print('***********************************************')

    ##### part 2

    #alchemy_api_key="xxxx"
    alchemyapi = AlchemyAPI()

    response = alchemyapi.keywords('text', h3s_combined, {
        'sentiment': 1,
        'maxRetrieve': 10
    })

    if response['status'] == 'OK':
        print('## Response Object ##')
        # print(json.dumps(response, indent=4))

        print('')
        print('## Keywords ##')
        for keyword in response['keywords']:
            print('text: ', keyword['text'].encode('utf-8'))
            print('relevance: ', keyword['relevance'])
            print('sentiment: ', keyword['sentiment']['type'])
            print('')
    else:
        print('Error in keyword extaction call: ', response['statusInfo'])
Beispiel #33
0
def alchemyExtract(cleanText,options):
    """Uses alchemyAPI to find keywords in the cleaned text. 
    In this case,options[1] should specify the number of keywords to be 
    used. Default to 4."""
    try:
        options[1]=int(options[1])
    except IndexError as e:
        options[1]=4
    except ValueError as e:
        print("You selected alchemy, and options[1] was not a valid integer.")
        raise e
    from alchemyapi import AlchemyAPI
    alch = AlchemyAPI()
    response = alch.keywords('text',cleanText,{'sentiment':1})
    rKeywords = response['keywords']
    finalKeywords = []
    if options[1]<len(rKeywords):
        for i in range(options[1]):
            finalKeywords.append(rKeywords[i]['text'])
    else:
        finalKeywords = rKeywords
    return "\n\nKeywords: " + ", ".join(finalKeywords) #figure out formatting for this later.
def nlp_process(ids,ids_hash):
    #instantiate an elasticsearch client
    es = Elasticsearch()

    #instantiate an alchemy client
    alchemyapi = AlchemyAPI()

    for item in ids:
        data = ' '.join(ids_hash[item])
        lowers = data.lower()
        alchem_data = []

        response = alchemyapi.keywords('text', lowers, {'sentiment': 1})

        if response['status'] == 'OK':
            print('#Success#')
            for keyword in response['keywords']:
                al_temp = defaultdict()

                al_temp['text'] = keyword['text'].encode('utf-8')
                al_temp['relevance'] = keyword['relevance']
                al_temp['sentiment'] = keyword['sentiment']['type']

                if 'score' in keyword['sentiment']:
                    al_temp['score'] = keyword['sentiment']['score']

                alchem_data.append(al_temp)
        else:
            print('Error in keyword extaction call: ', response['statusInfo'])
        print len(alchem_data)
        # prepare body for insertion
        doc = {
            "business_id" : item,
            "word_freq": alchem_data
        }
        exit()
        template = { "create": { "_index": "alchem", "_type": "doc"} }
        res = es.index(index="alchem", doc_type='doc', body=doc)
Beispiel #35
0
def alchemyExtract(cleanText, options):
    """Uses alchemyAPI to find keywords in the cleaned text. 
    In this case,options[1] should specify the number of keywords to be 
    used. Default to 4."""
    try:
        options[1] = int(options[1])
    except IndexError as e:
        options[1] = 4
    except ValueError as e:
        print("You selected alchemy, and options[1] was not a valid integer.")
        raise e
    from alchemyapi import AlchemyAPI
    alch = AlchemyAPI()
    response = alch.keywords('text', cleanText, {'sentiment': 1})
    rKeywords = response['keywords']
    finalKeywords = []
    if options[1] < len(rKeywords):
        for i in range(options[1]):
            finalKeywords.append(rKeywords[i]['text'])
    else:
        finalKeywords = rKeywords
    return "\n\nKeywords: " + ", ".join(
        finalKeywords)  #figure out formatting for this later.
Beispiel #36
0
def run_app():	

	##### part 1

	page = requests.get('https://en.wikipedia.org/wiki/Michael_Jordan')
	tree = html.fromstring(page.content)
	h3s = tree.xpath('//h3/span/text()')
	h3s_combined = ''

	for next_h3 in h3s:
		h3s_combined += ' '
		h3s_combined += next_h3
	
	print('***********************************************')
	print('*** The h3s on this page were: ', h3s_combined)
	print('***********************************************')

	##### part 2

	#alchemy_api_key="xxxx"
	alchemyapi = AlchemyAPI()

	response = alchemyapi.keywords('text', h3s_combined, {'sentiment': 1, 'maxRetrieve': 10})

	if response['status'] == 'OK':
		print('## Response Object ##')
		# print(json.dumps(response, indent=4))

		print('')
		print('## Keywords ##')
		for keyword in response['keywords']:
			print('text: ', keyword['text'].encode('utf-8'))
			print('relevance: ', keyword['relevance'])
			print('sentiment: ', keyword['sentiment']['type'])
			print('')
	else:
		print('Error in keyword extaction call: ', response['statusInfo'])
Beispiel #37
0
 def post(self):
     sid = self.get_argument('sid')
     abstract = self.get_argument('abstract')
     print abstract
     url = "https://api-us.clusterpoint.com/100629/stud/_search.json"
     headers = {"Authorization":"Basic bGVvcGFuaWdyYWhpQGdtYWlsLmNvbToyM2xlbzIz"}
     string = "<project><abstract>"+ abstract +"</abstract></project>"
     values = dict(query=string)
     r = requests.post(url,data=json.dumps(values), headers=headers)
     responseDict = json.loads(r.content)
     for each in responseDict['documents'][0]['project']:
         if each['abstract'] == abstract:
             mytext = each['abstract']
             alchemyapi = AlchemyAPI()
             response = alchemyapi.keywords('text', abstract, {'sentiment': 1})
             finalKeys = list()
             for each in response['keywords']:
                 try:
                     if float(each['relevance'])>0.5:
                         finalKeys.append(each['text'])
                         print finalKeys
                         self.render("tags.html", finalKeys=finalKeys)
                 except:
                     pass
Beispiel #38
0
def sentiment_alchemy(url):
    alchemyapi = AlchemyAPI()

    response = alchemyapi.sentiment('url', url)
    response['usage'] = None

    if response['status'] == 'OK':
        print('## Response Object ##')
        print(json.dumps(response, indent=4))

        print('')
        print('## Author ##')
        print('author: ', response.get('author', ''))
        print('')
    else:
        print('Error in author extraction call: ', response['statusInfo'])

    response = alchemyapi.keywords('url', url)
    del (response['usage'])

    if response['status'] == 'OK':
        print('## Response Object ##')
        print(json.dumps(response, indent=4))

        print('')
        print('## Keywords ##')
        for keyword in response['keywords']:
            print('text: ', keyword['text'].encode('utf-8'))
            print('relevance: ', keyword['relevance'])
            print('sentiment: ', keyword.get('sentiment', {}).get('type', ''))
            if 'score' in keyword.get('sentiment', {}):
                print('sentiment score: ' + keyword['sentiment']['score'])
            print('')
        else:
            print('Error in keyword extaction call: ',
                  response.get('statusInfo', ''))
Beispiel #39
0
print('Checking entities . . . ')
response = alchemyapi.entities('text', test_text);
assert(response['status'] == 'OK')
response = alchemyapi.entities('html', test_html);
assert(response['status'] == 'OK')
response = alchemyapi.entities('url', test_url);
assert(response['status'] == 'OK')
response = alchemyapi.entities('random', test_url);
assert(response['status'] == 'ERROR') 	#invalid flavor
print('Entity tests complete!')
print('')


#Keywords
print('Checking keywords . . . ')
response = alchemyapi.keywords('text', test_text);
assert(response['status'] == 'OK')
response = alchemyapi.keywords('html', test_html);
assert(response['status'] == 'OK')
response = alchemyapi.keywords('url', test_url);
assert(response['status'] == 'OK')
response = alchemyapi.keywords('random', test_url);
assert(response['status'] == 'ERROR') 	#invalid flavor
print('Keyword tests complete!')
print('')




#Concepts
print('Checking concepts . . . ')
Beispiel #40
0
import os, sys, json
from alchemyapi import AlchemyAPI

alchemyapi = AlchemyAPI()

fname = sys.argv[1]

with open(fname, 'r') as f:
    Text = f.readlines()

text_file = open(fname, 'w')

for fig in range(len(Text)):
    response = alchemyapi.keywords("text", Text[fig])
    if response['status'] == 'OK':
        for keyword in response['keywords']:
            text_file.write(
                str(fig) + '\t' + str(keyword['text'].encode('utf-8')) + '\t' +
                str(keyword['relevance']) + '\n')
    else:
        print('Error in keyword extaction call: ', response['statusInfo'])

text_file.close()
Beispiel #41
0
else:
    print('Error in sentiment analysis call: ', response['statusInfo'])

print('')
print('')
print('')
print('############################################')
print('#   Keyword Extraction Example             #')
print('############################################')
print('')
print('')

print('Processing text: ', demo_text)
print('')

response = alchemyapi.keywords('text', demo_text, {'sentiment': 1})

if response['status'] == 'OK':
    print('## Response Object ##')
    print(json.dumps(response, indent=4))

    print('')
    print('## Keywords ##')
    for keyword in response['keywords']:
        print('text: ', keyword['text'])
        print('relevance: ', keyword['relevance'])
        print(
            'sentiment: ', keyword['sentiment']['type'] + ' (' +
            keyword['sentiment']['score'] + ')')
        print('')
else:
Beispiel #42
0
                    ent_rele=[]
                    ent_type=[]
                    if response['status'] == 'OK':
                        flag=1
                        for entity in response['entities']:
                            ent.append(entity['text'])
                            ent_rele.append(entity['relevance'])
                            ent_type.append(entity['type'])
                    else:
                        print('Error in entity extraction call: ', response['statusInfo'])
                    if flag==1:
                        
                            #response = json.loads(json.dumps(alchemyapi.sentiment("text", trans_text)))
                ###### GETTING AN ERROR HERE FOR SOME REASON ######
                #senti=response["docSentiment"]["type"]
                            response = json.loads(json.dumps(alchemyapi.keywords('text', trans_text, {'sentiment': 1})))
                            #size=len(response['keywords'])
                            keywords=[]
                            if response['status'] == 'OK':
                                for word in response['keywords']:
                                    keywords.append(word['text'])
                            else:
                                print('Error in entity extraction call: ', response['statusInfo'])


                            response=json.loads(json.dumps(alchemyapi.concepts("text",trans_text)))
                            #size=len(response['concepts'])
                            concept=[]
                            if response['status'] == 'OK':

                                for con in response['concepts']:
Beispiel #43
0
        mixed = None
        alchemy_response = alchemyapi.sentiment_targeted(
            'url', response['items'][i]['link'], topic)
        if alchemy_response['status'] == 'OK':
            sentiment = alchemy_response['docSentiment']['type']
            if 'score' in alchemy_response['docSentiment']:
                score = alchemy_response['docSentiment']['score']
            if 'mixed' in alchemy_response['docSentiment']:
                mixed = alchemy_response['docSentiment']['mixed']
        else:
            print j + i + 1, 'Error in targeted sentiment analysis call: ', alchemy_response[
                'statusInfo']

        # Keyword sentiment analysis
        keyword_response = alchemyapi.keywords('url',
                                               response['items'][i]['link'],
                                               {'sentiment': 1})
        key_sentiment = None
        key_score = None
        key_mixed = None
        if keyword_response['status'] == 'OK':
            if 'keywords' in keyword_response:
                for keyword in keyword_response['keywords']:
                    if keyword['text'].lower() == topic.lower():
                        if 'sentiment' in keyword:
                            key_sentiment = keyword['sentiment']['type']
                            if 'score' in keyword['sentiment']:
                                key_score = keyword['sentiment']['score']
                            if 'mixed' in keyword['sentiment']:
                                key_mixed = keyword['sentiment']['mixed']
        else:
Beispiel #44
0
                print('relevance : ', concept['relevance'])
                print('')
                concept_temp.append(concept['text'])
                con_rel_temp.append(concept['relevance'])
                """
                concept_list[title_num][conc_num] = concept['text']
                relevance_list[title_num][conc_num]=concept['relevance']
                conc_num += 1
                """

        else:
            print('Error in concept tagging call: ', response['statusInfo'])
            print('다시 시도 하거나 다른 주소를 입력하세요.')

        response2 = alchemyapi.keywords('url',
                                        input_url,
                                        options={'maxRetrieve': input_maxC})
        if response2['status'] == 'OK':
            print('## Keywords ##')
            keyword_temp = []
            key_rel_temp = []
            #rel_temp=[]
            for keyword in response2['keywords']:
                print('text: ', keyword['text'])
                print('relevance: ', keyword['relevance'])
                print('')
                keyword_temp.append(keyword['text'])
                key_rel_temp.append(keyword['relevance'])
        else:
            print('Error in keywords call: ', response2['statusInfo'])
            print('다시 시도 하거나 다른 주소를 입력하세요.')
Beispiel #45
0
import bs4
from bs4 import BeautifulSoup
import urllib2


url= "http://businessoverbroadway.com/top-10-skills-in-data-science"
page = urllib2.urlopen(url)
soup = bs4.BeautifulSoup(page.read(), 'html.parser')
mytext = soup.find('p').getText()

print mytext



#Take this text and store it in the program to use in the next step.


# Part 2

from alchemyapi import AlchemyAPI
alchemyapi = AlchemyAPI()

response = alchemyapi.keywords('text', mytext)
print "Keywords"   '\t'  '\t' "Relevance"
for kyw in response['keywords'][1:10]:
        k1=  kyw['text']
        r1=  kyw['relevance']
        print "              "
        print "--------------------------------------"
        print k1, '\t', '\t', r1
Beispiel #46
0
            con_rel_temp = []
            key_rel_temp = []
            for concept in response['concepts']:
                print('text: ', concept['text'])
                print('relevance : ', concept['relevance'])
                print('')
                concept_temp.append(concept['text'])
                con_rel_temp.append(concept['relevance'])
                """
                concept_list[title_num][conc_num] = concept['text']
                relevance_list[title_num][conc_num]=concept['relevance']
                conc_num += 1
                """

            response = alchemyapi.keywords('text',
                                           input_text,
                                           options={'maxRetrieve': input_maxC})
            print('## Keywords ##')
            rel_temp = []
            for keyword in response['keywords']:
                print('text: ', keyword['text'])
                print('relevance: ', keyword['relevance'])
                print('')
                keyword_temp.append(keyword['text'])
                key_rel_temp.append(keyword['relevance'])
        else:
            print('Error in concept tagging call: ', response['statusInfo'])

        if 'y' == input('이 결과를 저장하시겠습니까?(y/n)'):
            title_input = input('제목을 입력하세요 : ')
            concept_list.append(concept_temp)
Beispiel #47
0
# For this assignment, I will read in the text of a magazine article and pass it through the Alchemy API
# to rank the keywords. First, we must read the text in from the html with the help of beautifulsoup.

import urllib
import bs4
from alchemyapi import AlchemyAPI

url = "http://www.theatlantic.com/magazine/archive/2015/09/the-coddling-of-the-american-mind/399356/"
f = urllib.urlopen(url)
website = f.read()
soup = bs4.BeautifulSoup(website, 'html.parser')
story = soup.find('div', {'class': 'article-body'}).getText()
story

# Now that we have the text of the article, we will pass it to the Alchemy API and print out the top 10
# ranked words and their respective relevance.

alchemyapi = AlchemyAPI()
keywords = alchemyapi.keywords("text", story, {'maxRetrieve': 10})
i = 0
for keyword in keywords[u'keywords']:
    i += 1
    print 'Rank:', i, '\t Relevance:', "\t %s \t %s" % (keyword['relevance'],
                                                        keyword['text'])
Beispiel #48
0
    def trigger(self, text):
        print('Request: {0}'.format(text))
        try:
            alchemyapi = AlchemyAPI()
        except Exception as e:
            print('Could not connect to AlchemyAPI. Details: {}'.format(e))
        relations = alchemyapi.relations('text', text)

        action = None # Action to take. Only 'Play' implemented so far.
        format = None # In pratice 'album' or 'track'.
        toPlay = None # Album or track name.

        if relations['status'] == 'OK':
            print('## Object ##')
            print(json.dumps(relations, indent=4))
            for relation in relations['relations']:
                if 'action' in relation:
                    print('Action: ', relation['action']['text'].encode('utf-8'))
                    if relation['action']['verb']['text'] in ('play', 'listen'):
                        action = 'PLAY'
        else:
            print('Error in relation extaction call: ', relations['statusInfo'])

        # If no action found, we abort.
        if not action:
            print('Could not find any action to take.')
            return

        # Detect the artist to play
        artists = []
        artist = None
        keywords = alchemyapi.keywords('text', text, {'sentiment': 1})

        if keywords['status'] == 'OK':
            print('## Response Object ##')
            print(json.dumps(keywords, indent=4))

            for keyword in keywords['keywords']:
                artists.append((keyword['relevance'], extractArtists(keyword['text'].encode('utf-8'))))
        else:
            print('Error in keyword extaction call: ', keywords['statusInfo'])

        print('Action: {0}'.format(action))
        print('Artists extracted from the request: {0}'.format(' '.join([str(n)+'('+c+')' for c,n in artists])))
        for e in artists:
            if not e[1]:
                continue
            else:
                artist = e[1][-1]
                break
        if not artist:
            print('Could not find any artist.')
            return
        print('Selected artist: {0}'.format(artist))
        print('To simplify I assume you are looking for an album and not a track.')
        albums = AudioDB_getAlbums(artist)

        if not albums:
            print('Could not find any album for {0}'.format(artist))
            return

        album = None
        for name, item in albums.iteritems():
            if name in text:
                album = item
                break

        if not album:
            print('Could not find any specific album, so will try to extract a track from the full discography')
            for _, album in albums.iteritems():
                tracks = AudioDB_getTracks(album['id'])
                for track in tracks:
                    if track in text:
                        format = 'song'
                        toPlay = track
                        break
            if not toPlay:
                format = 'album'
                toPlay = albums.itervalues().next()['name'] # We assume the artist has at least one album
                print('Could not find any album or track so I will play the album {0}'.format(toPlay))
        else:
            print('Selected album is {0}. Now, check the tracks.'.format(album))
            tracks = AudioDB_getTracks(album['id'])
            for track in tracks:
                if track in text:
                    format = 'song'
                    toPlay = track
                    break

            if not toPlay:
                print("Could not find a track, so I will play the whole album.")
                toPlay = album['name']
                format = 'album'
            else:
                print('The song to play will be {0}'.format(toPlay))


        print('Selected album/track: {0}'.format(toPlay))
        hint = 'album' if format == 'album' else ''
        args = {'q': ' '.join([toPlay, artist, hint]),
                'max_results': 1}

        print('Youtube query: {0}'.format(args['q']))

        try:
            video = get_video(args)
            if video:
                video_ID = video[0]
                print('Youtube Video ID: {0}'.format(video_ID))
                uri = 'http://www.youtube.com/watch?v={0}'.format(video_ID)
                res = self.call('music.fromyoutube', uri)
            else:
                print("Could not find any stream.")
        except HttpError, e:
            print("An HTTP error %d occurred:\n%s" % (e.resp.status, e.content))
Beispiel #49
0
class AlchemyPost:

    def __init__(self, post_tumblr, post_id, consumer_key, consumer_secret, oauth_token, oauth_secret):
        self.post_tumblr = post_tumblr
        self.post_id = post_id
        self._init_tumblr(consumer_key, consumer_secret, oauth_token, oauth_secret)
        self._init_alchemy()

    def _init_tumblr(self, consumer_key, consumer_secret, oauth_token, oauth_secret):
        self._client = pytumblr.TumblrRestClient(consumer_key, consumer_secret, oauth_token, oauth_secret)    

    def _init_alchemy(self):
        self.alchemyapi = AlchemyAPI()
        self.content = {}

    def analyze_post(self):
        self.post = self._get_content_post()
        self._alchemy_entities()
        self._alchemy_keywords()
        self._alchemy_concepts()
        self._alchemy_sentiment()
        self._alchemy_relations()
        self._alchemy_category()
        self._alchemy_feeds()
        self._alchemy_taxonomy()

    def print_content(self):
        print(json.dumps(self.content, indent=4))

    def _get_content_post(self):
        print "*",
        infos = self._get_infos_post() 
        self.title = ''
        self.tags = []
        if 'tags' in infos:
            self.tags = infos['tags']
        
        if infos['type'] == 'text':
            return self._get_content_text(infos)
        if infos['type'] == 'quote':
            return self._get_content_quote(infos)
        return ''

    def _get_infos_post(self):
         infos = self._client.posts(self.post_tumblr, id=self.post_id)
         if 'posts' in infos and len(infos['posts'])>0:
            return infos['posts'][0]
         return {}

    def _get_content_text(self, infos):
        content = "<h1>" + str(infos['title']) + "</h1>"
        content += " <br>" + str(infos['body'])
        content += " <br>" + " ".join(infos['tags'])
        return content

    def _get_content_quote(self, infos):
        content = str(infos['text'])
        content += " <br>" + str(infos['source'])
        content += " <br>" + " ".join(infos['tags'])
        return content

    def _alchemy_entities(self):
        print ".",
        response = self.alchemyapi.entities('html', self.post)
        if response['status'] != 'OK':
            return False
        self.content['entities'] = response['entities']
        return True

    def _alchemy_keywords(self):
        print ".",
        response = self.alchemyapi.keywords('html', self.post)
        if response['status'] != 'OK':
            return False
        self.content['keywords'] = response['keywords']
        return True

    def _alchemy_concepts(self):
        print ".",
        response = self.alchemyapi.concepts('html', self.post)
        if response['status'] != 'OK':
            return False
        self.content['concepts'] = response['concepts']
        return True

    def _alchemy_sentiment(self):
        print ".",
        response = self.alchemyapi.sentiment('html', self.post)
        if response['status'] != 'OK':
            return False
        self.content['sentiment'] = response['docSentiment']
        return True

    def _alchemy_relations(self):
        print ".",
        response = self.alchemyapi.relations('html', self.post)
        if response['status'] != 'OK':
            return False
        self.content['relations'] = response['relations'] 
        return True

    def _alchemy_category(self):
        print ".",
        response = self.alchemyapi.category('html', self.post)
        if response['status'] != 'OK':
            return False
        self.content['category'] = response['category'] 
        self.content['score'] = response['score'] 
        return True

    def _alchemy_feeds(self):
        print ".",
        response = self.alchemyapi.feeds('html', self.post)
        if response['status'] != 'OK':
            return False
        self.content['feeds'] = response['feeds'] 
        return True

    def _alchemy_taxonomy(self):
        print ".",
        response = self.alchemyapi.taxonomy('html', self.post)
        if response['status'] != 'OK':
            return False
        self.content['taxonomy'] = response['taxonomy'] 
        return True
Beispiel #50
0
#Entities
print('Checking entities . . . ')
response = alchemyapi.entities('text', test_text)
assert (response['status'] == 'OK')
response = alchemyapi.entities('html', test_html)
assert (response['status'] == 'OK')
response = alchemyapi.entities('url', test_url)
assert (response['status'] == 'OK')
response = alchemyapi.entities('random', test_url)
assert (response['status'] == 'ERROR')  #invalid flavor
print('Entity tests complete!')
print('')

#Keywords
print('Checking keywords . . . ')
response = alchemyapi.keywords('text', test_text)
assert (response['status'] == 'OK')
response = alchemyapi.keywords('html', test_html)
assert (response['status'] == 'OK')
response = alchemyapi.keywords('url', test_url)
assert (response['status'] == 'OK')
response = alchemyapi.keywords('random', test_url)
assert (response['status'] == 'ERROR')  #invalid flavor
print('Keyword tests complete!')
print('')

#Concepts
print('Checking concepts . . . ')
response = alchemyapi.concepts('text', test_text)
assert (response['status'] == 'OK')
response = alchemyapi.concepts('html', test_html)
def nlp_process(ids,ids_hash, tgt_file_name):
    # instantiate an elasticsearch client
    #es = Elasticsearch()

    outfile = open(tgt_file_name, 'w')

    # instantiate an alchemy client
    alchemyapi = AlchemyAPI()

    # loop through each business
    for business_id in ids:
        # loop through reviews for each star
        for star in range(1,6):

            # combine review text from all reviews rated same score
            print "processing business {} for review with {} star(s)".format(business_id, star)
            data = '.'.join(ids_hash[(business_id, star)])
            alchem_keywords = []
            alchem_concepts = []

            # perform analysis only if combined review length across all the reviews
            # within a star is more than 100 characters
            if len(data) >= 100:
                # call Alchemy API combined call to determing keywords, sentiment
                # and concepts (tags)
                response = alchemyapi.keywords('text', data, {'sentiment': 1, 'maxRetrieve': 100})

                # process response
                if response['status'] == 'OK':
                    print('#Success#')

                    # process keywords
                    for keyword in response['keywords']:
                        al_temp = defaultdict()

                        al_temp['text'] = keyword['text'].encode('utf-8')
                        al_temp['relevance'] = keyword['relevance']

                        if 'sentiment' in keyword:
                            al_temp['sentiment'] = keyword['sentiment'].get('type', 'neutral')

                            if 'score' in keyword['sentiment']:
                                al_temp['score'] = keyword['sentiment']['score']

                        alchem_keywords.append(al_temp)

                    # process concepts/tags
                    #for keyword in response['concepts']:
                    #    al_temp = defaultdict()

                    #    al_temp['text'] = keyword['text'].encode('utf-8')
                    #    al_temp['relevance'] = keyword['relevance']

                    #    alchem_concepts.append(al_temp)
                else:
                    print('Error in keyword extaction call: ', response['statusInfo'])
                print len(alchem_keywords), len(alchem_concepts)

                # prepare body for insertion
                if ( len(alchem_keywords) > 0):
                    doc = {
                        "business_id" : business_id,
                        "stars": star,
                        "word_freq": alchem_keywords,
                        "topics": alchem_concepts
                    }

                    # write to a file
                    json.dump(doc, outfile)
                    outfile.write('\n')
Beispiel #52
0


print('')
print('')
print('')
print('############################################')
print('#   Keyword Extraction Example             #')
print('############################################')
print('')
print('')

print('Processing text: ', demo_text)
print('')

response = alchemyapi.keywords('text',demo_text, { 'sentiment':1 })

if response['status'] == 'OK':
	print('## Response Object ##')
	print(json.dumps(response, indent=4))


	print('')
	print('## Keywords ##')
	for keyword in response['keywords']:
		print('text: ', keyword['text'].encode('utf-8'))
		print('relevance: ', keyword['relevance'])
		print('sentiment: ', keyword['sentiment']['type']) 
		if 'score' in keyword['sentiment']:
			print('sentiment score: ' + keyword['sentiment']['score'])
		print('')
Beispiel #53
0
for p in paragraphs:
    f = re.sub(r"\[.*\]", "", p.text.encode("utf-8"))  # regex
    # text_extracted = text_extracted + p.text.encode("utf-8")
    text_extracted = text_extracted + f

web_text = text_extracted

print '\n\n'
print "URL Source: " + url
print web_text
print '\n\n'

print '#### Top 10 Keywords ####'.center(80, " ")

response = alchemyapi.keywords(
    'text', web_text, {'sentiment': 1})  # From example.py of alchemyapi module

# Processing the XML

if response['status'] == 'OK':

    print '\n'
    print "Keyword".center(50, " ") + "Relevance".center(
        10, " ") + "Sentiment".center(10, " ")
    print "=" * 80  # underline
    i = 0
    for keyword in response[
            'keywords']:  # loop to specify top ten based on relevance
        i += 1
        if i > 10: break
        if 'score' in keyword['sentiment']:
Beispiel #54
0
# -*- coding: utf-8 -*-
from alchemyapi import AlchemyAPI
import json

if __name__ == "__main__":
    text = u'Artist tackles Chicago\'s pesky pothole problem _ by filling annoying craters with mosaics\r\n\r\nCHICAGO \u2013  The perfect pothole might not exist for many people \u2014 but for mosaic artist Jim Bachor, it\'s one with a nice oval shape. Bachor began filling those potholes a little more than a year ago, after one in front of his house became a hassle.\r\n\r\nBachor doesn\'t just fill them with cement, though. He\'s turned pothole-filling into a public art project \u2014 one with a sense of humor. He fills them with mosaics.\r\n\r\n"I just think it\'s fun to add that little bit of spark into (an) issue that people moan about," says the Chicago resident, whose work also hangs in galleries. He was first drawn to the ancient art form because of its ability to last.\r\n\r\nWith orange cones and vests displaying his last name, Bachor and his helpers look official enough to shut down a street section to work on filling a pothole.\r\n\r\nBachor uses the Chicago city flag design in his pothole art. Some versions hold phone numbers to local auto repair shops, while others simply read "POTHOLE." His most recent installment north of downtown Chicago \u2014 "#21914" \u2014 pokes fun at the huge number of potholes that exist in the city.\r\n\r\nWhile his mosaic art isn\'t a permanent solution to the city\'s pothole problem, it\'s at least a small fix, he says. The city hasn\'t shut down his project, and some community members have expressed gratitude.\r\n\r\nAfter his first project, one neighbor stopped to thank him. "And then 15 minutes later, he came back with a coffee and a Danish for me," Bachor says, "and so I thought that was really cool."\r\n\r\nGerry Shaheen, a resident of Peoria, Illinois, recently stopped to ask Bachor about his work, as the artist installed a mosaic. He says Bachor and his crew are welcome anytime to fill potholes in his city, one of many hit with an especially large number of the annoying craters after a hard winter.\r\n\r\n"I\'ll pave the way for them," Shaheen said with smirk. "No pun intended."'

    #text = u'''When three Chicago financial trading firms traveled to Washington, D.C., for a crucial meeting with federal regulators, they didn\'t go alone. They brought a man schooled in the ways of the capital who had recently received $182,000 in political contributions from them: Mayor Rahm Emanuel. At stake was a proposal that would have cut into the firms\' bottom lines by making them hold large cash reserves to protect against volatility in the fledgling, high-frequency trading market they had helped pioneer. Public officials of Emanuel\'s stature rarely show up at the arcane rule-making meetings of the Commodity Futures Trading Commission. But the newly elected mayor delivered a message in the windowless conference room that day, about the important role the trading firms and others like them play in Chicago\'s economy. When regulators drafted the final rules months after the September 2011 meeting, Chicago\'s trading firms got the break they wanted. Since the Washington meeting, the three firms — DRW Trading Group, Chicago Trading Co. and Infinium Capital — have donated an additional $187,800 to Emanuel\'s campaign funds, bringing their total support of the mayor to nearly $370,000. Two other Chicago firms that stood to benefit from the rule, PEAK6 and Chopper Trading, have given an additional $334,000 to Emanuel. All the trading firms but Infinium are among an elite corps of roughly 100 donors Emanuel turned to when he first ran for mayor and is relying on again for his re-election effort. Those donors, consisting of individuals, couples, business partners and firms, are responsible for more than $14 million of the $30.5 million he has raised since fall 2010. More than half of those loyal donors have received a tangible benefit from the mayor or his administration, ranging from city contracts to help with regulators, according to a Tribune analysis. In a monthslong examination of Emanuel\'s fundraising machine, the Tribune documented his connections to investment house executives, corporate CEOs and Washington insiders who sustain him with cash even though their interests lie far beyond the policy decisions issued from the mayor\'s fifth-floor office at City Hall. Emanuel declined to be interviewed for this story. His spokeswoman said his trip to Washington in September 2011 was focused on helping the city but declined to discuss what he did on behalf of the key donors. "The mayor had a series of meetings with federal officials in Washington, as he routinely does, to advocate for Chicago priorities — including federal funding for Chicago — and regarding matters critical to our city\'s interest,\" Kelley Quinn, Emanuel\'s communications director, said in a statement. \"In these meetings he discussed federal funding to invest in housing for low-income Chicagoans, funding to invest in Chicago parkland and open space, a potential national park designation for the historic Pullman neighborhood (on) the South Side, and issues impacting Chicago\'s trading industry that employs more than 33,000 people in the city." Those Washington meetings — among more than 100 he has had during more than 30 trips as mayor — are the result of years of cultivating relationships with fellow national leaders. Emanuel\'s resume is unmatched by any mayor in the country: senior adviser to then-President Bill Clinton, top-ranking Democratic congressman from Chicago and first White House chief of staff to President Barack Obama. Add to that years as a top strategist and fundraiser for the national Democratic Party and a two-year stint as an investment banker.'''
    text = text.encode('ascii', 'ignore')

    alchemyapi = AlchemyAPI()

    response = alchemyapi.keywords('text', text, {
        'sentiment': 1,
        'maxRetrieve': 20,
        'keywordExtractMode': 'strict'
    })

    if response['status'] == "OK":
        print(json.dumps(response, indent=4))

        for keyword in response['keywords']:
            print('Keyword text:', keyword['text'].encode('utf-8'))
            print('Keyword relevance:', keyword['relevance'])
    print("**********")
    response = alchemyapi.concepts('text', text, {'maxRetrieve': 10})

    if response['status'] == "OK":
        #print(json.dumps(response, indent = 4))
        keywords = [(concept['text'].encode('ascii'),
                     float(concept['relevance']))
Beispiel #55
0
url = "http://quora-api.herokuapp.com/users/" + sys.argv[1] + "/activity"
data = requests.get(url).json()
data = data['activity']
payload = {}
#count=0
#getDocCount()
for activities in data:
    title = activities['title']
    summary = activities['summary']
    print title
    document['title'] = title
    document['summary'] = summary
    labels = al.taxonomy("text", title)
    entities = al.entities("html", summary)
    keywords = al.keywords("html", summary)
    sentiment = al.sentiment("html", summary)
    #print labels['taxonomy']
    #count+=1
    payload['entities'] = {}
    payload['keywords'] = []
    payload['sentiment'] = {}
    docNode = createDocNode(document)
    try:
        print "Yo"
        labels = labels['taxonomy'][0]['label']
        print "Yo1"
        print labels
        labels = func(labels)
        print labels
        entities = entities['entities']
def extract_keywords():
    out_file = open(sys.argv[2], "w")
    dir_name = sys.argv[1]
    loc_hash = {}
    alchemyapi = AlchemyAPI()
    for file_in in glob.glob(dir_name + "/*.txt"):
        word_count = {}
        f_obj = open(file_in)
        loc = file_in.split("/")[1].split(".")[0]
        loc_hash[loc] = {}

        text = f_obj.read()
        text = text.replace("*******************", "")
        doc_len = len(text)
        print(str(doc_len))
        text_words = text.split()
        unigrams_freq = nltk.FreqDist(text_words)

        bigrams = bgs = nltk.bigrams(text_words)
        bigrams_freq = nltk.FreqDist(bigrams)

        loc_hash[loc]["len"] = doc_len

        print(" calling api for " + file_in + ".......")
        response = alchemyapi.keywords('text', text, {'sentiment': 0})
        if response['status'] == 'OK':
            loc_hash[loc]["keywords"] = {}
            for keyword in response['keywords']:
                words = keyword['text'].split()
                if len(words) < 2:
                    words = keyword['text'].split("/")
                if (len(words) <= 2):
                    loc_hash[loc]["keywords"][keyword['text']] = 0
                if len(words) > 1:
                    for i in range(len(words)):
                        new_word = words[i]
                        if new_word not in loc_hash[loc]["keywords"]:
                            loc_hash[loc]["keywords"][new_word] = 0
                        if (i != len(words) - 1):
                            new_word = words[i] + " " + words[i + 1]
                            if new_word not in loc_hash[loc]["keywords"]:
                                loc_hash[loc]["keywords"][new_word] = 0

        else:
            print('Error in keyword extaction call for file :' + file_in +
                  " : " + response['statusInfo'])
        myRE = re.compile('^[a-zA-Z]+$')
        for keyword in loc_hash[loc]["keywords"]:
            words = keyword.split()
            if (len(words) > 1):
                if (re.match(myRE, words[0])):
                    if (re.match(myRE, words[1])):
                        t = (words[0], words[1])
                        freq = bigrams_freq[t]
                        loc_hash[loc]["keywords"][keyword] = freq
            else:
                if (re.match(myRE, words[0])):
                    freq = unigrams_freq[words[0]]
                    loc_hash[loc]["keywords"][keyword] = freq

    json.dump(loc_hash, out_file)
Beispiel #57
0
        for item in json_data[business]["1"]:
            business_ones = business_ones + " " + item
    if "2" in json_data[business]:
        for item in json_data[business]["2"]:
            business_twos = business_twos + " " + item
    if "3" in json_data[business]:
        for item in json_data[business]["3"]:
            business_threes = business_threes + " " + item
    if "4" in json_data[business]:
        for item in json_data[business]["4"]:
            business_fours = business_fours + " " + item
    if "5" in json_data[business]:
        for item in json_data[business]["5"]:
            business_fives = business_fives + " " + item

    business_response_ones = alchemyapi.keywords("text", business_ones,
                                                 {'sentiment': 1})
    business_response_twos = alchemyapi.keywords("text", business_twos,
                                                 {'sentiment': 1})
    business_response_threes = alchemyapi.keywords("text", business_threes,
                                                   {'sentiment': 1})
    business_response_fours = alchemyapi.keywords("text", business_fours,
                                                  {'sentiment': 1})
    business_response_fives = alchemyapi.keywords("text", business_fives,
                                                  {'sentiment': 1})

    data = {}
    data['1'] = business_response_ones
    data['2'] = business_response_twos
    data['3'] = business_response_threes
    data['4'] = business_response_fours
    data['5'] = business_response_fives