def getData(filename):
with open(
'C:\\Users\\gatesyp\\Documents\\GitHub\\twiddit\\tweetsRepo\\' +
filename + '.txt', 'r') as myfile:
data = myfile.read().replace('\n', ' ')
alchemyapi = AlchemyAPI()
response = alchemyapi.entities('text', data, {'sentiment': 1})
if response['status'] == 'OK':
#print(json.dumps(response, indent=4))
for entity in response['entities']:
try:
result = testGoog.searchGoog(entity['text'])
if float(entity['relevance']) > 0.4:
#print(result[0]['url'])
#print(result[1]['url'])
redditHandler.subscribe(result[0]['url'], mybiszti,
lolipop123)
redditHandler.subscribe(result[1]['url'], mybiszti,
lolipop123)
except:
print("none or code 403")
def test_alchemy():
text_1 = "The decision by the independent MP Andrew Wilkie to withdraw his support for the minority Labor government sounded dramatic but it should not further threaten its stability. When, after the 2010 election, Wilkie, Rob Oakeshott, Tony Windsor and the Greens agreed to support Labor, they gave just two guarantees: confidence and supply"
text_2 = "inflation plan, initially hailed at home and abroad as the saviour of the economy, is limping towards its first anniversary amid soaring prices, widespread shortages and a foreign payments crisis. Announced last February 28 the plan prices, fixed the value of the new Cruzado currency and ended widespread indexation of the economy in a bid to halt the country's 250 pct inflation rate. But within a year the plan has all but collapsed. \"The situation now is worse than it was. Although there was inflation, at least the economy worked,\" a leading bank economist said. The crumbling of the plan has been accompanied by a dramatic reversal in the foreign trade account. In 1984 and 1985 Brazil's annual trade surpluses had been sufficient to cover the 12 billion dlrs needed to service its 109 billion dlr foreign debt. For the first nine months of 1986 all seemed to be on target for a repeat, with monthly surpluses averaging one billion dlrs. But as exports were diverted and imports increased to avoid further domestic shortages the trade surplus plunged to 211 mln dlrs in October and since then has averaged under 150 mln. Reuter "
alchemyapi = AlchemyAPI()
response = alchemyapi.entities('text', text_2, {'sentiment': 1})
pp.pprint(response)
print len(response['entities'])
def getAnnotation(text):
annotations = spotlight.annotate(
'http://spotlight.dbpedia.org/rest/annotate',
text,
confidence=0.25,
support=40)
annotationsSorted = sorted(annotations, key=lambda k: k['similarityScore'])
setSpotlight = set(map(lambda x: x['URI'], annotationsSorted))
"""
{ u'URI': u'http://dbpedia.org/resource/People',
u'offset': 321,
u'percentageOfSecondRank': -1.0,
u'similarityScore': 0.08647863566875458,
u'support': 426, #
u'surfaceForm': u'people',
u'types': u'DBpedia:TopicalConcept'}
"""
alchemyapi = AlchemyAPI()
response = alchemyapi.entities('text', text, {'sentiment': 1})
resFilt = filter(lambda x: 'disambiguated' in x, response['entities'])
key = ['dbpedia', 'geonames', 'yago', 'opencyc']
resFilt
entitySet = set()
for r in resFilt:
for k in key:
if k in r['disambiguated']:
entitySet.add(r['disambiguated'][k])
"""
{u'count': u'1',
u'disambiguated': {u'dbpedia': u'http://dbpedia.org/resource/Kathmandu',
u'freebase': u'http://rdf.freebase.com/ns/m.04cx5',
u'geo': u'27.716666666666665 85.36666666666666',
u'geonames': u'http://sws.geonames.org/1283240/',
u'name': u'Kathmandu',
u'subType': [u'TouristAttraction'],
u'website': u'http://www.kathmandu.gov.np/',
u'yago': u'http://yago-knowledge.org/resource/Kathmandu'},
u'relevance': u'0.33',
u'sentiment': {u'type': u'neutral'},
u'text': u'Kathmandu',
u'type': u'City'},
"""
entitySet.update(setSpotlight)
return entitySet, annotationsSorted, response
def extract_text_alchemy(url):
# for content processing
# to connect with alchemy and tag the content
alchemyapi = AlchemyAPI()
resp = alchemyapi.text('url', url)
response = alchemyapi.keywords("text", resp['text'])
if "keywords" in response.keys():
keywords = response["keywords"]
else:
print "No parsed data: ", url
print response
keywords = []
return keywords
import json
import urllib
import time
import sys
import requests
import unirest
import os
import json
import simplejson
import subprocess
import shlex
import numpy as np
#import alchemyapi_python.__future__ import print_function
from alchemyapi_python.alchemyapi import AlchemyAPI
alchemyapi = AlchemyAPI()
demo_text = 'Yesterday dumb Bob destroyed my fancy iPhone in beautiful Denver, Colorado. I guess I will have to head over to the Apple Store and buy a new one.'
response = alchemyapi.entities('text', demo_text, {'sentiment': 1})
key = 'd7d457mawe4jc68nja63sjcn'
search_url1 = 'http://api.rottentomatoes.com/api/public/v1.0/movies.json?apikey='+key+'&q='
search_url2 = '&page_limit=1&page='
rev_url1='http://api.rottentomatoes.com/api/public/v1.0/movies/'
rev_url2='/reviews.json?apikey='+key+'&page_limit=50'
mpaa_conversion = {"G":0, "PG":1, "PG-13":2,"R":3,"NC-17":4}
ratings_conversion = {"A":1.0,"B":0.7,"C":0.4,"D":0.2,"E":0}
def rt_features(search_str):
search_url = search_url1+search_str+search_url2
def find_keywords(text, citycode):
text = text.encode('ascii', 'ignore')
###extractor = extract.TermExtractor()
###keywords = extractor(text)
###s1 = sorted(keywords,key=lambda term: term[1])
###s1.reverse()
###s2 = sorted(keywords,key=lambda term: term[2])
###s2.reverse()
###maxkw = 3
###s1 = [(kw[0],kw[1]) for kw in s1[0:maxkw] ]
###s2 = [(kw[0],kw[2]) for kw in s2[0:maxkw] ]
###kw = s1
###for k in s2:
### if k not in kw:
### kw.append(k)
###kw = sorted(kw,key=lambda term: term[1])
###kw.reverse()
#kw = set(s1+s2)
alchemyapi = AlchemyAPI()
###response = alchemyapi.keywords('text',text,{'sentiment':1,'maxRetrieve':5})
response = alchemyapi.concepts('text', text, {'maxRetrieve': 5})
kw = []
if response['status'] == "OK":
#kw = [(keyword['text'].encode('ascii'),float(keyword['relevance'])) for keyword in response['keywords']]
kw = [(concept['text'].encode('ascii'), float(concept['relevance']))
for concept in response['concepts']]
print kw
results = {}
isFirst = True
# all data?
# opendata.socrata.com/browse
dataSetQueryURLs = {
'EDM':
'https://data.edmonton.ca/browse?limitTo=maps&sortBy=relevance&q=%s',
'CGO':
'https://data.cityofchicago.org/browse?limitTo=maps&sortBy=relevance&q=%s',
'NYC':
'https://nycopendata.socrata.com/browse?limitTo=maps&sortBy=relevance&q=%s',
'SFO':
'https://data.sfgov.org/browse?limitTo=maps&sortBy=relevance&q=%s',
'BOS':
'https://data.cityofboston.gov/browse?limitTo=maps&sortBy=relevance&q=%s',
'MTL':
'https://montreal.demo.socrata.com/browse?limitTo=maps&sortBy=relevance&q=%s',
}
dataSetQueryURL = dataSetQueryURLs[citycode]
linkclasses = {
'EDM': 'nameLink',
'CGO': 'name',
'NYC': 'nameLink',
'SFO': 'nameLink',
'BOS': 'name',
'MTL': 'nameLink',
}
linkclass = linkclasses[citycode]
for key in kw:
text = key[0]
score = key[1]
# save score somewhere?
url = dataSetQueryURL % urllib.quote(text)
content = urllib2.urlopen(url).read()
soup = BeautifulSoup(content)
links = soup.find_all("a", class_=linkclass)
links = links[0:3]
if len(links) > 0:
results[text] = []
for link in links:
results[text].append(link['href'])
if isFirst:
content = urllib2.urlopen(results[text][0]).read()
soup = BeautifulSoup(content)
embed = soup.find(id="embed_code")
embed = str(embed.contents[0])
embed = embed.replace('_width_px', '646')
embed = embed.replace('_height_px', '760')
results['the_featured_embed_'] = mark_safe(embed)
isFirst = False
return results
def __init__(self):
logging.debug("Loading Alchemy...")
self.alchemyapi = AlchemyAPI()
def post(self):
"""
Get wufoo data from webhook
"""
# potential data coming in from wufoo.com
data = request.form
# potential data coming in from leanworkbench.com
if request.data:
lwb_data = loads(request.data)
create = lwb_data.get("create")
else:
create = False
# if creating/registering survey to user
if create:
if current_user.is_anonymous():
return dumps([{"status": 400}])
else:
url = lwb_data.get("url")
handshake = lwb_data.get("handshake")
if not url:
return jsonify(emsg="No url given")
else:
print 'attempting to add survey'
try:
name = url.split('/')[-2]
new_survey = WufooSurveyModel(
username=current_user.email,
url=url,
name=name,
handshake=handshake)
db.session.add(new_survey)
db.session.commit()
return make_response(
dumps({
"status": 200,
"msg": "Survey successfully added"
}))
except:
traceback.print_exc()
return jsonify(status=500)
# if webhook and not the user registering the survey for the first time
else:
# parse json load
entry_id = data.get("EntryId")
form_structure = data.get("FormStructure")
form_structure_dict = loads(form_structure)
created_by = form_structure_dict.get('Email')
url = form_structure_dict.get("Url")
field_structure = data.get("FieldStructure")
field_structure_dict = loads(field_structure)
fields = field_structure_dict.get("Fields")
handshake = data.get("HandshakeKey")
# get survey
survey = WufooSurveyModel.query.filter_by(wufoo_email=created_by,
name=url).all()
if not survey:
print 'survey does not exist yet'
# if survey doesn't exist yet, pass
return jsonify(status="This survey does not exist yet.")
survey = survey[-1]
survey_handshake = survey.handshake
if handshake == "":
handshake = None
if survey_handshake != handshake:
print 'handshake not equal'
return jsonify(status="Handshake invalid")
# get textareas
textareas = []
for field in fields:
print field
field_type = field.get("Type")
if field_type == "textarea":
textareas.append(field.get("ID"))
alchemyapi = AlchemyAPI(os.getenv("ALCHEMYAPI_KEY"))
for key in data:
if key in textareas:
text = data[key]
response = alchemyapi.sentiment('text', text)
if response['status'] == 'OK':
docsentiment = response.get("docSentiment")
score = docsentiment.get("score")
sent_type = docsentiment.get("type")
new_sentiment = WufooTextareaSentiment(
score=score, sentiment_type=sent_type, text=text)
survey.textareas.append(new_sentiment)
db.session.add(survey)
db.session.add(new_sentiment)
db.session.commit()
else:
print 'alchemy failed'
def __init__(self):
self.alchemyapi = AlchemyAPI()
self.alchemyapi.apikey = 'a6e464bee67aebc9f2197ecffd88d48187bf469e'
