def collect(filename, query, limit, sentiment, partition):
lines = set()
# Collect the data from webhose.io with the given query up to the given limit
response = webhose.search(query)
while len(response.posts) > 0 and len(lines) < limit:
# Go over the list of posts returned from the response
for post in response.posts:
# Verify that the length of the text is not too short nor too long
if 1000 > len(post.text) > 50:
# Extracting the text from the post object and clean it
text = re.sub(r'(\([^\)]+\)|(stars|rating)\s*:\s*\S+)\s*$', '',
post.text.replace('\n', '').replace('\t', ''), 0,
re.I)
# add the post-text to the lines we are going to save in the train/test file
lines.add(text.encode('utf8'))
time.sleep(2)
print 'Getting %s' % response.next
# Request the next 100 results from webhose.io
response = response.get_next()
# Build the train file (first part of the returned documents)
with open(os.path.join(resources_dir, filename + '.train'),
'a+') as train_file:
for line in list(lines)[:int((len(lines)) * partition)]:
train_file.write('%s\t%s\n' % (sentiment, line))
# Build the test file (rest of the returned documents)
with open(os.path.join(resources_dir, filename + '.test'),
'a+') as test_file:
for line in list(lines)[int((len(lines)) * partition):]:
test_file.write('%s\t%s\n' % (sentiment, line))
def search(self, query):
"""
retrieves result for the searched term from web hose
uses webhose sdk to construct query object
:param query:
:return: search result as a json string
"""
post = webhose.search(query)
# print post
return post
def search(self, query):
"""
retrieves result for the searched term from web hose
uses webhose sdk to construct query object
:param query:
:return: search result as a json string
"""
post = webhose.search(query)
# print post
return post
def fetch_news_by_location(location):
geocode = geocoder.google(location)
query = {
'location': location,
'language': 'english',
'site_type': 'news',
'thread.country':geocode.country,
}
headlines = [x.title for x in webhose.search(query=query, since=int((datetime.now() - timedelta(days=3)).strftime("%s")))]
return headlines
def search():
form = SearchForm(request.form)
s = None
if form.validate():
q = form.s.data
session['q'] = q
res = webhose.search(q)
search_res = []
for i in range(0,3):
url = BeautifulSoup(urllib2.urlopen(res.posts[i].url).read()).head.find('meta', attrs={'http-equiv':'refresh'}).get("content")[7::]
comb = call_alchemy_combined.call(url) #alchemyapi.combined("url",url)
rs1 = comb['docSentiment'] if 'docSentiment' in comb else [] #alchemyapi.sentiment("url", url)
rt1 = comb['taxonomy'] if 'taxonomy' in comb else []
rr1 = comb['relations'] if 'relations' in comb else [] #alchemyapi.relations("url", url)
rc1 = comb['concepts'] if 'concepts' in comb else []
rk1 = comb['keywords'] if 'keywords' in comb else []
re1 = comb['entities'] if 'entities' in comb else []
em1 = comb['docEmotion'] if 'docEmotion' in comb else [] #alchemyapi.emotion("url", url)
#if(rs1['status'] == "OK"):
#if(rr1['status']== "OK"):
if(comb['status'] == "OK"):
for t in rr1:
if t['subject']['text']+"-"+t['action']['lemmatized']+("-"+t['object']['text'] if 'object' in t else "") in d:
d[t['subject']['text']+"-"+t['action']['lemmatized']+("-"+t['object']['text'] if 'object' in t else "")] = 1
if 'sentiment' in d:
d['sentiment'] = rs1['score']
for t in rt1:
if t['label'] in d:
d[t['label']] = t['score']
for c in rc1:
if 'dbpedia' in rc1:
if c['dbpedia'] in d:
d[c['dbpedia']] = c['relevance']
for k in rk1:
if k['text'] in k:
d[k['text']] = k['relevance']
for e in re1:
if e['text'] in d:
d[e['text']] = e['relevance']
for e in em1:
if e in d:
d[e] = emq[e]
o = clf.predict(d)[0]
sent = Sentiment(score=rs1['score'],type=rs1['type']) if rs1 is not None and len(rs1)>0 else Sentiment(score="0",type="")
search_res.append(SearchItem(url=url,objectivity=Objectivity(score=str(o)),sentiment=sent))
s = Search(query=q,searchItems=search_res)
s.save()
session['s'] = s
else:
session['s'] = None
return render_template("search/search.html", form=form, search=session['s'])
def extract_top_persons(top_person):
top_person = top_person.title()
print "extract_top_persons for " + top_person
days_back = 30
date_days_ago = datetime.now() - timedelta(days=days_back)
top_person = top_person.lower()
posts = []
top_person = top_person.replace("\"", "") # clean
r = webhose.search("persons:\"" + top_person + "\" domain_rank:<10000", since=int(time.mktime(date_days_ago.timetuple())))
for post in r.posts:
posts.append(post)
while days_back > 0 or (len(r.posts) == 0):
print "days_back = " + str(days_back)
days_back -= 5
r = webhose.search("persons:\"" + top_person + "\" domain_rank:<10000", since=int(time.mktime(date_days_ago.timetuple())))
for post in r.posts:
posts.append(post)
return get_top_persons(posts, top_person)
def test_next(self):
"""
check that if we use the 'since' parameter from one query, that we
don't get any overlap
"""
# run a "regular" query
webhose.config(os.environ['WEBHOSE_TOKEN'])
query = webhose.Query()
query.some_terms = ('boston','red sox')
query.language = 'english'
query.site_type = 'news'
response = webhose.search(query)
# grab some stuff that we need for testing
next_ts = response.next_ts
last_post_crawled = response.posts[99].crawled_parsed
# now run our second query
response = webhose.search(query, since=response.next_ts)
self.assertGreater(response.posts[99].crawled_parsed,
last_post_crawled)
def test_next(self):
"""
check that if we use the 'since' parameter from one query, that we
don't get any overlap
"""
# run a "regular" query
webhose.config(os.environ['WEBHOSE_TOKEN'])
query = webhose.Query()
query.some_terms = ('boston', 'red sox')
query.language = 'english'
query.site_type = 'news'
response = webhose.search(query)
# grab some stuff that we need for testing
next_ts = response.next_ts
last_post_crawled = response.posts[99].crawled_parsed
# now run our second query
response = webhose.search(query, since=response.next_ts)
self.assertGreater(response.posts[99].crawled_parsed,
last_post_crawled)
def fullretrievew(terms,stage1):
#Initiate Webhose search instance, return max results
webhose.config(token = 'b88b78c1-0dac-4793-913e-7d20e0559144')
re = webhose.search(terms)
open(stage1, 'w').close()
s1 = open(stage1, "a")
i = 0
for post in re:
i = i+1
try:
s1.write("-=-=-ARTICLE " + str(i) + ": " + str(post.thread.url.encode('utf-8')) + "\n")
except:
s1.write("-=-=-ARTICLE " + str(i) + "\n")
try:
s1.write("-=-=-TITLE: " + str(post.thread.title_full.encode('utf-8')) + "\n")
except:
s1.write("-=-=-TITLE: \n")
s1.write("-=-=-SNIPPET: \n")
try:
s1.write("-=-=-TEXT: " + str(post.text.encode('utf-8')) + "\n")
except:
s1.write("-=-=-TEXT: \n")
return
!pip install webhose
import webhose
webhose.config(token=os.environ["WEBHOSE_KEY"])
## Just make a call
#posts = webhose.search("Obama performance_score:>8")
q = webhose.Query()
#q.some_terms = ['"big data"','"machine learning"']
#q.title = '"big data" OR "machine learning"'
q.phrase = '"data science" OR "machine learning"'
print q.query_string()
results = webhose.search(q.query_string() + ' performance_score:>1')
for post in results.posts:
score = post.thread.performance_score
if (score > 0):
print(post.title + ' by ' + post.thread.site + ' with score ' + str(score))
#print(post.thread.main_image)
import webhose
webhose.config(token="ce676c6c-02c7-47f4-a4e3-6f332774a976")
for post in webhose.search("github"):
print(post.title)
import webhose
webhose.config(token="35699326-6aec-4b1e-8aa4-a0794ba56819")
r = webhose.search("python java")
for i in xrange(0, 20):
print r.posts[i].title
# print "\n"
def news_search(keyword):
webhose.config(token="97029546-2c7f-4116-a16d-e88dd66f09c2")
r = webhose.search(keyword)
for i in range(1):
print(r.posts[i].title)
def search():
form = SearchForm(request.form)
s = None
if form.validate():
q = form.s.data
session['q'] = q
res = webhose.search(q)
search_res = []
for i in range(0, 3):
url = BeautifulSoup(urllib2.urlopen(
res.posts[i].url).read()).head.find('meta',
attrs={
'http-equiv': 'refresh'
}).get("content")[7::]
comb = call_alchemy_combined.call(
url) #alchemyapi.combined("url",url)
rs1 = comb['docSentiment'] if 'docSentiment' in comb else [
] #alchemyapi.sentiment("url", url)
rt1 = comb['taxonomy'] if 'taxonomy' in comb else []
rr1 = comb['relations'] if 'relations' in comb else [
] #alchemyapi.relations("url", url)
rc1 = comb['concepts'] if 'concepts' in comb else []
rk1 = comb['keywords'] if 'keywords' in comb else []
re1 = comb['entities'] if 'entities' in comb else []
em1 = comb['docEmotion'] if 'docEmotion' in comb else [
] #alchemyapi.emotion("url", url)
#if(rs1['status'] == "OK"):
#if(rr1['status']== "OK"):
if (comb['status'] == "OK"):
for t in rr1:
if t['subject']['text'] + "-" + t['action'][
'lemmatized'] + ("-" + t['object']['text']
if 'object' in t else "") in d:
d[t['subject']['text'] + "-" +
t['action']['lemmatized'] +
("-" +
t['object']['text'] if 'object' in t else "")] = 1
if 'sentiment' in d:
d['sentiment'] = rs1['score']
for t in rt1:
if t['label'] in d:
d[t['label']] = t['score']
for c in rc1:
if 'dbpedia' in rc1:
if c['dbpedia'] in d:
d[c['dbpedia']] = c['relevance']
for k in rk1:
if k['text'] in k:
d[k['text']] = k['relevance']
for e in re1:
if e['text'] in d:
d[e['text']] = e['relevance']
for e in em1:
if e in d:
d[e] = emq[e]
o = clf.predict(d)[0]
sent = Sentiment(
score=rs1['score'], type=rs1['type']
) if rs1 is not None and len(rs1) > 0 else Sentiment(score="0",
type="")
search_res.append(
SearchItem(url=url,
objectivity=Objectivity(score=str(o)),
sentiment=sent))
s = Search(query=q, searchItems=search_res)
s.save()
session['s'] = s
else:
session['s'] = None
return render_template("search/search.html",
form=form,
search=session['s'])
def runQuery():
queryResults = webhose.search(str(query))
return queryResults
response = unirest.get("https://webhose.io/search?token=f9cf7cbd-5c93-4672-8cb0-f6da249d1808&format=json&q=amazon%20OR%20AMZN&sort=relevancy&ts=1478463348741",
headers={
"Accept": "text/plain"
}
)
print response.body
"""
import webhose
webhose.config(token='f9cf7cbd-5c93-4672-8cb0-f6da249d1808')
company_list = ["PayPal"]
news_content = {}
for org in company_list:
r = webhose.search(org)
news_content[org] = {}
articleNo = 1
while True:
for post in r:
news_content[org][articleNo] = {}
if post.language == 'english' and post.published[:4] == '2016':
timestamp = post.published[:10] + post.published[11:19]
news_content[org][articleNo][timestamp] = [
post.title, post.text
]
#filename = os.getcwd() + os.path.sep + ct._DATADICTIONARY + os.path.sep + '{0}.json'.format(org)
#with open(filename,"a") as f:
# f.write(str(str(post.title) + str(timestamp) + str('*' * 25) + str(post.text)))
import time
from datetime import datetime, timedelta
webhose.config(token='c6052904-f312-436b-a6d8-d915084ac866') # backup token
days_back = 30
date_days_ago = datetime.now() - timedelta(days=days_back)
day = datetime.today().day
topic = 'facebook'
lang = 'english'
country = 'US'
# positive stories
rp = webhose.search("organization.positive:\"" + topic + "\" language:\"" +
lang + "\" thread.country:\"" + country + "\" format:\"" +
"json" + "\" domain_rank:<100000",
since=int(time.mktime(date_days_ago.timetuple())))
# calculate total number of articles by day
for post in ra:
p = post.published
m = int(p[5:7])
if m == 11:
d = int(p[8:10])
else:
d = int(p[8:10]) + 30
aa[d - day] += 1
out0 = open('all.txt', 'w')
out0.truncate()
for i in aa:
out0.write(str(i))