def __init__(self, parent, id, title, size=(640, 480)):
wx.Frame.__init__(self, parent, id, title, size=size)
self.ts = TwitterSearch()
self.ts.set_search_term('beach')
self.ts.query_twitter()
self.ts.transform_tweets()
self.search_button = wx.Button(self, 4, 'Search', (0, 0), (90, 30))
self.search_text = wx.TextCtrl(self, 5, 'beach', (90, 0), (300, 30))
self.tweet_list = wx.ListBox(self,
1,
(390, 0),
(250, 480),
[t['from_user'] for t in self.ts.get_transformed_tweets()],
wx.LB_SINGLE)
self.tweet_text = wx.StaticText(self, 2, '', (0, 30), (390, 150), style=wx.ALIGN_LEFT)
self.timestamp_text = wx.StaticText(self, 3, '', (0, 190), (390, 150), style=wx.ALIGN_LEFT)
wx.EVT_LISTBOX(self, 1, self.OnSelect)
wx.EVT_BUTTON(self, 4, self.OnClick)
self.Show(True)
self.Centre()
def get(self):
TWITTER_DATETIME_PATTERN = '%a, %d %b %Y %H:%M:%S +0000'
try:
twitter_search = TwitterSearch()
filter_set = FilterSetFactory.createFilterSet()
company_name = self.request.get('company_name')
show_table = bool(self.request.get('show_table'))
query = Company.all()
query.filter('name =', company_name)
for company in query:
if show_table:
self.response.out.write("<h1>Crawler executing twitter search for [%s], since_id [%s], query [%s]</h1>" % ( company.name, company.since_id, company.query))
search_result_list = twitter_search.search(company)
if search_result_list is None:
log.info("Crawler found [0] tweets for company [%s] with since_id [%s]" % (company.name, company.since_id))
if show_table:
self.response.out.write('<h3>Crawler found zero results</h3>')
else:
log.info("Crawler found [%s] tweets for company [%s] with since_id [%s]" % (len(search_result_list),company.name, company.since_id))
if show_table:
self.response.out.write('<h3>Crawler found the following [%s] results</h3>' % len(search_result_list))
self.response.out.write('<table border="0"><tr>')
filter_set.applyFilters(search_result_list)
for tweet in search_result_list:
tweet_date = datetime.datetime.strptime(tweet['created_at'], TWITTER_DATETIME_PATTERN)
if show_table:
self.response.out.write('<td><img src="' + tweet['profile_image_url'] + '"></td>')
self.response.out.write('<td><b>' + tweet['from_user'] + '</b></td>')
self.response.out.write('<td><b>' + tweet['created_at'] + '</b></td>')
self.response.out.write('<td>' + tweet['text'] + '</td')
self.response.out.write('</tr><tr>')
company.record_count += 1
company.put()
rec = Record(company=company, source=tweet['source'], text=tweet['text'], date=tweet_date, user=tweet['from_user'], user_image_url=tweet['profile_image_url'])
rec.put()
if show_table:
self.response.out.write('</tr></table>')
else:
self.response.out.write('Done')
except DeadlineExceededError:
log.error("The crawler could not be completed in time.")
from twitter_search import TwitterSearch import sys if __name__ == '__main__': search = TwitterSearch(sys.argv[1]) search.transform_tweets(['profile_image_url']) tweet_number = 1 for tweet in search.get_transformed_tweets(): print "Tweet: %s" % tweet_number for key in search.get_transformed_tweets_keys(): print "%s: %s" % (key, tweet[key])
class WXTwitter(wx.Frame):
def __init__(self, parent, id, title, size=(640, 480)):
wx.Frame.__init__(self, parent, id, title, size=size)
self.ts = TwitterSearch()
self.ts.set_search_term('beach')
self.ts.query_twitter()
self.ts.transform_tweets()
self.search_button = wx.Button(self, 4, 'Search', (0, 0), (90, 30))
self.search_text = wx.TextCtrl(self, 5, 'beach', (90, 0), (300, 30))
self.tweet_list = wx.ListBox(self,
1,
(390, 0),
(250, 480),
[t['from_user'] for t in self.ts.get_transformed_tweets()],
wx.LB_SINGLE)
self.tweet_text = wx.StaticText(self, 2, '', (0, 30), (390, 150), style=wx.ALIGN_LEFT)
self.timestamp_text = wx.StaticText(self, 3, '', (0, 190), (390, 150), style=wx.ALIGN_LEFT)
wx.EVT_LISTBOX(self, 1, self.OnSelect)
wx.EVT_BUTTON(self, 4, self.OnClick)
self.Show(True)
self.Centre()
def OnSelect(self, event):
index = event.GetSelection()
tweet = self.ts.get_transformed_tweet_by_index(index)
self.tweet_text.Freeze()
self.tweet_text.SetLabel(tweet['text'])
self.tweet_text.Wrap(380)
self.tweet_text.Thaw()
self.timestamp_text.Freeze()
self.timestamp_text.SetLabel(tweet['created_at'])
self.timestamp_text.Wrap(380)
self.timestamp_text.Thaw()
def OnClick(self, event):
self.tweet_list.Clear()
self.ts.set_search_term(self.search_text.GetValue())
self.ts.query_twitter()
self.ts.transform_tweets()
for t in self.ts.get_transformed_tweets():
self.tweet_list.Append(t['from_user'])
def search(input):
x = TwitterSearch().search(input)
return x
import json
locations = [(40.787764, -73.962949, 'new_york'), # new york
(37.798926, -122.414257, 'san_fran'), # san fran
(34.034023, -118.242361, 'LA'), # LA
(43.696059, -79.413813, 'toronto') # Toronto
]
# locations = [(40.787764, -73.962949, 'new_york')] # new york
key_words = ["flu", "asthma", "allergy"]
# key_words = ["flu"]
twitter_connection = TwitterSearch()
NUM_RESULTS = 5000
RADIUS = 10
log = open("mining.log", "a+")
log = csv.writer(log)
for word in key_words:
for location in locations:
latitude = location[0]
longitude = location[1]
city = location[2]
search_word = word
