def get_history():
TWITTER_URL = "http://api.twitter.com/1/statuses/user_timeline.json?page=%s&count=200&screen_name=brianmckinney&include_rts=true"
log = logging.getLogger("jellyroll.providers.twitter")
done = False
page = 1
log.debug("Fetching Twitter Status")
while not done:
log.debug("Page " + str(page))
auth = tweepy.OAuthHandler(settings.TWITTER_CONSUMER_KEY, settings.TWITTER_CONSUMER_SECRET)
auth.set_access_token(settings.TWITTER_ACCESS_TOKEN, settings.TWITTER_ACCESS_TOKEN_SECRET)
api = tweepy.API(auth)
#json = utils.getjson(TWITTER_URL % page)
if len(json) == 0:
log.debug("Error: ran out of results exiting")
break
for status in json:
message = status['text']
message_text = smart_unicode(message)
url = smart_unicode(TWITTER_LINK_TEMPLATE % status['id'])
id = status['id']
# pubDate delivered as UTC
timestamp = dateutil.parser.parse(status['created_at'])
if utils.JELLYROLL_ADJUST_DATETIME:
timestamp = utils.utc_to_local_datetime(timestamp)
if not _status_exists(id):
_handle_status(id, message_text, url, timestamp)
page += 1
def update():
last_update_date = Item.objects.get_last_update_of_model(Message)
log.debug("Last update date: %s", last_update_date)
xml = utils.getxml(RECENT_STATUSES_URL % settings.TWITTER_USERNAME)
for status in xml.getiterator("item"):
message = status.find('title')
message_text = smart_unicode(message.text)
url = smart_unicode(status.find('link').text)
# pubDate delivered as UTC
timestamp = dateutil.parser.parse(status.find('pubDate').text)
if utils.JELLYROLL_ADJUST_DATETIME:
timestamp = utils.utc_to_local_datetime(timestamp)
if not _status_exists(message_text, url, timestamp):
_handle_status(message_text, url, timestamp)
def update():
last_update_date = Item.objects.get_last_update_of_model(Message)
log.debug("Last update date: %s", last_update_date)
xml = utils.getxml(RECENT_STATUSES_URL % settings.TWITTER_USERNAME)
for status in xml.getiterator("item"):
message = status.find('title')
message_text = smart_unicode(message.text)
url = smart_unicode(status.find('link').text)
# pubDate delivered as UTC
timestamp = dateutil.parser.parse(status.find('pubDate').text)
if utils.JELLYROLL_ADJUST_DATETIME:
timestamp = utils.utc_to_local_datetime(timestamp)
if not _status_exists(message_text, url, timestamp):
_handle_status(message_text, url, timestamp)
def update_message(self, data_iterator_func):
last_update_date = Item.objects.get_last_update_of_model(Message)
log.debug("Last update date: %s", last_update_date)
statuses = self.incoming['message'] = list()
for status in data_iterator_func("item"):
message = smart_unicode(status.find("title").text)
timestamp = dateutil.parser.parse(status.find('pubDate').text)
if utils.JELLYROLL_ADJUST_DATETIME:
timestamp = utils.utc_to_local_datetime(timestamp)
obj = {}
obj['message'], obj['links'], obj['tags'] = self.parse_message(message)
obj['timestamp'] = timestamp
obj['url'] = smart_unicode(status.find("link").text)
statuses.append( obj )
def update():
last_update_date = Item.objects.get_last_update_of_model(Message)
log.debug("Last update date: %s", last_update_date)
auth = tweepy.OAuthHandler(settings.TWITTER_CONSUMER_KEY, settings.TWITTER_CONSUMER_SECRET)
auth.set_access_token(settings.TWITTER_ACCESS_TOKEN, settings.TWITTER_ACCESS_TOKEN_SECRET)
api = tweepy.API(auth)
json = api.user_timeline(count=100)
for status in json:
message = status.text
message_text = smart_unicode(message)
url = smart_unicode(TWITTER_LINK_TEMPLATE % status.id_str)
id = status.id
# pubDate delivered as UTC
timestamp = dateutil.parser.parse(status.created_at.isoformat())
if utils.JELLYROLL_ADJUST_DATETIME:
timestamp = utils.utc_to_local_datetime(timestamp)
if not _status_exists(id):
_handle_status(id, message_text, url, timestamp)