def process_tweet(self, tweet):
self.logger.debug("incoming tweet with id %s" % tweet['id'])
try:
tw_obj = Update.objects.get(feed__type="TW", origin_id=tweet['id'])
return
except Update.DoesNotExist:
pass
tw_user = tweet['user']
try:
feed = Feed.objects.get(type="TW", origin_id=tw_user['id'])
except Feed.DoesNotExist:
self.logger.debug("feed %s (%s) not found" % (tw_user['id'], tw_user['screen_name']))
return
self.logger.debug("tweet for feed %s" % unicode(feed))
feed.last_update = datetime.datetime.now()
feed.interest = tw_user['followers_count']
feed.picture = tw_user['profile_image_url']
feed.account_name = tw_user['screen_name']
feed.save()
tw_obj = Update(feed=feed, origin_id=tweet['id'])
text = tweet['text']
tw_obj.text = text.replace('>', '>').replace('<', '<').replace(''', "'")
date = calendar.timegm(email.utils.parsedate(tweet['created_at']))
tw_obj.created_time = datetime.datetime.fromtimestamp(date)
try:
tw_obj.save()
except Exception as e:
self.logger.error(str(e))
raise
def process_tweet(self, tweet):
self.logger.debug("incoming tweet with id %s" % tweet['id'])
try:
tw_obj = Update.objects.get(feed__type="TW", origin_id=tweet['id'])
return
except Update.DoesNotExist:
pass
tw_user = tweet['user']
try:
feed = Feed.objects.get(type="TW", origin_id=tw_user['id'])
except Feed.DoesNotExist:
self.logger.debug("feed %s (%s) not found" %
(tw_user['id'], tw_user['screen_name']))
return
self.logger.debug("tweet for feed %s" % unicode(feed))
feed.last_update = datetime.datetime.now()
feed.interest = tw_user['followers_count']
feed.picture = tw_user['profile_image_url']
feed.account_name = tw_user['screen_name']
feed.save()
tw_obj = Update(feed=feed, origin_id=tweet['id'])
text = tweet['text']
tw_obj.text = text.replace('>',
'>').replace('<',
'<').replace(''', "'")
date = calendar.timegm(email.utils.parsedate(tweet['created_at']))
tw_obj.created_time = datetime.datetime.fromtimestamp(date)
try:
tw_obj.save()
except Exception as e:
self.logger.error(str(e))
raise
def process_twitter_timeline(self, twitter, feed):
self.stdout.write("Processing %s\n" % feed.account_name)
user_id = feed.origin_id
args = {'user_id': user_id, 'username': user_id, 'count': 100,
'trim_user': True}
tw_list = []
while True:
# retry two times if the twitter call fails
for i in range(3):
try:
tweets = twitter.getUserTimeline(**args)
except ValueError:
if i == 2:
raise
self.stderr.write("\tGot exception, retrying.\n")
continue
break
if 'error' in tweets:
self.stderr.write("\tERROR: %s\n" % tweets['error'])
if not 'Rate limit exceeded' in tweets['error']:
feed.update_error_count += 1
feed.save()
return
if not len(tweets):
break
for tw in tweets:
try:
mp_tw = Update.objects.get(feed=feed, origin_id=tw['id'])
except Update.DoesNotExist:
tw_list.insert(0, tw)
else:
break
else:
args['max_id'] = tw_list[0]['id'] - 1
continue
break
self.stdout.write("\tNew tweets: %d\n" % len(tw_list))
for tw in tw_list:
mp_tw = Update(feed=feed)
mp_tw.origin_id = tw['id']
text = tw['text']
mp_tw.text = text.replace('>', '>').replace('<', '<').replace(''', "'")
date = calendar.timegm(email.utils.parsedate(tw['created_at']))
mp_tw.created_time = datetime.datetime.fromtimestamp(date)
try:
mp_tw.save()
except:
self.stderr.write("%s\n" % str(tw))
raise
feed.last_update = datetime.datetime.now()
feed.save()
def process_twitter_timeline(self, twitter, feed):
self.stdout.write("Processing %s\n" % feed.account_name)
user_id = feed.origin_id
args = {"user_id": user_id, "username": user_id, "count": 100, "trim_user": True}
tw_list = []
while True:
tweets = twitter.getUserTimeline(**args)
if "error" in tweets:
self.stderr.write("\tERROR: %s\n" % tweets["error"])
if not "Rate limit exceeded" in tweets["error"]:
feed.update_error_count += 1
feed.save()
return
if not len(tweets):
break
for tw in tweets:
try:
mp_tw = Update.objects.get(feed=feed, origin_id=tw["id"])
except Update.DoesNotExist:
tw_list.insert(0, tw)
else:
break
else:
args["max_id"] = tw_list[0]["id"] - 1
continue
break
self.stdout.write("\tNew tweets: %d\n" % len(tw_list))
for tw in tw_list:
mp_tw = Update(feed=feed)
mp_tw.origin_id = tw["id"]
text = tw["text"]
mp_tw.text = text.replace(">", ">").replace("<", "<")
date = calendar.timegm(email.utils.parsedate(tw["created_at"]))
mp_tw.created_time = datetime.datetime.fromtimestamp(date)
try:
mp_tw.save()
except:
self.stderr.write("%s\n" % str(tw))
raise
feed.last_update = datetime.datetime.now()
feed.save()
def process_twitter_feed(self, feed):
self.logger.info("Processing Twitter feed %s" % feed.account_name)
user_id = feed.origin_id
args = {'user_id': user_id, 'username': user_id, 'count': 200,
'trim_user': True}
tw_list = []
try:
info = self.twitter.show_user(**args)
except TwythonError as e:
if 'Unauthorized:' in e.msg:
raise UpdateError(e.msg, can_continue=True)
self.logger.error("Got Twitter exception: %s" % e)
if "Rate limit exceeded" in e.msg:
raise UpdateError("Rate limit exceeded", can_continue=False, feed_ok=True)
raise UpdateError(e.msg)
feed.interest = info['followers_count']
feed.picture = info['profile_image_url']
feed.account_name = info['screen_name']
while True:
# retry two times if the twitter call fails
for i in range(3):
try:
tweets = self.twitter.get_user_timeline(**args)
except ValueError:
if i == 2:
raise
self.logger.warning("Got exception, retrying.")
continue
except TwythonError as e:
self.logger.error("Got Twitter exception: %s" % e)
if 'Unauthorized:' in e.msg:
raise UpdateError(e.msg, can_continue=True)
if "Rate limit exceeded" in e.msg:
raise UpdateError("Rate limit exceeded", can_continue=False, feed_ok=True)
raise UpdateError(e.msg)
break
if 'error' in tweets:
self.logger.error("%s" % tweets['error'])
if not 'Rate limit exceeded' in tweets['error']:
self.logger.error("Twitter error: %s" % tweets['error'])
raise UpdateError(tweets['error'])
else:
self.logger.error("Twitter rate limit exceeded")
raise UpdateError(tweets['error'])
if not len(tweets):
break
for tw in tweets:
try:
mp_tw = Update.objects.get(feed=feed, origin_id=tw['id'])
except Update.DoesNotExist:
tw_list.insert(0, tw)
else:
break
else:
args['max_id'] = tw_list[0]['id'] - 1
continue
break
self.logger.debug("New tweets: %d" % len(tw_list))
for tw in tw_list:
tw_obj = Update(feed=feed)
tw_obj.origin_id = tw['id']
text = tw['text']
tw_obj.text = text.replace('>', '>').replace('<', '<').replace(''', "'")
date = calendar.timegm(email.utils.parsedate(tw['created_at']))
tw_obj.created_time = datetime.datetime.fromtimestamp(date)
try:
tw_obj.save()
except Exception as e:
self.logger.error(str(e))
raise
feed.update_error_count = 0
feed.last_update = datetime.datetime.now()
feed.save()
def process_twitter_feed(self, feed):
self.logger.info("Processing Twitter feed %s" % feed.account_name)
user_id = feed.origin_id
args = {
'user_id': user_id,
'username': user_id,
'count': 200,
'trim_user': True
}
tw_list = []
try:
info = self.twitter.showUser(**args)
except TwythonError as e:
if 'Unauthorized:' in e.msg:
raise UpdateError(e.msg, can_continue=True)
self.logger.error("Got Twitter exception: %s" % e)
if "Rate limit exceeded" in e.msg:
raise UpdateError("Rate limit exceeded",
can_continue=False,
feed_ok=True)
raise UpdateError(e.msg)
feed.interest = info['followers_count']
feed.picture = info['profile_image_url']
feed.account_name = info['screen_name']
while True:
# retry two times if the twitter call fails
for i in range(3):
try:
tweets = self.twitter.getUserTimeline(**args)
except ValueError:
if i == 2:
raise
self.logger.warning("Got exception, retrying.")
continue
except TwythonError as e:
self.logger.error("Got Twitter exception: %s" % e)
if 'Unauthorized:' in e.msg:
raise UpdateError(e.msg, can_continue=True)
if "Rate limit exceeded" in e.msg:
raise UpdateError("Rate limit exceeded",
can_continue=False,
feed_ok=True)
raise UpdateError(e.msg)
break
if 'error' in tweets:
self.logger.error("%s" % tweets['error'])
if not 'Rate limit exceeded' in tweets['error']:
self.logger.error("Twitter error: %s" % tweets['error'])
raise UpdateError(tweets['error'])
else:
self.logger.error("Twitter rate limit exceeded")
raise UpdateError(tweets['error'])
if not len(tweets):
break
for tw in tweets:
try:
mp_tw = Update.objects.get(feed=feed, origin_id=tw['id'])
except Update.DoesNotExist:
tw_list.insert(0, tw)
else:
break
else:
args['max_id'] = tw_list[0]['id'] - 1
continue
break
self.logger.debug("New tweets: %d" % len(tw_list))
for tw in tw_list:
tw_obj = Update(feed=feed)
tw_obj.origin_id = tw['id']
text = tw['text']
tw_obj.text = text.replace('>',
'>').replace('<',
'<').replace(''', "'")
date = calendar.timegm(email.utils.parsedate(tw['created_at']))
tw_obj.created_time = datetime.datetime.fromtimestamp(date)
try:
tw_obj.save()
except Exception as e:
self.logger.error(str(e))
raise
feed.update_error_count = 0
feed.last_update = datetime.datetime.now()
feed.save()
