예제 #1
0
 def process_tweet(self, tweet):
     self.logger.debug("incoming tweet with id %s" % tweet['id'])
     try:
         tw_obj = Update.objects.get(feed__type="TW", origin_id=tweet['id'])
         return
     except Update.DoesNotExist:
         pass
     tw_user = tweet['user']
     try:
         feed = Feed.objects.get(type="TW", origin_id=tw_user['id'])
     except Feed.DoesNotExist:
         self.logger.debug("feed %s (%s) not found" % (tw_user['id'], tw_user['screen_name']))
         return
     self.logger.debug("tweet for feed %s" % unicode(feed))
     feed.last_update = datetime.datetime.now()
     feed.interest = tw_user['followers_count']
     feed.picture = tw_user['profile_image_url']
     feed.account_name = tw_user['screen_name']
     feed.save()
     tw_obj = Update(feed=feed, origin_id=tweet['id'])
     text = tweet['text']
     tw_obj.text = text.replace('&gt;', '>').replace('&lt;', '<').replace('&#39;', "'")
     date = calendar.timegm(email.utils.parsedate(tweet['created_at']))
     tw_obj.created_time = datetime.datetime.fromtimestamp(date)
     try:
         tw_obj.save()
     except Exception as e:
         self.logger.error(str(e))
         raise
예제 #2
0
 def process_tweet(self, tweet):
     self.logger.debug("incoming tweet with id %s" % tweet['id'])
     try:
         tw_obj = Update.objects.get(feed__type="TW", origin_id=tweet['id'])
         return
     except Update.DoesNotExist:
         pass
     tw_user = tweet['user']
     try:
         feed = Feed.objects.get(type="TW", origin_id=tw_user['id'])
     except Feed.DoesNotExist:
         self.logger.debug("feed %s (%s) not found" %
                           (tw_user['id'], tw_user['screen_name']))
         return
     self.logger.debug("tweet for feed %s" % unicode(feed))
     feed.last_update = datetime.datetime.now()
     feed.interest = tw_user['followers_count']
     feed.picture = tw_user['profile_image_url']
     feed.account_name = tw_user['screen_name']
     feed.save()
     tw_obj = Update(feed=feed, origin_id=tweet['id'])
     text = tweet['text']
     tw_obj.text = text.replace('&gt;',
                                '>').replace('&lt;',
                                             '<').replace('&#39;', "'")
     date = calendar.timegm(email.utils.parsedate(tweet['created_at']))
     tw_obj.created_time = datetime.datetime.fromtimestamp(date)
     try:
         tw_obj.save()
     except Exception as e:
         self.logger.error(str(e))
         raise
예제 #3
0
 def process_twitter_timeline(self, twitter, feed):
     self.stdout.write("Processing %s\n" % feed.account_name)
     user_id = feed.origin_id
     args = {'user_id': user_id, 'username': user_id, 'count': 100,
             'trim_user': True}
     tw_list = []
     while True:
         # retry two times if the twitter call fails
         for i in range(3):
             try:
                 tweets = twitter.getUserTimeline(**args)
             except ValueError:
                 if i == 2:
                     raise
                 self.stderr.write("\tGot exception, retrying.\n")
                 continue
             break
         if 'error' in tweets:
             self.stderr.write("\tERROR: %s\n" % tweets['error'])
             if not 'Rate limit exceeded' in tweets['error']:
                 feed.update_error_count += 1
                 feed.save()
             return
         if not len(tweets):
             break
         for tw in tweets:
             try:
                 mp_tw = Update.objects.get(feed=feed, origin_id=tw['id'])
             except Update.DoesNotExist:
                 tw_list.insert(0, tw)
             else:
                 break
         else:
             args['max_id'] = tw_list[0]['id'] - 1
             continue
         break
     self.stdout.write("\tNew tweets: %d\n" % len(tw_list))
     for tw in tw_list:
         mp_tw = Update(feed=feed)
         mp_tw.origin_id = tw['id']
         text = tw['text']
         mp_tw.text = text.replace('&gt;', '>').replace('&lt;', '<').replace('&#39;', "'")
         date = calendar.timegm(email.utils.parsedate(tw['created_at']))
         mp_tw.created_time = datetime.datetime.fromtimestamp(date)
         try:
             mp_tw.save()
         except:
             self.stderr.write("%s\n" % str(tw))
             raise
     feed.last_update = datetime.datetime.now()
     feed.save()
예제 #4
0
 def process_twitter_timeline(self, twitter, feed):
     self.stdout.write("Processing %s\n" % feed.account_name)
     user_id = feed.origin_id
     args = {"user_id": user_id, "username": user_id, "count": 100, "trim_user": True}
     tw_list = []
     while True:
         tweets = twitter.getUserTimeline(**args)
         if "error" in tweets:
             self.stderr.write("\tERROR: %s\n" % tweets["error"])
             if not "Rate limit exceeded" in tweets["error"]:
                 feed.update_error_count += 1
                 feed.save()
             return
         if not len(tweets):
             break
         for tw in tweets:
             try:
                 mp_tw = Update.objects.get(feed=feed, origin_id=tw["id"])
             except Update.DoesNotExist:
                 tw_list.insert(0, tw)
             else:
                 break
         else:
             args["max_id"] = tw_list[0]["id"] - 1
             continue
         break
     self.stdout.write("\tNew tweets: %d\n" % len(tw_list))
     for tw in tw_list:
         mp_tw = Update(feed=feed)
         mp_tw.origin_id = tw["id"]
         text = tw["text"]
         mp_tw.text = text.replace("&gt;", ">").replace("&lt;", "<")
         date = calendar.timegm(email.utils.parsedate(tw["created_at"]))
         mp_tw.created_time = datetime.datetime.fromtimestamp(date)
         try:
             mp_tw.save()
         except:
             self.stderr.write("%s\n" % str(tw))
             raise
     feed.last_update = datetime.datetime.now()
     feed.save()
예제 #5
0
 def process_twitter_feed(self, feed):
     self.logger.info("Processing Twitter feed %s" % feed.account_name)
     user_id = feed.origin_id
     args = {'user_id': user_id, 'username': user_id, 'count': 200,
             'trim_user': True}
     tw_list = []
     try:
         info = self.twitter.show_user(**args)
     except TwythonError as e:
         if 'Unauthorized:' in e.msg:
             raise UpdateError(e.msg, can_continue=True)
         self.logger.error("Got Twitter exception: %s" % e)
         if "Rate limit exceeded" in e.msg:
             raise UpdateError("Rate limit exceeded", can_continue=False, feed_ok=True)
         raise UpdateError(e.msg)
     feed.interest = info['followers_count']
     feed.picture = info['profile_image_url']
     feed.account_name = info['screen_name']
     while True:
         # retry two times if the twitter call fails
         for i in range(3):
             try:
                 tweets = self.twitter.get_user_timeline(**args)
             except ValueError:
                 if i == 2:
                     raise
                 self.logger.warning("Got exception, retrying.")
                 continue
             except TwythonError as e:
                 self.logger.error("Got Twitter exception: %s" % e)
                 if 'Unauthorized:' in e.msg:
                     raise UpdateError(e.msg, can_continue=True)
                 if "Rate limit exceeded" in e.msg:
                     raise UpdateError("Rate limit exceeded", can_continue=False, feed_ok=True)
                 raise UpdateError(e.msg)
             break
         if 'error' in tweets:
             self.logger.error("%s" % tweets['error'])
             if not 'Rate limit exceeded' in tweets['error']:
                 self.logger.error("Twitter error: %s" % tweets['error'])
                 raise UpdateError(tweets['error'])
             else:
                 self.logger.error("Twitter rate limit exceeded")
                 raise UpdateError(tweets['error'])
         if not len(tweets):
             break
         for tw in tweets:
             try:
                 mp_tw = Update.objects.get(feed=feed, origin_id=tw['id'])
             except Update.DoesNotExist:
                 tw_list.insert(0, tw)
             else:
                 break
         else:
             args['max_id'] = tw_list[0]['id'] - 1
             continue
         break
     self.logger.debug("New tweets: %d" % len(tw_list))
     for tw in tw_list:
         tw_obj = Update(feed=feed)
         tw_obj.origin_id = tw['id']
         text = tw['text']
         tw_obj.text = text.replace('&gt;', '>').replace('&lt;', '<').replace('&#39;', "'")
         date = calendar.timegm(email.utils.parsedate(tw['created_at']))
         tw_obj.created_time = datetime.datetime.fromtimestamp(date)
         try:
             tw_obj.save()
         except Exception as e:
             self.logger.error(str(e))
             raise
     feed.update_error_count = 0
     feed.last_update = datetime.datetime.now()
     feed.save()
예제 #6
0
 def process_twitter_feed(self, feed):
     self.logger.info("Processing Twitter feed %s" % feed.account_name)
     user_id = feed.origin_id
     args = {
         'user_id': user_id,
         'username': user_id,
         'count': 200,
         'trim_user': True
     }
     tw_list = []
     try:
         info = self.twitter.showUser(**args)
     except TwythonError as e:
         if 'Unauthorized:' in e.msg:
             raise UpdateError(e.msg, can_continue=True)
         self.logger.error("Got Twitter exception: %s" % e)
         if "Rate limit exceeded" in e.msg:
             raise UpdateError("Rate limit exceeded",
                               can_continue=False,
                               feed_ok=True)
         raise UpdateError(e.msg)
     feed.interest = info['followers_count']
     feed.picture = info['profile_image_url']
     feed.account_name = info['screen_name']
     while True:
         # retry two times if the twitter call fails
         for i in range(3):
             try:
                 tweets = self.twitter.getUserTimeline(**args)
             except ValueError:
                 if i == 2:
                     raise
                 self.logger.warning("Got exception, retrying.")
                 continue
             except TwythonError as e:
                 self.logger.error("Got Twitter exception: %s" % e)
                 if 'Unauthorized:' in e.msg:
                     raise UpdateError(e.msg, can_continue=True)
                 if "Rate limit exceeded" in e.msg:
                     raise UpdateError("Rate limit exceeded",
                                       can_continue=False,
                                       feed_ok=True)
                 raise UpdateError(e.msg)
             break
         if 'error' in tweets:
             self.logger.error("%s" % tweets['error'])
             if not 'Rate limit exceeded' in tweets['error']:
                 self.logger.error("Twitter error: %s" % tweets['error'])
                 raise UpdateError(tweets['error'])
             else:
                 self.logger.error("Twitter rate limit exceeded")
                 raise UpdateError(tweets['error'])
         if not len(tweets):
             break
         for tw in tweets:
             try:
                 mp_tw = Update.objects.get(feed=feed, origin_id=tw['id'])
             except Update.DoesNotExist:
                 tw_list.insert(0, tw)
             else:
                 break
         else:
             args['max_id'] = tw_list[0]['id'] - 1
             continue
         break
     self.logger.debug("New tweets: %d" % len(tw_list))
     for tw in tw_list:
         tw_obj = Update(feed=feed)
         tw_obj.origin_id = tw['id']
         text = tw['text']
         tw_obj.text = text.replace('&gt;',
                                    '>').replace('&lt;',
                                                 '<').replace('&#39;', "'")
         date = calendar.timegm(email.utils.parsedate(tw['created_at']))
         tw_obj.created_time = datetime.datetime.fromtimestamp(date)
         try:
             tw_obj.save()
         except Exception as e:
             self.logger.error(str(e))
             raise
     feed.update_error_count = 0
     feed.last_update = datetime.datetime.now()
     feed.save()