Пример #1
0
def search(twitter_id, params, object_type='', reply_to=None, queue=None, extra_headers=None, high_priority=False):
    """Twitter Search"""
    try:
        q = params.get('q', u'')
        key_prefix = high_priority and u'amqp.twitter.priosearch' or u'amqp.twitter.search'
        key = shorten_key(u'%s:%s:%s:%s' % (key_prefix, object_type, twitter_id, q))
        tstamp = cache.cache.get(key, None)
        if tstamp:
            _log.debug("Skipping already queued Twitter search for %s %s", object_type, twitter_id)
            return # this twitter search is already in the queue; don't requeue it
        headers = {'twitter_id':twitter_id, 'object_type':object_type}
        if extra_headers:
            headers.update(extra_headers)
        headers['cache_key'] = key
        correlation_id = uuid4().hex
        ds = DataStorage()
        ds[correlation_id] = {'timestamp':time.time()}
        msg = aq.Message(
            params,
            content_type='application/json',
            delivery_mode=_DELIVERY_MODE,
            correlation_id=correlation_id,
            timestamp=datetime.now(),
            application_headers=headers,
            reply_to=reply_to
        )
        q = queue or SearchQ(bind_queue=False)
        routing_key = high_priority and 'twitter.priosearch.search' or 'twitter.search.search'
        cache.cache.set(key, int(time.time()), 3600*5)
        q.send(msg, q.exchange_name, routing_key)
        ds.close()
        _log.debug("Twitter search initiated for twitter_id %s. High prio: %s", twitter_id, high_priority)
    except Exception, e:
        _log.error("Could not initiate Twitter search for twitter_id %s. High prio: %s", twitter_id, high_priority)
        _log.exception(e)
Пример #2
0
 def search(self, twitter_id, msg, hdrs, next, correlation_id):
     cache_key = hdrs.get('cache_key', None)
     if cache_key:
         cache.cache.delete(cache_key)
     ts = msg.properties.get('timestamp', None)
     if ts and ts < (datetime.now() - timedelta(seconds=3600*5)):
         _log.debug("Skipping stale search request for id %s", twitter_id)
         return None # Don't process stale requests
     params = msg.body
     tweets = utils.search(params)
     _log.debug("Found %s tweets for %s\n%s", len(tweets), twitter_id, msg.properties)
     ds = DataStorage()
     data = ds[correlation_id]
     if data is None:
         data = {'timestamp':time.time()}
     data['results'] = tweets
     data['timestamp'] = time.time()
     ds[correlation_id] = data
     ds.close()
     _log.debug("Sending search results for %s to %s", twitter_id, next)
     if next:
         hdrs['next'] = next
         reply = Message(
             '',
             delivery_mode=self.delivery_mode,
             content_type='text/plain',
             correlation_id=correlation_id,
             application_headers=hdrs,
             reply_to=None,
         )
         return reply
     return None
Пример #3
0
 def search(self, twitter_id, msg, hdrs, next, correlation_id):
     cache_key = hdrs.get('cache_key', None)
     if cache_key:
         cache.cache.delete(cache_key)
     ts = msg.properties.get('timestamp', None)
     if ts and ts < (datetime.now() - timedelta(seconds=3600 * 5)):
         _log.debug("Skipping stale search request for id %s", twitter_id)
         return None  # Don't process stale requests
     params = msg.body
     tweets = utils.search(params)
     _log.debug("Found %s tweets for %s\n%s", len(tweets), twitter_id,
                msg.properties)
     ds = DataStorage()
     data = ds[correlation_id]
     if data is None:
         data = {'timestamp': time.time()}
     data['results'] = tweets
     data['timestamp'] = time.time()
     ds[correlation_id] = data
     ds.close()
     _log.debug("Sending search results for %s to %s", twitter_id, next)
     if next:
         hdrs['next'] = next
         reply = Message(
             '',
             delivery_mode=self.delivery_mode,
             content_type='text/plain',
             correlation_id=correlation_id,
             application_headers=hdrs,
             reply_to=None,
         )
         return reply
     return None
Пример #4
0
 def get_friends(self, twitter_id, msg, hdrs, next, correlation_id):
     ds = DataStorage()
     data = ds[correlation_id]
     if data is None:
         _log.debug("Follower/Followee data lost. Skipping friendship computation for twitter_id %s", twitter_id)
         return None
     if 'followers' not in data or 'followees' not in data:
         _log.debug("Follower/Followee data unavailable. Skipping friendship computation for twitter_id %s", twitter_id)
         del ds[correlation_id]
         ds.close()
         return None
     followers = data['followers']
     followees = data['followees']
     del ds[correlation_id]
     ds.close()
     try:
         tw_profiles = TwitterProfile.objects.active().filter(appuser_id=unicode(twitter_id)).order_by('-pk')
         tw_profile = tw_profiles[:1].get()
     except TwitterProfile.DoesNotExist:
         return None
     friends = utils.get_friends(twitter_id, followees, followers)
     _log.debug("Twitter friends %s...", list(friends)[:5])
     self._save_twitter_friends(tw_profiles, friends)
     if next:
         hdrs['next'] = next
         reply = Message(
             '',
             delivery_mode=self.delivery_mode,
             content_type='text/plain',
             correlation_id=correlation_id,
             application_headers=hdrs,
             reply_to=None,
         )
         return reply # this will be sent to the exchange/queue to which next points
     return None
Пример #5
0
 def get_followees(self, twitter_id, msg, hdrs, next, correlation_id):
     access_token = hdrs.get('access_token', '')
     followees = utils.get_followees(twitter_id, access_token=access_token)
     ds = DataStorage()
     data = ds[correlation_id]
     if data is None:
         _log.debug(
             "Followee data lost. Skipping friendship computation for twitter_id %s",
             twitter_id)
         return None
     data['followees'] = followees
     data['timestamp'] = time.time()
     ds[correlation_id] = data
     ds.close()
     if next:
         hdrs['next'] = next
         reply = Message(
             '',
             delivery_mode=self.delivery_mode,
             content_type='text/plain',
             correlation_id=correlation_id,
             application_headers=hdrs,
             reply_to=
             'twitter/twitter.compute.save_friends',  # the function will be asked to send its reply here
         )
         return reply  # this will be sent to the exchange/queue to which next points
     return None
Пример #6
0
 def save_event_tweets(self, msg, hdrs, next):
     event_id = hdrs['twitter_id']
     do_reset = hdrs.get('do_reset', False)
     ds = DataStorage()
     data = ds[msg.correlation_id]
     if data is None:
         data = {'results': []}
     if data and 'results' in data:
         tweets = data['results']
         del ds[msg.correlation_id]
         ds.close()
         # if tweets or do_reset:
         utils.save_event_tweets(event_id, tweets, do_reset)
         _log.debug("Saved %s tweets for event %s", len(tweets), event_id)
         return None
Пример #7
0
 def save_event_tweets(self, msg, hdrs, next):
     event_id = hdrs['twitter_id']
     do_reset = hdrs.get('do_reset', False)
     ds = DataStorage()
     data = ds[msg.correlation_id]
     if data is None:
         data = {'results': []}
     if data and 'results' in data:
         tweets = data['results']
         del ds[msg.correlation_id]
         ds.close()
         # if tweets or do_reset:
         utils.save_event_tweets(event_id, tweets, do_reset)
         _log.debug("Saved %s tweets for event %s", len(tweets), event_id)
         return None
Пример #8
0
def refresh_friends(twitter_id,
                    queue=None,
                    extra_headers=None,
                    access_token=''):
    """Refresh friends of the given twitter user_id.

    The AMQP chain of functions is:

        1. twitter/twitter.get.get_followers (replies to: 2 with next reply to: 3)
        2. twitter/twitter.get.get_followees (replies to: 3 with next reply to: 4)
        3. twitter/twitter.compute.get_friends (replies to: 4 with next reply to: None)
        4. twitter/twitter.compute.save_friends (replies to: final_reply_to/None with next reply to: None)

    """
    try:
        kx = u""  # uuid4().hex
        correlation_id = shorten_key('twitter_refresh_friends:%s:%s' %
                                     (twitter_id, kx))
        ds = DataStorage()
        val = ds[correlation_id]
        if val is not None:
            if (time.time() - val['timestamp']) > THRESHOLD_SECONDS:
                del ds[correlation_id]  # clear out zombie data
            else:
                _log.debug(
                    "Another worker is refreshing friends for twitter_id %s",
                    twitter_id)
                return
        ds[correlation_id] = {'timestamp': time.time()}
        app_headers = {
            'twitter_id': twitter_id,
            'access_token': access_token,
        }
        if extra_headers:
            app_headers.update(extra_headers)
        msg = aq.Message('',
                         delivery_mode=_DELIVERY_MODE,
                         correlation_id=correlation_id,
                         application_headers=app_headers,
                         reply_to='twitter/twitter.get.get_followees')
        q = queue or GetQ(bind_queue=False)
        q.send(msg, q.exchange_name, 'twitter.get.get_followers')
        ds.close()
        _log.debug("Friend refresh initiated for twitter_id %s", twitter_id)
    except Exception, e:
        _log.debug("Could not refresh friends for twitter_id %s", twitter_id)
        _log.exception(e)
Пример #9
0
def search(twitter_id,
           params,
           object_type='',
           reply_to=None,
           queue=None,
           extra_headers=None,
           high_priority=False):
    """Twitter Search"""
    try:
        q = params.get('q', u'')
        key_prefix = high_priority and u'amqp.twitter.priosearch' or u'amqp.twitter.search'
        key = shorten_key(u'%s:%s:%s:%s' %
                          (key_prefix, object_type, twitter_id, q))
        tstamp = cache.cache.get(key, None)
        if tstamp:
            _log.debug("Skipping already queued Twitter search for %s %s",
                       object_type, twitter_id)
            return  # this twitter search is already in the queue; don't requeue it
        headers = {'twitter_id': twitter_id, 'object_type': object_type}
        if extra_headers:
            headers.update(extra_headers)
        headers['cache_key'] = key
        correlation_id = uuid4().hex
        ds = DataStorage()
        ds[correlation_id] = {'timestamp': time.time()}
        msg = aq.Message(params,
                         content_type='application/json',
                         delivery_mode=_DELIVERY_MODE,
                         correlation_id=correlation_id,
                         timestamp=datetime.now(),
                         application_headers=headers,
                         reply_to=reply_to)
        q = queue or SearchQ(bind_queue=False)
        routing_key = high_priority and 'twitter.priosearch.search' or 'twitter.search.search'
        cache.cache.set(key, int(time.time()), 3600 * 5)
        q.send(msg, q.exchange_name, routing_key)
        ds.close()
        _log.debug("Twitter search initiated for twitter_id %s. High prio: %s",
                   twitter_id, high_priority)
    except Exception, e:
        _log.error(
            "Could not initiate Twitter search for twitter_id %s. High prio: %s",
            twitter_id, high_priority)
        _log.exception(e)
Пример #10
0
def refresh_friends(twitter_id, queue=None, extra_headers=None, access_token=''):
    """Refresh friends of the given twitter user_id.

    The AMQP chain of functions is:

        1. twitter/twitter.get.get_followers (replies to: 2 with next reply to: 3)
        2. twitter/twitter.get.get_followees (replies to: 3 with next reply to: 4)
        3. twitter/twitter.compute.get_friends (replies to: 4 with next reply to: None)
        4. twitter/twitter.compute.save_friends (replies to: final_reply_to/None with next reply to: None)

    """
    try:
        kx = u"" # uuid4().hex
        correlation_id = shorten_key('twitter_refresh_friends:%s:%s' % (twitter_id, kx))
        ds = DataStorage()
        val = ds[correlation_id]
        if val is not None:
            if (time.time() - val['timestamp']) > THRESHOLD_SECONDS:
                del ds[correlation_id] # clear out zombie data
            else:
                _log.debug("Another worker is refreshing friends for twitter_id %s", twitter_id)
                return
        ds[correlation_id] = {'timestamp':time.time()}
        app_headers = {
            'twitter_id':twitter_id,
            'access_token':access_token,
        }
        if extra_headers:
            app_headers.update(extra_headers)
        msg = aq.Message(
            '',
            delivery_mode=_DELIVERY_MODE,
            correlation_id=correlation_id,
            application_headers=app_headers,
            reply_to='twitter/twitter.get.get_followees'
        )
        q = queue or GetQ(bind_queue=False)
        q.send(msg, q.exchange_name, 'twitter.get.get_followers')
        ds.close()
        _log.debug("Friend refresh initiated for twitter_id %s", twitter_id)
    except Exception, e:
        _log.debug("Could not refresh friends for twitter_id %s", twitter_id)
        _log.exception(e)
Пример #11
0
 def get_followers(self, twitter_id, msg, hdrs, next, correlation_id):
     access_token = hdrs.get('access_token', '')
     followers = utils.get_followers(twitter_id, access_token=access_token)
     ds = DataStorage()
     data = ds[correlation_id]
     if data is None:
         data = {}
     data['followers'] = followers
     data['timestamp'] = time.time()
     ds[correlation_id] = data
     ds.close()
     if next:
         hdrs['next'] = next
         reply = Message(
             '',
             delivery_mode=self.delivery_mode,
             content_type='text/plain',
             correlation_id=correlation_id,
             application_headers=hdrs,
             reply_to='twitter/twitter.compute.get_friends', # the function will be asked to send its reply here
         )
         return reply # this will be sent to the exchange/queue to which next points
     return None
Пример #12
0
 def get_friends(self, twitter_id, msg, hdrs, next, correlation_id):
     ds = DataStorage()
     data = ds[correlation_id]
     if data is None:
         _log.debug(
             "Follower/Followee data lost. Skipping friendship computation for twitter_id %s",
             twitter_id)
         return None
     if 'followers' not in data or 'followees' not in data:
         _log.debug(
             "Follower/Followee data unavailable. Skipping friendship computation for twitter_id %s",
             twitter_id)
         del ds[correlation_id]
         ds.close()
         return None
     followers = data['followers']
     followees = data['followees']
     del ds[correlation_id]
     ds.close()
     try:
         tw_profiles = TwitterProfile.objects.active().filter(
             appuser_id=unicode(twitter_id)).order_by('-pk')
         tw_profile = tw_profiles[:1].get()
     except TwitterProfile.DoesNotExist:
         return None
     friends = utils.get_friends(twitter_id, followees, followers)
     _log.debug("Twitter friends %s...", list(friends)[:5])
     self._save_twitter_friends(tw_profiles, friends)
     if next:
         hdrs['next'] = next
         reply = Message(
             '',
             delivery_mode=self.delivery_mode,
             content_type='text/plain',
             correlation_id=correlation_id,
             application_headers=hdrs,
             reply_to=None,
         )
         return reply  # this will be sent to the exchange/queue to which next points
     return None