def search(twitter_id, params, object_type='', reply_to=None, queue=None, extra_headers=None, high_priority=False):
"""Twitter Search"""
try:
q = params.get('q', u'')
key_prefix = high_priority and u'amqp.twitter.priosearch' or u'amqp.twitter.search'
key = shorten_key(u'%s:%s:%s:%s' % (key_prefix, object_type, twitter_id, q))
tstamp = cache.cache.get(key, None)
if tstamp:
_log.debug("Skipping already queued Twitter search for %s %s", object_type, twitter_id)
return # this twitter search is already in the queue; don't requeue it
headers = {'twitter_id':twitter_id, 'object_type':object_type}
if extra_headers:
headers.update(extra_headers)
headers['cache_key'] = key
correlation_id = uuid4().hex
ds = DataStorage()
ds[correlation_id] = {'timestamp':time.time()}
msg = aq.Message(
params,
content_type='application/json',
delivery_mode=_DELIVERY_MODE,
correlation_id=correlation_id,
timestamp=datetime.now(),
application_headers=headers,
reply_to=reply_to
)
q = queue or SearchQ(bind_queue=False)
routing_key = high_priority and 'twitter.priosearch.search' or 'twitter.search.search'
cache.cache.set(key, int(time.time()), 3600*5)
q.send(msg, q.exchange_name, routing_key)
ds.close()
_log.debug("Twitter search initiated for twitter_id %s. High prio: %s", twitter_id, high_priority)
except Exception, e:
_log.error("Could not initiate Twitter search for twitter_id %s. High prio: %s", twitter_id, high_priority)
_log.exception(e)
def search(self, twitter_id, msg, hdrs, next, correlation_id):
cache_key = hdrs.get('cache_key', None)
if cache_key:
cache.cache.delete(cache_key)
ts = msg.properties.get('timestamp', None)
if ts and ts < (datetime.now() - timedelta(seconds=3600*5)):
_log.debug("Skipping stale search request for id %s", twitter_id)
return None # Don't process stale requests
params = msg.body
tweets = utils.search(params)
_log.debug("Found %s tweets for %s\n%s", len(tweets), twitter_id, msg.properties)
ds = DataStorage()
data = ds[correlation_id]
if data is None:
data = {'timestamp':time.time()}
data['results'] = tweets
data['timestamp'] = time.time()
ds[correlation_id] = data
ds.close()
_log.debug("Sending search results for %s to %s", twitter_id, next)
if next:
hdrs['next'] = next
reply = Message(
'',
delivery_mode=self.delivery_mode,
content_type='text/plain',
correlation_id=correlation_id,
application_headers=hdrs,
reply_to=None,
)
return reply
return None
def search(self, twitter_id, msg, hdrs, next, correlation_id):
cache_key = hdrs.get('cache_key', None)
if cache_key:
cache.cache.delete(cache_key)
ts = msg.properties.get('timestamp', None)
if ts and ts < (datetime.now() - timedelta(seconds=3600 * 5)):
_log.debug("Skipping stale search request for id %s", twitter_id)
return None # Don't process stale requests
params = msg.body
tweets = utils.search(params)
_log.debug("Found %s tweets for %s\n%s", len(tweets), twitter_id,
msg.properties)
ds = DataStorage()
data = ds[correlation_id]
if data is None:
data = {'timestamp': time.time()}
data['results'] = tweets
data['timestamp'] = time.time()
ds[correlation_id] = data
ds.close()
_log.debug("Sending search results for %s to %s", twitter_id, next)
if next:
hdrs['next'] = next
reply = Message(
'',
delivery_mode=self.delivery_mode,
content_type='text/plain',
correlation_id=correlation_id,
application_headers=hdrs,
reply_to=None,
)
return reply
return None
def get_friends(self, twitter_id, msg, hdrs, next, correlation_id):
ds = DataStorage()
data = ds[correlation_id]
if data is None:
_log.debug("Follower/Followee data lost. Skipping friendship computation for twitter_id %s", twitter_id)
return None
if 'followers' not in data or 'followees' not in data:
_log.debug("Follower/Followee data unavailable. Skipping friendship computation for twitter_id %s", twitter_id)
del ds[correlation_id]
ds.close()
return None
followers = data['followers']
followees = data['followees']
del ds[correlation_id]
ds.close()
try:
tw_profiles = TwitterProfile.objects.active().filter(appuser_id=unicode(twitter_id)).order_by('-pk')
tw_profile = tw_profiles[:1].get()
except TwitterProfile.DoesNotExist:
return None
friends = utils.get_friends(twitter_id, followees, followers)
_log.debug("Twitter friends %s...", list(friends)[:5])
self._save_twitter_friends(tw_profiles, friends)
if next:
hdrs['next'] = next
reply = Message(
'',
delivery_mode=self.delivery_mode,
content_type='text/plain',
correlation_id=correlation_id,
application_headers=hdrs,
reply_to=None,
)
return reply # this will be sent to the exchange/queue to which next points
return None
def get_followees(self, twitter_id, msg, hdrs, next, correlation_id):
access_token = hdrs.get('access_token', '')
followees = utils.get_followees(twitter_id, access_token=access_token)
ds = DataStorage()
data = ds[correlation_id]
if data is None:
_log.debug(
"Followee data lost. Skipping friendship computation for twitter_id %s",
twitter_id)
return None
data['followees'] = followees
data['timestamp'] = time.time()
ds[correlation_id] = data
ds.close()
if next:
hdrs['next'] = next
reply = Message(
'',
delivery_mode=self.delivery_mode,
content_type='text/plain',
correlation_id=correlation_id,
application_headers=hdrs,
reply_to=
'twitter/twitter.compute.save_friends', # the function will be asked to send its reply here
)
return reply # this will be sent to the exchange/queue to which next points
return None
def save_event_tweets(self, msg, hdrs, next):
event_id = hdrs['twitter_id']
do_reset = hdrs.get('do_reset', False)
ds = DataStorage()
data = ds[msg.correlation_id]
if data is None:
data = {'results': []}
if data and 'results' in data:
tweets = data['results']
del ds[msg.correlation_id]
ds.close()
# if tweets or do_reset:
utils.save_event_tweets(event_id, tweets, do_reset)
_log.debug("Saved %s tweets for event %s", len(tweets), event_id)
return None
def save_event_tweets(self, msg, hdrs, next):
event_id = hdrs['twitter_id']
do_reset = hdrs.get('do_reset', False)
ds = DataStorage()
data = ds[msg.correlation_id]
if data is None:
data = {'results': []}
if data and 'results' in data:
tweets = data['results']
del ds[msg.correlation_id]
ds.close()
# if tweets or do_reset:
utils.save_event_tweets(event_id, tweets, do_reset)
_log.debug("Saved %s tweets for event %s", len(tweets), event_id)
return None
def refresh_friends(twitter_id,
queue=None,
extra_headers=None,
access_token=''):
"""Refresh friends of the given twitter user_id.
The AMQP chain of functions is:
1. twitter/twitter.get.get_followers (replies to: 2 with next reply to: 3)
2. twitter/twitter.get.get_followees (replies to: 3 with next reply to: 4)
3. twitter/twitter.compute.get_friends (replies to: 4 with next reply to: None)
4. twitter/twitter.compute.save_friends (replies to: final_reply_to/None with next reply to: None)
"""
try:
kx = u"" # uuid4().hex
correlation_id = shorten_key('twitter_refresh_friends:%s:%s' %
(twitter_id, kx))
ds = DataStorage()
val = ds[correlation_id]
if val is not None:
if (time.time() - val['timestamp']) > THRESHOLD_SECONDS:
del ds[correlation_id] # clear out zombie data
else:
_log.debug(
"Another worker is refreshing friends for twitter_id %s",
twitter_id)
return
ds[correlation_id] = {'timestamp': time.time()}
app_headers = {
'twitter_id': twitter_id,
'access_token': access_token,
}
if extra_headers:
app_headers.update(extra_headers)
msg = aq.Message('',
delivery_mode=_DELIVERY_MODE,
correlation_id=correlation_id,
application_headers=app_headers,
reply_to='twitter/twitter.get.get_followees')
q = queue or GetQ(bind_queue=False)
q.send(msg, q.exchange_name, 'twitter.get.get_followers')
ds.close()
_log.debug("Friend refresh initiated for twitter_id %s", twitter_id)
except Exception, e:
_log.debug("Could not refresh friends for twitter_id %s", twitter_id)
_log.exception(e)
def search(twitter_id,
params,
object_type='',
reply_to=None,
queue=None,
extra_headers=None,
high_priority=False):
"""Twitter Search"""
try:
q = params.get('q', u'')
key_prefix = high_priority and u'amqp.twitter.priosearch' or u'amqp.twitter.search'
key = shorten_key(u'%s:%s:%s:%s' %
(key_prefix, object_type, twitter_id, q))
tstamp = cache.cache.get(key, None)
if tstamp:
_log.debug("Skipping already queued Twitter search for %s %s",
object_type, twitter_id)
return # this twitter search is already in the queue; don't requeue it
headers = {'twitter_id': twitter_id, 'object_type': object_type}
if extra_headers:
headers.update(extra_headers)
headers['cache_key'] = key
correlation_id = uuid4().hex
ds = DataStorage()
ds[correlation_id] = {'timestamp': time.time()}
msg = aq.Message(params,
content_type='application/json',
delivery_mode=_DELIVERY_MODE,
correlation_id=correlation_id,
timestamp=datetime.now(),
application_headers=headers,
reply_to=reply_to)
q = queue or SearchQ(bind_queue=False)
routing_key = high_priority and 'twitter.priosearch.search' or 'twitter.search.search'
cache.cache.set(key, int(time.time()), 3600 * 5)
q.send(msg, q.exchange_name, routing_key)
ds.close()
_log.debug("Twitter search initiated for twitter_id %s. High prio: %s",
twitter_id, high_priority)
except Exception, e:
_log.error(
"Could not initiate Twitter search for twitter_id %s. High prio: %s",
twitter_id, high_priority)
_log.exception(e)
def refresh_friends(twitter_id, queue=None, extra_headers=None, access_token=''):
"""Refresh friends of the given twitter user_id.
The AMQP chain of functions is:
1. twitter/twitter.get.get_followers (replies to: 2 with next reply to: 3)
2. twitter/twitter.get.get_followees (replies to: 3 with next reply to: 4)
3. twitter/twitter.compute.get_friends (replies to: 4 with next reply to: None)
4. twitter/twitter.compute.save_friends (replies to: final_reply_to/None with next reply to: None)
"""
try:
kx = u"" # uuid4().hex
correlation_id = shorten_key('twitter_refresh_friends:%s:%s' % (twitter_id, kx))
ds = DataStorage()
val = ds[correlation_id]
if val is not None:
if (time.time() - val['timestamp']) > THRESHOLD_SECONDS:
del ds[correlation_id] # clear out zombie data
else:
_log.debug("Another worker is refreshing friends for twitter_id %s", twitter_id)
return
ds[correlation_id] = {'timestamp':time.time()}
app_headers = {
'twitter_id':twitter_id,
'access_token':access_token,
}
if extra_headers:
app_headers.update(extra_headers)
msg = aq.Message(
'',
delivery_mode=_DELIVERY_MODE,
correlation_id=correlation_id,
application_headers=app_headers,
reply_to='twitter/twitter.get.get_followees'
)
q = queue or GetQ(bind_queue=False)
q.send(msg, q.exchange_name, 'twitter.get.get_followers')
ds.close()
_log.debug("Friend refresh initiated for twitter_id %s", twitter_id)
except Exception, e:
_log.debug("Could not refresh friends for twitter_id %s", twitter_id)
_log.exception(e)
def get_followers(self, twitter_id, msg, hdrs, next, correlation_id):
access_token = hdrs.get('access_token', '')
followers = utils.get_followers(twitter_id, access_token=access_token)
ds = DataStorage()
data = ds[correlation_id]
if data is None:
data = {}
data['followers'] = followers
data['timestamp'] = time.time()
ds[correlation_id] = data
ds.close()
if next:
hdrs['next'] = next
reply = Message(
'',
delivery_mode=self.delivery_mode,
content_type='text/plain',
correlation_id=correlation_id,
application_headers=hdrs,
reply_to='twitter/twitter.compute.get_friends', # the function will be asked to send its reply here
)
return reply # this will be sent to the exchange/queue to which next points
return None
def get_friends(self, twitter_id, msg, hdrs, next, correlation_id):
ds = DataStorage()
data = ds[correlation_id]
if data is None:
_log.debug(
"Follower/Followee data lost. Skipping friendship computation for twitter_id %s",
twitter_id)
return None
if 'followers' not in data or 'followees' not in data:
_log.debug(
"Follower/Followee data unavailable. Skipping friendship computation for twitter_id %s",
twitter_id)
del ds[correlation_id]
ds.close()
return None
followers = data['followers']
followees = data['followees']
del ds[correlation_id]
ds.close()
try:
tw_profiles = TwitterProfile.objects.active().filter(
appuser_id=unicode(twitter_id)).order_by('-pk')
tw_profile = tw_profiles[:1].get()
except TwitterProfile.DoesNotExist:
return None
friends = utils.get_friends(twitter_id, followees, followers)
_log.debug("Twitter friends %s...", list(friends)[:5])
self._save_twitter_friends(tw_profiles, friends)
if next:
hdrs['next'] = next
reply = Message(
'',
delivery_mode=self.delivery_mode,
content_type='text/plain',
correlation_id=correlation_id,
application_headers=hdrs,
reply_to=None,
)
return reply # this will be sent to the exchange/queue to which next points
return None