def process_waitlist():
from server.db import redis
while True:
lock = redis.lock('locks:process_waitlist', timeout=600)
if lock.acquire(blocking=False):
try:
user_ids = wigo_db.sorted_set_range_by_score(
skey('user_queue'), 0, time(), 0, 50)
if user_ids:
for user_id in user_ids:
logger.info('unlocking user id {}'.format(user_id))
user = User.find(user_id)
if user.is_waiting():
user.status = 'active'
user.save()
# remove from wait list
wigo_db.sorted_set_remove(skey('user_queue'),
user.id,
replicate=False)
else:
break
finally:
lock.release()
def delete_conversation(cls, user, to_user):
from server.db import wigo_db
with wigo_db.transaction(commit_on_select=False):
wigo_db.sorted_set_remove(skey(user, 'conversations'), to_user.id)
wigo_db.delete(skey(user, 'conversation', to_user.id))
user.track_meta('last_message_change')
to_user.track_meta('last_message_change')
def predictions_listener(sender, instance, created):
if isinstance(instance, User):
if is_new_user(instance, created):
generate_friend_recs(instance, force=True)
elif isinstance(instance, Tap):
if Configuration.PREDICTION_IO_ENABLED:
capture_interaction.delay(instance.user_id, instance.tapped_id,
instance.created)
elif isinstance(instance, Message):
if Configuration.PREDICTION_IO_ENABLED:
capture_interaction.delay(instance.user_id,
instance.to_user_id,
instance.created)
elif isinstance(instance, Invite):
if Configuration.PREDICTION_IO_ENABLED:
capture_interaction.delay(instance.user_id,
instance.invited_id,
instance.created)
elif isinstance(instance, Friend) and instance.accepted:
if Configuration.PREDICTION_IO_ENABLED:
capture_interaction.delay(instance.user_id,
instance.friend_id,
instance.created,
action='view')
capture_interaction.delay(instance.friend_id,
instance.user_id,
instance.created,
action='view')
capture_interaction.delay(instance.user_id,
instance.friend_id,
instance.created,
action='buy')
capture_interaction.delay(instance.friend_id,
instance.user_id,
instance.created,
action='buy')
generate_friend_recs(instance.user)
generate_friend_recs(instance.friend)
wigo_db.sorted_set_remove(
skey('user', instance.user_id, 'friend', 'suggestions'),
instance.friend_id)
wigo_db.sorted_set_remove(
skey('user', instance.friend_id, 'friend', 'suggestions'),
instance.user_id)
def __clean_results(self, objects):
from server.db import wigo_db
if not objects:
return []
objects = [o for o in objects if o is not None]
objects = self.__secure_results(objects)
if self._model_class == User and self._user and self._distance:
objects = [
u for u in objects if Location.getLatLonDistance(
(self._user.group.latitude, self._user.group.longitude),
(u.group.latitude, u.group.longitude)) <= self._distance
]
elif self._model_class == Event and (self._user or self._group):
events = []
for e in objects:
if e.is_expired:
num_messages = get_cached_num_messages(
e.id, self._user.id if self._user else None)
num_attending = get_cached_num_attending(
e.id, self._user.id if self._user else None)
if num_messages == 0 or num_attending == 0:
if self._user:
logger.debug('cleaning event {} for user'.format(
e.id))
wigo_db.sorted_set_remove(
skey(self._user, 'events'), e.id)
elif self._group:
logger.debug('cleaning event {} for group'.format(
e.id))
wigo_db.sorted_set_remove(
skey(self._group, 'events'), e.id)
else:
events.append(e)
else:
events.append(e)
objects = events
return objects
def _do_generate_friend_recs(user_id,
num_friends_to_recommend=200,
force=False):
is_dev = Configuration.ENVIRONMENT == 'dev'
user = User.find(user_id)
suggestions_key = skey(user, 'friend', 'suggestions')
suggested = {}
friend_id_list = user.get_friend_ids()
friend_ids = set(friend_id_list)
deleted_suggest_ids = wigo_db.sorted_set_range(
skey(user, 'friend', 'suggestions', 'deleted'))
exclude = set(friend_id_list + user.get_blocked_ids() +
user.get_friend_request_ids() +
user.get_friend_requested_ids() + deleted_suggest_ids)
def is_limited(field, ttl=10):
last_check = user.get_meta(field)
if last_check:
last_check = datetime.utcfromtimestamp(float(last_check))
if not last_check:
return False
return last_check >= (datetime.utcnow() - timedelta(minutes=ttl))
def should_suggest(suggest_id):
if (suggest_id == user.id) or (suggest_id
in suggested) or (suggest_id
in exclude):
return False
return True
def get_num_friends_in_common(suggest_id):
with_friend_ids = set(
wigo_db.sorted_set_range(skey('user', suggest_id, 'friends'), 0,
-1))
return len(friend_ids & with_friend_ids)
p = wigo_db.redis.pipeline()
def add_friend(suggest_id, boost=0):
if boost == 0:
existing_score = suggested.get(suggest_id, -1)
if existing_score >= 10000:
boost = 10000
score = get_num_friends_in_common(suggest_id) + boost
p.zadd(suggestions_key, suggest_id, score)
suggested[suggest_id] = score
sleep(.1)
#################################################
# first clean up all the old suggestions
for suggest_id, score in wigo_db.sorted_set_iter(suggestions_key,
count=50):
if should_suggest(suggest_id):
# update the scores
boost = 10000 if score >= 10000 else 0
score = get_num_friends_in_common(suggest_id) + boost
if score != suggested.get(suggest_id, -1):
p.zadd(suggestions_key, suggest_id, score)
suggested[suggest_id] = score
else:
wigo_db.sorted_set_remove(suggestions_key,
suggest_id,
replicate=False)
##################################
# add facebook friends
if force or not is_limited('last_facebook_check', 30):
num_fb_recs = 0
facebook = Facebook(user.facebook_token, user.facebook_token_expires)
try:
token_expires = facebook.get_token_expiration()
if token_expires != user.facebook_token_expires:
user.facebook_token_expires = token_expires
user.save()
friends_to_add = set()
def iterate_facebook(next=None):
for facebook_id in facebook.get_friend_ids(next):
try:
friend = User.find(facebook_id=facebook_id)
if should_suggest(friend.id):
yield friend.id
except DoesNotExist:
pass
# iterate fb friends, starting at last stored next token
next_fb_path = user.get_meta('next_fb_friends_path')
for friend_id in iterate_facebook(next_fb_path):
friends_to_add.add(friend_id)
num_fb_recs += 1
if num_fb_recs >= 100:
break
# possibly rewrap the search
if num_fb_recs < 100 and next_fb_path:
# iterate again, without next, so starting at beginning
for friend_id in iterate_facebook():
friends_to_add.add(friend_id)
num_fb_recs += 1
if num_fb_recs >= 100:
break
# remove all the existing fb recommendations
p.zremrangebyscore(suggestions_key, 10000, '+inf')
# add these fb recommendations
for friend_id in friends_to_add:
add_friend(friend_id, 10000)
if facebook.next_path:
user.track_meta('next_fb_friends_path', facebook.next_path)
else:
user.remove_meta('next_fb_friends_path')
logger.info(
'generated {} facebook friend suggestions for user {}'.format(
num_fb_recs, user_id))
user.track_meta('last_facebook_check')
except FacebookTokenExpiredException:
logger.warn(
'error finding facebook friends to suggest for user {}, token expired'
.format(user_id))
user.track_meta('last_facebook_check')
user.facebook_token_expires = datetime.utcnow()
user.save()
except Exception, e:
logger.error('error finding facebook friends to suggest '
'for user {}, {}'.format(user_id, e.message))
def update_top_friends(user_id, interaction_scores):
last_active_window = datetime.utcnow() - timedelta(days=30)
friend_ids = wigo_db.sorted_set_rrange(skey('user', user_id, 'friends'))
friends = User.find(friend_ids)
# remove missing friends
with wigo_db.transaction(commit_on_select=False):
for index, f in enumerate(friends):
if f is None:
friend_id = friend_ids[index]
if not wigo_db.exists(skey('user', friend_id)):
wigo_db.sorted_set_remove(skey('user', user_id, 'friends'),
friend_id)
wigo_db.set_remove(
skey('user', user_id, 'friends', 'private'), friend_id)
for type in [
'top', 'alpha', 'friend_requests',
'friend_requested'
]:
wigo_db.sorted_set_remove(
skey('user', user_id, 'friends', type), friend_id)
# remove nulls
friends = [f for f in friends if f is not None]
# get all my friends last active dates
p = wigo_db.redis.pipeline()
for index, f in enumerate(friends):
p.hget(skey(f, 'meta'), 'last_active')
last_active_dates = p.execute()
for index, f in enumerate(friends):
last_active = last_active_dates[index]
last_active = datetime.utcfromtimestamp(
float(last_active)) if last_active else last_active_window
f.last_active = last_active
# bucket the friends
buckets = [[], [], [], [], []]
for f in friends:
score = interaction_scores.get(f.id, 0)
if score == 0:
buckets[0].append(f)
elif score <= 3:
buckets[1].append(f)
elif score <= 10:
buckets[2].append(f)
elif score <= 20:
buckets[3].append(f)
else:
buckets[4].append(f)
# score
def get_score(f):
score = 1
if f.gender == 'female':
score += 2
if f.last_active > last_active_window:
score += 2
return score
for b in buckets:
b.sort(key=lambda f: random() * get_score(f))
top_friends_key = skey('user', user_id, 'friends', 'top')
with wigo_db.transaction(commit_on_select=False):
score = 0
for b in buckets:
for f in b:
wigo_db.sorted_set_add(top_friends_key, f.id, score)
score += 1
def delete_user(user_id, group_id):
logger.info('deleting user {}'.format(user_id))
friend_ids = wigo_db.sorted_set_range(skey('user', user_id, 'friends'))
with wigo_db.transaction(commit_on_select=False):
# remove from attendees
for event_id, score in wigo_db.sorted_set_iter(
skey('user', user_id, 'events')):
wigo_db.sorted_set_remove(skey('event', event_id, 'attendees'),
user_id)
for friend_id in friend_ids:
wigo_db.sorted_set_remove(
skey('user', friend_id, 'event', event_id, 'attendees'),
user_id)
# remove event message votes
for message_id, score in wigo_db.sorted_set_iter(
skey('user', user_id, 'votes')):
wigo_db.sorted_set_remove(
skey('eventmessage', message_id, 'votes'), user_id)
for friend_id in friend_ids:
wigo_db.sorted_set_remove(
user_votes_key(friend_id, message_id), user_id)
# remove event messages
for message_id, score in wigo_db.sorted_set_iter(
skey('user', user_id, 'event_messages')):
message = EventMessage.find(message_id)
event_id = message.event_id
wigo_db.sorted_set_remove(skey('event', event_id, 'messages'),
message_id)
wigo_db.sorted_set_remove(
skey('event', event_id, 'messages', 'by_votes'), message_id)
for friend_id in friend_ids:
wigo_db.sorted_set_remove(
skey('user', friend_id, 'event', event_id, 'messages'),
message_id)
wigo_db.sorted_set_remove(
skey('user', friend_id, 'event', event_id, 'messages',
'by_votes'), message_id)
for friend_id in friend_ids:
# remove conversations
wigo_db.sorted_set_remove(skey('user', friend_id, 'conversations'),
user_id)
wigo_db.delete(skey('user', friend_id, 'conversation', user_id))
wigo_db.delete(skey('user', user_id, 'conversation', friend_id))
# remove friends
wigo_db.sorted_set_remove(skey('user', friend_id, 'friends'),
user_id)
wigo_db.sorted_set_remove(
skey('user', friend_id, 'friends', 'top'), user_id)
wigo_db.sorted_set_remove(
skey('user', friend_id, 'friends', 'alpha'), user_id)
wigo_db.set_remove(skey('user', friend_id, 'friends', 'private'),
user_id)
# remove friend requests
for friend_id in wigo_db.sorted_set_range(
skey('user', user_id, 'friend_requested')):
wigo_db.sorted_set_remove(
skey('user', friend_id, 'friend_requests'), user_id)
wigo_db.sorted_set_remove(
skey('user', friend_id, 'friend_requests', 'common'), user_id)
# remove messages
for message_id, score in wigo_db.sorted_set_iter(
skey('user', user_id, 'messages')):
wigo_db.delete(skey('message', message_id))
wigo_db.delete(skey('user', user_id, 'events'))
wigo_db.delete(skey('user', user_id, 'friends'))
wigo_db.delete(skey('user', user_id, 'friends', 'top'))
wigo_db.delete(skey('user', user_id, 'friends', 'private'))
wigo_db.delete(skey('user', user_id, 'friends', 'alpha'))
wigo_db.delete(skey('user', user_id, 'friends', 'friend_requested'))
wigo_db.delete(skey('user', user_id, 'friends', 'friend_requests'))
wigo_db.delete(
skey('user', user_id, 'friends', 'friend_requests', 'common'))
wigo_db.delete(skey('user', user_id, 'blocked'))
wigo_db.delete(skey('user', user_id, 'notifications'))
wigo_db.delete(skey('user', user_id, 'notifs'), replicate=False)
wigo_db.delete(skey('user', user_id, 'conversations'))
wigo_db.delete(skey('user', user_id, 'tapped'))
wigo_db.delete(skey('user', user_id, 'votes'))
wigo_db.delete(skey('user', user_id, 'messages'))
def delete(self, suggest_id):
user = g.user
wigo_db.sorted_set_remove(skey(user, 'friend', 'suggestions'), suggest_id)
wigo_db.sorted_set_add(skey(user, 'friend', 'suggestions', 'deleted'), suggest_id, time())
capture_interaction.delay(g.user.id, suggest_id, datetime.utcnow())
return {'success': True}