def process_waitlist(): from server.db import redis while True: lock = redis.lock('locks:process_waitlist', timeout=600) if lock.acquire(blocking=False): try: user_ids = wigo_db.sorted_set_range_by_score( skey('user_queue'), 0, time(), 0, 50) if user_ids: for user_id in user_ids: logger.info('unlocking user id {}'.format(user_id)) user = User.find(user_id) if user.is_waiting(): user.status = 'active' user.save() # remove from wait list wigo_db.sorted_set_remove(skey('user_queue'), user.id, replicate=False) else: break finally: lock.release()
def delete_conversation(cls, user, to_user): from server.db import wigo_db with wigo_db.transaction(commit_on_select=False): wigo_db.sorted_set_remove(skey(user, 'conversations'), to_user.id) wigo_db.delete(skey(user, 'conversation', to_user.id)) user.track_meta('last_message_change') to_user.track_meta('last_message_change')
def predictions_listener(sender, instance, created): if isinstance(instance, User): if is_new_user(instance, created): generate_friend_recs(instance, force=True) elif isinstance(instance, Tap): if Configuration.PREDICTION_IO_ENABLED: capture_interaction.delay(instance.user_id, instance.tapped_id, instance.created) elif isinstance(instance, Message): if Configuration.PREDICTION_IO_ENABLED: capture_interaction.delay(instance.user_id, instance.to_user_id, instance.created) elif isinstance(instance, Invite): if Configuration.PREDICTION_IO_ENABLED: capture_interaction.delay(instance.user_id, instance.invited_id, instance.created) elif isinstance(instance, Friend) and instance.accepted: if Configuration.PREDICTION_IO_ENABLED: capture_interaction.delay(instance.user_id, instance.friend_id, instance.created, action='view') capture_interaction.delay(instance.friend_id, instance.user_id, instance.created, action='view') capture_interaction.delay(instance.user_id, instance.friend_id, instance.created, action='buy') capture_interaction.delay(instance.friend_id, instance.user_id, instance.created, action='buy') generate_friend_recs(instance.user) generate_friend_recs(instance.friend) wigo_db.sorted_set_remove( skey('user', instance.user_id, 'friend', 'suggestions'), instance.friend_id) wigo_db.sorted_set_remove( skey('user', instance.friend_id, 'friend', 'suggestions'), instance.user_id)
def __clean_results(self, objects): from server.db import wigo_db if not objects: return [] objects = [o for o in objects if o is not None] objects = self.__secure_results(objects) if self._model_class == User and self._user and self._distance: objects = [ u for u in objects if Location.getLatLonDistance( (self._user.group.latitude, self._user.group.longitude), (u.group.latitude, u.group.longitude)) <= self._distance ] elif self._model_class == Event and (self._user or self._group): events = [] for e in objects: if e.is_expired: num_messages = get_cached_num_messages( e.id, self._user.id if self._user else None) num_attending = get_cached_num_attending( e.id, self._user.id if self._user else None) if num_messages == 0 or num_attending == 0: if self._user: logger.debug('cleaning event {} for user'.format( e.id)) wigo_db.sorted_set_remove( skey(self._user, 'events'), e.id) elif self._group: logger.debug('cleaning event {} for group'.format( e.id)) wigo_db.sorted_set_remove( skey(self._group, 'events'), e.id) else: events.append(e) else: events.append(e) objects = events return objects
def _do_generate_friend_recs(user_id, num_friends_to_recommend=200, force=False): is_dev = Configuration.ENVIRONMENT == 'dev' user = User.find(user_id) suggestions_key = skey(user, 'friend', 'suggestions') suggested = {} friend_id_list = user.get_friend_ids() friend_ids = set(friend_id_list) deleted_suggest_ids = wigo_db.sorted_set_range( skey(user, 'friend', 'suggestions', 'deleted')) exclude = set(friend_id_list + user.get_blocked_ids() + user.get_friend_request_ids() + user.get_friend_requested_ids() + deleted_suggest_ids) def is_limited(field, ttl=10): last_check = user.get_meta(field) if last_check: last_check = datetime.utcfromtimestamp(float(last_check)) if not last_check: return False return last_check >= (datetime.utcnow() - timedelta(minutes=ttl)) def should_suggest(suggest_id): if (suggest_id == user.id) or (suggest_id in suggested) or (suggest_id in exclude): return False return True def get_num_friends_in_common(suggest_id): with_friend_ids = set( wigo_db.sorted_set_range(skey('user', suggest_id, 'friends'), 0, -1)) return len(friend_ids & with_friend_ids) p = wigo_db.redis.pipeline() def add_friend(suggest_id, boost=0): if boost == 0: existing_score = suggested.get(suggest_id, -1) if existing_score >= 10000: boost = 10000 score = get_num_friends_in_common(suggest_id) + boost p.zadd(suggestions_key, suggest_id, score) suggested[suggest_id] = score sleep(.1) ################################################# # first clean up all the old suggestions for suggest_id, score in wigo_db.sorted_set_iter(suggestions_key, count=50): if should_suggest(suggest_id): # update the scores boost = 10000 if score >= 10000 else 0 score = get_num_friends_in_common(suggest_id) + boost if score != suggested.get(suggest_id, -1): p.zadd(suggestions_key, suggest_id, score) suggested[suggest_id] = score else: wigo_db.sorted_set_remove(suggestions_key, suggest_id, replicate=False) ################################## # add facebook friends if force or not is_limited('last_facebook_check', 30): num_fb_recs = 0 facebook = Facebook(user.facebook_token, user.facebook_token_expires) try: token_expires = facebook.get_token_expiration() if token_expires != user.facebook_token_expires: user.facebook_token_expires = token_expires user.save() friends_to_add = set() def iterate_facebook(next=None): for facebook_id in facebook.get_friend_ids(next): try: friend = User.find(facebook_id=facebook_id) if should_suggest(friend.id): yield friend.id except DoesNotExist: pass # iterate fb friends, starting at last stored next token next_fb_path = user.get_meta('next_fb_friends_path') for friend_id in iterate_facebook(next_fb_path): friends_to_add.add(friend_id) num_fb_recs += 1 if num_fb_recs >= 100: break # possibly rewrap the search if num_fb_recs < 100 and next_fb_path: # iterate again, without next, so starting at beginning for friend_id in iterate_facebook(): friends_to_add.add(friend_id) num_fb_recs += 1 if num_fb_recs >= 100: break # remove all the existing fb recommendations p.zremrangebyscore(suggestions_key, 10000, '+inf') # add these fb recommendations for friend_id in friends_to_add: add_friend(friend_id, 10000) if facebook.next_path: user.track_meta('next_fb_friends_path', facebook.next_path) else: user.remove_meta('next_fb_friends_path') logger.info( 'generated {} facebook friend suggestions for user {}'.format( num_fb_recs, user_id)) user.track_meta('last_facebook_check') except FacebookTokenExpiredException: logger.warn( 'error finding facebook friends to suggest for user {}, token expired' .format(user_id)) user.track_meta('last_facebook_check') user.facebook_token_expires = datetime.utcnow() user.save() except Exception, e: logger.error('error finding facebook friends to suggest ' 'for user {}, {}'.format(user_id, e.message))
def update_top_friends(user_id, interaction_scores): last_active_window = datetime.utcnow() - timedelta(days=30) friend_ids = wigo_db.sorted_set_rrange(skey('user', user_id, 'friends')) friends = User.find(friend_ids) # remove missing friends with wigo_db.transaction(commit_on_select=False): for index, f in enumerate(friends): if f is None: friend_id = friend_ids[index] if not wigo_db.exists(skey('user', friend_id)): wigo_db.sorted_set_remove(skey('user', user_id, 'friends'), friend_id) wigo_db.set_remove( skey('user', user_id, 'friends', 'private'), friend_id) for type in [ 'top', 'alpha', 'friend_requests', 'friend_requested' ]: wigo_db.sorted_set_remove( skey('user', user_id, 'friends', type), friend_id) # remove nulls friends = [f for f in friends if f is not None] # get all my friends last active dates p = wigo_db.redis.pipeline() for index, f in enumerate(friends): p.hget(skey(f, 'meta'), 'last_active') last_active_dates = p.execute() for index, f in enumerate(friends): last_active = last_active_dates[index] last_active = datetime.utcfromtimestamp( float(last_active)) if last_active else last_active_window f.last_active = last_active # bucket the friends buckets = [[], [], [], [], []] for f in friends: score = interaction_scores.get(f.id, 0) if score == 0: buckets[0].append(f) elif score <= 3: buckets[1].append(f) elif score <= 10: buckets[2].append(f) elif score <= 20: buckets[3].append(f) else: buckets[4].append(f) # score def get_score(f): score = 1 if f.gender == 'female': score += 2 if f.last_active > last_active_window: score += 2 return score for b in buckets: b.sort(key=lambda f: random() * get_score(f)) top_friends_key = skey('user', user_id, 'friends', 'top') with wigo_db.transaction(commit_on_select=False): score = 0 for b in buckets: for f in b: wigo_db.sorted_set_add(top_friends_key, f.id, score) score += 1
def delete_user(user_id, group_id): logger.info('deleting user {}'.format(user_id)) friend_ids = wigo_db.sorted_set_range(skey('user', user_id, 'friends')) with wigo_db.transaction(commit_on_select=False): # remove from attendees for event_id, score in wigo_db.sorted_set_iter( skey('user', user_id, 'events')): wigo_db.sorted_set_remove(skey('event', event_id, 'attendees'), user_id) for friend_id in friend_ids: wigo_db.sorted_set_remove( skey('user', friend_id, 'event', event_id, 'attendees'), user_id) # remove event message votes for message_id, score in wigo_db.sorted_set_iter( skey('user', user_id, 'votes')): wigo_db.sorted_set_remove( skey('eventmessage', message_id, 'votes'), user_id) for friend_id in friend_ids: wigo_db.sorted_set_remove( user_votes_key(friend_id, message_id), user_id) # remove event messages for message_id, score in wigo_db.sorted_set_iter( skey('user', user_id, 'event_messages')): message = EventMessage.find(message_id) event_id = message.event_id wigo_db.sorted_set_remove(skey('event', event_id, 'messages'), message_id) wigo_db.sorted_set_remove( skey('event', event_id, 'messages', 'by_votes'), message_id) for friend_id in friend_ids: wigo_db.sorted_set_remove( skey('user', friend_id, 'event', event_id, 'messages'), message_id) wigo_db.sorted_set_remove( skey('user', friend_id, 'event', event_id, 'messages', 'by_votes'), message_id) for friend_id in friend_ids: # remove conversations wigo_db.sorted_set_remove(skey('user', friend_id, 'conversations'), user_id) wigo_db.delete(skey('user', friend_id, 'conversation', user_id)) wigo_db.delete(skey('user', user_id, 'conversation', friend_id)) # remove friends wigo_db.sorted_set_remove(skey('user', friend_id, 'friends'), user_id) wigo_db.sorted_set_remove( skey('user', friend_id, 'friends', 'top'), user_id) wigo_db.sorted_set_remove( skey('user', friend_id, 'friends', 'alpha'), user_id) wigo_db.set_remove(skey('user', friend_id, 'friends', 'private'), user_id) # remove friend requests for friend_id in wigo_db.sorted_set_range( skey('user', user_id, 'friend_requested')): wigo_db.sorted_set_remove( skey('user', friend_id, 'friend_requests'), user_id) wigo_db.sorted_set_remove( skey('user', friend_id, 'friend_requests', 'common'), user_id) # remove messages for message_id, score in wigo_db.sorted_set_iter( skey('user', user_id, 'messages')): wigo_db.delete(skey('message', message_id)) wigo_db.delete(skey('user', user_id, 'events')) wigo_db.delete(skey('user', user_id, 'friends')) wigo_db.delete(skey('user', user_id, 'friends', 'top')) wigo_db.delete(skey('user', user_id, 'friends', 'private')) wigo_db.delete(skey('user', user_id, 'friends', 'alpha')) wigo_db.delete(skey('user', user_id, 'friends', 'friend_requested')) wigo_db.delete(skey('user', user_id, 'friends', 'friend_requests')) wigo_db.delete( skey('user', user_id, 'friends', 'friend_requests', 'common')) wigo_db.delete(skey('user', user_id, 'blocked')) wigo_db.delete(skey('user', user_id, 'notifications')) wigo_db.delete(skey('user', user_id, 'notifs'), replicate=False) wigo_db.delete(skey('user', user_id, 'conversations')) wigo_db.delete(skey('user', user_id, 'tapped')) wigo_db.delete(skey('user', user_id, 'votes')) wigo_db.delete(skey('user', user_id, 'messages'))
def delete(self, suggest_id): user = g.user wigo_db.sorted_set_remove(skey(user, 'friend', 'suggestions'), suggest_id) wigo_db.sorted_set_add(skey(user, 'friend', 'suggestions', 'deleted'), suggest_id, time()) capture_interaction.delay(g.user.id, suggest_id, datetime.utcnow()) return {'success': True}