def __get_events_keys(self): if self._model_class == EventMessage: query_class = EventMessage if self._user: return [ user_eventmessages_key(self._user, event, self._by_votes) for event in self._events ] else: return [ skey(event, 'messages', 'by_votes') if self._by_votes else skey(event, 'messages') for event in self._events ] elif self._model_class == EventAttendee: query_class = User if self._user: return [ user_attendees_key(self._user, event) for event in self._events ] else: return [skey(event, 'attendees') for event in self._events] else: raise ValueError('Invalid query')
def remove_index(self): super(Friend, self).remove_index() with self.db.transaction(commit_on_select=False): def cleanup(u1, u2): self.db.sorted_set_remove(skey(u1, 'friends'), u2.id) self.db.sorted_set_remove(skey(u1, 'friends', 'top'), u2.id) self.db.sorted_set_remove(skey(u1, 'friends', 'alpha'), u2.id, replicate=False) self.db.set_remove(skey(u1, 'friends', 'private'), u2.id, replicate=False) cleanup(self.user, self.friend) cleanup(self.friend, self.user) # clean it out of the current users friend_requests and friend_requested but # leave the request on the other side of the relationship so it still seems to be pending self.db.sorted_set_remove( skey('user', self.user_id, 'friend_requested'), self.friend_id) self.db.sorted_set_remove( skey('user', self.user_id, 'friend_requests'), self.friend_id) self.db.sorted_set_remove( skey('user', self.user_id, 'friend_requests', 'common'), self.friend_id)
def process_waitlist(): from server.db import redis while True: lock = redis.lock('locks:process_waitlist', timeout=600) if lock.acquire(blocking=False): try: user_ids = wigo_db.sorted_set_range_by_score( skey('user_queue'), 0, time(), 0, 50) if user_ids: for user_id in user_ids: logger.info('unlocking user id {}'.format(user_id)) user = User.find(user_id) if user.is_waiting(): user.status = 'active' user.save() # remove from wait list wigo_db.sorted_set_remove(skey('user_queue'), user.id, replicate=False) else: break finally: lock.release()
def get_num_attending(event_id, user_id=None): """ Should only be called on expired events. """ from server.db import wigo_db key = (skey('user', user_id, 'event', event_id, 'attendees') if user_id is not None else skey('event', event_id, 'attendees')) return wigo_db.get_sorted_set_size(key)
def delete_conversation(cls, user, to_user): from server.db import wigo_db with wigo_db.transaction(commit_on_select=False): wigo_db.sorted_set_remove(skey(user, 'conversations'), to_user.id) wigo_db.delete(skey(user, 'conversation', to_user.id)) user.track_meta('last_message_change') to_user.track_meta('last_message_change')
def index(self): super(Message, self).index() with self.db.transaction(commit_on_select=False): self.db.set( skey(self.user, 'conversation', self.to_user.id, 'last_message'), self.id) self.db.set( skey(self.to_user, 'conversation', self.user.id, 'last_message'), self.id)
def cleanup(u1, u2): self.db.sorted_set_remove(skey(u1, 'friends'), u2.id) self.db.sorted_set_remove(skey(u1, 'friends', 'top'), u2.id) self.db.sorted_set_remove(skey(u1, 'friends', 'alpha'), u2.id, replicate=False) self.db.set_remove(skey(u1, 'friends', 'private'), u2.id, replicate=False)
def privacy_changed(user_id): # tell all friends about the privacy change user = User.find(user_id) with wigo_db.transaction(commit_on_select=False): for friend in user.friends_iter(): if user.privacy == 'public': wigo_db.set_remove(skey(friend, 'friends', 'private'), user_id) else: wigo_db.set_add(skey(friend, 'friends', 'private'), user_id)
def get(self, event_id, headers): message_meta = {} message_ids = wigo_db.sorted_set_range(skey('event', event_id, 'messages')) for message_id in message_ids: message_meta[message_id] = { 'num_votes': wigo_db.get_sorted_set_size(skey('eventmessage', message_id, 'votes')), 'voted': wigo_db.sorted_set_is_member(skey('eventmessage', message_id, 'votes'), g.user.id) } return message_meta, 200, headers
def update_global_events(self, group=None): if group is None: group = self.group events_key = skey('group', group.id, 'events') attendees_key = skey(self, 'attendees') if group.id == self.group_id: distance = 0 event_name_key = skey(group, Event, Event.event_key(self.name, group)) else: event_name_key = None # get the distance to this event distance = Location.getLatLonDistance((self.group.latitude, self.group.longitude), (group.latitude, group.longitude)) with self.db.transaction(commit_on_select=False): if self.privacy == 'public': if event_name_key: self.db.set(event_name_key, self.id, self.expires, self.expires) num_attending = self.db.get_sorted_set_size(attendees_key) num_messages = get_cached_num_messages(self.id) if self.is_expired else 10 if self.is_new is False and self.owner_id is not None and (num_attending == 0 or num_messages == 0): self.db.sorted_set_remove(events_key, self.id) if group.id == self.group_id and self.is_global: self.db.sorted_set_remove(skey('global', 'events'), self.id) else: # special scoring of verified, and verified global events if self.is_verified: score = get_score_key(self.expires, 0 if self.is_global else distance, 500 + num_attending) else: score = get_score_key(self.expires, distance, num_attending) self.db.sorted_set_add(events_key, self.id, score) if group.id == self.group_id and self.is_global: self.db.sorted_set_add(skey('global', 'events'), self.id, score) else: # if the event is being made private, make sure it hasn't taken the name if event_name_key: try: existing_event = self.find(group=group, name=self.name) if existing_event.id == self.id: self.db.delete(event_name_key) except DoesNotExist: pass self.db.sorted_set_remove(events_key, self.id)
def setup(u1, u2): self.db.sorted_set_add(skey(u1, 'friends'), u2.id, epoch(self.created)) self.db.sorted_set_add(skey(u1, 'friends', 'top'), u2.id, 10000) self.db.sorted_set_add(skey(u1, 'friends', 'alpha'), u2.id, prefix_score(u2.full_name.lower()), replicate=False) if u2.privacy == 'private': self.db.set_add(skey(u1, 'friends', 'private'), u2.id, replicate=False)
def index(self): super(EventMessage, self).index() event = self.event # record to the global by_votes sort with self.db.transaction(commit_on_select=False): num_votes = self.db.get_sorted_set_size(skey(self, 'votes')) sub_sort = epoch(self.created) / epoch(event.expires + timedelta(days=365)) by_votes_key = skey(event, 'messages', 'by_votes') self.db.sorted_set_add(by_votes_key, self.id, num_votes + sub_sort) self.db.expire(by_votes_key, self.ttl()) self.record_for_user(self.user)
def index(self): super(EventMessageVote, self).index() user = self.user message = self.message event = message.event # record the vote into the global "by_votes" sort order with self.db.transaction(commit_on_select=False): num_votes = self.db.get_sorted_set_size(skey(message, 'votes')) sub_sort = epoch() / epoch(event.expires + timedelta(days=365)) by_votes = skey(event, 'messages', 'by_votes') self.db.sorted_set_add(by_votes, self.message_id, num_votes + sub_sort, replicate=False) self.db.expire(by_votes, self.ttl())
def predictions_listener(sender, instance, created): if isinstance(instance, User): if is_new_user(instance, created): generate_friend_recs(instance, force=True) elif isinstance(instance, Tap): if Configuration.PREDICTION_IO_ENABLED: capture_interaction.delay(instance.user_id, instance.tapped_id, instance.created) elif isinstance(instance, Message): if Configuration.PREDICTION_IO_ENABLED: capture_interaction.delay(instance.user_id, instance.to_user_id, instance.created) elif isinstance(instance, Invite): if Configuration.PREDICTION_IO_ENABLED: capture_interaction.delay(instance.user_id, instance.invited_id, instance.created) elif isinstance(instance, Friend) and instance.accepted: if Configuration.PREDICTION_IO_ENABLED: capture_interaction.delay(instance.user_id, instance.friend_id, instance.created, action='view') capture_interaction.delay(instance.friend_id, instance.user_id, instance.created, action='view') capture_interaction.delay(instance.user_id, instance.friend_id, instance.created, action='buy') capture_interaction.delay(instance.friend_id, instance.user_id, instance.created, action='buy') generate_friend_recs(instance.user) generate_friend_recs(instance.friend) wigo_db.sorted_set_remove( skey('user', instance.user_id, 'friend', 'suggestions'), instance.friend_id) wigo_db.sorted_set_remove( skey('user', instance.friend_id, 'friend', 'suggestions'), instance.user_id)
def __get_notifications(self): key = skey(self._user, 'notifs') min = self._min or '-inf' max = self._max or '+inf' count = self.db.get_sorted_set_size(key, min, max) if count == 0: return 0, self._page, [] pages = int(math.ceil(float(count) / self._limit)) start = (self._page - 1) * self._limit results = self.db.sorted_set_rrange_by_score(key, max, min, start, self._limit, dt=dict) instances = [] for result in results: instance = Notification(result) instance.prepared() instances.append(instance) return count, self._page, instances
def post(self, user_id): referred_by_id = self.get_id_field('referred_by_id') referred_by = User.find(referred_by_id) referred_key = skey(referred_by, 'referred') # record into the referrers list of users they referred if not wigo_db.sorted_set_is_member(referred_key, g.user.id): wigo_db.sorted_set_add(referred_key, g.user.id, time()) # record into the referrers rankings for the month now = datetime.now(Configuration.ANALYTICS_TIMEZONE) wigo_db.sorted_set_incr_score( skey('referrers', now.month, now.year), referred_by.id) wigo_db.sorted_set_incr_score(skey('referrers'), referred_by.id) return {'success': True}
def new_user(user_id, score=None): user = User.find(user_id) if user.status != 'waiting': return user_queue_key = skey('user_queue') if score is None: if user_id < 130000: score = time() + randint(60, 60 * 30) else: last_waiting = wigo_db.sorted_set_range(user_queue_key, -1, -1, True) if last_waiting: last_waiting_score = last_waiting[0][1] if last_waiting_score > (time() + (60 * 60 * 6)): score = last_waiting_score + 10 else: score = last_waiting_score + randint(0, 60) else: score = time() + randint(120, 60 * 20) wigo_db.sorted_set_add(user_queue_key, user_id, score, replicate=False) scheduler.schedule(datetime.utcfromtimestamp(score), process_waitlist, result_ttl=0, timeout=600)
def get(self): user = g.user generate_friend_recs(user) count, page, users = self.select().key(skey(user, 'friend', 'suggestions')).execute() if count == 0: count, page, users = self.select().group(g.group).execute() if count == 0: count, page, users = self.select().execute() fb, other = partition(users, lambda u: True if hasattr(u, 'score') and u.score >= 10000 else False) shuffle(fb) shuffle(other) users = fb + other for u in users: if hasattr(u, 'score'): score = u.score delattr(u, 'score') if score >= 10000: score -= 10000 if score < 100000: u.num_friends_in_common = int(score) else: u.num_friends_in_common = 0 else: u.num_friends_in_common = 0 return self.serialize_list(User, users, count, page), 200, { 'Cache-Control': 'max-age=60' }
def migrate_top_friends(): logconfig.configure('dev') users = 0 for user_id, score in wigo_db.sorted_set_iter(skey('user'), count=50): for friend_id, score in wigo_db.sorted_set_iter(skey( 'user', user_id, 'friends'), count=50): if not wigo_db.sorted_set_is_member( skey('user', user_id, 'friends', 'top'), friend_id): wigo_db.sorted_set_add(skey('user', user_id, 'friends', 'top'), friend_id, 1) users += 1 if (users % 100) == 0: logger.info('fixed {} users'.format(users))
def each_friends_friend(): for friend_id in friend_ids: friends_friends = wigo_db.sorted_set_rrange( skey('user', friend_id, 'friends'), 0, 50) for friends_friend in friends_friends: if should_suggest(friends_friend): yield friends_friend
def remove_index(self): super(EventMessageVote, self).remove_index() message = self.message event = message.event self.db.sorted_set_remove(skey(event, 'messages', 'by_votes'), self.message_id, replicate=False)
def index(self): super(EventAttendee, self).index() user = self.user event = self.event # check if the user is switching events for the date the event is on current_event_id = user.get_attending_id(event) if current_event_id and current_event_id != event.id: EventAttendee({'event_id': current_event_id, 'user_id': user.id}).delete() with self.db.transaction(commit_on_select=False): # first update the global state of the event attendees_key = skey(event, 'attendees') self.db.sorted_set_add(attendees_key, user.id, epoch(self.created)) self.db.expire(attendees_key, event.ttl()) # now update the users view of the events # record the exact event the user is currently attending user.set_attending(event) # record current user as an attendee attendees_key = user_attendees_key(user, event) self.db.sorted_set_add(attendees_key, user.id, 'inf') self.db.expire(attendees_key, event.ttl()) # record the event into the events the user can see with self.db.transaction(commit_on_select=False): event.update_global_events() event.update_user_events(user)
def decorated(*args, **kw): headers = {} kw['headers'] = headers if context_var == 'user' and 'user_id' in kw: user_id = kw['user_id'] if user_id == '(null)': return f(*args, **kw) context = g.user if user_id == 'me' else User.find(int(user_id)) else: context = getattr(g, context_var, None) if not context: return f(*args, **kw) last_change = wigo_db.get_redis().hget(skey(context, 'meta'), field) if last_change: last_change = datetime.utcfromtimestamp(float(last_change)) else: return f(*args, **kw) if last_change > datetime.utcnow(): # if last-change is set to the future, the intent is to disable if-modified-since # until that time. Last-Modified can't be set to the future or that doesn't work. headers['Last-Modified'] = http_date(datetime.utcnow()) else: headers['Last-Modified'] = http_date(last_change) if max_age: headers['Cache-Control'] = 'max-age={}'.format(max_age) if last_change and not is_resource_modified(request.environ, last_modified=last_change): return 'Not modified', 304, headers return f(*args, **kw)
def get_friend_ids_in_common(self, with_user_id): from server.db import wigo_db friend_ids = set(self.get_friend_ids()) with_friend_ids = set( wigo_db.sorted_set_range(skey('user', with_user_id, 'friends'), 0, -1)) return friend_ids & with_friend_ids
def get_user_id_for_key(key): from server.db import wigo_db model_ids = wigo_db.sorted_set_range(skey('user', key, 'key')) if model_ids: return model_ids[0] else: raise DoesNotExist()
def __get_conversation(self): if self._to_user: return self.__get_page( skey(self._user, 'conversation', self._to_user.id)) else: query = User.select().key(skey(self._user, 'conversations')).limit( self._limit).page(self._page).order(self._order) count, page, users = query.execute() message_ids = [] for user in users: last_message_id = self.db.get( skey(self._user, 'conversation', user.id, 'last_message')) message_ids.append(last_message_id) return count, page, Message.find(message_ids)
def get_tapped_ids(self): from server.db import wigo_db return wigo_db.sorted_set_range_by_score( skey(self, 'tapped'), epoch(self.group.get_day_start()), 'inf', limit=5000)
def __get_by_event(self): if self._model_class == EventMessage: if self._user: key = user_eventmessages_key(self._user, self._event, self._by_votes) else: key = skey(self._event, 'messages', 'by_votes') if self._by_votes else skey( self._event, 'messages') return self.__get_page(key) elif self._model_class == EventAttendee: if self._user: key = user_attendees_key(self._user, self._event) else: key = skey(self._event, 'attendees') return self.model_class(User).key(key).execute() else: raise ValueError('Invalid query')
def remove_index(self, group=None): super(Event, self).remove_index() if group is None: group = self.group with self.db.transaction(commit_on_select=False): self.db.delete(skey(self, 'attendees')) self.db.delete(skey(self, 'messages')) self.db.sorted_set_remove(skey(group, 'events'), self.id) if group.id == self.group_id: try: existing_event = self.find(group=group, name=self.name) if existing_event.id == self.id: self.db.delete(skey(group, Event, Event.event_key(self.name, group))) except: pass
def tell_friends_event_message(message_id): try: message = EventMessage.find(message_id) user_id = message.user_id except: return for friend_id, score in wigo_db.sorted_set_iter( skey('user', user_id, 'friends')): tell_friend_event_message.delay(message_id, friend_id)