def new_user(user_id, score=None):
user = User.find(user_id)
if user.status != 'waiting':
return
user_queue_key = skey('user_queue')
if score is None:
if user_id < 130000:
score = time() + randint(60, 60 * 30)
else:
last_waiting = wigo_db.sorted_set_range(user_queue_key, -1, -1,
True)
if last_waiting:
last_waiting_score = last_waiting[0][1]
if last_waiting_score > (time() + (60 * 60 * 6)):
score = last_waiting_score + 10
else:
score = last_waiting_score + randint(0, 60)
else:
score = time() + randint(120, 60 * 20)
wigo_db.sorted_set_add(user_queue_key, user_id, score, replicate=False)
scheduler.schedule(datetime.utcfromtimestamp(score),
process_waitlist,
result_ttl=0,
timeout=600)
def get_user_id_for_key(key):
from server.db import wigo_db
model_ids = wigo_db.sorted_set_range(skey('user', key, 'key'))
if model_ids:
return model_ids[0]
else:
raise DoesNotExist()
def get_friend_ids_in_common(self, with_user_id):
from server.db import wigo_db
friend_ids = set(self.get_friend_ids())
with_friend_ids = set(
wigo_db.sorted_set_range(skey('user', with_user_id, 'friends'), 0,
-1))
return friend_ids & with_friend_ids
def get(self, user_id, event_id, headers):
message_meta = {}
message_ids = wigo_db.sorted_set_range(user_eventmessages_key(g.user, event_id))
for message_id in message_ids:
message_meta[message_id] = {
'num_votes': wigo_db.get_sorted_set_size(user_votes_key(g.user, message_id)),
'voted': wigo_db.sorted_set_is_member(user_votes_key(g.user, message_id), g.user.id)
}
return message_meta, 200, headers
def get(self, event_id, headers):
message_meta = {}
message_ids = wigo_db.sorted_set_range(skey('event', event_id, 'messages'))
for message_id in message_ids:
message_meta[message_id] = {
'num_votes': wigo_db.get_sorted_set_size(skey('eventmessage', message_id, 'votes')),
'voted': wigo_db.sorted_set_is_member(skey('eventmessage', message_id, 'votes'), g.user.id)
}
return message_meta, 200, headers
def __filtered(self):
from server.db import wigo_db
from server.rdbms import db
try:
for kwarg in self._where:
applicable_indexes = [
key_tmpl
for key_tmpl, unique, expires in self._model_class.indexes
if kwarg in key_tmpl
]
if applicable_indexes:
for key_tmpl in applicable_indexes:
key = index_key(key_tmpl,
{kwarg: self._where.get(kwarg)})
model_ids = wigo_db.sorted_set_range(key)
if model_ids:
instances = self._model_class.find(model_ids)
return len(instances), self._page, instances
else:
return 0, self._page, []
else:
# not a redis indexed key, so look in the rdbms
with db.execution_context(False) as ctx:
keys = [
k[0]
for k in DataStrings.select(DataStrings.key).where(
DataStrings.value.contains(
{kwarg: self._where.get(kwarg)})).tuples()
]
# still fetch the actual objects from redis
results = wigo_db.mget(keys)
instances = [
self._model_class(result)
if result is not None else None for result in results
]
for instance in instances:
if instance is not None:
instance.prepared()
return len(instances), self._page, instances
except DoesNotExist:
return 0, self._page, []
def get(self, event_id):
event = Event.find(event_id)
page = self.get_page()
limit = self.get_limit()
start = (page - 1) * limit
num_friends = wigo_db.get_sorted_set_size(skey(g.user, 'friends'))
# find the users top 5 friends. this is users with > 3 interactions
top_5 = wigo_db.sorted_set_rrange_by_score(skey(
g.user, 'friends', 'top'),
'inf',
3,
limit=5)
if not top_5:
top_5 = wigo_db.sorted_set_rrange_by_score(skey(g.user, 'friends'),
'inf',
3,
limit=5)
friend_ids = wigo_db.sorted_set_range(skey(g.user, 'friends', 'alpha'),
start, start + (limit - 1))
for top_friend_id in top_5:
if top_friend_id in friend_ids:
friend_ids.remove(top_friend_id)
if page == 1 and top_5:
friend_ids = top_5 + friend_ids
users = User.find(friend_ids)
p = wigo_db.redis.pipeline()
for user in users:
p.zscore(skey(event, g.user, 'invited'), user.id if user else -1)
scores = p.execute()
for index, user in enumerate(users):
if user:
user.invited = scores[index] is not None
return self.serialize_list(self.model, users, num_friends, page)
def find(cls, *args, **kwargs):
from server.db import wigo_db
model_id = args[0] if args else kwargs.get('id')
model_ids = model_id if hasattr(model_id, '__iter__') else kwargs.get('ids')
if hasattr(model_ids, '__iter__'):
return cls.__get_by_ids(model_ids)
elif model_id:
return cls.__get_by_id(model_id)
# search the indexes
if kwargs:
for kwarg in kwargs:
applicable_indexes = [key_tmpl for key_tmpl, unique, expires in cls.indexes if kwarg in key_tmpl]
for key_tmpl in applicable_indexes:
key = index_key(key_tmpl, {kwarg: kwargs.get(kwarg)})
model_ids = wigo_db.sorted_set_range(key)
if model_ids:
return cls.find(model_ids[0])
raise DoesNotExist()
return None
def _do_generate_friend_recs(user_id,
num_friends_to_recommend=200,
force=False):
is_dev = Configuration.ENVIRONMENT == 'dev'
user = User.find(user_id)
suggestions_key = skey(user, 'friend', 'suggestions')
suggested = {}
friend_id_list = user.get_friend_ids()
friend_ids = set(friend_id_list)
deleted_suggest_ids = wigo_db.sorted_set_range(
skey(user, 'friend', 'suggestions', 'deleted'))
exclude = set(friend_id_list + user.get_blocked_ids() +
user.get_friend_request_ids() +
user.get_friend_requested_ids() + deleted_suggest_ids)
def is_limited(field, ttl=10):
last_check = user.get_meta(field)
if last_check:
last_check = datetime.utcfromtimestamp(float(last_check))
if not last_check:
return False
return last_check >= (datetime.utcnow() - timedelta(minutes=ttl))
def should_suggest(suggest_id):
if (suggest_id == user.id) or (suggest_id
in suggested) or (suggest_id
in exclude):
return False
return True
def get_num_friends_in_common(suggest_id):
with_friend_ids = set(
wigo_db.sorted_set_range(skey('user', suggest_id, 'friends'), 0,
-1))
return len(friend_ids & with_friend_ids)
p = wigo_db.redis.pipeline()
def add_friend(suggest_id, boost=0):
if boost == 0:
existing_score = suggested.get(suggest_id, -1)
if existing_score >= 10000:
boost = 10000
score = get_num_friends_in_common(suggest_id) + boost
p.zadd(suggestions_key, suggest_id, score)
suggested[suggest_id] = score
sleep(.1)
#################################################
# first clean up all the old suggestions
for suggest_id, score in wigo_db.sorted_set_iter(suggestions_key,
count=50):
if should_suggest(suggest_id):
# update the scores
boost = 10000 if score >= 10000 else 0
score = get_num_friends_in_common(suggest_id) + boost
if score != suggested.get(suggest_id, -1):
p.zadd(suggestions_key, suggest_id, score)
suggested[suggest_id] = score
else:
wigo_db.sorted_set_remove(suggestions_key,
suggest_id,
replicate=False)
##################################
# add facebook friends
if force or not is_limited('last_facebook_check', 30):
num_fb_recs = 0
facebook = Facebook(user.facebook_token, user.facebook_token_expires)
try:
token_expires = facebook.get_token_expiration()
if token_expires != user.facebook_token_expires:
user.facebook_token_expires = token_expires
user.save()
friends_to_add = set()
def iterate_facebook(next=None):
for facebook_id in facebook.get_friend_ids(next):
try:
friend = User.find(facebook_id=facebook_id)
if should_suggest(friend.id):
yield friend.id
except DoesNotExist:
pass
# iterate fb friends, starting at last stored next token
next_fb_path = user.get_meta('next_fb_friends_path')
for friend_id in iterate_facebook(next_fb_path):
friends_to_add.add(friend_id)
num_fb_recs += 1
if num_fb_recs >= 100:
break
# possibly rewrap the search
if num_fb_recs < 100 and next_fb_path:
# iterate again, without next, so starting at beginning
for friend_id in iterate_facebook():
friends_to_add.add(friend_id)
num_fb_recs += 1
if num_fb_recs >= 100:
break
# remove all the existing fb recommendations
p.zremrangebyscore(suggestions_key, 10000, '+inf')
# add these fb recommendations
for friend_id in friends_to_add:
add_friend(friend_id, 10000)
if facebook.next_path:
user.track_meta('next_fb_friends_path', facebook.next_path)
else:
user.remove_meta('next_fb_friends_path')
logger.info(
'generated {} facebook friend suggestions for user {}'.format(
num_fb_recs, user_id))
user.track_meta('last_facebook_check')
except FacebookTokenExpiredException:
logger.warn(
'error finding facebook friends to suggest for user {}, token expired'
.format(user_id))
user.track_meta('last_facebook_check')
user.facebook_token_expires = datetime.utcnow()
user.save()
except Exception, e:
logger.error('error finding facebook friends to suggest '
'for user {}, {}'.format(user_id, e.message))
def get_num_friends_in_common(suggest_id):
with_friend_ids = set(
wigo_db.sorted_set_range(skey('user', suggest_id, 'friends'), 0,
-1))
return len(friend_ids & with_friend_ids)
def delete_user(user_id, group_id):
logger.info('deleting user {}'.format(user_id))
friend_ids = wigo_db.sorted_set_range(skey('user', user_id, 'friends'))
with wigo_db.transaction(commit_on_select=False):
# remove from attendees
for event_id, score in wigo_db.sorted_set_iter(
skey('user', user_id, 'events')):
wigo_db.sorted_set_remove(skey('event', event_id, 'attendees'),
user_id)
for friend_id in friend_ids:
wigo_db.sorted_set_remove(
skey('user', friend_id, 'event', event_id, 'attendees'),
user_id)
# remove event message votes
for message_id, score in wigo_db.sorted_set_iter(
skey('user', user_id, 'votes')):
wigo_db.sorted_set_remove(
skey('eventmessage', message_id, 'votes'), user_id)
for friend_id in friend_ids:
wigo_db.sorted_set_remove(
user_votes_key(friend_id, message_id), user_id)
# remove event messages
for message_id, score in wigo_db.sorted_set_iter(
skey('user', user_id, 'event_messages')):
message = EventMessage.find(message_id)
event_id = message.event_id
wigo_db.sorted_set_remove(skey('event', event_id, 'messages'),
message_id)
wigo_db.sorted_set_remove(
skey('event', event_id, 'messages', 'by_votes'), message_id)
for friend_id in friend_ids:
wigo_db.sorted_set_remove(
skey('user', friend_id, 'event', event_id, 'messages'),
message_id)
wigo_db.sorted_set_remove(
skey('user', friend_id, 'event', event_id, 'messages',
'by_votes'), message_id)
for friend_id in friend_ids:
# remove conversations
wigo_db.sorted_set_remove(skey('user', friend_id, 'conversations'),
user_id)
wigo_db.delete(skey('user', friend_id, 'conversation', user_id))
wigo_db.delete(skey('user', user_id, 'conversation', friend_id))
# remove friends
wigo_db.sorted_set_remove(skey('user', friend_id, 'friends'),
user_id)
wigo_db.sorted_set_remove(
skey('user', friend_id, 'friends', 'top'), user_id)
wigo_db.sorted_set_remove(
skey('user', friend_id, 'friends', 'alpha'), user_id)
wigo_db.set_remove(skey('user', friend_id, 'friends', 'private'),
user_id)
# remove friend requests
for friend_id in wigo_db.sorted_set_range(
skey('user', user_id, 'friend_requested')):
wigo_db.sorted_set_remove(
skey('user', friend_id, 'friend_requests'), user_id)
wigo_db.sorted_set_remove(
skey('user', friend_id, 'friend_requests', 'common'), user_id)
# remove messages
for message_id, score in wigo_db.sorted_set_iter(
skey('user', user_id, 'messages')):
wigo_db.delete(skey('message', message_id))
wigo_db.delete(skey('user', user_id, 'events'))
wigo_db.delete(skey('user', user_id, 'friends'))
wigo_db.delete(skey('user', user_id, 'friends', 'top'))
wigo_db.delete(skey('user', user_id, 'friends', 'private'))
wigo_db.delete(skey('user', user_id, 'friends', 'alpha'))
wigo_db.delete(skey('user', user_id, 'friends', 'friend_requested'))
wigo_db.delete(skey('user', user_id, 'friends', 'friend_requests'))
wigo_db.delete(
skey('user', user_id, 'friends', 'friend_requests', 'common'))
wigo_db.delete(skey('user', user_id, 'blocked'))
wigo_db.delete(skey('user', user_id, 'notifications'))
wigo_db.delete(skey('user', user_id, 'notifs'), replicate=False)
wigo_db.delete(skey('user', user_id, 'conversations'))
wigo_db.delete(skey('user', user_id, 'tapped'))
wigo_db.delete(skey('user', user_id, 'votes'))
wigo_db.delete(skey('user', user_id, 'messages'))
def get_blocked_ids(self):
from server.db import wigo_db
return wigo_db.sorted_set_range(skey(self, 'blocked'), 0, -1)