def reap_chat(chat_id): user_list = UserListStore(NewparpRedis(connection_pool=redis_chat_pool), chat_id) old_user_ids = user_list.user_ids_online() if not old_user_ids: return with session_scope() as db: chat = db.query(Chat).filter(Chat.id == chat_id).one() chat_users = { _.user_id: _ for _ in db.query(ChatUser).filter(and_( ChatUser.chat_id == chat_id, ChatUser.user_id.in_(old_user_ids), )) } for socket_id, user_id in user_list.inconsistent_entries(): chat_user = chat_users[user_id] dead = user_list.socket_disconnect(socket_id, chat_user.number) if dead: logger.debug("dead: %s" % chat_user) # TODO optimise this when reaping several people at once? if chat_user.computed_group == "silent" or chat.type in ("pm", "roulette"): send_userlist(user_list, db, chat) else: send_message(db, reap_chat.redis, Message( chat=chat, user_id=chat_user.user_id, type="timeout", name=chat_user.name, text="%s's connection timed out." % chat_user.name, ), user_list)
def reap(): redis = reap.redis with session_scope() as db: current_time = int(time.time()) disconnected_users = set() # Long poll sessions. for dead in redis.zrangebyscore("chats_alive", 0, current_time): logger.info("Reaping %s" % dead) chat_id, session_id = dead.split('/') user_id = redis.hget("chat:%s:online" % chat_id, session_id) disconnected = disconnect(redis, chat_id, session_id) # Only send a timeout message if they were already online. if not disconnected: logger.info("Not sending timeout message.") continue disconnected_users.add((chat_id, user_id, False)) # Sockets. for dead in redis.zrangebyscore("sockets_alive", 0, current_time): logger.info("Reaping %s" % dead) chat_id, session_id, socket_id = dead.split('/') user_id = redis.hget("chat:%s:online" % chat_id, socket_id) disconnected = disconnect(redis, chat_id, socket_id) redis.srem("chat:%s:sockets:%s" % (chat_id, session_id), socket_id) redis.zrem("sockets_alive", "%s/%s/%s" % (chat_id, session_id, socket_id)) # Only send a timeout message if they were already online. if not disconnected: logger.info("Not sending timeout message.") continue disconnected_users.add((chat_id, user_id, True)) for chat_id, user_id, reaped_socket in disconnected_users: try: dead_chat_user = db.query(ChatUser).filter(and_( ChatUser.user_id == user_id, ChatUser.chat_id == chat_id, )).options(joinedload(ChatUser.chat), joinedload(ChatUser.user)).one() except NoResultFound: logger.error("Unable to find ChatUser (chat %s, user %s)." % (chat_id, user_id)) continue if reaped_socket: typing_key = "chat:%s:typing" % chat_id if redis.srem(typing_key, dead_chat_user.number): redis.publish("channel:%s:typing" % chat_id, json.dumps({ "typing": list(int(_) for _ in redis.smembers(typing_key)), })) if dead_chat_user.computed_group == "silent" or dead_chat_user.chat.type in ("pm", "roulette"): send_userlist(db, redis, dead_chat_user.chat) else: send_message(db, redis, Message( chat_id=chat_id, user_id=dead_chat_user.user_id, type="timeout", name=dead_chat_user.name, text="%s's connection timed out." % dead_chat_user.name, )) logger.info("Sent timeout message for ChatUser (chat %s, user %s)." % (chat_id, user_id))
def update_lastonline(): redis = update_lastonline.redis redis_chat = NewparpRedis(connection_pool=redis_chat_pool) if redis.exists("lock:lastonline"): return redis.setex("lock:lastonline", 60, 1) chat_ids = redis.hgetall("queue:lastonline") # Reset the list for the next iteration. redis.delete("queue:lastonline") for chat_id, posted in chat_ids.items(): online_user_ids = UserListStore(redis_chat, chat_id).user_ids_online() try: posted = datetime.datetime.utcfromtimestamp(float(posted)) except ValueError: continue with session_scope() as db: db.query(Chat).filter(Chat.id == chat_id).update({"last_message": posted}, synchronize_session=False) if len(online_user_ids) != 0: db.query(ChatUser).filter(and_( ChatUser.user_id.in_(online_user_ids), ChatUser.chat_id == chat_id, )).update({"last_online": posted}, synchronize_session=False) redis.delete("lock:lastonline")
def update_lastonline(): redis = update_lastonline.redis redis_chat = NewparpRedis(connection_pool=redis_chat_pool) if redis.exists("lock:lastonline"): return redis.setex("lock:lastonline", 60, 1) chat_ids = redis.hgetall("queue:lastonline") # Reset the list for the next iteration. redis.delete("queue:lastonline") for chat_id, posted in chat_ids.items(): online_user_ids = UserListStore(redis_chat, chat_id).user_ids_online() try: posted = datetime.datetime.utcfromtimestamp(float(posted)) except ValueError: continue with session_scope() as db: db.query(Chat).filter(Chat.id == chat_id).update( {"last_message": posted}, synchronize_session=False) if len(online_user_ids) != 0: db.query(ChatUser).filter( and_( ChatUser.user_id.in_(online_user_ids), ChatUser.chat_id == chat_id, )).update({"last_online": posted}, synchronize_session=False) redis.delete("lock:lastonline")
def unlist_chats(): with session_scope() as db: db.query(GroupChat).filter( and_( GroupChat.id == Chat.id, GroupChat.publicity == "listed", GroupChat.last_message < datetime.datetime.now() - datetime.timedelta(7))).update({"publicity": "unlisted"}) db.commit()
def unlist_chats(): with session_scope() as db: db.query(GroupChat).filter(and_( GroupChat.id == Chat.id, GroupChat.publicity == "listed", GroupChat.last_message < datetime.datetime.now() - datetime.timedelta(7) )).update({"publicity": "unlisted"}) db.commit()
def update_log_marker(chat_id, log_marker_type="page_with_system_messages"): redis = update_log_marker.redis chat_id = int(chat_id) if log_marker_type == "page_without_system_messages": message_filter = and_( Message.chat_id == chat_id, Message.type.in_(("ic", "ooc", "me")), ) else: message_filter = Message.chat_id == chat_id with session_scope() as db: # Fetch the last log marker. last_log_marker = (db.query(LogMarker).filter( and_( LogMarker.chat_id == chat_id, LogMarker.type == log_marker_type, )).options(joinedload(LogMarker.message)).order_by( LogMarker.number.desc()).first()) # See how much has happened since that marker. if last_log_marker: messages_since_last_marker = (db.query( func.count("*")).select_from(Message).filter( and_( message_filter, Message.posted >= last_log_marker.message.posted, )).scalar()) # And create a new one if necessary. if messages_since_last_marker > 200: db.add( LogMarker( chat_id=chat_id, type=log_marker_type, number=last_log_marker.number + 1, message_id=(db.query(Message.id).filter( and_( message_filter, Message.posted >= last_log_marker.message.posted, )).order_by( Message.posted).offset(200).limit(1).scalar()), )) # Or create initial log marker if there aren't any. else: first_message = (db.query(Message).filter(message_filter).order_by( Message.posted).first()) if first_message: db.add( LogMarker( chat_id=chat_id, type=log_marker_type, number=1, message_id=first_message.id, ))
def update_user_meta(): redis = update_user_meta.redis if redis.exists("lock:metaupdate"): return redis.setex("lock:metaupdate", 60, 1) meta_updates = redis.hgetall("queue:usermeta") # Reset the list for the next iteration. redis.delete("queue:usermeta") for key, meta in meta_updates.items(): try: meta = json.loads(meta) except ValueError: continue msgtype, userid = key.split(":", 2) with session_scope() as db: if msgtype == "user" and "last_ip" in meta: try: last_online = datetime.datetime.utcfromtimestamp( float(meta["last_online"])) except (ValueError, KeyError): last_online = datetime.datetime.now() db.query(User).filter(User.id == userid).update( { "last_online": last_online, "last_ip": meta["last_ip"] }, synchronize_session=False) elif msgtype == "chatuser": try: last_online = datetime.datetime.utcfromtimestamp( float(meta["last_online"])) except (ValueError, KeyError): last_online = datetime.datetime.now() db.query(ChatUser).filter( and_(ChatUser.user_id == userid, ChatUser.chat_id == meta["chat_id"])).update( {"last_online": last_online}, synchronize_session=False) redis.delete("lock:metaupdate")
def update_user_meta(): redis = update_user_meta.redis if redis.exists("lock:metaupdate"): return redis.setex("lock:metaupdate", 60, 1) meta_updates = redis.hgetall("queue:usermeta") # Reset the list for the next iteration. redis.delete("queue:usermeta") for key, meta in meta_updates.items(): try: meta = json.loads(meta) except ValueError: continue msgtype, userid = key.split(":", 2) with session_scope() as db: if msgtype == "user" and "last_ip" in meta: try: last_online = datetime.datetime.utcfromtimestamp(float(meta["last_online"])) except (ValueError, KeyError): last_online = datetime.datetime.now() db.query(User).filter(User.id == userid).update({ "last_online": last_online, "last_ip": meta["last_ip"] }, synchronize_session=False) elif msgtype == "chatuser": try: last_online = datetime.datetime.utcfromtimestamp(float(meta["last_online"])) except (ValueError, KeyError): last_online = datetime.datetime.now() db.query(ChatUser).filter(and_(ChatUser.user_id == userid, ChatUser.chat_id == meta["chat_id"])).update({ "last_online": last_online }, synchronize_session=False) redis.delete("lock:metaupdate")
def load_lists(self): if lists["reload"] == self.redis.get("spamless:reload"): return logger.info("reload") lists["reload"] = self.redis.get("spamless:reload") with session_scope() as db: filters = db.query(SpamlessFilter).all() lists["banned_names"] = [ re.compile(_.regex, re.IGNORECASE | re.MULTILINE) for _ in filters if _.type == "banned_names" ] lists["blacklist"] = [ (re.compile(_.regex, re.IGNORECASE | re.MULTILINE), _.points) for _ in filters if _.type == "blacklist" ] lists["warnlist"] = [ re.compile(_.regex, re.IGNORECASE | re.MULTILINE) for _ in filters if _.type == "warnlist" ]
def load_lists(self): if lists["reload"] == self.redis.get("spamless:reload"): return logger.info("reload") lists["reload"] = self.redis.get("spamless:reload") with session_scope() as db: filters = db.query(SpamlessFilter).all() lists["banned_names"] = [ re.compile(_.regex, re.IGNORECASE | re.MULTILINE) for _ in filters if _.type == "banned_names" ] lists["blacklist"] = [ (re.compile(_.regex, re.IGNORECASE | re.MULTILINE), _.points) for _ in filters if _.type == "blacklist" ] lists["warnlist"] = [ re.compile(_.regex, re.IGNORECASE | re.MULTILINE) for _ in filters if _.type == "warnlist" ]
def reap_chat(chat_id): user_list = UserListStore(NewparpRedis(connection_pool=redis_chat_pool), chat_id) old_user_ids = user_list.user_ids_online() if not old_user_ids: return with session_scope() as db: chat = db.query(Chat).filter(Chat.id == chat_id).one() chat_users = { _.user_id: _ for _ in db.query(ChatUser).filter( and_( ChatUser.chat_id == chat_id, ChatUser.user_id.in_(old_user_ids), )) } for socket_id, user_id in user_list.inconsistent_entries(): chat_user = chat_users[user_id] dead = user_list.socket_disconnect(socket_id, chat_user.number) if dead: logger.debug("dead: %s" % chat_user) # TODO optimise this when reaping several people at once? if chat_user.computed_group == "silent" or chat.type in ( "pm", "roulette"): send_userlist(user_list, db, chat) else: send_message( db, reap_chat.redis, Message( chat=chat, user_id=chat_user.user_id, type="timeout", name=chat_user.name, text="%s's connection timed out." % chat_user.name, ), user_list)
def run(self, chat_id, data): if len(data["messages"]) == 0: return self.load_lists() for message in data["messages"]: if message["user_number"] is None: continue message["hash"] = hashlib.md5( ":".join([message["color"], message["acronym"], message["text"]]) .encode("utf-8").lower() ).hexdigest() try: self.check_connection_spam(chat_id, message) self.check_banned_names(chat_id, message) self.check_message_filter(chat_id, message) self.check_warnlist(chat_id, message) except Mark as e: with session_scope() as db: q = db.query(Message).filter(Message.id == message["id"]).update({"spam_flag": str(e)}) message.update({"spam_flag": str(e)}) self.redis.publish("spamless:live", json.dumps(message)) except Silence as e: with session_scope() as db: # XXX maybe cache this? try: chat_user, user, chat = db.query( ChatUser, User, AnyChat, ).join( User, ChatUser.user_id == User.id, ).join( AnyChat, ChatUser.chat_id == AnyChat.id, ).filter(and_( ChatUser.chat_id == chat_id, ChatUser.number == message["user_number"], )).one() except NoResultFound: continue if chat.type != "group": flag_suffix = chat.type.upper() elif chat_user.computed_group in ("admin", "creator"): flag_suffix = chat_user.computed_group.upper() else: flag_suffix = "SILENCED" db.query(Message).filter(Message.id == message["id"]).update({ "spam_flag": str(e) + " " + flag_suffix, }) message.update({"spam_flag": str(e) + " " + flag_suffix}) self.redis.publish("spamless:live", json.dumps(message)) if flag_suffix == "SILENCED": db.query(ChatUser).filter(and_( ChatUser.chat_id == chat_id, ChatUser.number == message["user_number"] )).update({"group": "silent"}) send_message(db, self.redis, Message( chat_id=chat_id, type="spamless", name="The Spamless", acronym="\u264b", text="Spam has been detected and silenced. Please come [url=http://help.msparp.com/]here[/url] or ask a chat moderator to unsilence you if this was an accident.", color="626262" ), True)
def comparison_callback(results, searcher_id_1): redis = comparison_callback.redis if redis.exists("lock:matchmaker"): logger.debug("locked. calling again in 1 second.") comparison_callback.apply_async((results, searcher_id_1), countdown=1) return redis.setex("lock:matchmaker", 60, 1) # Check if there's a match. matched_searchers = [_ for _ in results if _[0] is not None] if not matched_searchers: logger.debug("no results") redis.delete("lock:matchmaker") return logger.debug("results: %s" % matched_searchers) shuffle(matched_searchers) # Fetch searcher 1. s1 = fetch_searcher(redis, searcher_id_1) if not all(s1[:-2]): logger.debug("%s has expired" % searcher_id_1) redis.delete("lock:matchmaker") return with session_scope() as db: # Pick a second searcher from the matches. for searcher_id_2, options in matched_searchers: s2 = fetch_searcher(redis, searcher_id_2) if all(s2[:-2]) and db.query(func.count("*")).select_from(Block).filter(or_( and_(Block.blocking_user_id == s1.user_id, Block.blocked_user_id == s2.user_id), and_(Block.blocking_user_id == s2.user_id, Block.blocked_user_id == s1.user_id), )).scalar() == 0: logger.debug("matched %s" % searcher_id_2) break else: logger.debug("all matches have expired") redis.delete("lock:matchmaker") return new_url = str(uuid4()).replace("-", "") logger.info("matched %s and %s, sending to %s." % (s1.id, s2.id, new_url)) new_chat = SearchedChat(url=new_url) db.add(new_chat) db.flush() s1_user = db.query(User).filter(User.id == s1.user_id).one() s2_user = db.query(User).filter(User.id == s2.user_id).one() db.add(ChatUser.from_user(s1_user, chat_id=new_chat.id, number=1, search_character_id=s1.search_character_id, **s1.character)) if s1_user != s2_user: db.add(ChatUser.from_user(s2_user, chat_id=new_chat.id, number=2, search_character_id=s2.search_character_id, **s2.character)) if options: db.add(Message( chat_id=new_chat.id, type="search_info", text=" ".join(option_messages[_] for _ in options), )) pipe = redis.pipeline() match_key = "matched:%s:%s" % tuple(sorted([s1.user_id, s2.user_id])) pipe.set(match_key, 1) pipe.expire(match_key, 1800) pipe.srem("searchers", s1.id, s2.id) match_message = """{"status":"matched","url":"%s"}""" % new_url pipe.publish("searcher:%s" % s1.id, match_message) pipe.publish("searcher:%s" % s2.id, match_message) pipe.delete("lock:matchmaker") pipe.execute()
def postgres_test(): with session_scope() as db: db.query(SearchCharacter).first()
def run(self, chat_id, data): if len(data["messages"]) == 0: return self.load_lists() for message in data["messages"]: if message["user_number"] is None: continue self.redis.set("spamless:last_id", message["id"]) message["hash"] = hashlib.md5( ":".join([message["color"], message["acronym"], message["text"]]) .encode("utf-8").lower() ).hexdigest() try: self.check_connection_spam(chat_id, message) self.check_banned_names(chat_id, message) self.check_message_filter(chat_id, message) self.check_warnlist(chat_id, message) except Mark as e: with session_scope() as db: q = db.query(Message).filter(Message.id == message["id"]).update({"spam_flag": str(e)}) message.update({"spam_flag": str(e)}) self.redis.publish("spamless:live", json.dumps(message)) except Silence as e: with session_scope() as db: # XXX maybe cache this? try: chat_user, user, chat = db.query( ChatUser, User, AnyChat, ).join( User, ChatUser.user_id == User.id, ).join( AnyChat, ChatUser.chat_id == AnyChat.id, ).filter(and_( ChatUser.chat_id == chat_id, ChatUser.number == message["user_number"], )).one() except NoResultFound: continue if chat.type != "group": flag_suffix = chat.type.upper() elif chat_user.computed_group in ("admin", "creator"): flag_suffix = chat_user.computed_group.upper() else: flag_suffix = "SILENCED" db.query(Message).filter(Message.id == message["id"]).update({ "spam_flag": str(e) + " " + flag_suffix, }) message.update({"spam_flag": str(e) + " " + flag_suffix}) self.redis.publish("spamless:live", json.dumps(message)) if flag_suffix == "SILENCED": db.query(ChatUser).filter(and_( ChatUser.chat_id == chat_id, ChatUser.number == message["user_number"] )).update({"group": "silent"}) send_message(db, self.redis, Message( chat_id=chat_id, type="spamless", name="The Spamless", acronym="\u264b", text="Spam has been detected and silenced. Please come [url=http://help.msparp.com/]here[/url] or ask a chat moderator to unsilence you if this was an accident.", color="626262" ), force_userlist=True)
def update_log_marker(chat_id, log_marker_type="page_with_system_messages"): redis = update_log_marker.redis chat_id = int(chat_id) if log_marker_type == "page_without_system_messages": message_filter = and_( Message.chat_id == chat_id, Message.type.in_(("ic", "ooc", "me")), ) else: message_filter = Message.chat_id == chat_id with session_scope() as db: # Fetch the last log marker. last_log_marker = ( db.query(LogMarker) .filter(and_( LogMarker.chat_id == chat_id, LogMarker.type == log_marker_type, )) .options(joinedload(LogMarker.message)) .order_by(LogMarker.number.desc()).first() ) # See how much has happened since that marker. if last_log_marker: messages_since_last_marker = ( db.query(func.count("*")).select_from(Message) .filter(and_( message_filter, Message.posted >= last_log_marker.message.posted, )).scalar() ) # And create a new one if necessary. if messages_since_last_marker > 200: db.add(LogMarker( chat_id=chat_id, type=log_marker_type, number=last_log_marker.number + 1, message_id=( db.query(Message.id) .filter(and_( message_filter, Message.posted >= last_log_marker.message.posted, )).order_by(Message.posted) .offset(200).limit(1).scalar() ), )) # Or create initial log marker if there aren't any. else: first_message = ( db.query(Message) .filter(message_filter) .order_by(Message.posted).first() ) if first_message: db.add(LogMarker( chat_id=chat_id, type=log_marker_type, number=1, message_id=first_message.id, ))