def reap_chat(chat_id):
user_list = UserListStore(NewparpRedis(connection_pool=redis_chat_pool), chat_id)
old_user_ids = user_list.user_ids_online()
if not old_user_ids:
return
with session_scope() as db:
chat = db.query(Chat).filter(Chat.id == chat_id).one()
chat_users = {
_.user_id: _
for _ in db.query(ChatUser).filter(and_(
ChatUser.chat_id == chat_id,
ChatUser.user_id.in_(old_user_ids),
))
}
for socket_id, user_id in user_list.inconsistent_entries():
chat_user = chat_users[user_id]
dead = user_list.socket_disconnect(socket_id, chat_user.number)
if dead:
logger.debug("dead: %s" % chat_user)
# TODO optimise this when reaping several people at once?
if chat_user.computed_group == "silent" or chat.type in ("pm", "roulette"):
send_userlist(user_list, db, chat)
else:
send_message(db, reap_chat.redis, Message(
chat=chat,
user_id=chat_user.user_id,
type="timeout",
name=chat_user.name,
text="%s's connection timed out." % chat_user.name,
), user_list)
def reap():
redis = reap.redis
with session_scope() as db:
current_time = int(time.time())
disconnected_users = set()
# Long poll sessions.
for dead in redis.zrangebyscore("chats_alive", 0, current_time):
logger.info("Reaping %s" % dead)
chat_id, session_id = dead.split('/')
user_id = redis.hget("chat:%s:online" % chat_id, session_id)
disconnected = disconnect(redis, chat_id, session_id)
# Only send a timeout message if they were already online.
if not disconnected:
logger.info("Not sending timeout message.")
continue
disconnected_users.add((chat_id, user_id, False))
# Sockets.
for dead in redis.zrangebyscore("sockets_alive", 0, current_time):
logger.info("Reaping %s" % dead)
chat_id, session_id, socket_id = dead.split('/')
user_id = redis.hget("chat:%s:online" % chat_id, socket_id)
disconnected = disconnect(redis, chat_id, socket_id)
redis.srem("chat:%s:sockets:%s" % (chat_id, session_id), socket_id)
redis.zrem("sockets_alive", "%s/%s/%s" % (chat_id, session_id, socket_id))
# Only send a timeout message if they were already online.
if not disconnected:
logger.info("Not sending timeout message.")
continue
disconnected_users.add((chat_id, user_id, True))
for chat_id, user_id, reaped_socket in disconnected_users:
try:
dead_chat_user = db.query(ChatUser).filter(and_(
ChatUser.user_id == user_id,
ChatUser.chat_id == chat_id,
)).options(joinedload(ChatUser.chat), joinedload(ChatUser.user)).one()
except NoResultFound:
logger.error("Unable to find ChatUser (chat %s, user %s)." % (chat_id, user_id))
continue
if reaped_socket:
typing_key = "chat:%s:typing" % chat_id
if redis.srem(typing_key, dead_chat_user.number):
redis.publish("channel:%s:typing" % chat_id, json.dumps({
"typing": list(int(_) for _ in redis.smembers(typing_key)),
}))
if dead_chat_user.computed_group == "silent" or dead_chat_user.chat.type in ("pm", "roulette"):
send_userlist(db, redis, dead_chat_user.chat)
else:
send_message(db, redis, Message(
chat_id=chat_id,
user_id=dead_chat_user.user_id,
type="timeout",
name=dead_chat_user.name,
text="%s's connection timed out." % dead_chat_user.name,
))
logger.info("Sent timeout message for ChatUser (chat %s, user %s)." % (chat_id, user_id))
def update_lastonline():
redis = update_lastonline.redis
redis_chat = NewparpRedis(connection_pool=redis_chat_pool)
if redis.exists("lock:lastonline"):
return
redis.setex("lock:lastonline", 60, 1)
chat_ids = redis.hgetall("queue:lastonline")
# Reset the list for the next iteration.
redis.delete("queue:lastonline")
for chat_id, posted in chat_ids.items():
online_user_ids = UserListStore(redis_chat, chat_id).user_ids_online()
try:
posted = datetime.datetime.utcfromtimestamp(float(posted))
except ValueError:
continue
with session_scope() as db:
db.query(Chat).filter(Chat.id == chat_id).update({"last_message": posted}, synchronize_session=False)
if len(online_user_ids) != 0:
db.query(ChatUser).filter(and_(
ChatUser.user_id.in_(online_user_ids),
ChatUser.chat_id == chat_id,
)).update({"last_online": posted}, synchronize_session=False)
redis.delete("lock:lastonline")
def update_lastonline():
redis = update_lastonline.redis
redis_chat = NewparpRedis(connection_pool=redis_chat_pool)
if redis.exists("lock:lastonline"):
return
redis.setex("lock:lastonline", 60, 1)
chat_ids = redis.hgetall("queue:lastonline")
# Reset the list for the next iteration.
redis.delete("queue:lastonline")
for chat_id, posted in chat_ids.items():
online_user_ids = UserListStore(redis_chat, chat_id).user_ids_online()
try:
posted = datetime.datetime.utcfromtimestamp(float(posted))
except ValueError:
continue
with session_scope() as db:
db.query(Chat).filter(Chat.id == chat_id).update(
{"last_message": posted}, synchronize_session=False)
if len(online_user_ids) != 0:
db.query(ChatUser).filter(
and_(
ChatUser.user_id.in_(online_user_ids),
ChatUser.chat_id == chat_id,
)).update({"last_online": posted},
synchronize_session=False)
redis.delete("lock:lastonline")
def unlist_chats():
with session_scope() as db:
db.query(GroupChat).filter(
and_(
GroupChat.id == Chat.id, GroupChat.publicity == "listed",
GroupChat.last_message < datetime.datetime.now() -
datetime.timedelta(7))).update({"publicity": "unlisted"})
db.commit()
def unlist_chats():
with session_scope() as db:
db.query(GroupChat).filter(and_(
GroupChat.id == Chat.id,
GroupChat.publicity == "listed",
GroupChat.last_message < datetime.datetime.now() - datetime.timedelta(7)
)).update({"publicity": "unlisted"})
db.commit()
def update_log_marker(chat_id, log_marker_type="page_with_system_messages"):
redis = update_log_marker.redis
chat_id = int(chat_id)
if log_marker_type == "page_without_system_messages":
message_filter = and_(
Message.chat_id == chat_id,
Message.type.in_(("ic", "ooc", "me")),
)
else:
message_filter = Message.chat_id == chat_id
with session_scope() as db:
# Fetch the last log marker.
last_log_marker = (db.query(LogMarker).filter(
and_(
LogMarker.chat_id == chat_id,
LogMarker.type == log_marker_type,
)).options(joinedload(LogMarker.message)).order_by(
LogMarker.number.desc()).first())
# See how much has happened since that marker.
if last_log_marker:
messages_since_last_marker = (db.query(
func.count("*")).select_from(Message).filter(
and_(
message_filter,
Message.posted >= last_log_marker.message.posted,
)).scalar())
# And create a new one if necessary.
if messages_since_last_marker > 200:
db.add(
LogMarker(
chat_id=chat_id,
type=log_marker_type,
number=last_log_marker.number + 1,
message_id=(db.query(Message.id).filter(
and_(
message_filter,
Message.posted >=
last_log_marker.message.posted,
)).order_by(
Message.posted).offset(200).limit(1).scalar()),
))
# Or create initial log marker if there aren't any.
else:
first_message = (db.query(Message).filter(message_filter).order_by(
Message.posted).first())
if first_message:
db.add(
LogMarker(
chat_id=chat_id,
type=log_marker_type,
number=1,
message_id=first_message.id,
))
def update_user_meta():
redis = update_user_meta.redis
if redis.exists("lock:metaupdate"):
return
redis.setex("lock:metaupdate", 60, 1)
meta_updates = redis.hgetall("queue:usermeta")
# Reset the list for the next iteration.
redis.delete("queue:usermeta")
for key, meta in meta_updates.items():
try:
meta = json.loads(meta)
except ValueError:
continue
msgtype, userid = key.split(":", 2)
with session_scope() as db:
if msgtype == "user" and "last_ip" in meta:
try:
last_online = datetime.datetime.utcfromtimestamp(
float(meta["last_online"]))
except (ValueError, KeyError):
last_online = datetime.datetime.now()
db.query(User).filter(User.id == userid).update(
{
"last_online": last_online,
"last_ip": meta["last_ip"]
},
synchronize_session=False)
elif msgtype == "chatuser":
try:
last_online = datetime.datetime.utcfromtimestamp(
float(meta["last_online"]))
except (ValueError, KeyError):
last_online = datetime.datetime.now()
db.query(ChatUser).filter(
and_(ChatUser.user_id == userid,
ChatUser.chat_id == meta["chat_id"])).update(
{"last_online": last_online},
synchronize_session=False)
redis.delete("lock:metaupdate")
def update_user_meta():
redis = update_user_meta.redis
if redis.exists("lock:metaupdate"):
return
redis.setex("lock:metaupdate", 60, 1)
meta_updates = redis.hgetall("queue:usermeta")
# Reset the list for the next iteration.
redis.delete("queue:usermeta")
for key, meta in meta_updates.items():
try:
meta = json.loads(meta)
except ValueError:
continue
msgtype, userid = key.split(":", 2)
with session_scope() as db:
if msgtype == "user" and "last_ip" in meta:
try:
last_online = datetime.datetime.utcfromtimestamp(float(meta["last_online"]))
except (ValueError, KeyError):
last_online = datetime.datetime.now()
db.query(User).filter(User.id == userid).update({
"last_online": last_online,
"last_ip": meta["last_ip"]
}, synchronize_session=False)
elif msgtype == "chatuser":
try:
last_online = datetime.datetime.utcfromtimestamp(float(meta["last_online"]))
except (ValueError, KeyError):
last_online = datetime.datetime.now()
db.query(ChatUser).filter(and_(ChatUser.user_id == userid, ChatUser.chat_id == meta["chat_id"])).update({
"last_online": last_online
}, synchronize_session=False)
redis.delete("lock:metaupdate")
def load_lists(self):
if lists["reload"] == self.redis.get("spamless:reload"):
return
logger.info("reload")
lists["reload"] = self.redis.get("spamless:reload")
with session_scope() as db:
filters = db.query(SpamlessFilter).all()
lists["banned_names"] = [
re.compile(_.regex, re.IGNORECASE | re.MULTILINE)
for _ in filters if _.type == "banned_names"
]
lists["blacklist"] = [
(re.compile(_.regex, re.IGNORECASE | re.MULTILINE), _.points)
for _ in filters if _.type == "blacklist"
]
lists["warnlist"] = [
re.compile(_.regex, re.IGNORECASE | re.MULTILINE)
for _ in filters if _.type == "warnlist"
]
def load_lists(self):
if lists["reload"] == self.redis.get("spamless:reload"):
return
logger.info("reload")
lists["reload"] = self.redis.get("spamless:reload")
with session_scope() as db:
filters = db.query(SpamlessFilter).all()
lists["banned_names"] = [
re.compile(_.regex, re.IGNORECASE | re.MULTILINE)
for _ in filters if _.type == "banned_names"
]
lists["blacklist"] = [
(re.compile(_.regex, re.IGNORECASE | re.MULTILINE), _.points)
for _ in filters if _.type == "blacklist"
]
lists["warnlist"] = [
re.compile(_.regex, re.IGNORECASE | re.MULTILINE)
for _ in filters if _.type == "warnlist"
]
def reap_chat(chat_id):
user_list = UserListStore(NewparpRedis(connection_pool=redis_chat_pool),
chat_id)
old_user_ids = user_list.user_ids_online()
if not old_user_ids:
return
with session_scope() as db:
chat = db.query(Chat).filter(Chat.id == chat_id).one()
chat_users = {
_.user_id: _
for _ in db.query(ChatUser).filter(
and_(
ChatUser.chat_id == chat_id,
ChatUser.user_id.in_(old_user_ids),
))
}
for socket_id, user_id in user_list.inconsistent_entries():
chat_user = chat_users[user_id]
dead = user_list.socket_disconnect(socket_id, chat_user.number)
if dead:
logger.debug("dead: %s" % chat_user)
# TODO optimise this when reaping several people at once?
if chat_user.computed_group == "silent" or chat.type in (
"pm", "roulette"):
send_userlist(user_list, db, chat)
else:
send_message(
db, reap_chat.redis,
Message(
chat=chat,
user_id=chat_user.user_id,
type="timeout",
name=chat_user.name,
text="%s's connection timed out." % chat_user.name,
), user_list)
def run(self, chat_id, data):
if len(data["messages"]) == 0:
return
self.load_lists()
for message in data["messages"]:
if message["user_number"] is None:
continue
message["hash"] = hashlib.md5(
":".join([message["color"], message["acronym"], message["text"]])
.encode("utf-8").lower()
).hexdigest()
try:
self.check_connection_spam(chat_id, message)
self.check_banned_names(chat_id, message)
self.check_message_filter(chat_id, message)
self.check_warnlist(chat_id, message)
except Mark as e:
with session_scope() as db:
q = db.query(Message).filter(Message.id == message["id"]).update({"spam_flag": str(e)})
message.update({"spam_flag": str(e)})
self.redis.publish("spamless:live", json.dumps(message))
except Silence as e:
with session_scope() as db:
# XXX maybe cache this?
try:
chat_user, user, chat = db.query(
ChatUser, User, AnyChat,
).join(
User, ChatUser.user_id == User.id,
).join(
AnyChat, ChatUser.chat_id == AnyChat.id,
).filter(and_(
ChatUser.chat_id == chat_id,
ChatUser.number == message["user_number"],
)).one()
except NoResultFound:
continue
if chat.type != "group":
flag_suffix = chat.type.upper()
elif chat_user.computed_group in ("admin", "creator"):
flag_suffix = chat_user.computed_group.upper()
else:
flag_suffix = "SILENCED"
db.query(Message).filter(Message.id == message["id"]).update({
"spam_flag": str(e) + " " + flag_suffix,
})
message.update({"spam_flag": str(e) + " " + flag_suffix})
self.redis.publish("spamless:live", json.dumps(message))
if flag_suffix == "SILENCED":
db.query(ChatUser).filter(and_(
ChatUser.chat_id == chat_id,
ChatUser.number == message["user_number"]
)).update({"group": "silent"})
send_message(db, self.redis, Message(
chat_id=chat_id,
type="spamless",
name="The Spamless",
acronym="\u264b",
text="Spam has been detected and silenced. Please come [url=http://help.msparp.com/]here[/url] or ask a chat moderator to unsilence you if this was an accident.",
color="626262"
), True)
def comparison_callback(results, searcher_id_1):
redis = comparison_callback.redis
if redis.exists("lock:matchmaker"):
logger.debug("locked. calling again in 1 second.")
comparison_callback.apply_async((results, searcher_id_1), countdown=1)
return
redis.setex("lock:matchmaker", 60, 1)
# Check if there's a match.
matched_searchers = [_ for _ in results if _[0] is not None]
if not matched_searchers:
logger.debug("no results")
redis.delete("lock:matchmaker")
return
logger.debug("results: %s" % matched_searchers)
shuffle(matched_searchers)
# Fetch searcher 1.
s1 = fetch_searcher(redis, searcher_id_1)
if not all(s1[:-2]):
logger.debug("%s has expired" % searcher_id_1)
redis.delete("lock:matchmaker")
return
with session_scope() as db:
# Pick a second searcher from the matches.
for searcher_id_2, options in matched_searchers:
s2 = fetch_searcher(redis, searcher_id_2)
if all(s2[:-2]) and db.query(func.count("*")).select_from(Block).filter(or_(
and_(Block.blocking_user_id == s1.user_id, Block.blocked_user_id == s2.user_id),
and_(Block.blocking_user_id == s2.user_id, Block.blocked_user_id == s1.user_id),
)).scalar() == 0:
logger.debug("matched %s" % searcher_id_2)
break
else:
logger.debug("all matches have expired")
redis.delete("lock:matchmaker")
return
new_url = str(uuid4()).replace("-", "")
logger.info("matched %s and %s, sending to %s." % (s1.id, s2.id, new_url))
new_chat = SearchedChat(url=new_url)
db.add(new_chat)
db.flush()
s1_user = db.query(User).filter(User.id == s1.user_id).one()
s2_user = db.query(User).filter(User.id == s2.user_id).one()
db.add(ChatUser.from_user(s1_user, chat_id=new_chat.id, number=1, search_character_id=s1.search_character_id, **s1.character))
if s1_user != s2_user:
db.add(ChatUser.from_user(s2_user, chat_id=new_chat.id, number=2, search_character_id=s2.search_character_id, **s2.character))
if options:
db.add(Message(
chat_id=new_chat.id,
type="search_info",
text=" ".join(option_messages[_] for _ in options),
))
pipe = redis.pipeline()
match_key = "matched:%s:%s" % tuple(sorted([s1.user_id, s2.user_id]))
pipe.set(match_key, 1)
pipe.expire(match_key, 1800)
pipe.srem("searchers", s1.id, s2.id)
match_message = """{"status":"matched","url":"%s"}""" % new_url
pipe.publish("searcher:%s" % s1.id, match_message)
pipe.publish("searcher:%s" % s2.id, match_message)
pipe.delete("lock:matchmaker")
pipe.execute()
def postgres_test():
with session_scope() as db:
db.query(SearchCharacter).first()
def run(self, chat_id, data):
if len(data["messages"]) == 0:
return
self.load_lists()
for message in data["messages"]:
if message["user_number"] is None:
continue
self.redis.set("spamless:last_id", message["id"])
message["hash"] = hashlib.md5(
":".join([message["color"], message["acronym"], message["text"]])
.encode("utf-8").lower()
).hexdigest()
try:
self.check_connection_spam(chat_id, message)
self.check_banned_names(chat_id, message)
self.check_message_filter(chat_id, message)
self.check_warnlist(chat_id, message)
except Mark as e:
with session_scope() as db:
q = db.query(Message).filter(Message.id == message["id"]).update({"spam_flag": str(e)})
message.update({"spam_flag": str(e)})
self.redis.publish("spamless:live", json.dumps(message))
except Silence as e:
with session_scope() as db:
# XXX maybe cache this?
try:
chat_user, user, chat = db.query(
ChatUser, User, AnyChat,
).join(
User, ChatUser.user_id == User.id,
).join(
AnyChat, ChatUser.chat_id == AnyChat.id,
).filter(and_(
ChatUser.chat_id == chat_id,
ChatUser.number == message["user_number"],
)).one()
except NoResultFound:
continue
if chat.type != "group":
flag_suffix = chat.type.upper()
elif chat_user.computed_group in ("admin", "creator"):
flag_suffix = chat_user.computed_group.upper()
else:
flag_suffix = "SILENCED"
db.query(Message).filter(Message.id == message["id"]).update({
"spam_flag": str(e) + " " + flag_suffix,
})
message.update({"spam_flag": str(e) + " " + flag_suffix})
self.redis.publish("spamless:live", json.dumps(message))
if flag_suffix == "SILENCED":
db.query(ChatUser).filter(and_(
ChatUser.chat_id == chat_id,
ChatUser.number == message["user_number"]
)).update({"group": "silent"})
send_message(db, self.redis, Message(
chat_id=chat_id,
type="spamless",
name="The Spamless",
acronym="\u264b",
text="Spam has been detected and silenced. Please come [url=http://help.msparp.com/]here[/url] or ask a chat moderator to unsilence you if this was an accident.",
color="626262"
), force_userlist=True)
def update_log_marker(chat_id, log_marker_type="page_with_system_messages"):
redis = update_log_marker.redis
chat_id = int(chat_id)
if log_marker_type == "page_without_system_messages":
message_filter = and_(
Message.chat_id == chat_id,
Message.type.in_(("ic", "ooc", "me")),
)
else:
message_filter = Message.chat_id == chat_id
with session_scope() as db:
# Fetch the last log marker.
last_log_marker = (
db.query(LogMarker)
.filter(and_(
LogMarker.chat_id == chat_id,
LogMarker.type == log_marker_type,
))
.options(joinedload(LogMarker.message))
.order_by(LogMarker.number.desc()).first()
)
# See how much has happened since that marker.
if last_log_marker:
messages_since_last_marker = (
db.query(func.count("*")).select_from(Message)
.filter(and_(
message_filter,
Message.posted >= last_log_marker.message.posted,
)).scalar()
)
# And create a new one if necessary.
if messages_since_last_marker > 200:
db.add(LogMarker(
chat_id=chat_id,
type=log_marker_type,
number=last_log_marker.number + 1,
message_id=(
db.query(Message.id)
.filter(and_(
message_filter,
Message.posted >= last_log_marker.message.posted,
)).order_by(Message.posted)
.offset(200).limit(1).scalar()
),
))
# Or create initial log marker if there aren't any.
else:
first_message = (
db.query(Message)
.filter(message_filter)
.order_by(Message.posted).first()
)
if first_message:
db.add(LogMarker(
chat_id=chat_id,
type=log_marker_type,
number=1,
message_id=first_message.id,
))