def queue_user_meta(context, redis: NewparpRedis, last_ip: str):
redis.hset(
"queue:usermeta", "user:%s" % (context.user.id),
json.dumps({
"last_online": str(time.time()),
"last_ip": last_ip,
}))
def init_db(redis: NewparpRedis):
print("Attempting to init database.")
lock = str(uuid.uuid4())
redis.setex("lock:initdb", 60, lock)
time.sleep(random.uniform(1, 2))
if redis.get("lock:initdb") == lock:
print("Got the init lock, initating database.")
subprocess.call(["python3", os.path.dirname(os.path.realpath(__file__)) + "/../newparp/model/init_db.py"])
else:
print("Didn't get the init lock, waiting 5 seconds.")
time.sleep(5)
def update_lastonline():
redis = update_lastonline.redis
redis_chat = NewparpRedis(connection_pool=redis_chat_pool)
if redis.exists("lock:lastonline"):
return
redis.setex("lock:lastonline", 60, 1)
chat_ids = redis.hgetall("queue:lastonline")
# Reset the list for the next iteration.
redis.delete("queue:lastonline")
for chat_id, posted in chat_ids.items():
online_user_ids = UserListStore(redis_chat, chat_id).user_ids_online()
try:
posted = datetime.datetime.utcfromtimestamp(float(posted))
except ValueError:
continue
with session_scope() as db:
db.query(Chat).filter(Chat.id == chat_id).update(
{"last_message": posted}, synchronize_session=False)
if len(online_user_ids) != 0:
db.query(ChatUser).filter(
and_(
ChatUser.user_id.in_(online_user_ids),
ChatUser.chat_id == chat_id,
)).update({"last_online": posted},
synchronize_session=False)
redis.delete("lock:lastonline")
def get_ip_banned(ip_address: str,
db: Session,
redis: NewparpRedis,
use_cache: bool = True) -> bool:
cached_bans = redis.get("bans:%s" % (ip_address))
if cached_bans:
try:
return int(cached_bans) > 0
except (ValueError, TypeError):
pass
ip_bans = db.query(func.count('*')).select_from(IPBan).filter(
IPBan.address.op(">>=")(ip_address)).scalar()
banned = ip_bans > 0
redis.setex("bans:%s" % (ip_address), 60, ip_bans)
return banned
def init_db(redis: NewparpRedis):
print("Attempting to init database.")
lock = str(uuid.uuid4())
redis.setex("lock:initdb", 60, lock)
time.sleep(random.uniform(1, 2))
if redis.get("lock:initdb") == lock:
print("Got the init lock, initating database.")
subprocess.call([
"python3",
os.path.dirname(os.path.realpath(__file__)) +
"/../newparp/model/init_db.py"
])
else:
print("Didn't get the init lock, waiting 5 seconds.")
time.sleep(5)
def generate_counters():
logger.info("Generating user counters.")
redis_chat = NewparpRedis(connection_pool=redis_chat_pool)
user_ids_online = list(
UserListStore.multi_user_ids_online(
redis_chat,
UserListStore.scan_active_chats(redis_chat),
))
generate_counters.redis.set(
"connected_users",
len(set.union(*user_ids_online)) if user_ids_online else 0,
)
def broadcast_post():
title = request.form.get("title", "Global Announcement").strip()
text = request.form["text"].strip()
if not text:
abort(400)
if request.form["color"][0] == "#":
color = request.form["color"][1:]
else:
color = request.form["color"]
if not color_validator.match(color):
abort(400)
if request.form["headercolor"][0] == "#":
headercolor = request.form["headercolor"][1:]
else:
headercolor = request.form["headercolor"]
if not color_validator.match(headercolor):
abort(400)
g.db.add(
AdminLogEntry(
action_user=g.user,
type="broadcast",
description=text,
))
message_json = json.dumps({
"messages": [{
"id": None,
"user_number": None,
"posted": time.time(),
"type": "global",
"color": color,
"headercolor": headercolor,
"acronym": "",
"name": "",
"text": text,
"title": title,
"important": "important" in request.form
}]
})
for chat_id in UserListStore.scan_active_chats(
NewparpRedis(connection_pool=redis_chat_pool)):
g.redis.publish("channel:%s" % chat_id, message_json)
return redirect(url_for("admin_broadcast"))
def reap_chat(chat_id):
user_list = UserListStore(NewparpRedis(connection_pool=redis_chat_pool),
chat_id)
old_user_ids = user_list.user_ids_online()
if not old_user_ids:
return
with session_scope() as db:
chat = db.query(Chat).filter(Chat.id == chat_id).one()
chat_users = {
_.user_id: _
for _ in db.query(ChatUser).filter(
and_(
ChatUser.chat_id == chat_id,
ChatUser.user_id.in_(old_user_ids),
))
}
for socket_id, user_id in user_list.inconsistent_entries():
chat_user = chat_users[user_id]
dead = user_list.socket_disconnect(socket_id, chat_user.number)
if dead:
logger.debug("dead: %s" % chat_user)
# TODO optimise this when reaping several people at once?
if chat_user.computed_group == "silent" or chat.type in (
"pm", "roulette"):
send_userlist(user_list, db, chat)
else:
send_message(
db, reap_chat.redis,
Message(
chat=chat,
user_id=chat_user.user_id,
type="timeout",
name=chat_user.name,
text="%s's connection timed out." % chat_user.name,
), user_list)
def send_message(db, redis, message, user_list=None, force_userlist=False):
db.add(message)
db.flush()
message_dict = message.to_dict()
# Cache before sending.
cache_key = "chat:%s" % message.chat_id
redis.zadd(cache_key, message.id, json.dumps(message_dict))
redis.zremrangebyrank(cache_key, 0, -51)
# Prepare pubsub message
redis_message = {
"messages": [message_dict],
}
# Reload userlist if necessary.
if message.type in (
"join",
"disconnect",
"timeout",
"user_info",
"user_group",
"user_action",
) or force_userlist:
if user_list is None:
user_list = UserListStore(
NewparpRedis(connection_pool=redis_chat_pool), message.chat_id)
redis_message["users"] = get_userlist(user_list, db)
# Reload chat metadata if necessary.
if message.type == "chat_meta":
redis_message["chat"] = message.chat.to_dict()
redis.publish("channel:%s" % message.chat_id, json.dumps(redis_message))
# Send notifications.
if message.type in ("ic", "ooc", "me", "spamless"):
# Queue an update for the last_online field.
# TODO move the PM stuff here too
redis.hset(
"queue:lastonline", message.chat.id,
time.mktime(message.posted.timetuple()) +
float(message.posted.microsecond) / 1000000)
if user_list is None:
user_list = UserListStore(
NewparpRedis(connection_pool=redis_chat_pool), message.chat_id)
online_user_ids = user_list.user_ids_online()
if message.chat.type == "pm":
offline_chat_users = db.query(ChatUser).filter(
and_(
~ChatUser.user_id.in_(online_user_ids),
ChatUser.chat_id == message.chat.id,
))
for chat_user in offline_chat_users:
# Only send a notification if it's not already unread.
if message.chat.last_message <= chat_user.last_online:
redis.publish("channel:pm:%s" % chat_user.user_id,
"{\"pm\":\"1\"}")
# And send the message to spamless last.
# 1 second delay to prevent the task from executing before we commit the message.
celery.send_task("newparp.tasks.spamless.CheckSpamTask",
args=(message.chat_id, redis_message),
countdown=1)
if redis.get("lock:initdb") == lock:
print("Got the init lock, initating database.")
subprocess.call([
"python3",
os.path.dirname(os.path.realpath(__file__)) +
"/../newparp/model/init_db.py"
])
else:
print("Didn't get the init lock, waiting 5 seconds.")
time.sleep(5)
try:
db = sm()
redis = NewparpRedis(connection_pool=redis_pool)
# Redis online check.
while True:
try:
redis.info()
break
except RedisError as e:
print("Encountered a Redis error, waiting a second.")
print(str(e))
time.sleep(1)
# Database online and initation check.
while True:
try:
db.query(SearchCharacterGroup).first()
authorize_joining,
get_userlist,
kick_check,
send_join_message,
send_userlist,
send_quit_message,
)
from newparp.helpers.matchmaker import validate_searcher_exists, refresh_searcher
from newparp.helpers.users import queue_user_meta
from newparp.model import sm, AnyChat, ChatUser, User, SearchCharacter
from newparp.model.connections import redis_pool, redis_chat_pool, NewparpRedis
from newparp.model.user_list import UserListStore, PingTimeoutException
from newparp.tasks.matchmaker import new_searcher
redis = NewparpRedis(connection_pool=redis_pool)
redis_chat = NewparpRedis(connection_pool=redis_chat_pool)
thread_pool = ThreadPoolExecutor()
origin_regex = re.compile("^https?:\/\/%s$" % os.environ["BASE_DOMAIN"].replace(".", "\."))
sockets = set()
DEBUG = "DEBUG" in os.environ or "--debug" in sys.argv
class ChatHandler(WebSocketHandler):
def groups(fmt=None):
style_filter = set()
for style in GroupChat.style.type.enums:
if style in request.args:
style_filter.add(style)
if not style_filter:
if g.user is not None:
style_filter = g.user.group_chat_styles
else:
style_filter.add("script")
level_filter = set()
for level in GroupChat.level.type.enums:
if level in request.args:
level_filter.add(level)
if not level_filter:
if g.user is not None:
level_filter = g.user.group_chat_levels
else:
level_filter.add("sfw")
if g.user is not None:
g.user.group_chat_styles = style_filter
g.user.group_chat_levels = level_filter
groups_query = g.db.query(GroupChat).filter(
and_(
GroupChat.publicity.in_(("listed", "pinned")),
GroupChat.style.in_(style_filter),
GroupChat.level.in_(level_filter),
)).all()
online_userlists = UserListStore.multi_user_ids_online(
NewparpRedis(connection_pool=redis_chat_pool),
(group.id for group in groups_query),
)
groups = []
for group, online_users in zip(groups_query, online_userlists):
online_user_count = len(set(online_users))
if online_user_count > 0:
groups.append((group, online_user_count))
groups.sort(key=lambda _: (_[0].publicity, _[1]), reverse=True)
chat_dicts = []
for chat, online in groups:
cd = chat.to_dict()
cd["online"] = online
chat_dicts.append(cd)
if fmt == "json":
return jsonify({
"chats": chat_dicts,
})
return render_template(
"groups.html",
level_options=level_options,
groups=chat_dicts,
style_filter=style_filter,
level_filter=level_filter,
)
lock = str(uuid.uuid4())
redis.setex("lock:initdb", 60, lock)
time.sleep(random.uniform(1, 2))
if redis.get("lock:initdb") == lock:
print("Got the init lock, initating database.")
subprocess.call(["python3", os.path.dirname(os.path.realpath(__file__)) + "/../newparp/model/init_db.py"])
else:
print("Didn't get the init lock, waiting 5 seconds.")
time.sleep(5)
try:
db = sm()
redis = NewparpRedis(connection_pool=redis_pool)
# Redis online check.
while True:
try:
redis.info()
break
except RedisError as e:
print("Encountered a Redis error, waiting a second.")
print(str(e))
time.sleep(1)
# Database online and initation check.
while True:
try:
db.query(SearchCharacterGroup).first()
def join(client, chat: Chat):
client.get("/" + chat.url)
user_list = UserListStore(NewparpRedis(connection_pool=redis_chat_pool),
chat.id)
user_list.socket_join(str(uuid.uuid4()), g.session_id, g.user_id)
def chat_list(fmt=None, type=None, page=1):
if type is None:
type = "all"
try:
ChatClass = chat_classes[type]
except KeyError:
abort(404)
chats = g.db.query(ChatUser, ChatClass).join(ChatClass).filter(
and_(
ChatUser.user_id == g.user.id,
ChatUser.subscribed == True,
))
if type == "unread":
chats = chats.filter(ChatClass.last_message > ChatUser.last_online)
chats = chats.order_by(ChatClass.last_message.desc(), ).offset(
(page - 1) * 50).limit(50).all()
if len(chats) == 0 and page != 1:
abort(404)
chat_count = g.db.query(func.count('*')).select_from(ChatUser).filter(
and_(
ChatUser.user_id == g.user.id,
ChatUser.subscribed == True,
))
if type == "unread":
chat_count = chat_count.join(ChatClass).filter(
ChatClass.last_message > ChatUser.last_online, )
elif type is not None:
chat_count = chat_count.join(ChatClass)
chat_count = chat_count.scalar()
online_userlists = UserListStore.multi_user_ids_online(
NewparpRedis(connection_pool=redis_chat_pool),
(c[1].id for c in chats),
)
chat_dicts = []
for (chat_user, chat), online_user_ids in zip(chats, online_userlists):
if chat.type == "pm":
pm_chat_user = g.db.query(ChatUser).filter(
and_(
ChatUser.chat_id == chat.id,
ChatUser.user_id != g.user.id,
)).options(joinedload(ChatUser.user)).first()
else:
pm_chat_user = None
cd = chat.to_dict(
pm_user=pm_chat_user.user if pm_chat_user is not None else None)
cd["online"] = len(set(online_user_ids))
if chat.type == "pm":
cd["partner_online"] = pm_chat_user.user.id in (
int(_) for _ in online_user_ids)
cd["unread"] = chat.last_message > chat_user.last_online
chat_dicts.append({
"chat_user":
chat_user.to_dict(include_title_and_notes=True),
"chat":
cd,
})
if fmt == "json":
return jsonify({
"total": chat_count,
"chats": chat_dicts,
})
paginator = paginate.Page(
[],
page=page,
items_per_page=50,
item_count=chat_count,
url_maker=lambda page: url_for("rp_chat_list", page=page, type=type),
)
return render_template(
"chat_list.html",
level_options=level_options,
type=type,
chats=chat_dicts,
paginator=paginator,
chat_classes=chat_classes,
)
def decorated_function(url, fmt=None, *args, **kwargs):
# Helper for doing some special URL stuff with PM chats.
# Normally we just query for a Chat object with the url. However, PM chat
# URLs take the form "pm/<username>", so we have to look up the username,
# find the User it belongs to, and use our URL and theirs to create a
# special URL.
if url == "pm":
abort(404)
elif url.startswith("pm/"):
username = url[3:]
if username == "":
abort(404)
# You can't PM yourself.
if g.user is None or username.lower() == g.user.username.lower():
abort(404)
try:
pm_user = g.db.query(User).filter(
func.lower(User.username) == username.lower()).one()
except NoResultFound:
abort(404)
# Fix case if necessary.
if pm_user.username != username:
if request.method != "GET":
abort(404)
return redirect(
url_for(request.endpoint,
url="pm/" + pm_user.username,
fmt=fmt))
# Generate URL from our user ID and their user ID.
# Sort so they're always in the same order.
pm_url = "pm/" + ("/".join(
sorted([str(g.user.id), str(pm_user.id)])))
try:
chat = g.db.query(PMChat).filter(PMChat.url == pm_url, ).one()
except NoResultFound:
# Only create a new PMChat on the main chat page.
if request.endpoint != "rp_chat":
abort(404)
chat = PMChat(url=pm_url)
g.db.add(chat)
g.db.flush()
# Create ChatUser for the other user.
pm_chat_user = ChatUser.from_user(pm_user,
chat_id=chat.id,
number=1,
subscribed=True)
g.db.add(pm_chat_user)
g.db.flush()
return f(chat, pm_user, url, fmt, *args, **kwargs)
# Force lower case.
if url != url.lower():
if request.method != "GET":
abort(404)
return redirect(url_for(request.endpoint, url=url.lower(),
fmt=fmt))
try:
chat = g.db.query(AnyChat).filter(AnyChat.url == url).one()
except NoResultFound:
abort(404)
g.chat = chat
g.chat_id = chat.id
g.user_list = UserListStore(
NewparpRedis(connection_pool=redis_chat_pool), chat.id)
try:
authorize_joining(g.db, g)
except BannedException:
if request.endpoint != "rp_chat" or chat.url == "theoubliette":
abort(403)
if request.method != "GET":
abort(404)
return redirect(
url_for(request.endpoint, url="theoubliette", fmt=fmt))
except UnauthorizedException:
if request.endpoint != "rp_chat_unsubscribe":
return render_template("chat/private.html", chat=chat), 403
except TooManyPeopleException:
if request.endpoint == "rp_chat":
return render_template("chat/too_many_people.html",
chat=chat), 403
return f(chat, None, url, fmt, *args, **kwargs)
def reap():
redis_chat = NewparpRedis(connection_pool=redis_chat_pool)
for chat_id in UserListStore.scan_active_chats(redis_chat):
reap_chat.delay(chat_id)
def redis():
return NewparpRedis(connection_pool=redis_pool)
KickedException,
authorize_joining,
kick_check,
join,
send_userlist,
get_userlist,
disconnect,
send_quit_message,
)
from newparp.helpers.matchmaker import validate_searcher_exists, refresh_searcher
from newparp.helpers.users import queue_user_meta
from newparp.model import sm, AnyChat, Ban, ChatUser, Message, User, SearchCharacter
from newparp.model.connections import redis_pool, NewparpRedis
from newparp.tasks.matchmaker import new_searcher
redis = NewparpRedis(connection_pool=redis_pool)
thread_pool = ThreadPoolExecutor()
origin_regex = re.compile("^https?:\/\/%s$" %
os.environ["BASE_DOMAIN"].replace(".", "\."))
sockets = set()
DEBUG = "DEBUG" in os.environ or "--debug" in sys.argv
class ChatHandler(WebSocketHandler):
@property
def db(self):
if hasattr(self, "_db") and self._db is not None:
return self._db
def send_message(db, redis, message, user_list=None, force_userlist=False):
db.add(message)
db.flush()
message_dict = message.to_dict()
if user_list is None:
redis_chat = NewparpRedis(connection_pool=redis_chat_pool)
user_list = UserListStore(redis_chat, message.chat_id)
else:
redis_chat = user_list.redis
# Cache before sending.
cache_key = "chat:%s" % message.chat_id
redis_chat.zadd(cache_key, message.id, json.dumps(message_dict))
redis_chat.zremrangebyrank(cache_key, 0, -51)
redis_chat.expire(cache_key, 604800)
# Prepare pubsub message
redis_message = {
"messages": [message_dict],
}
# Reload userlist if necessary.
if message.type in (
"join",
"disconnect",
"timeout",
"user_info",
"user_group",
"user_action",
) or force_userlist:
redis_message["users"] = get_userlist(user_list, db)
# Reload chat metadata if necessary.
if message.type == "chat_meta":
redis_message["chat"] = message.chat.to_dict()
redis.publish("channel:%s" % message.chat_id, json.dumps(redis_message))
# Send notifications.
if message.type in ("ic", "ooc", "me", "spamless"):
# Queue an update for the last_online field.
# TODO move the PM stuff here too
redis.hset("queue:lastonline", message.chat.id, time.mktime(message.posted.timetuple()) + float(message.posted.microsecond) / 1000000)
online_user_ids = user_list.user_ids_online()
if message.chat.type == "pm":
offline_chat_users = db.query(ChatUser).filter(and_(
~ChatUser.user_id.in_(online_user_ids),
ChatUser.chat_id == message.chat.id,
))
for chat_user in offline_chat_users:
# Only send a notification if it's not already unread.
if message.chat.last_message <= chat_user.last_online:
redis.publish("channel:pm:%s" % chat_user.user_id, "{\"pm\":\"1\"}")
# And send the message to spamless last.
# 1 second delay to prevent the task from executing before we commit the message.
celery.send_task("newparp.tasks.spamless.CheckSpamTask", args=(message.chat_id, redis_message), countdown=1)