def open_session(self, app, request):
session_id = request.cookies.get(app.session_cookie_name)
expires = int(now() + 1000 * datetime.timedelta(
minutes=self.EXPIRES_MINUTES).total_seconds())
if not session_id:
# No session, generate one for the client
return self.session_class(session_id=self.generate_session_id(),
new=True,
expires=expires)
else:
# Session id supplied, search for it in storage
session = self.find_session_for_id(session_id)
if session is not None:
# Was a valid session id, check data
if now() < session.expires:
return self.session_class(initial=session,
session_id=session.session_id,
expires=session.expires)
# Invalid session id or expired, generate a new one
# Force so that it will always be processed in save session, either to be deleted or set
return self.session_class(session_id=self.generate_session_id(),
new=True,
was_invalid=True,
expires=expires)
def create_thread(board: BoardModel, post: PostModel) \
-> Tuple[PostResultModel, int, int]:
start_time = now()
with session() as s:
board_orm_model = s.query(BoardOrmModel).filter_by(id=board.id).one()
thread_orm_model = ThreadOrmModel()
thread_orm_model.last_modified = now()
thread_orm_model.refno = 0
thread_orm_model.board_id = board.id
post_orm_model = post.to_orm_model()
post_orm_model.thread = thread_orm_model
post_orm_model.refno = 1
s.add(thread_orm_model)
# Atomically update the refno counter
board_orm_model.refno_counter = BoardOrmModel.refno_counter + 1
s.commit()
# Set it to the board after the commit to make sure there aren't any duplicates
thread_refno = thread_orm_model.refno = board_orm_model.refno_counter
# Attach file to the post id
if post.file:
file_orm_model = post.file.to_orm_model()
file_orm_model.post_id = post_orm_model.id
s.add(file_orm_model)
if post.moderator:
post_orm_model.moderator_id = post.moderator.id
# Purge overflowed threads
threads_refnos_to_purge = _purge_threads(s, board, board.config.pages,
board.config.per_page)
s.commit()
insert_time = now() - start_time
start_time = now()
for purging_refno in threads_refnos_to_purge:
cache.delete(cache_key('thread', board.name, purging_refno))
cache.delete(cache_key('thread_stub', board.name, purging_refno))
thread = ThreadModel.from_orm_model(thread_orm_model)
_invalidate_thread_cache(s, thread, board)
_invalidate_board_pages_catalog_cache(s, board)
document_cache.purge_board(board)
cache_time = now() - start_time
res = PostResultModel.from_board_name_thread_refno_post_refno(
board.name, thread_refno, 1)
return res, insert_time, cache_time
def create_post(board: BoardModel, thread: ThreadModel, post: PostModel, sage: bool) \
-> Tuple[PostResultModel, int, int]:
start_time = now()
with session() as s:
post_orm_model = post.to_orm_model()
s.add(post_orm_model)
to_thread_orm_model = s.query(ThreadOrmModel).filter_by(
id=thread.id).one()
post_orm_model.thread = to_thread_orm_model
post_orm_model.refno = 0
# Atomically update the refno counter
to_thread_orm_model.refno_counter = ThreadOrmModel.refno_counter + 1
s.commit()
# Set it to the post after the commit to make sure there aren't any duplicates
post_refno = post_orm_model.refno = to_thread_orm_model.refno_counter
post_id = post_orm_model.id
# Attach file to the post id
if post.file:
file_orm_model = post.file.to_orm_model()
file_orm_model.post_id = post_id
s.add(file_orm_model)
if post.moderator:
post_orm_model.moderator_id = post.moderator.id
modify_date = now()
# Use the refno to avoid a count(*)
if not sage and post_refno <= board.config.bump_limit:
to_thread_orm_model.last_modified = modify_date
s.commit()
insert_time = now() - start_time
start_time = now()
_invalidate_thread_cache(s, thread, board)
_invalidate_board_pages_catalog_cache(s, board)
purge_thread_future = document_cache.purge_thread(board, thread)
# Wait for the thread to be purged, otherwise the chance exists that the client reloads a cached version.
# This only holds up the posting client, others have the updated memcache available.
purge_thread_future.result()
# Don't wait for this
document_cache.purge_board(board)
cache_time = now() - start_time
res = PostResultModel.from_board_name_thread_refno_post_refno(
board.name, thread.refno, post_refno)
return res, insert_time, cache_time
def _queue_file(post_details):
file = request.files['file']
start_time = now()
thumbnail_size = configuration.app.thumbnail_reply if post_details.thread_refno else configuration.app.thumbnail_op
try:
post_details.uploaded_file, upload_queue_files = file_service.prepare_upload(file, thumbnail_size)
except ArgumentError as e:
raise BadRequestError(e.message)
post_details.file_time = now() - start_time
return upload_queue_files
def create_thread(board: BoardModel, post: PostModel) \
-> Tuple[PostResultModel, int, int]:
start_time = now()
with session() as s:
board_orm_model = s.query(BoardOrmModel).filter_by(id=board.id).one()
thread_orm_model = ThreadOrmModel()
thread_orm_model.last_modified = now()
thread_orm_model.refno = 0
thread_orm_model.board_id = board.id
post_orm_model = post.to_orm_model()
post_orm_model.thread = thread_orm_model
post_orm_model.refno = 1
s.add(thread_orm_model)
# Atomically update the refno counter
board_orm_model.refno_counter = BoardOrmModel.refno_counter + 1
s.commit()
# Set it to the board after the commit to make sure there aren't any duplicates
thread_refno = thread_orm_model.refno = board_orm_model.refno_counter
# Attach file to the post id
if post.file:
file_orm_model = post.file.to_orm_model()
file_orm_model.post_id = post_orm_model.id
s.add(file_orm_model)
if post.moderator:
post_orm_model.moderator_id = post.moderator.id
# Purge overflowed threads
threads_refnos_to_purge = _purge_threads(s, board, board.config.pages, board.config.per_page)
s.commit()
insert_time = now() - start_time
start_time = now()
for purging_refno in threads_refnos_to_purge:
cache.delete(cache_key('thread', board.name, purging_refno))
cache.delete(cache_key('thread_stub', board.name, purging_refno))
thread = ThreadModel.from_orm_model(thread_orm_model)
_invalidate_thread_cache(s, thread, board)
_invalidate_board_pages_catalog_cache(s, board)
document_cache.purge_board(board)
cache_time = now() - start_time
res = PostResultModel.from_board_name_thread_refno_post_refno(board.name, thread_refno, 1)
return res, insert_time, cache_time
def create_post(board: BoardModel, thread: ThreadModel, post: PostModel, sage: bool) \
-> Tuple[PostResultModel, int, int]:
start_time = now()
with session() as s:
post_orm_model = post.to_orm_model()
s.add(post_orm_model)
to_thread_orm_model = s.query(ThreadOrmModel).filter_by(id=thread.id).one()
post_orm_model.thread = to_thread_orm_model
post_orm_model.refno = 0
# Atomically update the refno counter
to_thread_orm_model.refno_counter = ThreadOrmModel.refno_counter + 1
s.commit()
# Set it to the post after the commit to make sure there aren't any duplicates
post_refno = post_orm_model.refno = to_thread_orm_model.refno_counter
post_id = post_orm_model.id
# Attach file to the post id
if post.file:
file_orm_model = post.file.to_orm_model()
file_orm_model.post_id = post_id
s.add(file_orm_model)
if post.moderator:
post_orm_model.moderator_id = post.moderator.id
modify_date = now()
# Use the refno to avoid a count(*)
if not sage and post_refno <= board.config.bump_limit:
to_thread_orm_model.last_modified = modify_date
s.commit()
insert_time = now() - start_time
start_time = now()
_invalidate_thread_cache(s, thread, board)
_invalidate_board_pages_catalog_cache(s, board)
purge_thread_future = document_cache.purge_thread(board, thread)
# Wait for the thread to be purged, otherwise the chance exists that the client reloads a cached version.
# This only holds up the posting client, others have the updated memcache available.
purge_thread_future.result()
# Don't wait for this
document_cache.purge_board(board)
cache_time = now() - start_time
res = PostResultModel.from_board_name_thread_refno_post_refno(board.name, thread.refno, post_refno)
return res, insert_time, cache_time
def is_request_suspended(ip4: int, board: BoardModel, thread: ThreadModel) -> Tuple[bool, int]:
timeout = NEW_THREAD_COOLDOWN if thread is None else NEW_POST_COOLDOWN
from_time = now() - timeout
post_list = posts.find_posts_by_ip4_from_time(ip4, from_time, by_thread=thread)
if post_list:
most_recent = post_list[0]
time_left = (most_recent.date + timeout - now()) // 1000
return True, time_left
return False, 0
def is_request_suspended(ip4: int, board: BoardModel,
thread: ThreadModel) -> Tuple[bool, int]:
timeout = NEW_THREAD_COOLDOWN if thread is None else NEW_POST_COOLDOWN
from_time = now() - timeout
post_list = posts.find_posts_by_ip4_from_time(ip4,
from_time,
by_thread=thread)
if post_list:
most_recent = post_list[0]
time_left = (most_recent.date + timeout - now()) // 1000
return True, time_left
return False, 0
def _is_verifications_valid(verifying_client: VerifyingClient, verifications: VerificationsModel):
time = now()
ip_correct = verifications.ip4 == verifying_client.ip4
expired = verifications.expires < time
return ip_correct and not expired
def report_post(post: PostModel):
existing_report = reports.find_by_post(post)
if existing_report:
reports.increase_report_count(existing_report)
else:
report = ReportModel.from_post_count_date(post, 1, now())
reports.create(report)
def add_ban(ban: BanModel) -> BanModel:
if ban.length > MAX_BAN_TIME:
raise ArgumentError(MESSAGE_BAN_TOO_LONG)
if ban.ip4_end is not None and ban.ip4_end <= ban.ip4:
raise ArgumentError(MESSAGE_IP4_ILLEGAL_RANGE)
if ban.board:
board = board_service.find_board(ban.board)
if not board:
raise ArgumentError(MESSAGE_BOARD_NOT_FOUND)
if ban.reason and len(ban.reason) > MAX_REASON_LENGTH:
raise ArgumentError(MESSAGE_BAN_TEXT_TOO_LONG)
ban.date = now()
ban = bans.create_ban(ban)
for_board_text = ' on {}'.format(ban.board) if ban.board else ''
ip4_end_text = ip4_to_str(ban.ip4_end) if ban.ip4_end is not None else '-'
f = 'ban add {} from {} to {}{} for {} hours reason {}'
text = f.format(ban.id, ip4_to_str(ban.ip4), ip4_end_text, for_board_text,
ban.length / 60 / 60 / 1000, ban.reason)
mod_log(text)
return ban
def report_post(post: PostModel):
existing_report = reports.find_by_post(post)
if existing_report:
reports.increase_report_count(existing_report)
else:
report = ReportModel.from_post_count_date(post, 1, now())
reports.create(report)
def is_verified(verifying_client: VerifyingClient) -> bool:
verification_model = None
verification_model_cache = cache.get(
cache_key('verifications', verifying_client.verification_id))
if verification_model_cache:
verification_model = VerificationsModel.from_cache(
verification_model_cache)
if not verification_model:
with session() as s:
q = s.query(VerificationOrmModel)
q = q.filter_by(verification_id=verifying_client.verification_id)
verifications_orm_model = q.one_or_none()
if verifications_orm_model:
verification_model = VerificationsModel.from_orm_model(
verifications_orm_model)
cached = verification_model.to_cache()
timeout = max(1, (verification_model.expires - now()) // 1000)
cache.set(cache_key('verifications', verification_model.id),
cached,
timeout=timeout)
s.commit()
return verification_model and _is_verifications_valid(
verifying_client, verification_model)
def add_ban(ban: BanModel) -> BanModel:
if ban.length > MAX_BAN_TIME:
raise ArgumentError(MESSAGE_BAN_TOO_LONG)
if ban.ip4_end is not None and ban.ip4_end <= ban.ip4:
raise ArgumentError(MESSAGE_IP4_ILLEGAL_RANGE)
if ban.board:
board = board_service.find_board(ban.board)
if not board:
raise ArgumentError(MESSAGE_BOARD_NOT_FOUND)
if ban.reason and len(ban.reason) > MAX_REASON_LENGTH:
raise ArgumentError(MESSAGE_BAN_TEXT_TOO_LONG)
ban.date = now()
ban = bans.create_ban(ban)
for_board_text = ' on {}'.format(ban.board) if ban.board else ''
ip4_end_text = ip4_to_str(ban.ip4_end) if ban.ip4_end is not None else '-'
f = 'ban add {} from {} to {}{} for {} hours reason {}'
text = f.format(ban.id, ip4_to_str(ban.ip4), ip4_end_text, for_board_text, ban.length / 60 / 60 / 1000, ban.reason)
mod_log(text)
return ban
def get(self, key):
try:
expires, item = self.items[key]
if now() < expires:
return item
except KeyError:
pass
return None
def get(self, key):
try:
expires, item = self.items[key]
if now() < expires:
return item
except KeyError:
pass
return None
def _is_verifications_valid(verifying_client: VerifyingClient,
verifications: VerificationsModel):
time = now()
ip_correct = verifications.ip4 == verifying_client.ip4
expired = verifications.expires < time
return ip_correct and not expired
def _convert_exception(exception):
try:
raise exception
except RequestBannedException:
raise BadRequestError(MESSAGE_REQUEST_BANNED)
except RequestSuspendedException as e:
raise BadRequestError(MESSAGE_REQUEST_SUSPENDED.format(time_remaining(now() + 1000 * e.suspend_time)))
except ArgumentError as e:
raise BadRequestError(e.message)
def open_session(self, app, request):
session_id = request.cookies.get(app.session_cookie_name)
expires = int(now() + 1000 * datetime.timedelta(minutes=self.EXPIRES_MINUTES).total_seconds())
if not session_id:
# No session, generate one for the client
return self.session_class(session_id=self.generate_session_id(), new=True, expires=expires)
else:
# Session id supplied, search for it in storage
session = self.find_session_for_id(session_id)
if session is not None:
# Was a valid session id, check data
if now() < session.expires:
return self.session_class(initial=session, session_id=session.session_id, expires=session.expires)
# Invalid session id or expired, generate a new one
# Force so that it will always be processed in save session, either to be deleted or set
return self.session_class(session_id=self.generate_session_id(), new=True, was_invalid=True, expires=expires)
def log(log_type: ModeratorLogType, moderator: ModeratorModel,
board: BoardModel, text: str):
log_model = ModeratorLogModel.from_date_type_text_moderator_board(
date=now(),
type=log_type.value,
text=text,
moderator=moderator,
board=board)
moderator_logs.create(log_model)
def set_verified(verifying_client: VerifyingClient) -> VerificationsModel:
with session() as s:
expires = now() + VERIFICATION_DURATION
vid = verifying_client.verification_id
ip4 = verifying_client.ip4
model = VerificationsModel.from_id_ip4_expires(vid, ip4, expires)
orm_model = model.to_orm_model()
s.add(orm_model)
s.flush()
m = VerificationsModel.from_orm_model(orm_model)
s.commit()
return m
def set_verified(verifying_client: VerifyingClient) -> VerificationsModel:
with session() as s:
expires = now() + VERIFICATION_DURATION
vid = verifying_client.verification_id
ip4 = verifying_client.ip4
model = VerificationsModel.from_id_ip4_expires(vid, ip4, expires)
orm_model = model.to_orm_model()
s.add(orm_model)
s.flush()
m = VerificationsModel.from_orm_model(orm_model)
s.commit()
return m
def row(self, ban: BanOrmModel):
if ban.length > 0:
expire_time = ban.date + ban.length
until = formatted_time(expire_time) + ' - '
if expire_time - now() < 0:
until += 'Expired, not viewed'
else:
until += time_remaining(expire_time) + ' remaining'
else:
until = 'Does not expire'
delete_button = '<button class="confirm-button" name="ban_id" value="' + str(
ban.id) + '">Lift ban</button>'
return (ip4_to_str(ban.ip4),
ip4_to_str(ban.ip4_end) if ban.ip4_end is not None else '',
formatted_time(ban.date), until, ban.board
or '', ban.reason, Markup(delete_button))
def time_remaining(t):
remaining = t - now()
past = False
if remaining < 0:
past = True
remaining = -remaining
ms_in_day = 1000 * 60 * 60 * 24
days = int(remaining // ms_in_day)
remaining -= days * ms_in_day
ms_in_hour = 1000 * 60 * 60
hours = int(remaining // ms_in_hour)
remaining -= hours * ms_in_hour
ms_in_minute = 1000 * 60
minutes = int(remaining // ms_in_minute)
remaining -= minutes * ms_in_minute
ms_in_second = 1000
seconds = int(remaining // ms_in_second)
remaining -= seconds * ms_in_second
text = ''
if not days and not hours and not minutes:
text += '{} second{}'.format(seconds, '' if seconds == 1 else 's')
else:
if days > 0:
text += '{} day{}'.format(days, '' if days == 1 else 's')
if hours > 0:
text += ', '
else:
text += ' and '
if hours > 0:
text += '{} hour{} and '.format(hours, '' if hours == 1 else 's')
text += '{} minute{}'.format(minutes, '' if minutes == 1 else 's')
if past:
text += ' ago'
return text
def is_verified(verifying_client: VerifyingClient) -> bool:
verification_model = None
verification_model_cache = cache.get(cache_key('verifications', verifying_client.verification_id))
if verification_model_cache:
verification_model = VerificationsModel.from_cache(verification_model_cache)
if not verification_model:
with session() as s:
q = s.query(VerificationOrmModel)
q = q.filter_by(verification_id=verifying_client.verification_id)
verifications_orm_model = q.one_or_none()
if verifications_orm_model:
verification_model = VerificationsModel.from_orm_model(verifications_orm_model)
cached = verification_model.to_cache()
timeout = max(1, (verification_model.expires - now()) // 1000)
cache.set(cache_key('verifications', verification_model.id), cached, timeout=timeout)
s.commit()
return verification_model and _is_verifications_valid(verifying_client, verification_model)
def row(self, ban: BanOrmModel):
if ban.length > 0:
expire_time = ban.date + ban.length
until = formatted_time(expire_time) + ' - '
if expire_time - now() < 0:
until += 'Expired, not viewed'
else:
until += time_remaining(expire_time) + ' remaining'
else:
until = 'Does not expire'
delete_button = '<button class="confirm-button" name="ban_id" value="' + str(ban.id) + '">Lift ban</button>'
return (
ip4_to_str(ban.ip4),
ip4_to_str(ban.ip4_end) if ban.ip4_end is not None else '',
formatted_time(ban.date),
until,
ban.board or '',
ban.reason,
Markup(delete_button)
)
def verify(self, response):
res = requests.post('https://www.google.com/recaptcha/api/siteverify', data={
'secret': self.secret,
'response': response
})
res_json = res.json()
timestamp_iso = 'challenge_ts' in res_json and res_json['challenge_ts']
if not timestamp_iso:
return False
timestamp = dateutil.parser.parse(timestamp_iso)
time_ago = now() - int(timestamp.timestamp() * 1000)
if time_ago > 1000 * 60 * 30:
# Expired
return False
if 'success' in res_json and res_json['success'] is True:
return True
return False
def verify(self, response):
res = requests.post('https://www.google.com/recaptcha/api/siteverify',
data={
'secret': self.secret,
'response': response
})
res_json = res.json()
timestamp_iso = 'challenge_ts' in res_json and res_json['challenge_ts']
if not timestamp_iso:
return False
timestamp = dateutil.parser.parse(timestamp_iso)
time_ago = now() - int(timestamp.timestamp() * 1000)
if time_ago > 1000 * 60 * 30:
# Expired
return False
if 'success' in res_json and res_json['success'] is True:
return True
return False
def set(self, key, value, timeout=15000):
self.items[key] = (now() + timeout, value)
def ban_expired(ban: BanModel) -> bool:
if ban.length == 0:
return False
return now() > ban.date + ban.length
def create_post(post_details: PostDetails) -> PostResultModel:
start_time = now()
board = board_service.find_board(post_details.board_name)
if not board:
raise ArgumentError(MESSAGE_BOARD_NOT_FOUND)
to_thread = None
if post_details.thread_refno is not None:
to_thread = posts.find_thread_by_board_name_thread_refno(board.name, post_details.thread_refno)
if to_thread is None:
raise ArgumentError(MESSAGE_THREAD_NOT_FOUND)
_check_post_details(post_details, to_thread, board)
plugin_manager.execute_hook('on_handle_post', post_details)
site_config = site_service.get_site_config()
default_name = site_config.default_name
# Get moderator if mod_id was set
moderator = None
if post_details.mod_id is not None:
moderator = moderator_service.find_moderator_id(post_details.mod_id)
if moderator is None:
raise Exception('Moderator not found')
post = PostModel()
post.date = now()
post.ip4 = post_details.ip4
if moderator is not None and moderator_service.moderates_board(moderator, board):
post.moderator = moderator
_handle_text(post, post_details)
sage = _handle_name(post, post_details, default_name)
_handle_subject(post, post_details, to_thread)
_handle_password(post, post_details)
if post_details.uploaded_file is not None:
# TODO
post.file = FileModel()
post.file.location = post_details.uploaded_file.location
post.file.thumbnail_location = post_details.uploaded_file.thumbnail_location
post.file.original_name = post_details.uploaded_file.original_name
post.file.width = post_details.uploaded_file.width
post.file.height = post_details.uploaded_file.height
post.file.size = post_details.uploaded_file.size
post.file.thumbnail_width = post_details.uploaded_file.thumbnail_width
post.file.thumbnail_height = post_details.uploaded_file.thumbnail_height
handle_time = now() - start_time
if to_thread is None:
res, insert_time, cache_time = posts.create_thread(board, post)
else:
res, insert_time, cache_time = posts.create_post(board, to_thread, post, sage)
_log_post(post_details, res, handle_time + insert_time, cache_time)
return res
def ban_expired(ban: BanModel) -> bool:
if ban.length == 0:
return False
return now() > ban.date + ban.length
def create_post(post_details: PostDetails) -> PostResultModel:
start_time = now()
board = board_service.find_board(post_details.board_name)
if not board:
raise ArgumentError(MESSAGE_BOARD_NOT_FOUND)
to_thread = None
if post_details.thread_refno is not None:
to_thread = posts.find_thread_by_board_name_thread_refno(
board.name, post_details.thread_refno)
if to_thread is None:
raise ArgumentError(MESSAGE_THREAD_NOT_FOUND)
_check_post_details(post_details, to_thread, board)
plugin_manager.execute_hook('on_handle_post', post_details)
site_config = site_service.get_site_config()
default_name = site_config.default_name
# Get moderator if mod_id was set
moderator = None
if post_details.mod_id is not None:
moderator = moderator_service.find_moderator_id(post_details.mod_id)
if moderator is None:
raise Exception('Moderator not found')
post = PostModel()
post.date = now()
post.ip4 = post_details.ip4
if moderator is not None and moderator_service.moderates_board(
moderator, board):
post.moderator = moderator
_handle_text(post, post_details)
sage = _handle_name(post, post_details, default_name)
_handle_subject(post, post_details, to_thread)
_handle_password(post, post_details)
if post_details.uploaded_file is not None:
# TODO
post.file = FileModel()
post.file.location = post_details.uploaded_file.location
post.file.thumbnail_location = post_details.uploaded_file.thumbnail_location
post.file.original_name = post_details.uploaded_file.original_name
post.file.width = post_details.uploaded_file.width
post.file.height = post_details.uploaded_file.height
post.file.size = post_details.uploaded_file.size
post.file.thumbnail_width = post_details.uploaded_file.thumbnail_width
post.file.thumbnail_height = post_details.uploaded_file.thumbnail_height
handle_time = now() - start_time
if to_thread is None:
res, insert_time, cache_time = posts.create_thread(board, post)
else:
res, insert_time, cache_time = posts.create_post(
board, to_thread, post, sage)
_log_post(post_details, res, handle_time + insert_time, cache_time)
return res
def set(self, key, value, timeout=15000):
self.items[key] = (now() + timeout, value)
def log(log_type: ModeratorLogType, moderator: ModeratorModel, board: BoardModel, text: str):
log_model = ModeratorLogModel.from_date_type_text_moderator_board(
date=now(), type=log_type.value, text=text, moderator=moderator, board=board)
moderator_logs.create(log_model)
