def open_session(self, app, request): session_id = request.cookies.get(app.session_cookie_name) expires = int(now() + 1000 * datetime.timedelta( minutes=self.EXPIRES_MINUTES).total_seconds()) if not session_id: # No session, generate one for the client return self.session_class(session_id=self.generate_session_id(), new=True, expires=expires) else: # Session id supplied, search for it in storage session = self.find_session_for_id(session_id) if session is not None: # Was a valid session id, check data if now() < session.expires: return self.session_class(initial=session, session_id=session.session_id, expires=session.expires) # Invalid session id or expired, generate a new one # Force so that it will always be processed in save session, either to be deleted or set return self.session_class(session_id=self.generate_session_id(), new=True, was_invalid=True, expires=expires)
def create_thread(board: BoardModel, post: PostModel) \ -> Tuple[PostResultModel, int, int]: start_time = now() with session() as s: board_orm_model = s.query(BoardOrmModel).filter_by(id=board.id).one() thread_orm_model = ThreadOrmModel() thread_orm_model.last_modified = now() thread_orm_model.refno = 0 thread_orm_model.board_id = board.id post_orm_model = post.to_orm_model() post_orm_model.thread = thread_orm_model post_orm_model.refno = 1 s.add(thread_orm_model) # Atomically update the refno counter board_orm_model.refno_counter = BoardOrmModel.refno_counter + 1 s.commit() # Set it to the board after the commit to make sure there aren't any duplicates thread_refno = thread_orm_model.refno = board_orm_model.refno_counter # Attach file to the post id if post.file: file_orm_model = post.file.to_orm_model() file_orm_model.post_id = post_orm_model.id s.add(file_orm_model) if post.moderator: post_orm_model.moderator_id = post.moderator.id # Purge overflowed threads threads_refnos_to_purge = _purge_threads(s, board, board.config.pages, board.config.per_page) s.commit() insert_time = now() - start_time start_time = now() for purging_refno in threads_refnos_to_purge: cache.delete(cache_key('thread', board.name, purging_refno)) cache.delete(cache_key('thread_stub', board.name, purging_refno)) thread = ThreadModel.from_orm_model(thread_orm_model) _invalidate_thread_cache(s, thread, board) _invalidate_board_pages_catalog_cache(s, board) document_cache.purge_board(board) cache_time = now() - start_time res = PostResultModel.from_board_name_thread_refno_post_refno( board.name, thread_refno, 1) return res, insert_time, cache_time
def create_post(board: BoardModel, thread: ThreadModel, post: PostModel, sage: bool) \ -> Tuple[PostResultModel, int, int]: start_time = now() with session() as s: post_orm_model = post.to_orm_model() s.add(post_orm_model) to_thread_orm_model = s.query(ThreadOrmModel).filter_by( id=thread.id).one() post_orm_model.thread = to_thread_orm_model post_orm_model.refno = 0 # Atomically update the refno counter to_thread_orm_model.refno_counter = ThreadOrmModel.refno_counter + 1 s.commit() # Set it to the post after the commit to make sure there aren't any duplicates post_refno = post_orm_model.refno = to_thread_orm_model.refno_counter post_id = post_orm_model.id # Attach file to the post id if post.file: file_orm_model = post.file.to_orm_model() file_orm_model.post_id = post_id s.add(file_orm_model) if post.moderator: post_orm_model.moderator_id = post.moderator.id modify_date = now() # Use the refno to avoid a count(*) if not sage and post_refno <= board.config.bump_limit: to_thread_orm_model.last_modified = modify_date s.commit() insert_time = now() - start_time start_time = now() _invalidate_thread_cache(s, thread, board) _invalidate_board_pages_catalog_cache(s, board) purge_thread_future = document_cache.purge_thread(board, thread) # Wait for the thread to be purged, otherwise the chance exists that the client reloads a cached version. # This only holds up the posting client, others have the updated memcache available. purge_thread_future.result() # Don't wait for this document_cache.purge_board(board) cache_time = now() - start_time res = PostResultModel.from_board_name_thread_refno_post_refno( board.name, thread.refno, post_refno) return res, insert_time, cache_time
def _queue_file(post_details): file = request.files['file'] start_time = now() thumbnail_size = configuration.app.thumbnail_reply if post_details.thread_refno else configuration.app.thumbnail_op try: post_details.uploaded_file, upload_queue_files = file_service.prepare_upload(file, thumbnail_size) except ArgumentError as e: raise BadRequestError(e.message) post_details.file_time = now() - start_time return upload_queue_files
def create_thread(board: BoardModel, post: PostModel) \ -> Tuple[PostResultModel, int, int]: start_time = now() with session() as s: board_orm_model = s.query(BoardOrmModel).filter_by(id=board.id).one() thread_orm_model = ThreadOrmModel() thread_orm_model.last_modified = now() thread_orm_model.refno = 0 thread_orm_model.board_id = board.id post_orm_model = post.to_orm_model() post_orm_model.thread = thread_orm_model post_orm_model.refno = 1 s.add(thread_orm_model) # Atomically update the refno counter board_orm_model.refno_counter = BoardOrmModel.refno_counter + 1 s.commit() # Set it to the board after the commit to make sure there aren't any duplicates thread_refno = thread_orm_model.refno = board_orm_model.refno_counter # Attach file to the post id if post.file: file_orm_model = post.file.to_orm_model() file_orm_model.post_id = post_orm_model.id s.add(file_orm_model) if post.moderator: post_orm_model.moderator_id = post.moderator.id # Purge overflowed threads threads_refnos_to_purge = _purge_threads(s, board, board.config.pages, board.config.per_page) s.commit() insert_time = now() - start_time start_time = now() for purging_refno in threads_refnos_to_purge: cache.delete(cache_key('thread', board.name, purging_refno)) cache.delete(cache_key('thread_stub', board.name, purging_refno)) thread = ThreadModel.from_orm_model(thread_orm_model) _invalidate_thread_cache(s, thread, board) _invalidate_board_pages_catalog_cache(s, board) document_cache.purge_board(board) cache_time = now() - start_time res = PostResultModel.from_board_name_thread_refno_post_refno(board.name, thread_refno, 1) return res, insert_time, cache_time
def create_post(board: BoardModel, thread: ThreadModel, post: PostModel, sage: bool) \ -> Tuple[PostResultModel, int, int]: start_time = now() with session() as s: post_orm_model = post.to_orm_model() s.add(post_orm_model) to_thread_orm_model = s.query(ThreadOrmModel).filter_by(id=thread.id).one() post_orm_model.thread = to_thread_orm_model post_orm_model.refno = 0 # Atomically update the refno counter to_thread_orm_model.refno_counter = ThreadOrmModel.refno_counter + 1 s.commit() # Set it to the post after the commit to make sure there aren't any duplicates post_refno = post_orm_model.refno = to_thread_orm_model.refno_counter post_id = post_orm_model.id # Attach file to the post id if post.file: file_orm_model = post.file.to_orm_model() file_orm_model.post_id = post_id s.add(file_orm_model) if post.moderator: post_orm_model.moderator_id = post.moderator.id modify_date = now() # Use the refno to avoid a count(*) if not sage and post_refno <= board.config.bump_limit: to_thread_orm_model.last_modified = modify_date s.commit() insert_time = now() - start_time start_time = now() _invalidate_thread_cache(s, thread, board) _invalidate_board_pages_catalog_cache(s, board) purge_thread_future = document_cache.purge_thread(board, thread) # Wait for the thread to be purged, otherwise the chance exists that the client reloads a cached version. # This only holds up the posting client, others have the updated memcache available. purge_thread_future.result() # Don't wait for this document_cache.purge_board(board) cache_time = now() - start_time res = PostResultModel.from_board_name_thread_refno_post_refno(board.name, thread.refno, post_refno) return res, insert_time, cache_time
def is_request_suspended(ip4: int, board: BoardModel, thread: ThreadModel) -> Tuple[bool, int]: timeout = NEW_THREAD_COOLDOWN if thread is None else NEW_POST_COOLDOWN from_time = now() - timeout post_list = posts.find_posts_by_ip4_from_time(ip4, from_time, by_thread=thread) if post_list: most_recent = post_list[0] time_left = (most_recent.date + timeout - now()) // 1000 return True, time_left return False, 0
def is_request_suspended(ip4: int, board: BoardModel, thread: ThreadModel) -> Tuple[bool, int]: timeout = NEW_THREAD_COOLDOWN if thread is None else NEW_POST_COOLDOWN from_time = now() - timeout post_list = posts.find_posts_by_ip4_from_time(ip4, from_time, by_thread=thread) if post_list: most_recent = post_list[0] time_left = (most_recent.date + timeout - now()) // 1000 return True, time_left return False, 0
def _is_verifications_valid(verifying_client: VerifyingClient, verifications: VerificationsModel): time = now() ip_correct = verifications.ip4 == verifying_client.ip4 expired = verifications.expires < time return ip_correct and not expired
def report_post(post: PostModel): existing_report = reports.find_by_post(post) if existing_report: reports.increase_report_count(existing_report) else: report = ReportModel.from_post_count_date(post, 1, now()) reports.create(report)
def add_ban(ban: BanModel) -> BanModel: if ban.length > MAX_BAN_TIME: raise ArgumentError(MESSAGE_BAN_TOO_LONG) if ban.ip4_end is not None and ban.ip4_end <= ban.ip4: raise ArgumentError(MESSAGE_IP4_ILLEGAL_RANGE) if ban.board: board = board_service.find_board(ban.board) if not board: raise ArgumentError(MESSAGE_BOARD_NOT_FOUND) if ban.reason and len(ban.reason) > MAX_REASON_LENGTH: raise ArgumentError(MESSAGE_BAN_TEXT_TOO_LONG) ban.date = now() ban = bans.create_ban(ban) for_board_text = ' on {}'.format(ban.board) if ban.board else '' ip4_end_text = ip4_to_str(ban.ip4_end) if ban.ip4_end is not None else '-' f = 'ban add {} from {} to {}{} for {} hours reason {}' text = f.format(ban.id, ip4_to_str(ban.ip4), ip4_end_text, for_board_text, ban.length / 60 / 60 / 1000, ban.reason) mod_log(text) return ban
def report_post(post: PostModel): existing_report = reports.find_by_post(post) if existing_report: reports.increase_report_count(existing_report) else: report = ReportModel.from_post_count_date(post, 1, now()) reports.create(report)
def is_verified(verifying_client: VerifyingClient) -> bool: verification_model = None verification_model_cache = cache.get( cache_key('verifications', verifying_client.verification_id)) if verification_model_cache: verification_model = VerificationsModel.from_cache( verification_model_cache) if not verification_model: with session() as s: q = s.query(VerificationOrmModel) q = q.filter_by(verification_id=verifying_client.verification_id) verifications_orm_model = q.one_or_none() if verifications_orm_model: verification_model = VerificationsModel.from_orm_model( verifications_orm_model) cached = verification_model.to_cache() timeout = max(1, (verification_model.expires - now()) // 1000) cache.set(cache_key('verifications', verification_model.id), cached, timeout=timeout) s.commit() return verification_model and _is_verifications_valid( verifying_client, verification_model)
def add_ban(ban: BanModel) -> BanModel: if ban.length > MAX_BAN_TIME: raise ArgumentError(MESSAGE_BAN_TOO_LONG) if ban.ip4_end is not None and ban.ip4_end <= ban.ip4: raise ArgumentError(MESSAGE_IP4_ILLEGAL_RANGE) if ban.board: board = board_service.find_board(ban.board) if not board: raise ArgumentError(MESSAGE_BOARD_NOT_FOUND) if ban.reason and len(ban.reason) > MAX_REASON_LENGTH: raise ArgumentError(MESSAGE_BAN_TEXT_TOO_LONG) ban.date = now() ban = bans.create_ban(ban) for_board_text = ' on {}'.format(ban.board) if ban.board else '' ip4_end_text = ip4_to_str(ban.ip4_end) if ban.ip4_end is not None else '-' f = 'ban add {} from {} to {}{} for {} hours reason {}' text = f.format(ban.id, ip4_to_str(ban.ip4), ip4_end_text, for_board_text, ban.length / 60 / 60 / 1000, ban.reason) mod_log(text) return ban
def get(self, key): try: expires, item = self.items[key] if now() < expires: return item except KeyError: pass return None
def get(self, key): try: expires, item = self.items[key] if now() < expires: return item except KeyError: pass return None
def _is_verifications_valid(verifying_client: VerifyingClient, verifications: VerificationsModel): time = now() ip_correct = verifications.ip4 == verifying_client.ip4 expired = verifications.expires < time return ip_correct and not expired
def _convert_exception(exception): try: raise exception except RequestBannedException: raise BadRequestError(MESSAGE_REQUEST_BANNED) except RequestSuspendedException as e: raise BadRequestError(MESSAGE_REQUEST_SUSPENDED.format(time_remaining(now() + 1000 * e.suspend_time))) except ArgumentError as e: raise BadRequestError(e.message)
def open_session(self, app, request): session_id = request.cookies.get(app.session_cookie_name) expires = int(now() + 1000 * datetime.timedelta(minutes=self.EXPIRES_MINUTES).total_seconds()) if not session_id: # No session, generate one for the client return self.session_class(session_id=self.generate_session_id(), new=True, expires=expires) else: # Session id supplied, search for it in storage session = self.find_session_for_id(session_id) if session is not None: # Was a valid session id, check data if now() < session.expires: return self.session_class(initial=session, session_id=session.session_id, expires=session.expires) # Invalid session id or expired, generate a new one # Force so that it will always be processed in save session, either to be deleted or set return self.session_class(session_id=self.generate_session_id(), new=True, was_invalid=True, expires=expires)
def log(log_type: ModeratorLogType, moderator: ModeratorModel, board: BoardModel, text: str): log_model = ModeratorLogModel.from_date_type_text_moderator_board( date=now(), type=log_type.value, text=text, moderator=moderator, board=board) moderator_logs.create(log_model)
def set_verified(verifying_client: VerifyingClient) -> VerificationsModel: with session() as s: expires = now() + VERIFICATION_DURATION vid = verifying_client.verification_id ip4 = verifying_client.ip4 model = VerificationsModel.from_id_ip4_expires(vid, ip4, expires) orm_model = model.to_orm_model() s.add(orm_model) s.flush() m = VerificationsModel.from_orm_model(orm_model) s.commit() return m
def set_verified(verifying_client: VerifyingClient) -> VerificationsModel: with session() as s: expires = now() + VERIFICATION_DURATION vid = verifying_client.verification_id ip4 = verifying_client.ip4 model = VerificationsModel.from_id_ip4_expires(vid, ip4, expires) orm_model = model.to_orm_model() s.add(orm_model) s.flush() m = VerificationsModel.from_orm_model(orm_model) s.commit() return m
def row(self, ban: BanOrmModel): if ban.length > 0: expire_time = ban.date + ban.length until = formatted_time(expire_time) + ' - ' if expire_time - now() < 0: until += 'Expired, not viewed' else: until += time_remaining(expire_time) + ' remaining' else: until = 'Does not expire' delete_button = '<button class="confirm-button" name="ban_id" value="' + str( ban.id) + '">Lift ban</button>' return (ip4_to_str(ban.ip4), ip4_to_str(ban.ip4_end) if ban.ip4_end is not None else '', formatted_time(ban.date), until, ban.board or '', ban.reason, Markup(delete_button))
def time_remaining(t): remaining = t - now() past = False if remaining < 0: past = True remaining = -remaining ms_in_day = 1000 * 60 * 60 * 24 days = int(remaining // ms_in_day) remaining -= days * ms_in_day ms_in_hour = 1000 * 60 * 60 hours = int(remaining // ms_in_hour) remaining -= hours * ms_in_hour ms_in_minute = 1000 * 60 minutes = int(remaining // ms_in_minute) remaining -= minutes * ms_in_minute ms_in_second = 1000 seconds = int(remaining // ms_in_second) remaining -= seconds * ms_in_second text = '' if not days and not hours and not minutes: text += '{} second{}'.format(seconds, '' if seconds == 1 else 's') else: if days > 0: text += '{} day{}'.format(days, '' if days == 1 else 's') if hours > 0: text += ', ' else: text += ' and ' if hours > 0: text += '{} hour{} and '.format(hours, '' if hours == 1 else 's') text += '{} minute{}'.format(minutes, '' if minutes == 1 else 's') if past: text += ' ago' return text
def is_verified(verifying_client: VerifyingClient) -> bool: verification_model = None verification_model_cache = cache.get(cache_key('verifications', verifying_client.verification_id)) if verification_model_cache: verification_model = VerificationsModel.from_cache(verification_model_cache) if not verification_model: with session() as s: q = s.query(VerificationOrmModel) q = q.filter_by(verification_id=verifying_client.verification_id) verifications_orm_model = q.one_or_none() if verifications_orm_model: verification_model = VerificationsModel.from_orm_model(verifications_orm_model) cached = verification_model.to_cache() timeout = max(1, (verification_model.expires - now()) // 1000) cache.set(cache_key('verifications', verification_model.id), cached, timeout=timeout) s.commit() return verification_model and _is_verifications_valid(verifying_client, verification_model)
def row(self, ban: BanOrmModel): if ban.length > 0: expire_time = ban.date + ban.length until = formatted_time(expire_time) + ' - ' if expire_time - now() < 0: until += 'Expired, not viewed' else: until += time_remaining(expire_time) + ' remaining' else: until = 'Does not expire' delete_button = '<button class="confirm-button" name="ban_id" value="' + str(ban.id) + '">Lift ban</button>' return ( ip4_to_str(ban.ip4), ip4_to_str(ban.ip4_end) if ban.ip4_end is not None else '', formatted_time(ban.date), until, ban.board or '', ban.reason, Markup(delete_button) )
def verify(self, response): res = requests.post('https://www.google.com/recaptcha/api/siteverify', data={ 'secret': self.secret, 'response': response }) res_json = res.json() timestamp_iso = 'challenge_ts' in res_json and res_json['challenge_ts'] if not timestamp_iso: return False timestamp = dateutil.parser.parse(timestamp_iso) time_ago = now() - int(timestamp.timestamp() * 1000) if time_ago > 1000 * 60 * 30: # Expired return False if 'success' in res_json and res_json['success'] is True: return True return False
def verify(self, response): res = requests.post('https://www.google.com/recaptcha/api/siteverify', data={ 'secret': self.secret, 'response': response }) res_json = res.json() timestamp_iso = 'challenge_ts' in res_json and res_json['challenge_ts'] if not timestamp_iso: return False timestamp = dateutil.parser.parse(timestamp_iso) time_ago = now() - int(timestamp.timestamp() * 1000) if time_ago > 1000 * 60 * 30: # Expired return False if 'success' in res_json and res_json['success'] is True: return True return False
def set(self, key, value, timeout=15000): self.items[key] = (now() + timeout, value)
def ban_expired(ban: BanModel) -> bool: if ban.length == 0: return False return now() > ban.date + ban.length
def create_post(post_details: PostDetails) -> PostResultModel: start_time = now() board = board_service.find_board(post_details.board_name) if not board: raise ArgumentError(MESSAGE_BOARD_NOT_FOUND) to_thread = None if post_details.thread_refno is not None: to_thread = posts.find_thread_by_board_name_thread_refno(board.name, post_details.thread_refno) if to_thread is None: raise ArgumentError(MESSAGE_THREAD_NOT_FOUND) _check_post_details(post_details, to_thread, board) plugin_manager.execute_hook('on_handle_post', post_details) site_config = site_service.get_site_config() default_name = site_config.default_name # Get moderator if mod_id was set moderator = None if post_details.mod_id is not None: moderator = moderator_service.find_moderator_id(post_details.mod_id) if moderator is None: raise Exception('Moderator not found') post = PostModel() post.date = now() post.ip4 = post_details.ip4 if moderator is not None and moderator_service.moderates_board(moderator, board): post.moderator = moderator _handle_text(post, post_details) sage = _handle_name(post, post_details, default_name) _handle_subject(post, post_details, to_thread) _handle_password(post, post_details) if post_details.uploaded_file is not None: # TODO post.file = FileModel() post.file.location = post_details.uploaded_file.location post.file.thumbnail_location = post_details.uploaded_file.thumbnail_location post.file.original_name = post_details.uploaded_file.original_name post.file.width = post_details.uploaded_file.width post.file.height = post_details.uploaded_file.height post.file.size = post_details.uploaded_file.size post.file.thumbnail_width = post_details.uploaded_file.thumbnail_width post.file.thumbnail_height = post_details.uploaded_file.thumbnail_height handle_time = now() - start_time if to_thread is None: res, insert_time, cache_time = posts.create_thread(board, post) else: res, insert_time, cache_time = posts.create_post(board, to_thread, post, sage) _log_post(post_details, res, handle_time + insert_time, cache_time) return res
def ban_expired(ban: BanModel) -> bool: if ban.length == 0: return False return now() > ban.date + ban.length
def create_post(post_details: PostDetails) -> PostResultModel: start_time = now() board = board_service.find_board(post_details.board_name) if not board: raise ArgumentError(MESSAGE_BOARD_NOT_FOUND) to_thread = None if post_details.thread_refno is not None: to_thread = posts.find_thread_by_board_name_thread_refno( board.name, post_details.thread_refno) if to_thread is None: raise ArgumentError(MESSAGE_THREAD_NOT_FOUND) _check_post_details(post_details, to_thread, board) plugin_manager.execute_hook('on_handle_post', post_details) site_config = site_service.get_site_config() default_name = site_config.default_name # Get moderator if mod_id was set moderator = None if post_details.mod_id is not None: moderator = moderator_service.find_moderator_id(post_details.mod_id) if moderator is None: raise Exception('Moderator not found') post = PostModel() post.date = now() post.ip4 = post_details.ip4 if moderator is not None and moderator_service.moderates_board( moderator, board): post.moderator = moderator _handle_text(post, post_details) sage = _handle_name(post, post_details, default_name) _handle_subject(post, post_details, to_thread) _handle_password(post, post_details) if post_details.uploaded_file is not None: # TODO post.file = FileModel() post.file.location = post_details.uploaded_file.location post.file.thumbnail_location = post_details.uploaded_file.thumbnail_location post.file.original_name = post_details.uploaded_file.original_name post.file.width = post_details.uploaded_file.width post.file.height = post_details.uploaded_file.height post.file.size = post_details.uploaded_file.size post.file.thumbnail_width = post_details.uploaded_file.thumbnail_width post.file.thumbnail_height = post_details.uploaded_file.thumbnail_height handle_time = now() - start_time if to_thread is None: res, insert_time, cache_time = posts.create_thread(board, post) else: res, insert_time, cache_time = posts.create_post( board, to_thread, post, sage) _log_post(post_details, res, handle_time + insert_time, cache_time) return res
def set(self, key, value, timeout=15000): self.items[key] = (now() + timeout, value)
def log(log_type: ModeratorLogType, moderator: ModeratorModel, board: BoardModel, text: str): log_model = ModeratorLogModel.from_date_type_text_moderator_board( date=now(), type=log_type.value, text=text, moderator=moderator, board=board) moderator_logs.create(log_model)