def init_app(self, app): """ :param app: :class:`flask.Flask` instance to rate limit. """ self.enabled = app.config.setdefault(C.ENABLED, True) self._swallow_errors = app.config.setdefault( C.SWALLOW_ERRORS, self._swallow_errors ) self._headers_enabled = ( self._headers_enabled or app.config.setdefault(C.HEADERS_ENABLED, False) ) self._storage_options.update( app.config.get(C.STORAGE_OPTIONS, {}) ) self._storage = storage_from_string( self._storage_uri or app.config.setdefault(C.STORAGE_URL, 'memory://'), ** self._storage_options ) strategy = ( self._strategy or app.config.setdefault(C.STRATEGY, 'fixed-window') ) if strategy not in STRATEGIES: raise ConfigurationError("Invalid rate limiting strategy %s" % strategy) self._limiter = STRATEGIES[strategy](self._storage) self._header_mapping.update({ HEADERS.RESET : self._header_mapping.get(HEADERS.RESET,None) or app.config.setdefault(C.HEADER_RESET, "X-RateLimit-Reset"), HEADERS.REMAINING : self._header_mapping.get(HEADERS.REMAINING,None) or app.config.setdefault(C.HEADER_REMAINING, "X-RateLimit-Remaining"), HEADERS.LIMIT : self._header_mapping.get(HEADERS.LIMIT,None) or app.config.setdefault(C.HEADER_LIMIT, "X-RateLimit-Limit"), HEADERS.RETRY_AFTER : self._header_mapping.get(HEADERS.RETRY_AFTER,None) or app.config.setdefault(C.HEADER_RETRY_AFTER, "Retry-After"), }) self._retry_after = ( self._retry_after or app.config.get(C.HEADER_RETRY_AFTER_VALUE) ) conf_limits = app.config.get(C.GLOBAL_LIMITS, None) if not self._global_limits and conf_limits: self._global_limits = [ ExtLimit( limit, self._key_func, None, False, None, None, None ) for limit in parse_many(conf_limits) ] fallback_limits = app.config.get(C.IN_MEMORY_FALLBACK, None) if not self._in_memory_fallback and fallback_limits: self._in_memory_fallback = [ ExtLimit( limit, self._key_func, None, False, None, None, None ) for limit in parse_many(fallback_limits) ] if self._auto_check: app.before_request(self.__check_request_limit) app.after_request(self.__inject_headers) if self._in_memory_fallback: self._fallback_storage = MemoryStorage() self._fallback_limiter = STRATEGIES[strategy](self._fallback_storage) # purely for backward compatibility as stated in flask documentation if not hasattr(app, 'extensions'): app.extensions = {} # pragma: no cover app.extensions['limiter'] = self
def __init__( self, # app: Starlette = None, key_func: Callable[..., str], default_limits: List[StrOrCallableStr] = [], application_limits: List[StrOrCallableStr] = [], headers_enabled: bool = False, strategy: Optional[str] = None, storage_uri: Optional[str] = None, storage_options: Dict[str, str] = {}, auto_check: bool = True, swallow_errors: bool = False, in_memory_fallback: List[StrOrCallableStr] = [], in_memory_fallback_enabled: bool = False, retry_after: Optional[str] = None, key_prefix: str = "", enabled: bool = True, config_filename: Optional[str] = None, ) -> None: """ Configure the rate limiter at app level """ # assert app is not None, "Passing the app instance to the limiter is required" # self.app = app # app.state.limiter = self self.logger = logging.getLogger("slowapi") self.app_config = Config( config_filename if config_filename is not None else ".env") self.enabled = enabled self._default_limits = [] self._application_limits = [] self._in_memory_fallback: List[LimitGroup] = [] self._in_memory_fallback_enabled = (in_memory_fallback_enabled or len(in_memory_fallback) > 0) self._exempt_routes: Set[str] = set() self._request_filters: List[Callable[..., bool]] = [] self._headers_enabled = headers_enabled self._header_mapping: Dict[int, str] = {} self._retry_after: Optional[str] = retry_after self._strategy = strategy self._storage_uri = storage_uri self._storage_options = storage_options self._auto_check = auto_check self._swallow_errors = swallow_errors self._key_func = key_func self._key_prefix = key_prefix for limit in set(default_limits): self._default_limits.extend([ LimitGroup(limit, self._key_func, None, False, None, None, None, False) ]) for limit in application_limits: self._application_limits.extend([ LimitGroup(limit, self._key_func, "global", False, None, None, None, False) ]) for limit in in_memory_fallback: self._in_memory_fallback.extend([ LimitGroup(limit, self._key_func, None, False, None, None, None, False) ]) self._route_limits: Dict[str, List[Limit]] = {} self._dynamic_route_limits: Dict[str, List[LimitGroup]] = {} # a flag to note if the storage backend is dead (not available) self._storage_dead: bool = False self._fallback_limiter = None self.__check_backend_count = 0 self.__last_check_backend = time.time() self.__marked_for_limiting: Dict[str, List[Callable]] = {} class BlackHoleHandler(logging.StreamHandler): def emit(*_): return self.logger.addHandler(BlackHoleHandler()) self.enabled = self.get_app_config(C.ENABLED, self.enabled) self._swallow_errors = self.get_app_config(C.SWALLOW_ERRORS, self._swallow_errors) self._headers_enabled = self._headers_enabled or self.get_app_config( C.HEADERS_ENABLED, False) self._storage_options.update(self.get_app_config( C.STORAGE_OPTIONS, {})) self._storage: Storage = storage_from_string( self._storage_uri or self.get_app_config(C.STORAGE_URL, "memory://"), **self._storage_options, ) strategy = self._strategy or self.get_app_config( C.STRATEGY, "fixed-window") if strategy not in STRATEGIES: raise ConfigurationError("Invalid rate limiting strategy %s" % strategy) self._limiter: RateLimiter = STRATEGIES[strategy](self._storage) self._header_mapping.update({ HEADERS.RESET: self._header_mapping.get( HEADERS.RESET, self.get_app_config(C.HEADER_RESET, "X-RateLimit-Reset"), ), HEADERS.REMAINING: self._header_mapping.get( HEADERS.REMAINING, self.get_app_config(C.HEADER_REMAINING, "X-RateLimit-Remaining"), ), HEADERS.LIMIT: self._header_mapping.get( HEADERS.LIMIT, self.get_app_config(C.HEADER_LIMIT, "X-RateLimit-Limit"), ), HEADERS.RETRY_AFTER: self._header_mapping.get( HEADERS.RETRY_AFTER, self.get_app_config(C.HEADER_RETRY_AFTER, "Retry-After"), ), }) self._retry_after = self._retry_after or self.get_app_config( C.HEADER_RETRY_AFTER_VALUE) self._key_prefix = self._key_prefix or self.get_app_config( C.KEY_PREFIX) app_limits: Optional[StrOrCallableStr] = self.get_app_config( C.APPLICATION_LIMITS, None) if not self._application_limits and app_limits: self._application_limits = [ LimitGroup(app_limits, self._key_func, "global", False, None, None, None, False) ] conf_limits: Optional[StrOrCallableStr] = self.get_app_config( C.DEFAULT_LIMITS, None) if not self._default_limits and conf_limits: self._default_limits = [ LimitGroup(conf_limits, self._key_func, None, False, None, None, None, False) ] fallback_enabled = self.get_app_config(C.IN_MEMORY_FALLBACK_ENABLED, False) fallback_limits: Optional[StrOrCallableStr] = self.get_app_config( C.IN_MEMORY_FALLBACK, None) if not self._in_memory_fallback and fallback_limits: self._in_memory_fallback = [ LimitGroup( fallback_limits, self._key_func, None, False, None, None, None, False, ) ] if not self._in_memory_fallback_enabled: self._in_memory_fallback_enabled = ( fallback_enabled or len(self._in_memory_fallback) > 0) if self._in_memory_fallback_enabled: self._fallback_storage = MemoryStorage() self._fallback_limiter = STRATEGIES[strategy]( self._fallback_storage)
async def before_server_start(app_: Sanic, loop): mzk.set_process_name(f'{app_.name} --- listener {mzk.get_my_pid()}') mzk.print_info(f'Starting Sanic server. -- {mzk.get_my_pid()}') app_.system_config: mzk.MocaConfig = mzk.MocaConfig(core.SYSTEM_CONFIG, manual_reload=True) app_.ip_blacklist: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile( core.IP_BLACKLIST_FILE, manual_reload=True, remove_duplicates=True, ) app_.api_key_config: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile( core.API_KEY_FILE, manual_reload=True) app_.twitter: mzk.MocaTwitter = mzk.MocaTwitter( core.TWITTER_CONFIG['CONSUMER_KEY'], core.TWITTER_CONFIG['CONSUMER_SECRET'], core.TWITTER_CONFIG['ACCESS_TOKEN'], core.TWITTER_CONFIG['ACCESS_TOKEN_SECRET']) app_.dict_cache = {} app_.secure_log = mzk.MocaFileLog(core.LOG_DIR.joinpath('secure.log')) app_.scheduler = mzk.MocaScheduler() if core.SERVER_CONFIG['rate_limiter_redis_storage'] is None: app_._storage_for_rate_limiter = MemoryStorage() else: app_._storage_for_rate_limiter = RedisStorage( core.SERVER_CONFIG['rate_limiter_redis_storage']) app_.rate_limiter = FixedWindowElasticExpiryRateLimiter( app_._storage_for_rate_limiter) try: app_.mysql = mzk.MocaMysql( core.DB_CONFIG['mysql']['host'], int(core.DB_CONFIG['mysql']['port']), core.DB_CONFIG['mysql']['user'], core.DB_CONFIG['mysql']['password'], core.DB_CONFIG['mysql']['database'], int(core.DB_CONFIG['mysql']['min_size']), int(core.DB_CONFIG['mysql']['max_size']), ) app_.mysql.force_sync = mzk.try_to_bool( core.DB_CONFIG['mysql']['force_sync']) except KeyError as e: mzk.print_error( f'Mysql database configuration error. missing key: {e}') mzk.sys_exit(1) except MySQLError as e: mzk.print_error( "Can't connect to MySQL database, Please check your database configuration." ) mzk.print_error("And make sure your database is online.") mzk.print_error( "You can use 'python3 moca.py test-mysql-con' to check your database." ) mzk.print_error(f"<MySQLError: {e}>") mzk.sys_exit(1) try: app_.redis = mzk.MocaRedis( core.DB_CONFIG['redis']['host'], int(core.DB_CONFIG['redis']['port']), int(core.DB_CONFIG['redis']['db']), core.DB_CONFIG['redis']['password'], int(core.DB_CONFIG['mysql']['min_size']), int(core.DB_CONFIG['mysql']['max_size']), ) app_.redis.prefix = core.DB_CONFIG['redis']['prefix'] await app_.redis.test_con() except KeyError as e: mzk.print_error( f'Redis database configuration error. missing key: {e}') mzk.sys_exit(1) except (RedisError, ConnectionRefusedError) as e: mzk.print_error( "Can't connect to Redis database, Please check your database configuration." ) mzk.print_error("And make sure your database is online.") mzk.print_error( "You can use 'python3 moca.py test-redis-con' to check your database." ) mzk.print_error(f"<(RedisError, ConnectionRefusedError): {e}>") mzk.sys_exit(1) try: app_.simple_cache = mzk.MocaSimpleCache( int(core.DB_CONFIG['simple_cache']['pool_size']), int(core.DB_CONFIG['simple_cache']['page_size']), ) except KeyError as e: mzk.print_error(f'SimpleCache configuration error. missing key: {e}') mzk.sys_exit(1) def __reload_timer(application: Sanic) -> None: while True: mzk.sleep(1) application.system_config.reload_file() application.ip_blacklist.reload_file() application.api_key_config.reload_file() app_._timer_thread = Thread(target=__reload_timer, args=(app_, ), daemon=True) app_._timer_thread.start()
def setUp(self): self.storage = MemoryStorage()
async def before_server_start(app_: Sanic, loop): mzk.set_process_name(f'{app_.name} --- listener {mzk.get_my_pid()}') mzk.print_info(f'Starting Sanic server. -- {mzk.get_my_pid()}') app_.system_config: mzk.MocaConfig = mzk.MocaConfig(core.SYSTEM_CONFIG, manual_reload=True) app_.ip_blacklist: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile( core.IP_BLACKLIST_FILE, manual_reload=True, remove_duplicates=True, ) app_.api_key_config: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile( core.API_KEY_FILE, manual_reload=True) app_.flags = mzk.MocaSynchronizedJSONDictFile(core.FLAGS_FILE, manual_reload=True) app_.dict_cache = {} app_.secure_log = mzk.MocaFileLog(core.LOG_DIR.joinpath('secure.log')) app_.scheduler = mzk.MocaScheduler() if core.SERVER_CONFIG['rate_limiter_redis_storage'] is None: app_._storage_for_rate_limiter = MemoryStorage() else: app_._storage_for_rate_limiter = RedisStorage( core.SERVER_CONFIG['rate_limiter_redis_storage']) app_.rate_limiter = FixedWindowElasticExpiryRateLimiter( app_._storage_for_rate_limiter) try: app_.mysql = mzk.MocaMysql( core.DB_CONFIG['mysql']['host'], int(core.DB_CONFIG['mysql']['port']), core.DB_CONFIG['mysql']['user'], core.DB_CONFIG['mysql']['password'], core.DB_CONFIG['mysql']['database'], int(core.DB_CONFIG['mysql']['min_size']), int(core.DB_CONFIG['mysql']['max_size']), ) app_.mysql.force_sync = mzk.try_to_bool( core.DB_CONFIG['mysql']['force_sync']) except KeyError as e: mzk.print_error( f'Mysql database configuration error. missing key: {e}') mzk.sys_exit(1) except MySQLError as e: mzk.print_error( "Can't connect to MySQL database, Please check your database configuration." ) mzk.print_error("And make sure your database is online.") mzk.print_error( "You can use 'python3 moca.py test-mysql-con' to check your database." ) mzk.print_error(f"<MySQLError: {e}>") mzk.sys_exit(1) def __reload_timer(application: Sanic) -> None: while True: mzk.sleep(1) application.system_config.reload_file() application.ip_blacklist.reload_file() application.api_key_config.reload_file() application.flags.reload_file() app_._timer_thread = Thread(target=__reload_timer, args=(app_, ), daemon=True) app_._timer_thread.start() app_.bots = {} def reload_bot(the_updated_key, old_value, new_value, *args, **kwargs) -> None: application = kwargs['app'] for bot_dir in core.STORAGE_DIR.iterdir(): if bot_dir.is_dir(): application.bots[bot_dir.name] = mzk.MocaBot( bot_dir.name, bot_dir) con = application.mysql.get_a_new_con() cursor = con.cursor() cursor.execute(core.GET_BOTS_QUERY) res = cursor.fetchall() con.close() application.dict_cache['id'] = {} application.dict_cache['name'] = {} for info in res: application.dict_cache['id'][info[0]] = info[1] application.dict_cache['name'][info[1]] = info[0] reload_bot(None, None, None, app=app_) app_.flags.set('moca_bot_reload', False) app_.flags.add_handler('moca_bot_reload' + str(mzk.get_my_pid()), 'moca_bot_reload', reload_bot, kwargs={'app': app_})