def _validate_rate_limit(_ctx, _param, value): """Validate rate limit string.""" if value is None: return None try: limits.parse_many(value) except ValueError: raise click.BadParameter('Rate limit format: n/second, m/minute, etc.') return value
def _check_rate_limit_format(limit: str) -> bool: if isinstance(limit, str) and (len(limit) <= 256): try: if limit != '*': parse_many(limit) return True except ValueError: return False else: return False
def __init__(self, storage, limit, identifiers=None): if identifiers is None: identifiers = [] self._window = MovingWindowRateLimiter(storage) self._limits = parse_many(limit) self._identifiers = identifiers
def __iter__(self): limit_items = parse_many(self.__limit_provider( ) if callable(self.__limit_provider) else self.__limit_provider) for limit in limit_items: yield Limit(limit, self.key_function, self.__scope, self.per_method, self.methods, self.error_message, self.exempt_when)
def __iter__(self) -> Iterator[Limit]: if callable(self.__limit_provider): if "key" in inspect.signature(self.__limit_provider).parameters.keys(): assert ( "request" in inspect.signature(self.key_function).parameters.keys() ), f"Limit provider function {self.key_function.__name__} needs a `request` argument" if self.request is None: raise Exception("`request` object can't be None") limit_raw = self.__limit_provider(self.key_function(self.request)) else: limit_raw = self.__limit_provider() else: limit_raw = self.__limit_provider limit_items: List[RateLimitItem] = parse_many(limit_raw) for limit in limit_items: yield Limit( limit, self.key_function, self.__scope, self.per_method, self.methods, self.error_message, self.exempt_when, self.override_defaults, )
def __init__(self, storage, limit, identifiers=None): if identifiers is None: identifiers = [] self._window = MovingWindowRateLimiter(storage) self._limits = parse_many(limit) self._identifiers = identifiers
async def check_api_key(self, api_key: str, permission: str, request: Request) -> Tuple[int, str]: """ Return: status_code, response_message. Status Code ----------- -1: unknown api key. 0: allowed. 1: api_key format error. 2: invalid referer. 3: too many requests. 4: you reached the access limit. 5: permission error. """ if not self._check_api_key_format(api_key): return 1, 'api_key format error.' cache = await self._redis.get('moca-api-key-cache-' + api_key) if cache is None: pool = await self._mysql.get_aio_pool() async with pool.acquire() as con: async with con.cursor() as cursor: await cursor.execute(MocaAccess._GET_API_KEY_INFO, (api_key, )) res = await cursor.fetchall() if len(res) > 0: data = list(res[0]) data[2] = parse_many(data[2]) await self._redis.save('moca-api-key-cache-' + api_key, data) else: return -1, 'unknown api key.' else: data = cache referer = str( request.headers.get('referer', request.headers.get('origin'))) if (data[1] != '*') and (not referer.startswith(data[1])): return 2, 'invalid referer.' if data[2] != '*': for item in data[2]: if not self._api_key_window.hit( item, self.get_remote_address(request)): return 3, 'too many requests.' count = await self._redis.increment('moca-api-key-count-' + api_key) if self._sync_count >= 0: self._sync_count -= 1 else: self._sync_count = 64 pool = await self._mysql.get_aio_pool() async with pool.acquire() as con: async with con.cursor() as cursor: await cursor.execute(MocaAccess._UPDATE_COUNT, (count, api_key)) await con.commit() if count > data[3]: return 4, 'you reached the access limit.' if ('-RO-' not in data[5]) and (permission not in data[5]): return 5, 'permission error.' return 0, 'allowed.'
def __init__(self, storage, limit, *, identifiers=None, metrics): if identifiers is None: identifiers = [] self._storage = storage self._window = MovingWindowRateLimiter(storage) self._limits = parse_many(limit) self._identifiers = identifiers self._metrics = metrics
def make_rate_limiter(scope, limits): """Create a rate limiter. Multiple limits can be separated with a semicolon; in that case all limits are checked until one succeeds. This allows specifying a somewhat strict limit, but then a higher limit over a longer period of time to allow for bursts. """ limits = list(parse_many(limits)) if limits is not None else None return RateLimit(limiter, limiter._key_func, scope, limits)
def __init__(self, redis: MocaRedis, mysql: MocaMysql, global_rate_limit: str): MocaNamedInstance.__init__(self) MocaClassCache.__init__(self) self._redis: MocaRedis = redis self._mysql: MocaMysql = mysql self._redis_storage: RedisStorage = self._redis.get_redis_storage() self._api_key_window: FixedWindowElasticExpiryRateLimiter = FixedWindowElasticExpiryRateLimiter( self._redis_storage) self._ip_window: FixedWindowElasticExpiryRateLimiter = FixedWindowElasticExpiryRateLimiter( self._redis_storage) self._sync_count: int = 64 self._global_rate_limit = parse_many(global_rate_limit)
def __iter__(self) -> Iterator[Limit]: limit_items: List[RateLimitItem] = parse_many(self.__limit_provider( ) if callable(self.__limit_provider) else self.__limit_provider) for limit in limit_items: yield Limit( limit, self.key_function, self.__scope, self.per_method, self.methods, self.error_message, self.exempt_when, )
async def iterate(self, request: Request) -> Iterator[Limit]: limit_items: List[RateLimitItem] = parse_many( await self.__limit_provider(request) # noqa if inspect.iscoroutinefunction(self.__limit_provider ) else self.__limit_provider) for lmt in limit_items: yield Limit( lmt, self.key_function, self.__scope, self.per_method, self.methods, self.error_message, self.exempt_when, self.override_defaults, )
async def api_key_checker(request: Request): """A api-key filter.""" if request.method.upper() == 'OPTIONS': return text('success.') if not request.raw_url.startswith(b'/static') and \ not request.raw_url.startswith(b'/web') and \ not request.raw_url.startswith(b'/status') and \ not request.raw_url.startswith(b'/moca-twitter/static/icons/') and \ request.method.upper() != 'OPTIONS': received_key = mzk.get_args(request, ('api_key', str, None, {'max_length': 1024}))[0] ip = mzk.get_remote_address(request) if received_key is None: raise Forbidden('Missing API-KEY.') found = False for api_key_info in request.app.api_key_config.list: if api_key_info.get('key') == received_key: # found a api key. found = True if api_key_info.get('status', False): # check api key status. rate_limit = api_key_info.get('rate') if rate_limit != '*': for item in parse_many(rate_limit): # check rate limit. if not request.app.rate_limiter.hit(item, received_key): abort(429, 'Too many requests.') else: pass # '*' means unlimited. allowed = False target = request.raw_url.decode() for path_info in api_key_info.get('allowed_path'): if ':' in path_info: path, path_rate = path_info.split(':') else: path, path_rate, = path_info, '*' if target.startswith(path): if path_rate == '*': allowed = True else: for item in parse_many(path_rate): # check rate limit. if not request.app.rate_limiter.hit(item, f'{received_key}-{ip}-{path}'): abort(429, 'Too many requests.') allowed = True break if not allowed: raise Forbidden("Your API-KEY can't access to this path.") else: pass # allowed ip = mzk.get_remote_address(request) if api_key_info.get('ip') != '*' and ip not in api_key_info.get('ip'): raise Forbidden(f"Your API-KEY can't use from this ip address. ({ip})") else: pass # allowed required = api_key_info.get('required') for key, value in required['headers'].items(): if request.headers.get(key) != value: raise Forbidden('Missing required header.') for key, value in required['args'].items(): if mzk.get_args(request, key)[0] != value: raise Forbidden('Missing required argument.') if api_key_info.get('delay', 0) != 0: await sleep(api_key_info.get('delay')) # --- success. do nothing. --- else: raise Forbidden('Your API-KEY is not online.') break if not found: ip = mzk.get_remote_address(request) request.app.secure_log.write_log( f"Received a unknown API-KEY: <{received_key}> from {ip}.", mzk.LogLevel.WARNING ) if request.app.dict_cache.get('unknown_api_key') is None: request.app.dict_cache['unknown_api_key'] = {} try: request.app.dict_cache['unknown_api_key'][ip] += 1 except KeyError: request.app.dict_cache['unknown_api_key'][ip] = 1 if request.app.dict_cache['unknown_api_key'][ip] > request.app.system_config.get_config( 'block_ip_when_received_invalid_system_auth', int, 0 ): request.app.ip_blacklist.append(ip) request.app.secure_log.write_log( f"Add {ip} to the blacklist. <api_key_checker>", mzk.LogLevel.WARNING ) raise Forbidden('Unknown API-KEY.')
def rate_limit(self, limit: str, request: Request, key: str = '') -> bool: for item in parse_many(limit): if not self._ip_window.hit(item, self.get_remote_address(request), key): return False return True
async def run_commands(request: Request) -> HTTPResponse: """Run registered commands.""" cmd_name, password, args = mzk.get_args( request, ('cmd_name|name', str, None, { 'max_length': 64 }), ('password|pass', str, None, { 'max_length': 1024 }), ('arguments|args', str, '', { 'max_length': 8192, 'invalid_str': ["'", "\\", ";"] }), ) ip = mzk.get_remote_address(request) if cmd_name is None: raise Forbidden( 'cmd-name parameter is required and must be less than 64 or equal to 64 characters.' ) cmd = request.app.commands.get(cmd_name, None) if cmd is None: raise Forbidden('Unknown command.') elif not cmd.get('status', True): raise Forbidden('This command is offline.') elif cmd.get('pass', None) != password and request.app.system_config.get_config( 'root_pass') != password: raise Forbidden('Password error.') elif cmd.get('ip', None) is not None \ and cmd['ip'] != '*' \ and ip not in cmd['ip']: raise Forbidden("This command can't use from your ip address.") rate_limit = cmd.get('rate', '*') if rate_limit != '*': for item in parse_many(rate_limit): # check rate limit. if not request.app.rate_limiter.hit(item, cmd_name): abort(429, 'Too many requests.') rate_per_ip = cmd.get('rate_per_ip', '*') if rate_per_ip != '*': for item in parse_many(rate_per_ip): # check rate limit. if not request.app.rate_limiter.hit(item, f'{cmd_name}-{ip}'): abort(429, 'Too many requests.') try: if "cmd" in cmd: if cmd['cmd'].startswith('[moca]'): func = functions.get(cmd['cmd'][6:]) if func is None: raise ServerError('Unknown function.') return func(request, args) else: if args != '': res = mzk.check_output(f"{cmd['cmd']} '{args}'", shell=True) else: res = mzk.check_output(f"{cmd['cmd']}", shell=True) return text(res.decode()) elif "cmd_path" in cmd: if cmd["cmd_path"].startswith("/"): if args != '': res = mzk.check_output(f"{cmd['cmd_path']} '{args}'", shell=True) else: res = mzk.check_output(f"{cmd['cmd_path']}", shell=True) else: if args != '': res = mzk.check_output( f"{core.COMMANDS_DIR.joinpath(cmd['cmd_path'])} '{args}'", shell=True) else: res = mzk.check_output( f"{core.COMMANDS_DIR.joinpath(cmd['cmd_path'])}", shell=True) return text(res.decode()) else: raise ServerError('Command format error.') except CalledProcessError: raise ServerError("Can't execute this command.")