def _validate_rate_limit(_ctx, _param, value):
"""Validate rate limit string."""
if value is None:
return None
try:
limits.parse_many(value)
except ValueError:
raise click.BadParameter('Rate limit format: n/second, m/minute, etc.')
return value
def _check_rate_limit_format(limit: str) -> bool:
if isinstance(limit, str) and (len(limit) <= 256):
try:
if limit != '*':
parse_many(limit)
return True
except ValueError:
return False
else:
return False
def __init__(self, storage, limit, identifiers=None):
if identifiers is None:
identifiers = []
self._window = MovingWindowRateLimiter(storage)
self._limits = parse_many(limit)
self._identifiers = identifiers
def __iter__(self):
limit_items = parse_many(self.__limit_provider(
) if callable(self.__limit_provider) else self.__limit_provider)
for limit in limit_items:
yield Limit(limit, self.key_function, self.__scope,
self.per_method, self.methods, self.error_message,
self.exempt_when)
def __iter__(self) -> Iterator[Limit]:
if callable(self.__limit_provider):
if "key" in inspect.signature(self.__limit_provider).parameters.keys():
assert (
"request" in inspect.signature(self.key_function).parameters.keys()
), f"Limit provider function {self.key_function.__name__} needs a `request` argument"
if self.request is None:
raise Exception("`request` object can't be None")
limit_raw = self.__limit_provider(self.key_function(self.request))
else:
limit_raw = self.__limit_provider()
else:
limit_raw = self.__limit_provider
limit_items: List[RateLimitItem] = parse_many(limit_raw)
for limit in limit_items:
yield Limit(
limit,
self.key_function,
self.__scope,
self.per_method,
self.methods,
self.error_message,
self.exempt_when,
self.override_defaults,
)
def __init__(self, storage, limit, identifiers=None):
if identifiers is None:
identifiers = []
self._window = MovingWindowRateLimiter(storage)
self._limits = parse_many(limit)
self._identifiers = identifiers
async def check_api_key(self, api_key: str, permission: str,
request: Request) -> Tuple[int, str]:
"""
Return: status_code, response_message.
Status Code
-----------
-1: unknown api key.
0: allowed.
1: api_key format error.
2: invalid referer.
3: too many requests.
4: you reached the access limit.
5: permission error.
"""
if not self._check_api_key_format(api_key):
return 1, 'api_key format error.'
cache = await self._redis.get('moca-api-key-cache-' + api_key)
if cache is None:
pool = await self._mysql.get_aio_pool()
async with pool.acquire() as con:
async with con.cursor() as cursor:
await cursor.execute(MocaAccess._GET_API_KEY_INFO,
(api_key, ))
res = await cursor.fetchall()
if len(res) > 0:
data = list(res[0])
data[2] = parse_many(data[2])
await self._redis.save('moca-api-key-cache-' + api_key,
data)
else:
return -1, 'unknown api key.'
else:
data = cache
referer = str(
request.headers.get('referer', request.headers.get('origin')))
if (data[1] != '*') and (not referer.startswith(data[1])):
return 2, 'invalid referer.'
if data[2] != '*':
for item in data[2]:
if not self._api_key_window.hit(
item, self.get_remote_address(request)):
return 3, 'too many requests.'
count = await self._redis.increment('moca-api-key-count-' + api_key)
if self._sync_count >= 0:
self._sync_count -= 1
else:
self._sync_count = 64
pool = await self._mysql.get_aio_pool()
async with pool.acquire() as con:
async with con.cursor() as cursor:
await cursor.execute(MocaAccess._UPDATE_COUNT,
(count, api_key))
await con.commit()
if count > data[3]:
return 4, 'you reached the access limit.'
if ('-RO-' not in data[5]) and (permission not in data[5]):
return 5, 'permission error.'
return 0, 'allowed.'
def __init__(self, storage, limit, *, identifiers=None, metrics):
if identifiers is None:
identifiers = []
self._storage = storage
self._window = MovingWindowRateLimiter(storage)
self._limits = parse_many(limit)
self._identifiers = identifiers
self._metrics = metrics
def make_rate_limiter(scope, limits):
"""Create a rate limiter.
Multiple limits can be separated with a semicolon; in that case
all limits are checked until one succeeds. This allows specifying
a somewhat strict limit, but then a higher limit over a longer period
of time to allow for bursts.
"""
limits = list(parse_many(limits)) if limits is not None else None
return RateLimit(limiter, limiter._key_func, scope, limits)
def __init__(self, redis: MocaRedis, mysql: MocaMysql,
global_rate_limit: str):
MocaNamedInstance.__init__(self)
MocaClassCache.__init__(self)
self._redis: MocaRedis = redis
self._mysql: MocaMysql = mysql
self._redis_storage: RedisStorage = self._redis.get_redis_storage()
self._api_key_window: FixedWindowElasticExpiryRateLimiter = FixedWindowElasticExpiryRateLimiter(
self._redis_storage)
self._ip_window: FixedWindowElasticExpiryRateLimiter = FixedWindowElasticExpiryRateLimiter(
self._redis_storage)
self._sync_count: int = 64
self._global_rate_limit = parse_many(global_rate_limit)
def __iter__(self) -> Iterator[Limit]:
limit_items: List[RateLimitItem] = parse_many(self.__limit_provider(
) if callable(self.__limit_provider) else self.__limit_provider)
for limit in limit_items:
yield Limit(
limit,
self.key_function,
self.__scope,
self.per_method,
self.methods,
self.error_message,
self.exempt_when,
)
async def iterate(self, request: Request) -> Iterator[Limit]:
limit_items: List[RateLimitItem] = parse_many(
await self.__limit_provider(request) # noqa
if inspect.iscoroutinefunction(self.__limit_provider
) else self.__limit_provider)
for lmt in limit_items:
yield Limit(
lmt,
self.key_function,
self.__scope,
self.per_method,
self.methods,
self.error_message,
self.exempt_when,
self.override_defaults,
)
async def api_key_checker(request: Request):
"""A api-key filter."""
if request.method.upper() == 'OPTIONS':
return text('success.')
if not request.raw_url.startswith(b'/static') and \
not request.raw_url.startswith(b'/web') and \
not request.raw_url.startswith(b'/status') and \
not request.raw_url.startswith(b'/moca-twitter/static/icons/') and \
request.method.upper() != 'OPTIONS':
received_key = mzk.get_args(request, ('api_key', str, None, {'max_length': 1024}))[0]
ip = mzk.get_remote_address(request)
if received_key is None:
raise Forbidden('Missing API-KEY.')
found = False
for api_key_info in request.app.api_key_config.list:
if api_key_info.get('key') == received_key: # found a api key.
found = True
if api_key_info.get('status', False): # check api key status.
rate_limit = api_key_info.get('rate')
if rate_limit != '*':
for item in parse_many(rate_limit): # check rate limit.
if not request.app.rate_limiter.hit(item, received_key):
abort(429, 'Too many requests.')
else:
pass # '*' means unlimited.
allowed = False
target = request.raw_url.decode()
for path_info in api_key_info.get('allowed_path'):
if ':' in path_info:
path, path_rate = path_info.split(':')
else:
path, path_rate, = path_info, '*'
if target.startswith(path):
if path_rate == '*':
allowed = True
else:
for item in parse_many(path_rate): # check rate limit.
if not request.app.rate_limiter.hit(item, f'{received_key}-{ip}-{path}'):
abort(429, 'Too many requests.')
allowed = True
break
if not allowed:
raise Forbidden("Your API-KEY can't access to this path.")
else:
pass # allowed
ip = mzk.get_remote_address(request)
if api_key_info.get('ip') != '*' and ip not in api_key_info.get('ip'):
raise Forbidden(f"Your API-KEY can't use from this ip address. ({ip})")
else:
pass # allowed
required = api_key_info.get('required')
for key, value in required['headers'].items():
if request.headers.get(key) != value:
raise Forbidden('Missing required header.')
for key, value in required['args'].items():
if mzk.get_args(request, key)[0] != value:
raise Forbidden('Missing required argument.')
if api_key_info.get('delay', 0) != 0:
await sleep(api_key_info.get('delay'))
# --- success. do nothing. ---
else:
raise Forbidden('Your API-KEY is not online.')
break
if not found:
ip = mzk.get_remote_address(request)
request.app.secure_log.write_log(
f"Received a unknown API-KEY: <{received_key}> from {ip}.",
mzk.LogLevel.WARNING
)
if request.app.dict_cache.get('unknown_api_key') is None:
request.app.dict_cache['unknown_api_key'] = {}
try:
request.app.dict_cache['unknown_api_key'][ip] += 1
except KeyError:
request.app.dict_cache['unknown_api_key'][ip] = 1
if request.app.dict_cache['unknown_api_key'][ip] > request.app.system_config.get_config(
'block_ip_when_received_invalid_system_auth', int, 0
):
request.app.ip_blacklist.append(ip)
request.app.secure_log.write_log(
f"Add {ip} to the blacklist. <api_key_checker>",
mzk.LogLevel.WARNING
)
raise Forbidden('Unknown API-KEY.')
def rate_limit(self, limit: str, request: Request, key: str = '') -> bool:
for item in parse_many(limit):
if not self._ip_window.hit(item, self.get_remote_address(request),
key):
return False
return True
async def run_commands(request: Request) -> HTTPResponse:
"""Run registered commands."""
cmd_name, password, args = mzk.get_args(
request,
('cmd_name|name', str, None, {
'max_length': 64
}),
('password|pass', str, None, {
'max_length': 1024
}),
('arguments|args', str, '', {
'max_length': 8192,
'invalid_str': ["'", "\\", ";"]
}),
)
ip = mzk.get_remote_address(request)
if cmd_name is None:
raise Forbidden(
'cmd-name parameter is required and must be less than 64 or equal to 64 characters.'
)
cmd = request.app.commands.get(cmd_name, None)
if cmd is None:
raise Forbidden('Unknown command.')
elif not cmd.get('status', True):
raise Forbidden('This command is offline.')
elif cmd.get('pass',
None) != password and request.app.system_config.get_config(
'root_pass') != password:
raise Forbidden('Password error.')
elif cmd.get('ip', None) is not None \
and cmd['ip'] != '*' \
and ip not in cmd['ip']:
raise Forbidden("This command can't use from your ip address.")
rate_limit = cmd.get('rate', '*')
if rate_limit != '*':
for item in parse_many(rate_limit): # check rate limit.
if not request.app.rate_limiter.hit(item, cmd_name):
abort(429, 'Too many requests.')
rate_per_ip = cmd.get('rate_per_ip', '*')
if rate_per_ip != '*':
for item in parse_many(rate_per_ip): # check rate limit.
if not request.app.rate_limiter.hit(item, f'{cmd_name}-{ip}'):
abort(429, 'Too many requests.')
try:
if "cmd" in cmd:
if cmd['cmd'].startswith('[moca]'):
func = functions.get(cmd['cmd'][6:])
if func is None:
raise ServerError('Unknown function.')
return func(request, args)
else:
if args != '':
res = mzk.check_output(f"{cmd['cmd']} '{args}'",
shell=True)
else:
res = mzk.check_output(f"{cmd['cmd']}", shell=True)
return text(res.decode())
elif "cmd_path" in cmd:
if cmd["cmd_path"].startswith("/"):
if args != '':
res = mzk.check_output(f"{cmd['cmd_path']} '{args}'",
shell=True)
else:
res = mzk.check_output(f"{cmd['cmd_path']}", shell=True)
else:
if args != '':
res = mzk.check_output(
f"{core.COMMANDS_DIR.joinpath(cmd['cmd_path'])} '{args}'",
shell=True)
else:
res = mzk.check_output(
f"{core.COMMANDS_DIR.joinpath(cmd['cmd_path'])}",
shell=True)
return text(res.decode())
else:
raise ServerError('Command format error.')
except CalledProcessError:
raise ServerError("Can't execute this command.")
