def test_fixed_window_with_elastic_expiry_memcache(self):
storage = MemcachedStorage('memcached://localhost:11211')
limiter = FixedWindowElasticExpiryRateLimiter(storage)
limit = RateLimitItemPerSecond(10, 2)
self.assertTrue(all([limiter.hit(limit) for _ in range(0,10)]))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
def test_fixed_window_with_elastic_expiry_redis(self):
storage = RedisStorage('redis://localhost:6379')
limiter = FixedWindowElasticExpiryRateLimiter(storage)
limit = RateLimitItemPerSecond(10, 2)
self.assertTrue(all([limiter.hit(limit) for _ in range(0, 10)]))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
def test_fixed_window_with_elastic_expiry_memcache(self):
storage = MemcachedStorage('memcached://localhost:11211')
limiter = FixedWindowElasticExpiryRateLimiter(storage)
limit = RateLimitItemPerSecond(10, 2)
self.assertTrue(all([limiter.hit(limit) for _ in range(0, 10)]))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
def test_fixed_window_with_elastic_expiry_redis(self):
storage = RedisStorage('redis://localhost:6379')
limiter = FixedWindowElasticExpiryRateLimiter(storage)
limit = RateLimitItemPerSecond(10, 2)
self.assertTrue(all([limiter.hit(limit) for _ in range(0,10)]))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
def test():
storage = RedisStorage("redis://127.0.0.1/3")
strategy = FixedWindowElasticExpiryRateLimiter(storage)
# one_per_second = RateLimitItemPerSecond(20, 1)
one_per_min = RateLimitItemPerMinute(100, 1)
r = strategy.hit(one_per_min, "k", "v")
if r == True:
print(True)
else:
print("请求满了")
def test_fixed_window_with_elastic_expiry(self):
storage = MemcachedStorage("memcached://localhost:22122")
limiter = FixedWindowElasticExpiryRateLimiter(storage)
per_sec = RateLimitItemPerSecond(2, 2)
self.assertTrue(limiter.hit(per_sec))
self.assertTrue(limiter.hit(per_sec))
time.sleep(1)
self.assertFalse(limiter.test(per_sec))
time.sleep(1)
self.assertTrue(limiter.test(per_sec))
def __init__(self, redis: MocaRedis, mysql: MocaMysql,
global_rate_limit: str):
MocaNamedInstance.__init__(self)
MocaClassCache.__init__(self)
self._redis: MocaRedis = redis
self._mysql: MocaMysql = mysql
self._redis_storage: RedisStorage = self._redis.get_redis_storage()
self._api_key_window: FixedWindowElasticExpiryRateLimiter = FixedWindowElasticExpiryRateLimiter(
self._redis_storage)
self._ip_window: FixedWindowElasticExpiryRateLimiter = FixedWindowElasticExpiryRateLimiter(
self._redis_storage)
self._sync_count: int = 64
self._global_rate_limit = parse_many(global_rate_limit)
def test_fixed_window_with_elastic_expiry_redis_sentinel(self):
storage = RedisSentinelStorage(
"redis+sentinel://localhost:26379",
service_name="localhost-redis-sentinel"
)
limiter = FixedWindowElasticExpiryRateLimiter(storage)
limit = RateLimitItemPerSecond(10, 2)
self.assertTrue(all([limiter.hit(limit) for _ in range(0,10)]))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 0)
def test_fixed_window_with_elastic_expiry_memcache_concurrency(self):
storage = MemcachedStorage('memcached://localhost:11211')
limiter = FixedWindowElasticExpiryRateLimiter(storage)
start = int(time.time())
limit = RateLimitItemPerSecond(100, 2)
def _c():
for i in range(0,50):
limiter.hit(limit)
t1, t2 = threading.Thread(target=_c), threading.Thread(target=_c)
t1.start(), t2.start()
[t1.join(), t2.join()]
self.assertEqual(limiter.get_window_stats(limit)[1], 0)
self.assertTrue(start + 2 <= limiter.get_window_stats(limit)[0] <= start + 3)
self.assertEqual(storage.get(limit.key_for()), 100)
async def before_server_start(app_: Sanic, loop):
mzk.set_process_name(f'MocaFileLog({core.VERSION}) --- {app_._log_name}')
mzk.print_info(f'Starting Sanic server. -- {mzk.get_my_pid()}')
app_.system_config: mzk.MocaConfig = mzk.MocaConfig(core.SYSTEM_CONFIG,
manual_reload=True)
app_.ip_blacklist: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile(
core.IP_BLACKLIST_FILE,
manual_reload=True,
remove_duplicates=True,
)
app_.api_key_config: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile(
core.API_KEY_FILE, manual_reload=True)
app_.dict_cache = {}
if app_._log_file_path.startswith('/'):
file = app_._log_file_path
else:
file = core.CLIENT_LOG_DIR.joinpath(app_._log_file_path)
app_.moca_log = mzk.MocaFileLog(file, app_._log_level)
app_.secure_log = mzk.MocaFileLog(core.LOG_DIR.joinpath('secure.log'))
app_.scheduler = mzk.MocaScheduler()
app_.log_list = []
if core.SERVER_CONFIG['rate_limiter_redis_storage'] is None:
app_._storage_for_rate_limiter = MemoryStorage()
else:
app_._storage_for_rate_limiter = RedisStorage(
core.SERVER_CONFIG['rate_limiter_redis_storage'])
app_.rate_limiter = FixedWindowElasticExpiryRateLimiter(
app_._storage_for_rate_limiter)
if core.LOG_CONFIG[app_._log_name].get('google_spread_sheets_auth',
None) is not None:
scope = [
'https://spreadsheets.google.com/feeds',
'https://www.googleapis.com/auth/drive'
]
app_._credentials = ServiceAccountCredentials.from_json_keyfile_name(
str(
core.CONFIG_DIR.joinpath(core.LOG_CONFIG[app_._log_name].get(
'google_spread_sheets_auth'))),
scope,
)
app_._gc = authorize(app_._credentials)
app_.workbook = app_._gc.open_by_key(
core.LOG_CONFIG[app_._log_name].get('spread_sheets_key'))
else:
app_.workbook = None
def __reload_timer(application: Sanic) -> None:
while True:
mzk.sleep(1)
application.system_config.reload_file()
application.ip_blacklist.reload_file()
application.api_key_config.reload_file()
app_._timer_thread = Thread(target=__reload_timer,
args=(app_, ),
daemon=True)
app_._timer_thread.start()
def test_fixed_window_with_elastic_expiry_memcache_concurrency(self):
storage = MemcachedStorage('memcached://localhost:11211')
limiter = FixedWindowElasticExpiryRateLimiter(storage)
start = int(time.time())
limit = RateLimitItemPerSecond(100, 2)
def _c():
for i in range(0, 50):
limiter.hit(limit)
t1, t2 = threading.Thread(target=_c), threading.Thread(target=_c)
t1.start(), t2.start()
[t1.join(), t2.join()]
self.assertEqual(limiter.get_window_stats(limit)[1], 0)
self.assertTrue(
start + 2 <= limiter.get_window_stats(limit)[0] <= start + 3)
self.assertEqual(storage.get(limit.key_for()), 100)
def test_fixed_window_with_elastic_expiry_in_memory(self):
storage = MemoryStorage()
limiter = FixedWindowElasticExpiryRateLimiter(storage)
with hiro.Timeline().freeze() as timeline:
start = int(time.time())
limit = RateLimitItemPerSecond(10, 2)
self.assertTrue(all([limiter.hit(limit) for _ in range(0, 10)]))
timeline.forward(1)
self.assertFalse(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 0)
# three extensions to the expiry
self.assertEqual(limiter.get_window_stats(limit)[0], start + 3)
timeline.forward(1)
self.assertFalse(limiter.hit(limit))
timeline.forward(3)
start = int(time.time())
self.assertTrue(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 9)
self.assertEqual(limiter.get_window_stats(limit)[0], start + 2)
def test_fixed_window_with_elastic_expiry_redis_sentinel(self):
storage = RedisSentinelStorage("redis+sentinel://localhost:26379",
service_name="localhost-redis-sentinel")
limiter = FixedWindowElasticExpiryRateLimiter(storage)
limit = RateLimitItemPerSecond(10, 2)
self.assertTrue(all([limiter.hit(limit) for _ in range(0, 10)]))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
time.sleep(1)
self.assertFalse(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 0)
def test_fixed_window_with_elastic_expiry_in_memory(self):
storage = MemoryStorage()
limiter = FixedWindowElasticExpiryRateLimiter(storage)
with hiro.Timeline().freeze() as timeline:
start = int(time.time())
limit = RateLimitItemPerSecond(10, 2)
self.assertTrue(all([limiter.hit(limit) for _ in range(0,10)]))
timeline.forward(1)
self.assertFalse(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 0)
# three extensions to the expiry
self.assertEqual(limiter.get_window_stats(limit)[0], start + 3)
timeline.forward(1)
self.assertFalse(limiter.hit(limit))
timeline.forward(3)
start = int(time.time())
self.assertTrue(limiter.hit(limit))
self.assertEqual(limiter.get_window_stats(limit)[1], 9)
self.assertEqual(limiter.get_window_stats(limit)[0], start + 2)
async def before_server_start(app_: Sanic, loop):
mzk.set_process_name(f'{app_.name} --- listener {mzk.get_my_pid()}')
mzk.print_info(f'Starting Sanic server. -- {mzk.get_my_pid()}')
app_.system_config: mzk.MocaConfig = mzk.MocaConfig(core.SYSTEM_CONFIG,
manual_reload=True)
app_.commands: mzk.MocaSynchronizedJSONDictFile = mzk.MocaSynchronizedJSONDictFile(
core.COMMANDS_CONFIG, manual_reload=True)
app_.ip_blacklist: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile(
core.IP_BLACKLIST_FILE,
manual_reload=True,
remove_duplicates=True,
)
app_.api_key_config: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile(
core.API_KEY_FILE, manual_reload=True)
app_.dict_cache = {}
app_.secure_log = mzk.MocaFileLog(core.LOG_DIR.joinpath('secure.log'))
app_.scheduler = mzk.MocaScheduler()
if core.SERVER_CONFIG['rate_limiter_redis_storage'] is None:
app_._storage_for_rate_limiter = MemoryStorage()
else:
app_._storage_for_rate_limiter = RedisStorage(
core.SERVER_CONFIG['rate_limiter_redis_storage'])
app_.rate_limiter = FixedWindowElasticExpiryRateLimiter(
app_._storage_for_rate_limiter)
def __reload_timer(application: Sanic) -> None:
while True:
mzk.sleep(1)
application.system_config.reload_file()
application.commands.reload_file()
application.ip_blacklist.reload_file()
application.api_key_config.reload_file()
app_._timer_thread = Thread(target=__reload_timer,
args=(app_, ),
daemon=True)
app_._timer_thread.start()
async def before_server_start(app_: Sanic, loop):
mzk.set_process_name(f'{app_.name} --- listener {mzk.get_my_pid()}')
mzk.print_info(f'Starting Sanic server. -- {mzk.get_my_pid()}')
app_.system_config: mzk.MocaConfig = mzk.MocaConfig(core.SYSTEM_CONFIG,
manual_reload=True)
app_.ip_blacklist: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile(
core.IP_BLACKLIST_FILE,
manual_reload=True,
remove_duplicates=True,
)
app_.api_key_config: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile(
core.API_KEY_FILE, manual_reload=True)
app_.flags = mzk.MocaSynchronizedJSONDictFile(core.FLAGS_FILE,
manual_reload=True)
app_.dict_cache = {}
app_.secure_log = mzk.MocaFileLog(core.LOG_DIR.joinpath('secure.log'))
app_.scheduler = mzk.MocaScheduler()
if core.SERVER_CONFIG['rate_limiter_redis_storage'] is None:
app_._storage_for_rate_limiter = MemoryStorage()
else:
app_._storage_for_rate_limiter = RedisStorage(
core.SERVER_CONFIG['rate_limiter_redis_storage'])
app_.rate_limiter = FixedWindowElasticExpiryRateLimiter(
app_._storage_for_rate_limiter)
try:
app_.mysql = mzk.MocaMysql(
core.DB_CONFIG['mysql']['host'],
int(core.DB_CONFIG['mysql']['port']),
core.DB_CONFIG['mysql']['user'],
core.DB_CONFIG['mysql']['password'],
core.DB_CONFIG['mysql']['database'],
int(core.DB_CONFIG['mysql']['min_size']),
int(core.DB_CONFIG['mysql']['max_size']),
)
app_.mysql.force_sync = mzk.try_to_bool(
core.DB_CONFIG['mysql']['force_sync'])
except KeyError as e:
mzk.print_error(
f'Mysql database configuration error. missing key: {e}')
mzk.sys_exit(1)
except MySQLError as e:
mzk.print_error(
"Can't connect to MySQL database, Please check your database configuration."
)
mzk.print_error("And make sure your database is online.")
mzk.print_error(
"You can use 'python3 moca.py test-mysql-con' to check your database."
)
mzk.print_error(f"<MySQLError: {e}>")
mzk.sys_exit(1)
def __reload_timer(application: Sanic) -> None:
while True:
mzk.sleep(1)
application.system_config.reload_file()
application.ip_blacklist.reload_file()
application.api_key_config.reload_file()
application.flags.reload_file()
app_._timer_thread = Thread(target=__reload_timer,
args=(app_, ),
daemon=True)
app_._timer_thread.start()
app_.bots = {}
def reload_bot(the_updated_key, old_value, new_value, *args,
**kwargs) -> None:
application = kwargs['app']
for bot_dir in core.STORAGE_DIR.iterdir():
if bot_dir.is_dir():
application.bots[bot_dir.name] = mzk.MocaBot(
bot_dir.name, bot_dir)
con = application.mysql.get_a_new_con()
cursor = con.cursor()
cursor.execute(core.GET_BOTS_QUERY)
res = cursor.fetchall()
con.close()
application.dict_cache['id'] = {}
application.dict_cache['name'] = {}
for info in res:
application.dict_cache['id'][info[0]] = info[1]
application.dict_cache['name'][info[1]] = info[0]
reload_bot(None, None, None, app=app_)
app_.flags.set('moca_bot_reload', False)
app_.flags.add_handler('moca_bot_reload' + str(mzk.get_my_pid()),
'moca_bot_reload',
reload_bot,
kwargs={'app': app_})
async def before_server_start(app_: Sanic, loop):
mzk.set_process_name(f'{app_.name} --- listener {mzk.get_my_pid()}')
mzk.print_info(f'Starting Sanic server. -- {mzk.get_my_pid()}')
app_.system_config: mzk.MocaConfig = mzk.MocaConfig(core.SYSTEM_CONFIG,
manual_reload=True)
app_.ip_blacklist: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile(
core.IP_BLACKLIST_FILE,
manual_reload=True,
remove_duplicates=True,
)
app_.api_key_config: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile(
core.API_KEY_FILE, manual_reload=True)
app_.twitter: mzk.MocaTwitter = mzk.MocaTwitter(
core.TWITTER_CONFIG['CONSUMER_KEY'],
core.TWITTER_CONFIG['CONSUMER_SECRET'],
core.TWITTER_CONFIG['ACCESS_TOKEN'],
core.TWITTER_CONFIG['ACCESS_TOKEN_SECRET'])
app_.dict_cache = {}
app_.secure_log = mzk.MocaFileLog(core.LOG_DIR.joinpath('secure.log'))
app_.scheduler = mzk.MocaScheduler()
if core.SERVER_CONFIG['rate_limiter_redis_storage'] is None:
app_._storage_for_rate_limiter = MemoryStorage()
else:
app_._storage_for_rate_limiter = RedisStorage(
core.SERVER_CONFIG['rate_limiter_redis_storage'])
app_.rate_limiter = FixedWindowElasticExpiryRateLimiter(
app_._storage_for_rate_limiter)
try:
app_.mysql = mzk.MocaMysql(
core.DB_CONFIG['mysql']['host'],
int(core.DB_CONFIG['mysql']['port']),
core.DB_CONFIG['mysql']['user'],
core.DB_CONFIG['mysql']['password'],
core.DB_CONFIG['mysql']['database'],
int(core.DB_CONFIG['mysql']['min_size']),
int(core.DB_CONFIG['mysql']['max_size']),
)
app_.mysql.force_sync = mzk.try_to_bool(
core.DB_CONFIG['mysql']['force_sync'])
except KeyError as e:
mzk.print_error(
f'Mysql database configuration error. missing key: {e}')
mzk.sys_exit(1)
except MySQLError as e:
mzk.print_error(
"Can't connect to MySQL database, Please check your database configuration."
)
mzk.print_error("And make sure your database is online.")
mzk.print_error(
"You can use 'python3 moca.py test-mysql-con' to check your database."
)
mzk.print_error(f"<MySQLError: {e}>")
mzk.sys_exit(1)
try:
app_.redis = mzk.MocaRedis(
core.DB_CONFIG['redis']['host'],
int(core.DB_CONFIG['redis']['port']),
int(core.DB_CONFIG['redis']['db']),
core.DB_CONFIG['redis']['password'],
int(core.DB_CONFIG['mysql']['min_size']),
int(core.DB_CONFIG['mysql']['max_size']),
)
app_.redis.prefix = core.DB_CONFIG['redis']['prefix']
await app_.redis.test_con()
except KeyError as e:
mzk.print_error(
f'Redis database configuration error. missing key: {e}')
mzk.sys_exit(1)
except (RedisError, ConnectionRefusedError) as e:
mzk.print_error(
"Can't connect to Redis database, Please check your database configuration."
)
mzk.print_error("And make sure your database is online.")
mzk.print_error(
"You can use 'python3 moca.py test-redis-con' to check your database."
)
mzk.print_error(f"<(RedisError, ConnectionRefusedError): {e}>")
mzk.sys_exit(1)
try:
app_.simple_cache = mzk.MocaSimpleCache(
int(core.DB_CONFIG['simple_cache']['pool_size']),
int(core.DB_CONFIG['simple_cache']['page_size']),
)
except KeyError as e:
mzk.print_error(f'SimpleCache configuration error. missing key: {e}')
mzk.sys_exit(1)
def __reload_timer(application: Sanic) -> None:
while True:
mzk.sleep(1)
application.system_config.reload_file()
application.ip_blacklist.reload_file()
application.api_key_config.reload_file()
app_._timer_thread = Thread(target=__reload_timer,
args=(app_, ),
daemon=True)
app_._timer_thread.start()
