def test_fixed_window_with_elastic_expiry_memcache_concurrency(self): storage = MemcachedStorage('memcached://localhost:11211') limiter = FixedWindowElasticExpiryRateLimiter(storage) start = int(time.time()) limit = RateLimitItemPerSecond(100, 2) def _c(): for i in range(0, 50): limiter.hit(limit) t1, t2 = threading.Thread(target=_c), threading.Thread(target=_c) t1.start(), t2.start() [t1.join(), t2.join()] self.assertEqual(limiter.get_window_stats(limit)[1], 0) self.assertTrue( start + 2 <= limiter.get_window_stats(limit)[0] <= start + 3) self.assertEqual(storage.get(limit.key_for()), 100)
def test_fixed_window_with_elastic_expiry_in_memory(self): storage = MemoryStorage() limiter = FixedWindowElasticExpiryRateLimiter(storage) with hiro.Timeline().freeze() as timeline: start = int(time.time()) limit = RateLimitItemPerSecond(10, 2) self.assertTrue(all([limiter.hit(limit) for _ in range(0, 10)])) timeline.forward(1) self.assertFalse(limiter.hit(limit)) self.assertEqual(limiter.get_window_stats(limit)[1], 0) # three extensions to the expiry self.assertEqual(limiter.get_window_stats(limit)[0], start + 3) timeline.forward(1) self.assertFalse(limiter.hit(limit)) timeline.forward(3) start = int(time.time()) self.assertTrue(limiter.hit(limit)) self.assertEqual(limiter.get_window_stats(limit)[1], 9) self.assertEqual(limiter.get_window_stats(limit)[0], start + 2)
async def before_server_start(app_: Sanic, loop): mzk.set_process_name(f'{app_.name} --- listener {mzk.get_my_pid()}') mzk.print_info(f'Starting Sanic server. -- {mzk.get_my_pid()}') app_.system_config: mzk.MocaConfig = mzk.MocaConfig(core.SYSTEM_CONFIG, manual_reload=True) app_.commands: mzk.MocaSynchronizedJSONDictFile = mzk.MocaSynchronizedJSONDictFile( core.COMMANDS_CONFIG, manual_reload=True) app_.ip_blacklist: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile( core.IP_BLACKLIST_FILE, manual_reload=True, remove_duplicates=True, ) app_.api_key_config: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile( core.API_KEY_FILE, manual_reload=True) app_.dict_cache = {} app_.secure_log = mzk.MocaFileLog(core.LOG_DIR.joinpath('secure.log')) app_.scheduler = mzk.MocaScheduler() if core.SERVER_CONFIG['rate_limiter_redis_storage'] is None: app_._storage_for_rate_limiter = MemoryStorage() else: app_._storage_for_rate_limiter = RedisStorage( core.SERVER_CONFIG['rate_limiter_redis_storage']) app_.rate_limiter = FixedWindowElasticExpiryRateLimiter( app_._storage_for_rate_limiter) def __reload_timer(application: Sanic) -> None: while True: mzk.sleep(1) application.system_config.reload_file() application.commands.reload_file() application.ip_blacklist.reload_file() application.api_key_config.reload_file() app_._timer_thread = Thread(target=__reload_timer, args=(app_, ), daemon=True) app_._timer_thread.start()
async def before_server_start(app_: Sanic, loop): mzk.set_process_name(f'{app_.name} --- listener {mzk.get_my_pid()}') mzk.print_info(f'Starting Sanic server. -- {mzk.get_my_pid()}') app_.system_config: mzk.MocaConfig = mzk.MocaConfig(core.SYSTEM_CONFIG, manual_reload=True) app_.ip_blacklist: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile( core.IP_BLACKLIST_FILE, manual_reload=True, remove_duplicates=True, ) app_.api_key_config: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile( core.API_KEY_FILE, manual_reload=True) app_.twitter: mzk.MocaTwitter = mzk.MocaTwitter( core.TWITTER_CONFIG['CONSUMER_KEY'], core.TWITTER_CONFIG['CONSUMER_SECRET'], core.TWITTER_CONFIG['ACCESS_TOKEN'], core.TWITTER_CONFIG['ACCESS_TOKEN_SECRET']) app_.dict_cache = {} app_.secure_log = mzk.MocaFileLog(core.LOG_DIR.joinpath('secure.log')) app_.scheduler = mzk.MocaScheduler() if core.SERVER_CONFIG['rate_limiter_redis_storage'] is None: app_._storage_for_rate_limiter = MemoryStorage() else: app_._storage_for_rate_limiter = RedisStorage( core.SERVER_CONFIG['rate_limiter_redis_storage']) app_.rate_limiter = FixedWindowElasticExpiryRateLimiter( app_._storage_for_rate_limiter) try: app_.mysql = mzk.MocaMysql( core.DB_CONFIG['mysql']['host'], int(core.DB_CONFIG['mysql']['port']), core.DB_CONFIG['mysql']['user'], core.DB_CONFIG['mysql']['password'], core.DB_CONFIG['mysql']['database'], int(core.DB_CONFIG['mysql']['min_size']), int(core.DB_CONFIG['mysql']['max_size']), ) app_.mysql.force_sync = mzk.try_to_bool( core.DB_CONFIG['mysql']['force_sync']) except KeyError as e: mzk.print_error( f'Mysql database configuration error. missing key: {e}') mzk.sys_exit(1) except MySQLError as e: mzk.print_error( "Can't connect to MySQL database, Please check your database configuration." ) mzk.print_error("And make sure your database is online.") mzk.print_error( "You can use 'python3 moca.py test-mysql-con' to check your database." ) mzk.print_error(f"<MySQLError: {e}>") mzk.sys_exit(1) try: app_.redis = mzk.MocaRedis( core.DB_CONFIG['redis']['host'], int(core.DB_CONFIG['redis']['port']), int(core.DB_CONFIG['redis']['db']), core.DB_CONFIG['redis']['password'], int(core.DB_CONFIG['mysql']['min_size']), int(core.DB_CONFIG['mysql']['max_size']), ) app_.redis.prefix = core.DB_CONFIG['redis']['prefix'] await app_.redis.test_con() except KeyError as e: mzk.print_error( f'Redis database configuration error. missing key: {e}') mzk.sys_exit(1) except (RedisError, ConnectionRefusedError) as e: mzk.print_error( "Can't connect to Redis database, Please check your database configuration." ) mzk.print_error("And make sure your database is online.") mzk.print_error( "You can use 'python3 moca.py test-redis-con' to check your database." ) mzk.print_error(f"<(RedisError, ConnectionRefusedError): {e}>") mzk.sys_exit(1) try: app_.simple_cache = mzk.MocaSimpleCache( int(core.DB_CONFIG['simple_cache']['pool_size']), int(core.DB_CONFIG['simple_cache']['page_size']), ) except KeyError as e: mzk.print_error(f'SimpleCache configuration error. missing key: {e}') mzk.sys_exit(1) def __reload_timer(application: Sanic) -> None: while True: mzk.sleep(1) application.system_config.reload_file() application.ip_blacklist.reload_file() application.api_key_config.reload_file() app_._timer_thread = Thread(target=__reload_timer, args=(app_, ), daemon=True) app_._timer_thread.start()
async def before_server_start(app_: Sanic, loop): mzk.set_process_name(f'{app_.name} --- listener {mzk.get_my_pid()}') mzk.print_info(f'Starting Sanic server. -- {mzk.get_my_pid()}') app_.system_config: mzk.MocaConfig = mzk.MocaConfig(core.SYSTEM_CONFIG, manual_reload=True) app_.ip_blacklist: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile( core.IP_BLACKLIST_FILE, manual_reload=True, remove_duplicates=True, ) app_.api_key_config: mzk.MocaSynchronizedJSONListFile = mzk.MocaSynchronizedJSONListFile( core.API_KEY_FILE, manual_reload=True) app_.flags = mzk.MocaSynchronizedJSONDictFile(core.FLAGS_FILE, manual_reload=True) app_.dict_cache = {} app_.secure_log = mzk.MocaFileLog(core.LOG_DIR.joinpath('secure.log')) app_.scheduler = mzk.MocaScheduler() if core.SERVER_CONFIG['rate_limiter_redis_storage'] is None: app_._storage_for_rate_limiter = MemoryStorage() else: app_._storage_for_rate_limiter = RedisStorage( core.SERVER_CONFIG['rate_limiter_redis_storage']) app_.rate_limiter = FixedWindowElasticExpiryRateLimiter( app_._storage_for_rate_limiter) try: app_.mysql = mzk.MocaMysql( core.DB_CONFIG['mysql']['host'], int(core.DB_CONFIG['mysql']['port']), core.DB_CONFIG['mysql']['user'], core.DB_CONFIG['mysql']['password'], core.DB_CONFIG['mysql']['database'], int(core.DB_CONFIG['mysql']['min_size']), int(core.DB_CONFIG['mysql']['max_size']), ) app_.mysql.force_sync = mzk.try_to_bool( core.DB_CONFIG['mysql']['force_sync']) except KeyError as e: mzk.print_error( f'Mysql database configuration error. missing key: {e}') mzk.sys_exit(1) except MySQLError as e: mzk.print_error( "Can't connect to MySQL database, Please check your database configuration." ) mzk.print_error("And make sure your database is online.") mzk.print_error( "You can use 'python3 moca.py test-mysql-con' to check your database." ) mzk.print_error(f"<MySQLError: {e}>") mzk.sys_exit(1) def __reload_timer(application: Sanic) -> None: while True: mzk.sleep(1) application.system_config.reload_file() application.ip_blacklist.reload_file() application.api_key_config.reload_file() application.flags.reload_file() app_._timer_thread = Thread(target=__reload_timer, args=(app_, ), daemon=True) app_._timer_thread.start() app_.bots = {} def reload_bot(the_updated_key, old_value, new_value, *args, **kwargs) -> None: application = kwargs['app'] for bot_dir in core.STORAGE_DIR.iterdir(): if bot_dir.is_dir(): application.bots[bot_dir.name] = mzk.MocaBot( bot_dir.name, bot_dir) con = application.mysql.get_a_new_con() cursor = con.cursor() cursor.execute(core.GET_BOTS_QUERY) res = cursor.fetchall() con.close() application.dict_cache['id'] = {} application.dict_cache['name'] = {} for info in res: application.dict_cache['id'][info[0]] = info[1] application.dict_cache['name'][info[1]] = info[0] reload_bot(None, None, None, app=app_) app_.flags.set('moca_bot_reload', False) app_.flags.add_handler('moca_bot_reload' + str(mzk.get_my_pid()), 'moca_bot_reload', reload_bot, kwargs={'app': app_})