def test_registered_defaults(self): def check_key_is_absnet(key): try: swift_info[key] except KeyError as err: if key not in err: raise test_limits = { 'account_ratelimit': 1, 'max_sleep_time_seconds': 60, 'container_ratelimit_0': 0, 'container_ratelimit_10': 10, 'container_ratelimit_50': 50, 'container_listing_ratelimit_0': 0, 'container_listing_ratelimit_10': 10, 'container_listing_ratelimit_50': 50 } ratelimit.filter_factory(test_limits)('have to pass in an app') swift_info = utils.get_swift_info() self.assertTrue('ratelimit' in swift_info) self.assertEqual(swift_info['ratelimit']['account_ratelimit'], 1.0) self.assertEqual(swift_info['ratelimit']['max_sleep_time_seconds'], 60.0) self.assertEqual(swift_info['ratelimit']['container_ratelimits'][0][0], 0) self.assertEqual(swift_info['ratelimit']['container_ratelimits'][0][1], 0.0) self.assertEqual(swift_info['ratelimit']['container_ratelimits'][1][0], 10) self.assertEqual(swift_info['ratelimit']['container_ratelimits'][1][1], 10.0) self.assertEqual(swift_info['ratelimit']['container_ratelimits'][2][0], 50) self.assertEqual(swift_info['ratelimit']['container_ratelimits'][2][1], 50.0) self.assertEqual( swift_info['ratelimit']['container_listing_ratelimits'][0][0], 0) self.assertEqual( swift_info['ratelimit']['container_listing_ratelimits'][0][1], 0.0) self.assertEqual( swift_info['ratelimit']['container_listing_ratelimits'][1][0], 10) self.assertEqual( swift_info['ratelimit']['container_listing_ratelimits'][1][1], 10.0) self.assertEqual( swift_info['ratelimit']['container_listing_ratelimits'][2][0], 50) self.assertEqual( swift_info['ratelimit']['container_listing_ratelimits'][2][1], 50.0) # these were left out on purpose for key in [ 'log_sleep_time_seconds', 'clock_accuracy', 'rate_buffer_seconds', 'ratelimit_whitelis', 'ratelimit_blacklist' ]: check_key_is_absnet(key)
def test_registered_defaults(self): def check_key_is_absent(key): try: swift_info[key] except KeyError as err: if key not in str(err): raise test_limits = {'account_ratelimit': 1, 'max_sleep_time_seconds': 60, 'container_ratelimit_0': 0, 'container_ratelimit_10': 10, 'container_ratelimit_50': 50, 'container_listing_ratelimit_0': 0, 'container_listing_ratelimit_10': 10, 'container_listing_ratelimit_50': 50} ratelimit.filter_factory(test_limits)('have to pass in an app') swift_info = utils.get_swift_info() self.assertIn('ratelimit', swift_info) self.assertEqual(swift_info['ratelimit'] ['account_ratelimit'], 1.0) self.assertEqual(swift_info['ratelimit'] ['max_sleep_time_seconds'], 60.0) self.assertEqual(swift_info['ratelimit'] ['container_ratelimits'][0][0], 0) self.assertEqual(swift_info['ratelimit'] ['container_ratelimits'][0][1], 0.0) self.assertEqual(swift_info['ratelimit'] ['container_ratelimits'][1][0], 10) self.assertEqual(swift_info['ratelimit'] ['container_ratelimits'][1][1], 10.0) self.assertEqual(swift_info['ratelimit'] ['container_ratelimits'][2][0], 50) self.assertEqual(swift_info['ratelimit'] ['container_ratelimits'][2][1], 50.0) self.assertEqual(swift_info['ratelimit'] ['container_listing_ratelimits'][0][0], 0) self.assertEqual(swift_info['ratelimit'] ['container_listing_ratelimits'][0][1], 0.0) self.assertEqual(swift_info['ratelimit'] ['container_listing_ratelimits'][1][0], 10) self.assertEqual(swift_info['ratelimit'] ['container_listing_ratelimits'][1][1], 10.0) self.assertEqual(swift_info['ratelimit'] ['container_listing_ratelimits'][2][0], 50) self.assertEqual(swift_info['ratelimit'] ['container_listing_ratelimits'][2][1], 50.0) # these were left out on purpose for key in ['log_sleep_time_seconds', 'clock_accuracy', 'rate_buffer_seconds', 'ratelimit_whitelis', 'ratelimit_blacklist']: check_key_is_absent(key)
def test_account_ratelimit(self): current_rate = 5 num_calls = 50 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) with mock.patch('swift.common.middleware.ratelimit.get_container_info', lambda *args, **kwargs: {}): with mock.patch( 'swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): for meth, exp_time in [('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]: req = Request.blank('/v/a%s/c' % meth) req.method = meth req.environ['swift.cache'] = FakeMemcache() make_app_call = lambda: self.test_ratelimit( req.environ, start_response) begin = time.time() self._run(make_app_call, num_calls, current_rate, check_time=bool(exp_time)) self.assertEquals(round(time.time() - begin, 1), exp_time) self._reset_time()
def test_ratelimit_max_rate_double(self): global time_ticker global time_override current_rate = 2 conf_dict = { 'account_ratelimit': current_rate, 'clock_accuracy': 100, 'max_sleep_time_seconds': 1 } self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) self.test_ratelimit.log_sleep_time_seconds = .00001 req = Request.blank('/v1/a/c') req.method = 'PUT' req.environ['swift.cache'] = FakeMemcache() time_override = [0, 0, 0, 0, None] # simulates 4 requests coming in at same time, then sleeping with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): r = self.test_ratelimit(req.environ.copy(), start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ.copy(), start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ.copy(), start_response) self.assertEqual(r[0], b'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ.copy(), start_response) self.assertEqual(r[0], b'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ.copy(), start_response) self.assertEqual(r[0], b'Some Content')
def test_get_ratelimitable_key_tuples(self): current_rate = 13 conf_dict = {'account_ratelimit': current_rate, 'container_ratelimit_3': 200} fake_memcache = FakeMemcache() fake_memcache.store[get_cache_key('a', 'c')] = \ {'object_count': '5'} the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app.memcache_client = fake_memcache environ = {'swift.cache': fake_memcache, 'PATH_INFO': '/v1/a/c/o'} with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): self.assertEqual(len(the_app.get_ratelimitable_key_tuples( FakeReq('DELETE', environ), 'a', None, None)), 0) self.assertEqual(len(the_app.get_ratelimitable_key_tuples( FakeReq('PUT', environ), 'a', 'c', None)), 1) self.assertEqual(len(the_app.get_ratelimitable_key_tuples( FakeReq('DELETE', environ), 'a', 'c', None)), 1) self.assertEqual(len(the_app.get_ratelimitable_key_tuples( FakeReq('GET', environ), 'a', 'c', 'o')), 0) self.assertEqual(len(the_app.get_ratelimitable_key_tuples( FakeReq('PUT', environ), 'a', 'c', 'o')), 1) self.assertEqual(len(the_app.get_ratelimitable_key_tuples( FakeReq('PUT', environ), 'a', 'c', None, global_ratelimit=10)), 2) self.assertEqual(the_app.get_ratelimitable_key_tuples( FakeReq('PUT', environ), 'a', 'c', None, global_ratelimit=10)[1], ('ratelimit/global-write/a', 10)) self.assertEqual(len(the_app.get_ratelimitable_key_tuples( FakeReq('PUT', environ), 'a', 'c', None, global_ratelimit='notafloat')), 1)
def test_ratelimit_blacklist(self): global time_ticker current_rate = 2 conf_dict = {'account_ratelimit': current_rate, 'max_sleep_time_seconds': 2, 'account_whitelist': 'a', 'account_blacklist': 'b'} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) self.test_ratelimit.logger = FakeLogger() self.test_ratelimit.BLACK_LIST_SLEEP = 0 ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/b/c') req.environ['swift.cache'] = FakeMemcache() class rate_caller(Thread): def __init__(self, parent): Thread.__init__(self) self.parent = parent def run(self): self.result = self.parent.test_ratelimit(req.environ, start_response) nt = 5 threads = [] for i in range(nt): rc = rate_caller(self) rc.start() threads.append(rc) for thread in threads: thread.join() the_497s = [ t for t in threads if ''.join(t.result).startswith('Your account')] self.assertEquals(len(the_497s), 5) self.assertEquals(time_ticker, 0)
def test_ratelimit_max_rate_multiple_acc(self): num_calls = 4 current_rate = 2 conf_dict = {'account_ratelimit': current_rate, 'max_sleep_time_seconds': 2} fake_memcache = FakeMemcache() the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app.memcache_client = fake_memcache class rate_caller(threading.Thread): def __init__(self, name): self.myname = name threading.Thread.__init__(self) def run(self): for j in range(num_calls): self.result = the_app.handle_ratelimit( FakeReq('PUT'), self.myname, 'c', None) with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): nt = 15 begin = time.time() threads = [] for i in range(nt): rc = rate_caller('a%s' % i) rc.start() threads.append(rc) for thread in threads: thread.join() time_took = time.time() - begin self.assertEqual(1.5, round(time_took, 1))
def test_get_ratelimitable_key_tuples(self): current_rate = 13 conf_dict = {"account_ratelimit": current_rate, "container_ratelimit_3": 200} fake_memcache = FakeMemcache() fake_memcache.store[get_cache_key("a", "c")] = {"object_count": "5"} the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app.memcache_client = fake_memcache req = lambda: None req.environ = {"swift.cache": fake_memcache, "PATH_INFO": "/v1/a/c/o"} with mock.patch("swift.common.middleware.ratelimit.get_account_info", lambda *args, **kwargs: {}): req.method = "DELETE" self.assertEqual(len(the_app.get_ratelimitable_key_tuples(req, "a", None, None)), 0) req.method = "PUT" self.assertEqual(len(the_app.get_ratelimitable_key_tuples(req, "a", "c", None)), 1) req.method = "DELETE" self.assertEqual(len(the_app.get_ratelimitable_key_tuples(req, "a", "c", None)), 1) req.method = "GET" self.assertEqual(len(the_app.get_ratelimitable_key_tuples(req, "a", "c", "o")), 0) req.method = "PUT" self.assertEqual(len(the_app.get_ratelimitable_key_tuples(req, "a", "c", "o")), 1) req.method = "PUT" self.assertEqual(len(the_app.get_ratelimitable_key_tuples(req, "a", "c", None, global_ratelimit=10)), 2) self.assertEqual( the_app.get_ratelimitable_key_tuples(req, "a", "c", None, global_ratelimit=10)[1], ("ratelimit/global-write/a", 10), ) req.method = "PUT" self.assertEqual( len(the_app.get_ratelimitable_key_tuples(req, "a", "c", None, global_ratelimit="notafloat")), 1 )
def test_ratelimit_max_rate_double(self): global time_ticker global time_override current_rate = 2 conf_dict = {"account_ratelimit": current_rate, "clock_accuracy": 100, "max_sleep_time_seconds": 1} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) self.test_ratelimit.log_sleep_time_seconds = 0.00001 req = Request.blank("/v/a/c") req.method = "PUT" req.environ["swift.cache"] = FakeMemcache() time_override = [0, 0, 0, 0, None] # simulates 4 requests coming in at same time, then sleeping with mock.patch("swift.common.middleware.ratelimit.get_account_info", lambda *args, **kwargs: {}): r = self.test_ratelimit(req.environ, start_response) mock_sleep(0.1) r = self.test_ratelimit(req.environ, start_response) mock_sleep(0.1) r = self.test_ratelimit(req.environ, start_response) self.assertEqual(r[0], "Slow down") mock_sleep(0.1) r = self.test_ratelimit(req.environ, start_response) self.assertEqual(r[0], "Slow down") mock_sleep(0.1) r = self.test_ratelimit(req.environ, start_response) self.assertEqual(r[0], "204 No Content")
def test_ratelimit_max_rate_double_container(self): global time_ticker global time_override current_rate = 2 conf_dict = { 'container_ratelimit_0': current_rate, 'clock_accuracy': 100, 'max_sleep_time_seconds': 1 } self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) self.test_ratelimit.log_sleep_time_seconds = .00001 req = Request.blank('/v/a/c/o') req.method = 'PUT' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].set(get_container_memcache_key('a', 'c'), {'container_size': 1}) time_override = [0, 0, 0, 0, None] # simulates 4 requests coming in at same time, then sleeping with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], '204 No Content')
def test_ratelimit_max_rate_double_container(self): global time_ticker global time_override current_rate = 2 conf_dict = {'container_ratelimit_0': current_rate, 'clock_accuracy': 100, 'max_sleep_time_seconds': 1} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) self.test_ratelimit.log_sleep_time_seconds = .00001 req = Request.blank('/v1/a/c/o') req.method = 'PUT' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].set( get_cache_key('a', 'c'), {'object_count': 1}) time_override = [0, 0, 0, 0, None] # simulates 4 requests coming in at same time, then sleeping with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEqual(r[0], b'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEqual(r[0], b'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEqual(r[0], b'Some Content')
def test_ratelimit_max_rate_multiple_acc(self): num_calls = 4 current_rate = 2 conf_dict = {"account_ratelimit": current_rate, "max_sleep_time_seconds": 2} fake_memcache = FakeMemcache() the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app.memcache_client = fake_memcache req = lambda: None req.method = "PUT" req.environ = {} class rate_caller(threading.Thread): def __init__(self, name): self.myname = name threading.Thread.__init__(self) def run(self): for j in range(num_calls): self.result = the_app.handle_ratelimit(req, self.myname, "c", None) with mock.patch("swift.common.middleware.ratelimit.get_account_info", lambda *args, **kwargs: {}): nt = 15 begin = time.time() threads = [] for i in range(nt): rc = rate_caller("a%s" % i) rc.start() threads.append(rc) for thread in threads: thread.join() time_took = time.time() - begin self.assertEqual(1.5, round(time_took, 1))
def test_ratelimit_max_rate_double_container_listing(self): global time_ticker global time_override current_rate = 2 conf_dict = {"container_listing_ratelimit_0": current_rate, "clock_accuracy": 100, "max_sleep_time_seconds": 1} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) self.test_ratelimit.log_sleep_time_seconds = 0.00001 req = Request.blank("/v/a/c") req.method = "GET" req.environ["swift.cache"] = FakeMemcache() req.environ["swift.cache"].set(get_cache_key("a", "c"), {"object_count": 1}) with mock.patch("swift.common.middleware.ratelimit.get_account_info", lambda *args, **kwargs: {}): time_override = [0, 0, 0, 0, None] # simulates 4 requests coming in at same time, then sleeping r = self.test_ratelimit(req.environ, start_response) mock_sleep(0.1) r = self.test_ratelimit(req.environ, start_response) mock_sleep(0.1) r = self.test_ratelimit(req.environ, start_response) self.assertEqual(r[0], "Slow down") mock_sleep(0.1) r = self.test_ratelimit(req.environ, start_response) self.assertEqual(r[0], "Slow down") mock_sleep(0.1) r = self.test_ratelimit(req.environ, start_response) self.assertEqual(r[0], "204 No Content") mc = self.test_ratelimit.memcache_client try: self.test_ratelimit.memcache_client = None self.assertEqual(self.test_ratelimit.handle_ratelimit(req, "n", "c", None), None) finally: self.test_ratelimit.memcache_client = mc
def test_registered_defaults(self): def check_key_is_absnet(key): try: swift_info[key] except KeyError as err: if key not in err: raise test_limits = { "account_ratelimit": 1, "max_sleep_time_seconds": 60, "container_ratelimit_0": 0, "container_ratelimit_10": 10, "container_ratelimit_50": 50, "container_listing_ratelimit_0": 0, "container_listing_ratelimit_10": 10, "container_listing_ratelimit_50": 50, } ratelimit.filter_factory(test_limits)("have to pass in an app") swift_info = utils.get_swift_info() self.assertTrue("ratelimit" in swift_info) self.assertEqual(swift_info["ratelimit"]["account_ratelimit"], 1.0) self.assertEqual(swift_info["ratelimit"]["max_sleep_time_seconds"], 60.0) self.assertEqual(swift_info["ratelimit"]["container_ratelimits"][0][0], 0) self.assertEqual(swift_info["ratelimit"]["container_ratelimits"][0][1], 0.0) self.assertEqual(swift_info["ratelimit"]["container_ratelimits"][1][0], 10) self.assertEqual(swift_info["ratelimit"]["container_ratelimits"][1][1], 10.0) self.assertEqual(swift_info["ratelimit"]["container_ratelimits"][2][0], 50) self.assertEqual(swift_info["ratelimit"]["container_ratelimits"][2][1], 50.0) self.assertEqual(swift_info["ratelimit"]["container_listing_ratelimits"][0][0], 0) self.assertEqual(swift_info["ratelimit"]["container_listing_ratelimits"][0][1], 0.0) self.assertEqual(swift_info["ratelimit"]["container_listing_ratelimits"][1][0], 10) self.assertEqual(swift_info["ratelimit"]["container_listing_ratelimits"][1][1], 10.0) self.assertEqual(swift_info["ratelimit"]["container_listing_ratelimits"][2][0], 50) self.assertEqual(swift_info["ratelimit"]["container_listing_ratelimits"][2][1], 50.0) # these were left out on purpose for key in [ "log_sleep_time_seconds", "clock_accuracy", "rate_buffer_seconds", "ratelimit_whitelis", "ratelimit_blacklist", ]: check_key_is_absnet(key)
def test_get_ratelimitable_key_tuples(self): current_rate = 13 conf_dict = { 'account_ratelimit': current_rate, 'container_ratelimit_3': 200 } fake_memcache = FakeMemcache() fake_memcache.store[get_container_memcache_key('a', 'c')] = \ {'object_count': '5'} the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app.memcache_client = fake_memcache req = lambda: None req.environ = {'swift.cache': fake_memcache, 'PATH_INFO': '/v1/a/c/o'} with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): req.method = 'DELETE' self.assertEquals( len(the_app.get_ratelimitable_key_tuples(req, 'a', None, None)), 0) req.method = 'PUT' self.assertEquals( len(the_app.get_ratelimitable_key_tuples(req, 'a', 'c', None)), 1) req.method = 'DELETE' self.assertEquals( len(the_app.get_ratelimitable_key_tuples(req, 'a', 'c', None)), 1) req.method = 'GET' self.assertEquals( len(the_app.get_ratelimitable_key_tuples(req, 'a', 'c', 'o')), 0) req.method = 'PUT' self.assertEquals( len(the_app.get_ratelimitable_key_tuples(req, 'a', 'c', 'o')), 1) def get_fake_ratelimit(*args, **kwargs): return {'sysmeta': {'global-write-ratelimit': 10}} with mock.patch('swift.common.middleware.ratelimit.get_account_info', get_fake_ratelimit): req.method = 'PUT' self.assertEquals( len(the_app.get_ratelimitable_key_tuples(req, 'a', 'c', None)), 2) self.assertEquals( the_app.get_ratelimitable_key_tuples(req, 'a', 'c', None)[1], ('ratelimit/global-write/a', 10)) def get_fake_ratelimit(*args, **kwargs): return {'sysmeta': {'global-write-ratelimit': 'notafloat'}} with mock.patch('swift.common.middleware.ratelimit.get_account_info', get_fake_ratelimit): req.method = 'PUT' self.assertEquals( len(the_app.get_ratelimitable_key_tuples(req, 'a', 'c', None)), 1)
def test_get_maxrate(self): conf_dict = {"container_ratelimit_10": 200, "container_ratelimit_50": 100, "container_ratelimit_75": 30} test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) test_ratelimit.logger = FakeLogger() self.assertEqual(ratelimit.get_maxrate(test_ratelimit.container_ratelimits, 0), None) self.assertEqual(ratelimit.get_maxrate(test_ratelimit.container_ratelimits, 5), None) self.assertEqual(ratelimit.get_maxrate(test_ratelimit.container_ratelimits, 10), 200) self.assertEqual(ratelimit.get_maxrate(test_ratelimit.container_ratelimits, 60), 72) self.assertEqual(ratelimit.get_maxrate(test_ratelimit.container_ratelimits, 160), 30)
def test_no_memcache(self): current_rate = 13 num_calls = 5 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/a') req.environ['swift.cache'] = None make_app_call = lambda: self.test_ratelimit(req.environ, start_response) self._run(make_app_call, num_calls, current_rate)
def test_no_memcache(self): current_rate = 13 num_calls = 5 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/a') req.environ['swift.cache'] = None make_app_call = lambda: self.test_ratelimit(req.environ, start_response ) self._run(make_app_call, num_calls, current_rate)
def test_no_memcache(self): current_rate = 13 num_calls = 5 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) req = Request.blank('/v1/a') req.environ['swift.cache'] = None make_app_call = lambda: self.test_ratelimit(req.environ, start_response ) begin = time.time() self._run(make_app_call, num_calls, current_rate, check_time=False) time_took = time.time() - begin self.assertEqual(round(time_took, 1), 0) # no memcache, no limiting
def test_no_memcache(self): current_rate = 13 num_calls = 5 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) req = Request.blank('/v1/a') req.environ['swift.cache'] = None make_app_call = lambda: self.test_ratelimit(req.environ, start_response) begin = time.time() self._run(make_app_call, num_calls, current_rate, check_time=False) time_took = time.time() - begin self.assertEqual(round(time_took, 1), 0) # no memcache, no limiting
def test_ratelimit_old_memcache_format(self): current_rate = 13 conf_dict = {"account_ratelimit": current_rate, "container_ratelimit_3": 200} fake_memcache = FakeMemcache() fake_memcache.store[get_cache_key("a", "c")] = {"container_size": 5} the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app.memcache_client = fake_memcache req = lambda: None req.method = "PUT" req.environ = {"PATH_INFO": "/v1/a/c/o", "swift.cache": fake_memcache} with mock.patch("swift.common.middleware.ratelimit.get_account_info", lambda *args, **kwargs: {}): tuples = the_app.get_ratelimitable_key_tuples(req, "a", "c", "o") self.assertEqual(tuples, [("ratelimit/a/c", 200.0)])
def test_ratelimit_old_white_black_list(self): global time_ticker current_rate = 2 conf_dict = { "account_ratelimit": current_rate, "max_sleep_time_seconds": 2, "account_whitelist": "a", "account_blacklist": "b", } self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) req = Request.blank("/") with mock.patch.object(self.test_ratelimit, "memcache_client", FakeMemcache()): self.assertEqual(self.test_ratelimit.handle_ratelimit(req, "a", "c", "o"), None) self.assertEqual(self.test_ratelimit.handle_ratelimit(req, "b", "c", "o").status_int, 497)
def test_already_handled(self): current_rate = 13 num_calls = 5 conf_dict = {'container_listing_ratelimit_0': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) fake_cache = FakeMemcache() fake_cache.set(get_cache_key('a', 'c'), {'object_count': 1}) req = Request.blank('/v1/a/c', environ={'swift.cache': fake_cache}) req.environ['swift.ratelimit.handled'] = True make_app_call = lambda: self.test_ratelimit(req.environ, start_response ) begin = time.time() self._run(make_app_call, num_calls, current_rate, check_time=False) time_took = time.time() - begin self.assertEqual(round(time_took, 1), 0) # no memcache, no limiting
def test_ratelimit_old_memcache_format(self): current_rate = 13 conf_dict = {'account_ratelimit': current_rate, 'container_ratelimit_3': 200} fake_memcache = FakeMemcache() fake_memcache.store[get_cache_key('a', 'c')] = \ {'container_size': 5} the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app.memcache_client = fake_memcache req = FakeReq('PUT', { 'PATH_INFO': '/v1/a/c/o', 'swift.cache': fake_memcache}) with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): tuples = the_app.get_ratelimitable_key_tuples(req, 'a', 'c', 'o') self.assertEqual(tuples, [('ratelimit/a/c', 200.0)])
def test_ratelimit_set_incr(self): current_rate = 5 num_calls = 50 conf_dict = {"account_ratelimit": current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) req = Request.blank("/v/a/c") req.method = "PUT" req.environ["swift.cache"] = FakeMemcache() req.environ["swift.cache"].init_incr_return_neg = True make_app_call = lambda: self.test_ratelimit(req.environ, start_response) begin = time.time() with mock.patch("swift.common.middleware.ratelimit.get_account_info", lambda *args, **kwargs: {}): self._run(make_app_call, num_calls, current_rate, check_time=False) self.assertEqual(round(time.time() - begin, 1), 9.8)
def test_ratelimit_set_incr(self): current_rate = 5 num_calls = 50 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/a/c') req.method = 'PUT' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].init_incr_return_neg = True make_app_call = lambda: self.test_ratelimit(req.environ, start_response) begin = time.time() self._run(make_app_call, num_calls, current_rate, check_time=False) self.assertEquals(round(time.time() - begin, 1), 9.8)
def test_restarting_memcache(self): current_rate = 2 num_calls = 5 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/a') req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].error_on_incr = True make_app_call = lambda: self.test_ratelimit(req.environ, start_response) begin = time.time() self._run(make_app_call, num_calls, current_rate, check_time=False) time_took = time.time() - begin self.assert_(round(time_took, 1) == 0) # no memcache, no limiting
def test_restarting_memcache(self): current_rate = 2 num_calls = 5 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/a') req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].error_on_incr = True make_app_call = lambda: self.test_ratelimit(req.environ, start_response ) begin = time.time() self._run(make_app_call, num_calls, current_rate, check_time=False) time_took = time.time() - begin self.assert_(round(time_took, 1) == 0) # no memcache, no limiting
def test_get_ratelimitable_key_tuples(self): current_rate = 13 conf_dict = {'account_ratelimit': current_rate, 'container_ratelimit_3': 200} fake_memcache = FakeMemcache() fake_memcache.store[get_container_memcache_key('a', 'c')] = \ {'object_count': '5'} the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app.memcache_client = fake_memcache req = lambda: None req.environ = {'swift.cache': fake_memcache, 'PATH_INFO': '/v1/a/c/o'} with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): req.method = 'DELETE' self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', None, None)), 0) req.method = 'PUT' self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', 'c', None)), 1) req.method = 'DELETE' self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', 'c', None)), 1) req.method = 'GET' self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', 'c', 'o')), 0) req.method = 'PUT' self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', 'c', 'o')), 1) def get_fake_ratelimit(*args, **kwargs): return {'sysmeta': {'global-write-ratelimit': 10}} with mock.patch('swift.common.middleware.ratelimit.get_account_info', get_fake_ratelimit): req.method = 'PUT' self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', 'c', None)), 2) self.assertEquals(the_app.get_ratelimitable_key_tuples( req, 'a', 'c', None)[1], ('ratelimit/global-write/a', 10)) def get_fake_ratelimit(*args, **kwargs): return {'sysmeta': {'global-write-ratelimit': 'notafloat'}} with mock.patch('swift.common.middleware.ratelimit.get_account_info', get_fake_ratelimit): req.method = 'PUT' self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', 'c', None)), 1)
def test_ratelimit_set_incr(self): current_rate = 5 num_calls = 50 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) req = Request.blank('/v1/a/c') req.method = 'PUT' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].init_incr_return_neg = True make_app_call = lambda: self.test_ratelimit(req.environ, start_response ) begin = time.time() with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): self._run(make_app_call, num_calls, current_rate, check_time=False) self.assertEqual(round(time.time() - begin, 1), 9.8)
def test_ratelimit_old_memcache_format(self): current_rate = 13 conf_dict = {'account_ratelimit': current_rate, 'container_ratelimit_3': 200} fake_memcache = FakeMemcache() fake_memcache.store[get_cache_key('a', 'c')] = \ {'container_size': 5} the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app.memcache_client = fake_memcache req = lambda: None req.method = 'PUT' req.environ = {'PATH_INFO': '/v1/a/c/o', 'swift.cache': fake_memcache} with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): tuples = the_app.get_ratelimitable_key_tuples(req, 'a', 'c', 'o') self.assertEqual(tuples, [('ratelimit/a/c', 200.0)])
def test_get_maxrate(self): conf_dict = {'container_ratelimit_10': 200, 'container_ratelimit_50': 100, 'container_ratelimit_75': 30} test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) test_ratelimit.logger = debug_logger() self.assertIsNone(ratelimit.get_maxrate( test_ratelimit.container_ratelimits, 0)) self.assertIsNone(ratelimit.get_maxrate( test_ratelimit.container_ratelimits, 5)) self.assertEqual(ratelimit.get_maxrate( test_ratelimit.container_ratelimits, 10), 200) self.assertEqual(ratelimit.get_maxrate( test_ratelimit.container_ratelimits, 60), 72) self.assertEqual(ratelimit.get_maxrate( test_ratelimit.container_ratelimits, 160), 30)
def test_restarting_memcache(self): current_rate = 2 num_calls = 5 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) req = Request.blank('/v1/a/c') req.method = 'PUT' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].error_on_incr = True make_app_call = lambda: self.test_ratelimit(req.environ, start_response) begin = time.time() with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): self._run(make_app_call, num_calls, current_rate, check_time=False) time_took = time.time() - begin self.assertEqual(round(time_took, 1), 0) # no memcache, no limit
def test_restarting_memcache(self): current_rate = 2 num_calls = 5 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) req = Request.blank('/v/a/c') req.method = 'PUT' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].error_on_incr = True make_app_call = lambda: self.test_ratelimit(req.environ, start_response ) begin = time.time() with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): self._run(make_app_call, num_calls, current_rate, check_time=False) time_took = time.time() - begin self.assertEqual(round(time_took, 1), 0) # no memcache, no limit
def test_account_ratelimit(self): current_rate = 5 num_calls = 50 conf_dict = {"account_ratelimit": current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) with mock.patch("swift.common.middleware.ratelimit.get_container_info", lambda *args, **kwargs: {}): with mock.patch("swift.common.middleware.ratelimit.get_account_info", lambda *args, **kwargs: {}): for meth, exp_time in [("DELETE", 9.8), ("GET", 0), ("POST", 0), ("PUT", 9.8)]: req = Request.blank("/v/a%s/c" % meth) req.method = meth req.environ["swift.cache"] = FakeMemcache() make_app_call = lambda: self.test_ratelimit(req.environ, start_response) begin = time.time() self._run(make_app_call, num_calls, current_rate, check_time=bool(exp_time)) self.assertEqual(round(time.time() - begin, 1), exp_time) self._reset_time()
def test_ratelimit_old_white_black_list(self): global time_ticker current_rate = 2 conf_dict = { 'account_ratelimit': current_rate, 'max_sleep_time_seconds': 2, 'account_whitelist': 'a', 'account_blacklist': 'b' } self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) req = Request.blank('/') with mock.patch.object(self.test_ratelimit, 'memcache_client', FakeMemcache()): self.assertEqual( self.test_ratelimit.handle_ratelimit(req, 'a', 'c', 'o'), None) self.assertEqual( self.test_ratelimit.handle_ratelimit(req, 'b', 'c', 'o').status_int, 497)
def test_ratelimit_old_white_black_list(self): global time_ticker current_rate = 2 conf_dict = {'account_ratelimit': current_rate, 'max_sleep_time_seconds': 2, 'account_whitelist': 'a', 'account_blacklist': 'b'} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) req = Request.blank('/') with mock.patch.object(self.test_ratelimit, 'memcache_client', FakeMemcache()): self.assertEqual( self.test_ratelimit.handle_ratelimit(req, 'a', 'c', 'o'), None) self.assertEqual( self.test_ratelimit.handle_ratelimit( req, 'b', 'c', 'o').status_int, 497)
def test_account_ratelimit(self): current_rate = 5 num_calls = 50 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) for meth, exp_time in [('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]: req = Request.blank('/v/a%s/c' % meth) req.method = meth req.environ['swift.cache'] = FakeMemcache() make_app_call = lambda: self.test_ratelimit(req.environ, start_response) begin = time.time() self._run(make_app_call, num_calls, current_rate, check_time=bool(exp_time)) self.assertEquals(round(time.time() - begin, 1), exp_time) self._reset_time()
def test_call_invalid_path(self): env = {'REQUEST_METHOD': 'GET', 'SCRIPT_NAME': '', 'PATH_INFO': '//v1/AUTH_1234567890', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '80', 'swift.cache': FakeMemcache(), 'SERVER_PROTOCOL': 'HTTP/1.0'} app = lambda *args, **kwargs: ['fake_app'] rate_mid = ratelimit.filter_factory({})(app) class a_callable(object): def __call__(self, *args, **kwargs): pass resp = rate_mid.__call__(env, a_callable()) self.assertEqual('fake_app', resp[0])
def test_ratelimit_blacklist(self): global time_ticker current_rate = 2 conf_dict = { 'account_ratelimit': current_rate, 'max_sleep_time_seconds': 2, 'account_whitelist': 'a', 'account_blacklist': 'b' } self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) self.test_ratelimit.logger = FakeLogger() self.test_ratelimit.BLACK_LIST_SLEEP = 0 ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/b/c') req.environ['swift.cache'] = FakeMemcache() class rate_caller(Thread): def __init__(self, parent): Thread.__init__(self) self.parent = parent def run(self): self.result = self.parent.test_ratelimit( req.environ, start_response) def get_fake_ratelimit(*args, **kwargs): return {'sysmeta': {'global-write-ratelimit': 'BLACKLIST'}} with mock.patch('swift.common.middleware.ratelimit.get_account_info', get_fake_ratelimit): nt = 5 threads = [] for i in range(nt): rc = rate_caller(self) rc.start() threads.append(rc) for thread in threads: thread.join() the_497s = [ t for t in threads if ''.join(t.result).startswith('Your account') ] self.assertEquals(len(the_497s), 5) self.assertEquals(time_ticker, 0)
def test_call_invalid_path(self): env = { "REQUEST_METHOD": "GET", "SCRIPT_NAME": "", "PATH_INFO": "//v1/AUTH_1234567890", "SERVER_NAME": "127.0.0.1", "SERVER_PORT": "80", "swift.cache": FakeMemcache(), "SERVER_PROTOCOL": "HTTP/1.0", } app = lambda *args, **kwargs: ["fake_app"] rate_mid = ratelimit.filter_factory({})(app) class a_callable(object): def __call__(self, *args, **kwargs): pass resp = rate_mid.__call__(env, a_callable()) self.assertTrue("fake_app" == resp[0])
def test_ratelimit_max_rate_double_container_listing(self): global time_ticker global time_override current_rate = 2 conf_dict = { 'container_listing_ratelimit_0': current_rate, 'clock_accuracy': 100, 'max_sleep_time_seconds': 1 } self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) self.test_ratelimit.log_sleep_time_seconds = .00001 req = Request.blank('/v/a/c') req.method = 'GET' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].set( ratelimit.get_container_memcache_key('a', 'c'), {'container_size': 1}) time_override = [0, 0, 0, 0, None] # simulates 4 requests coming in at same time, then sleeping r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], '204 No Content') mc = self.test_ratelimit.memcache_client try: self.test_ratelimit.memcache_client = None self.assertEquals( self.test_ratelimit.handle_ratelimit(req, 'n', 'c', None), None) finally: self.test_ratelimit.memcache_client = mc
def test_ratelimit_max_rate_double_container_listing(self): global time_ticker global time_override current_rate = 2 conf_dict = {'container_listing_ratelimit_0': current_rate, 'clock_accuracy': 100, 'max_sleep_time_seconds': 1} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) self.test_ratelimit.log_sleep_time_seconds = .00001 req = Request.blank('/v/a/c') req.method = 'GET' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].set( get_container_memcache_key('a', 'c'), {'container_size': 1}) with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): time_override = [0, 0, 0, 0, None] # simulates 4 requests coming in at same time, then sleeping r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], '204 No Content') mc = self.test_ratelimit.memcache_client try: self.test_ratelimit.memcache_client = None self.assertEquals( self.test_ratelimit.handle_ratelimit(req, 'n', 'c', None), None) finally: self.test_ratelimit.memcache_client = mc
def test_ratelimit_whitelist_sysmeta(self): global time_ticker current_rate = 2 conf_dict = { 'account_ratelimit': current_rate, 'max_sleep_time_seconds': 2, 'account_whitelist': 'a', 'account_blacklist': 'b' } self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) req = Request.blank('/v1/a/c') req.environ['swift.cache'] = FakeMemcache() class rate_caller(threading.Thread): def __init__(self, parent): threading.Thread.__init__(self) self.parent = parent def run(self): self.result = self.parent.test_ratelimit( req.environ, start_response) def get_fake_ratelimit(*args, **kwargs): return {'sysmeta': {'global-write-ratelimit': 'WHITELIST'}} with mock.patch('swift.common.middleware.ratelimit.get_account_info', get_fake_ratelimit): nt = 5 threads = [] for i in range(nt): rc = rate_caller(self) rc.start() threads.append(rc) for thread in threads: thread.join() the_498s = [ t for t in threads if b''.join(t.result).startswith(b'Slow down') ] self.assertEqual(len(the_498s), 0) self.assertEqual(time_ticker, 0)
def test_ratelimit_blacklist(self): global time_ticker current_rate = 2 conf_dict = {'account_ratelimit': current_rate, 'max_sleep_time_seconds': 2, 'account_whitelist': 'a', 'account_blacklist': 'b'} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) self.test_ratelimit.logger = FakeLogger() self.test_ratelimit.BLACK_LIST_SLEEP = 0 req = Request.blank('/v1/b/c') req.environ['swift.cache'] = FakeMemcache() class rate_caller(threading.Thread): def __init__(self, parent): threading.Thread.__init__(self) self.parent = parent def run(self): self.result = self.parent.test_ratelimit(req.environ, start_response) def get_fake_ratelimit(*args, **kwargs): return {'sysmeta': {'global-write-ratelimit': 'BLACKLIST'}} with mock.patch('swift.common.middleware.ratelimit.get_account_info', get_fake_ratelimit): nt = 5 threads = [] for i in range(nt): rc = rate_caller(self) rc.start() threads.append(rc) for thread in threads: thread.join() the_497s = [ t for t in threads if b''.join(t.result).startswith(b'Your account')] self.assertEqual(len(the_497s), 5) self.assertEqual(time_ticker, 0)
def test_call_non_swift_api_path(self): env = {'REQUEST_METHOD': 'GET', 'SCRIPT_NAME': '', 'PATH_INFO': '/ive/got/a/lovely/bunch/of/coconuts', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '80', 'swift.cache': FakeMemcache(), 'SERVER_PROTOCOL': 'HTTP/1.0'} app = lambda *args, **kwargs: ['some response'] rate_mid = ratelimit.filter_factory({})(app) class a_callable(object): def __call__(self, *args, **kwargs): pass with mock.patch('swift.common.middleware.ratelimit.get_account_info', side_effect=Exception("you shouldn't call this")): resp = rate_mid(env, a_callable()) self.assertEqual(resp[0], 'some response')
def test_ratelimit_whitelist_sysmeta(self): global time_ticker current_rate = 2 conf_dict = {'account_ratelimit': current_rate, 'max_sleep_time_seconds': 2, 'account_whitelist': 'a', 'account_blacklist': 'b'} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/a/c') req.environ['swift.cache'] = FakeMemcache() class rate_caller(Thread): def __init__(self, parent): Thread.__init__(self) self.parent = parent def run(self): self.result = self.parent.test_ratelimit(req.environ, start_response) def get_fake_ratelimit(*args, **kwargs): return {'sysmeta': {'global-write-ratelimit': 'WHITELIST'}} with mock.patch('swift.common.middleware.ratelimit.get_account_info', get_fake_ratelimit): nt = 5 threads = [] for i in range(nt): rc = rate_caller(self) rc.start() threads.append(rc) for thread in threads: thread.join() the_498s = [ t for t in threads if ''.join(t.result).startswith('Slow down')] self.assertEquals(len(the_498s), 0) self.assertEquals(time_ticker, 0)
def test_ratelimit_max_rate_double_container_listing(self): global time_ticker global time_override current_rate = 2 conf_dict = { 'container_listing_ratelimit_0': current_rate, 'clock_accuracy': 100, 'max_sleep_time_seconds': 1 } self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) self.test_ratelimit.log_sleep_time_seconds = .00001 req = Request.blank('/v1/a/c') req.method = 'GET' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].set(get_cache_key('a', 'c'), {'object_count': 1}) with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): time_override = [0, 0, 0, 0, None] # simulates 4 requests coming in at same time, then sleeping r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEqual(r[0], 'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEqual(r[0], 'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEqual(r[0], '204 No Content') mc = self.test_ratelimit.memcache_client try: self.test_ratelimit.memcache_client = None self.assertIsNone( self.test_ratelimit.handle_ratelimit(req, 'n', 'c', None)) finally: self.test_ratelimit.memcache_client = mc
def test_ratelimit_blacklist(self): global time_ticker current_rate = 2 conf_dict = { "account_ratelimit": current_rate, "max_sleep_time_seconds": 2, "account_whitelist": "a", "account_blacklist": "b", } self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) self.test_ratelimit.logger = FakeLogger() self.test_ratelimit.BLACK_LIST_SLEEP = 0 ratelimit.http_connect = mock_http_connect(204) req = Request.blank("/v/b/c") req.environ["swift.cache"] = FakeMemcache() class rate_caller(threading.Thread): def __init__(self, parent): threading.Thread.__init__(self) self.parent = parent def run(self): self.result = self.parent.test_ratelimit(req.environ, start_response) def get_fake_ratelimit(*args, **kwargs): return {"sysmeta": {"global-write-ratelimit": "BLACKLIST"}} with mock.patch("swift.common.middleware.ratelimit.get_account_info", get_fake_ratelimit): nt = 5 threads = [] for i in range(nt): rc = rate_caller(self) rc.start() threads.append(rc) for thread in threads: thread.join() the_497s = [t for t in threads if "".join(t.result).startswith("Your account")] self.assertEqual(len(the_497s), 5) self.assertEqual(time_ticker, 0)
def test_account_ratelimit(self): current_rate = 5 num_calls = 50 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) with mock.patch('swift.common.middleware.ratelimit.get_container_info', lambda *args, **kwargs: {}): with mock.patch( 'swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): for meth, exp_time in [('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]: req = Request.blank('/v1/a%s/c' % meth) req.method = meth req.environ['swift.cache'] = FakeMemcache() make_app_call = lambda: self.test_ratelimit(req.environ, start_response) begin = time.time() self._run(make_app_call, num_calls, current_rate, check_time=bool(exp_time)) self.assertEqual(round(time.time() - begin, 1), exp_time) self._reset_time()
def test_ratelimit_whitelist(self): global time_ticker current_rate = 2 conf_dict = { 'account_ratelimit': current_rate, 'max_sleep_time_seconds': 2, 'account_whitelist': 'a', 'account_blacklist': 'b' } self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/a/c') req.environ['swift.cache'] = FakeMemcache() class rate_caller(Thread): def __init__(self, parent): Thread.__init__(self) self.parent = parent def run(self): self.result = self.parent.test_ratelimit( req.environ, start_response) nt = 5 threads = [] for i in range(nt): rc = rate_caller(self) rc.start() threads.append(rc) for thread in threads: thread.join() the_498s = [ t for t in threads if ''.join(t.result).startswith('Slow down') ] self.assertEquals(len(the_498s), 0) self.assertEquals(time_ticker, 0)