def test_peek(self): PyRateLimit.init(redis_host="localhost", redis_port=6379) limit = PyRateLimit(period=10, limit=10) for x in range(0, 10): self.assertTrue(limit.attempt('test_namespace2')) self.assertTrue(limit.is_rate_limited('test_namespace2')) time.sleep(10) self.assertFalse(limit.is_rate_limited('test_namespace2'))
def test_peek(self): PyRateLimit.init(redis_host="localhost", redis_port=6379) limit = PyRateLimit() limit.create(10, # rate limit period in seconds 10) # no of attempts in the time period for x in range(0, 10): self.assertTrue(limit.attempt('test_namespace2')) self.assertTrue(limit.is_rate_limited('test_namespace2')) time.sleep(10) self.assertFalse(limit.is_rate_limited('test_namespace2'))
def test_peek(self): PyRateLimit.init(redis_host="localhost", redis_port=6379) limit = PyRateLimit() limit.create( 10, # rate limit period in seconds 10) # no of attempts in the time period for x in range(0, 10): self.assertTrue(limit.attempt('test_namespace2')) self.assertTrue(limit.is_rate_limited('test_namespace2')) time.sleep(10) self.assertFalse(limit.is_rate_limited('test_namespace2'))
def test_throttle(self): PyRateLimit.init(redis_host="localhost", redis_port=6379) limit = PyRateLimit(period=10, limit=10) for x in range(0, 20): time.sleep(.5) if x < 10: self.assertTrue(limit.attempt('test_namespace')) else: self.assertFalse(limit.attempt('test_namespace')) time.sleep(6) self.assertTrue(limit.attempt('test_namespace'))
def test_throttle(self): PyRateLimit.init(redis_host="localhost", redis_port=6379) limit = PyRateLimit() limit.create(10, # rate limit period in seconds 10) # no of attempts in the time period for x in range(0, 20): time.sleep(.5) if x < 10: self.assertTrue(limit.attempt('test_namespace')) else: self.assertFalse(limit.attempt('test_namespace')) time.sleep(6) self.assertTrue(limit.attempt('test_namespace'))
def test_throttle(self): PyRateLimit.init(redis_host="localhost", redis_port=6379) limit = PyRateLimit() limit.create( 10, # rate limit period in seconds 10) # no of attempts in the time period for x in range(0, 20): time.sleep(.5) if x < 10: self.assertTrue(limit.attempt('test_namespace')) else: self.assertFalse(limit.attempt('test_namespace')) time.sleep(6) self.assertTrue(limit.attempt('test_namespace'))
def __init__(self, crawler_name, seconds=1, minutes=0, hours=0, rate=10, max_tokens=10): super().__init__(crawler_name, seconds, minutes, hours, rate, max_tokens) PyRateLimit.init( redis_host=settings.REDIS_HOST_PRIMARY, redis_port=settings.REDIS_PORT_PRIMARY, redis_password=settings.REDIS_PASS_PRIMARY, )
def __init__(self, *args): self.logger = logging.getLogger(__name__) self.base_url = 'https://www.virustotal.com/vtapi/v2/' self.headers = { "Accept-Encoding": "gzip, deflate", "User-Agent": "Anteater" } self.HTTP_OK = 200 self.HTTP_BAD = 400 self.HTTP_FORBIDDEN = 403 self.HTTP_RATE_EXCEEDED = 204 self.public_api_sleep_time = 20 self.logger = logging.getLogger(__name__) self.uuid = uuid.uuid4() self.config = six.moves.configparser.SafeConfigParser() self.config.read('anteater.conf') try: conn = redis.StrictRedis(host='localhost', port=6379, password='') conn.ping() PyRateLimit.init(redis_host="localhost", redis_port=6379) except Exception as ex: self.logger.error('Error: %s', ex) exit('Failed to connect, terminating.') self.limit = PyRateLimit() try: vt_rate_type = self.config.get('config', 'vt_rate_type') except six.moves.configparser.NoSectionError: self.logger.error( "A config section is required for vt_rate_type with a public | private option " ) sys.exit(1) patten = re.compile(r'\bpublic\b|\bprivate\b') if not patten.match(vt_rate_type): self.logger.error("Unrecognized %s option for vt_rate_type", vt_rate_type) sys.exit(1) if vt_rate_type == 'public': self.limit.create(21, 1) elif vt_rate_type == 'private': self.limit.create(1, 1)