def ratelimit(request: HTTPServerRequest, area: str = "global") -> bool: """Test if a requesting IP is ratelimited for a certain area. Areas are different functionalities of the website, for example 'view' or 'input' to differentiate between creating new pastes (low volume) or high volume viewing. Important: this does not work well for IPv6 addresses where really when a ratelimit is hit, we should walk up the CIDRs (/64, /48, etc) as those ranges usually belong to the same person. If this is encountered in real life this function will have to be expanded.""" if area not in ratelimit_area: ratelimit_area[area] = {} # TODO handle valueerror as validationerror? address = ipaddress.ip_address(request.remote_ip) if address not in ratelimit_area[area]: ratelimit_area[area][address] = token_bucket.Limiter( configuration.ratelimit[area]["refill"], configuration.ratelimit[area]["capacity"], token_bucket.MemoryStorage(), ) if not ratelimit_area[area][address].consume(1): log.warning("%s hit rate limit for %r", address, area) return True return False
def __init__(self, api_key): self.BASE_URL = 'https://www.giantbomb.com/api' self.api_key = api_key # Ratelimit burst limit 200, renews at 200 / 1hr self.bucket_storage = token_bucket.MemoryStorage() self.ratelimit = token_bucket.Limiter(200 / (60 * 60), 200, self.bucket_storage)
def test_token_bucket(self): storage = token_bucket.MemoryStorage() limiter = token_bucket.Limiter(10, 10, storage) for i in range(30): if limiter.consume('key'): print("Write to kafka") else: print("Filter") import time time.sleep(.05)
def __init__(self, credentials: PhemexCredentials = PublicCredentials(), api_url='https://api.phemex.com', request_timeout: int = 30, rate: float = 2.5, capacity: int = 200): self.credentials = credentials self.api_url = api_url.rstrip('/') self.request_timeout = request_timeout self.session = Session() self.storage = token_bucket.MemoryStorage() self.limiter = token_bucket.Limiter(rate, capacity, self.storage)
def test_general_functionality(rate, capacity): key = 'key' storage = token_bucket.MemoryStorage() limiter = token_bucket.Limiter(rate, capacity, storage) assert storage.get_token_count(key) == 0 consume_one = functools.partial(limiter.consume, key) # NOTE(kgriffs) Trigger creation of the bucket and then # sleep to ensure it is at full capacity before testing it. consume_one() time.sleep(float(capacity) / rate) # NOTE(kgriffs): This works because we can consume at a much # higher rate relative to the replenishment rate, such that we # easily consume the total capacity before a single token can # be replenished. def consume_all(): for i in range(capacity + 3): conforming = consume_one() # NOTE(kgriffs): One past the end should be non-conforming, # but sometimes an extra token or two can be generated, so # only check a couple past the end for non-conforming. if i < capacity: assert conforming elif i > capacity + 1: assert not conforming # Check non-conforming after consuming all of the tokens consume_all() # Let the bucket replenish 1 token time.sleep(1.0 / rate) assert consume_one() # NOTE(kgriffs): Occasionally enough time will have elapsed to # cause an additional token to be generated. Clear that one # out if it is there. consume_one() assert storage.get_token_count(key) < 1.0 # NOTE(kgriffs): Let the bucket replenish all the tokens; do this # twice to verify that the bucket is limited to capacity. for __ in range(2): time.sleep(float(capacity) / rate) storage.replenish(key, rate, capacity) assert int(storage.get_token_count(key)) == capacity consume_all()
def handle_command_line_args(): parser = argparse.ArgumentParser() parser.add_argument("-v", "--verbose", help="print all debug messages", action="store_true") parser.add_argument("--fahad_keyword", help="five-letter word to replace with 'Fahad'", type=str, default="their") parser.add_argument("--bandwidth", help="limits client's allowed bandwidth (bytes/sec)", type=int, required=False) parser.add_argument("--burst_rate", help="max bandwidth client is allowed (bytes/sec)", type=int, required=False) parser.add_argument("-a", "--server_address", help="server's address (eg: localhost)", type=str, default="localhost") parser.add_argument("-b", "--blacklist_sites", help="name of file containing list of blacklisted hostnames", type=str, default=None) parser.add_argument("port", help="port to run on", type=int) args = parser.parse_args() global VERBOSE if args.verbose: VERBOSE = True global RATE if args.bandwidth: RATE = args.bandwidth global CAPACITY if args.burst_rate: CAPACITY = args.burst_rate global USING_RATE_LIMITING if args.burst_rate and args.bandwidth: USING_RATE_LIMITING = True global MODIFY_CONTENT global CONTENT_REPLACE_TARGET if args.fahad_keyword: MODIFY_CONTENT = True CONTENT_REPLACE_TARGET = args.fahad_keyword[:5] # limit to 5 letters global limiter global storage if USING_RATE_LIMITING: storage = token_bucket.MemoryStorage() limiter = token_bucket.Limiter(RATE, CAPACITY, storage) global USING_BLACKLIST if args.blacklist_sites is not None: USING_BLACKLIST = True parse_blacklisted_hostnames(args.blacklist_sites) return args
def test_consume_multiple_tokens_at_a_time(capacity): rate = 100 num_tokens = capacity key = 'key' storage = token_bucket.MemoryStorage() limiter = token_bucket.Limiter(rate, capacity, storage) assert not limiter.consume(key, num_tokens=capacity + 1) # NOTE(kgriffs): Should be able to conform indefinitely since we # are matching the replenishment rate; verify for five seconds # only. for __ in range(int(rate * 5 / num_tokens)): assert limiter.consume(key, num_tokens=num_tokens) assert storage.get_token_count(key) < 1.0 # Sleep long enough to generate num_tokens time.sleep(1.0 / rate * num_tokens)
def test_conforming_ratio(): rate = 100 capacity = 10 key = 'key' target_ratio = 0.5 ratio_max = 0.62 num_threads = 4 storage = token_bucket.MemoryStorage() limiter = token_bucket.Limiter(rate, capacity, storage) # NOTE(kgriffs): Rather than using a lock to protect some counters, # rely on the GIL and count things up after the fact. conforming_states = [] # NOTE(kgriffs): Start with an empty bucket while limiter.consume(key): pass def loop(): # NOTE(kgriffs): Run for 10 seconds for __ in range(int(rate * 10 / target_ratio / num_threads)): conforming_states.append(limiter.consume(key)) # NOTE(kgriffs): Only generate some of the tokens needed, so # that some requests will end up being non-conforming. time.sleep(1.0 / rate * target_ratio * num_threads) _run_threaded(loop, num_threads) total_conforming = 0 for c in conforming_states: if c: total_conforming += 1 actual_ratio = float(total_conforming) / len(conforming_states) # NOTE(kgriffs): We don't expect to be super precise due to # the inprecision of time.sleep() and also having to take into # account execution time of the other instructions in the # loop. We do expect a few more conforming states vs. non- # conforming since the sleep time + overall execution time # makes the threads run a little behind the replenishment rate. assert target_ratio < actual_ratio < ratio_max
def test_different_keys(): rate = 10 capacity = 10 storage = token_bucket.MemoryStorage() limiter = token_bucket.Limiter(rate, capacity, storage) keys = [ uuid.uuid4().bytes, u'3084"5tj jafsb: f', b'77752098', u'whiz:bang', b'x' ] # The last two should be non-conforming for i in range(capacity + 2): for k in keys: conforming = limiter.consume(k) if i < capacity: assert conforming else: assert not conforming
def __init__(self, ctx, localpath, s3path): self.ctx = ctx self.config = ctx.obj['CONFIG'] self.localpath = localpath self.s3path = s3path self.include_patterns = self.config['watcher']['include_patterns'] self.exclude_patterns = self.config['watcher']['exclude_patterns'] exclude_directories = self.config['watcher']['exclude_directories'] case_sensitive = self.config['watcher']['case_sensitive'] watchdog.events.PatternMatchingEventHandler.__init__(self, patterns=self.include_patterns, ignore_patterns=self.exclude_patterns, ignore_directories=exclude_directories, case_sensitive=case_sensitive) self.syncop = partial(_do_sync, ctx, self.localpath, self.s3path, include_patterns=self.include_patterns, exclude_patterns=self.exclude_patterns) storage = token_bucket.MemoryStorage() per_second_rate = (float(self.config['global']['max_syncs_per_minute'])/60.0)*NUM_TOKENS_PER_PUSH logger.debug("Rate limiting to [{}] tokens per second".format(per_second_rate)) self.limiter = token_bucket.Limiter(per_second_rate, 60, storage) #do one sync to begin with self.syncop()
def test_replenishment(): capacity = 100 rate = 100 num_threads = 4 trials = 100 storage = token_bucket.MemoryStorage() def loop(): for i in range(trials): key = str(i) for __ in range(int(capacity / num_threads)): storage.replenish(key, rate, capacity) time.sleep(1.0 / rate) _run_threaded(loop, num_threads) # NOTE(kgriffs): Ensure that a race condition did not result in # not all the tokens being replenished for i in range(trials): key = str(i) assert storage.get_token_count(key) == capacity
def test_negative_count(rate, capacity, max_tokens_to_consume): # NOTE(kgriffs): Usually there will be a much larger number of # keys in a production system, but keep to just five to increase # the likelihood of collisions. keys = [uuid.uuid4().bytes for __ in range(5)] num_threads = 100 storage = token_bucket.MemoryStorage() limiter = token_bucket.Limiter(rate, capacity, storage) token_counts = [] def loop(): for __ in range(1000): key = random.choice(keys) num_tokens = random.randint(1, max_tokens_to_consume) # NOTE(kgriffs): The race condition is only possible when # conforming. if limiter.consume(key, num_tokens): token_counts.append(storage.get_token_count(key)) # NOTE(kgriffs): Sleep for only a short time to increase the # likelihood for contention, while keeping to something # more realistic than a no-op (max of 1 ms) time.sleep(random.random() / 1000) _run_threaded(loop, num_threads) negative_counts = [c for c in token_counts if c < 0] if negative_counts: # NOTE(kgriffs): Negatives should be rare ratio_of_negatives = float(len(negative_counts)) / len(token_counts) assert ratio_of_negatives < 0.008 # NOTE(kgriffs): Shouldn't ever have more than a few extra tokens # removed. Allow for 2-3 collisions at max token removal assert (max_tokens_to_consume * -2) < min(negative_counts)
def set_rate(self, rate): self.limiter = token_bucket.Limiter(rate, rate, token_bucket.MemoryStorage()) self.rate = rate
def test_input_validation_rate_and_capacity(rate, capacity, etype): with pytest.raises(etype): token_bucket.Limiter(rate, capacity, token_bucket.MemoryStorage())
# List of active connections - dicts, with IP, port, socket object, # thread signal (to kill corresponding thread), and Common Name # example_connection = { # 'ip' : '10.0.0.2', # 'port' : 2000, # 'socket' : conn, # 'kill_signal' : False, # 'CN' : 'client' # } active_connections = [] rate_per_key = 10 # frames per second max_capacity = 20 # token capacity - defines burst size buckets = token_bucket.Limiter(rate_per_key, max_capacity, token_bucket.MemoryStorage()) # Initialize a TAP interface and make it non-blocking tap = TunTapDevice(flags=IFF_TAP | IFF_NO_PI, name='tap0') tap.up() fcntl.fcntl(tap.fileno(), fcntl.F_SETFL, os.O_NONBLOCK) tap_thread = threading.Thread(target=ThreadTapFunction, args=( tap, active_connections, )) tap_thread.daemon = True tap_thread.start() t = threading.Thread(target=ServerFunction,
def __init__(self, bot): self.bot = bot self.inprogressEdits = {} # !profile ratelimits self.bucket_storage = token_bucket.MemoryStorage() self.profile_bucket = token_bucket.Limiter( 1 / 30, 2, self.bucket_storage) # burst limit 2, renews at 1 / 30 s # Profile generation self.twemojiPath = 'resources/twemoji/assets/72x72/' self.bot_contributors = [ 125233822760566784, # MattBSG 123879073972748290, # Lyrus 108429628560924672, # Alex from Alaska 115840403458097161, # FlapSnapple ] # Profile generation - precaching self.profileFonts = self._load_fonts({ 'meta': ('Regular', 36), 'user': ('Regular', 48), 'subtext': ('Light', 48), 'medium': ('Light', 36), 'small': ('Light', 30), }) with open("resources/profiles/themes.yml", 'r') as stream: self.themes = yaml.safe_load(stream) with open("resources/profiles/borders.yml", 'r') as stream: self.borders = yaml.safe_load(stream) with open("resources/profiles/backgrounds.yml", 'r') as stream: self.backgrounds = yaml.safe_load(stream) for bg_name in self.backgrounds.keys(): self.backgrounds[bg_name][ "image"] = self._render_background_image(bg_name) for theme in self.themes.keys(): self.themes[theme]['pfpBackground'] = Image.open( f'resources/profiles/layout/{theme}/pfp-background.png' ).convert('RGBA') self.themes[theme]['missingImage'] = (Image.open( f'resources/profiles/layout/{theme}/missing-game.png').convert( "RGBA").resize((45, 45))) self.themes[theme]['profileStatic'] = self._init_profile_static( theme) # Do this last self.trophyImgCache = {} self.borderImgCache = {} self.flagImgCache = {} self.gameImgCache = {} # Friend Code Regexs (\u2014 = em-dash) self.friendCodeRegex = { # Profile setup/editor (lenient) "profile": re.compile( r'(?:sw)?[ \-\u2014_]?(\d{4})[ \-\u2014_]?(\d{4})[ \-\u2014_]?(\d{4})', re.I), # Chat filter, "It appears you've sent a friend code." Requires separators and discards select prefixes. # Discarded prefixes: MA/MO (AC Designer), DA (AC Dream Address). "chatFilter": re.compile( r'(sw|m[^ao]|d[^a]|[^MD]\w|^\w|^)[ \-\u2014_]?\d{4}[ \-\u2014_]\d{4}[ \-\u2014_]\d{4}', re.I + re.M), } self.special_trophies = { 'bot': lambda member, guild: member.bot, 'owner': lambda member, guild: member.id == guild.owner.id, 'developer': lambda member, guild: member.id in self.bot_contributors, 'chat-mod': lambda member, guild: guild.get_role(config.chatmod ) in member.roles, 'sub-mod': lambda member, guild: guild.get_role(config.submod ) in member.roles, 'mod-emeritus': lambda member, guild: guild.get_role(config.modemeritus) in member. roles or guild.get_role(config.submodemeritus) in member.roles, 'helpful-user': lambda member, guild: guild.get_role(config.helpfulUser ) in member.roles, 'booster': lambda member, guild: guild.get_role(config.boostRole ) in member.roles, 'verified': lambda member, guild: guild.get_role(config.verified ) in member.roles, }
tls = TLSTunnel () tls.initialize() tls.create_tap() if tls.args.mode == 'client': while (True): try: tls.cli_start_new_client() except Exception as e: print("Fail to establish connection or connection closed. Waiting a bit before retrying. " + str(e)) sleep(5) elif tls.args.mode == 'server': print ("tb", tls.tb_rate, tls.tb_burst) tls.buckets = token_bucket.Limiter(tls.tb_rate, tls.tb_burst, token_bucket.MemoryStorage()) # start tap reading and sending to clients via TLS tap_thread = threading.Thread(target=TLSTunnel.srvThreadTapReadFunction, args=(tls.tap, tls.active_connections,)) tap_thread.daemon = True; tap_thread.start() # start server thread waiting for new clients and spawn new tap read from client and write to tap t = threading.Thread(target=tls.srvThreadServerFunction, args=(tls.args.listen_addr, tls.args.listen_port, tls.active_connections, tls.buckets,)) t.daemon = True; t.start() # printout stats and don't exit while True: sleep(30) print("\nActive connections:") for i, connection in enumerate(tls.active_connections): print("{} - {}:{} ({})".format(i, connection['ip'], connection['port'], connection['CN']))
def test_input_validation_on_consume(key, num_tokens, etype): limiter = token_bucket.Limiter(1, 1, token_bucket.MemoryStorage()) with pytest.raises(etype): limiter.consume(key, num_tokens)