def ratelimit(request: HTTPServerRequest, area: str = "global") -> bool:
"""Test if a requesting IP is ratelimited for a certain area. Areas are
different functionalities of the website, for example 'view' or 'input' to
differentiate between creating new pastes (low volume) or high volume
viewing.
Important: this does not work well for IPv6 addresses where really when a
ratelimit is hit, we should walk up the CIDRs (/64, /48, etc) as those
ranges usually belong to the same person. If this is encountered in real
life this function will have to be expanded."""
if area not in ratelimit_area:
ratelimit_area[area] = {}
# TODO handle valueerror as validationerror?
address = ipaddress.ip_address(request.remote_ip)
if address not in ratelimit_area[area]:
ratelimit_area[area][address] = token_bucket.Limiter(
configuration.ratelimit[area]["refill"],
configuration.ratelimit[area]["capacity"],
token_bucket.MemoryStorage(),
)
if not ratelimit_area[area][address].consume(1):
log.warning("%s hit rate limit for %r", address, area)
return True
return False
def __init__(self, api_key):
self.BASE_URL = 'https://www.giantbomb.com/api'
self.api_key = api_key
# Ratelimit burst limit 200, renews at 200 / 1hr
self.bucket_storage = token_bucket.MemoryStorage()
self.ratelimit = token_bucket.Limiter(200 / (60 * 60), 200, self.bucket_storage)
def test_token_bucket(self):
storage = token_bucket.MemoryStorage()
limiter = token_bucket.Limiter(10, 10, storage)
for i in range(30):
if limiter.consume('key'):
print("Write to kafka")
else:
print("Filter")
import time
time.sleep(.05)
def __init__(self,
credentials: PhemexCredentials = PublicCredentials(),
api_url='https://api.phemex.com',
request_timeout: int = 30,
rate: float = 2.5,
capacity: int = 200):
self.credentials = credentials
self.api_url = api_url.rstrip('/')
self.request_timeout = request_timeout
self.session = Session()
self.storage = token_bucket.MemoryStorage()
self.limiter = token_bucket.Limiter(rate, capacity, self.storage)
def test_general_functionality(rate, capacity):
key = 'key'
storage = token_bucket.MemoryStorage()
limiter = token_bucket.Limiter(rate, capacity, storage)
assert storage.get_token_count(key) == 0
consume_one = functools.partial(limiter.consume, key)
# NOTE(kgriffs) Trigger creation of the bucket and then
# sleep to ensure it is at full capacity before testing it.
consume_one()
time.sleep(float(capacity) / rate)
# NOTE(kgriffs): This works because we can consume at a much
# higher rate relative to the replenishment rate, such that we
# easily consume the total capacity before a single token can
# be replenished.
def consume_all():
for i in range(capacity + 3):
conforming = consume_one()
# NOTE(kgriffs): One past the end should be non-conforming,
# but sometimes an extra token or two can be generated, so
# only check a couple past the end for non-conforming.
if i < capacity:
assert conforming
elif i > capacity + 1:
assert not conforming
# Check non-conforming after consuming all of the tokens
consume_all()
# Let the bucket replenish 1 token
time.sleep(1.0 / rate)
assert consume_one()
# NOTE(kgriffs): Occasionally enough time will have elapsed to
# cause an additional token to be generated. Clear that one
# out if it is there.
consume_one()
assert storage.get_token_count(key) < 1.0
# NOTE(kgriffs): Let the bucket replenish all the tokens; do this
# twice to verify that the bucket is limited to capacity.
for __ in range(2):
time.sleep(float(capacity) / rate)
storage.replenish(key, rate, capacity)
assert int(storage.get_token_count(key)) == capacity
consume_all()
def handle_command_line_args():
parser = argparse.ArgumentParser()
parser.add_argument("-v", "--verbose", help="print all debug messages", action="store_true")
parser.add_argument("--fahad_keyword", help="five-letter word to replace with 'Fahad'", type=str, default="their")
parser.add_argument("--bandwidth", help="limits client's allowed bandwidth (bytes/sec)", type=int, required=False)
parser.add_argument("--burst_rate", help="max bandwidth client is allowed (bytes/sec)", type=int, required=False)
parser.add_argument("-a", "--server_address", help="server's address (eg: localhost)", type=str, default="localhost")
parser.add_argument("-b", "--blacklist_sites", help="name of file containing list of blacklisted hostnames", type=str, default=None)
parser.add_argument("port", help="port to run on", type=int)
args = parser.parse_args()
global VERBOSE
if args.verbose:
VERBOSE = True
global RATE
if args.bandwidth:
RATE = args.bandwidth
global CAPACITY
if args.burst_rate:
CAPACITY = args.burst_rate
global USING_RATE_LIMITING
if args.burst_rate and args.bandwidth:
USING_RATE_LIMITING = True
global MODIFY_CONTENT
global CONTENT_REPLACE_TARGET
if args.fahad_keyword:
MODIFY_CONTENT = True
CONTENT_REPLACE_TARGET = args.fahad_keyword[:5] # limit to 5 letters
global limiter
global storage
if USING_RATE_LIMITING:
storage = token_bucket.MemoryStorage()
limiter = token_bucket.Limiter(RATE, CAPACITY, storage)
global USING_BLACKLIST
if args.blacklist_sites is not None:
USING_BLACKLIST = True
parse_blacklisted_hostnames(args.blacklist_sites)
return args
def test_consume_multiple_tokens_at_a_time(capacity):
rate = 100
num_tokens = capacity
key = 'key'
storage = token_bucket.MemoryStorage()
limiter = token_bucket.Limiter(rate, capacity, storage)
assert not limiter.consume(key, num_tokens=capacity + 1)
# NOTE(kgriffs): Should be able to conform indefinitely since we
# are matching the replenishment rate; verify for five seconds
# only.
for __ in range(int(rate * 5 / num_tokens)):
assert limiter.consume(key, num_tokens=num_tokens)
assert storage.get_token_count(key) < 1.0
# Sleep long enough to generate num_tokens
time.sleep(1.0 / rate * num_tokens)
def test_conforming_ratio():
rate = 100
capacity = 10
key = 'key'
target_ratio = 0.5
ratio_max = 0.62
num_threads = 4
storage = token_bucket.MemoryStorage()
limiter = token_bucket.Limiter(rate, capacity, storage)
# NOTE(kgriffs): Rather than using a lock to protect some counters,
# rely on the GIL and count things up after the fact.
conforming_states = []
# NOTE(kgriffs): Start with an empty bucket
while limiter.consume(key):
pass
def loop():
# NOTE(kgriffs): Run for 10 seconds
for __ in range(int(rate * 10 / target_ratio / num_threads)):
conforming_states.append(limiter.consume(key))
# NOTE(kgriffs): Only generate some of the tokens needed, so
# that some requests will end up being non-conforming.
time.sleep(1.0 / rate * target_ratio * num_threads)
_run_threaded(loop, num_threads)
total_conforming = 0
for c in conforming_states:
if c:
total_conforming += 1
actual_ratio = float(total_conforming) / len(conforming_states)
# NOTE(kgriffs): We don't expect to be super precise due to
# the inprecision of time.sleep() and also having to take into
# account execution time of the other instructions in the
# loop. We do expect a few more conforming states vs. non-
# conforming since the sleep time + overall execution time
# makes the threads run a little behind the replenishment rate.
assert target_ratio < actual_ratio < ratio_max
def test_different_keys():
rate = 10
capacity = 10
storage = token_bucket.MemoryStorage()
limiter = token_bucket.Limiter(rate, capacity, storage)
keys = [
uuid.uuid4().bytes, u'3084"5tj jafsb: f', b'77752098', u'whiz:bang',
b'x'
]
# The last two should be non-conforming
for i in range(capacity + 2):
for k in keys:
conforming = limiter.consume(k)
if i < capacity:
assert conforming
else:
assert not conforming
def __init__(self, ctx, localpath, s3path):
self.ctx = ctx
self.config = ctx.obj['CONFIG']
self.localpath = localpath
self.s3path = s3path
self.include_patterns = self.config['watcher']['include_patterns']
self.exclude_patterns = self.config['watcher']['exclude_patterns']
exclude_directories = self.config['watcher']['exclude_directories']
case_sensitive = self.config['watcher']['case_sensitive']
watchdog.events.PatternMatchingEventHandler.__init__(self, patterns=self.include_patterns, ignore_patterns=self.exclude_patterns,
ignore_directories=exclude_directories, case_sensitive=case_sensitive)
self.syncop = partial(_do_sync, ctx, self.localpath, self.s3path, include_patterns=self.include_patterns, exclude_patterns=self.exclude_patterns)
storage = token_bucket.MemoryStorage()
per_second_rate = (float(self.config['global']['max_syncs_per_minute'])/60.0)*NUM_TOKENS_PER_PUSH
logger.debug("Rate limiting to [{}] tokens per second".format(per_second_rate))
self.limiter = token_bucket.Limiter(per_second_rate, 60, storage)
#do one sync to begin with
self.syncop()
def test_negative_count(rate, capacity, max_tokens_to_consume):
# NOTE(kgriffs): Usually there will be a much larger number of
# keys in a production system, but keep to just five to increase
# the likelihood of collisions.
keys = [uuid.uuid4().bytes for __ in range(5)]
num_threads = 100
storage = token_bucket.MemoryStorage()
limiter = token_bucket.Limiter(rate, capacity, storage)
token_counts = []
def loop():
for __ in range(1000):
key = random.choice(keys)
num_tokens = random.randint(1, max_tokens_to_consume)
# NOTE(kgriffs): The race condition is only possible when
# conforming.
if limiter.consume(key, num_tokens):
token_counts.append(storage.get_token_count(key))
# NOTE(kgriffs): Sleep for only a short time to increase the
# likelihood for contention, while keeping to something
# more realistic than a no-op (max of 1 ms)
time.sleep(random.random() / 1000)
_run_threaded(loop, num_threads)
negative_counts = [c for c in token_counts if c < 0]
if negative_counts:
# NOTE(kgriffs): Negatives should be rare
ratio_of_negatives = float(len(negative_counts)) / len(token_counts)
assert ratio_of_negatives < 0.008
# NOTE(kgriffs): Shouldn't ever have more than a few extra tokens
# removed. Allow for 2-3 collisions at max token removal
assert (max_tokens_to_consume * -2) < min(negative_counts)
def __init__(self, bot):
self.bot = bot
self.inprogressEdits = {}
# !profile ratelimits
self.bucket_storage = token_bucket.MemoryStorage()
self.profile_bucket = token_bucket.Limiter(
1 / 30, 2,
self.bucket_storage) # burst limit 2, renews at 1 / 30 s
# Profile generation
self.twemojiPath = 'resources/twemoji/assets/72x72/'
self.bot_contributors = [
125233822760566784, # MattBSG
123879073972748290, # Lyrus
108429628560924672, # Alex from Alaska
115840403458097161, # FlapSnapple
]
# Profile generation - precaching
self.profileFonts = self._load_fonts({
'meta': ('Regular', 36),
'user': ('Regular', 48),
'subtext': ('Light', 48),
'medium': ('Light', 36),
'small': ('Light', 30),
})
with open("resources/profiles/themes.yml", 'r') as stream:
self.themes = yaml.safe_load(stream)
with open("resources/profiles/borders.yml", 'r') as stream:
self.borders = yaml.safe_load(stream)
with open("resources/profiles/backgrounds.yml", 'r') as stream:
self.backgrounds = yaml.safe_load(stream)
for bg_name in self.backgrounds.keys():
self.backgrounds[bg_name][
"image"] = self._render_background_image(bg_name)
for theme in self.themes.keys():
self.themes[theme]['pfpBackground'] = Image.open(
f'resources/profiles/layout/{theme}/pfp-background.png'
).convert('RGBA')
self.themes[theme]['missingImage'] = (Image.open(
f'resources/profiles/layout/{theme}/missing-game.png').convert(
"RGBA").resize((45, 45)))
self.themes[theme]['profileStatic'] = self._init_profile_static(
theme) # Do this last
self.trophyImgCache = {}
self.borderImgCache = {}
self.flagImgCache = {}
self.gameImgCache = {}
# Friend Code Regexs (\u2014 = em-dash)
self.friendCodeRegex = {
# Profile setup/editor (lenient)
"profile":
re.compile(
r'(?:sw)?[ \-\u2014_]?(\d{4})[ \-\u2014_]?(\d{4})[ \-\u2014_]?(\d{4})',
re.I),
# Chat filter, "It appears you've sent a friend code." Requires separators and discards select prefixes.
# Discarded prefixes: MA/MO (AC Designer), DA (AC Dream Address).
"chatFilter":
re.compile(
r'(sw|m[^ao]|d[^a]|[^MD]\w|^\w|^)[ \-\u2014_]?\d{4}[ \-\u2014_]\d{4}[ \-\u2014_]\d{4}',
re.I + re.M),
}
self.special_trophies = {
'bot':
lambda member, guild: member.bot,
'owner':
lambda member, guild: member.id == guild.owner.id,
'developer':
lambda member, guild: member.id in self.bot_contributors,
'chat-mod':
lambda member, guild: guild.get_role(config.chatmod
) in member.roles,
'sub-mod':
lambda member, guild: guild.get_role(config.submod
) in member.roles,
'mod-emeritus':
lambda member, guild: guild.get_role(config.modemeritus) in member.
roles or guild.get_role(config.submodemeritus) in member.roles,
'helpful-user':
lambda member, guild: guild.get_role(config.helpfulUser
) in member.roles,
'booster':
lambda member, guild: guild.get_role(config.boostRole
) in member.roles,
'verified':
lambda member, guild: guild.get_role(config.verified
) in member.roles,
}
def set_rate(self, rate):
self.limiter = token_bucket.Limiter(rate, rate, token_bucket.MemoryStorage())
self.rate = rate
def test_input_validation_storage_type():
class DoesNotInheritFromStorageBase(object):
pass
with pytest.raises(TypeError):
token_bucket.Limiter(1, 1, DoesNotInheritFromStorageBase())
# List of active connections - dicts, with IP, port, socket object,
# thread signal (to kill corresponding thread), and Common Name
# example_connection = {
# 'ip' : '10.0.0.2',
# 'port' : 2000,
# 'socket' : conn,
# 'kill_signal' : False,
# 'CN' : 'client'
# }
active_connections = []
rate_per_key = 10 # frames per second
max_capacity = 20 # token capacity - defines burst size
buckets = token_bucket.Limiter(rate_per_key, max_capacity,
token_bucket.MemoryStorage())
# Initialize a TAP interface and make it non-blocking
tap = TunTapDevice(flags=IFF_TAP | IFF_NO_PI, name='tap0')
tap.up()
fcntl.fcntl(tap.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
tap_thread = threading.Thread(target=ThreadTapFunction,
args=(
tap,
active_connections,
))
tap_thread.daemon = True
tap_thread.start()
t = threading.Thread(target=ServerFunction,
def test_input_validation_rate_and_capacity(rate, capacity, etype):
with pytest.raises(etype):
token_bucket.Limiter(rate, capacity, token_bucket.MemoryStorage())
tls = TLSTunnel ()
tls.initialize()
tls.create_tap()
if tls.args.mode == 'client':
while (True):
try:
tls.cli_start_new_client()
except Exception as e:
print("Fail to establish connection or connection closed. Waiting a bit before retrying. " + str(e))
sleep(5)
elif tls.args.mode == 'server':
print ("tb", tls.tb_rate, tls.tb_burst)
tls.buckets = token_bucket.Limiter(tls.tb_rate, tls.tb_burst, token_bucket.MemoryStorage())
# start tap reading and sending to clients via TLS
tap_thread = threading.Thread(target=TLSTunnel.srvThreadTapReadFunction, args=(tls.tap, tls.active_connections,))
tap_thread.daemon = True; tap_thread.start()
# start server thread waiting for new clients and spawn new tap read from client and write to tap
t = threading.Thread(target=tls.srvThreadServerFunction, args=(tls.args.listen_addr, tls.args.listen_port, tls.active_connections, tls.buckets,))
t.daemon = True; t.start()
# printout stats and don't exit
while True:
sleep(30)
print("\nActive connections:")
for i, connection in enumerate(tls.active_connections):
print("{} - {}:{} ({})".format(i, connection['ip'], connection['port'], connection['CN']))
def test_input_validation_on_consume(key, num_tokens, etype):
limiter = token_bucket.Limiter(1, 1, token_bucket.MemoryStorage())
with pytest.raises(etype):
limiter.consume(key, num_tokens)
