def test_decrypt(self, backend):
f1 = Fernet(base64.urlsafe_b64encode(b"\x00" * 32), backend=backend)
f2 = Fernet(base64.urlsafe_b64encode(b"\x01" * 32), backend=backend)
f = MultiFernet([f1, f2])
assert f.decrypt(f1.encrypt(b"abc")) == b"abc"
assert f.decrypt(f2.encrypt(b"abc")) == b"abc"
with pytest.raises(InvalidToken):
f.decrypt(b"\x00" * 16)
def test_rotate(self, backend):
f1 = Fernet(base64.urlsafe_b64encode(b"\x00" * 32), backend=backend)
f2 = Fernet(base64.urlsafe_b64encode(b"\x01" * 32), backend=backend)
mf1 = MultiFernet([f1])
mf2 = MultiFernet([f2, f1])
plaintext = b"abc"
mf1_ciphertext = mf1.encrypt(plaintext)
assert mf2.decrypt(mf1_ciphertext) == plaintext
rotated = mf2.rotate(mf1_ciphertext)
assert rotated != mf1_ciphertext
assert mf2.decrypt(rotated) == plaintext
with pytest.raises(InvalidToken):
mf1.decrypt(rotated)
class EncryptingPacker(object):
"""Implement conversion of Python objects to/from encrypted bytestrings.
:param str key: a `Fernet`_ key to use for encryption and decryption
:param list old_keys: additional `Fernet`_ keys to use for decryption
.. note::
Encrypted messages contain the timestamp at which they were generated
*in plaintext*. See `our audit`_ for discussion of this and other
considerations with `Fernet`_.
.. _Fernet: https://cryptography.io/en/latest/fernet/
.. _our audit: https://github.com/gratipay/gratipay.com/pull/3998#issuecomment-216227070
"""
def __init__(self, key, *old_keys):
keys = [key] + list(old_keys)
self.fernet = MultiFernet([Fernet(k) for k in keys])
def pack(self, obj):
"""Given a JSON-serializable object, return a `Fernet`_ token.
"""
obj = json.dumps(obj) # serialize to unicode
obj = obj.encode('utf8') # convert to bytes
obj = self.fernet.encrypt(obj) # encrypt
return obj
def unpack(self, token):
"""Given a `Fernet`_ token with JSON in the ciphertext, return a Python object.
"""
obj = token
if not type(obj) is bytes:
raise TypeError("need bytes, got {}".format(type(obj)))
obj = self.fernet.decrypt(obj) # decrypt
obj = obj.decode('utf8') # convert to unicode
obj = json.loads(obj) # deserialize from unicode
return obj
from cryptography.fernet import Fernet, MultiFernet # The sample code is extracted from the book Python Cryptography # The book can be downloaded from https://leanpub.com/cryptop # Online Crypto Playgroud https://8gwifi.org # Author Anish Nath key1 = Fernet(Fernet.generate_key()) key2 = Fernet(Fernet.generate_key()) key3 = Fernet(Fernet.generate_key()) plaintext = "Hello 8gwifi.org" f = MultiFernet([key1, key2, key3]) token = f.encrypt(plaintext) d = f.decrypt(token) assert d, plaintext # Rotating key key4 = Fernet(Fernet.generate_key()) key5 = Fernet(Fernet.generate_key()) f2 = MultiFernet([key4, key5, key1, key2, key3]) rotated = f2.rotate(token) d = f2.decrypt(token) assert d, plaintext
class RedisChannelLayer(BaseChannelLayer):
"""
Redis channel layer.
It routes all messages into remote Redis server. Support for
sharding among different Redis installations and message
encryption are provided. Both synchronous and asynchronous (via
Twisted) approaches are implemented.
"""
blpop_timeout = 5
global_statistics_expiry = 86400
channel_statistics_expiry = 3600
global_stats_key = '#global#' # needs to be invalid as a channel name
def __init__(
self,
expiry=60,
hosts=None,
prefix="asgi:",
group_expiry=86400,
capacity=100,
channel_capacity=None,
symmetric_encryption_keys=None,
stats_prefix="asgi-meta:",
connection_kwargs=None,
):
super(RedisChannelLayer, self).__init__(
expiry=expiry,
group_expiry=group_expiry,
capacity=capacity,
channel_capacity=channel_capacity,
)
self.hosts = self._setup_hosts(hosts)
self.prefix = prefix
assert isinstance(self.prefix, six.text_type), "Prefix must be unicode"
# Precalculate some values for ring selection
self.ring_size = len(self.hosts)
# Create connections ahead of time (they won't call out just yet, but
# we want to connection-pool them later)
socket_timeout = connection_kwargs and connection_kwargs.get(
"socket_timeout", None)
if socket_timeout and socket_timeout < self.blpop_timeout:
raise ValueError("The socket timeout must be at least %s seconds" %
self.blpop_timeout)
self._connection_list = self._generate_connections(
redis_kwargs=connection_kwargs or {}, )
# Normal channels choose a host index by cycling through the available hosts
self._receive_index_generator = itertools.cycle(range(len(self.hosts)))
self._send_index_generator = itertools.cycle(range(len(self.hosts)))
# Decide on a unique client prefix to use in ! sections
# TODO: ensure uniqueness better, e.g. Redis keys with SETNX
self.client_prefix = "".join(
random.choice(string.ascii_letters) for i in range(8))
self._register_scripts()
self._setup_encryption(symmetric_encryption_keys)
self.stats_prefix = stats_prefix
def _setup_hosts(self, hosts):
# Make sure they provided some hosts, or provide a default
final_hosts = list()
if not hosts:
hosts = [("localhost", 6379)]
if isinstance(hosts, six.string_types):
# user accidentally used one host string instead of providing a list of hosts
raise ValueError(
'ASGI Redis hosts must be specified as an iterable list of hosts.'
)
for entry in hosts:
if isinstance(entry, six.string_types):
final_hosts.append(entry)
else:
final_hosts.append("redis://%s:%d/0" % (entry[0], entry[1]))
return final_hosts
def _register_scripts(self):
connection = self.connection(None)
self.chansend = connection.register_script(self.lua_chansend)
self.lpopmany = connection.register_script(self.lua_lpopmany)
self.delprefix = connection.register_script(self.lua_delprefix)
self.incrstatcounters = connection.register_script(
self.lua_incrstatcounters)
self.chansend.sha = hashlib.sha1(b(self.chansend.script)).hexdigest()
self.lpopmany.sha = hashlib.sha1(b(self.lpopmany.script)).hexdigest()
self.delprefix.sha = hashlib.sha1(b(self.delprefix.script)).hexdigest()
self.incrstatcounters.sha = hashlib.sha1(
b(self.incrstatcounters.script)).hexdigest()
def _setup_encryption(self, symmetric_encryption_keys):
# See if we can do encryption if they asked
if symmetric_encryption_keys:
if isinstance(symmetric_encryption_keys, six.string_types):
raise ValueError(
"symmetric_encryption_keys must be a list of possible keys"
)
try:
from cryptography.fernet import MultiFernet
except ImportError:
raise ValueError(
"Cannot run with encryption without 'cryptography' installed."
)
sub_fernets = [
self.make_fernet(key) for key in symmetric_encryption_keys
]
self.crypter = MultiFernet(sub_fernets)
else:
self.crypter = None
def _generate_connections(self, redis_kwargs):
return [
redis.Redis.from_url(host, **redis_kwargs) for host in self.hosts
]
### ASGI API ###
extensions = ["groups", "flush", "statistics"]
try:
import txredisapi
except ImportError:
pass
else:
extensions.append("twisted")
def send(self, channel, message):
# Typecheck
assert isinstance(message, dict), "message is not a dict"
assert self.valid_channel_name(channel), "Channel name not valid"
# Make sure the message does not contain reserved keys
assert "__asgi_channel__" not in message
# If it's a process-local channel, strip off local part and stick full name in message
if "!" in channel:
message = dict(message.items())
message['__asgi_channel__'] = channel
channel = self.non_local_name(channel)
# Write out message into expiring key (avoids big items in list)
# TODO: Use extended set, drop support for older redis?
message_key = self.prefix + uuid.uuid4().hex
channel_key = self.prefix + channel
# Pick a connection to the right server - consistent for response
# channels, random for normal channels
if "!" in channel or "?" in channel:
index = self.consistent_hash(channel)
connection = self.connection(index)
else:
index = next(self._send_index_generator)
connection = self.connection(index)
# Use the Lua function to do the set-and-push
try:
self.chansend(
keys=[message_key, channel_key],
args=[
self.serialize(message), self.expiry,
self.get_capacity(channel)
],
client=connection,
)
self._incr_statistics_counter(
stat_name=self.STAT_MESSAGES_COUNT,
channel=channel,
connection=connection,
)
except redis.exceptions.ResponseError as e:
# The Lua script handles capacity checking and sends the "full" error back
if e.args[0] == "full":
self._incr_statistics_counter(
stat_name=self.STAT_CHANNEL_FULL,
channel=channel,
connection=connection,
)
raise self.ChannelFull
elif "unknown command" in e.args[0]:
raise UnsupportedRedis(
"Redis returned an error (%s). Please ensure you're running a "
" version of redis that is supported by asgi_redis." %
e.args[0])
else:
# Let any other exception bubble up
raise
def receive(self, channels, block=False):
# List name get
indexes = self._receive_list_names(channels)
# Short circuit if no channels
if indexes is None:
return None, None
# Get a message from one of our channels
while True:
got_expired_content = False
# Try each index:channels pair at least once or until a result is returned
for index, list_names in indexes.items():
# Shuffle list_names to avoid the first ones starving others of workers
random.shuffle(list_names)
# Open a connection
connection = self.connection(index)
# Pop off any waiting message
if block:
result = connection.blpop(list_names,
timeout=self.blpop_timeout)
else:
result = self.lpopmany(keys=list_names, client=connection)
if result:
content = connection.get(result[1])
connection.delete(result[1])
if content is None:
# If the content key expired, keep going.
got_expired_content = True
continue
# Return the channel it's from and the message
channel = result[0][len(self.prefix):].decode("utf8")
message = self.deserialize(content)
# If there is a full channel name stored in the message, unpack it.
if "__asgi_channel__" in message:
channel = message['__asgi_channel__']
del message['__asgi_channel__']
return channel, message
# If we only got expired content, try again
if got_expired_content:
continue
else:
return None, None
def _receive_list_names(self, channels):
"""
Inner logic of receive; takes channels, groups by shard, and
returns {connection_index: list_names ...} if a query is needed or
None for a vacuously empty response.
"""
# Short circuit if no channels
if not channels:
return None
# Check channel names are valid
channels = list(channels)
assert all(
self.valid_channel_name(channel, receive=True)
for channel in channels), "One or more channel names invalid"
# Work out what servers to listen on for the given channels
indexes = {}
index = next(self._receive_index_generator)
for channel in channels:
if "!" in channel or "?" in channel:
indexes.setdefault(self.consistent_hash(channel),
[]).append(self.prefix + channel, )
else:
indexes.setdefault(index, []).append(self.prefix + channel, )
return indexes
def new_channel(self, pattern):
assert isinstance(pattern, six.text_type)
# Keep making channel names till one isn't present.
while True:
random_string = "".join(
random.choice(string.ascii_letters) for i in range(12))
assert pattern.endswith("?")
new_name = pattern + random_string
# Get right connection
index = self.consistent_hash(new_name)
connection = self.connection(index)
# Check to see if it's in the connected Redis.
# This fails to stop collisions for sharding where the channel is
# non-single-listener, but that seems very unlikely.
key = self.prefix + new_name
if not connection.exists(key):
return new_name
### ASGI Group extension ###
def group_add(self, group, channel):
"""
Adds the channel to the named group for at least 'expiry'
seconds (expiry defaults to message expiry if not provided).
"""
assert self.valid_group_name(group), "Group name not valid"
assert self.valid_channel_name(channel), "Channel name not valid"
group_key = self._group_key(group)
connection = self.connection(self.consistent_hash(group))
# Add to group sorted set with creation time as timestamp
connection.zadd(group_key, **{channel: time.time()})
# Set both expiration to be group_expiry, since everything in
# it at this point is guaranteed to expire before that
connection.expire(group_key, self.group_expiry)
def group_discard(self, group, channel):
"""
Removes the channel from the named group if it is in the group;
does nothing otherwise (does not error)
"""
assert self.valid_group_name(group), "Group name not valid"
assert self.valid_channel_name(channel), "Channel name not valid"
key = self._group_key(group)
self.connection(self.consistent_hash(group)).zrem(
key,
channel,
)
def group_channels(self, group):
"""
Returns all channels in the group as an iterable.
"""
key = self._group_key(group)
connection = self.connection(self.consistent_hash(group))
# Discard old channels based on group_expiry
connection.zremrangebyscore(key, 0,
int(time.time()) - self.group_expiry)
# Return current lot
return [x.decode("utf8") for x in connection.zrange(
key,
0,
-1,
)]
def send_group(self, group, message):
"""
Sends a message to the entire group.
"""
assert self.valid_group_name(group), "Group name not valid"
# TODO: More efficient implementation (lua script per shard?)
for channel in self.group_channels(group):
try:
self.send(channel, message)
except self.ChannelFull:
pass
def _group_key(self, group):
return ("%s:group:%s" % (self.prefix, group)).encode("utf8")
### Flush extension ###
def flush(self):
"""
Deletes all messages and groups on all shards.
"""
for connection in self._connection_list:
self.delprefix(keys=[],
args=[self.prefix + "*"],
client=connection)
self.delprefix(keys=[],
args=[self.stats_prefix + "*"],
client=connection)
### Twisted extension ###
@defer.inlineCallbacks
def receive_twisted(self, channels):
"""
Twisted-native implementation of receive.
"""
# List name get
indexes = self._receive_list_names(channels)
# Short circuit if no channels
if indexes is None:
defer.returnValue((None, None))
# Get a message from one of our channels
while True:
got_expired_content = False
# Try each index:channels pair at least once or until a result is returned
for index, list_names in indexes.items():
# Shuffle list_names to avoid the first ones starving others of workers
random.shuffle(list_names)
# Get a sync connection for conn details
sync_connection = self.connection(index)
twisted_connection = yield txredisapi.ConnectionPool(
host=sync_connection.connection_pool.
connection_kwargs['host'],
port=sync_connection.connection_pool.
connection_kwargs['port'],
dbid=sync_connection.connection_pool.
connection_kwargs['db'],
password=sync_connection.connection_pool.
connection_kwargs['password'],
)
try:
# Pop off any waiting message
result = yield twisted_connection.blpop(
list_names, timeout=self.blpop_timeout)
if result:
content = yield twisted_connection.get(result[1])
# If the content key expired, keep going.
if content is None:
got_expired_content = True
continue
# Return the channel it's from and the message
channel = result[0][len(self.prefix):]
message = self.deserialize(content)
# If there is a full channel name stored in the message, unpack it.
if "__asgi_channel__" in message:
channel = message['__asgi_channel__']
del message['__asgi_channel__']
defer.returnValue((channel, message))
finally:
yield twisted_connection.disconnect()
# If we only got expired content, try again
if got_expired_content:
continue
else:
defer.returnValue((None, None))
### statistics extension ###
STAT_MESSAGES_COUNT = 'messages_count'
STAT_MESSAGES_PENDING = 'messages_pending'
STAT_MESSAGES_MAX_AGE = 'messages_max_age'
STAT_CHANNEL_FULL = 'channel_full_count'
def global_statistics(self):
"""
Returns dictionary of statistics across all channels on all shards.
Return value is a dictionary with following fields:
* messages_count, the number of messages processed since server start
* channel_full_count, the number of times ChannelFull exception has been risen since server start
This implementation does not provide calculated per second values.
Due perfomance concerns, does not provide aggregated messages_pending and messages_max_age,
these are only avaliable per channel.
"""
return self._count_global_stats(self._connection_list)
def _count_global_stats(self, connection_list):
statistics = {
self.STAT_MESSAGES_COUNT: 0,
self.STAT_CHANNEL_FULL: 0,
}
prefix = self.stats_prefix + self.global_stats_key
for connection in connection_list:
messages_count, channel_full_count = connection.mget(
':'.join((prefix, self.STAT_MESSAGES_COUNT)),
':'.join((prefix, self.STAT_CHANNEL_FULL)),
)
statistics[self.STAT_MESSAGES_COUNT] += int(messages_count or 0)
statistics[self.STAT_CHANNEL_FULL] += int(channel_full_count or 0)
return statistics
def channel_statistics(self, channel):
"""
Returns dictionary of statistics for specified channel.
Return value is a dictionary with following fields:
* messages_count, the number of messages processed since server start
* messages_pending, the current number of messages waiting
* messages_max_age, how long the oldest message has been waiting, in seconds
* channel_full_count, the number of times ChannelFull exception has been risen since server start
This implementation does not provide calculated per second values
"""
if "!" in channel or "?" in channel:
connections = [self.connection(self.consistent_hash(channel))]
else:
# if we don't know where it is, we have to check in all shards
connections = self._connection_list
return self._count_channel_stats(channel, connections)
def _count_channel_stats(self, channel, connections):
statistics = {
self.STAT_MESSAGES_COUNT: 0,
self.STAT_MESSAGES_PENDING: 0,
self.STAT_MESSAGES_MAX_AGE: 0,
self.STAT_CHANNEL_FULL: 0,
}
prefix = self.stats_prefix + channel
channel_key = self.prefix + channel
for connection in connections:
messages_count, channel_full_count = connection.mget(
':'.join((prefix, self.STAT_MESSAGES_COUNT)),
':'.join((prefix, self.STAT_CHANNEL_FULL)),
)
statistics[self.STAT_MESSAGES_COUNT] += int(messages_count or 0)
statistics[self.STAT_CHANNEL_FULL] += int(channel_full_count or 0)
statistics[self.STAT_MESSAGES_PENDING] += connection.llen(
channel_key)
oldest_message = connection.lindex(channel_key, 0)
if oldest_message:
messages_age = self.expiry - connection.ttl(oldest_message)
statistics[self.STAT_MESSAGES_MAX_AGE] = max(
statistics[self.STAT_MESSAGES_MAX_AGE], messages_age)
return statistics
def _incr_statistics_counter(self, stat_name, channel, connection):
""" helper function to intrement counter stats in one go """
self.incrstatcounters(
keys=[
"{prefix}{channel}:{stat_name}".format(
prefix=self.stats_prefix,
channel=channel,
stat_name=stat_name,
), "{prefix}{global_key}:{stat_name}".format(
prefix=self.stats_prefix,
global_key=self.global_stats_key,
stat_name=stat_name,
)
],
args=[
self.channel_statistics_expiry, self.global_statistics_expiry
],
client=connection,
)
### Serialization ###
def serialize(self, message):
"""
Serializes message to a byte string.
"""
value = msgpack.packb(message, use_bin_type=True)
if self.crypter:
value = self.crypter.encrypt(value)
return value
def deserialize(self, message):
"""
Deserializes from a byte string.
"""
if self.crypter:
message = self.crypter.decrypt(message, self.expiry + 10)
return msgpack.unpackb(message, encoding="utf8")
### Redis Lua scripts ###
# Single-command channel send. Returns error if over capacity.
# Keys: message, channel_list
# Args: content, expiry, capacity
lua_chansend = """
if redis.call('llen', KEYS[2]) >= tonumber(ARGV[3]) then
return redis.error_reply("full")
end
redis.call('set', KEYS[1], ARGV[1])
redis.call('expire', KEYS[1], ARGV[2])
redis.call('rpush', KEYS[2], KEYS[1])
redis.call('expire', KEYS[2], ARGV[2] + 1)
"""
# Single-command to increment counter stats.
# Keys: channel_stat, global_stat
# Args: channel_stat_expiry, global_stat_expiry
lua_incrstatcounters = """
redis.call('incr', KEYS[1])
redis.call('expire', KEYS[1], ARGV[1])
redis.call('incr', KEYS[2])
redis.call('expire', KEYS[2], ARGV[2])
"""
lua_lpopmany = """
for keyCount = 1, #KEYS do
local result = redis.call('LPOP', KEYS[keyCount])
if result then
return {KEYS[keyCount], result}
end
end
return {nil, nil}
"""
lua_delprefix = """
local keys = redis.call('keys', ARGV[1])
for i=1,#keys,5000 do
redis.call('del', unpack(keys, i, math.min(i+4999, #keys)))
end
"""
### Internal functions ###
def consistent_hash(self, value):
"""
Maps the value to a node value between 0 and 4095
using CRC, then down to one of the ring nodes.
"""
if isinstance(value, six.text_type):
value = value.encode("utf8")
bigval = binascii.crc32(value) & 0xfff
ring_divisor = 4096 / float(self.ring_size)
return int(bigval / ring_divisor)
def random_index(self):
return random.randint(0, len(self.hosts) - 1)
def connection(self, index):
"""
Returns the correct connection for the current thread.
Pass key to use a server based on consistent hashing of the key value;
pass None to use a random server instead.
"""
# If index is explicitly None, pick a random server
if index is None:
index = self.random_index()
# Catch bad indexes
if not 0 <= index < self.ring_size:
raise ValueError("There are only %s hosts - you asked for %s!" %
(self.ring_size, index))
return self._connection_list[index]
def make_fernet(self, key):
"""
Given a single encryption key, returns a Fernet instance using it.
"""
from cryptography.fernet import Fernet
if isinstance(key, six.text_type):
key = key.encode("utf8")
formatted_key = base64.urlsafe_b64encode(hashlib.sha256(key).digest())
return Fernet(formatted_key)
def __str__(self):
return "%s(hosts=%s)" % (self.__class__.__name__, self.hosts)
class RedisChannelLayer(BaseChannelLayer):
"""
ORM-backed channel environment. For development use only; it will span
multiple processes fine, but it's going to be pretty bad at throughput.
"""
blpop_timeout = 5
def __init__(self,
expiry=60,
hosts=None,
prefix="asgi:",
group_expiry=86400,
capacity=100,
channel_capacity=None,
symmetric_encryption_keys=None):
super(RedisChannelLayer, self).__init__(
expiry=expiry,
group_expiry=group_expiry,
capacity=capacity,
channel_capacity=channel_capacity,
)
# Make sure they provided some hosts, or provide a default
if not hosts:
hosts = [("localhost", 6379)]
self.hosts = []
for entry in hosts:
if isinstance(entry, six.string_types):
self.hosts.append(entry)
else:
self.hosts.append("redis://%s:%d/0" % (entry[0], entry[1]))
self.prefix = prefix
assert isinstance(self.prefix, six.text_type), "Prefix must be unicode"
# Precalculate some values for ring selection
self.ring_size = len(self.hosts)
self.ring_divisor = int(math.ceil(4096 / float(self.ring_size)))
# Create connections ahead of time (they won't call out just yet, but
# we want to connection-pool them later)
self._connection_list = self._generate_connections()
# Decide on a unique client prefix to use in ! sections
# TODO: ensure uniqueness better, e.g. Redis keys with SETNX
self.client_prefix = "".join(
random.choice(string.ascii_letters) for i in range(8))
# Register scripts
connection = self.connection(None)
self.chansend = connection.register_script(self.lua_chansend)
self.lpopmany = connection.register_script(self.lua_lpopmany)
self.delprefix = connection.register_script(self.lua_delprefix)
# See if we can do encryption if they asked
if symmetric_encryption_keys:
if isinstance(symmetric_encryption_keys, six.string_types):
raise ValueError(
"symmetric_encryption_keys must be a list of possible keys"
)
try:
from cryptography.fernet import MultiFernet
except ImportError:
raise ValueError(
"Cannot run with encryption without 'cryptography' installed."
)
sub_fernets = [
self.make_fernet(key) for key in symmetric_encryption_keys
]
self.crypter = MultiFernet(sub_fernets)
else:
self.crypter = None
def _generate_connections(self):
return [redis.Redis.from_url(host) for host in self.hosts]
### ASGI API ###
extensions = ["groups", "flush", "twisted"]
def send(self, channel, message):
# Typecheck
assert isinstance(message, dict), "message is not a dict"
assert self.valid_channel_name(channel), "Channel name not valid"
# Write out message into expiring key (avoids big items in list)
# TODO: Use extended set, drop support for older redis?
message_key = self.prefix + uuid.uuid4().hex
channel_key = self.prefix + channel
# Pick a connection to the right server - consistent for response
# channels, random for normal channels
if "!" in channel or "?" in channel:
index = self.consistent_hash(channel)
connection = self.connection(index)
else:
connection = self.connection(None)
# Use the Lua function to do the set-and-push
try:
self.chansend(
keys=[message_key, channel_key],
args=[
self.serialize(message), self.expiry,
self.get_capacity(channel)
],
client=connection,
)
except redis.exceptions.ResponseError as e:
# The Lua script handles capacity checking and sends the "full" error back
if e.args[0] == "full":
raise self.ChannelFull
def receive_many(self, channels, block=False):
# List name get
indexes = self._receive_many_list_names(channels)
# Short circuit if no channels
if indexes is None:
return None, None
# Get a message from one of our channels
while True:
# Select a random connection to use
index = random.choice(list(indexes.keys()))
list_names = indexes[index]
# Shuffle list_names to avoid the first ones starving others of workers
random.shuffle(list_names)
# Open a connection
connection = self.connection(index)
# Pop off any waiting message
if block:
result = connection.blpop(list_names,
timeout=self.blpop_timeout)
else:
result = self.lpopmany(keys=list_names, client=connection)
if result:
content = connection.get(result[1])
# If the content key expired, keep going.
if content is None:
continue
# Return the channel it's from and the message
return result[0][len(self.prefix):].decode(
"utf8"), self.deserialize(content)
else:
return None, None
def _receive_many_list_names(self, channels):
"""
Inner logic of receive_many; takes channels, groups by shard, and
returns {connection_index: list_names ...} if a query is needed or
None for a vacuously empty response.
"""
# Short circuit if no channels
if not channels:
return None
# Check channel names are valid
channels = list(channels)
assert all(
self.valid_channel_name(channel)
for channel in channels), "One or more channel names invalid"
# Work out what servers to listen on for the given channels
indexes = {}
random_index = self.random_index()
for channel in channels:
if "!" in channel or "?" in channel:
indexes.setdefault(self.consistent_hash(channel),
[]).append(self.prefix + channel, )
else:
indexes.setdefault(random_index,
[]).append(self.prefix + channel, )
return indexes
def new_channel(self, pattern):
assert isinstance(pattern, six.text_type)
# Keep making channel names till one isn't present.
while True:
random_string = "".join(
random.choice(string.ascii_letters) for i in range(12))
assert pattern.endswith("!") or pattern.endswith("?")
new_name = pattern + random_string
# Get right connection
index = self.consistent_hash(new_name)
connection = self.connection(index)
# Check to see if it's in the connected Redis.
# This fails to stop collisions for sharding where the channel is
# non-single-listener, but that seems very unlikely.
key = self.prefix + new_name
if not connection.exists(key):
return new_name
### ASGI Group extension ###
def group_add(self, group, channel):
"""
Adds the channel to the named group for at least 'expiry'
seconds (expiry defaults to message expiry if not provided).
"""
assert self.valid_group_name(group), "Group name not valid"
assert self.valid_channel_name(channel), "Channel name not valid"
group_key = self._group_key(group)
connection = self.connection(self.consistent_hash(group))
# Add to group sorted set with creation time as timestamp
connection.zadd(group_key, **{channel: time.time()})
# Set both expiration to be group_expiry, since everything in
# it at this point is guaranteed to expire before that
connection.expire(group_key, self.group_expiry)
def group_discard(self, group, channel):
"""
Removes the channel from the named group if it is in the group;
does nothing otherwise (does not error)
"""
assert self.valid_group_name(group), "Group name not valid"
assert self.valid_channel_name(channel), "Channel name not valid"
key = self._group_key(group)
self.connection(self.consistent_hash(group)).zrem(
key,
channel,
)
def group_channels(self, group):
"""
Returns all channels in the group as an iterable.
"""
key = self._group_key(group)
connection = self.connection(self.consistent_hash(group))
# Discard old channels based on group_expiry
connection.zremrangebyscore(key, 0,
int(time.time()) - self.group_expiry)
# Return current lot
return [x.decode("utf8") for x in connection.zrange(
key,
0,
-1,
)]
def send_group(self, group, message):
"""
Sends a message to the entire group.
"""
assert self.valid_group_name(group), "Group name not valid"
# TODO: More efficient implementation (lua script per shard?)
for channel in self.group_channels(group):
try:
self.send(channel, message)
except self.ChannelFull:
pass
def _group_key(self, group):
return ("%s:group:%s" % (self.prefix, group)).encode("utf8")
### Flush extension ###
def flush(self):
"""
Deletes all messages and groups on all shards.
"""
for connection in self._connection_list:
self.delprefix(keys=[],
args=[self.prefix + "*"],
client=connection)
### Twisted extension ###
@defer.inlineCallbacks
def receive_many_twisted(self, channels):
"""
Twisted-native implementation of receive_many.
"""
# List name get
indexes = self._receive_many_list_names(channels)
# Short circuit if no channels
if indexes is None:
defer.returnValue((None, None))
# Get a message from one of our channels
while True:
# Select a random connection to use
index = random.choice(list(indexes.keys()))
list_names = indexes[index]
# Shuffle list_names to avoid the first ones starving others of workers
random.shuffle(list_names)
# Get a sync connection for conn details
sync_connection = self.connection(index)
twisted_connection = yield txredisapi.ConnectionPool(
host=sync_connection.connection_pool.connection_kwargs['host'],
port=sync_connection.connection_pool.connection_kwargs['port'],
dbid=sync_connection.connection_pool.connection_kwargs['db'],
)
try:
# Pop off any waiting message
result = yield twisted_connection.blpop(
list_names, timeout=self.blpop_timeout)
if result:
content = yield twisted_connection.get(result[1])
# If the content key expired, keep going.
if content is None:
continue
# Return the channel it's from and the message
defer.returnValue((result[0][len(self.prefix):],
self.deserialize(content)))
else:
defer.returnValue((None, None))
finally:
yield twisted_connection.disconnect()
### Serialization ###
def serialize(self, message):
"""
Serializes message to a byte string.
"""
value = msgpack.packb(message, use_bin_type=True)
if self.crypter:
value = self.crypter.encrypt(value)
return value
def deserialize(self, message):
"""
Deserializes from a byte string.
"""
if self.crypter:
message = self.crypter.decrypt(message, self.expiry + 10)
return msgpack.unpackb(message, encoding="utf8")
### Redis Lua scripts ###
# Single-command channel send. Returns error if over capacity.
# Keys: message, channel_list
# Args: content, expiry, capacity
lua_chansend = """
if redis.call('llen', KEYS[2]) >= tonumber(ARGV[3]) then
return redis.error_reply("full")
end
redis.call('set', KEYS[1], ARGV[1])
redis.call('expire', KEYS[1], ARGV[2])
redis.call('rpush', KEYS[2], KEYS[1])
redis.call('expire', KEYS[2], ARGV[2] + 1)
"""
lua_lpopmany = """
for keyCount = 1, #KEYS do
local result = redis.call('LPOP', KEYS[keyCount])
if result then
return {KEYS[keyCount], result}
end
end
return {nil, nil}
"""
lua_delprefix = """
local keys = redis.call('keys', ARGV[1])
for i=1,#keys,5000 do
redis.call('del', unpack(keys, i, math.min(i+4999, #keys)))
end
"""
### Internal functions ###
def consistent_hash(self, value):
"""
Maps the value to a node value between 0 and 4095
using MD5, then down to one of the ring nodes.
"""
if isinstance(value, six.text_type):
value = value.encode("utf8")
bigval = binascii.crc32(value) & 0xffffffff
return (bigval // 0x100000) // self.ring_divisor
def random_index(self):
return random.randint(0, len(self.hosts) - 1)
def connection(self, index):
"""
Returns the correct connection for the current thread.
Pass key to use a server based on consistent hashing of the key value;
pass None to use a random server instead.
"""
# If index is explicitly None, pick a random server
if index is None:
index = self.random_index()
# Catch bad indexes
if not 0 <= index < self.ring_size:
raise ValueError("There are only %s hosts - you asked for %s!" %
(self.ring_size, index))
return self._connection_list[index]
def make_fernet(self, key):
"""
Given a single encryption key, returns a Fernet instance using it.
"""
from cryptography.fernet import Fernet
if isinstance(key, six.text_type):
key = key.encode("utf8")
formatted_key = base64.urlsafe_b64encode(hashlib.sha256(key).digest())
return Fernet(formatted_key)
def __str__(self):
return "%s(hosts=%s)" % (self.__class__.__name__, self.hosts)
class RedisChannelLayer(BaseChannelLayer):
"""
Redis channel layer.
It routes all messages into remote Redis server. Support for
sharding among different Redis installations and message
encryption are provided.
"""
blpop_timeout = 5
queue_get_timeout = 10
def __init__(
self,
hosts=None,
prefix="asgi:",
expiry=60,
group_expiry=86400,
capacity=100,
channel_capacity=None,
symmetric_encryption_keys=None,
):
# Store basic information
self.expiry = expiry
self.group_expiry = group_expiry
self.capacity = capacity
self.channel_capacity = self.compile_capacities(channel_capacity or {})
self.prefix = prefix
assert isinstance(self.prefix, str), "Prefix must be unicode"
# Cached redis connection pools and the event loop they are from
self.pools = {}
self.pools_loop = None
# Configure the host objects
self.hosts = self.decode_hosts(hosts)
self.ring_size = len(self.hosts)
# Normal channels choose a host index by cycling through the available hosts
self._receive_index_generator = itertools.cycle(range(len(self.hosts)))
self._send_index_generator = itertools.cycle(range(len(self.hosts)))
# Decide on a unique client prefix to use in ! sections
# TODO: ensure uniqueness better, e.g. Redis keys with SETNX
self.client_prefix = "".join(
random.choice(string.ascii_letters) for i in range(8))
# Set up any encryption objects
self._setup_encryption(symmetric_encryption_keys)
# Number of coroutines trying to receive right now
self.receive_count = 0
# Event loop they are trying to receive on
self.receive_event_loop = None
# Main receive loop running
self.receive_loop_task = None
# Buffered messages by process-local channel name
self.receive_buffer = collections.defaultdict(asyncio.Queue)
def decode_hosts(self, hosts):
"""
Takes the value of the "hosts" argument passed to the class and returns
a list of kwargs to use for the Redis connection constructor.
"""
# If no hosts were provided, return a default value
if not hosts:
return [{"address": ("localhost", 6379)}]
# If they provided just a string, scold them.
if isinstance(hosts, (str, bytes)):
raise ValueError(
"You must pass a list of Redis hosts, even if there is only one."
)
# Decode each hosts entry into a kwargs dict
result = []
for entry in hosts:
if isinstance(entry, dict):
result.append(entry)
else:
result.append({
"address": entry,
})
return result
def _setup_encryption(self, symmetric_encryption_keys):
# See if we can do encryption if they asked
if symmetric_encryption_keys:
if isinstance(symmetric_encryption_keys, (str, bytes)):
raise ValueError(
"symmetric_encryption_keys must be a list of possible keys"
)
try:
from cryptography.fernet import MultiFernet
except ImportError:
raise ValueError(
"Cannot run with encryption without 'cryptography' installed."
)
sub_fernets = [
self.make_fernet(key) for key in symmetric_encryption_keys
]
self.crypter = MultiFernet(sub_fernets)
else:
self.crypter = None
### Channel layer API ###
extensions = ["groups", "flush"]
async def send(self, channel, message):
"""
Send a message onto a (general or specific) channel.
"""
# Typecheck
assert isinstance(message, dict), "message is not a dict"
assert self.valid_channel_name(channel), "Channel name not valid"
# Make sure the message does not contain reserved keys
assert "__asgi_channel__" not in message
# If it's a process-local channel, strip off local part and stick full name in message
channel_non_local_name = channel
if "!" in channel:
message = dict(message.items())
message["__asgi_channel__"] = channel
channel_non_local_name = self.non_local_name(channel)
# Write out message into expiring key (avoids big items in list)
channel_key = self.prefix + channel_non_local_name
# Pick a connection to the right server - consistent for specific
# channels, random for general channels
if "!" in channel:
index = self.consistent_hash(channel)
else:
index = next(self._send_index_generator)
async with self.connection(index) as connection:
# Check the length of the list before send
# This can allow the list to leak slightly over capacity, but that's fine.
if await connection.llen(channel_key) >= self.get_capacity(
channel):
raise ChannelFull()
# Push onto the list then set it to expire in case it's not consumed
await connection.rpush(channel_key, self.serialize(message))
await connection.expire(channel_key, int(self.expiry))
async def receive(self, channel):
"""
Receive the first message that arrives on the channel.
If more than one coroutine waits on the same channel, the first waiter
will be given the message when it arrives.
"""
# Make sure the channel name is valid then get the non-local part
# and thus its index
assert self.valid_channel_name(channel)
if "!" in channel:
real_channel = self.non_local_name(channel)
assert real_channel.endswith(self.client_prefix +
"!"), "Wrong client prefix"
# Enter receiving section
loop = asyncio.get_event_loop()
self.receive_count += 1
try:
if self.receive_count == 1:
# If we're the first coroutine in, make a receive loop!
general_channel = self.non_local_name(channel)
self.receive_loop_task = loop.create_task(
self.receive_loop(general_channel))
self.receive_event_loop = loop
else:
# Otherwise, check our event loop matches
if self.receive_event_loop != loop:
raise RuntimeError(
"Two event loops are trying to receive() on one channel layer at once!"
)
if self.receive_loop_task.done():
# Maybe raise an exception from the task
self.receive_loop_task.result()
# Raise our own exception if that failed
raise RuntimeError("Redis receive loop exited early")
# Wait for our message to appear
while True:
try:
message = await asyncio.wait_for(
self.receive_buffer[channel].get(),
self.queue_get_timeout)
if self.receive_buffer[channel].empty():
del self.receive_buffer[channel]
return message
except asyncio.TimeoutError:
# See if we need to propagate a dead receiver exception
if self.receive_loop_task.done():
self.receive_loop_task.result()
finally:
self.receive_count -= 1
# If we were the last out, stop the receive loop
if self.receive_count == 0:
self.receive_loop_task.cancel()
else:
# Do a plain direct receive
return (await self.receive_single(channel))[1]
async def receive_loop(self, general_channel):
"""
Continuous-receiving loop that makes sure something is fetching results
for the channel passed in.
"""
assert general_channel.endswith(
"!"
), "receive_loop not called on general queue of process-local channel"
while True:
real_channel, message = await self.receive_single(general_channel)
if type(real_channel) is list:
for channel in real_channel:
await self.receive_buffer[channel].put(message)
else:
await self.receive_buffer[real_channel].put(message)
async def receive_single(self, channel):
"""
Receives a single message off of the channel and returns it.
"""
# Check channel name
assert self.valid_channel_name(channel,
receive=True), "Channel name invalid"
# Work out the connection to use
if "!" in channel:
assert channel.endswith("!")
index = self.consistent_hash(channel)
else:
index = next(self._receive_index_generator)
# Get that connection and receive off of it
async with self.connection(index) as connection:
channel_key = self.prefix + channel
content = None
while content is None:
content = await connection.blpop(channel_key,
timeout=self.blpop_timeout)
# Message decode
message = self.deserialize(content[1])
# TODO: message expiry?
# If there is a full channel name stored in the message, unpack it.
if "__asgi_channel__" in message:
channel = message["__asgi_channel__"]
del message["__asgi_channel__"]
return channel, message
async def new_channel(self, prefix="specific"):
"""
Returns a new channel name that can be used by something in our
process as a specific channel.
"""
# TODO: Guarantee uniqueness better?
return "%s.%s!%s" % (
prefix,
self.client_prefix,
"".join(random.choice(string.ascii_letters) for i in range(12)),
)
### Flush extension ###
async def flush(self):
"""
Deletes all messages and groups on all shards.
"""
# Lua deletion script
delete_prefix = """
local keys = redis.call('keys', ARGV[1])
for i=1,#keys,5000 do
redis.call('del', unpack(keys, i, math.min(i+4999, #keys)))
end
"""
# Go through each connection and remove all with prefix
for i in range(self.ring_size):
async with self.connection(i) as connection:
await connection.eval(delete_prefix,
keys=[],
args=[self.prefix + "*"])
### Groups extension ###
async def group_add(self, group, channel):
"""
Adds the channel name to a group.
"""
# Check the inputs
assert self.valid_group_name(group), "Group name not valid"
assert self.valid_channel_name(channel), "Channel name not valid"
# Get a connection to the right shard
group_key = self._group_key(group)
async with self.connection(self.consistent_hash(group)) as connection:
# Add to group sorted set with creation time as timestamp
await connection.zadd(
group_key,
time.time(),
channel,
)
# Set expiration to be group_expiry, since everything in
# it at this point is guaranteed to expire before that
await connection.expire(group_key, self.group_expiry)
async def group_discard(self, group, channel):
"""
Removes the channel from the named group if it is in the group;
does nothing otherwise (does not error)
"""
assert self.valid_group_name(group), "Group name not valid"
assert self.valid_channel_name(channel), "Channel name not valid"
key = self._group_key(group)
async with self.connection(self.consistent_hash(group)) as connection:
await connection.zrem(
key,
channel,
)
async def group_send(self, group, message):
"""
Sends a message to the entire group.
"""
assert self.valid_group_name(group), "Group name not valid"
# Retrieve list of all channel names
key = self._group_key(group)
async with self.connection(self.consistent_hash(group)) as connection:
# Discard old channels based on group_expiry
await connection.zremrangebyscore(key,
min=0,
max=int(time.time()) -
self.group_expiry)
channel_names = [
x.decode("utf8") for x in await connection.zrange(key, 0, -1)
]
connection_to_channel_keys, channel_keys_to_message, channel_keys_to_capacity = \
self._map_channel_keys_to_connection(channel_names, message)
for connection_index, channel_redis_keys in connection_to_channel_keys.items(
):
# Create a LUA script specific for this connection.
# Make sure to use the message specific to this channel, it is
# stored in channel_to_message dict and contains the
# __asgi_channel__ key.
group_send_lua = """
for i=1,#KEYS do
if redis.call('LLEN', KEYS[i]) < tonumber(ARGV[i + #KEYS]) then
redis.call('RPUSH', KEYS[i], ARGV[i])
redis.call('EXPIRE', KEYS[i], %d)
end
end
""" % self.expiry
# We need to filter the messages to keep those related to the connection
args = [
channel_keys_to_message[channel_key]
for channel_key in channel_redis_keys
]
# We need to send the capacity for each channel
args += [
channel_keys_to_capacity[channel_key]
for channel_key in channel_redis_keys
]
# channel_keys does not contain a single redis key more than once
async with self.connection(connection_index) as connection:
await connection.eval(group_send_lua,
keys=channel_redis_keys,
args=args)
def _map_channel_to_connection(self, channel_names, message):
"""
For a list of channel names, bucket each one to a dict keyed by the
connection index
Also for each channel create a message specific to that channel, adding
the __asgi_channel__ key to the message
We also return a mapping from channel names to their corresponding Redis
keys, and a mapping of channels to their capacity
"""
connection_to_channels = collections.defaultdict(list)
channel_to_message = dict()
channel_to_capacity = dict()
channel_to_key = dict()
for channel in channel_names:
channel_non_local_name = channel
if "!" in channel:
message = dict(message.items())
message["__asgi_channel__"] = channel
channel_non_local_name = self.non_local_name(channel)
channel_key = self.prefix + channel_non_local_name
idx = self.consistent_hash(channel_non_local_name)
connection_to_channels[idx].append(channel_key)
channel_to_capacity[channel] = self.get_capacity(channel)
channel_to_message[channel] = self.serialize(message)
# We build a
channel_to_key[channel] = channel_key
return connection_to_channels, channel_to_message, channel_to_capacity, channel_to_key
def _map_channel_keys_to_connection(self, channel_names, message):
"""
For a list of channel names, GET
1. list of their redis keys bucket each one to a dict keyed by the connection index
2. for each unique channel redis key create a serialized message specific to that redis key, by adding
the list of channels mapped to that redis key in __asgi_channel__ key to the message
3. returns a mapping of redis channels keys to their capacity
"""
# Connection dict keyed by index to list of redis keys mapped on that index
connection_to_channel_keys = collections.defaultdict(list)
# Message dict maps redis key to the message that needs to be send on that key
channel_key_to_message = dict()
# Channel key mapped to its capacity
channel_key_to_capacity = dict()
# For each channel
for channel in channel_names:
channel_non_local_name = channel
if "!" in channel:
channel_non_local_name = self.non_local_name(channel)
# Get its redis key
channel_key = self.prefix + channel_non_local_name
# Have we come across the same redis key?
if channel_key not in channel_key_to_message.keys():
# If not, fill the corresponding dicts
message = dict(message.items())
message["__asgi_channel__"] = [channel]
channel_key_to_message[channel_key] = message
channel_key_to_capacity[channel_key] = self.get_capacity(
channel)
idx = self.consistent_hash(channel_non_local_name)
connection_to_channel_keys[idx].append(channel_key)
else:
# Yes, Append the channel in message dict
channel_key_to_message[channel_key]["__asgi_channel__"].append(
channel)
# Now that we know what message needs to be send on a redis key we serialize it
for key in channel_key_to_message.keys():
# Serialize the message stored for each redis key
channel_key_to_message[key] = self.serialize(
channel_key_to_message[key])
return connection_to_channel_keys, channel_key_to_message, channel_key_to_capacity
def _group_key(self, group):
"""
Common function to make the storage key for the group.
"""
return ("%s:group:%s" % (self.prefix, group)).encode("utf8")
### Serialization ###
def serialize(self, message):
"""
Serializes message to a byte string.
"""
value = msgpack.packb(message, use_bin_type=True)
if self.crypter:
value = self.crypter.encrypt(value)
return value
def deserialize(self, message):
"""
Deserializes from a byte string.
"""
if self.crypter:
message = self.crypter.decrypt(message, self.expiry + 10)
return msgpack.unpackb(message, encoding="utf8")
### Internal functions ###
def consistent_hash(self, value):
"""
Maps the value to a node value between 0 and 4095
using CRC, then down to one of the ring nodes.
"""
if isinstance(value, str):
value = value.encode("utf8")
bigval = binascii.crc32(value) & 0xfff
ring_divisor = 4096 / float(self.ring_size)
return int(bigval / ring_divisor)
def make_fernet(self, key):
"""
Given a single encryption key, returns a Fernet instance using it.
"""
from cryptography.fernet import Fernet
if isinstance(key, str):
key = key.encode("utf8")
formatted_key = base64.urlsafe_b64encode(hashlib.sha256(key).digest())
return Fernet(formatted_key)
def __str__(self):
return "%s(hosts=%s)" % (self.__class__.__name__, self.hosts)
### Connection handling ###
def connection(self, index):
"""
Returns the correct connection for the index given.
Lazily instantiates pools.
"""
# Catch bad indexes
if not 0 <= index < self.ring_size:
raise ValueError("There are only %s hosts - you asked for %s!" %
(self.ring_size, index))
# Make a context manager
return self.ConnectionContextManager(self.hosts[index])
class ConnectionContextManager:
"""
Async context manager for connections
"""
def __init__(self, kwargs):
self.kwargs = kwargs
async def __aenter__(self):
self.conn = await aioredis.create_redis(**self.kwargs)
return self.conn
async def __aexit__(self, exc_type, exc, tb):
self.conn.close()
class RedisChannelLayer(BaseChannelLayer):
"""
Redis channel layer.
It routes all messages into remote Redis server. Support for
sharding among different Redis installations and message
encryption are provided.
"""
blpop_timeout = 5
local_poll_interval = 0.01
def __init__(
self,
hosts=None,
prefix="asgi:",
expiry=60,
group_expiry=86400,
capacity=100,
channel_capacity=None,
symmetric_encryption_keys=None,
):
# Store basic information
self.expiry = expiry
self.group_expiry = group_expiry
self.capacity = capacity
self.channel_capacity = self.compile_capacities(channel_capacity or {})
self.prefix = prefix
assert isinstance(self.prefix, str), "Prefix must be unicode"
# Cached redis connection pools and the event loop they are from
self.pools = {}
self.pools_loop = None
# Configure the host objects
self.hosts = self.decode_hosts(hosts)
self.ring_size = len(self.hosts)
# Normal channels choose a host index by cycling through the available hosts
self._receive_index_generator = itertools.cycle(range(len(self.hosts)))
self._send_index_generator = itertools.cycle(range(len(self.hosts)))
# Decide on a unique client prefix to use in ! sections
# TODO: ensure uniqueness better, e.g. Redis keys with SETNX
self.client_prefix = "".join(random.choice(string.ascii_letters) for i in range(8))
# Set up any encryption objects
self._setup_encryption(symmetric_encryption_keys)
# Buffered messages by process-local channel name
self.receive_buffer = {}
# Coroutines currently receiving the process-local channel.
self.receive_tasks = {}
def decode_hosts(self, hosts):
"""
Takes the value of the "hosts" argument passed to the class and returns
a list of kwargs to use for the Redis connection constructor.
"""
# If no hosts were provided, return a default value
if not hosts:
return {"address": ("localhost", 6379)}
# If they provided just a string, scold them.
if isinstance(hosts, (str, bytes)):
raise ValueError("You must pass a list of Redis hosts, even if there is only one.")
# Decode each hosts entry into a kwargs dict
result = []
for entry in hosts:
result.append({
"address": entry,
})
return result
def _setup_encryption(self, symmetric_encryption_keys):
# See if we can do encryption if they asked
if symmetric_encryption_keys:
if isinstance(symmetric_encryption_keys, (str, bytes)):
raise ValueError("symmetric_encryption_keys must be a list of possible keys")
try:
from cryptography.fernet import MultiFernet
except ImportError:
raise ValueError("Cannot run with encryption without 'cryptography' installed.")
sub_fernets = [self.make_fernet(key) for key in symmetric_encryption_keys]
self.crypter = MultiFernet(sub_fernets)
else:
self.crypter = None
### Channel layer API ###
extensions = ["groups", "flush"]
async def send(self, channel, message):
"""
Send a message onto a (general or specific) channel.
"""
# Typecheck
assert isinstance(message, dict), "message is not a dict"
assert self.valid_channel_name(channel), "Channel name not valid"
# Make sure the message does not contain reserved keys
assert "__asgi_channel__" not in message
# If it's a process-local channel, strip off local part and stick full name in message
if "!" in channel:
message = dict(message.items())
message["__asgi_channel__"] = channel
channel = self.non_local_name(channel)
# Write out message into expiring key (avoids big items in list)
channel_key = self.prefix + channel
# Pick a connection to the right server - consistent for specific
# channels, random for general channels
if "!" in channel:
index = self.consistent_hash(channel)
pool = await self.connection(index)
else:
index = next(self._send_index_generator)
pool = await self.connection(index)
with (await pool) as connection:
# Check the length of the list before send
# This can allow the list to leak slightly over capacity, but that's fine.
if await connection.llen(channel_key) >= self.get_capacity(channel):
raise ChannelFull()
# Push onto the list then set it to expire in case it's not consumed
await connection.rpush(channel_key, self.serialize(message))
await connection.expire(channel_key, int(self.expiry))
async def receive(self, channel):
"""
Receive the first message that arrives on the channel.
If more than one coroutine waits on the same channel, the first waiter
will be given the message when it arrives.
"""
# Make sure the channel name is valid then get the non-local part
# and thus its index
assert self.valid_channel_name(channel)
if "!" in channel:
real_channel = self.non_local_name(channel)
assert real_channel.endswith(self.client_prefix + "!"), "Wrong client prefix"
# Make sure a receive task is running
task = self.receive_tasks.get(real_channel, None)
if task is not None and task.done():
task = None
if task is None:
self.receive_tasks[real_channel] = asyncio.ensure_future(
self.receive_loop(real_channel),
)
# Wait on the receive buffer's contents
return await self.receive_buffer_lpop(channel)
else:
# Do a plain direct receive
return (await self.receive_single(channel))[1]
async def receive_loop(self, channel):
"""
Continuous-receiving loop that fetches results into the receive buffer.
"""
assert "!" in channel, "receive_loop called on non-process-local channel"
while True:
# Catch RuntimeErrors from the loop stopping while we release
# a connection. Wish there was a cleaner solution here.
real_channel, message = await self.receive_single(channel)
self.receive_buffer.setdefault(real_channel, []).append(message)
async def receive_single(self, channel):
"""
Receives a single message off of the channel and returns it.
"""
# Check channel name
assert self.valid_channel_name(channel, receive=True), "Channel name invalid"
# Work out the connection to use
if "!" in channel:
assert channel.endswith("!")
index = self.consistent_hash(channel)
else:
index = next(self._receive_index_generator)
# Get that connection and receive off of it
pool = await self.connection(index)
with (await pool) as connection:
channel_key = self.prefix + channel
content = None
while content is None:
content = await connection.blpop(channel_key, timeout=self.blpop_timeout)
# Message decode
message = self.deserialize(content[1])
# TODO: message expiry?
# If there is a full channel name stored in the message, unpack it.
if "__asgi_channel__" in message:
channel = message["__asgi_channel__"]
del message["__asgi_channel__"]
return channel, message
async def receive_buffer_lpop(self, channel):
"""
Atomic, async method that returns the left-hand item in a receive buffer.
"""
# TODO: Use locks or something, not a poll
while True:
if self.receive_buffer.get(channel, None):
message = self.receive_buffer[channel][0]
if len(self.receive_buffer[channel]) == 1:
del self.receive_buffer[channel]
else:
self.receive_buffer[channel] = self.receive_buffer[channel][1:]
return message
else:
# See if we need to propagate a dead receiver exception
real_channel = self.non_local_name(channel)
task = self.receive_tasks.get(real_channel, None)
if task is not None and task.done():
task.result()
# Sleep poll
await asyncio.sleep(self.local_poll_interval)
async def new_channel(self, prefix="specific."):
"""
Returns a new channel name that can be used by something in our
process as a specific channel.
"""
# TODO: Guarantee uniqueness better?
return "%s.%s!%s" % (
prefix,
self.client_prefix,
"".join(random.choice(string.ascii_letters) for i in range(12)),
)
### Flush extension ###
async def flush(self):
"""
Deletes all messages and groups on all shards.
"""
# Lua deletion script
delete_prefix = """
local keys = redis.call('keys', ARGV[1])
for i=1,#keys,5000 do
redis.call('del', unpack(keys, i, math.min(i+4999, #keys)))
end
"""
# Go through each connection and remove all with prefix
for i in range(self.ring_size):
connection = await self.connection(i)
await connection.eval(
delete_prefix,
keys=[],
args=[self.prefix + "*"]
)
async def close(self):
# Stop all reader tasks
for task in self.receive_tasks.values():
task.cancel()
asyncio.wait(self.receive_tasks.values())
self.receive_tasks = {}
# Close up all pools
for pool in self.pools.values():
pool.close()
await pool.wait_closed()
### Groups extension ###
async def group_add(self, group, channel):
"""
Adds the channel name to a group.
"""
# Check the inputs
assert self.valid_group_name(group), "Group name not valid"
assert self.valid_channel_name(channel), "Channel name not valid"
# Get a connection to the right shard
group_key = self._group_key(group)
pool = await self.connection(self.consistent_hash(group))
with (await pool) as connection:
# Add to group sorted set with creation time as timestamp
await connection.zadd(
group_key,
time.time(),
channel,
)
# Set expiration to be group_expiry, since everything in
# it at this point is guaranteed to expire before that
await connection.expire(group_key, self.group_expiry)
async def group_discard(self, group, channel):
"""
Removes the channel from the named group if it is in the group;
does nothing otherwise (does not error)
"""
assert self.valid_group_name(group), "Group name not valid"
assert self.valid_channel_name(channel), "Channel name not valid"
key = self._group_key(group)
pool = await self.connection(self.consistent_hash(group))
await pool.zrem(
key,
channel,
)
async def group_send(self, group, message):
"""
Sends a message to the entire group.
"""
assert self.valid_group_name(group), "Group name not valid"
# Retrieve list of all channel names
key = self._group_key(group)
pool = await self.connection(self.consistent_hash(group))
with (await pool) as connection:
# Discard old channels based on group_expiry
await connection.zremrangebyscore(key, min=0, max=int(time.time()) - self.group_expiry)
# Return current lot
channel_names = [
x.decode("utf8") for x in
await connection.zrange(key, 0, -1)
]
# TODO: More efficient implementation (lua script per shard?)
for channel in channel_names:
try:
await self.send(channel, message)
except ChannelFull:
pass
def _group_key(self, group):
"""
Common function to make the storage key for the group.
"""
return ("%s:group:%s" % (self.prefix, group)).encode("utf8")
### Serialization ###
def serialize(self, message):
"""
Serializes message to a byte string.
"""
value = msgpack.packb(message, use_bin_type=True)
if self.crypter:
value = self.crypter.encrypt(value)
return value
def deserialize(self, message):
"""
Deserializes from a byte string.
"""
if self.crypter:
message = self.crypter.decrypt(message, self.expiry + 10)
return msgpack.unpackb(message, encoding="utf8")
### Internal functions ###
def consistent_hash(self, value):
"""
Maps the value to a node value between 0 and 4095
using CRC, then down to one of the ring nodes.
"""
if isinstance(value, str):
value = value.encode("utf8")
bigval = binascii.crc32(value) & 0xfff
ring_divisor = 4096 / float(self.ring_size)
return int(bigval / ring_divisor)
async def connection(self, index):
"""
Returns the correct connection for the index given.
Lazily instantiates pools.
"""
# Catch bad indexes
if not 0 <= index < self.ring_size:
raise ValueError("There are only %s hosts - you asked for %s!" % (self.ring_size, index))
# Check to see if the stored pools are for the right event loop
# TODO: Maybe cache from multiple event loops to avoid AsyncToSync wiping
# out the main thread's cache (but we'd need to cap the number of entries
# with an LRU strategy or something)
if self.pools_loop != asyncio.get_event_loop():
self.pools = {}
self.pools_loop = asyncio.get_event_loop()
# Make the new pool if it does not exist
if index not in self.pools:
self.pools[index] = await aioredis.create_redis_pool(**self.hosts[index])
return self.pools[index]
def make_fernet(self, key):
"""
Given a single encryption key, returns a Fernet instance using it.
"""
from cryptography.fernet import Fernet
if isinstance(key, str):
key = key.encode("utf8")
formatted_key = base64.urlsafe_b64encode(hashlib.sha256(key).digest())
return Fernet(formatted_key)
def __str__(self):
return "%s(hosts=%s)" % (self.__class__.__name__, self.hosts)
print("Timestamp of the encrypted string: %s" % timestamp)
decrypted_string = key_object.decrypt(encrypted_string)
print("Decrypted String: %s" % decrypted_string)
# Rotation
third_key = Fernet(Fernet.generate_key())
key_object_2 = MultiFernet([third_key, first_key, second_key])
rotated = key_object_2.rotate(encrypted_string)
decrypted_string = key_object_2.decrypt(rotated)
print("Decrypted String: %s" % decrypted_string)
# Fernet Encryption using Passwords
import base64
import os
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
class SecretStore:
def __init__(self, *master_keys, encrypted_store: dict = None):
if not len(master_keys):
raise ValueError('at least one master key must be passed')
self.crypt = MultiFernet([Fernet(key) for key in master_keys])
if not encrypted_store:
self.encrypted_store = dict()
else:
self.encrypted_store = encrypted_store
@staticmethod
def generate_master_key():
return Fernet.generate_key()
@staticmethod
def add_master_key(key_yaml_path):
master_key = SecretStore.generate_master_key()
try:
master_keys = SecretStore._load_keys(key_yaml_path)
except OSError:
master_keys = []
master_keys = [master_key] + master_keys
SecretStore._save_as_yaml(key_yaml_path, 'keys', master_keys)
return master_keys
@staticmethod
def _load_keys(key_yaml_path):
with open(key_yaml_path, 'r') as key_file:
master_keys = yaml.load(key_file)['keys']
return master_keys
@classmethod
def load_from_yaml(cls,
key_yaml_path,
store_yaml_path=None,
encrypted=True):
master_keys = SecretStore._load_keys(key_yaml_path)
secret_store = cls(*master_keys)
if store_yaml_path:
secret_store.load_as_yaml(store_yaml_path, encrypted=encrypted)
return secret_store
def encrypt_copy(self, plain_store, *path):
for key in plain_store:
value = plain_store[key]
if isinstance(value, bytes) or isinstance(value, str):
self.set_secret(value, *path, key)
else:
self.encrypt_copy(value, *(list(path) + [key]))
def set_secret(self, secret, *path):
if not len(path):
raise ValueError('path to secret must not be empty')
if not (isinstance(secret, bytes) or isinstance(secret, str)):
raise ValueError(
'secret must be bytes or str, but {0} is passed'.format(
type(secret)))
if isinstance(secret, str):
secret = secret.encode('utf-8')
encrypted_secret = self.crypt.encrypt(secret)
store = self.encrypted_store
for key in path[:-1]:
store = store.setdefault(key, dict())
store[path[-1]] = encrypted_secret
def get_secret(self, *path):
encrypted_secret = self.get_encrypted_secret(*path)
return self.crypt.decrypt(encrypted_secret)
def delete_secret(self, *path):
if not len(path):
raise ValueError('path to secret must not be empty')
store = self.encrypted_store
for key in path[:-1]:
store = store[key]
del store[path[-1]]
def get_encrypted_secret(self, *path):
if not len(path):
raise ValueError('path to secret must not be empty')
store = self.encrypted_store
for key in path[:-1]:
store = store[key]
encrypted_secret = store[path[-1]]
return encrypted_secret
def load_as_yaml(self, yaml_path, encrypted=True):
with open(yaml_path, 'r') as secret_file:
secret_storage = yaml.load(secret_file)
if encrypted:
self.encrypted_store = secret_storage['encrypted_store']
else:
self.encrypt_copy(secret_storage['encrypted_store'])
def save_as_yaml(self, yaml_path):
SecretStore._save_as_yaml(yaml_path, 'encrypted_store',
self.encrypted_store)
def print_as_yaml(self):
print(yaml.dump(self.encrypted_store, default_flow_style=False))
@staticmethod
def _wrap_payload(payload_key, payload):
now = datetime.now()
timestamp = now.replace(tzinfo=timezone.utc).timestamp()
wrapper = {
'meta': {
'method': 'fernet',
'timestamp': timestamp,
'timezone': 'utc'
},
payload_key: payload
}
return wrapper
@staticmethod
def _save_as_yaml(yaml_path, payload_key, payload):
content = SecretStore._wrap_payload(payload_key, payload)
with open(yaml_path, 'w') as yaml_file:
yaml.dump(content, yaml_file, default_flow_style=False)
class SecretStore:
def __init__(self, *master_keys, encrypted_store: dict = None):
if not len(master_keys):
raise ValueError('at least one master key must be passed')
self.crypt = MultiFernet([Fernet(key) for key in master_keys])
if not encrypted_store:
self.encrypted_store = dict()
else:
self.encrypted_store = encrypted_store
@staticmethod
def generate_master_key():
return Fernet.generate_key()
@staticmethod
def add_master_key(key_yaml_path):
master_key = SecretStore.generate_master_key()
try:
master_keys = SecretStore._load_keys(key_yaml_path)
except OSError:
master_keys = []
master_keys = [master_key] + master_keys
SecretStore._save_as_yaml(key_yaml_path, 'keys', master_keys)
return master_keys
@staticmethod
def _load_keys(key_yaml_path):
with open(key_yaml_path, 'r') as key_file:
master_keys = yaml.load(key_file)['keys']
return master_keys
@classmethod
def load_from_yaml(cls, key_yaml_path, store_yaml_path=None, encrypted=True):
master_keys = SecretStore._load_keys(key_yaml_path)
secret_store = cls(*master_keys)
if store_yaml_path:
secret_store.load_as_yaml(store_yaml_path, encrypted=encrypted)
return secret_store
def encrypt_copy(self, plain_store, *path):
for key in plain_store:
value = plain_store[key]
if isinstance(value, bytes) or isinstance(value, str):
self.set_secret(value, *path, key)
else:
self.encrypt_copy(value, *(list(path) + [key]))
def set_secret(self, secret, *path):
if not len(path):
raise ValueError('path to secret must not be empty')
if not (isinstance(secret, bytes) or isinstance(secret, str)):
raise ValueError(
'secret must be bytes or str, but {0} is passed'.format(
type(secret)))
if isinstance(secret, str):
secret = secret.encode('utf-8')
encrypted_secret = self.crypt.encrypt(secret)
store = self.encrypted_store
for key in path[:-1]:
store = store.setdefault(key, dict())
store[path[-1]] = encrypted_secret
def get_secret(self, *path):
encrypted_secret = self.get_encrypted_secret(*path)
return self.crypt.decrypt(encrypted_secret)
def delete_secret(self, *path):
if not len(path):
raise ValueError('path to secret must not be empty')
store = self.encrypted_store
for key in path[:-1]:
store = store[key]
del store[path[-1]]
def get_encrypted_secret(self, *path):
if not len(path):
raise ValueError('path to secret must not be empty')
store = self.encrypted_store
for key in path[:-1]:
store = store[key]
encrypted_secret = store[path[-1]]
return encrypted_secret
def load_as_yaml(self, yaml_path, encrypted=True):
with open(yaml_path, 'r') as secret_file:
secret_storage = yaml.load(secret_file)
if encrypted:
self.encrypted_store = secret_storage['encrypted_store']
else:
self.encrypt_copy(secret_storage['encrypted_store'])
def save_as_yaml(self, yaml_path):
SecretStore._save_as_yaml(yaml_path, 'encrypted_store', self.encrypted_store)
def print_as_yaml(self):
print(yaml.dump(self.encrypted_store, default_flow_style=False))
@staticmethod
def _wrap_payload(payload_key, payload):
now = datetime.now()
timestamp = now.replace(tzinfo=timezone.utc).timestamp()
wrapper = {
'meta': {
'method': 'fernet',
'timestamp': timestamp,
'timezone': 'utc'
},
payload_key: payload
}
return wrapper
@staticmethod
def _save_as_yaml(yaml_path, payload_key, payload):
content = SecretStore._wrap_payload(payload_key, payload)
with open(yaml_path, 'w') as yaml_file:
yaml.dump(content, yaml_file, default_flow_style=False)
# The sample code is extracted from the book Python Cryptography # The book can be downloaded from https://leanpub.com/cryptop # Online Crypto Playgroud https://8gwifi.org # Author Anish Nath from cryptography.fernet import Fernet, MultiFernet key1 = Fernet(Fernet.generate_key()) key2 = Fernet(Fernet.generate_key()) key3 = Fernet(Fernet.generate_key()) plaintext = "Hello 8gwifi.org" f = MultiFernet([key1, key2, key3]) token = f.encrypt(plaintext) d = f.decrypt(token) assert d, plaintext
return None
event_session = dataSession(urx)
pub_key_store_en = event_session.query(K).filter(K.active==True).with_entities(K.key_id, K.pub_key).all()
event_session.close()
event_session = dataSession(urx)
pvt_key_store_en = event_session.query(K).filter(K.deprecated==False).with_entities(K.key_id, K.pvt_key).all()
event_session = dataSession(urx)
unlock_set = [Fernet(app_key.encode()), Fernet(global_key.encode())]
mft = MFT(unlock_set)
for _ in pvt_key_store_en:
_tup_ = {'key_id': _[0]}
_pvt_ = serialization.load_pem_private_key(
_[1],
password=mft.decrypt(master_key.encode()),
backend=default_backend()
)
_pem_ =_pvt_.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.PKCS8,
encryption_algorithm=serialization.NoEncryption()
).decode()
_tup_['pvt_key'] = _pem_
pvt_key_store.append(_tup_)
pvt_key_store_en.clear()
for _ in pub_key_store_en:
_tup_ = {
'key_id' : _[0],
class AutopushSettings(object):
"""Main Autopush Settings Object"""
debug = attrib(default=False) # type: bool
fernet = attrib(init=False) # type: MultiFernet
_crypto_key = attrib(convert=_init_crypto_key,
default=None) # type: List[str]
bear_hash_key = attrib(default=Factory(list)) # type: List[str]
hostname = attrib(default=None) # type: Optional[str]
port = attrib(default=None) # type: Optional[int]
_resolve_hostname = attrib(default=False) # type: bool
router_scheme = attrib(default=None) # type: Optional[str]
router_hostname = attrib(default=None) # type: Optional[str]
router_port = attrib(default=None) # type: Optional[int]
endpoint_scheme = attrib(default=None) # type: Optional[str]
endpoint_hostname = attrib(default=None) # type: Optional[str]
endpoint_port = attrib(default=None) # type: Optional[int]
proxy_protocol_port = attrib(default=None) # type: Optional[int]
memusage_port = attrib(default=None) # type: Optional[int]
statsd_host = attrib(default="localhost") # type: str
statsd_port = attrib(default=8125) # type: int
datadog_api_key = attrib(default=None) # type: Optional[str]
datadog_app_key = attrib(default=None) # type: Optional[str]
datadog_flush_interval = attrib(default=None) # type: Optional[int]
router_tablename = attrib(default="router") # type: str
router_read_throughput = attrib(default=5) # type: int
router_write_throughput = attrib(default=5) # type: int
storage_tablename = attrib(default="storage") # type: str
storage_read_throughput = attrib(default=5) # type: int
storage_write_throughput = attrib(default=5) # type: int
message_tablename = attrib(default="message") # type: str
message_read_throughput = attrib(default=5) # type: int
message_write_throughput = attrib(default=5) # type: int
preflight_uaid = attrib(
default="deadbeef00000000deadbeef00000000") # type: str
ssl_key = attrib(default=None) # type: Optional[str]
ssl_cert = attrib(default=None) # type: Optional[str]
ssl_dh_param = attrib(default=None) # type: Optional[str]
router_ssl_key = attrib(default=None) # type: Optional[str]
router_ssl_cert = attrib(default=None) # type: Optional[str]
client_certs = attrib(default=None) # type: Optional[Dict[str, str]]
router_url = attrib(init=False) # type: str
endpoint_url = attrib(init=False) # type: str
ws_url = attrib(init=False) # type: str
router_conf = attrib(default=Factory(dict)) # type: JSONDict
# twisted Agent's connectTimeout
connect_timeout = attrib(default=0.5) # type: float
max_data = attrib(default=4096) # type: int
env = attrib(default='development') # type: str
ami_id = attrib(default=None) # type: Optional[str]
cors = attrib(default=False) # type: bool
hello_timeout = attrib(default=0) # type: int
# Force timeout in idle seconds
wake_timeout = attrib(default=0) # type: int
msg_limit = attrib(default=100) # type: int
auto_ping_interval = attrib(default=None) # type: Optional[int]
auto_ping_timeout = attrib(default=None) # type: Optional[int]
max_connections = attrib(default=None) # type: Optional[int]
close_handshake_timeout = attrib(default=None) # type: Optional[int]
# Generate messages per legacy rules, only used for testing to
# generate legacy data.
_notification_legacy = attrib(default=False) # type: bool
def __attrs_post_init__(self):
"""Initialize the Settings object"""
# Setup hosts/ports/urls
if not self.hostname:
self.hostname = socket.gethostname()
if self._resolve_hostname:
self.hostname = resolve_ip(self.hostname)
if not self.endpoint_hostname:
self.endpoint_hostname = self.hostname
if not self.router_hostname:
self.router_hostname = self.hostname
self.router_url = canonical_url(self.router_scheme or 'http',
self.router_hostname, self.router_port)
self.endpoint_url = canonical_url(self.endpoint_scheme or 'http',
self.endpoint_hostname,
self.endpoint_port)
# not accurate under autoendpoint (like router_url)
self.ws_url = "{}://{}:{}/".format('wss' if self.ssl_key else 'ws',
self.hostname, self.port)
self.fernet = MultiFernet([Fernet(key) for key in self._crypto_key])
@property
def enable_tls_auth(self):
"""Whether TLS authentication w/ client certs is enabled"""
return self.client_certs is not None
@classmethod
def from_argparse(cls, ns, **kwargs):
# type: (Namespace, **Any) -> AutopushSettings
"""Create an instance from argparse/additional kwargs"""
router_conf = {}
if ns.key_hash:
db.key_hash = ns.key_hash
# Some routers require a websocket to timeout on idle
# (e.g. UDP)
if ns.wake_pem is not None and ns.wake_timeout != 0:
router_conf["simplepush"] = {
"idle": ns.wake_timeout,
"server": ns.wake_server,
"cert": ns.wake_pem
}
if ns.apns_creds:
# if you have the critical elements for each external
# router, create it
try:
router_conf["apns"] = json.loads(ns.apns_creds)
except (ValueError, TypeError):
raise InvalidSettings(
"Invalid JSON specified for APNS config options")
if ns.gcm_enabled:
# Create a common gcmclient
try:
sender_ids = json.loads(ns.senderid_list)
except (ValueError, TypeError):
raise InvalidSettings(
"Invalid JSON specified for senderid_list")
try:
# This is an init check to verify that things are
# configured correctly. Otherwise errors may creep in
# later that go unaccounted.
sender_ids[sender_ids.keys()[0]]
except (IndexError, TypeError):
raise InvalidSettings("No GCM SenderIDs specified or found.")
router_conf["gcm"] = {
"ttl": ns.gcm_ttl,
"dryrun": ns.gcm_dryrun,
"max_data": ns.max_data,
"collapsekey": ns.gcm_collapsekey,
"senderIDs": sender_ids
}
client_certs = None
# endpoint only
if getattr(ns, 'client_certs', None):
try:
client_certs_arg = json.loads(ns.client_certs)
except (ValueError, TypeError):
raise InvalidSettings(
"Invalid JSON specified for client_certs")
if client_certs_arg:
if not ns.ssl_key:
raise InvalidSettings("client_certs specified without SSL "
"enabled (no ssl_key specified)")
client_certs = {}
for name, sigs in client_certs_arg.iteritems():
if not isinstance(sigs, list):
raise InvalidSettings(
"Invalid JSON specified for client_certs")
for sig in sigs:
sig = sig.upper()
if (not name or not CLIENT_SHA256_RE.match(sig)
or sig in client_certs):
raise InvalidSettings(
"Invalid client_certs argument")
client_certs[sig] = name
if ns.fcm_enabled:
# Create a common gcmclient
if not ns.fcm_auth:
raise InvalidSettings("No Authorization Key found for FCM")
if not ns.fcm_senderid:
raise InvalidSettings("No SenderID found for FCM")
router_conf["fcm"] = {
"ttl": ns.fcm_ttl,
"dryrun": ns.fcm_dryrun,
"max_data": ns.max_data,
"collapsekey": ns.fcm_collapsekey,
"auth": ns.fcm_auth,
"senderid": ns.fcm_senderid
}
ami_id = None
# Not a fan of double negatives, but this makes more
# understandable args
if not ns.no_aws:
ami_id = get_amid()
return cls(crypto_key=ns.crypto_key,
datadog_api_key=ns.datadog_api_key,
datadog_app_key=ns.datadog_app_key,
datadog_flush_interval=ns.datadog_flush_interval,
hostname=ns.hostname,
statsd_host=ns.statsd_host,
statsd_port=ns.statsd_port,
router_conf=router_conf,
router_tablename=ns.router_tablename,
storage_tablename=ns.storage_tablename,
storage_read_throughput=ns.storage_read_throughput,
storage_write_throughput=ns.storage_write_throughput,
message_tablename=ns.message_tablename,
message_read_throughput=ns.message_read_throughput,
message_write_throughput=ns.message_write_throughput,
router_read_throughput=ns.router_read_throughput,
router_write_throughput=ns.router_write_throughput,
resolve_hostname=ns.resolve_hostname,
wake_timeout=ns.wake_timeout,
ami_id=ami_id,
client_certs=client_certs,
msg_limit=ns.msg_limit,
connect_timeout=ns.connection_timeout,
memusage_port=ns.memusage_port,
ssl_key=ns.ssl_key,
ssl_cert=ns.ssl_cert,
ssl_dh_param=ns.ssl_dh_param,
**kwargs)
def make_simplepush_endpoint(self, uaid, chid):
"""Create a simplepush endpoint"""
root = self.endpoint_url + "/spush/"
base = (uaid.replace('-', '').decode("hex") +
chid.replace('-', '').decode("hex"))
return root + 'v1/' + self.fernet.encrypt(base).strip('=')
def make_endpoint(self, uaid, chid, key=None):
"""Create an v1 or v2 WebPush endpoint from the identifiers.
Both endpoints use bytes instead of hex to reduce ID length.
v1 is the uaid + chid
v2 is the uaid + chid + sha256(key).bytes
:param uaid: User Agent Identifier
:param chid: Channel or Subscription ID
:param key: Optional Base64 URL-encoded application server key
:returns: Push endpoint
"""
root = self.endpoint_url + '/wpush/'
base = (uaid.replace('-', '').decode("hex") +
chid.replace('-', '').decode("hex"))
if key is None:
return root + 'v1/' + self.fernet.encrypt(base).strip('=')
raw_key = base64url_decode(key.encode('utf8'))
ep = self.fernet.encrypt(base + sha256(raw_key).digest()).strip('=')
return root + 'v2/' + ep
def parse_endpoint(self,
metrics,
token,
version="v1",
ckey_header=None,
auth_header=None):
"""Parse an endpoint into component elements of UAID, CHID and optional
key hash if v2
:param token: The obscured subscription data.
:param version: This is the API version of the token.
:param ckey_header: the Crypto-Key header bearing the public key
(from Crypto-Key: p256ecdsa=)
:param auth_header: The Authorization header bearing the VAPID info
:raises ValueError: In the case of a malformed endpoint.
:returns: a dict containing (uaid=UAID, chid=CHID, public_key=KEY)
"""
token = self.fernet.decrypt(repad(token).encode('utf8'))
public_key = None
if ckey_header:
try:
crypto_key = CryptoKey(ckey_header)
except CryptoKeyException:
raise InvalidTokenException("Invalid key data")
public_key = crypto_key.get_label('p256ecdsa')
if auth_header:
vapid_auth = parse_auth_header(auth_header)
if not vapid_auth:
raise VapidAuthException("Invalid Auth token")
metrics.increment("notification.auth",
tags="vapid:{version},scheme:{scheme}".format(
**vapid_auth).split(","))
# pull the public key from the VAPID auth header if needed
try:
if vapid_auth['version'] != 1:
public_key = vapid_auth['k']
except KeyError:
raise VapidAuthException("Missing Public Key")
if version == 'v1' and len(token) != 32:
raise InvalidTokenException("Corrupted push token")
if version == 'v2':
if not auth_header:
raise VapidAuthException("Missing Authorization Header")
if len(token) != 64:
raise InvalidTokenException("Corrupted push token")
if not public_key:
raise VapidAuthException("Invalid key data")
try:
decoded_key = base64url_decode(public_key)
except TypeError:
raise VapidAuthException("Invalid key data")
if not constant_time.bytes_eq(
sha256(decoded_key).digest(), token[32:]):
raise VapidAuthException("Key mismatch")
return dict(uaid=token[:16].encode('hex'),
chid=token[16:32].encode('hex'),
version=version,
public_key=public_key)
class Cryptograph:
"""Symmetric encryption and decryption for the storage of sensitive data.
We currently rely on Fernet, which was the algorithm adopted by Gratipay:
https://github.com/gratipay/gratipay.com/pull/3998#issuecomment-216227070
For encryption Fernet uses the AES cipher in CBC mode with PKCS7 padding and
a 128 bits key. For authentication it uses HMAC-SHA256 with another 128 bits
key.
Fernet messages contain the timestamp at which they were generated *in plain
text*. This isn't a problem for us since we want to store the time at which
the data was encrypted in order to facilitate key rotation.
We use CBOR (Concise Binary Object Representation) to serialize objects
before encryption. Compared to JSON, CBOR is faster to parse and serialize,
more compact, and extensible (it can represent any data type using "tags").
More info on CBOR: http://cbor.io/ https://tools.ietf.org/html/rfc7049
"""
KEY_ROTATION_DELAY = timedelta(weeks=1)
def __init__(self):
if website.env.aws_secret_access_key:
sm = self.secrets_manager = boto3.client('secretsmanager',
region_name='eu-west-1')
secret = sm.get_secret_value(SecretId='Fernet')
rotation_start = secret['CreatedDate'].date()
keys = secret['SecretString'].split()
else:
self.secrets_manager = None
parts = os.environ['SECRET_FERNET_KEYS'].split()
rotation_start = date(*map(int, parts[0].split('-')))
keys = parts[1:]
self.fernet_rotation_start = rotation_start
self.fernet_keys = [k.encode('ascii') for k in keys]
self.fernet = MultiFernet([Fernet(k) for k in self.fernet_keys])
def encrypt_dict(self, dic, allow_single_key=False):
"""Serialize and encrypt a dictionary for storage in the database.
Encrypting partially predictable data may help an attacker break the
encryption key, so to make our data less predictable we randomize the
order of the dict's items before serializing it.
For this to be effective the CBOR serializer must not sort the items
again in an attempt to produce Canonical CBOR, so we explicitly pass
`canonical=False` to the `cbor.dumps` function.
In addition, the dict must not contain only one key if that key is
predictable, so a `CryptoWarning` is emitted when `dic` only contains
one key, unless `allow_single_key` is set to `True`.
"""
dic = self.randomize_dict(dic, allow_single_key=allow_single_key)
serialized = cbor.dumps(dic, canonical=False)
encrypted = self.fernet.encrypt(serialized)
return Encrypted(('fernet', encrypted, utcnow()))
def decrypt(self, scheme, payload):
"""Decrypt and reconstruct an object stored in the database.
"""
if scheme == 'fernet':
decrypted = self.fernet.decrypt(payload)
else:
raise ValueError('unknown encryption scheme %r' % scheme)
return cbor.loads(decrypted)
@staticmethod
def randomize_dict(dic, allow_single_key=False):
"""Randomize the order of a dictionary's items.
Emits a `CryptoWarning` if `dic` only contains one key, unless
`allow_single_key` is set to `True`.
"""
if not isinstance(dic, dict):
raise TypeError("expected a dict, got %s" % type(dic))
# Compute the number of random bytes needed based on the size of the dict
n = len(dic)
if n < 2:
# Can't randomize the order if the dict contains less than 2 items
if n == 1 and not allow_single_key:
warnings.warn("dict only contains one key", CryptoWarning)
return dic
n = int(log(n, 2) // 8) + 2
# Return a new ordered dict sorted randomly
return OrderedDict(t[1] for t in sorted((urandom(n), item)
for item in dic.items()))
def rotate_key(self):
"""Generate a new key and send it to the secrets manager.
"""
keys = b' '.join([Fernet.generate_key()] + self.fernet_keys).decode()
if self.secrets_manager:
self.secrets_manager.update_secret(SecretId='Fernet',
SecretString=keys)
else:
keys = utcnow().date().isoformat() + ' ' + keys
print("No secrets manager, updating the key storage is up to you.")
return keys
def rotate_message(self, msg, force=False):
"""Re-encrypt a single message using the current primary key.
The original timestamp included in the message is always preserved.
Moreover the entire message is returned unchanged if it was already
encrypted from the latest key and `force` is `False` (the default).
`InvalidToken` is raised if decryption fails.
"""
timestamp, data = Fernet._get_unverified_token_data(msg)
for i, fernet in enumerate(self.fernet._fernets):
try:
p = fernet._decrypt_data(data, timestamp, None, None)
except InvalidToken:
continue
if i == 0 and not force:
# This message was encrypted using the latest key, return it
return msg
break
else:
raise InvalidToken
iv = os.urandom(16)
return self.fernet._fernets[0]._encrypt_from_parts(p, timestamp, iv)
def rotate_stored_data(self, wait=True):
"""Re-encrypt all the sensitive information stored in our database.
This function is a special kind of "cron job" that returns one of two
constants from the `liberapay.cron` module: `CRON_ENCORE`, indicating
that the function needs to be run again to continue its work, or
`CRON_STOP`, indicating that all the ciphertexts are up-to-date (or that
it isn't time to rotate yet).
Rows are processed in batches of 50. Timestamps are used to keep track of
progress and to avoid overwriting new data with re-encrypted old data.
The update only starts one week after the new key was generated, unless
`wait` is set to `False`. This delay is to "ensure" that the previous
key is no longer being used to encrypt new data.
"""
update_start = self.fernet_rotation_start + self.KEY_ROTATION_DELAY
if wait:
if utcnow().date() < update_start:
return CRON_STOP
with website.db.get_cursor() as cursor:
batch = cursor.all(
"""
SELECT id, info
FROM identities
WHERE (info).ts <= %s
ORDER BY (info).ts ASC
LIMIT 50
""", (update_start, ))
if not batch:
return CRON_STOP
sql = """
UPDATE identities
SET info = ('fernet', %s, current_timestamp)::encrypted
WHERE id = %s
AND (info).ts = %s;
"""
args_list = [(self.rotate_message(r.info.payload), r.id, r.info.ts)
for r in batch]
execute_batch(cursor, sql, args_list)
return CRON_ENCORE
#Ele pega uma list de instâncias de Fernet from cryptography.fernet import Fernet, MultiFernet key1 = Fernet(Fernet.generate_key()) key2 = Fernet(Fernet.generate_key()) f = MultiFernet([key1, key2]) token = f.encrypt(b'Secret message!') print(token) #O multiFernet tenta descriptografar com uma chave de cada vez #Uma exceção é lançada caso a chave correta não seja encontrada #A rotação de tokens facilita a substituição de chaves antigas print(f.decrypt(token)) ''' MultiFernet é uma prática recomendada e uma maneira de higiene criptográfica projetada para limitar os danos em caso de um evento não detectado e aumentar a dificuldade dos ataques ''' #Criando uma nova chave key3 = Fernet(Fernet.generate_key()) #lista de chaves f2 = MultiFernet([key3, key1, key2]) #Gira um token criptografando-o #Se um token foi rotacionado com sucesso, o token rotacionado será retornado.
# 加密信息
token = F.encrypt(info.encode())
print("token: ", token)
# 解密信息
de_info = F.decrypt(token)
print(de_info.decode())
# MultiFernet
key1 = Fernet.generate_key()
print("key1: ", key1)
F1 = Fernet(key1)
key2 = Fernet.generate_key()
print("key2: ", key2)
F2 = Fernet(key2)
# MultiFernet performs all encryption options using the first key in the list provided.
# MultiFernet attempts to decrypt tokens with each key in turn.
FM = MultiFernet([F1, F2])
tokenFM = FM.encrypt(info.encode())
print("tokenFM: ", tokenFM)
de_info = FM.decrypt(tokenFM)
# MultiFernet rotate, rotate a token by decrypt and re-encrypting it under the MultiFernet instance’s primary key.
print("FM.decrypt: ", de_info.decode())
key3 = Fernet.generate_key()
print("key3: ", key3)
F3 = Fernet(key3)
FM2 = MultiFernet([F3, F1, F2])
rotated = FM2.rotate(tokenFM)
print("rotated: ", rotated)
de_info = FM2.decrypt(rotated)
print(de_info.decode())
class RedisChannelLayer(BaseChannelLayer):
"""
ORM-backed channel environment. For development use only; it will span
multiple processes fine, but it's going to be pretty bad at throughput.
"""
blpop_timeout = 5
def __init__(self, expiry=60, hosts=None, prefix="asgi:", group_expiry=86400, capacity=100, channel_capacity=None,
symmetric_encryption_keys=None):
super(RedisChannelLayer, self).__init__(
expiry=expiry,
group_expiry=group_expiry,
capacity=capacity,
channel_capacity=channel_capacity,
)
# Make sure they provided some hosts, or provide a default
if not hosts:
hosts = [("localhost", 6379)]
self.hosts = []
for entry in hosts:
if isinstance(entry, six.string_types):
self.hosts.append(entry)
else:
self.hosts.append("redis://%s:%d/0" % (entry[0],entry[1]))
self.prefix = prefix
assert isinstance(self.prefix, six.text_type), "Prefix must be unicode"
# Precalculate some values for ring selection
self.ring_size = len(self.hosts)
self.ring_divisor = int(math.ceil(4096 / float(self.ring_size)))
# Create connections ahead of time (they won't call out just yet, but
# we want to connection-pool them later)
self._connection_list = [
redis.Redis.from_url(host)
for host in self.hosts
]
# Decide on a unique client prefix to use in ! sections
# TODO: ensure uniqueness better, e.g. Redis keys with SETNX
self.client_prefix = "".join(random.choice(string.ascii_letters) for i in range(8))
# Register scripts
connection = self.connection(None)
self.chansend = connection.register_script(self.lua_chansend)
self.lpopmany = connection.register_script(self.lua_lpopmany)
self.delprefix = connection.register_script(self.lua_delprefix)
# See if we can do encryption if they asked
if symmetric_encryption_keys:
if isinstance(symmetric_encryption_keys, six.string_types):
raise ValueError("symmetric_encryption_keys must be a list of possible keys")
try:
from cryptography.fernet import MultiFernet
except ImportError:
raise ValueError("Cannot run with encryption without 'cryptography' installed.")
sub_fernets = [self.make_fernet(key) for key in symmetric_encryption_keys]
self.crypter = MultiFernet(sub_fernets)
else:
self.crypter = None
### ASGI API ###
extensions = ["groups", "flush"]
def send(self, channel, message):
# Typecheck
assert isinstance(message, dict), "message is not a dict"
assert self.valid_channel_name(channel), "Channel name not valid"
# Write out message into expiring key (avoids big items in list)
# TODO: Use extended set, drop support for older redis?
message_key = self.prefix + uuid.uuid4().hex
channel_key = self.prefix + channel
# Pick a connection to the right server - consistent for response
# channels, random for normal channels
if "!" in channel or "?" in channel:
index = self.consistent_hash(channel)
connection = self.connection(index)
else:
connection = self.connection(None)
# Use the Lua function to do the set-and-push
try:
self.chansend(
keys=[message_key, channel_key],
args=[self.serialize(message), self.expiry, self.get_capacity(channel)],
)
except redis.exceptions.ResponseError as e:
# The Lua script handles capacity checking and sends the "full" error back
if e.args[0] == "full":
raise self.ChannelFull
def receive_many(self, channels, block=False):
if not channels:
return None, None
channels = list(channels)
assert all(self.valid_channel_name(channel) for channel in channels), "One or more channel names invalid"
# Work out what servers to listen on for the given channels
indexes = {}
random_index = self.random_index()
for channel in channels:
if "!" in channel or "?" in channel:
indexes.setdefault(self.consistent_hash(channel), []).append(channel)
else:
indexes.setdefault(random_index, []).append(channel)
# Get a message from one of our channels
while True:
# Select a random connection to use
index = random.choice(list(indexes.keys()))
connection = self.connection(index)
channels = indexes[index]
# Shuffle channels to avoid the first ones starving others of workers
random.shuffle(channels)
# Pop off any waiting message
list_names = [self.prefix + channel for channel in channels]
if block:
result = connection.blpop(list_names, timeout=self.blpop_timeout)
else:
result = self.lpopmany(keys=list_names, client=connection)
if result:
content = connection.get(result[1])
# If the content key expired, keep going.
if content is None:
continue
# Return the channel it's from and the message
return result[0][len(self.prefix):].decode("utf8"), self.deserialize(content)
else:
return None, None
def new_channel(self, pattern):
assert isinstance(pattern, six.text_type)
# Keep making channel names till one isn't present.
while True:
random_string = "".join(random.choice(string.ascii_letters) for i in range(12))
assert pattern.endswith("!") or pattern.endswith("?")
new_name = pattern + random_string
# Get right connection
index = self.consistent_hash(new_name)
connection = self.connection(index)
# Check to see if it's in the connected Redis.
# This fails to stop collisions for sharding where the channel is
# non-single-listener, but that seems very unlikely.
key = self.prefix + new_name
if not connection.exists(key):
return new_name
### ASGI Group extension ###
def group_add(self, group, channel):
"""
Adds the channel to the named group for at least 'expiry'
seconds (expiry defaults to message expiry if not provided).
"""
assert self.valid_group_name(group), "Group name not valid"
assert self.valid_channel_name(channel), "Channel name not valid"
group_key = self._group_key(group)
connection = self.connection(self.consistent_hash(group))
# Add to group sorted set with creation time as timestamp
connection.zadd(
group_key,
**{channel: time.time()}
)
# Set both expiration to be group_expiry, since everything in
# it at this point is guaranteed to expire before that
connection.expire(group_key, self.group_expiry)
# Also add to a normal set that contains all the groups a channel is in
# (as yet unused)
channel_key = self._channel_groups_key(channel)
connection = self.connection(self.consistent_hash(channel))
connection.sadd(channel_key, group)
connection.expire(channel_key, self.group_expiry)
def group_discard(self, group, channel):
"""
Removes the channel from the named group if it is in the group;
does nothing otherwise (does not error)
"""
assert self.valid_group_name(group), "Group name not valid"
assert self.valid_channel_name(channel), "Channel name not valid"
key = self._group_key(group)
self.connection(self.consistent_hash(group)).zrem(
key,
channel,
)
def group_channels(self, group):
"""
Returns all channels in the group as an iterable.
"""
key = self._group_key(group)
connection = self.connection(self.consistent_hash(group))
# Discard old channels based on group_expiry
connection.zremrangebyscore(key, 0, int(time.time()) - self.group_expiry)
# Return current lot
return [x.decode("utf8") for x in connection.zrange(
key,
0,
-1,
)]
def send_group(self, group, message):
"""
Sends a message to the entire group.
"""
assert self.valid_group_name(group), "Group name not valid"
# TODO: More efficient implementation (lua script per shard?)
for channel in self.group_channels(group):
try:
self.send(channel, message)
except self.ChannelFull:
pass
def _group_key(self, group):
return ("%s:group:%s" % (self.prefix, group)).encode("utf8")
def _channel_groups_key(self, group):
return ("%s:chgroups:%s" % (self.prefix, group)).encode("utf8")
### Flush extension ###
def flush(self):
"""
Deletes all messages and groups on all shards.
"""
for connection in self._connection_list:
self.delprefix(keys=[], args=[self.prefix+"*"], client=connection)
### Serialization ###
def serialize(self, message):
"""
Serializes message to a byte string.
"""
value = msgpack.packb(message, use_bin_type=True)
if self.crypter:
value = self.crypter.encrypt(value)
return value
def deserialize(self, message):
"""
Deserializes from a byte string.
"""
if self.crypter:
message = self.crypter.decrypt(message, self.expiry + 10)
return msgpack.unpackb(message, encoding="utf8")
### Redis Lua scripts ###
# Single-command channel send. Returns error if over capacity.
# Keys: message, channel_list
# Args: content, expiry, capacity
lua_chansend = """
if redis.call('llen', KEYS[2]) >= tonumber(ARGV[3]) then
return redis.error_reply("full")
end
redis.call('set', KEYS[1], ARGV[1])
redis.call('expire', KEYS[1], ARGV[2])
redis.call('rpush', KEYS[2], KEYS[1])
redis.call('expire', KEYS[2], ARGV[2] + 1)
"""
lua_lpopmany = """
for keyCount = 1, #KEYS do
local result = redis.call('LPOP', KEYS[keyCount])
if result then
return {KEYS[keyCount], result}
end
end
return {nil, nil}
"""
lua_delprefix = """
local keys = redis.call('keys', ARGV[1])
for i=1,#keys,5000 do
redis.call('del', unpack(keys, i, math.min(i+4999, #keys)))
end
"""
### Internal functions ###
def consistent_hash(self, value):
"""
Maps the value to a node value between 0 and 4095
using MD5, then down to one of the ring nodes.
"""
if isinstance(value, six.text_type):
value = value.encode("utf8")
bigval = binascii.crc32(value) & 0xffffffff
return (bigval // 0x100000) // self.ring_divisor
def random_index(self):
return random.randint(0, len(self.hosts) - 1)
def connection(self, index):
"""
Returns the correct connection for the current thread.
Pass key to use a server based on consistent hashing of the key value;
pass None to use a random server instead.
"""
# If index is explicitly None, pick a random server
if index is None:
index = self.random_index()
# Catch bad indexes
if not 0 <= index < self.ring_size:
raise ValueError("There are only %s hosts - you asked for %s!" % (self.ring_size, index))
return self._connection_list[index]
def make_fernet(self, key):
"""
Given a single encryption key, returns a Fernet instance using it.
"""
from cryptography.fernet import Fernet
if isinstance(key, six.text_type):
key = key.encode("utf8")
formatted_key = base64.urlsafe_b64encode(hashlib.sha256(key).digest())
return Fernet(formatted_key)
def __str__(self):
return "%s(hosts=%s)" % (self.__class__.__name__, self.hosts)
class Cryptograph(object):
"""Symmetric encryption and decryption for the storage of sensitive data.
We currently rely on Fernet, which was the algorithm adopted by Gratipay:
https://github.com/gratipay/gratipay.com/pull/3998#issuecomment-216227070
For encryption Fernet uses the AES cipher in CBC mode with PKCS7 padding and
a 128 bits key. For authentication it uses HMAC-SHA256 with another 128 bits
key.
Fernet messages contain the timestamp at which they were generated *in plain
text*. This isn't a problem for us since we want to store the time at which
the data was encrypted in order to facilitate key rotation.
We use CBOR (Concise Binary Object Representation) to serialize objects
before encryption. Compared to JSON, CBOR is faster to parse and serialize,
more compact, and extensible (it can represent any data type using "tags").
More info on CBOR: http://cbor.io/ https://tools.ietf.org/html/rfc7049
"""
KEY_ROTATION_DELAY = timedelta(weeks=1)
def __init__(self):
if website.env.aws_secret_access_key:
sm = self.secrets_manager = boto3.client('secretsmanager', region_name='eu-west-1')
secret = sm.get_secret_value(SecretId='Fernet')
rotation_start = secret['CreatedDate'].date()
keys = secret['SecretString'].split()
else:
self.secrets_manager = None
parts = os.environ['SECRET_FERNET_KEYS'].split()
rotation_start = date(*map(int, parts[0].split('-')))
keys = parts[1:]
self.fernet_rotation_start = rotation_start
self.fernet_keys = [k.encode('ascii') for k in keys]
self.fernet = MultiFernet([Fernet(k) for k in self.fernet_keys])
def encrypt_dict(self, dic, allow_single_key=False):
"""Serialize and encrypt a dictionary for storage in the database.
Encrypting partially predictable data may help an attacker break the
encryption key, so to make our data less predictable we randomize the
order of the dict's items before serializing it.
For this to be effective the CBOR serializer must not sort the items
again in an attempt to produce Canonical CBOR, so we explicitly pass
`canonical=False` to the `cbor.dumps` function.
In addition, the dict must not contain only one key if that key is
predictable, so a `CryptoWarning` is emitted when `dic` only contains
one key, unless `allow_single_key` is set to `True`.
"""
dic = self.randomize_dict(dic, allow_single_key=allow_single_key)
serialized = cbor.dumps(dic, canonical=False)
encrypted = self.fernet.encrypt(serialized)
return Encrypted(dict(scheme='fernet', payload=encrypted, ts=utcnow()))
def decrypt(self, scheme, payload):
"""Decrypt and reconstruct an object stored in the database.
"""
if scheme == 'fernet':
decrypted = self.fernet.decrypt(payload)
else:
raise ValueError('unknown encryption scheme %r' % scheme)
return cbor.loads(decrypted)
@staticmethod
def randomize_dict(dic, allow_single_key=False):
"""Randomize the order of a dictionary's items.
Emits a `CryptoWarning` if `dic` only contains one key, unless
`allow_single_key` is set to `True`.
"""
if not isinstance(dic, dict):
raise TypeError("expected a dict, got %s" % type(dic))
# Compute the number of random bytes needed based on the size of the dict
n = len(dic)
if n < 2:
# Can't randomize the order if the dict contains less than 2 items
if n == 1 and not allow_single_key:
warnings.warn("dict only contains one key", CryptoWarning)
return dic
n = int(log(n, 2) // 8) + 2
# Return a new ordered dict sorted randomly
return OrderedDict(
t[1] for t in sorted((urandom(n), item) for item in dic.items())
)
def rotate_key(self):
"""Generate a new key and send it to the secrets manager.
"""
keys = b' '.join([Fernet.generate_key()] + self.fernet_keys).decode()
if self.secrets_manager:
self.secrets_manager.update_secret(SecretId='Fernet', SecretString=keys)
else:
keys = utcnow().date().isoformat() + ' ' + keys
print("No secrets manager, updating the key storage is up to you.")
return keys
def rotate_message(self, msg, force=False):
"""Re-encrypt a single message using the current primary key.
The original timestamp included in the message is always preserved.
Moreover the entire message is returned unchanged if it was already
encrypted from the latest key and `force` is `False` (the default).
`InvalidToken` is raised if decryption fails.
"""
timestamp, data = Fernet._get_unverified_token_data(msg)
for i, fernet in enumerate(self.fernet._fernets):
try:
p = fernet._decrypt_data(data, timestamp, None)
except InvalidToken:
continue
if i == 0 and not force:
# This message was encrypted using the latest key, return it
return msg
break
else:
raise InvalidToken
iv = os.urandom(16)
return self.fernet._fernets[0]._encrypt_from_parts(p, timestamp, iv)
def rotate_stored_data(self, wait=True):
"""Re-encrypt all the sensitive information stored in our database.
This function is a special kind of "cron job" that returns one of two
constants from the `liberapay.cron` module: `CRON_ENCORE`, indicating
that the function needs to be run again to continue its work, or
`CRON_STOP`, indicating that all the ciphertexts are up-to-date (or that
it isn't time to rotate yet).
Rows are processed in batches of 50. Timestamps are used to keep track of
progress and to avoid overwriting new data with re-encrypted old data.
The update only starts one week after the new key was generated, unless
`wait` is set to `False`. This delay is to "ensure" that the previous
key is no longer being used to encrypt new data.
"""
update_start = self.fernet_rotation_start + self.KEY_ROTATION_DELAY
if wait:
if utcnow().date() < update_start:
return CRON_STOP
with website.db.get_cursor() as cursor:
batch = cursor.all("""
SELECT id, info
FROM identities
WHERE (info).ts <= %s
ORDER BY (info).ts ASC
LIMIT 50
""", (update_start,))
if not batch:
return CRON_STOP
sql = """
UPDATE identities
SET info = ('fernet', %s, current_timestamp)::encrypted
WHERE id = %s
AND (info).ts = %s;
"""
args_list = [
(self.rotate_message(r.info.payload), r.id, r.info.ts)
for r in batch
]
execute_batch(cursor, sql, args_list)
return CRON_ENCORE
def multiDecryptInput(privateKeys, token):
initiateKeys = []
for i in privateKeys:
initiateKeys.append(Fernet(i))
crypt = MultiFernet(initiateKeys)
return crypt.decrypt(token)
def decrypt(key1, key2, encryptedMessage):
k1 = Fernet(key1)
k2 = Fernet(key2)
decryptoKey = MultiFernet([k1, k2])
message = decryptoKey.decrypt(encryptedMessage)
return message
class StateEncryptor(object):
def __init__(self,
state_aging_tolerance: int = 120,
key_renewal_frequency: int = 30):
if state_aging_tolerance <= 0:
raise ValueError("Bad state_aging_tolerance")
if key_renewal_frequency < 0:
raise ValueError("Bad key_renewal_frequency")
self.__state_aging_tolerance: int = state_aging_tolerance
self.__key_renewal_frequency: int = key_renewal_frequency
self.__max_keys = 1
if key_renewal_frequency > 0:
if state_aging_tolerance < key_renewal_frequency:
self.__max_keys += 1
else:
self.__max_keys += state_aging_tolerance // key_renewal_frequency
if self.__max_keys >= REASONABLE_NUMBER_OF_KEYS:
raise ValueError(f"Keeping {self.__max_keys} keys is ridiculous.")
self.__keys: List[Fernet] = []
self.__renew_key()
self.__encryptor: MultiFernet = MultiFernet(self.__keys)
self.__thread_stop_event = threading.Event()
def __renew_key(self):
self.__keys.insert(0, Fernet(Fernet.generate_key()))
self.__keys = self.__keys[:self.__max_keys]
logging.debug(self.__keys)
self.__encryptor = MultiFernet(self.__keys)
def start(self):
schedule.every(self.__key_renewal_frequency).seconds.do(
self.__renew_key)
thread = threading.Thread(target=self.__run_scheduler)
thread.start()
def stop(self):
self.__thread_stop_event.set()
def __run_scheduler(self):
while not self.__thread_stop_event.is_set():
schedule.run_pending()
time.sleep(SCHEDULER_SLEEP_TIME)
@property
def max_keys(self):
return self.__max_keys
def encrypt_state(self, state: object) -> bytes:
state_serialization = pickle.dumps(state)
fernet_token = self.__encryptor.encrypt(state_serialization)
return fernet_token
def decrypt_state(self, encrypted_state: bytes) -> object:
try:
decrypted_state = self.__encryptor.decrypt(
encrypted_state, self.__state_aging_tolerance)
state = pickle.loads(decrypted_state)
except InvalidToken as e:
raise ValueError("Cannot decrypt state")
return state
class AutopushSettings(object):
"""Main Autopush Settings Object"""
options = ["crypto_key", "hostname", "min_ping_interval", "max_data"]
def __init__(
self,
crypto_key=None,
datadog_api_key=None,
datadog_app_key=None,
datadog_flush_interval=None,
hostname=None,
port=None,
router_scheme=None,
router_hostname=None,
router_port=None,
endpoint_scheme=None,
endpoint_hostname=None,
endpoint_port=None,
router_conf=None,
router_tablename="router",
router_read_throughput=5,
router_write_throughput=5,
storage_tablename="storage",
storage_read_throughput=5,
storage_write_throughput=5,
message_tablename="message",
message_read_throughput=5,
message_write_throughput=5,
statsd_host="localhost",
statsd_port=8125,
resolve_hostname=False,
max_data=4096,
# Reflected up from UDP Router
wake_timeout=0,
env='development',
enable_cors=False,
hello_timeout=0,
bear_hash_key=None,
preflight_uaid="deadbeef00000000deadbeef00000000",
ami_id=None,
client_certs=None,
msg_limit=100,
debug=False,
connect_timeout=0.5,
):
"""Initialize the Settings object
Upon creation, the HTTP agent will initialize, all configured routers
will be setup and started, logging will be started, and the database
will have a preflight check done.
"""
# Use a persistent connection pool for HTTP requests.
pool = HTTPConnectionPool(reactor)
if not debug:
pool._factory = QuietClientFactory
self.agent = Agent(reactor, connectTimeout=connect_timeout, pool=pool)
if not crypto_key:
crypto_key = [Fernet.generate_key()]
if not isinstance(crypto_key, list):
crypto_key = [crypto_key]
self.update(crypto_key=crypto_key)
self.crypto_key = crypto_key
if bear_hash_key is None:
bear_hash_key = []
if not isinstance(bear_hash_key, list):
bear_hash_key = [bear_hash_key]
self.bear_hash_key = bear_hash_key
self.max_data = max_data
self.clients = {}
# Setup hosts/ports/urls
default_hostname = socket.gethostname()
self.hostname = hostname or default_hostname
if resolve_hostname:
self.hostname = resolve_ip(self.hostname)
# Metrics setup
if datadog_api_key:
self.metrics = DatadogMetrics(
hostname=self.hostname,
api_key=datadog_api_key,
app_key=datadog_app_key,
flush_interval=datadog_flush_interval,
)
elif statsd_host:
self.metrics = TwistedMetrics(statsd_host, statsd_port)
else:
self.metrics = SinkMetrics()
self.port = port
self.endpoint_hostname = endpoint_hostname or self.hostname
self.router_hostname = router_hostname or self.hostname
if router_conf is None:
router_conf = {}
self.router_conf = router_conf
self.router_url = canonical_url(router_scheme or 'http',
self.router_hostname, router_port)
self.endpoint_url = canonical_url(endpoint_scheme or 'http',
self.endpoint_hostname,
endpoint_port)
self.enable_tls_auth = client_certs is not None
self.client_certs = client_certs
# Database objects
self.router_table = get_router_table(router_tablename,
router_read_throughput,
router_write_throughput)
self.storage_table = get_storage_table(storage_tablename,
storage_read_throughput,
storage_write_throughput)
self.message_table = get_rotating_message_table(
message_tablename,
message_read_throughput=message_read_throughput,
message_write_throughput=message_write_throughput)
self._message_prefix = message_tablename
self.message_limit = msg_limit
self.storage = Storage(self.storage_table, self.metrics)
self.router = Router(self.router_table, self.metrics)
# Used to determine whether a connection is out of date with current
# db objects. There are three noteworty cases:
# 1 "Last Month" the table requires a rollover.
# 2 "This Month" the most common case.
# 3 "Next Month" where the system will soon be rolling over, but with
# timing, some nodes may roll over sooner. Ensuring the next month's
# table is present before the switchover is the main reason for this,
# just in case some nodes do switch sooner.
self.create_initial_message_tables()
# Run preflight check
preflight_check(self.storage, self.router, preflight_uaid)
# CORS
self.cors = enable_cors
# Force timeout in idle seconds
self.wake_timeout = wake_timeout
# Setup the routers
self.routers = dict()
self.routers["simplepush"] = SimpleRouter(
self, router_conf.get("simplepush"))
self.routers["webpush"] = WebPushRouter(self, None)
if 'apns' in router_conf:
self.routers["apns"] = APNSRouter(self, router_conf["apns"])
if 'gcm' in router_conf:
self.routers["gcm"] = GCMRouter(self, router_conf["gcm"])
# Env
self.env = env
self.hello_timeout = hello_timeout
self.ami_id = ami_id
# Generate messages per legacy rules, only used for testing to
# generate legacy data.
self._notification_legacy = False
@property
def message(self):
"""Property that access the current message table"""
return self.message_tables[self.current_msg_month]
@message.setter
def message(self, value):
"""Setter to set the current message table"""
self.message_tables[self.current_msg_month] = value
def _tomorrow(self):
return datetime.date.today() + datetime.timedelta(days=1)
def create_initial_message_tables(self):
"""Initializes a dict of the initial rotating messages tables.
An entry for last months table, an entry for this months table,
an entry for tomorrow, if tomorrow is a new month.
"""
today = datetime.date.today()
last_month = get_rotating_message_table(self._message_prefix, -1)
this_month = get_rotating_message_table(self._message_prefix)
self.current_month = today.month
self.current_msg_month = this_month.table_name
self.message_tables = {
last_month.table_name: Message(last_month, self.metrics),
this_month.table_name: Message(this_month, self.metrics)
}
if self._tomorrow().month != today.month:
next_month = get_rotating_message_table(self._message_prefix,
delta=1)
self.message_tables[next_month.table_name] = Message(
next_month, self.metrics)
@inlineCallbacks
def update_rotating_tables(self):
"""This method is intended to be tasked to run periodically off the
twisted event hub to rotate tables.
When today is a new month from yesterday, then we swap out all the
table objects on the settings object.
"""
today = datetime.date.today()
tomorrow = self._tomorrow()
if ((tomorrow.month != today.month)
and sorted(self.message_tables.keys())[-1] != tomorrow.month):
next_month = yield deferToThread(get_rotating_message_table,
self._message_prefix, 0, tomorrow)
self.message_tables[next_month.table_name] = Message(
next_month, self.metrics)
if today.month == self.current_month:
# No change in month, we're fine.
returnValue(False)
# Get tables for the new month, and verify they exist before we try to
# switch over
message_table = yield deferToThread(get_rotating_message_table,
self._message_prefix)
# Both tables found, safe to switch-over
self.current_month = today.month
self.current_msg_month = message_table.table_name
self.message_tables[self.current_msg_month] = \
Message(message_table, self.metrics)
returnValue(True)
def update(self, **kwargs):
"""Update the arguments, if a ``crypto_key`` is in kwargs then the
``self.fernet`` attribute will be initialized"""
for key, val in kwargs.items():
if key == "crypto_key":
fkeys = []
if not isinstance(val, list):
val = [val]
for v in val:
fkeys.append(Fernet(v))
self.fernet = MultiFernet(fkeys)
else:
setattr(self, key, val)
def make_simplepush_endpoint(self, uaid, chid):
"""Create a simplepush endpoint"""
root = self.endpoint_url + "/spush/"
base = (uaid.replace('-', '').decode("hex") +
chid.replace('-', '').decode("hex"))
return root + 'v1/' + self.fernet.encrypt(base).strip('=')
def make_endpoint(self, uaid, chid, key=None):
"""Create an v1 or v2 WebPush endpoint from the identifiers.
Both endpoints use bytes instead of hex to reduce ID length.
v1 is the uaid + chid
v2 is the uaid + chid + sha256(key).bytes
:param uaid: User Agent Identifier
:param chid: Channel or Subscription ID
:param key: Optional Base64 URL-encoded application server key
:returns: Push endpoint
"""
root = self.endpoint_url + '/wpush/'
base = (uaid.replace('-', '').decode("hex") +
chid.replace('-', '').decode("hex"))
if key is None:
return root + 'v1/' + self.fernet.encrypt(base).strip('=')
raw_key = base64url_decode(key.encode('utf8'))
ep = self.fernet.encrypt(base + sha256(raw_key).digest()).strip('=')
return root + 'v2/' + ep
def parse_endpoint(self,
token,
version="v1",
ckey_header=None,
auth_header=None):
"""Parse an endpoint into component elements of UAID, CHID and optional
key hash if v2
:param token: The obscured subscription data.
:param version: This is the API version of the token.
:param ckey_header: the Crypto-Key header bearing the public key
(from Crypto-Key: p256ecdsa=)
:param auth_header: The Authorization header bearing the VAPID info
:raises ValueError: In the case of a malformed endpoint.
:returns: a dict containing (uaid=UAID, chid=CHID, public_key=KEY)
"""
token = self.fernet.decrypt(repad(token).encode('utf8'))
public_key = None
if ckey_header:
try:
crypto_key = CryptoKey(ckey_header)
except CryptoKeyException:
raise InvalidTokenException("Invalid key data")
public_key = crypto_key.get_label('p256ecdsa')
if version == 'v1' and len(token) != 32:
raise InvalidTokenException("Corrupted push token")
if version == 'v2':
if not auth_header:
raise VapidAuthException("Missing Authorization Header")
if len(token) != 64:
raise InvalidTokenException("Corrupted push token")
if not public_key:
raise VapidAuthException("Invalid key data")
try:
decoded_key = base64url_decode(public_key)
except TypeError:
raise VapidAuthException("Invalid key data")
if not constant_time.bytes_eq(
sha256(decoded_key).digest(), token[32:]):
raise VapidAuthException("Key mismatch")
return dict(uaid=token[:16].encode('hex'),
chid=token[16:32].encode('hex'),
version=version,
public_key=public_key)
class RedisChannelLayer(BaseChannelLayer):
"""
Redis channel layer.
It routes all messages into remote Redis server. Support for
sharding among different Redis installations and message
encryption are provided.
"""
brpop_timeout = 5
def __init__(
self,
hosts=None,
prefix="asgi:",
expiry=60,
group_expiry=86400,
capacity=100,
channel_capacity=None,
symmetric_encryption_keys=None,
):
# Store basic information
self.expiry = expiry
self.group_expiry = group_expiry
self.capacity = capacity
self.channel_capacity = self.compile_capacities(channel_capacity or {})
self.prefix = prefix
assert isinstance(self.prefix, str), "Prefix must be unicode"
# Configure the host objects
self.hosts = self.decode_hosts(hosts)
self.ring_size = len(self.hosts)
# Cached redis connection pools and the event loop they are from
self.pools = [ConnectionPool(host) for host in self.hosts]
# Normal channels choose a host index by cycling through the available hosts
self._receive_index_generator = itertools.cycle(range(len(self.hosts)))
self._send_index_generator = itertools.cycle(range(len(self.hosts)))
# Decide on a unique client prefix to use in ! sections
# TODO: ensure uniqueness better, e.g. Redis keys with SETNX
self.client_prefix = "".join(
random.choice(string.ascii_letters) for i in range(8))
# Set up any encryption objects
self._setup_encryption(symmetric_encryption_keys)
# Number of coroutines trying to receive right now
self.receive_count = 0
# The receive lock
self.receive_lock = None
# Event loop they are trying to receive on
self.receive_event_loop = None
# Buffered messages by process-local channel name
self.receive_buffer = collections.defaultdict(asyncio.Queue)
# Detached channel cleanup tasks
self.receive_cleaners = []
# Per-channel cleanup locks to prevent a receive starting and moving
# a message back into the main queue before its cleanup has completed
self.receive_clean_locks = ChannelLock()
def decode_hosts(self, hosts):
"""
Takes the value of the "hosts" argument passed to the class and returns
a list of kwargs to use for the Redis connection constructor.
"""
# If no hosts were provided, return a default value
if not hosts:
return [{"address": ("localhost", 6379)}]
# If they provided just a string, scold them.
if isinstance(hosts, (str, bytes)):
raise ValueError(
"You must pass a list of Redis hosts, even if there is only one."
)
# Decode each hosts entry into a kwargs dict
result = []
for entry in hosts:
if isinstance(entry, dict):
result.append(entry)
else:
result.append({"address": entry})
return result
def _setup_encryption(self, symmetric_encryption_keys):
# See if we can do encryption if they asked
if symmetric_encryption_keys:
if isinstance(symmetric_encryption_keys, (str, bytes)):
raise ValueError(
"symmetric_encryption_keys must be a list of possible keys"
)
try:
from cryptography.fernet import MultiFernet
except ImportError:
raise ValueError(
"Cannot run with encryption without 'cryptography' installed."
)
sub_fernets = [
self.make_fernet(key) for key in symmetric_encryption_keys
]
self.crypter = MultiFernet(sub_fernets)
else:
self.crypter = None
### Channel layer API ###
extensions = ["groups", "flush"]
async def send(self, channel, message):
"""
Send a message onto a (general or specific) channel.
"""
# Typecheck
assert isinstance(message, dict), "message is not a dict"
assert self.valid_channel_name(channel), "Channel name not valid"
# Make sure the message does not contain reserved keys
assert "__asgi_channel__" not in message
# If it's a process-local channel, strip off local part and stick full name in message
channel_non_local_name = channel
if "!" in channel:
message = dict(message.items())
message["__asgi_channel__"] = channel
channel_non_local_name = self.non_local_name(channel)
# Write out message into expiring key (avoids big items in list)
channel_key = self.prefix + channel_non_local_name
# Pick a connection to the right server - consistent for specific
# channels, random for general channels
if "!" in channel:
index = self.consistent_hash(channel)
else:
index = next(self._send_index_generator)
async with self.connection(index) as connection:
# Check the length of the list before send
# This can allow the list to leak slightly over capacity, but that's fine.
if await connection.llen(channel_key) >= self.get_capacity(
channel):
raise ChannelFull()
# Push onto the list then set it to expire in case it's not consumed
await connection.lpush(channel_key, self.serialize(message))
await connection.expire(channel_key, int(self.expiry))
def _backup_channel_name(self, channel):
"""
Construct the key used as a backup queue for the given channel.
"""
return channel + "$inflight"
async def _brpop_with_clean(self, index, channel, timeout):
"""
Perform a Redis BRPOP and manage the backup processing queue.
In case of cancellation, make sure the message is not lost.
"""
# The script will pop messages from the processing queue and push them in front
# of the main message queue in the proper order; BRPOP must *not* be called
# because that would deadlock the server
cleanup_script = """
local backed_up = redis.call('LRANGE', ARGV[2], 0, -1)
for i = #backed_up, 1, -1 do
redis.call('LPUSH', ARGV[1], backed_up[i])
end
redis.call('DEL', ARGV[2])
"""
backup_queue = self._backup_channel_name(channel)
async with self.connection(index) as connection:
# Cancellation here doesn't matter, we're not doing anything destructive
# and the script executes atomically...
await connection.eval(cleanup_script,
keys=[],
args=[channel, backup_queue])
# ...and it doesn't matter here either, the message will be safe in the backup.
return await connection.brpoplpush(channel,
backup_queue,
timeout=timeout)
async def _clean_receive_backup(self, index, channel):
"""
Pop the oldest message off the channel backup queue.
The result isn't interesting as it was already processed.
"""
async with self.connection(index) as connection:
await connection.brpop(self._backup_channel_name(channel))
async def receive(self, channel):
"""
Receive the first message that arrives on the channel.
If more than one coroutine waits on the same channel, the first waiter
will be given the message when it arrives.
"""
# Make sure the channel name is valid then get the non-local part
# and thus its index
assert self.valid_channel_name(channel)
if "!" in channel:
real_channel = self.non_local_name(channel)
assert real_channel.endswith(self.client_prefix +
"!"), "Wrong client prefix"
# Enter receiving section
loop = asyncio.get_event_loop()
self.receive_count += 1
try:
if self.receive_count == 1:
# If we're the first coroutine in, create the receive lock!
self.receive_lock = asyncio.Lock()
self.receive_event_loop = loop
else:
# Otherwise, check our event loop matches
if self.receive_event_loop != loop:
raise RuntimeError(
"Two event loops are trying to receive() on one channel layer at once!"
)
# Wait for our message to appear
message = None
while self.receive_buffer[channel].empty():
tasks = [
self.receive_lock.acquire(),
self.receive_buffer[channel].get(),
]
tasks = [asyncio.ensure_future(task) for task in tasks]
try:
done, pending = await asyncio.wait(
tasks, return_when=asyncio.FIRST_COMPLETED)
for task in pending:
# Cancel all pending tasks.
task.cancel()
except asyncio.CancelledError:
# Ensure all tasks are cancelled if we are cancelled.
# Also see: https://bugs.python.org/issue23859
del self.receive_buffer[channel]
for task in tasks:
if not task.cancel():
assert task.done()
if task.result() is True:
self.receive_lock.release()
raise
message, token, exception = None, None, None
for task in done:
try:
result = task.result()
except Exception as error: # NOQA
# We should not propagate exceptions immediately as otherwise this may cause
# the lock to be held and never be released.
exception = error
continue
if result is True:
token = result
else:
assert isinstance(result, dict)
message = result
if message or exception:
if token:
# We will not be receving as we already have the message.
self.receive_lock.release()
if exception:
raise exception
else:
break
else:
assert token
# We hold the receive lock, receive and then release it.
try:
# There is no interruption point from when the message is
# unpacked in receive_single to when we get back here, so
# the following lines are essentially atomic.
message_channel, message = await self.receive_single(
real_channel)
if type(message_channel) is list:
for chan in message_channel:
self.receive_buffer[chan].put_nowait(
message)
else:
self.receive_buffer[
message_channel].put_nowait(message)
message = None
except:
del self.receive_buffer[channel]
raise
finally:
self.receive_lock.release()
# We know there's a message available, because there
# couldn't have been any interruption between empty() and here
if message is None:
message = self.receive_buffer[channel].get_nowait()
if self.receive_buffer[channel].empty():
del self.receive_buffer[channel]
return message
finally:
self.receive_count -= 1
# If we were the last out, drop the receive lock
if self.receive_count == 0:
assert not self.receive_lock.locked()
self.receive_lock = None
self.receive_event_loop = None
else:
# Do a plain direct receive
return (await self.receive_single(channel))[1]
async def receive_single(self, channel):
"""
Receives a single message off of the channel and returns it.
"""
# Check channel name
assert self.valid_channel_name(channel,
receive=True), "Channel name invalid"
# Work out the connection to use
if "!" in channel:
assert channel.endswith("!")
index = self.consistent_hash(channel)
else:
index = next(self._receive_index_generator)
channel_key = self.prefix + channel
content = None
await self.receive_clean_locks.acquire(channel_key)
try:
while content is None:
# Nothing is lost here by cancellations, messages will still
# be in the backup queue.
content = await self._brpop_with_clean(
index, channel_key, timeout=self.brpop_timeout)
# Fire off a task to clean the message from its backup queue.
# Per-channel locking isn't needed, because the backup is a queue
# and additionally, we don't care about the order; all processed
# messages need to be removed, no matter if the current one is
# removed after the next one.
# NOTE: Duplicate messages will be received eventually if any
# of these cleaners are cancelled.
cleaner = asyncio.ensure_future(
self._clean_receive_backup(index, channel_key))
self.receive_cleaners.append(cleaner)
def _cleanup_done(cleaner):
self.receive_cleaners.remove(cleaner)
self.receive_clean_locks.release(channel_key)
cleaner.add_done_callback(_cleanup_done)
except Exception:
self.receive_clean_locks.release(channel_key)
raise
# Message decode
message = self.deserialize(content)
# TODO: message expiry?
# If there is a full channel name stored in the message, unpack it.
if "__asgi_channel__" in message:
channel = message["__asgi_channel__"]
del message["__asgi_channel__"]
return channel, message
async def new_channel(self, prefix="specific"):
"""
Returns a new channel name that can be used by something in our
process as a specific channel.
"""
# TODO: Guarantee uniqueness better?
return "%s.%s!%s" % (
prefix,
self.client_prefix,
"".join(random.choice(string.ascii_letters) for i in range(12)),
)
### Flush extension ###
async def flush(self):
"""
Deletes all messages and groups on all shards.
"""
# Make sure all channel cleaners have finished before removing
# keys from under their feet.
await self.wait_received()
# Lua deletion script
delete_prefix = """
local keys = redis.call('keys', ARGV[1])
for i=1,#keys,5000 do
redis.call('del', unpack(keys, i, math.min(i+4999, #keys)))
end
"""
# Go through each connection and remove all with prefix
for i in range(self.ring_size):
async with self.connection(i) as connection:
await connection.eval(delete_prefix,
keys=[],
args=[self.prefix + "*"])
# Now clear the pools as well
await self.close_pools()
async def close_pools(self):
"""
Close all connections in the event loop pools.
"""
# Flush all cleaners, in case somebody just wanted to close the
# pools without flushing first.
await self.wait_received()
for pool in self.pools:
await pool.close()
async def wait_received(self):
"""
Wait for all channel cleanup functions to finish.
"""
if self.receive_cleaners:
await asyncio.wait(self.receive_cleaners[:])
### Groups extension ###
async def group_add(self, group, channel):
"""
Adds the channel name to a group.
"""
# Check the inputs
assert self.valid_group_name(group), "Group name not valid"
assert self.valid_channel_name(channel), "Channel name not valid"
# Get a connection to the right shard
group_key = self._group_key(group)
async with self.connection(self.consistent_hash(group)) as connection:
# Add to group sorted set with creation time as timestamp
await connection.zadd(group_key, time.time(), channel)
# Set expiration to be group_expiry, since everything in
# it at this point is guaranteed to expire before that
await connection.expire(group_key, self.group_expiry)
async def group_discard(self, group, channel):
"""
Removes the channel from the named group if it is in the group;
does nothing otherwise (does not error)
"""
assert self.valid_group_name(group), "Group name not valid"
assert self.valid_channel_name(channel), "Channel name not valid"
key = self._group_key(group)
async with self.connection(self.consistent_hash(group)) as connection:
await connection.zrem(key, channel)
async def group_send(self, group, message):
"""
Sends a message to the entire group.
"""
assert self.valid_group_name(group), "Group name not valid"
# Retrieve list of all channel names
key = self._group_key(group)
async with self.connection(self.consistent_hash(group)) as connection:
# Discard old channels based on group_expiry
await connection.zremrangebyscore(key,
min=0,
max=int(time.time()) -
self.group_expiry)
channel_names = [
x.decode("utf8") for x in await connection.zrange(key, 0, -1)
]
connection_to_channel_keys, channel_keys_to_message, channel_keys_to_capacity = self._map_channel_keys_to_connection(
channel_names, message)
for connection_index, channel_redis_keys in connection_to_channel_keys.items(
):
# Create a LUA script specific for this connection.
# Make sure to use the message specific to this channel, it is
# stored in channel_to_message dict and contains the
# __asgi_channel__ key.
group_send_lua = ("""
for i=1,#KEYS do
if redis.call('LLEN', KEYS[i]) < tonumber(ARGV[i + #KEYS]) then
redis.call('LPUSH', KEYS[i], ARGV[i])
redis.call('EXPIRE', KEYS[i], %d)
end
end
""" % self.expiry)
# We need to filter the messages to keep those related to the connection
args = [
channel_keys_to_message[channel_key]
for channel_key in channel_redis_keys
]
# We need to send the capacity for each channel
args += [
channel_keys_to_capacity[channel_key]
for channel_key in channel_redis_keys
]
# channel_keys does not contain a single redis key more than once
async with self.connection(connection_index) as connection:
await connection.eval(group_send_lua,
keys=channel_redis_keys,
args=args)
def _map_channel_to_connection(self, channel_names, message):
"""
For a list of channel names, bucket each one to a dict keyed by the
connection index
Also for each channel create a message specific to that channel, adding
the __asgi_channel__ key to the message
We also return a mapping from channel names to their corresponding Redis
keys, and a mapping of channels to their capacity
"""
connection_to_channels = collections.defaultdict(list)
channel_to_message = dict()
channel_to_capacity = dict()
channel_to_key = dict()
for channel in channel_names:
channel_non_local_name = channel
if "!" in channel:
message = dict(message.items())
message["__asgi_channel__"] = channel
channel_non_local_name = self.non_local_name(channel)
channel_key = self.prefix + channel_non_local_name
idx = self.consistent_hash(channel_non_local_name)
connection_to_channels[idx].append(channel_key)
channel_to_capacity[channel] = self.get_capacity(channel)
channel_to_message[channel] = self.serialize(message)
# We build a
channel_to_key[channel] = channel_key
return (
connection_to_channels,
channel_to_message,
channel_to_capacity,
channel_to_key,
)
def _map_channel_keys_to_connection(self, channel_names, message):
"""
For a list of channel names, GET
1. list of their redis keys bucket each one to a dict keyed by the connection index
2. for each unique channel redis key create a serialized message specific to that redis key, by adding
the list of channels mapped to that redis key in __asgi_channel__ key to the message
3. returns a mapping of redis channels keys to their capacity
"""
# Connection dict keyed by index to list of redis keys mapped on that index
connection_to_channel_keys = collections.defaultdict(list)
# Message dict maps redis key to the message that needs to be send on that key
channel_key_to_message = dict()
# Channel key mapped to its capacity
channel_key_to_capacity = dict()
# For each channel
for channel in channel_names:
channel_non_local_name = channel
if "!" in channel:
channel_non_local_name = self.non_local_name(channel)
# Get its redis key
channel_key = self.prefix + channel_non_local_name
# Have we come across the same redis key?
if channel_key not in channel_key_to_message.keys():
# If not, fill the corresponding dicts
message = dict(message.items())
message["__asgi_channel__"] = [channel]
channel_key_to_message[channel_key] = message
channel_key_to_capacity[channel_key] = self.get_capacity(
channel)
idx = self.consistent_hash(channel_non_local_name)
connection_to_channel_keys[idx].append(channel_key)
else:
# Yes, Append the channel in message dict
channel_key_to_message[channel_key]["__asgi_channel__"].append(
channel)
# Now that we know what message needs to be send on a redis key we serialize it
for key in channel_key_to_message.keys():
# Serialize the message stored for each redis key
channel_key_to_message[key] = self.serialize(
channel_key_to_message[key])
return (
connection_to_channel_keys,
channel_key_to_message,
channel_key_to_capacity,
)
def _group_key(self, group):
"""
Common function to make the storage key for the group.
"""
return ("%s:group:%s" % (self.prefix, group)).encode("utf8")
### Serialization ###
def serialize(self, message):
"""
Serializes message to a byte string.
"""
value = msgpack.packb(message, use_bin_type=True)
if self.crypter:
value = self.crypter.encrypt(value)
return value
def deserialize(self, message):
"""
Deserializes from a byte string.
"""
if self.crypter:
message = self.crypter.decrypt(message, self.expiry + 10)
return msgpack.unpackb(message, raw=False)
### Internal functions ###
def consistent_hash(self, value):
"""
Maps the value to a node value between 0 and 4095
using CRC, then down to one of the ring nodes.
"""
if isinstance(value, str):
value = value.encode("utf8")
bigval = binascii.crc32(value) & 0xFFF
ring_divisor = 4096 / float(self.ring_size)
return int(bigval / ring_divisor)
def make_fernet(self, key):
"""
Given a single encryption key, returns a Fernet instance using it.
"""
from cryptography.fernet import Fernet
if isinstance(key, str):
key = key.encode("utf8")
formatted_key = base64.urlsafe_b64encode(hashlib.sha256(key).digest())
return Fernet(formatted_key)
def __str__(self):
return "%s(hosts=%s)" % (self.__class__.__name__, self.hosts)
### Connection handling ###
def connection(self, index):
"""
Returns the correct connection for the index given.
Lazily instantiates pools.
"""
# Catch bad indexes
if not 0 <= index < self.ring_size:
raise ValueError("There are only %s hosts - you asked for %s!" %
(self.ring_size, index))
# Make a context manager
return self.ConnectionContextManager(self.pools[index])
class ConnectionContextManager:
"""
Async context manager for connections
"""
def __init__(self, pool):
self.pool = pool
async def __aenter__(self):
self.conn = await self.pool.pop()
return self.conn
async def __aexit__(self, exc_type, exc, tb):
if exc:
self.pool.conn_error(self.conn)
else:
self.pool.push(self.conn)
self.conn = None
class AutopushSettings(object):
"""Main Autopush Settings Object"""
options = ["crypto_key", "hostname", "min_ping_interval",
"max_data"]
def __init__(self,
crypto_key=None,
datadog_api_key=None,
datadog_app_key=None,
datadog_flush_interval=None,
hostname=None,
port=None,
router_scheme=None,
router_hostname=None,
router_port=None,
endpoint_scheme=None,
endpoint_hostname=None,
endpoint_port=None,
router_conf={},
router_tablename="router",
router_read_throughput=5,
router_write_throughput=5,
storage_tablename="storage",
storage_read_throughput=5,
storage_write_throughput=5,
message_tablename="message",
message_read_throughput=5,
message_write_throughput=5,
statsd_host="localhost",
statsd_port=8125,
resolve_hostname=False,
max_data=4096,
# Reflected up from UDP Router
wake_timeout=0,
env='development',
enable_cors=False,
s3_bucket=DEFAULT_BUCKET,
senderid_expry=SENDERID_EXPRY,
senderid_list={},
hello_timeout=0,
bear_hash_key=None,
preflight_uaid="deadbeef00000000deadbeef000000000",
):
"""Initialize the Settings object
Upon creation, the HTTP agent will initialize, all configured routers
will be setup and started, logging will be started, and the database
will have a preflight check done.
"""
# Use a persistent connection pool for HTTP requests.
pool = HTTPConnectionPool(reactor)
self.agent = Agent(reactor, connectTimeout=5, pool=pool)
# Metrics setup
if datadog_api_key:
self.metrics = DatadogMetrics(
api_key=datadog_api_key,
app_key=datadog_app_key,
flush_interval=datadog_flush_interval
)
elif statsd_host:
self.metrics = TwistedMetrics(statsd_host, statsd_port)
else:
self.metrics = SinkMetrics()
if not crypto_key:
crypto_key = [Fernet.generate_key()]
if not isinstance(crypto_key, list):
crypto_key = [crypto_key]
self.update(crypto_key=crypto_key)
self.crypto_key = crypto_key
if bear_hash_key is None:
bear_hash_key = []
if not isinstance(bear_hash_key, list):
bear_hash_key = [bear_hash_key]
self.bear_hash_key = bear_hash_key
self.max_data = max_data
self.clients = {}
# Setup hosts/ports/urls
default_hostname = socket.gethostname()
self.hostname = hostname or default_hostname
if resolve_hostname:
self.hostname = resolve_ip(self.hostname)
self.port = port
self.endpoint_hostname = endpoint_hostname or self.hostname
self.router_hostname = router_hostname or self.hostname
self.router_conf = router_conf
self.router_url = canonical_url(
router_scheme or 'http',
self.router_hostname,
router_port
)
self.endpoint_url = canonical_url(
endpoint_scheme or 'http',
self.endpoint_hostname,
endpoint_port
)
# Database objects
self.router_table = get_router_table(router_tablename,
router_read_throughput,
router_write_throughput)
self.storage_table = get_storage_table(
storage_tablename,
storage_read_throughput,
storage_write_throughput)
self.message_table = get_rotating_message_table(
message_tablename)
self._message_prefix = message_tablename
self.storage = Storage(self.storage_table, self.metrics)
self.router = Router(self.router_table, self.metrics)
# Used to determine whether a connection is out of date with current
# db objects. There are three noteworty cases:
# 1 "Last Month" the table requires a rollover.
# 2 "This Month" the most common case.
# 3 "Next Month" where the system will soon be rolling over, but with
# timing, some nodes may roll over sooner. Ensuring the next month's
# table is present before the switchover is the main reason for this,
# just in case some nodes do switch sooner.
self.create_initial_message_tables()
# Run preflight check
preflight_check(self.storage, self.router, preflight_uaid)
# CORS
self.cors = enable_cors
# Force timeout in idle seconds
self.wake_timeout = wake_timeout
# Setup the routers
self.routers = {}
self.routers["simplepush"] = SimpleRouter(
self,
router_conf.get("simplepush")
)
self.routers["webpush"] = WebPushRouter(self, None)
if 'apns' in router_conf:
self.routers["apns"] = APNSRouter(self, router_conf["apns"])
if 'gcm' in router_conf:
self.routers["gcm"] = GCMRouter(self, router_conf["gcm"])
# Env
self.env = env
self.hello_timeout = hello_timeout
@property
def message(self):
"""Property that access the current message table"""
return self.message_tables[self.current_msg_month]
@message.setter
def message(self, value):
"""Setter to set the current message table"""
self.message_tables[self.current_msg_month] = value
def _tomorrow(self):
return datetime.date.today() + datetime.timedelta(days=1)
def create_initial_message_tables(self):
"""Initializes a dict of the initial rotating messages tables.
An entry for last months table, an entry for this months table,
an entry for tomorrow, if tomorrow is a new month.
"""
today = datetime.date.today()
last_month = get_rotating_message_table(self._message_prefix, -1)
this_month = get_rotating_message_table(self._message_prefix)
self.current_month = today.month
self.current_msg_month = this_month.table_name
self.message_tables = {
last_month.table_name: Message(last_month, self.metrics),
this_month.table_name: Message(this_month, self.metrics)
}
if self._tomorrow().month != today.month:
next_month = get_rotating_message_table(delta=1)
self.message_tables[next_month.table_name] = Message(
next_month, self.metrics)
@inlineCallbacks
def update_rotating_tables(self):
"""This method is intended to be tasked to run periodically off the
twisted event hub to rotate tables.
When today is a new month from yesterday, then we swap out all the
table objects on the settings object.
"""
today = datetime.date.today()
tomorrow = self._tomorrow()
if ((tomorrow.month != today.month) and
sorted(self.message_tables.keys())[-1] !=
tomorrow.month):
next_month = get_rotating_message_table(
self._message_prefix, 0, tomorrow)
self.message_tables[next_month.table_name] = Message(
next_month, self.metrics)
if today.month == self.current_month:
# No change in month, we're fine.
returnValue(False)
# Get tables for the new month, and verify they exist before we try to
# switch over
message_table = yield deferToThread(get_rotating_message_table,
self._message_prefix)
# Both tables found, safe to switch-over
self.current_month = today.month
self.current_msg_month = message_table.table_name
self.message_tables[self.current_msg_month] = \
Message(message_table, self.metrics)
returnValue(True)
def update(self, **kwargs):
"""Update the arguments, if a ``crypto_key`` is in kwargs then the
``self.fernet`` attribute will be initialized"""
for key, val in kwargs.items():
if key == "crypto_key":
fkeys = []
if not isinstance(val, list):
val = [val]
for v in val:
fkeys.append(Fernet(v))
self.fernet = MultiFernet(fkeys)
else:
setattr(self, key, val)
def make_simplepush_endpoint(self, uaid, chid):
"""Create a simplepush endpoint"""
root = self.endpoint_url + "/spush/"
base = (uaid.replace('-', '').decode("hex") +
chid.replace('-', '').decode("hex"))
return root + 'v1/' + self.fernet.encrypt(base).strip('=')
def make_endpoint(self, uaid, chid, key=None):
"""Create an v1 or v2 WebPush endpoint from the identifiers.
Both endpoints use bytes instead of hex to reduce ID length.
v0 is uaid.hex + ':' + chid.hex and is deprecated.
v1 is the uaid + chid
v2 is the uaid + chid + sha256(key).bytes
:param uaid: User Agent Identifier
:param chid: Channel or Subscription ID
:param key: Optional Base64 URL-encoded application server key
:returns: Push endpoint
"""
root = self.endpoint_url + '/push/'
base = (uaid.replace('-', '').decode("hex") +
chid.replace('-', '').decode("hex"))
if key is None:
return root + 'v1/' + self.fernet.encrypt(base).strip('=')
raw_key = base64url_decode(key.encode('utf8'))
ep = self.fernet.encrypt(base + sha256(raw_key).digest()).strip('=')
return root + 'v2/' + ep
def parse_endpoint(self, token, version="v0", ckey_header=None):
"""Parse an endpoint into component elements of UAID, CHID and optional
key hash if v2
:param token: The obscured subscription data.
:param version: This is the API version of the token.
:param ckey_header: the Crypto-Key header bearing the public key
(from Crypto-Key: p256ecdsa=)
:raises ValueError: In the case of a malformed endpoint.
:returns: a dict containing (uaid=UAID, chid=CHID, public_key=KEY)
"""
token = self.fernet.decrypt(repad(token).encode('utf8'))
public_key = None
if ckey_header:
try:
crypto_key = CryptoKey(ckey_header)
except CryptoKeyException:
raise InvalidTokenException("Invalid key data")
label = crypto_key.get_label('p256ecdsa')
try:
public_key = base64url_decode(label)
except:
# Ignore missing and malformed app server keys.
pass
if version == 'v0':
if not VALID_V0_TOKEN.match(token):
raise InvalidTokenException("Corrupted push token")
items = token.split(':')
return dict(uaid=items[0], chid=items[1], public_key=public_key)
if version == 'v1' and len(token) != 32:
raise InvalidTokenException("Corrupted push token")
if version == 'v2':
if len(token) != 64:
raise InvalidTokenException("Corrupted push token")
if not public_key:
raise InvalidTokenException("Invalid key data")
if not constant_time.bytes_eq(sha256(public_key).digest(),
token[32:]):
raise InvalidTokenException("Key mismatch")
return dict(uaid=token[:16].encode('hex'),
chid=token[16:32].encode('hex'),
public_key=public_key)
class DataManager(object):
"""Holds the internal state for a single Dask Gateway.
Keeps the memory representation in-sync with the database.
"""
def __init__(self, url="sqlite:///:memory:", encrypt_keys=(), **kwargs):
if url.startswith("sqlite"):
kwargs["connect_args"] = {"check_same_thread": False}
if is_in_memory_db(url):
kwargs["poolclass"] = StaticPool
self.fernet = None
else:
self.fernet = MultiFernet([Fernet(key) for key in encrypt_keys])
engine = create_engine(url, **kwargs)
if url.startswith("sqlite"):
register_foreign_keys(engine)
metadata.create_all(engine)
self.db = engine
self.username_to_user = {}
self.cookie_to_user = {}
self.token_to_cluster = {}
self.name_to_cluster = {}
self.id_to_cluster = {}
def load_database_state(self):
# Load all existing users into memory
id_to_user = {}
for u in self.db.execute(users.select()):
user = User(id=u.id, name=u.name, cookie=u.cookie)
self.username_to_user[user.name] = user
self.cookie_to_user[user.cookie] = user
id_to_user[user.id] = user
# Next load all existing clusters into memory
for c in self.db.execute(clusters.select()):
user = id_to_user[c.user_id]
tls_cert, tls_key = self.decode_tls_credentials(c.tls_credentials)
token = self.decode_token(c.token)
cluster = Cluster(
id=c.id,
name=c.name,
user=user,
token=token,
options=c.options,
status=c.status,
state=c.state,
scheduler_address=c.scheduler_address,
dashboard_address=c.dashboard_address,
api_address=c.api_address,
tls_cert=tls_cert,
tls_key=tls_key,
adaptive=c.adaptive,
memory=c.memory,
cores=c.cores,
start_time=c.start_time,
stop_time=c.stop_time,
)
self.id_to_cluster[cluster.id] = cluster
self.token_to_cluster[cluster.token] = cluster
self.name_to_cluster[cluster.name] = cluster
user.clusters[cluster.name] = cluster
# Next load all existing workers into memory
for w in self.db.execute(workers.select()):
cluster = self.id_to_cluster[w.cluster_id]
worker = Worker(
id=w.id,
name=w.name,
status=w.status,
cluster=cluster,
state=w.state,
memory=w.memory,
cores=w.cores,
start_time=w.start_time,
stop_time=w.stop_time,
)
cluster.workers[worker.name] = worker
if w.status == WorkerStatus.STARTING:
cluster.pending.add(worker.name)
def cleanup_expired(self, max_age_in_seconds):
cutoff = timestamp() - max_age_in_seconds * 1000
with self.db.begin() as conn:
to_delete = conn.execute(
select([clusters.c.id
]).where(clusters.c.stop_time < cutoff)).fetchall()
if to_delete:
to_delete = [i for i, in to_delete]
conn.execute(
clusters.delete().where(clusters.c.id == bindparam("id")),
[{
"id": i
} for i in to_delete],
)
for i in to_delete:
cluster = self.id_to_cluster.pop(i)
del self.token_to_cluster[cluster.token]
del self.name_to_cluster[cluster.name]
del cluster.user.clusters[cluster.name]
return len(to_delete)
def encrypt(self, b):
"""Encrypt bytes ``b``. If encryption is disabled this is a no-op"""
return b if self.fernet is None else self.fernet.encrypt(b)
def decrypt(self, b):
"""Decrypt bytes ``b``. If encryption is disabled this is a no-op"""
return b if self.fernet is None else self.fernet.decrypt(b)
def encode_tls_credentials(self, tls_cert, tls_key):
return self.encrypt(b";".join((tls_cert, tls_key)))
def decode_tls_credentials(self, data):
return self.decrypt(data).split(b";")
def encode_token(self, token):
return self.encrypt(token.encode("utf8"))
def decode_token(self, data):
return self.decrypt(data).decode()
def user_from_cookie(self, cookie):
"""Lookup a user from a cookie"""
return self.cookie_to_user.get(cookie)
def get_or_create_user(self, username):
"""Lookup a user if they exist, otherwise create a new user"""
user = self.username_to_user.get(username)
if user is None:
cookie = uuid.uuid4().hex
res = self.db.execute(users.insert().values(name=username,
cookie=cookie))
user = User(id=res.inserted_primary_key[0],
name=username,
cookie=cookie)
self.cookie_to_user[cookie] = user
self.username_to_user[username] = user
return user
def cluster_from_token(self, token):
"""Lookup a cluster from a token"""
return self.token_to_cluster.get(token)
def cluster_from_name(self, name):
"""Lookup a cluster by name"""
return self.name_to_cluster.get(name)
def active_clusters(self):
for user in self.username_to_user.values():
for cluster in user.clusters.values():
if cluster.is_active():
yield cluster
def create_cluster(self, user, options, memory, cores):
"""Create a new cluster for a user"""
cluster_name = uuid.uuid4().hex
token = uuid.uuid4().hex
tls_cert, tls_key = new_keypair(cluster_name)
# Encode the tls credentials for storing in the database
tls_credentials = self.encode_tls_credentials(tls_cert, tls_key)
enc_token = self.encode_token(token)
common = {
"name": cluster_name,
"options": options,
"status": ClusterStatus.STARTING,
"state": {},
"scheduler_address": "",
"dashboard_address": "",
"api_address": "",
"memory": memory,
"cores": cores,
"adaptive": False,
"start_time": timestamp(),
}
with self.db.begin() as conn:
res = conn.execute(clusters.insert().values(
user_id=user.id,
tls_credentials=tls_credentials,
token=enc_token,
**common,
))
cluster = Cluster(
id=res.inserted_primary_key[0],
user=user,
token=token,
tls_cert=tls_cert,
tls_key=tls_key,
**common,
)
self.id_to_cluster[cluster.id] = cluster
self.token_to_cluster[token] = cluster
self.name_to_cluster[cluster_name] = cluster
user.clusters[cluster_name] = cluster
return cluster
def create_worker(self, cluster, memory, cores):
"""Create a new worker for a cluster"""
worker_name = uuid.uuid4().hex
common = {
"name": worker_name,
"status": WorkerStatus.STARTING,
"state": {},
"memory": memory,
"cores": cores,
"start_time": timestamp(),
}
with self.db.begin() as conn:
res = conn.execute(workers.insert().values(cluster_id=cluster.id,
**common))
worker = Worker(id=res.inserted_primary_key[0],
cluster=cluster,
**common)
cluster.pending.add(worker.name)
cluster.workers[worker.name] = worker
return worker
def update_cluster(self, cluster, **kwargs):
"""Update a cluster's state"""
with self.db.begin() as conn:
conn.execute(clusters.update().where(
clusters.c.id == cluster.id).values(**kwargs))
for k, v in kwargs.items():
setattr(cluster, k, v)
def update_worker(self, worker, **kwargs):
"""Update a worker's state"""
with self.db.begin() as conn:
conn.execute(workers.update().where(
workers.c.id == worker.id).values(**kwargs))
for k, v in kwargs.items():
setattr(worker, k, v)
