def serialize(self, topic, value):
if (isinstance(topic,(bytes,bytearray))):
self._parent._logger.debug("passed a topic in bytes (should be string)")
topic = topic.decode('utf-8')
if value is None:
return None
if (isinstance(value,(KafkaCryptoMessage,))):
if (value.isCleartext(retry=False)):
value = value.getMessage(retry=False)
else:
# already serialized and encrypted
return bytes(value)
if (not isinstance(value,(bytes,bytearray))):
raise KafkaCryptoSerializeError("Passed value is not bytes or a KafkaCryptoMessage")
root = self._parent.get_root(topic)
self._parent._lock.acquire()
try:
#
# Slow path: if we don't already have a generator for this topic,
# we have to make one.
#
if not (root in self._parent._cur_pgens.keys()):
self._parent.get_producer(root)
# Use generator (new or existing)
keyidx = self._parent._cur_pgens[root]
pgen = self._parent._pgens[root][keyidx]
gen = pgen[self._kv]
salt = gen.salt()
key,nonce = gen.generate()
msg = b'\x01' + msgpack.packb([keyidx,salt,pysodium.crypto_secretbox(value,nonce,key)], use_bin_type=True)
except Exception as e:
self._parent._logger.warning("".join(format_exception_shim(e)))
finally:
self._parent._lock.release()
return msg
def reencrypt_request(self,
topic,
cryptoexchange,
msgkey=None,
msgval=None):
if (isinstance(topic, (bytes, bytearray))):
self._logger.debug("topic provided as bytes (should be string)")
topic = topic.decode('utf-8')
#
# msgval should be a msgpacked chain chaining to a provisioner.
# The last item in the array is the encrypted public key to retransmit.
#
try:
pk = None
try:
pk, pkprint = process_chain(msgval,
topic,
'key-encrypt-subscribe',
allowlist=self.__allowlist,
denylist=self.__denylist)
except ProcessChainError as pce:
raise pce
except:
pass
if (pk is None):
if not (pkprint is None):
raise ProcessChainError("Request did not validate: ",
pkprint)
else:
raise ValueError("Request did not validate!")
msg = cryptoexchange.signed_epk(topic, epk=pk[2])
except Exception as e:
self._logger.warning("".join(format_exception_shim(e)))
return (None, None)
return (msgkey, msg)
def encrypt_keys(self, keyidxs, keys, topic, msgval=None):
if (isinstance(topic,(bytes,bytearray))):
self._logger.debug("passed a topic in bytes (should be string)")
topic = topic.decode('utf-8')
#
# msgval should be a msgpacked chain.
# The public key in the array is the public key to send the key to, using a
# common DH-derived key between that public key and our private encryption key.
# Then there is at least one more additional item, random bytes:
# (3) random bytes
# Currently items after this are ignored, and reserved for future use.
#
try:
with self.__allowdenylist_lock:
pk,_ = process_chain(msgval,topic,'key-encrypt-request',allowlist=self.__allowlist,denylist=self.__denylist)
# Construct shared secret as sha256(topic || random0 || random1 || our_private*their_public)
epk = self.__cryptokey.get_epk(topic, 'encrypt_keys')
pks = [pk[2]]
eks = self.__cryptokey.use_epk(topic, 'encrypt_keys',pks)
ek = eks[0]
eks[0] = epk
random0 = pk[3]
random1 = pysodium.randombytes(self.__randombytes)
ss = pysodium.crypto_hash_sha256(topic.encode('utf-8') + random0 + random1 + ek)[0:pysodium.crypto_secretbox_KEYBYTES]
nonce = pysodium.randombytes(pysodium.crypto_secretbox_NONCEBYTES)
# encrypt keys and key indexes (MAC appended, nonce prepended)
msg = []
for i in range(0,len(keyidxs)):
msg.append(keyidxs[i])
msg.append(keys[i])
msg = msgpack.packb(msg, use_bin_type=True)
msg = nonce + pysodium.crypto_secretbox(msg,nonce,ss)
# this is then put in a msgpack array with the appropriate max_age, poison, and public key(s)
poison = msgpack.packb([['topics',[topic]],['usages',['key-encrypt']]], use_bin_type=True)
msg = msgpack.packb([time()+self.__maxage,poison,eks,[random0,random1],msg], use_bin_type=True)
# and signed with our signing key
msg = self.__cryptokey.sign_spk(msg)
# and finally put as last member of a msgpacked array chaining to ROT
with self.__spk_chain_lock:
tchain = self.__spk_chain.copy()
if (len(tchain) == 0):
poison = msgpack.packb([['topics',[topic]],['usages',['key-encrypt']],['pathlen',1]], use_bin_type=True)
lastcert = msgpack.packb([time()+self.__maxage,poison,self.__cryptokey.get_spk()], use_bin_type=True)
_,tempsk = pysodium.crypto_sign_seed_keypair(unhexlify(b'4c194f7de97c67626cc43fbdaf93dffbc4735352b37370072697d44254e1bc6c'))
tchain.append(pysodium.crypto_sign(lastcert,tempsk))
provision = msgpack.packb([msgpack.packb([0,b'\x90',self.__cryptokey.get_spk()]),self.__cryptokey.sign_spk(lastcert)], use_bin_type=True)
self._logger.warning("Current signing chain is empty. Use %s to provision access and then remove temporary root of trust from allowedlist.", provision.hex())
tchain.append(msg)
msg = msgpack.packb(tchain, use_bin_type=True)
except Exception as e:
self._logger.warning("".join(format_exception_shim(e)))
return None
return msg
def __update_spk_chain(self, newchain):
#
# We have a new candidate chain to replace the current one (if any). We
# first must check it is a valid chain ending in our signing public key.
#
if (newchain is None):
return None
try:
with self.__allowdenylist_lock:
pk,pkprint = process_chain(newchain,None,None,allowlist=self.__allowlist,denylist=self.__denylist)
if (len(pk) < 3 or pk[2] != self.__cryptokey.get_spk()):
if not (pkprint is None):
raise ProcessChainError("New chain does not match current signing publickey:", pkprint)
else:
raise ValueError("New chain does not match current signing public key,")
# If we get here, it is a valid chain. So now we need
# to see if it is "superior" than our current chain.
# This means a larger minimum max_age.
min_max_age = 0
with self.__spk_chain_lock:
for cpk in self.__spk_chain:
if (min_max_age == 0 or cpk[0]<min_max_age):
min_max_age = cpk[0]
if (pk[0] < min_max_age):
raise ProcessChainError("New chain has shorter expiry time than current chain.", pkprint)
self.__spk_chain = msgpack.unpackb(newchain,raw=True)
self._logger.warning("Utilizing new chain: %s", str(pkprint))
# Check if we are capable of direct key requests (default is "no")
with self.__spk_chain_lock:
self.__spk_direct_request = False
self._logger.debug("Defaulting new chain to not handle direct requests.")
try:
with self.__allowdenylist_lock:
pk,_ = process_chain(newchain,None,'key-encrypt-request',allowlist=self.__allowlist,denylist=self.__denylist)
if len(pk) >= 3:
with self.__spk_chain_lock:
self._logger.info(" New chain supports direct key requests. Enabling.")
self.__spk_direct_request = True
except Exception as e:
# exceptions when checking direct mean it is not supported
with self.__spk_chain_lock:
self._logger.info(" New chain does not support direct key requests. Disabling.")
self.__spk_direct_request = False
return newchain
except Exception as e:
self._logger.warning("".join(format_exception_shim(e)))
return None
def decrypt_keys(self, topic, msgval=None):
if (isinstance(topic,(bytes,bytearray))):
self._logger.debug("passed a topic in bytes (should be string)")
topic = topic.decode('utf-8')
#
# msgval should be a msgpacked chain.
# The public key in the array is a set of public key(s) to combine with our
# encryption key to get the secret to decrypt the key. If we do not
# have an encryption key for the topic, we cannot do it until we have one.
# The next item is then the pair of random values for generating the shared
# secret, followed by the actual key message.
#
try:
with self.__allowdenylist_lock:
pk,pkprint = process_chain(msgval,topic,'key-encrypt',allowlist=self.__allowlist,denylist=self.__denylist)
if (len(pk) < 5):
if not (pkprint is None):
raise ProcessChainError("Unexpected number of chain elements:", pkprint)
else:
raise ValueError("Unexpected number of chain elements!")
random0 = pk[3][0]
random1 = pk[3][1]
nonce = pk[4][0:pysodium.crypto_secretbox_NONCEBYTES]
msg = pk[4][pysodium.crypto_secretbox_NONCEBYTES:]
eks = self.__cryptokey.use_epk(topic, 'decrypt_keys', pk[2], clear=False)
for ck in eks:
# Construct candidate shared secrets as sha256(topic || random0 || random1 || our_private*their_public)
ss = pysodium.crypto_hash_sha256(topic.encode('utf-8') + random0 + random1 + ck)[0:pysodium.crypto_secretbox_KEYBYTES]
# decrypt and return key
try:
msg = msgpack.unpackb(pysodium.crypto_secretbox_open(msg,nonce,ss),raw=True)
rvs = {}
for i in range(0,len(msg),2):
rvs[msg[i]] = msg[i+1]
if len(rvs) < 1 or 2*len(rvs) != len(msg):
raise ValueError
except:
pass
else:
# clear the esk/epk we just used
self.__cryptokey.use_epk(topic, 'decrypt_keys', [])
return rvs
raise ValueError
except Exception as e:
self._logger.warning("".join(format_exception_shim(e)))
pass
return None
def add_denylist(self, deny):
self.__allowdenylist_lock.acquire()
try:
pk,_ = process_chain(deny,None,'key-denylist',allowlist=self.__allowlist,denylist=self.__denylist)
if (len(pk) >= 4):
apk = msgpack.unpackb(pk[3],raw=True)
if pk[2] != apk[2]:
self._logger.info("Mismatch in keys for denylist %s vs %s.",pk[2].hex(),apk[2].hex())
raise ValueError("Mismatch in keys for denylist.")
if key_in_list(pk[3],self.__denylist)==None:
self.__denylist.append(pk[3])
else:
self._logger.info("Key %s already in denylist",pk[2].hex())
raise ValueError("Key already in denylist!")
self._logger.warning("Added key %s to denylist",pk[2].hex())
return pk[3]
except Exception as e:
self._logger.warning("".join(format_exception_shim(e)))
return None
finally:
self.__allowdenylist_lock.release()
def _chain_server(self):
while True:
self._logger.info("Checking Key Lifetimes")
chainkeys = self._cryptostore.load_section('chainkeys',
defaults=False)
for ck in chainkeys.keys():
cv = msgpack.unpackb(chainkeys[ck], raw=True)
self._logger.info("Checking Key: %s", cv[2])
if cv[0] < time() + self._lifetime * self._refresh_fraction:
try:
# Time to renew this key
self._logger.warning("Key expires soon, renewing %s",
cv)
msg = msgpack.packb(
[time() + self._lifetime, cv[1], cv[2]],
use_bin_type=True)
chain = self._cryptokey.sign_spk(msg)
chain = msgpack.packb(
msgpack.unpackb(self._our_chain, raw=False) +
[chain],
use_bin_type=True)
# Validate
pk, _ = process_chain(chain,
None,
None,
allowlist=self._allowlist)
# Broadcast
self._producer.send('chains', key=cv[2], value=chain)
self._producer.flush(timeout=self._flush_time)
# save
self._cryptostore.store_value(ck,
msg,
section='chainkeys')
except Exception as e:
self._logger.warning("".join(format_exception_shim(e)))
self._logger.info("Done Checking.")
sleep(self._interval_secs)
def signed_epk(self, topic, epk=None):
if (isinstance(topic,(bytes,bytearray))):
self._logger.debug("passed a topic in bytes (should be string)")
topic = topic.decode('utf-8')
#
# returns the public key of a current or given ephemeral key for the specified topic
# (generating a new one if not present), signed by our signing key,
# with a fresh random value, and with the chain to the ROT prepended.
#
try:
if epk is None:
epk = self.__cryptokey.get_epk(topic,'decrypt_keys')
random0 = pysodium.randombytes(self.__randombytes)
# we allow either direct-to-producer or via-controller key establishment
poison = msgpack.packb([['topics',[topic]],['usages',['key-encrypt-request','key-encrypt-subscribe']]], use_bin_type=True)
msg = msgpack.packb([time()+self.__maxage,poison,epk,random0], use_bin_type=True)
# and signed with our signing key
msg = self.__cryptokey.sign_spk(msg)
# and finally put as last member of a msgpacked array chaining to ROT
with self.__spk_chain_lock:
tchain = self.__spk_chain.copy()
if (len(tchain) == 0):
# Use default for direct use when empty.
self.__spk_direct_request = True
poison = msgpack.packb([['topics',[topic]],['usages',['key-encrypt-request','key-encrypt-subscribe']],['pathlen',1]], use_bin_type=True)
lastcert = msgpack.packb([time()+self.__maxage,poison,self.__cryptokey.get_spk()], use_bin_type=True)
_,tempsk = pysodium.crypto_sign_seed_keypair(unhexlify(b'4c194f7de97c67626cc43fbdaf93dffbc4735352b37370072697d44254e1bc6c'))
tchain.append(pysodium.crypto_sign(lastcert,tempsk))
provision = msgpack.packb([msgpack.packb([0,b'\x90',self.__cryptokey.get_spk()]),self.__cryptokey.sign_spk(lastcert)], use_bin_type=True)
self._logger.warning("Current signing chain is empty. Use %s to provision access and then remove temporary root of trust from allowedlist.", provision.hex())
tchain.append(msg)
msg = msgpack.packb(tchain, use_bin_type=True)
return msg
except Exception as e:
self._logger.warning("".join(format_exception_shim(e)))
pass
return None
def deserialize(self, topic, bytes_):
if (isinstance(topic,(bytes,bytearray))):
self._parent._logger.debug("passed a topic in bytes (should be string)")
topic = topic.decode('utf-8')
if bytes_ is None:
return None
root = self._parent.get_root(topic)
if len(bytes_) < 1 or bytes_[0] != 1:
return KafkaCryptoMessage.fromBytes(bytes_,deser=self,topic=topic)
try:
msg = msgpack.unpackb(bytes_[1:],raw=True)
if (len(msg) != 3):
raise KafkaCryptoSerializeError("Malformed Message!")
except Exception as e:
self._parent._logger.debug("".join(format_exception_shim(e)))
return KafkaCryptoMessage.fromBytes(bytes_,deser=self,topic=topic)
ki = msg[0]
salt = msg[1]
msg = msg[2]
self._parent._lock.acquire()
i = 1
initial_waiter = False
try:
while ((not (root in self._parent._cgens.keys()) or not (ki in self._parent._cgens[root].keys())) and i>0):
if (not (root in self._parent._cwaits.keys()) or not (ki in self._parent._cwaits[root].keys())):
self._parent._logger.debug("Setting initial wait for root=%s, key index=%s.", root, ki)
initial_waiter = True
# first time we see a topic/key index pair, we wait the initial interval for key exchange
i = self._parent.DESER_INITIAL_WAIT_INTERVALS
if not (root in self._parent._cwaits.keys()):
self._parent._cwaits[root] = {}
self._parent._cwaits[root][ki] = i
elif initial_waiter:
self._parent._cwaits[root][ki] -= 1
elif (root in self._parent._cwaits.keys()) and (ki in self._parent._cwaits[root].keys()):
i = self._parent._cwaits[root][ki]
else:
i = self.MAX_WAIT_INTERVALS
ntp = (root+self._parent.TOPIC_SUFFIX_KEYS)
if not (ntp in self._parent._tps):
self._parent._tps[ntp] = TopicPartition(ntp,0)
self._parent._tps_offsets[ntp] = 0
self._parent._tps_updated = True
else:
if not (root in self._parent._subs_needed):
self._parent._subs_needed.append(root)
i=i-1
if (i > 0):
self._parent._lock.release()
sleep(self.WAIT_INTERVAL)
self._parent._lock.acquire()
if not (root in self._parent._cgens.keys()) or not (ki in self._parent._cgens[root].keys()):
self._parent._logger.debug("No decryption key found for root=%s, key index=%s. Returning encrypted message.", root, ki)
raise ValueError
gen = self._parent._cgens[root][ki][self._kv]
key,nonce = gen.generate(salt=salt)
msg = KafkaCryptoMessage(pysodium.crypto_secretbox_open(msg,nonce,key),ipt=True)
except:
return KafkaCryptoMessage.fromBytes(bytes_,deser=self,topic=topic)
finally:
self._parent._lock.release()
return msg
def _process_mgmt_messages(self):
while True:
# First, process messages
# we are the only thread ever using _kc, _kp, so we do not need the lock to use them
msgs = self._kc.poll(timeout_ms=self.MGMT_POLL_INTERVAL, max_records=self.MGMT_POLL_RECORDS)
# but to actually process messages, we need the lock
for tp,msgset in msgs.items():
self._lock.acquire()
for msg in msgset:
topic = msg.topic
if (isinstance(topic,(bytes,bytearray))):
self._logger.debug("passed a topic in bytes (should be string)")
topic = topic.decode('utf-8')
self._tps_offsets[topic] = msg.offset+1
self._logger.debug("Processing message: %s", msg)
if topic[-len(self.TOPIC_SUFFIX_REQS):] == self.TOPIC_SUFFIX_REQS:
root = topic[:-len(self.TOPIC_SUFFIX_REQS)]
# A new receiver: send all requested keys
try:
kreq = msgpack.unpackb(msg.key,raw=True)
if root in self._pgens.keys():
ki = []
s = []
for ski,sk in self._pgens[root].items():
if ski in kreq:
ki.append(ski)
s.append(sk['secret'])
if len(ki) > 0:
k = msgpack.packb(ki, use_bin_type=True)
v = self._cryptoexchange.encrypt_keys(ki, s, root, msgval=msg.value)
if not (v is None):
self._logger.info("Sending current encryption keys for root=%s to new receiver, msgkey=%s.", root, k)
self._kp.send(root + self.TOPIC_SUFFIX_KEYS, key=k, value=v)
else:
self._logger.info("Failed sending current encryption keys for root=%s to new receiver.", root)
else:
self._logger.info("No keys for root=%s to send to new receiver.", root)
except Exception as e:
self._parent._logger.warning("".join(format_exception_shim(e)))
elif topic[-len(self.TOPIC_SUFFIX_KEYS):] == self.TOPIC_SUFFIX_KEYS:
root = topic[:-len(self.TOPIC_SUFFIX_KEYS)]
# New key(s)
nks = self._cryptoexchange.decrypt_keys(root,msgval=msg.value)
if not (nks is None):
for nki,nk in nks.items():
self._logger.info("Received new encryption key for root=%s, key index=%s, msgkey=%s", root, nki, msg.key)
# do not clopper other keys that may exist
if not (root in self._cgens.keys()):
self._cgens[root] = {}
# but do clobber the same topic,keyID entry
self._cgens[root][nki] = {}
self._cgens[root][nki]['key'], self._cgens[root][nki]['value'] = KeyGenerator.get_key_value_generators(nk)
self._cgens[root][nki]['secret'] = nk
# now that we have this key index, clear from request lists
if root in self._cwaits.keys():
self._cwaits[root].pop(nki, None)
if root in self._subs_last:
eki = set(self._subs_last[root][1])
eki.difference_update(set(nks.keys()))
if len(eki) > 0:
self._logger.warning("For root=%s, the keys %s were requested but not received.", root, eki)
self._subs_last.pop(root,None)
elif topic == self.MGMT_TOPIC_CHAINS:
# New candidate public key chain
self._logger.info("Received new chain message: %s", msg)
if msg.key == self._cryptokey.get_spk():
self._logger.debug("Key matches ours. Validating Chain.")
newchain = self._cryptoexchange.replace_spk_chain(msg.value)
if not (newchain is None):
self._logger.info("New chain is superior, using it.")
self._cryptostore.store_value('chain',newchain,section='crypto')
elif topic == self.MGMT_TOPIC_ALLOWLIST:
self._logger.info("Received new allowlist message: %s", msg)
allow = self._cryptoexchange.add_allowlist(msg.value)
if not (allow is None):
c = pysodium.crypto_hash_sha256(allow)
self._cryptostore.store_value(c,allow,section='allowlist')
elif topic == self.MGMT_TOPIC_DENYLIST:
self._logger.info("Received new denylist message: %s", msg)
deny = self._cryptoexchange.add_denylist(msg.value)
if not (deny is None):
c = pysodium.crypto_hash_sha256(deny)
self._cryptostore.store_value(c,deny,section='denylist')
else:
# unknown object
log_limited(self._logger.warning, "Unknown topic type in message: %s", msg)
self._lock.release()
# Flush producer
try:
self._kp.flush(timeout=self.MGMT_FLUSH_TIME)
except Exception as e:
self._parent._logger.warning("".join(format_exception_shim(e)))
# Second, deal with subscription changes
self._lock.acquire()
self._logger.debug("Processing subscription changes.")
if self._tps_updated == True:
self._logger.debug("Subscriptions changed, adjusting.")
tpo = []
for tk in self._tps.keys():
tpo.append(TopicPartitionOffset(self._tps[tk].topic, self._tps[tk].partition, self._tps_offsets[tk]))
self._kc.assign_and_seek(tpo)
self._logger.debug("Subscriptions adjusted.")
if (self._kc.config['group_id'] is None):
self._logger.info("No group_id, seeking to beginning.")
self._kc.seek_to_beginning()
for tk in self._tps.keys():
if (tk[-len(self.TOPIC_SUFFIX_KEYS):] == self.TOPIC_SUFFIX_KEYS):
root = tk[:-len(self.TOPIC_SUFFIX_KEYS)]
if not (root in self._subs_needed):
self._subs_needed.append(root)
self._tps_updated = False
self._lock.release()
# Third, deal with topics needing subscriptions
self._lock.acquire()
self._logger.debug("Process subscriptions.")
subs_needed_next = []
for root in self._subs_needed:
if root in self._cwaits.keys():
self._logger.debug("Attempting (Re)subscribe to root=%s", root)
kis = list(self._cwaits[root].keys())
if len(kis) > 0:
if (not (root in self._subs_last.keys()) or self._subs_last[root][0]+self.CRYPTO_SUB_INTERVAL<time()):
k = msgpack.packb(kis, use_bin_type=True)
v = self._cryptoexchange.signed_epk(root)
if not (k is None) and not (v is None):
self._logger.info("Sending new subscribe request for root=%s, msgkey=%s", root, k)
self._kp.send(root + self.TOPIC_SUFFIX_SUBS, key=k, value=v)
if self._cryptoexchange.direct_request_spk_chain():
# if it may succeed, send directly as well
self._kp.send(root + self.TOPIC_SUFFIX_REQS, key=k, value=v)
self._subs_last[root] = [time(),kis]
else:
self._logger.info("Failed to send new subscribe request for root=%s", root)
subs_needed_next.append(root)
else:
self._logger.debug("Deferring (re)subscriptions for root=%s due to pending key request.", root)
subs_needed_next.append(root)
else:
self._logger.debug("No new keys needed for root=%s", root)
self._subs_needed = subs_needed_next
self._lock.release()
# Flush producer
try:
self._kp.flush(timeout=self.MGMT_FLUSH_TIME)
except Exception as e:
self._parent._logger.warning("".join(format_exception_shim(e)))
# Fourth, periodically increment ratchet and prune old keys
self._logger.debug("Checking ratchet time.")
self._lock.acquire()
if self._last_time+self.CRYPTO_RATCHET_INTERVAL < time():
self._logger.info("Periodic ratchet increment.")
self._last_time = time()
# prune
for root,pgens in self._pgens.items():
rki = []
for ki,kv in pgens.items():
if (kv['birth']+self.CRYPTO_MAX_PGEN_AGE<time()):
rki.append(ki)
for ki in rki:
self._pgens[root].pop(ki)
# increment
self._cur_pgens = {}
self._seed.increment()
self._lock.release()
# Fifth, write new producer keys if requested
self._logger.debug("Checking producer keys.")
self._lock.acquire()
if self._new_pgens:
self._logger.info("(Re)writing producer keys.")
self._new_pgens = False
kvs = {}
kvs['pgens'] = []
for root,pgens in self._pgens.items():
for ki,kv in pgens.items():
kvs['pgens'].append([root,ki,kv['secret'],kv['birth']])
# stored pgens do not have generators as they should never be used for active production
# (but secret stays around so lost consumers can catch up)
self._logger.info("Saving %s old production keys.", len(kvs['pgens']))
self._cryptostore.store_opaque_value('oldkeys',msgpack.packb(kvs, use_bin_type=True),section="crypto")
self._lock.release()
def _process_mgmt_messages(self):
while True:
# First, (Re)subscribe if needed
if ((time()-self._last_subscribed_time) >= self.MGMT_SUBSCRIBE_INTERVAL):
self._logger.debug("Initiating resubscribe...")
trx = "(.*\\" + self.TOPIC_SUFFIX_SUBS + "$)"
self._kc.subscribe(topics=[self.MGMT_TOPIC_CHAINS,self.MGMT_TOPIC_ALLOWLIST,self.MGMT_TOPIC_DENYLIST],pattern=trx)
self._last_subscribed_time = time()
self._logger.info("Resubscribed to topics.")
# Second, process messages
# we are the only thread ever using _kc, _kp, so we do not need the lock to use them
self._logger.debug("Initiating poll...")
msgs = self._kc.poll(timeout_ms=self.MGMT_POLL_INTERVAL, max_records=self.MGMT_POLL_RECORDS)
self._logger.debug("Poll complete with %i msgsets.", len(msgs))
# but to actually process messages, we need the lock
for tp,msgset in msgs.items():
self._logger.debug("Topic %s Partition %i has %i messages", tp.topic, tp.partition, len(msgset))
self._lock.acquire()
for msg in msgset:
self._logger.debug("Processing message: %s", msg)
topic = msg.topic
if (isinstance(topic,(bytes,bytearray))):
self._logger.debug("topic provided as bytes instead of string")
topic = topic.decode('utf-8')
if topic[-len(self.TOPIC_SUFFIX_SUBS):] == self.TOPIC_SUFFIX_SUBS:
root = topic[:-len(self.TOPIC_SUFFIX_SUBS)]
self._logger.debug("Processing subscribe message, root=%s, msgkey=%s", root, msg.key)
# New consumer encryption key. Validate
k,v = self._provisioners.reencrypt_request(root, cryptoexchange=self._cryptoexchange, msgkey=msg.key, msgval=msg.value)
# Valid request, resign and retransmit
if (not (k is None)) or (not (v is None)):
self._logger.info("Valid consumer key request on topic=%s, root=%s, msgkey=%s. Resending to topic=%s, msgkey=%s", topic, root, msg.key, root + self.TOPIC_SUFFIX_REQS, k)
self._kp.send(root + self.TOPIC_SUFFIX_REQS, key=k, value=v)
else:
self._logger.info("Invalid consumer key request on topic=%s, root=%s in message: %s", topic, root, msg)
elif topic == self.MGMT_TOPIC_CHAINS:
# New candidate public key chain
self._logger.info("Received new chain message: %s", msg)
if msg.key == self._cryptokey.get_spk():
self._logger.debug("Key matches ours. Validating Chain.")
newchain = self._cryptoexchange.replace_spk_chain(msg.value)
if not (newchain is None):
self._logger.info("New chain is superior, using it.")
self._cryptostore.store_value('chain',newchain,section='crypto')
elif topic == self.MGMT_TOPIC_ALLOWLIST:
self._logger.info("Received new allowlist message: %s", msg)
allow = self._cryptoexchange.add_allowlist(msg.value)
if not (allow is None):
c = pysodium.crypto_hash_sha256(allow)
self._cryptostore.store_value(c,allow,section='allowlist')
elif topic == self.MGMT_TOPIC_DENYLIST:
self._logger.info("Received new denylist message: %s", msg)
deny = self._cryptoexchange.add_denylist(msg.value)
if not (deny is None):
c = pysodium.crypto_hash_sha256(deny)
self._cryptostore.store_value(c,deny,section='denylist')
else:
# unknown object
log_limited(self._logger.warning, "Unknown topic type in message: %s", msg)
self._lock.release()
# Third, flush producer
try:
self._kp.flush(timeout=self.MGMT_FLUSH_TIME)
except Exception as e:
self._logger.warning("".join(format_exception_shim(e)))
# Fourth, commit offsets
if (self._kc.config['group_id'] is not None):
self._kc.commit()
