def add_denylist(self, deny):
self.__allowdenylist_lock.acquire()
try:
pk, _ = process_chain(deny,
None,
'key-denylist',
allowlist=self.__allowlist,
denylist=self.__denylist)
if (len(pk) >= 4):
apk = msgpack.unpackb(pk[3], raw=True)
if pk[2] != apk[2]:
self._logger.info(
"Mismatch in keys for denylist %s vs %s.", pk[2].hex(),
apk[2].hex())
raise ValueError("Mismatch in keys for denylist.")
if key_in_list(pk[3], self.__denylist) == None:
self.__denylist.append(pk[3])
else:
self._logger.info("Key %s already in denylist",
pk[2].hex())
raise ValueError("Key already in denylist!")
self._logger.warning("Added key %s to denylist", pk[2].hex())
return pk[3]
except Exception as e:
self._logger.warning("".join(
traceback.format_exception(etype=type(e),
value=e,
tb=e.__traceback__)))
return None
finally:
self.__allowdenylist_lock.release()
def reencrypt_request(self,
topic,
cryptoexchange,
msgkey=None,
msgval=None):
if (isinstance(topic, (bytes, bytearray))):
self._logger.debug("topic provided as bytes (should be string)")
topic = topic.decode('utf-8')
#
# msgval should be a msgpacked chain chaining to a provisioner.
# The last item in the array is the encrypted public key to retransmit.
#
try:
pk = None
try:
pk, pkprint = process_chain(msgval,
topic,
'key-encrypt-subscribe',
allowlist=self.__allowlist,
denylist=self.__denylist)
except ProcessChainError as pce:
raise pce
except:
pass
if (pk is None):
if not (pkprint is None):
raise ProcessChainError("Request did not validate: ",
pkprint)
else:
raise ValueError("Request did not validate!")
msg = cryptoexchange.signed_epk(topic, epk=pk[2])
except Exception as e:
self._logger.warning("".join(format_exception_shim(e)))
return (None, None)
return (msgkey, msg)
def __update_spk_chain(self, newchain):
#
# We have a new candidate chain to replace the current one (if any). We
# first must check it is a valid chain ending in our signing public key.
#
if (newchain is None):
return None
try:
with self.__allowdenylist_lock:
pk,pkprint = process_chain(newchain,None,None,allowlist=self.__allowlist,denylist=self.__denylist)
if (len(pk) < 3 or pk[2] != self.__cryptokey.get_spk()):
if not (pkprint is None):
raise ProcessChainError("New chain does not match current signing publickey:", pkprint)
else:
raise ValueError("New chain does not match current signing public key,")
# If we get here, it is a valid chain. So now we need
# to see if it is "superior" than our current chain.
# This means a larger minimum max_age.
min_max_age = 0
with self.__spk_chain_lock:
for cpk in self.__spk_chain:
if (min_max_age == 0 or cpk[0]<min_max_age):
min_max_age = cpk[0]
if (pk[0] < min_max_age):
raise ProcessChainError("New chain has shorter expiry time than current chain.", pkprint)
self.__spk_chain = msgpack.unpackb(newchain,raw=True)
self._logger.warning("Utilizing new chain: %s", str(pkprint))
# Check if we are capable of direct key requests (default is "no")
with self.__spk_chain_lock:
self.__spk_direct_request = False
self._logger.debug("Defaulting new chain to not handle direct requests.")
try:
with self.__allowdenylist_lock:
pk,_ = process_chain(newchain,None,'key-encrypt-request',allowlist=self.__allowlist,denylist=self.__denylist)
if len(pk) >= 3:
with self.__spk_chain_lock:
self._logger.info(" New chain supports direct key requests. Enabling.")
self.__spk_direct_request = True
except Exception as e:
# exceptions when checking direct mean it is not supported
with self.__spk_chain_lock:
self._logger.info(" New chain does not support direct key requests. Disabling.")
self.__spk_direct_request = False
return newchain
except Exception as e:
self._logger.warning("".join(format_exception_shim(e)))
return None
def encrypt_keys(self, keyidxs, keys, topic, msgval=None):
if (isinstance(topic,(bytes,bytearray))):
self._logger.debug("passed a topic in bytes (should be string)")
topic = topic.decode('utf-8')
#
# msgval should be a msgpacked chain.
# The public key in the array is the public key to send the key to, using a
# common DH-derived key between that public key and our private encryption key.
# Then there is at least one more additional item, random bytes:
# (3) random bytes
# Currently items after this are ignored, and reserved for future use.
#
try:
with self.__allowdenylist_lock:
pk,_ = process_chain(msgval,topic,'key-encrypt-request',allowlist=self.__allowlist,denylist=self.__denylist)
# Construct shared secret as sha256(topic || random0 || random1 || our_private*their_public)
epk = self.__cryptokey.get_epk(topic, 'encrypt_keys')
pks = [pk[2]]
eks = self.__cryptokey.use_epk(topic, 'encrypt_keys',pks)
ek = eks[0]
eks[0] = epk
random0 = pk[3]
random1 = pysodium.randombytes(self.__randombytes)
ss = pysodium.crypto_hash_sha256(topic.encode('utf-8') + random0 + random1 + ek)[0:pysodium.crypto_secretbox_KEYBYTES]
nonce = pysodium.randombytes(pysodium.crypto_secretbox_NONCEBYTES)
# encrypt keys and key indexes (MAC appended, nonce prepended)
msg = []
for i in range(0,len(keyidxs)):
msg.append(keyidxs[i])
msg.append(keys[i])
msg = msgpack.packb(msg, use_bin_type=True)
msg = nonce + pysodium.crypto_secretbox(msg,nonce,ss)
# this is then put in a msgpack array with the appropriate max_age, poison, and public key(s)
poison = msgpack.packb([['topics',[topic]],['usages',['key-encrypt']]], use_bin_type=True)
msg = msgpack.packb([time()+self.__maxage,poison,eks,[random0,random1],msg], use_bin_type=True)
# and signed with our signing key
msg = self.__cryptokey.sign_spk(msg)
# and finally put as last member of a msgpacked array chaining to ROT
with self.__spk_chain_lock:
tchain = self.__spk_chain.copy()
if (len(tchain) == 0):
poison = msgpack.packb([['topics',[topic]],['usages',['key-encrypt']],['pathlen',1]], use_bin_type=True)
lastcert = msgpack.packb([time()+self.__maxage,poison,self.__cryptokey.get_spk()], use_bin_type=True)
_,tempsk = pysodium.crypto_sign_seed_keypair(unhexlify(b'4c194f7de97c67626cc43fbdaf93dffbc4735352b37370072697d44254e1bc6c'))
tchain.append(pysodium.crypto_sign(lastcert,tempsk))
provision = msgpack.packb([msgpack.packb([0,b'\x90',self.__cryptokey.get_spk()]),self.__cryptokey.sign_spk(lastcert)], use_bin_type=True)
self._logger.warning("Current signing chain is empty. Use %s to provision access and then remove temporary root of trust from allowedlist.", provision.hex())
tchain.append(msg)
msg = msgpack.packb(tchain, use_bin_type=True)
except Exception as e:
self._logger.warning("".join(format_exception_shim(e)))
return None
return msg
def decrypt_keys(self, topic, msgval=None):
if (isinstance(topic,(bytes,bytearray))):
self._logger.debug("passed a topic in bytes (should be string)")
topic = topic.decode('utf-8')
#
# msgval should be a msgpacked chain.
# The public key in the array is a set of public key(s) to combine with our
# encryption key to get the secret to decrypt the key. If we do not
# have an encryption key for the topic, we cannot do it until we have one.
# The next item is then the pair of random values for generating the shared
# secret, followed by the actual key message.
#
try:
with self.__allowdenylist_lock:
pk,pkprint = process_chain(msgval,topic,'key-encrypt',allowlist=self.__allowlist,denylist=self.__denylist)
if (len(pk) < 5):
if not (pkprint is None):
raise ProcessChainError("Unexpected number of chain elements:", pkprint)
else:
raise ValueError("Unexpected number of chain elements!")
random0 = pk[3][0]
random1 = pk[3][1]
nonce = pk[4][0:pysodium.crypto_secretbox_NONCEBYTES]
msg = pk[4][pysodium.crypto_secretbox_NONCEBYTES:]
eks = self.__cryptokey.use_epk(topic, 'decrypt_keys', pk[2], clear=False)
for ck in eks:
# Construct candidate shared secrets as sha256(topic || random0 || random1 || our_private*their_public)
ss = pysodium.crypto_hash_sha256(topic.encode('utf-8') + random0 + random1 + ck)[0:pysodium.crypto_secretbox_KEYBYTES]
# decrypt and return key
try:
msg = msgpack.unpackb(pysodium.crypto_secretbox_open(msg,nonce,ss),raw=True)
rvs = {}
for i in range(0,len(msg),2):
rvs[msg[i]] = msg[i+1]
if len(rvs) < 1 or 2*len(rvs) != len(msg):
raise ValueError
except:
pass
else:
# clear the esk/epk we just used
self.__cryptokey.use_epk(topic, 'decrypt_keys', [])
return rvs
raise ValueError
except Exception as e:
self._logger.warning("".join(format_exception_shim(e)))
pass
return None
def __update_spk_chain(self, newchain):
#
# We have a new candidate chain to replace the current one (if any). We
# first must check it is a valid chain ending in our signing public key.
#
if (newchain is None):
return None
try:
with self.__allowdenylist_lock:
pk, pkprint = process_chain(newchain,
None,
None,
allowlist=self.__allowlist,
denylist=self.__denylist)
if (len(pk) < 3 or pk[2] != self.__cryptokey.get_spk()):
if not (pkprint is None):
raise ProcessChainError(
"New chain does not match current signing publickey:",
pkprint)
else:
raise ValueError(
"New chain does not match current signing public key,")
# If we get here, it is a valid chain. So now we need
# to see if it is "superior" than our current chain.
# This means a larger minimum max_age.
with self.__spk_chain_lock:
min_max_age = 0
for cpk in self.__spk_chain:
if (min_max_age == 0 or cpk[0] < min_max_age):
min_max_age = cpk[0]
if (pk[0] < min_max_age):
raise ProcessChainError(
"New chain has shorter expiry time than current chain.",
pkprint)
self.__spk_chain = msgpack.unpackb(newchain, raw=True)
self._logger.warning("Utilizing new chain: %s", str(pkprint))
return newchain
except Exception as e:
self._logger.warning("".join(
traceback.format_exception(etype=type(e),
value=e,
tb=e.__traceback__)))
return None
def _chain_server(self):
while True:
self._logger.info("Checking Key Lifetimes")
chainkeys = self._cryptostore.load_section('chainkeys',
defaults=False)
for ck in chainkeys.keys():
cv = msgpack.unpackb(chainkeys[ck], raw=True)
self._logger.info("Checking Key: %s", cv[2])
if cv[0] < time() + self._lifetime * self._refresh_fraction:
try:
# Time to renew this key
self._logger.warning("Key expires soon, renewing %s",
cv)
msg = msgpack.packb(
[time() + self._lifetime, cv[1], cv[2]],
use_bin_type=True)
chain = self._cryptokey.sign_spk(msg)
chain = msgpack.packb(
msgpack.unpackb(self._our_chain, raw=False) +
[chain],
use_bin_type=True)
# Validate
pk, _ = process_chain(chain,
None,
None,
allowlist=self._allowlist)
# Broadcast
self._producer.send('chains', key=cv[2], value=chain)
self._producer.flush(timeout=self._flush_time)
# save
self._cryptostore.store_value(ck,
msg,
section='chainkeys')
except Exception as e:
self._logger.warning("".join(format_exception_shim(e)))
self._logger.info("Done Checking.")
sleep(self._interval_secs)
password = ''
while len(password) < 12:
password = getpass('Provisioning Password (12+ chars): ')
prov = PasswordProvisioner(password, _rot)
# Check we have appropriate chains
if (choice == 1):
# Controllers must be signed by ROT
_msgchkrot = _msgrot
else:
# Everyone else by the Chain ROT (may = ROT)
_msgchkrot = _msgchainrot
assert (
len(_msgchains[key]) > 0
), 'A trusted chain for ' + key + ' is missing. Use generate-chains.py (and possibly sign with another key), and add to provision.py.'
pk = process_chain(_msgchains[key], None, None, allowlist=[_msgchkrot])
assert (
len(pk) >= 3 and pk[2] == prov._pk[_keys[key]]
), 'Malformed chain for ' + key + '. Did you enter your password correctly and have msgchain rot set appropriately?'
topics = None
while topics is None:
topic = input('Enter a space separated list of topics this ' + key +
' will use: ')
topics = list(set(map(lambda i: i.split('.', 1)[0], topic.split())))
if (len(topics) < 1):
ans = input('Are you sure you want to allow all topics (y/N)?')
if (len(ans) == 0 or ans[0].lower() != 'y'):
topics = None
else:
topics = ['^.*$']
elif (choice == 4 and not limited):
key = 'prodcon'
else:
assert False, 'Invalid combination of choices!'
# Check we have appropriate chains
if (choice == 1):
# Controllers must be signed by ROT
_msgchkrot = _msgrot
else:
# Everyone else by Chain ROT (may = ROT)
_msgchkrot = _msgchainrot
assert (
len(_msgchains[key]) > 0
), 'A trusted chain for ' + key + ' is missing. This should not happen with simple-provision, please report as a bug.'
pk = process_chain(_msgchains[key], None, None, allowlist=[_msgchkrot])[0]
assert (
len(pk) >= 3 and pk[2] == prov._pk[_keys[key]]
), 'Malformed chain for ' + key + '. Did you enter your passwords correctly?'
topics = None
while topics is None:
topic = input('Enter a space separated list of topics this ' + key +
' will use: ')
topics = list(set(map(lambda i: i.split('.', 1)[0], topic.split())))
if (len(topics) < 1):
ans = input('Are you sure you want to allow all topics (Y/n)?')
if len(ans) > 0 and (ans[0].lower() == 'n'):
topics = None
else:
topics = ['^.*$']
def __init__(self, nodeID, config=None, cryptokey=None):
if ((not isinstance(nodeID, (str)) or len(nodeID) < 1)
and (nodeID != None or config is None)):
raise KafkaCryptoChainServerError(
"Node ID " + str(nodeID) + " not a string or not specified!")
if (config is None):
config = nodeID + ".config"
self._logger = logging.getLogger(__name__)
if (hasattr(config, 'load_section')
and inspect.isroutine(config.load_section)
and hasattr(config, 'load_value')
and inspect.isroutine(config.load_value)
and hasattr(config, 'store_value')
and inspect.isroutine(config.store_value)
and hasattr(config, 'load_opaque_value')
and inspect.isroutine(config.load_opaque_value)
and hasattr(config, 'store_opaque_value')
and inspect.isroutine(config.store_opaque_value)
and hasattr(config, 'set_cryptokey')
and inspect.isroutine(config.set_cryptokey)):
self._cryptostore = config
else:
self._cryptostore = CryptoStore(nodeID, config)
nodeID = self._cryptostore.get_nodeID()
self._nodeID = nodeID
if (cryptokey is None):
cryptokey = self._cryptostore.load_value('cryptokey')
if cryptokey.startswith('file#'):
cryptokey = cryptokey[5:]
if (isinstance(cryptokey, (str))):
cryptokey = CryptoKey(file=cryptokey)
if (not hasattr(cryptokey, 'get_spk')
or not inspect.isroutine(cryptokey.get_spk)
or not hasattr(cryptokey, 'sign_spk')
or not inspect.isroutine(cryptokey.sign_spk)
or not hasattr(cryptokey, 'get_epk')
or not inspect.isroutine(cryptokey.get_epk)
or not hasattr(cryptokey, 'use_epk')
or not inspect.isroutine(cryptokey.use_epk)
or not hasattr(cryptokey, 'wrap_opaque')
or not inspect.isroutine(cryptokey.wrap_opaque)
or not hasattr(cryptokey, 'unwrap_opaque')
or not inspect.isroutine(cryptokey.unwrap_opaque)):
raise KafkaCryptoChainServerError(
"Invalid cryptokey source supplied!")
self._cryptokey = cryptokey
self._cryptostore.set_cryptokey(self._cryptokey)
# Load our custom configuration
self._interval_secs = self._cryptostore.load_value('interval_secs',
default=300)
self._lifetime = self._cryptostore.load_value('lifetime',
default=604800)
self._refresh_fraction = self._cryptostore.load_value(
'refresh_fraction', default=0.143)
self._flush_time = self._cryptostore.load_value('flush_time',
default=2.0)
# Load our signing key and trimmings
self._our_chain = self._cryptostore.load_value('chain',
section='crypto')
try:
msgpack.unpackb(self._our_chain, raw=False)
except:
self._logger.warning(
"Chain server chain is in legacy format. This should be corrected."
)
self._our_chain = msgpack.packb(msgpack.unpackb(self._our_chain,
raw=True),
use_bin_type=True)
self._allowlist = self._cryptostore.load_section('allowlist',
defaults=False)
if not (self._allowlist is None):
self._allowlist = self._allowlist.values()
# Validate
pk, pkprint = process_chain(self._our_chain,
None,
None,
allowlist=self._allowlist)
if (pk[2] != self._cryptokey.get_spk()):
raise KafkaCryptoChainServerError(
"Chain does not match public key: " + str(pkprint))
# Connect to Kafka
self._producer = KafkaProducer(
**self._cryptostore.get_kafka_config('producer'))
# Run Thread
self._mgmt_thread = Thread(target=self._chain_server, daemon=True)
self._mgmt_thread.start()
