def _validate_tls_cert_config(self, config):
ca_cert = config.get('tls_ca_cert', None)
if ca_cert:
try:
x509.load_pem_x509_certificate(ca_cert.encode('ascii'),
crypto_backend())
except Exception as e:
raise ValidationError(
"'tls_ca_cert' must include the certificate in PEM format: \"%s\""
% e)
client_cert = config.get('tls_client_cert', None)
if client_cert:
cert_data = client_cert.encode('ascii')
try:
x509.load_pem_x509_certificate(cert_data, crypto_backend())
except Exception as e:
raise ValidationError(
"'tls_client_cert' must include the certificate in PEM format: \"%s\""
% e)
try:
load_pem_private_key(cert_data,
password=None,
backend=crypto_backend())
except Exception as e:
raise ValidationError(
"'tls_client_cert' must include a PEM-encoded private key: \"%s\""
% e)
def build_signature_verifier(cls, public_key_binary, signature):
public_key = crypto_serial.load_ssh_public_key(
public_key_binary, crypto_backend()
)
return public_key.verifier(
signature,
crypto_padding.PKCS1v15(),
crypto_hashes.SHA512()
)
def make_hash(self, binary):
sha256 = crypto_hashes.Hash(
crypto_hashes.SHA256(), backend=crypto_backend()
)
sha256.update(binary)
digest_binary = sha256.finalize()
hex_binary = codecs.encode(digest_binary, self.HEX_ENCODING)
return hex_binary.decode(self.ASCII_ENCODING)
def _validate_tls_cert_config(self, config):
ca_cert = config.get('tls_ca_cert', None)
if ca_cert:
try:
x509.load_pem_x509_certificate(ca_cert.encode('ascii'), crypto_backend())
except Exception as e:
raise ValidationError("'tls_ca_cert' must include the certificate in PEM format: \"%s\"" % e)
client_cert = config.get('tls_client_cert', None)
if client_cert:
cert_data = client_cert.encode('ascii')
try:
x509.load_pem_x509_certificate(cert_data, crypto_backend())
except Exception as e:
raise ValidationError("'tls_client_cert' must include the certificate in PEM format: \"%s\"" % e)
try:
load_pem_private_key(cert_data, password=None, backend=crypto_backend())
except Exception as e:
raise ValidationError("'tls_client_cert' must include a PEM-encoded private key: \"%s\"" % e)
def _sign_text(self, text, private_key_binary):
private_key = crypto_serial.load_pem_private_key(
private_key_binary, password=None, backend=crypto_backend()
)
signer = private_key.signer(
crypto_padding.PKCS1v15(), crypto_hashes.SHA512()
)
signer.update(text.encode(self.UTF8_ENCODING))
return signer.finalize()
def _key_from_password(cls,
password: str,
salt: bytes,
*,
length=_ENC_PBKDF2_LEN_BYTES,
iterations=_ENC_PBKDF2_ITERS) -> bytes:
return PBKDF2HMAC(algorithm=cls._ENC_PBKDF2_HASH,
length=length,
salt=salt,
iterations=iterations,
backend=crypto_backend()).derive(
password.encode('UTF-8'))
def _make_key_pair(self):
private_key = crypto_rsa.generate_private_key(
self.PUBLIC_EXPONENT, self.KEY_SIZE, crypto_backend()
)
private_key_binary = private_key.private_bytes(
encoding=crypto_serial.Encoding.PEM,
format=crypto_serial.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=crypto_serial.NoEncryption()
)
public_key_binary = PublicKeySshCodec().encode(
private_key.public_key()
)
pair = KeyPair(
private=private_key_binary,
public=public_key_binary
)
return pair
def __init__(self, key_info):
"""Creates a verifier that uses a PEM-encoded RSA public key.
Args:
- key_info: KeyInfo protobuf message
Raises:
- PemError: If the key has an invalid encoding
- UnsupportedAlgorithmError: If the key uses an unsupported algorithm
"""
if (key_info.type != client_pb2.KeyInfo.RSA):
raise error.UnsupportedAlgorithmError(
"Expected RSA key, but got key type %d" % key_info.type)
pem_key = str(key_info.pem_key)
try:
self.__key = crypto_backend().load_pem_public_key(pem_key)
except ValueError as e:
raise pem.PemError(e)
except cryptography.exceptions.UnsupportedAlgorithm as e:
raise error.UnsupportedAlgorithmError(e)
def __init__(self, key_info):
"""Creates a verifier that uses a PEM-encoded ECDSA public key.
Args:
- key_info: KeyInfo protobuf message
Raises:
- PemError: If the key has an invalid encoding
- UnsupportedAlgorithmError: If the key uses an unsupported algorithm
"""
if (key_info.type != client_pb2.KeyInfo.ECDSA):
raise error.UnsupportedAlgorithmError(
"Expected ECDSA key, but got key type %d" % key_info.type)
pem_key = str(key_info.pem_key)
try:
self.__key = crypto_backend().load_pem_public_key(pem_key)
except ValueError as e:
raise pem.PemError(e)
except cryptography.exceptions.UnsupportedAlgorithm as e:
raise error.UnsupportedAlgorithmError(e)
def update(self, data):
"""Decrypt cipher text
Cipher text must be passed to this function in the order in
which it was output from the encryption.update function.
data:
(A portion of) the cipher text to be decrypted. data
value has to be contained in a bytes, bytearray or memoryview object.
returns:
any plain text produced by the call
"""
if not isinstance(data, (bytes, bytearray, memoryview)):
raise RuntimeError(
"Data must be bytes, bytearray, or memoryview objects")
#
# each encryption has a header on it that identifies the algorithm
# used and an encryption of the data key that was used to encrypt
# the original plain text. there is no guarantee how much of that
# data will be passed to this function or how many times this
# function will be called to process all of the data. to that end,
# this function buffers data internally, when it is unable to
# process it.
#
# the function buffers data internally until the entire header is
# received. once the header has been received, the encrypted data
# key is sent to the server for decryption. after the header has
# been successfully handled, this function always decrypts all of
# the data in its internal buffer *except* for however many bytes
# are specified by the algorithm's tag size. see the end() function
# for details.
#
self._buf += data
pt = b''
# if there is no key or 'dec' member of key, then the code
# is still trying to build a complete header
if not hasattr(self, '_key') or not 'dec' in self._key:
fmt = '!BBBBH'
fmtlen = struct.calcsize(fmt)
# does the buffer contain enough of the header to
# determine the lengths of the initialization vector
# and the key?
if len(self._buf) >= fmtlen:
ver, flags, alg, veclen, keylen = struct.unpack(
fmt, self._buf[:fmtlen])
# For VER 0, lsb of indicates AAD or not
if (ver != 0) or (flags & ~algorithm.UBIQ_HEADER_V0_FLAG_AAD):
raise RuntimeError('invalid encryption header')
# does the buffer contain the entire header?
if len(self._buf) >= fmtlen + veclen + keylen:
# Get the Header for AAD purposes. Only needed if
# version != 0, but get it now anyways
aad = self._buf[:fmtlen + veclen + keylen]
# extract the initialization vector and the key
vec = self._buf[fmtlen:fmtlen + veclen]
key = self._buf[fmtlen + veclen:fmtlen + veclen + keylen]
# remove the header from the buffer
self._buf = self._buf[fmtlen + veclen + keylen:]
# generate a local identifier for the key
sha = crypto.hashes.Hash(crypto.hashes.SHA256(),
backend=crypto_backend())
sha.update(key)
client_id = sha.finalize()
# if the object already has a key (from a previous
# decryption), is the key in this header the same as
# that previous one?
#
# if not, clear out the existing key
if hasattr(self, '_key'):
if self._key['client_id'] != client_id:
self.reset()
# if the object (still) has a key, then it can be
# reused--see below. if not, then request a decryption
# of the key in the current header from the server
if not hasattr(self, '_key'):
url = self._endpoint_base() + '/decryption/key'
response = requests.post(
url,
data=json.dumps({
'encrypted_data_key':
base64.b64encode(key).decode('utf-8')
}).encode('utf-8'),
auth=http_auth(self._papi, self._sapi))
if response.status_code != http.HTTPStatus.OK:
raise urllib.error.HTTPError(
url, response.status_code,
http.HTTPStatus(response.status_code).phrase,
response.headers, response.content)
content = json.loads(response.content.decode('utf-8'))
self._key = {}
self._key['algo'] = algorithm(alg)
# the client's id for recognizing key reuse
self._key['client_id'] = client_id
# the server's id for sending updates
self._key['finger_print'] = content['key_fingerprint']
self._key['session'] = content['encryption_session']
# decrypt the client's private key (sent
# by the server)
prvkey = crypto.serialization.load_pem_private_key(
content['encrypted_private_key'].encode('utf-8'),
self._srsa.encode('utf-8'), crypto_backend())
# use the private key to decrypt the data key
self._key['raw'] = prvkey.decrypt(
base64.b64decode(content['wrapped_data_key']),
crypto.asymmetric.padding.OAEP(
mgf=crypto.asymmetric.padding.MGF1(
algorithm=crypto.hashes.SHA1()),
algorithm=crypto.hashes.SHA1(),
label=None))
# this key hasn't been used (yet)
self._key['uses'] = 0
# if the key object exists, create a new decryptor
# with the initialization vector from the header and
# the decrypted key (which is either new from the
# server or cached from the previous decryption). in
# either case, increment the key usage
if hasattr(self, '_key'):
self._key['dec'] = self._key['algo'].decryptor(
self._key['raw'], vec)
self._key['uses'] += 1
if (flags & algorithm.UBIQ_HEADER_V0_FLAG_AAD):
self._key['dec'].authenticate_additional_data(aad)
# if the object has a key and a decryptor, then decrypt whatever
# data is in the buffer, less any data that needs to be saved to
# serve as the tag.
if hasattr(self, '_key') and 'dec' in self._key:
sz = len(self._buf) - self._key['algo'].len['tag']
if sz > 0:
pt = self._key['dec'].update(self._buf[:sz])
self._buf = self._buf[sz:]
return pt
class Archive:
VERSION = 0x0
_COMP_MODE = 0x1
_ENC_MODE = 0x1
_HEADER_FORMAT = '!BBB'
_CRYPTO_BACKEND = crypto_backend()
_PUBKEY_SIZE_BYTES = 32
_IV_SIZE_BYTES = 16
_CIPHER = AES
_CIPHER_KEY_SIZE_BYTES = 32
_CIPHER_IV_SIZE_BYTES = 12
_AUTH_TAG_SIZE_BYTES = 16
_KEY_DERIVATION_HASH = hashes.SHA256
def __init__(self, cfg: ArchiveConfig, *, chunk_size=DEFAULT_CHUNK_SIZE):
self._comp_cfg = cfg.compression
self._enc_cfg = cfg.encryption
self._chunk_size = chunk_size
self._enc_pubkey = (self._load_pubkey(self._enc_cfg.public_key)
if self._enc_cfg else None)
def store_file(self, file_or_path) -> typing.Iterable[bytes]:
if isinstance(file_or_path, io.BufferedIOBase):
yield from self._store_file(file_or_path)
else:
with open(file_or_path, 'rb') as fobj:
yield from self._store_file(fobj)
@classmethod
def unarchive(cls,
file_or_path,
*,
input_size: typing.Optional[int] = None,
enc_privkey: typing.Optional[x25519.X25519PrivateKey] = None,
chunk_size=DEFAULT_CHUNK_SIZE) -> typing.Iterable[bytes]:
if isinstance(file_or_path, io.BufferedIOBase):
yield from cls._unarchive(file_or_path, input_size, enc_privkey,
chunk_size)
else:
with open(file_or_path, 'rb') as fobj:
yield from cls._unarchive(fobj, input_size, enc_privkey,
chunk_size)
def _store_file(self, fobj: io.BufferedIOBase) -> typing.Iterable[bytes]:
need_compression = self._comp_cfg is not None
orig_size = None
if need_compression and fobj.seekable():
orig_size = self._get_size_till_eof(fobj)
if orig_size < self._comp_cfg.min_size:
need_compression = False
need_encryption = self._enc_cfg is not None
yield self._make_header(comp=need_compression, enc=need_encryption)
chunks = self._iter_file_chunk(fobj, self._chunk_size)
if need_compression:
chunks = self._compress(chunks, orig_size)
if need_encryption:
chunks = self._encrypt(chunks)
yield from chunks
@classmethod
def _unarchive(cls, fobj: io.BufferedIOBase,
input_size: typing.Optional[int],
enc_privkey: typing.Optional[x25519.X25519PrivateKey],
chunk_size: int) -> typing.Iterable[bytes]:
header = cls._parse_header(cls._read_exact(fobj, 3))
if not header:
raise RuntimeError('invalid archive')
compression, encryption = header
if encryption:
if enc_privkey is None:
raise RuntimeError('enc_privkey is required')
if input_size is not None:
input_size -= 3 # the header
chunks = cls._decrypt(fobj, input_size, enc_privkey, chunk_size)
else:
chunks = cls._iter_file_chunk(fobj, chunk_size)
if compression:
chunks = cls._decompress(chunks)
yield from chunks
@classmethod
def _iter_file_chunk(cls, fobj: io.BufferedIOBase,
chunk_size: int) -> typing.Iterable[bytes]:
while True:
data = fobj.read(chunk_size)
if len(data) == 0:
break
yield data
def _compress(self,
data: typing.Iterable[bytes],
orig_size=None) -> typing.Iterable[bytes]:
with LZ4FrameCompressor(auto_flush=True) as comp:
yield comp.begin(orig_size or 0)
for chunk in data:
yield comp.compress(chunk)
yield comp.flush()
@classmethod
def _decompress(cls,
data: typing.Iterable[bytes]) -> typing.Iterable[bytes]:
with LZ4FrameDecompressor() as decomp:
for chunk in data:
if not chunk:
break
yield decomp.decompress(chunk)
assert decomp.eof
def _encrypt(self, data: typing.Iterable[bytes]) -> typing.Iterable[bytes]:
eprikey = x25519.X25519PrivateKey.generate()
epubkey_bytes = eprikey.public_key().public_bytes(
encoding=serialization.Encoding.Raw,
format=serialization.PublicFormat.Raw)
yield epubkey_bytes
assert self._PUBKEY_SIZE_BYTES == len(epubkey_bytes)
iv = os.urandom(self._IV_SIZE_BYTES)
yield iv
shared_secret = eprikey.exchange(self._enc_pubkey)
cipher_key = self._derive_keys(shared_secret, iv)
cipher_iv = iv[:self._CIPHER_IV_SIZE_BYTES]
encryptor = Cipher(algorithm=self._CIPHER(cipher_key),
mode=GCM(cipher_iv),
backend=self._CRYPTO_BACKEND).encryptor()
for chunk in data:
yield encryptor.update(chunk)
yield encryptor.finalize()
yield encryptor.tag
@classmethod
def _decrypt(cls, fobj: io.BufferedIOBase,
input_size: typing.Optional[int],
privkey: x25519.X25519PrivateKey,
chunk_size: int) -> typing.Iterable[bytes]:
if input_size is None:
input_size = cls._get_size_till_eof(fobj)
epubkey = x25519.X25519PublicKey.from_public_bytes(
cls._read_exact(fobj, cls._PUBKEY_SIZE_BYTES))
input_size -= cls._PUBKEY_SIZE_BYTES
shared_secret = privkey.exchange(epubkey)
iv = cls._read_exact(fobj, cls._IV_SIZE_BYTES)
input_size -= cls._IV_SIZE_BYTES
cipher_key = cls._derive_keys(shared_secret, iv)
cipher_iv = iv[:cls._CIPHER_IV_SIZE_BYTES]
decryptor = Cipher(algorithm=cls._CIPHER(cipher_key),
mode=GCM(cipher_iv),
backend=cls._CRYPTO_BACKEND).decryptor()
if input_size < cls._AUTH_TAG_SIZE_BYTES:
raise RuntimeError('input_size is too short')
bufmv = memoryview(bytearray(chunk_size))
auth_tag = b''
while input_size > 0:
bytes_read = fobj.readinto(bufmv)
if bytes_read == 0:
break
input_size -= bytes_read
if input_size <= cls._AUTH_TAG_SIZE_BYTES:
auth_tag_part_len = cls._AUTH_TAG_SIZE_BYTES - input_size
auth_tag += bufmv[bytes_read - auth_tag_part_len:bytes_read]
auth_tag += fobj.read()
yield decryptor.update(bufmv[:bytes_read - auth_tag_part_len])
else:
yield decryptor.update(bufmv[:bytes_read])
yield decryptor.finalize_with_tag(auth_tag)
@classmethod
def _make_header(cls, *, comp: bool, enc: bool) -> bytes:
return struct.pack(cls._HEADER_FORMAT, cls.VERSION,
cls._COMP_MODE if comp else 0,
cls._ENC_MODE if enc else 0)
@classmethod
def _parse_header(
cls, header: bytes) -> typing.Optional[typing.Tuple[bool, bool]]:
ver, comp, enc = struct.unpack(cls._HEADER_FORMAT, header)
if ver != cls.VERSION:
return None
return comp, enc
@classmethod
def _load_pubkey(cls, pubkey_file) -> x25519.X25519PublicKey:
with open(pubkey_file, 'rb') as fobj:
b = base64.standard_b64decode(fobj.read())
return x25519.X25519PublicKey.from_public_bytes(b)
@classmethod
def _derive_keys(cls, shared_secret, iv) -> bytes:
return HKDF(algorithm=cls._KEY_DERIVATION_HASH(),
length=cls._CIPHER_KEY_SIZE_BYTES,
salt=iv,
info=None,
backend=cls._CRYPTO_BACKEND).derive(shared_secret)
@staticmethod
def _get_size_till_eof(fobj: io.IOBase) -> int:
start = fobj.tell()
fobj.seek(0, io.SEEK_END)
end = fobj.tell()
fobj.seek(start)
return end - start
@staticmethod
def _read_exact(fobj: io.BufferedIOBase, size: int) -> bytes:
buf = bytearray(size)
bufmv = memoryview(buf)
while size > 0:
bytes_read = fobj.readinto(bufmv[-size:])
if bytes_read == 0:
raise RuntimeError('EOF occurred')
size -= bytes_read
return bytes(buf)
def _load_public_key(self, public_key_binary):
return crypto_serial.load_ssh_public_key(
public_key_binary, crypto_backend()
)
def __init__(self, creds, uses):
"""Initialize the encryption object
papi:
The client's public API key (used to identify the
client to the server)
sapi:
The client's secret API key (used to authenticate HTTP requests)
srsa:
The client's secret RSA encryption key/password (used to decrypt
the client's RSA key from the server). This key is not retained
by this object.
uses:
The number of separate encryptions the caller wishes to perform
with the key. This number may be limited by the server.
host:
A string of the form 'host[:port]' with the []'s denoting an
optional portion of the string indicating the server to which
to make the request.
"""
# If the host does not begin with either http or https
# insert https://
self._host = creds.host
if (not self._host.lower().startswith('http')):
self._host = "https://" + self._host
self._papi = creds.access_key_id
self._sapi = creds.secret_signing_key
#
# request a new encryption key from the server. if the request
# fails, the function raises a urllib.error.HTTPError indicating
# the status code returned by the server. this exception is
# propagated back to the caller
#
url = self._endpoint_base() + '/encryption/key'
response = requests.post(url,
data=json.dumps({
'uses': uses
}).encode('utf-8'),
auth=http_auth(self._papi, self._sapi))
if response.status_code != http.HTTPStatus.CREATED:
raise urllib.error.HTTPError(
url, response.status_code,
http.HTTPStatus(response.status_code).phrase, response.headers,
response.content)
#
# the code below largely assumes that the server returns
# a json object that contains the members and is formatted
# according to the Ubiq REST specification. if it doesn't
# the code raises an exception about missing keys and those
# exceptions are propagated back to the caller
#
content = json.loads(response.content.decode('utf-8'))
#
# decrypt the client's private key. if the decryption fails,
# the function raises a ValueError which is propagated up.
#
prvkey = crypto.serialization.load_pem_private_key(
content['encrypted_private_key'].encode('utf-8'),
creds.secret_crypto_access_key.encode('utf-8'), crypto_backend())
self._key = {}
self._key['id'] = content['key_fingerprint']
self._key['session'] = content['encryption_session']
self._key['security_model'] = content['security_model']
self._key['algorithm'] = self._key['security_model'][
'algorithm'].lower()
self._key['max_uses'] = content['max_uses']
self._key['uses'] = 0
#
# use the client's private key to decrypt the data key to
# be used for encryption
#
self._key['raw'] = prvkey.decrypt(
base64.b64decode(content['wrapped_data_key']),
crypto.asymmetric.padding.OAEP(mgf=crypto.asymmetric.padding.MGF1(
algorithm=crypto.hashes.SHA1()),
algorithm=crypto.hashes.SHA1(),
label=None))
#
# the service also returns the encryption key encrypted by
# its own master key. this value is attached to each cipher
# text created by this object
#
self._key['encrypted'] = base64.b64decode(
content['encrypted_data_key'])
self._algo = algorithm(self._key['algorithm'])
def __init__(self, private_key_binary):
self._private_key = crypto_serial.load_pem_private_key(
private_key_binary, password=None, backend=crypto_backend()
)