def EventMsg(data, name=None):
"""Format a Sever-Sent-Event message.
:param data: message data, will be JSON-encoded.
:param name: (optional) name of the event type.
:rtype: str
"""
# We need to serialize the message data to a string. SSE doesn't say that
# we must use JSON for that, but it's a convenient choice.
# print("sending sse data: %s" % data)
if isinstance(data, int) or isinstance(data, float):
jsonData = data
else:
try:
jsonData = json.dumps(data)
# print("sending sse jsonData1: %s" % jsonData)
# assert '\n' not in jsonData
except Exception as e:
jsonData = unicode_to_bytes(data)
# print("sending sse jsonData2: %s" % jsonData)
# assert '\n' not in jsonData
# Newlines make SSE messages slightly more complicated. Fortunately for us,
# we don't have any in the messages we're using.
if name:
output = 'event: %s\n' % (name,)
else:
output = ''
output += 'data: %s\n\n' % (jsonData,)
return unicode_to_bytes(output)
def WebBroadcastMsg(data, name=None):
"""
Format a Sever-Sent-Event message.
:param data: message data, will be JSON-encoded.
:param name: (optional) name of the event type.
:rtype: str
"""
if isinstance(data, int) or isinstance(data, float):
jsonData = data
else:
try:
jsonData = json.dumps(data)
# print("sending sse jsonData1: %s" % jsonData)
# assert '\n' not in jsonData
except Exception as e:
jsonData = unicode_to_bytes(data)
# print("sending sse jsonData2: %s" % jsonData)
# assert '\n' not in jsonData
if name:
output = f"event: {name}\n"
else:
output = ""
output += f"data: {jsonData}\n\n"
return unicode_to_bytes(output)
def decrypt(self,
data,
passphrase: Optional[str] = None,
cipher: Optional[str] = None):
"""
Decrypt something, defaults aes256.
Example: data = self._Encryption.decrypt("x831jsd91je", cipher="aes256")
Ciphers available:
aes128
aes192
aes256
:param data: Any type of data can be encrypted. Text, binary.
:param passphrase: A passphrase to use. If missing, uses system passphrase.
:return: String containing the encrypted content.
"""
if data is None:
return None
data = unicode_to_bytes(data)
cipher = self.validate_cipher(cipher)
if passphrase is None:
passphrase = self.__aes_key
if cipher.startswith("aes"):
results = self.decrypt_aes(data, passphrase, cipher)
return results
def register_exchange_queue_binding(self, exchange_name: str, queue_name: str, routing_key: str,
register_persist: bool):
"""
Request a new exchange->queue binding. Simply adds to the queue to be processed when the
connection is fully established.
:param exchange_name:
:param queue_name:
:param routing_key:
:param register_persist:
:return:
"""
logger.debug("Factory:register_exchange_queue_binding - exchange_name:{exchange_name}, queue_name:{queue_name}",
exchange_name=exchange_name, queue_name=queue_name)
eqb_name = sha256(unicode_to_bytes(exchange_name + queue_name + routing_key)).hexdigest()
if eqb_name in self.exchange_queue_bindings:
raise YomboWarning(
f"AMQP Protocol:{self.AMQPClient.client_id} - Already has an exchange-queue-routing key binding: "
f"{exchange_name}-{queue_name}-{routing_key}", 200, "register_exchange", "AMQPFactory")
self.exchange_queue_bindings[exchange_name+queue_name] = {
"registered": False,
"queued": False,
"exchange_name": exchange_name,
"queue_name": queue_name,
"routing_key": routing_key,
"register_persist": register_persist,
}
self.check_registrations("binding")
def generate_csr(self, args):
"""
This function shouldn't be called directly. Instead, use the queue
"self.generate_csr_queue.put(request, callback, callback_args)" or
"self._SSLCerts.generate_csr_queue.put()".
Requests certs to be made. Will return right away with a request ID. A callback can be set to return
the cert once it's complete.
:return:
"""
logger.info("Generate_CSR called with args: {args}", args=args)
kwargs = self.check_csr_input(args)
if kwargs['key_type'] == 'rsa':
kwargs['key_type'] = crypto.TYPE_RSA
else:
kwargs['key_type'] = crypto.TYPE_DSA
gwid = "gw_%s" % self.gateway_id[0:10]
req = crypto.X509Req()
req.get_subject().CN = kwargs['cn']
req.get_subject().countryName = 'US'
req.get_subject().stateOrProvinceName = 'California'
req.get_subject().localityName = 'Sacramento'
req.get_subject().organizationName = 'Yombo'
req.get_subject().organizationalUnitName = gwid
# Appends SAN to have 'DNS:'
if kwargs['sans'] is not None:
san_string = []
for i in kwargs['sans']:
san_string.append("DNS: %s" % i)
san_string = ", ".join(san_string)
x509_extensions = [
crypto.X509Extension(b'subjectAltName', False,
unicode_to_bytes(san_string))
]
req.add_extensions(x509_extensions)
key = yield threads.deferToThread(
self._generate_key, **{
'key_type': kwargs['key_type'],
'key_size': kwargs['key_size']
})
req.set_pubkey(key)
req.sign(key, "sha256")
csr = crypto.dump_certificate_request(crypto.FILETYPE_PEM, req)
key_file = crypto.dump_privatekey(crypto.FILETYPE_PEM, key)
if kwargs['csr_file'] is not None:
save_file(kwargs['csr_file'], csr)
if kwargs['key_file'] is not None:
save_file(kwargs['key_file'], key_file)
return {'csr': csr, 'key': key_file}
def encrypt_aes(cls, data, passphrase, cipher):
"""
Encrypt data with passphrase. Currently, only supporting aes 156.
"""
passphrase = cls._aes_expand_key(passphrase, cipher)
iv = Random.new().read(AES.block_size)
aescipher = AES.new(
passphrase, AES.MODE_CBC,
iv) # Create a AES cipher object with the key using the mode CBC
raw = unicode_to_bytes(data)
ciphered_data = aescipher.encrypt(aespad(raw, AES.block_size))
return aescipher.iv + ciphered_data
def import_trust(self, trust_string):
"""
Sets the trust of a key.
#TODO: This function is blocking! Adjust to non-blocking. See below.
"""
# print("SEtting trust level: %s" % trust_string)
p = yield Popen([f"gpg --import-ownertrust --homedir {self.working_dir}/etc/gpg"], shell=True,
stdin=PIPE, stdout=PIPE, close_fds=True)
(child_stdout, child_stdin) = (p.stdout, p.stdin)
child_stdin.write(unicode_to_bytes(f"{trust_string}\n"))
child_stdin.close()
result = child_stdout.read()
logger.info("GPG Trust change: {result}", result=result)
def display_encrypted(self, in_text):
"""
Makes an input field friend version of an input. If encrypted, returns
"-----ENCRYPTED DATA-----", otherwise returns the text unchanged.
:param in_text:
:return:
"""
in_text_search = unicode_to_bytes(in_text)
if in_text_search.startswith(b"-----BEGIN PGP MESSAGE-----"):
return "-----ENCRYPTED DATA-----"
else:
return in_text
def _aes_expand_key(passphrase, cipher):
"""
Internal function to standardize the key size. Uses sha256 to generate a hash from the key, and then trims
as needed.
:param passphrase: passphrase to use.
:param cipher: 16, 24, 32
:return:
"""
cipher = int(cipher[3:])
passphrase = hashlib.sha256(unicode_to_bytes(passphrase)).digest()
if cipher == 128:
passphrase = passphrase[:16]
elif cipher == 192:
passphrase = passphrase[:24]
# Remaining is 256, or 32 bytes.
return passphrase
def sha512_compact(value, encoder: Optional[str] = None, convert_to_unicode: Optional[bool] = True):
"""
Returns a shorter sha512 - 86 characters long instead of 128.
This uses a base62 encoding which uses the entire alphabet, with mixed case.
Returned length is 86 characters.
:param value:
:return:
"""
if encoder is None:
encoder = "base62"
if convert_to_unicode is None:
convert_to_unicode = True
if value is None:
return None
return encode_binary(sha512(unicode_to_bytes(value)).digest(), encoder, convert_to_unicode)
def render_encode_output(self, request, content):
"""
Checks the request headers for the encoding string. If it's gzip, it will be compressed.
:param request:
:param content:
:return:
"""
if content is not None and len(
content) > 500: # don't bother compressing small data
response_content_encoding_headers = b','.join(
request.responseHeaders.getRawHeaders(b'content-encoding', []))
if bool(gzipCheckRegex.search(
response_content_encoding_headers)) is False:
request_accept_encoding_headers = b','.join(
request.requestHeaders.getRawHeaders(
b'accept-encoding', []))
if gzipCheckRegex.search(request_accept_encoding_headers):
encoding = request.responseHeaders.getRawHeaders(
b'content-encoding')
if encoding:
encoding = b','.join(encoding + [b'gzip'])
else:
encoding = b'gzip'
request.responseHeaders.setRawHeaders(
b'content-encoding', [encoding])
content = gzip.compress(unicode_to_bytes(content))
# print(f"render_encode_output: content type: {type(content)}")
# print(f"REO: {request.responseHeaders._rawHeaders}")
if content is None:
request.responseHeaders.setRawHeaders(b'content-length', [str(0)])
else:
request.responseHeaders.setRawHeaders(b'content-length',
[str(len(content))])
# print(f"REO: {request.responseHeaders._rawHeaders}")
return content
def sha512_crypt_mosquitto(password, salt_base64=None, rounds=None):
"""
Used for generating a crypted version for mosquitto pasword file.
:param password:
:param salt_base64:
:param rounds:
:return:
"""
if salt_base64 is None:
rand = random.SystemRandom()
salt_base64 = ''.join([
rand.choice(string.ascii_letters + string.digits)
for _ in range(16)
])
salt = base64.b64decode(salt_base64)
m = hashlib.sha512()
m.update(unicode_to_bytes(password))
m.update(salt)
hashed_password = m.digest()
encoded_password = bytes_to_unicode(base64.b64encode(hashed_password))
return '$6$' + salt_base64 + "$" + encoded_password
def update_attributes(self, attributes):
"""
Update various attributes. Should only be used by the SQLCerts system when loading updated things from
a library or module.
The attributes have already been screened by the parent.
:param attributes:
:return:
"""
if "update_callback" in attributes:
self.update_callback = attributes["update_callback"]
if "update_callback_type" in attributes:
self.update_callback_type = attributes["update_callback_type"]
if "update_callback_component" in attributes:
self.update_callback_component = attributes["update_callback_component"]
if "update_callback_function" in attributes:
self.update_callback_function = attributes["update_callback_function"]
if "key_size" in attributes:
self.key_size = int(attributes["key_size"]) if attributes["key_size"] else None
if "key_type" in attributes:
self.key_type = attributes["key_type"]
if "current_cert" in attributes:
self.current_cert = attributes["current_cert"]
if "current_chain" in attributes:
self.current_chain = attributes["current_chain"]
if "current_key" in attributes:
self.current_key = attributes["current_key"]
self.set_crypto()
if "current_status" in attributes:
self.current_status = attributes["current_status"]
if "current_status_msg" in attributes:
self.current_status_msg = attributes["current_status_msg"]
if "current_csr_hash" in attributes:
self.current_csr_hash = attributes["current_csr_hash"]
if "current_created_at" in attributes:
self.current_created_at = int(attributes["current_created_at"]) if attributes["current_created_at"] else None
if "current_expires_at" in attributes:
self.current_expires_at = int(attributes["current_expires_at"]) if attributes["current_expires_at"] else None
if "current_signed_at" in attributes:
self.current_signed_at = int(attributes["current_signed_at"]) if attributes["current_signed_at"] else None
if "current_submitted_at" in attributes:
self.current_submitted_at = int(attributes["current_submitted_at"]) if attributes["current_submitted_at"] else None
if "current_fqdn" in attributes:
self.current_fqdn = attributes["current_fqdn"]
if "current_is_valid" in attributes:
self.current_is_valid = attributes["current_is_valid"]
if "next_status" in attributes:
self.next_status = attributes["next_status"]
if "next_status_msg" in attributes:
self.next_status_msg = attributes["next_status_msg"]
if "next_csr_hash" in attributes:
self.next_csr_hash = attributes["next_csr_hash"]
if "next_csr" in attributes:
self.next_csr = attributes["next_csr"]
if "next_cert" in attributes:
self.next_cert = attributes["next_cert"]
if "next_chain" in attributes:
self.next_chain = attributes["next_chain"]
if "next_key" in attributes:
self.next_key = attributes["next_key"]
if "next_created_at" in attributes:
self.next_created_at = int(attributes["next_created_at"]) if attributes["next_created_at"] else None
if "next_expires_at" in attributes:
self.next_expires_at = int(attributes["next_expires_at"]) if attributes["next_expires_at"] else None
if "next_signed_at" in attributes:
self.next_signed_at = int(attributes["next_signed_at"]) if attributes["next_signed_at"] else None
if "next_submitted_at" in attributes:
self.next_submitted_at = int(attributes["next_submitted_at"]) if attributes["next_submitted_at"] else None
if "next_fqdn" in attributes:
self.next_fqdn = attributes["next_fqdn"]
if "next_is_valid" in attributes:
self.next_is_valid = attributes["next_is_valid"]
# do some back checks.
if self.next_csr is not None and self.next_csr_hash is None:
self.next_csr_hash = sha256_compact(unicode_to_bytes(self.next_csr))
self.dirty = True
def page_restore_details(webinterface, request, session):
"""
Prompts user to upload the configuration file to restore the gateway.
:param webinterface:
:param request:
:param session:
:return:
"""
try:
restorefile = request.args.get("restorefile")[0]
try:
restorefile = json.loads(restorefile)
logger.info("Received configuration backup file.")
try:
if restorefile["hash"] != hashlib.sha256(
unicode_to_bytes(
restorefile["data"])).hexdigest():
webinterface.add_alert(
"Backup file appears to be corrupt: Invalid checksum."
)
return webinterface.redirect("/setup_wizard/start")
restorefile["data"] = base64.b64decode(
unicode_to_bytes(restorefile["data"]))
except Exception as e:
logger.warn("Unable to b64decode data: {e}", e=e)
webinterface.add_alert(
"Unable to properly decode pass 1 of data segment of restore file."
)
return webinterface.redirect("/setup_wizard/start")
session.set("restore_backup_file", restorefile)
except Exception as e:
logger.warn("Unable to parse JSON phase 2: {e}", e=e)
webinterface.add_alert("Invalid restore file contents.")
return webinterface.redirect("/setup_wizard/start")
except Exception:
restorefile = session.get("restore_backup_file", None)
if restorefile is None:
webinterface.add_alert("No restore file found.")
return webinterface.redirect("/setup_wizard/start")
required_keys = ("encrypted", "time", "file_type", "created",
"backup_version")
if all(required in restorefile
for required in required_keys) is False:
webinterface.add_alert(
"Backup file appears to be missing important parts.")
return webinterface.redirect("/setup_wizard/start")
if restorefile["encrypted"] is True:
try:
password = request.args.get("password", )[0]
except Exception:
password = session.get("restorepassword", None)
if password is None:
page = webinterface.get_template(
request, webinterface.wi_dir +
"/pages/setup_wizard/restore_password.html")
return page.render(alerts=session.get_alerts(),
restore=restorefile)
try:
decrypted = yield webinterface._Tools.data_unpickle(
restorefile["data"],
"msgpack_aes256_zip_base85",
passphrase=password,
)
restorefile["data_processed"] = json.loads(
bytes_to_unicode(decrypted))
except Exception as e:
logger.warn("Unable to decrypt restoration file: {e}", e=e)
webinterface.add_alert(
"It appears the password is incorrect.", "danger")
page = webinterface.get_template(
request, webinterface.wi_dir +
"/pages/setup_wizard/restore_password.html")
return page.render(alerts=session.get_alerts(),
restore=restorefile)
else: #no password
restorefile["data_processed"] = json.loads(
bytes_to_unicode(restorefile["data"]))
session.set("restore_backup_file", restorefile)
page = webinterface.get_template(
request,
webinterface.wi_dir + "/pages/restore/restore_ready.html")
return page.render(alerts=session.get_alerts(),
restore=session.get("restore_backup_file"))
def data_unpickle(cls,
data: Any,
content_type: Optional[str] = None,
passphrase: Optional[str] = None,
use_decimal: Optional[bool] = None):
"""
Unpack data packed with data_pickle. See data_pickle() for content_type options.
:param data:
:param content_type:
:param passphrase: Passphrase will be sent to the encryption function.
:return:
"""
if data is None:
return None
data = bytes_to_unicode(data)
if content_type is None:
content_type = "msgpack_base85"
elif content_type == "string":
return str(data)
elif content_type == "bool":
return bool(data)
content_type = content_type.lower()
# Sometimes empty dictionaries are encoded... This is a simple shortcut.
if content_type == "msgpack_base85_zip" and data == "cwTD&004mifd":
return {}
elif content_type == "msgpack_base85" and data == "fB":
return {}
if "json" in content_type and "msgack" in content_type:
raise YomboWarning(
"Unpickle data can only have json or msgpack, not both.")
encoder_count = 0
for encoder_check in ("base32", "base62", "base64", "base85"):
if encoder_check in content_type:
encoder_count += 1
if encoder_count > 1:
raise YomboWarning(
"Pickle data can only one of: base32, base62, base64 or base85, not multiple."
)
if "base32" in content_type:
data = base64.b32decode(data)
if "base62" in content_type:
data = base62.decodebytes(data)
if "base64" in content_type:
data = data + "=" * (-len(data) % 4)
data = base64.b64decode(data)
elif "base85" in content_type:
data = base64.b85decode(data)
for cipher in AVAILABLE_CIPHERS:
if cipher in content_type:
try:
print(f"tools.unpickle, about to decrypt: {data}")
data = cls._Encryption.decrypt_aes(data,
passphrase=passphrase,
cipher=cipher)
print(f"tools.unpickle, about to decrypt, done: {data}")
except Exception:
break
if "lz4" in content_type:
try:
data = lz4.frame.decompress(data, passphrase=passphrase)
except Exception as e:
raise YomboWarning(f"Error lz4 decompress {content_type}: {e}")
elif "gzip" in content_type or "zip" in content_type:
try:
data = zlib.decompress(data)
except Exception as e:
raise YomboWarning(
f"Error zlib decompress {content_type}: {e}")
try:
data = zlib.decompress(data)
except Exception as e:
pass
if isinstance(data, str) or isinstance(data, bytes):
if "json" in content_type:
data = unicode_to_bytes(data)
try:
data = bytes_to_unicode(
json.loads(data, use_decimal=use_decimal))
except Exception as e:
raise YomboWarning(
f"data_unpickle received non-json content: {data}")
elif "msgpack" in content_type:
data = unicode_to_bytes(data)
try:
data = bytes_to_unicode(msgpack.unpackb(data))
except Exception as e:
raise YomboWarning(
f"data_unpickle received non-msgpack content: {data}")
return data
def data_pickle(cls,
data: Any,
content_type: Optional[str] = None,
compression_level: Optional[int] = None,
local: Optional[bool] = None,
passphrase: Optional[str] = None,
use_decimal: Optional[bool] = None) -> Union[str, bytes]:
"""
Encodes data based on value of the content_type type. The default is "msgpack_base85". This easily allows data
to be sent externally or even the database. The ordering of the content_type does not matter.
Format of the content_type:
{pickle-method}_{compress method}_{encode-method}.
Pickle methods:
* msgpack - Use msgpack to pickle the data. More space efficient than json.
* json - Use JSON to pickle the data.
Compression can also be applied, simply append:
* lz4 - Use the lz4 compression
* zip - Use the gzip compression
Encryption is also supported:
* aes256, aes192, aes128
Encode methods:
* base32 - Encode with base32 - Biggest size.
* base62 - Encode with base62 - Like base64, but URL safe.
* base64 - Encode with base64
* base85 - Encode with base85
Examples:
* msgpack_zip - Pickle with msgpack, then compress with gzip.
* zip_msgpack - Same as above, just different order.
* msgpack_base85_zip - Pickle with msgpack, then compress with gzip, then encode with base85
* msgpack_zip_aes256_base85 - Pickle with msgpack, compress with gzip, encrypt with aes, then encode with base85
* json_lz4 - Pickle with json, then compress with z-standard.
Non-encoded results typically return bytes, while encoded results return strings.
:param data: String, list, or dictionary to be encoded.
:param content_type: Optional encode method.
:param compression_level: Sets a compression level - default depends on compressor.
:param passphrase: Passphrase will be sent to the encryption function.
:return: bytes of the encoded data that can be used with data_unpickle.
"""
if data is None:
return None
if content_type is None:
content_type = "msgpack_base85"
elif content_type == "string":
return str(data)
elif content_type == "bool":
return bool(data)
content_type = content_type.lower()
if "json" in content_type and "msgack" in content_type:
raise YomboWarning(
"Pickle data can only have json or msgpack, not both.")
encoder_count = 0
for encoder_check in ("base32", "base62", "base64", "base85"):
if encoder_check in content_type:
encoder_count += 1
if encoder_count > 1:
raise YomboWarning(
"Pickle data can only one of: base32, base62, base64 or base85, not multiple."
)
if "json" in content_type:
try:
data = json.dumps(data,
separators=(",", ":"),
use_decimal=use_decimal)
except Exception as e:
raise YomboWarning(f"Error encoding json: {e}")
elif "msgpack" in content_type:
try:
data = msgpack.packb(data)
except Exception as e:
raise YomboWarning(f"Error encoding msgpack: {e}")
if "lz4" in content_type:
if compression_level is None:
compression_level = 2
elif compression_level > 1 or compression_level < 9:
compression_level = 2
try:
data = lz4.frame.compress(unicode_to_bytes(data),
compression_level)
except Exception as e:
raise YomboWarning(f"Error encoding {content_type}: {e}")
elif "gzip" in content_type or "zip" in content_type:
if compression_level is None:
compression_level = 5
elif compression_level > 1 or compression_level < 9:
compression_level = 5
try:
data = zlib.compress(unicode_to_bytes(data), compression_level)
except Exception as e:
raise YomboWarning(f"Error encoding {content_type}: {e}")
for cipher in AVAILABLE_CIPHERS:
if cipher in content_type:
try:
data = cls._Encryption.encrypt_aes(data,
passphrase=passphrase,
cipher=cipher)
except Exception:
break
if "base32" in content_type:
data = bytes_to_unicode(base64.b32encode(data))
if local is True:
data = data.rstrip("=")
elif "base62" in content_type:
data = bytes_to_unicode(base62.encodebytes(data))
if local is True:
data = data.rstrip("=")
elif "base64" in content_type:
data = bytes_to_unicode(base64.b64encode(data))
if local is True:
data = data.rstrip("=")
elif "base85" in content_type:
data = bytes_to_unicode(base64.b85encode(data))
return data
def generate_csr(self, args):
"""
This function shouldn't be called directly. Instead, use the queue
"self.generate_csr_queue.put(request, callback, callback_args)" or
"self._SSLCerts.generate_csr_queue.put()".
Requests certs to be made. Will return right away with a request ID. A callback can be set to return
the cert once it's complete.
:return:
"""
logger.debug("Generate_CSR called with args: {args}", args=args)
kwargs = self.check_csr_input(args)
if kwargs["key_type"] == "rsa":
kwargs["key_type"] = crypto.TYPE_RSA
else:
kwargs["key_type"] = crypto.TYPE_DSA
req = crypto.X509Req()
req.get_subject().CN = kwargs["cn"]
req.get_subject().countryName = "US"
req.get_subject().stateOrProvinceName = "California"
req.get_subject().localityName = "Sacramento"
req.get_subject().organizationName = "Yombo"
req.get_subject(
).organizationalUnitName = "Gateway " + self.gateway_id[0:15]
# Appends SAN to have "DNS:"
if kwargs["sans"] is not None:
san_string = []
for i in kwargs["sans"]:
san_string.append(f"DNS: {i}")
san_string = ", ".join(san_string)
x509_extensions = [
crypto.X509Extension(b"subjectAltName", False,
unicode_to_bytes(san_string))
]
req.add_extensions(x509_extensions)
start = time()
key = yield threads.deferToThread(
self._generate_key, **{
"key_type": kwargs["key_type"],
"key_size": kwargs["key_size"]
})
duration = round(float(time()) - start, 4)
self._Events.new("sslcerts", "generate_new",
(args["sslname"], kwargs["cn"], san_string, duration))
req.set_pubkey(key)
req.sign(key, "sha256")
csr = crypto.dump_certificate_request(crypto.FILETYPE_PEM, req)
key_file = crypto.dump_privatekey(crypto.FILETYPE_PEM, key)
if kwargs["csr_file"] is not None:
yield save_file(kwargs["csr_file"], csr)
if kwargs["key_file"] is not None:
yield save_file(kwargs["key_file"], key_file)
return {
"csr": csr,
"csr_hash": sha256_compact(unicode_to_bytes(csr)),
"key": key_file
}