def EventMsg(data, name=None): """Format a Sever-Sent-Event message. :param data: message data, will be JSON-encoded. :param name: (optional) name of the event type. :rtype: str """ # We need to serialize the message data to a string. SSE doesn't say that # we must use JSON for that, but it's a convenient choice. # print("sending sse data: %s" % data) if isinstance(data, int) or isinstance(data, float): jsonData = data else: try: jsonData = json.dumps(data) # print("sending sse jsonData1: %s" % jsonData) # assert '\n' not in jsonData except Exception as e: jsonData = unicode_to_bytes(data) # print("sending sse jsonData2: %s" % jsonData) # assert '\n' not in jsonData # Newlines make SSE messages slightly more complicated. Fortunately for us, # we don't have any in the messages we're using. if name: output = 'event: %s\n' % (name,) else: output = '' output += 'data: %s\n\n' % (jsonData,) return unicode_to_bytes(output)
def WebBroadcastMsg(data, name=None): """ Format a Sever-Sent-Event message. :param data: message data, will be JSON-encoded. :param name: (optional) name of the event type. :rtype: str """ if isinstance(data, int) or isinstance(data, float): jsonData = data else: try: jsonData = json.dumps(data) # print("sending sse jsonData1: %s" % jsonData) # assert '\n' not in jsonData except Exception as e: jsonData = unicode_to_bytes(data) # print("sending sse jsonData2: %s" % jsonData) # assert '\n' not in jsonData if name: output = f"event: {name}\n" else: output = "" output += f"data: {jsonData}\n\n" return unicode_to_bytes(output)
def decrypt(self, data, passphrase: Optional[str] = None, cipher: Optional[str] = None): """ Decrypt something, defaults aes256. Example: data = self._Encryption.decrypt("x831jsd91je", cipher="aes256") Ciphers available: aes128 aes192 aes256 :param data: Any type of data can be encrypted. Text, binary. :param passphrase: A passphrase to use. If missing, uses system passphrase. :return: String containing the encrypted content. """ if data is None: return None data = unicode_to_bytes(data) cipher = self.validate_cipher(cipher) if passphrase is None: passphrase = self.__aes_key if cipher.startswith("aes"): results = self.decrypt_aes(data, passphrase, cipher) return results
def register_exchange_queue_binding(self, exchange_name: str, queue_name: str, routing_key: str, register_persist: bool): """ Request a new exchange->queue binding. Simply adds to the queue to be processed when the connection is fully established. :param exchange_name: :param queue_name: :param routing_key: :param register_persist: :return: """ logger.debug("Factory:register_exchange_queue_binding - exchange_name:{exchange_name}, queue_name:{queue_name}", exchange_name=exchange_name, queue_name=queue_name) eqb_name = sha256(unicode_to_bytes(exchange_name + queue_name + routing_key)).hexdigest() if eqb_name in self.exchange_queue_bindings: raise YomboWarning( f"AMQP Protocol:{self.AMQPClient.client_id} - Already has an exchange-queue-routing key binding: " f"{exchange_name}-{queue_name}-{routing_key}", 200, "register_exchange", "AMQPFactory") self.exchange_queue_bindings[exchange_name+queue_name] = { "registered": False, "queued": False, "exchange_name": exchange_name, "queue_name": queue_name, "routing_key": routing_key, "register_persist": register_persist, } self.check_registrations("binding")
def generate_csr(self, args): """ This function shouldn't be called directly. Instead, use the queue "self.generate_csr_queue.put(request, callback, callback_args)" or "self._SSLCerts.generate_csr_queue.put()". Requests certs to be made. Will return right away with a request ID. A callback can be set to return the cert once it's complete. :return: """ logger.info("Generate_CSR called with args: {args}", args=args) kwargs = self.check_csr_input(args) if kwargs['key_type'] == 'rsa': kwargs['key_type'] = crypto.TYPE_RSA else: kwargs['key_type'] = crypto.TYPE_DSA gwid = "gw_%s" % self.gateway_id[0:10] req = crypto.X509Req() req.get_subject().CN = kwargs['cn'] req.get_subject().countryName = 'US' req.get_subject().stateOrProvinceName = 'California' req.get_subject().localityName = 'Sacramento' req.get_subject().organizationName = 'Yombo' req.get_subject().organizationalUnitName = gwid # Appends SAN to have 'DNS:' if kwargs['sans'] is not None: san_string = [] for i in kwargs['sans']: san_string.append("DNS: %s" % i) san_string = ", ".join(san_string) x509_extensions = [ crypto.X509Extension(b'subjectAltName', False, unicode_to_bytes(san_string)) ] req.add_extensions(x509_extensions) key = yield threads.deferToThread( self._generate_key, **{ 'key_type': kwargs['key_type'], 'key_size': kwargs['key_size'] }) req.set_pubkey(key) req.sign(key, "sha256") csr = crypto.dump_certificate_request(crypto.FILETYPE_PEM, req) key_file = crypto.dump_privatekey(crypto.FILETYPE_PEM, key) if kwargs['csr_file'] is not None: save_file(kwargs['csr_file'], csr) if kwargs['key_file'] is not None: save_file(kwargs['key_file'], key_file) return {'csr': csr, 'key': key_file}
def encrypt_aes(cls, data, passphrase, cipher): """ Encrypt data with passphrase. Currently, only supporting aes 156. """ passphrase = cls._aes_expand_key(passphrase, cipher) iv = Random.new().read(AES.block_size) aescipher = AES.new( passphrase, AES.MODE_CBC, iv) # Create a AES cipher object with the key using the mode CBC raw = unicode_to_bytes(data) ciphered_data = aescipher.encrypt(aespad(raw, AES.block_size)) return aescipher.iv + ciphered_data
def import_trust(self, trust_string): """ Sets the trust of a key. #TODO: This function is blocking! Adjust to non-blocking. See below. """ # print("SEtting trust level: %s" % trust_string) p = yield Popen([f"gpg --import-ownertrust --homedir {self.working_dir}/etc/gpg"], shell=True, stdin=PIPE, stdout=PIPE, close_fds=True) (child_stdout, child_stdin) = (p.stdout, p.stdin) child_stdin.write(unicode_to_bytes(f"{trust_string}\n")) child_stdin.close() result = child_stdout.read() logger.info("GPG Trust change: {result}", result=result)
def display_encrypted(self, in_text): """ Makes an input field friend version of an input. If encrypted, returns "-----ENCRYPTED DATA-----", otherwise returns the text unchanged. :param in_text: :return: """ in_text_search = unicode_to_bytes(in_text) if in_text_search.startswith(b"-----BEGIN PGP MESSAGE-----"): return "-----ENCRYPTED DATA-----" else: return in_text
def _aes_expand_key(passphrase, cipher): """ Internal function to standardize the key size. Uses sha256 to generate a hash from the key, and then trims as needed. :param passphrase: passphrase to use. :param cipher: 16, 24, 32 :return: """ cipher = int(cipher[3:]) passphrase = hashlib.sha256(unicode_to_bytes(passphrase)).digest() if cipher == 128: passphrase = passphrase[:16] elif cipher == 192: passphrase = passphrase[:24] # Remaining is 256, or 32 bytes. return passphrase
def sha512_compact(value, encoder: Optional[str] = None, convert_to_unicode: Optional[bool] = True): """ Returns a shorter sha512 - 86 characters long instead of 128. This uses a base62 encoding which uses the entire alphabet, with mixed case. Returned length is 86 characters. :param value: :return: """ if encoder is None: encoder = "base62" if convert_to_unicode is None: convert_to_unicode = True if value is None: return None return encode_binary(sha512(unicode_to_bytes(value)).digest(), encoder, convert_to_unicode)
def render_encode_output(self, request, content): """ Checks the request headers for the encoding string. If it's gzip, it will be compressed. :param request: :param content: :return: """ if content is not None and len( content) > 500: # don't bother compressing small data response_content_encoding_headers = b','.join( request.responseHeaders.getRawHeaders(b'content-encoding', [])) if bool(gzipCheckRegex.search( response_content_encoding_headers)) is False: request_accept_encoding_headers = b','.join( request.requestHeaders.getRawHeaders( b'accept-encoding', [])) if gzipCheckRegex.search(request_accept_encoding_headers): encoding = request.responseHeaders.getRawHeaders( b'content-encoding') if encoding: encoding = b','.join(encoding + [b'gzip']) else: encoding = b'gzip' request.responseHeaders.setRawHeaders( b'content-encoding', [encoding]) content = gzip.compress(unicode_to_bytes(content)) # print(f"render_encode_output: content type: {type(content)}") # print(f"REO: {request.responseHeaders._rawHeaders}") if content is None: request.responseHeaders.setRawHeaders(b'content-length', [str(0)]) else: request.responseHeaders.setRawHeaders(b'content-length', [str(len(content))]) # print(f"REO: {request.responseHeaders._rawHeaders}") return content
def sha512_crypt_mosquitto(password, salt_base64=None, rounds=None): """ Used for generating a crypted version for mosquitto pasword file. :param password: :param salt_base64: :param rounds: :return: """ if salt_base64 is None: rand = random.SystemRandom() salt_base64 = ''.join([ rand.choice(string.ascii_letters + string.digits) for _ in range(16) ]) salt = base64.b64decode(salt_base64) m = hashlib.sha512() m.update(unicode_to_bytes(password)) m.update(salt) hashed_password = m.digest() encoded_password = bytes_to_unicode(base64.b64encode(hashed_password)) return '$6$' + salt_base64 + "$" + encoded_password
def update_attributes(self, attributes): """ Update various attributes. Should only be used by the SQLCerts system when loading updated things from a library or module. The attributes have already been screened by the parent. :param attributes: :return: """ if "update_callback" in attributes: self.update_callback = attributes["update_callback"] if "update_callback_type" in attributes: self.update_callback_type = attributes["update_callback_type"] if "update_callback_component" in attributes: self.update_callback_component = attributes["update_callback_component"] if "update_callback_function" in attributes: self.update_callback_function = attributes["update_callback_function"] if "key_size" in attributes: self.key_size = int(attributes["key_size"]) if attributes["key_size"] else None if "key_type" in attributes: self.key_type = attributes["key_type"] if "current_cert" in attributes: self.current_cert = attributes["current_cert"] if "current_chain" in attributes: self.current_chain = attributes["current_chain"] if "current_key" in attributes: self.current_key = attributes["current_key"] self.set_crypto() if "current_status" in attributes: self.current_status = attributes["current_status"] if "current_status_msg" in attributes: self.current_status_msg = attributes["current_status_msg"] if "current_csr_hash" in attributes: self.current_csr_hash = attributes["current_csr_hash"] if "current_created_at" in attributes: self.current_created_at = int(attributes["current_created_at"]) if attributes["current_created_at"] else None if "current_expires_at" in attributes: self.current_expires_at = int(attributes["current_expires_at"]) if attributes["current_expires_at"] else None if "current_signed_at" in attributes: self.current_signed_at = int(attributes["current_signed_at"]) if attributes["current_signed_at"] else None if "current_submitted_at" in attributes: self.current_submitted_at = int(attributes["current_submitted_at"]) if attributes["current_submitted_at"] else None if "current_fqdn" in attributes: self.current_fqdn = attributes["current_fqdn"] if "current_is_valid" in attributes: self.current_is_valid = attributes["current_is_valid"] if "next_status" in attributes: self.next_status = attributes["next_status"] if "next_status_msg" in attributes: self.next_status_msg = attributes["next_status_msg"] if "next_csr_hash" in attributes: self.next_csr_hash = attributes["next_csr_hash"] if "next_csr" in attributes: self.next_csr = attributes["next_csr"] if "next_cert" in attributes: self.next_cert = attributes["next_cert"] if "next_chain" in attributes: self.next_chain = attributes["next_chain"] if "next_key" in attributes: self.next_key = attributes["next_key"] if "next_created_at" in attributes: self.next_created_at = int(attributes["next_created_at"]) if attributes["next_created_at"] else None if "next_expires_at" in attributes: self.next_expires_at = int(attributes["next_expires_at"]) if attributes["next_expires_at"] else None if "next_signed_at" in attributes: self.next_signed_at = int(attributes["next_signed_at"]) if attributes["next_signed_at"] else None if "next_submitted_at" in attributes: self.next_submitted_at = int(attributes["next_submitted_at"]) if attributes["next_submitted_at"] else None if "next_fqdn" in attributes: self.next_fqdn = attributes["next_fqdn"] if "next_is_valid" in attributes: self.next_is_valid = attributes["next_is_valid"] # do some back checks. if self.next_csr is not None and self.next_csr_hash is None: self.next_csr_hash = sha256_compact(unicode_to_bytes(self.next_csr)) self.dirty = True
def page_restore_details(webinterface, request, session): """ Prompts user to upload the configuration file to restore the gateway. :param webinterface: :param request: :param session: :return: """ try: restorefile = request.args.get("restorefile")[0] try: restorefile = json.loads(restorefile) logger.info("Received configuration backup file.") try: if restorefile["hash"] != hashlib.sha256( unicode_to_bytes( restorefile["data"])).hexdigest(): webinterface.add_alert( "Backup file appears to be corrupt: Invalid checksum." ) return webinterface.redirect("/setup_wizard/start") restorefile["data"] = base64.b64decode( unicode_to_bytes(restorefile["data"])) except Exception as e: logger.warn("Unable to b64decode data: {e}", e=e) webinterface.add_alert( "Unable to properly decode pass 1 of data segment of restore file." ) return webinterface.redirect("/setup_wizard/start") session.set("restore_backup_file", restorefile) except Exception as e: logger.warn("Unable to parse JSON phase 2: {e}", e=e) webinterface.add_alert("Invalid restore file contents.") return webinterface.redirect("/setup_wizard/start") except Exception: restorefile = session.get("restore_backup_file", None) if restorefile is None: webinterface.add_alert("No restore file found.") return webinterface.redirect("/setup_wizard/start") required_keys = ("encrypted", "time", "file_type", "created", "backup_version") if all(required in restorefile for required in required_keys) is False: webinterface.add_alert( "Backup file appears to be missing important parts.") return webinterface.redirect("/setup_wizard/start") if restorefile["encrypted"] is True: try: password = request.args.get("password", )[0] except Exception: password = session.get("restorepassword", None) if password is None: page = webinterface.get_template( request, webinterface.wi_dir + "/pages/setup_wizard/restore_password.html") return page.render(alerts=session.get_alerts(), restore=restorefile) try: decrypted = yield webinterface._Tools.data_unpickle( restorefile["data"], "msgpack_aes256_zip_base85", passphrase=password, ) restorefile["data_processed"] = json.loads( bytes_to_unicode(decrypted)) except Exception as e: logger.warn("Unable to decrypt restoration file: {e}", e=e) webinterface.add_alert( "It appears the password is incorrect.", "danger") page = webinterface.get_template( request, webinterface.wi_dir + "/pages/setup_wizard/restore_password.html") return page.render(alerts=session.get_alerts(), restore=restorefile) else: #no password restorefile["data_processed"] = json.loads( bytes_to_unicode(restorefile["data"])) session.set("restore_backup_file", restorefile) page = webinterface.get_template( request, webinterface.wi_dir + "/pages/restore/restore_ready.html") return page.render(alerts=session.get_alerts(), restore=session.get("restore_backup_file"))
def data_unpickle(cls, data: Any, content_type: Optional[str] = None, passphrase: Optional[str] = None, use_decimal: Optional[bool] = None): """ Unpack data packed with data_pickle. See data_pickle() for content_type options. :param data: :param content_type: :param passphrase: Passphrase will be sent to the encryption function. :return: """ if data is None: return None data = bytes_to_unicode(data) if content_type is None: content_type = "msgpack_base85" elif content_type == "string": return str(data) elif content_type == "bool": return bool(data) content_type = content_type.lower() # Sometimes empty dictionaries are encoded... This is a simple shortcut. if content_type == "msgpack_base85_zip" and data == "cwTD&004mifd": return {} elif content_type == "msgpack_base85" and data == "fB": return {} if "json" in content_type and "msgack" in content_type: raise YomboWarning( "Unpickle data can only have json or msgpack, not both.") encoder_count = 0 for encoder_check in ("base32", "base62", "base64", "base85"): if encoder_check in content_type: encoder_count += 1 if encoder_count > 1: raise YomboWarning( "Pickle data can only one of: base32, base62, base64 or base85, not multiple." ) if "base32" in content_type: data = base64.b32decode(data) if "base62" in content_type: data = base62.decodebytes(data) if "base64" in content_type: data = data + "=" * (-len(data) % 4) data = base64.b64decode(data) elif "base85" in content_type: data = base64.b85decode(data) for cipher in AVAILABLE_CIPHERS: if cipher in content_type: try: print(f"tools.unpickle, about to decrypt: {data}") data = cls._Encryption.decrypt_aes(data, passphrase=passphrase, cipher=cipher) print(f"tools.unpickle, about to decrypt, done: {data}") except Exception: break if "lz4" in content_type: try: data = lz4.frame.decompress(data, passphrase=passphrase) except Exception as e: raise YomboWarning(f"Error lz4 decompress {content_type}: {e}") elif "gzip" in content_type or "zip" in content_type: try: data = zlib.decompress(data) except Exception as e: raise YomboWarning( f"Error zlib decompress {content_type}: {e}") try: data = zlib.decompress(data) except Exception as e: pass if isinstance(data, str) or isinstance(data, bytes): if "json" in content_type: data = unicode_to_bytes(data) try: data = bytes_to_unicode( json.loads(data, use_decimal=use_decimal)) except Exception as e: raise YomboWarning( f"data_unpickle received non-json content: {data}") elif "msgpack" in content_type: data = unicode_to_bytes(data) try: data = bytes_to_unicode(msgpack.unpackb(data)) except Exception as e: raise YomboWarning( f"data_unpickle received non-msgpack content: {data}") return data
def data_pickle(cls, data: Any, content_type: Optional[str] = None, compression_level: Optional[int] = None, local: Optional[bool] = None, passphrase: Optional[str] = None, use_decimal: Optional[bool] = None) -> Union[str, bytes]: """ Encodes data based on value of the content_type type. The default is "msgpack_base85". This easily allows data to be sent externally or even the database. The ordering of the content_type does not matter. Format of the content_type: {pickle-method}_{compress method}_{encode-method}. Pickle methods: * msgpack - Use msgpack to pickle the data. More space efficient than json. * json - Use JSON to pickle the data. Compression can also be applied, simply append: * lz4 - Use the lz4 compression * zip - Use the gzip compression Encryption is also supported: * aes256, aes192, aes128 Encode methods: * base32 - Encode with base32 - Biggest size. * base62 - Encode with base62 - Like base64, but URL safe. * base64 - Encode with base64 * base85 - Encode with base85 Examples: * msgpack_zip - Pickle with msgpack, then compress with gzip. * zip_msgpack - Same as above, just different order. * msgpack_base85_zip - Pickle with msgpack, then compress with gzip, then encode with base85 * msgpack_zip_aes256_base85 - Pickle with msgpack, compress with gzip, encrypt with aes, then encode with base85 * json_lz4 - Pickle with json, then compress with z-standard. Non-encoded results typically return bytes, while encoded results return strings. :param data: String, list, or dictionary to be encoded. :param content_type: Optional encode method. :param compression_level: Sets a compression level - default depends on compressor. :param passphrase: Passphrase will be sent to the encryption function. :return: bytes of the encoded data that can be used with data_unpickle. """ if data is None: return None if content_type is None: content_type = "msgpack_base85" elif content_type == "string": return str(data) elif content_type == "bool": return bool(data) content_type = content_type.lower() if "json" in content_type and "msgack" in content_type: raise YomboWarning( "Pickle data can only have json or msgpack, not both.") encoder_count = 0 for encoder_check in ("base32", "base62", "base64", "base85"): if encoder_check in content_type: encoder_count += 1 if encoder_count > 1: raise YomboWarning( "Pickle data can only one of: base32, base62, base64 or base85, not multiple." ) if "json" in content_type: try: data = json.dumps(data, separators=(",", ":"), use_decimal=use_decimal) except Exception as e: raise YomboWarning(f"Error encoding json: {e}") elif "msgpack" in content_type: try: data = msgpack.packb(data) except Exception as e: raise YomboWarning(f"Error encoding msgpack: {e}") if "lz4" in content_type: if compression_level is None: compression_level = 2 elif compression_level > 1 or compression_level < 9: compression_level = 2 try: data = lz4.frame.compress(unicode_to_bytes(data), compression_level) except Exception as e: raise YomboWarning(f"Error encoding {content_type}: {e}") elif "gzip" in content_type or "zip" in content_type: if compression_level is None: compression_level = 5 elif compression_level > 1 or compression_level < 9: compression_level = 5 try: data = zlib.compress(unicode_to_bytes(data), compression_level) except Exception as e: raise YomboWarning(f"Error encoding {content_type}: {e}") for cipher in AVAILABLE_CIPHERS: if cipher in content_type: try: data = cls._Encryption.encrypt_aes(data, passphrase=passphrase, cipher=cipher) except Exception: break if "base32" in content_type: data = bytes_to_unicode(base64.b32encode(data)) if local is True: data = data.rstrip("=") elif "base62" in content_type: data = bytes_to_unicode(base62.encodebytes(data)) if local is True: data = data.rstrip("=") elif "base64" in content_type: data = bytes_to_unicode(base64.b64encode(data)) if local is True: data = data.rstrip("=") elif "base85" in content_type: data = bytes_to_unicode(base64.b85encode(data)) return data
def generate_csr(self, args): """ This function shouldn't be called directly. Instead, use the queue "self.generate_csr_queue.put(request, callback, callback_args)" or "self._SSLCerts.generate_csr_queue.put()". Requests certs to be made. Will return right away with a request ID. A callback can be set to return the cert once it's complete. :return: """ logger.debug("Generate_CSR called with args: {args}", args=args) kwargs = self.check_csr_input(args) if kwargs["key_type"] == "rsa": kwargs["key_type"] = crypto.TYPE_RSA else: kwargs["key_type"] = crypto.TYPE_DSA req = crypto.X509Req() req.get_subject().CN = kwargs["cn"] req.get_subject().countryName = "US" req.get_subject().stateOrProvinceName = "California" req.get_subject().localityName = "Sacramento" req.get_subject().organizationName = "Yombo" req.get_subject( ).organizationalUnitName = "Gateway " + self.gateway_id[0:15] # Appends SAN to have "DNS:" if kwargs["sans"] is not None: san_string = [] for i in kwargs["sans"]: san_string.append(f"DNS: {i}") san_string = ", ".join(san_string) x509_extensions = [ crypto.X509Extension(b"subjectAltName", False, unicode_to_bytes(san_string)) ] req.add_extensions(x509_extensions) start = time() key = yield threads.deferToThread( self._generate_key, **{ "key_type": kwargs["key_type"], "key_size": kwargs["key_size"] }) duration = round(float(time()) - start, 4) self._Events.new("sslcerts", "generate_new", (args["sslname"], kwargs["cn"], san_string, duration)) req.set_pubkey(key) req.sign(key, "sha256") csr = crypto.dump_certificate_request(crypto.FILETYPE_PEM, req) key_file = crypto.dump_privatekey(crypto.FILETYPE_PEM, key) if kwargs["csr_file"] is not None: yield save_file(kwargs["csr_file"], csr) if kwargs["key_file"] is not None: yield save_file(kwargs["key_file"], key_file) return { "csr": csr, "csr_hash": sha256_compact(unicode_to_bytes(csr)), "key": key_file }