def sha512_compact(value):
"""
Returns a shorter sha512 - 86 characters long instead of 128.
This uses a base62 encoding which uses the entire alphabet, with mixed case.
Returned length is 86 characters.
:param value:
:return:
"""
return base62.encodebytes(sha512(unicode_to_bytes(value)).digest())
def encode_binary(data, encoder: Optional[str] = None, convert_to_unicode: Optional[bool] = True):
"""Converts to text."""
if encoder is None:
encoder = "base62"
if encoder == "base62":
data = base62.encodebytes(data)
elif encoder == "base64":
data = base64.b64encode(data)
elif encoder == "base85":
data = base64.b85encode(data)
else:
raise YomboWarning("Base compactor type: {encoder}")
if convert_to_unicode in (None, True):
return bytes_to_unicode(data)
return data
def sha256_compact(value):
return base62.encodebytes(sha256(unicode_to_bytes(value)).digest())
def data_pickle(cls,
data: Any,
content_type: Optional[str] = None,
compression_level: Optional[int] = None,
local: Optional[bool] = None,
passphrase: Optional[str] = None,
use_decimal: Optional[bool] = None) -> Union[str, bytes]:
"""
Encodes data based on value of the content_type type. The default is "msgpack_base85". This easily allows data
to be sent externally or even the database. The ordering of the content_type does not matter.
Format of the content_type:
{pickle-method}_{compress method}_{encode-method}.
Pickle methods:
* msgpack - Use msgpack to pickle the data. More space efficient than json.
* json - Use JSON to pickle the data.
Compression can also be applied, simply append:
* lz4 - Use the lz4 compression
* zip - Use the gzip compression
Encryption is also supported:
* aes256, aes192, aes128
Encode methods:
* base32 - Encode with base32 - Biggest size.
* base62 - Encode with base62 - Like base64, but URL safe.
* base64 - Encode with base64
* base85 - Encode with base85
Examples:
* msgpack_zip - Pickle with msgpack, then compress with gzip.
* zip_msgpack - Same as above, just different order.
* msgpack_base85_zip - Pickle with msgpack, then compress with gzip, then encode with base85
* msgpack_zip_aes256_base85 - Pickle with msgpack, compress with gzip, encrypt with aes, then encode with base85
* json_lz4 - Pickle with json, then compress with z-standard.
Non-encoded results typically return bytes, while encoded results return strings.
:param data: String, list, or dictionary to be encoded.
:param content_type: Optional encode method.
:param compression_level: Sets a compression level - default depends on compressor.
:param passphrase: Passphrase will be sent to the encryption function.
:return: bytes of the encoded data that can be used with data_unpickle.
"""
if data is None:
return None
if content_type is None:
content_type = "msgpack_base85"
elif content_type == "string":
return str(data)
elif content_type == "bool":
return bool(data)
content_type = content_type.lower()
if "json" in content_type and "msgack" in content_type:
raise YomboWarning(
"Pickle data can only have json or msgpack, not both.")
encoder_count = 0
for encoder_check in ("base32", "base62", "base64", "base85"):
if encoder_check in content_type:
encoder_count += 1
if encoder_count > 1:
raise YomboWarning(
"Pickle data can only one of: base32, base62, base64 or base85, not multiple."
)
if "json" in content_type:
try:
data = json.dumps(data,
separators=(",", ":"),
use_decimal=use_decimal)
except Exception as e:
raise YomboWarning(f"Error encoding json: {e}")
elif "msgpack" in content_type:
try:
data = msgpack.packb(data)
except Exception as e:
raise YomboWarning(f"Error encoding msgpack: {e}")
if "lz4" in content_type:
if compression_level is None:
compression_level = 2
elif compression_level > 1 or compression_level < 9:
compression_level = 2
try:
data = lz4.frame.compress(unicode_to_bytes(data),
compression_level)
except Exception as e:
raise YomboWarning(f"Error encoding {content_type}: {e}")
elif "gzip" in content_type or "zip" in content_type:
if compression_level is None:
compression_level = 5
elif compression_level > 1 or compression_level < 9:
compression_level = 5
try:
data = zlib.compress(unicode_to_bytes(data), compression_level)
except Exception as e:
raise YomboWarning(f"Error encoding {content_type}: {e}")
for cipher in AVAILABLE_CIPHERS:
if cipher in content_type:
try:
data = cls._Encryption.encrypt_aes(data,
passphrase=passphrase,
cipher=cipher)
except Exception:
break
if "base32" in content_type:
data = bytes_to_unicode(base64.b32encode(data))
if local is True:
data = data.rstrip("=")
elif "base62" in content_type:
data = bytes_to_unicode(base62.encodebytes(data))
if local is True:
data = data.rstrip("=")
elif "base64" in content_type:
data = bytes_to_unicode(base64.b64encode(data))
if local is True:
data = data.rstrip("=")
elif "base85" in content_type:
data = bytes_to_unicode(base64.b85encode(data))
return data