def helper_compression_negative_level_notdefault(self):
if zstd.ZSTD_version_number() < 10304:
return raise_skip(
"PyZstd was build with old version of ZSTD library (%s) without support of negative compression levels."
% zstd.ZSTD_version())
CDATA = zstd.compress(tDATA, -1)
self.assertNotEqual(CDATA, zstd.compress(tDATA, 0))
def handle_chunk_data(data: bytes) -> bytes:
packet_chunk_data = PacketChunkData(data)
packet_chunk_data.unpack_custom_packet_data()
cached_ys = []
for y, section in enumerate(packet_chunk_data.sections):
# if section exists, put the data into database
if section:
coords = packet_chunk_data.get_coords_bytes(y)
get_session_info('chunk_section_db').put(coords,
zstd.compress(section))
cached_ys.append(y)
# iter cached section mask
for y in range(16):
if packet_chunk_data.cached_section_mask.get(y):
coords = packet_chunk_data.get_coords_bytes(y)
packet_chunk_data.sections[y] = zstd.decompress(
get_session_info('chunk_section_db').get(coords))
# send ack packet
if cached_ys:
packet_chunk_data_ack = PacketChunkDataAck()
packet_chunk_data_ack.dimension = get_session_info('dimension')
packet_chunk_data_ack.chunk_x = packet_chunk_data.x
packet_chunk_data_ack.chunk_z = packet_chunk_data.z
for y in cached_ys:
packet_chunk_data_ack.section_ys.append(y)
try:
send_packet(CHUNK_DATA_ACK, packet_chunk_data_ack.pack_packet())
except OSError:
pass
return packet_chunk_data.pack_vanilla_packet_data()
def helper_compression_level20(self):
if sys.hexversion < 0x03000000:
DATA = "This is must be very very long string to be compressed by zstd. AAAAAAAAAAAAARGGHHH!!! Just hope its enough length. И немного юникода."
else:
DATA = b"This is must be very very long string to be compressed by zstd. AAAAAAAAAAAAARGGHHH!!! Just hope its enough length." + " И немного юникода.".encode(
)
self.assertEqual(DATA, zstd.decompress(zstd.compress(DATA, 20)))
def commit(self):
compression_level = 3
cdata = zstd.compress(json.dumps(self.data).encode('UTF-8'), compression_level)
with open(self.out_dir + '/cc_' + str(self.i) + '.json.zst', 'wb') as fh:
fh.write(cdata)
self.i += 1
self.data = []
def process(self, data: Any) -> Any:
""" Implements a Processor for compressing model parameters. """
output = zstd.compress(pickle.dumps(data), self.compression_level)
logging.info("[Client #%d] Compressed model parameters.",
self.client_id)
return output
def apply_zstd_compression(decompressed_input_bin) -> tuple:
"""
Apply ZSTD compression to the input
Args:
:param decompressed_input_bin: the binary to be compressed
:return: a tuple (compressed_result, ZSTD)
"""
return zstd.compress(decompressed_input_bin), ZSTD
def save_compressed_data(data, file_name):
# comp_ctx = zstd.ZstdCompressor(write_content_size=True, level=6)
# compressed = comp_ctx.compress(pickle.dumps(data, protocol=-1))
compressed = zstd.compress(pickle.dumps(data, protocol=-1))
# compressed = pickle.dumps(data, protocol=-1)
f = open(file_name, 'wb')
f.write(compressed)
f.close()
def compress(value):
"""Serialize and Zstandard compress the value
Args:
value: Any value, could be list, dict, string, int
Returns: Compressed serialized bytes
"""
return zstd.compress(value, 9, 2)
def process(self, data: Any) -> Any:
""" Implements a Processor for compressing numpy array. """
if isinstance(data, list):
ret = []
datashape_feature = data[0][0].shape
datatype_feature = data[0][0].dtype
ret.append((datashape_feature, datatype_feature))
for logits, targets in data:
datashape_target = targets.shape
datatype_target = targets.dtype
datacom_feature = zstd.compress(logits, self.compression_ratio)
datacom_target = zstd.compress(targets, self.compression_ratio)
ret.append((datacom_feature, datacom_target, datashape_target,
datatype_target))
else:
ret = (data.shape, data.dtype,
zstd.compress(data, self.compression_ratio))
return ret
def pack_data(obj):
if obj is None:
return None
# pickled = pickle.dumps(obj) # 1169_airlines explodes here with RAM < ?
if isinstance(obj, Chain):
pickled = pickle.dumps(obj)
fast_reduced = lz.compress(b'Chain' + pickled, compression_level=1)
else:
h, w = obj.shape
fast_reduced = lz.compress(obj.reshape(w * h), compression_level=1)
return zs.compress(fast_reduced)
def main(prefix, logfile, delay, batch_size):
print(
f"Writing events to {prefix} with {delay} seconds between batches of {batch_size}"
)
sqs = None
# local-grapl prefix is reserved for running Grapl locally
if prefix == "local-grapl":
s3 = boto3.client(
"s3",
endpoint_url="http://localhost:9000",
aws_access_key_id="minioadmin",
aws_secret_access_key="minioadmin",
region_name="us-east-3",
)
sqs = boto3.client(
"sqs",
endpoint_url="http://localhost:9324",
region_name="us-east-1",
aws_access_key_id="dummy_cred_aws_access_key_id",
aws_secret_access_key="dummy_cred_aws_secret_access_key",
)
else:
s3 = boto3.client("s3")
with open(logfile, "rb") as b:
body = b.readlines()
body = [line for line in body]
def chunker(seq, size):
return [seq[pos:pos + size] for pos in range(0, len(seq), size)]
for chunks in chunker(body, batch_size):
c_body = zstd.compress(b"\n".join(chunks).replace(b"\n\n", b"\n"), 4)
epoch = int(time.time())
key = (str(epoch - (epoch % (24 * 60 * 60))) + "/sysmon/" +
str(epoch) + rand_str(3))
s3.put_object(Body=c_body,
Bucket="{}-sysmon-log-bucket".format(prefix),
Key=key)
# local-grapl relies on manual eventing
if sqs:
sqs.send_message(
QueueUrl=
"http://localhost:9324/queue/grapl-sysmon-graph-generator-queue",
MessageBody=into_sqs_message(
bucket="{}-sysmon-log-bucket".format(prefix), key=key),
)
time.sleep(delay)
print(f"Completed uploading at {time.ctime()}")
def main(prefix, logfile):
print(f"Writing events to {prefix}")
sqs = None
# local-grapl prefix is reserved for running Grapl locally
if prefix == 'local-grapl':
s3 = boto3.client(
's3',
endpoint_url="http://localhost:9000",
aws_access_key_id='minioadmin',
aws_secret_access_key='minioadmin',
region_name="us-east-3",
)
sqs = boto3.client(
'sqs',
endpoint_url="http://localhost:9324",
region_name="us-east-1",
aws_access_key_id='dummy_cred_aws_access_key_id',
aws_secret_access_key='dummy_cred_aws_secret_access_key',
)
else:
s3 = boto3.client('s3')
with open(logfile, 'rb') as b:
body = b.readlines()
body = [line for line in body]
def chunker(seq, size):
return [seq[pos:pos + size] for pos in range(0, len(seq), size)]
for chunks in chunker(body, 150):
c_body = zstd.compress(b"\n".join(chunks).replace(b"\n\n", b"\n"), 4)
epoch = int(time.time())
key = str(epoch - (epoch % (24 * 60 * 60))) + "/sysmon/" + str(epoch) + rand_str(3)
s3.put_object(
Body=c_body,
Bucket="{}-sysmon-log-bucket".format(prefix),
Key=key
)
# local-grapl relies on manual eventing
if sqs:
sqs.send_message(
QueueUrl='http://localhost:9324/queue/sysmon-graph-generator-queue',
MessageBody=into_sqs_message(
bucket="{}-sysmon-log-bucket".format(prefix),
key=key
)
)
print(f'Completed uploading at {time.ctime()}')
def get_compressed_data(self, extra_header_size):
rv = BytesIO()
data = self.get_value()
compressed = zstd.compress(data)
header_size = extra_header_size + 4 + 4 # sizes
write_binary_uint32(header_size + len(compressed), rv)
write_binary_uint32(len(data), rv)
rv.write(compressed)
return rv.getvalue()
def unpack_replay(outdir, replay):
json_header, body = read_header_and_body(replay)
binary_header = body.pop('header')
commands = get_command_timeseries(body.pop('body'))
blob = pickle.dumps({
'json': json_header,
'binary': binary_header,
'remaining': body,
'commands': commands
})
compressed = zstd.compress(blob)
base, ext = path.splitext(path.basename(replay))
with open(path.join(outdir, base + '.pickle'), 'wb') as handle:
handle.write(compressed)
def start(self, password=""):
ramdomKey = self.genRamdomKey()
packetKey = self.cryptographyRamdomKey(keyL2=self.adjustPassword(
self.genL2Key(password)),
ramdomKey=ramdomKey)
files = self.readFiles()
compressFile = zstd.compress(files, 9)
cryptoFile = self.cryptographyFiles(files=compressFile,
password=ramdomKey)
compressFile = None
packetFinal = self.packetGenerator(cryptoFile, packetKey)
cryptoFile = None
# self.feedBackResults(files,cryptoFile,compressFile,packetFinal)
self.deleteOriginalFiles()
self.writeCryptoFile(packetFinal)
packetFinal = None
print('Criptografia finalizada!')
def main(prefix, logfile):
print(f"Writing events to {prefix}")
sqs = None
# local-grapl prefix is reserved for running Grapl locally
if prefix == "local-grapl":
s3 = boto3.client(
"s3",
endpoint_url="http://localhost:9000",
aws_access_key_id="minioadmin",
aws_secret_access_key="minioadmin",
)
sqs = boto3.client("sqs", endpoint_url="http://localhost:9324")
else:
s3 = boto3.client("s3")
with open(logfile, "rb") as b:
body = json.load(b)
body = [line for line in body]
def chunker(seq, size):
return [seq[pos : pos + size] for pos in range(0, len(seq), size)]
for chunks in chunker(body, 50):
c_body = zstd.compress(json.dumps(chunks), 4)
epoch = int(time.time())
key = (
str(epoch - (epoch % (24 * 60 * 60)))
+ "/injected/"
+ str(epoch)
+ rand_str(3)
)
s3.put_object(Body=c_body, Bucket="{}-raw-log-bucket".format(prefix), Key=key)
# local-grapl relies on manual eventing
if sqs:
sqs.send_message(
QueueUrl="http://localhost:9324/queue/grapl-generic-graph-generator-queue",
MessageBody=into_sqs_message(
bucket="{}-sysmon-log-bucket".format(prefix), key=key
),
)
print(time.ctime())
def main(prefix):
s3 = boto3.client('s3')
with open('./eventlog.xml', 'rb') as b:
body = b.readlines()
body = [line for line in body]
def chunker(seq, size):
return [seq[pos:pos + size] for pos in range(0, len(seq), size)]
for chunks in chunker(body, 200):
c_body = zstd.compress(b"\n".join(chunks), 4)
epoch = int(time.time())
s3.put_object(Body=c_body,
Bucket="{}-sysmon-log-bucket".format(prefix),
Key=str(epoch - (epoch % (24 * 60 * 60))) + "/sysmon/" +
str(epoch) + rand_str(3))
def main(deployment_name, logfile):
print(f"Writing events to {deployment_name}")
sqs = None
# local-grapl deployment_name is reserved for running Grapl locally
if deployment_name == "local-grapl":
s3 = boto3.client(
"s3",
endpoint_url=os.environ["S3_ENDPOINT"],
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],
aws_secret_access_key=os.environ["S3_ACCESS_KEY_SECRET"])
sqs = boto3.client("sqs", endpoint_url=os.environ["SQS_ENDPOINT"])
else:
s3 = boto3.client("s3")
with open(logfile, "rb") as b:
body = json.load(b)
body = [line for line in body]
def chunker(seq, size):
return [seq[pos:pos + size] for pos in range(0, len(seq), size)]
for chunks in chunker(body, 50):
c_body = zstd.compress(json.dumps(chunks), 4)
epoch = int(time.time())
key = (str(epoch - (epoch % (24 * 60 * 60))) + "/injected/" +
str(epoch) + rand_str(3))
s3.put_object(Body=c_body,
Bucket="{}-raw-log-bucket".format(deployment_name),
Key=key)
# local-grapl relies on manual eventing
if sqs:
sqs.send_message(
QueueUrl=
f"{os.environ['SQS_ENDPOINT']}/queue/grapl-generic-graph-generator-queue",
MessageBody=into_sqs_message(
bucket="{}-sysmon-log-bucket".format(deployment_name),
key=key),
)
print(time.ctime())
def _compress(decompressed_input_bin: bin, compress_scheme=LZ4) -> bin:
"""
This function compresses a binary using LZ4
Args:
decompressed_input_bin (bin): binary to be compressed
compress_scheme: the compression method to use
Returns:
bin: a compressed binary
"""
if compress_scheme == LZ4:
return lz4.frame.compress(decompressed_input_bin)
elif compress_scheme == ZSTD:
return zstd.compress(decompressed_input_bin)
else:
raise CompressionNotFoundException("compression scheme note found for"
" compression code:" +
str(compress_scheme))
def on_matched_cbor(client: mqtt.Client, userdata, message):
signal = cbor.loads(message.payload, tag_hook=uncborify)
batch.append(signal)
logging.debug(signal)
cbor_batch = cbor.dumps(
batch,
timezone=datetime.timezone.utc,
datetime_as_timestamp=True,
default=cborify,
)
batch_s = len(cbor_batch)
lzma_s = len(lzma.compress(cbor_batch)) / batch_s
zlib_s = len(zlib.compress(cbor_batch, 9)) / batch_s
lz4_s = len(lz4.frame.compress(cbor_batch, compression_level=lz4.frame.COMPRESSIONLEVEL_MAX)) / batch_s
zstd_s = len(zstd.compress(cbor_batch, 22)) / batch_s
brotli_s = len(brotli.compress(cbor_batch, quality=11)) / batch_s
if len(batch) < 10 or (len(batch) % 10) == 0:
print(f"{len(batch)}, {batch_s}, {lzma_s}, {zlib_s}, {lz4_s}, {zstd_s}, {brotli_s}")
def main():
s3 = boto3.client('s3')
with open('./events3.xml', 'r') as b:
body = b.readlines()
def chunker(seq, size):
return [seq[pos:pos + size] for pos in range(0, len(seq), size)]
for chunks in chunker(body, 100):
c_body = zstd.compress("\n".join(chunks), 4)
epoch = int(time.time())
res = s3.put_object(
Body=c_body,
Bucket="grapl-sysmon-log-bucket",
Key=str(epoch - (epoch % (24 * 60 * 60))) + "/sysmon/" +
str(epoch) + rand_str(3)
)
def main(prefix):
s3 = boto3.client('s3')
with open('./injected_logs.json', 'rb') as b:
body = json.load(b)
body = [line for line in body]
def chunker(seq, size):
return [seq[pos:pos + size] for pos in range(0, len(seq), size)]
for chunks in chunker(body, 50):
c_body = zstd.compress(json.dumps(chunks), 4)
epoch = int(time.time())
s3.put_object(Body=c_body,
Bucket="{}-raw-log-bucket".format(prefix),
Key=str(epoch - (epoch % (24 * 60 * 60))) +
"/injected/" + str(epoch) + rand_str(3))
print(time.ctime())
def dump_layer(self, data, many, **kwargs):
"""
Called automatically to export the tileset from memory.
"""
perform_post_dump = True if "performPostDump" not in self.context.keys(
) else self.context["performPostDump"]
if perform_post_dump:
if "tintcolor" in data.keys():
data["tintcolor"] = str(data["tintcolor"])
if "transparentcolor" in data.keys():
data["transparentcolor"] = str(data["transparentcolor"])
# Load compressed data if necessary
compression = constants.Compression.none.value if "compression" not in data.keys(
) else data["compression"]
if compression == constants.Compression.zlib.value:
out_data = json.dumps(data["data"]).encode("utf-8")
out_data = zlib.compress(out_data)
out_data = base64.b64encode(out_data)
data["data"] = out_data.decode("utf-8")
elif compression == constants.Compression.gzip.value:
out_data = json.dumps(data["data"]).encode("utf-8")
out_data = gzip.compress(out_data)
out_data = base64.b64encode(out_data)
data["data"] = out_data.decode("utf-8")
elif compression == constants.Compression.zstd.value:
if zstd is None:
raise ModuleNotFoundError(
"Library zstd not installed: https://pypi.org/project/zstd/"
)
out_data = json.dumps(data["data"]).encode("utf-8")
out_data = zstd.compress(out_data)
out_data = base64.b64encode(out_data)
data["data"] = out_data.decode("utf-8")
elif compression == constants.Compression.none.value:
pass
else:
raise ValueError("Unsupported compression type: %s" %
item["compression"])
return data
def main(deployment_name):
s3 = boto3.client("s3")
with open("./eventlog.xml", "rb") as b:
body = b.readlines()
body = [line for line in body]
def chunker(seq, size):
return [seq[pos:pos + size] for pos in range(0, len(seq), size)]
for chunks in chunker(body, 50):
c_body = zstd.compress(b"\n".join(chunks), 4)
epoch = int(time.time())
s3.put_object(
Body=c_body,
Bucket="{}-sysmon-log-bucket".format(deployment_name),
Key=str(epoch - (epoch % (24 * 60 * 60))) + "/sysmon/" +
str(epoch) + rand_str(3),
)
print(time.ctime())
def zstd_encode(data, level=5, out=None):
"""Compress ZStandard."""
return zstd.compress(data, level)
def compressed_mesh(self):
if((self._compressed_mesh == None) and (self._mesh != None)):
self._compressed_mesh = zstd.compress(self._mesh, self._mesh_compression_level)
return self._compressed_mesh
def compressed_data(self):
if((self._compressed_data == None) and (self._data != None)):
self._compressed_data = zstd.compress(self._data, self._compression_level)
return self._compressed_data
def encode_chunk(self, input: List[bytes]) -> bytes:
# zstd encoded line delineated xml
return cast(
bytes, zstd.compress(b"\n".join(input).replace(b"\n\n", b"\n"), 4))
def compress_data(self, data):
return zstd.compress(bytes(data))
def test_raw(self): DATA = b"abc def" self.assertEqual(DATA, zstd.decompress(zstd.compress(DATA)))
def zstd_compress(data, level=None):
if level is not None:
return zstd.compress(data, level)
return zstd.compress(data)
