def decompress(self, source, cursor, compressedbytes, uncompressedbytes=None):
if self.algo == uproot.const.kZLIB:
from zlib import decompress as zlib_decompress
return zlib_decompress(cursor.bytes(source, compressedbytes))
elif self.algo == uproot.const.kLZMA:
try:
from lzma import decompress as lzma_decompress
except ImportError:
try:
from backports.lzma import decompress as lzma_decompress
except ImportError:
raise ImportError("Install lzma package with:\n pip install backports.lzma\nor\n conda install -c conda-forge backports.lzma\n(or just use Python >= 3.3).")
return lzma_decompress(cursor.bytes(source, compressedbytes))
elif self.algo == uproot.const.kOldCompressionAlgo:
raise NotImplementedError("ROOT's \"old\" algorithm (fCompress 300) is not supported")
elif self.algo == uproot.const.kLZ4:
try:
from lz4.block import decompress as lz4_decompress
except ImportError:
raise ImportError("Install lz4 package with:\n pip install lz4\nor\n conda install -c anaconda lz4")
if uncompressedbytes is None:
raise ValueError("lz4 needs to know the uncompressed number of bytes")
return lz4_decompress(cursor.bytes(source, compressedbytes), uncompressed_size=uncompressedbytes)
else:
raise ValueError("unrecognized compression algorithm: {0}".format(self.algo))
def decode_row(self, row):
if row is None:
return None
columns = row.cells[self.column_family]
try:
cell = columns[self.data_column][0]
except KeyError:
return None
# Check if a TTL column exists
# for this row. If there is,
# we can use the `timestamp` property of the
# cells to see if we should return the
# row or not.
if self.ttl_column in columns:
# If we needed the actual value, we could unpack it.
# ttl = struct.unpack('<I', columns[self.ttl_column][0].value)[0]
if cell.timestamp < timezone.now():
return None
data = cell.value
# Read our flags
flags = 0
if self.flags_column in columns:
flags = struct.unpack("B", columns[self.flags_column][0].value)[0]
# Check for a compression flag on, if so
# decompress the data.
if flags & self._FLAG_COMPRESSED:
data = zlib_decompress(data)
return json_loads(data)
def on_message(self, client, userdata, message):
try:
usablePayload = message.payload.decode('utf-8')
form = '' # Successfully decoded as utf8
except:
try: # jMQTT will try automaticaly to decompress the payload (requested in issue #135)
unzip = zlib_decompress(message.payload, wbits=-15)
usablePayload = unzip.decode('utf-8')
form = ' (decompressed)'
except: # If payload cannot be decoded or decompressed it is returned in base64
usablePayload = b2a_base64(message.payload,
newline=False).decode('utf-8')
form = ' (bin in base64)'
logging.info(
'BrkId: % 4s : Message received (topic="%s", payload="%s"%s, QoS=%s, retain=%s)',
self.id, message.topic, usablePayload, form, message.qos,
bool(message.retain))
self.q.put(
json.dumps({
"cmd": "messageIn",
"topic": message.topic,
"payload": usablePayload,
"qos": message.qos,
"retain": bool(message.retain)
}))
def decode_row(self, row):
if row is None:
return None
columns = row.cells[self.column_family]
try:
cell = columns[self.data_column][0]
except KeyError:
return None
# Check if a TTL column exists
# for this row. If there is,
# we can use the `timestamp` property of the
# cells to see if we should return the
# row or not.
if self.ttl_column in columns:
# If we needed the actual value, we could unpack it.
# ttl = struct.unpack('<I', columns[self.ttl_column][0].value)[0]
if cell.timestamp < timezone.now():
return None
data = cell.value
# Read our flags
flags = 0
if self.flags_column in columns:
flags = struct.unpack('B', columns[self.flags_column][0].value)[0]
# Check for a compression flag on, if so
# decompress the data.
if flags & self._FLAG_COMPRESSED:
data = zlib_decompress(data)
return json_loads(data)
def decompress(data, level):
#log.info("decompress(%s bytes, %s) type=%s", len(data), get_compression_type(level))
if level & LZ4_FLAG:
if not has_lz4:
raise InvalidCompressionException("lz4 is not available")
if not use_lz4:
raise InvalidCompressionException("lz4 is not enabled")
size = LZ4_HEADER.unpack_from(data[:4])[0]
#it would be better to use the max_size we have in protocol,
#but this hardcoded value will do for now
if size > MAX_SIZE:
sizemb = size // 1024 // 1024
maxmb = MAX_SIZE // 1024 // 1024
raise Exception(
"uncompressed data is too large: %iMB, limit is %iMB" %
(sizemb, maxmb))
return LZ4_uncompress(data)
if level & LZO_FLAG:
if not has_lzo:
raise InvalidCompressionException("lzo is not available")
if not use_lzo:
raise InvalidCompressionException("lzo is not enabled")
return LZO_decompress(data)
if level & BROTLI_FLAG:
if not has_brotli:
raise InvalidCompressionException("brotli is not available")
if not use_brotli:
raise InvalidCompressionException("brotli is not enabled")
return brotli_decompress(data)
if not use_zlib:
raise InvalidCompressionException("zlib is not enabled")
if isinstance(data, memoryview):
data = data.tobytes()
return zlib_decompress(data)
def inline_array_events_s3(options, today_log, array_files_list, enc_key,
connection):
verbose = options.verbose
to_sort = set()
try:
for index, filename in enumerate(array_files_list):
# Format: 'https://bucket.s3.amazonaws.com/gamefolder/arrayevents/date(seconds)/objectid.bin?
# AWSAccessKeyId=keyid&Expires=timestamp&Signature=signature'
# The objectid doesn't correspond to a database entry but it used for uniqueness and timestamp
filename_cleaned = filename.split('?', 1)[0].rsplit('/', 1)[-1]
event_objectid = filename_cleaned.split('.', 1)[0]
timestamp = get_objectid_timestamp(event_objectid)
formatted_timestamp = strftime('%Y-%m-%d %H:%M:%S',
gmtime(timestamp))
if verbose:
log('Requesting events file ' + str(index + 1) +
' submitted at ' + formatted_timestamp)
r = connection.request('GET', filename, redirect=False)
# pylint: disable=E1103
if r.status != 200:
error_msg = 'Couldn\'t download event %d.' % (index + 1)
if r.data.get('msg', None):
error_msg += ' ' + r.data['msg']
error(str(r.status) + error_msg)
exit(-1)
# pylint: enable=E1103
r_data = decrypt_data(r.data, enc_key)
r_data = json_loads(zlib_decompress(r_data))
if not isinstance(r_data, list):
r_data = [r_data]
for event in r_data:
slug = event['slug']
del event['slug']
event['time'] = strftime('%Y-%m-%d %H:%M:%S',
gmtime(event['time']))
if slug not in today_log:
today_log[slug] = {'playEvents': [], 'customEvents': []}
today_log[slug]['customEvents'].append(event)
# Maintaining a list of slugs to sort the customEvents by date for so that added array events appear in
# order but we do not unneccesarily sort large lists if an array event wasn't added to it
to_sort.add(slug)
for slug in to_sort:
today_log[slug]['customEvents'].sort(key=lambda k: k['time'])
return today_log
except (HTTPError, SSLError) as e:
error(e)
exit(-1)
def _set_body(self, data):
"""Set body."""
if self.compressed == 'gzip':
self.body += gzip_decompress(data)
elif self.compressed == 'deflate':
self.body += zlib_decompress(data)
else:
self.body += data
def inline_array_events_s3(options, today_log, array_files_list, enc_key, connection):
verbose = options.verbose
to_sort = set()
try:
for index, filename in enumerate(array_files_list):
# Format: 'https://bucket.s3.amazonaws.com/gamefolder/arrayevents/date(seconds)/objectid.bin?
# AWSAccessKeyId=keyid&Expires=timestamp&Signature=signature'
# The objectid doesn't correspond to a database entry but it used for uniqueness and timestamp
filename_cleaned = filename.split("?", 1)[0].rsplit("/", 1)[-1]
event_objectid = filename_cleaned.split(".", 1)[0]
timestamp = get_objectid_timestamp(event_objectid)
formatted_timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime(timestamp))
if verbose:
log("Requesting events file " + str(index + 1) + " submitted at " + formatted_timestamp)
r = connection.request("GET", filename, redirect=False)
# pylint: disable=E1103
if r.status != 200:
error_msg = "Couldn't download event %d." % (index + 1)
if r.data.get("msg", None):
error_msg += " " + r.data["msg"]
error(str(r.status) + error_msg)
exit(-1)
# pylint: enable=E1103
r_data = decrypt_data(r.data, enc_key)
r_data = json_loads(zlib_decompress(r_data))
if not isinstance(r_data, list):
r_data = [r_data]
for event in r_data:
slug = event["slug"]
del event["slug"]
event["time"] = strftime("%Y-%m-%d %H:%M:%S", gmtime(event["time"]))
if slug not in today_log:
today_log[slug] = {"playEvents": [], "customEvents": []}
today_log[slug]["customEvents"].append(event)
# Maintaining a list of slugs to sort the customEvents by date for so that added array events appear in
# order but we do not unneccesarily sort large lists if an array event wasn't added to it
to_sort.add(slug)
for slug in to_sort:
today_log[slug]["customEvents"].sort(key=lambda k: k["time"])
return today_log
except (HTTPError, SSLError) as e:
error(e)
exit(-1)
def inline_array_events_local(options, today_log, array_files_list, enc_key):
verbose = options.verbose
to_sort = set()
try:
index = 0
for index, filename in enumerate(array_files_list):
# Format: 'eventlogspath/gamefolder/arrayevents/date(seconds)/objectid.bin'
# The objectid doesn't correspond to a database entry but is used for uniqueness and timestamp
filename = filename.replace('\\', '/')
event_objectid = filename.rsplit('/', 1)[-1].split('.', 1)[0]
timestamp = get_objectid_timestamp(event_objectid)
formatted_timestamp = strftime('%Y-%m-%d %H:%M:%S',
gmtime(timestamp))
if verbose:
log('Retrieving array events file ' + str(index + 1) +
' submitted at ' + formatted_timestamp)
with open(filename, 'rb') as fin:
file_content = fin.read()
file_content = decrypt_data(file_content, enc_key)
file_content = json_loads(zlib_decompress(file_content))
if not isinstance(file_content, list):
file_content = [file_content]
for event in file_content:
slug = event['slug']
del event['slug']
# Some older files had no timestamp in the file data itself in which case we use the timestamp
# on the filename
if 'time' in event:
event['time'] = strftime('%Y-%m-%d %H:%M:%S',
gmtime(event['time']))
else:
event['time'] = formatted_timestamp
if slug not in today_log:
today_log[slug] = {'playEvents': [], 'customEvents': []}
today_log[slug]['customEvents'].append(event)
# Maintaining a list of slugs to sort the customEvents by date for so that added array events appear in
# order but we do not unneccesarily sort large lists if an array event wasn't added to it
to_sort.add(slug)
for slug in to_sort:
today_log[slug]['customEvents'].sort(key=lambda k: k['time'])
return today_log
except (IOError, OSError) as e:
error(e)
exit(-1)
def inline_array_events_local(options, today_log, array_files_list, enc_key):
verbose = options.verbose
to_sort = set()
try:
index = 0
for index, filename in enumerate(array_files_list):
# Format: 'eventlogspath/gamefolder/arrayevents/date(seconds)/objectid.bin'
# The objectid doesn't correspond to a database entry but is used for uniqueness and timestamp
filename = filename.replace('\\', '/')
event_objectid = filename.rsplit('/', 1)[-1].split('.', 1)[0]
timestamp = get_objectid_timestamp(event_objectid)
formatted_timestamp = strftime('%Y-%m-%d %H:%M:%S', gmtime(timestamp))
if verbose:
log('Retrieving array events file ' + str(index + 1) + ' submitted at ' + formatted_timestamp)
with open(filename, 'rb') as fin:
file_content = fin.read()
file_content = decrypt_data(file_content, enc_key)
file_content = json_loads(zlib_decompress(file_content))
if not isinstance(file_content, list):
file_content = [file_content]
for event in file_content:
slug = event['slug']
del event['slug']
# Some older files had no timestamp in the file data itself in which case we use the timestamp
# on the filename
if 'time' in event:
event['time'] = strftime('%Y-%m-%d %H:%M:%S', gmtime(event['time']))
else:
event['time'] = formatted_timestamp
if slug not in today_log:
today_log[slug] = { 'playEvents': [], 'customEvents': [] }
today_log[slug]['customEvents'].append(event)
# Maintaining a list of slugs to sort the customEvents by date for so that added array events appear in
# order but we do not unneccesarily sort large lists if an array event wasn't added to it
to_sort.add(slug)
for slug in to_sort:
today_log[slug]['customEvents'].sort(key=lambda k: k['time'])
return today_log
except (IOError, OSError) as e:
error(e)
exit(-1)
def get_log_files_s3(options, files_list, enc_key, connection):
verbose = options.verbose
silent = options.silent
overwrite = options.overwrite
output_dir = options.outputdir
filename_prefix = options.project + '-'
try:
for filename in files_list:
# Format v1: 'https://bucket.s3.amazonaws.com/gamefolder/events-yyyy-mm-dd.json?AWSAccessKeyId=keyid
# &Expires=timestamp&Signature=signature'
# Format v2: 'https://bucket.s3.amazonaws.com/gamefolder/events-yyyy-mm-dd.bin?AWSAccessKeyId=keyid
# &Expires=timestamp&Signature=signature'
# Convert to 'gameslug-events-yyyy-mm-dd.json'
filename_cleaned = filename.split('?', 1)[0].rsplit('/', 1)[-1]
filename_patched = filename_prefix + filename_cleaned.split(
'.', 1)[0] + '.json'
output_path = normpath(path_join(output_dir, filename_patched))
if not overwrite and path_exists(output_path):
if not silent:
warning('Skipping existing file: %s' % output_path)
continue
if verbose:
log('Requesting file: %s' % filename_patched)
r = connection.request('GET', filename, redirect=False)
# pylint: disable=E1103
if r.status != 200:
error_msg = 'Couldn\'t download %s.' % filename_patched
if r.data.get('msg', None):
error_msg += ' ' + r.data['msg']
error(str(r.status) + error_msg)
exit(-1)
# pylint: enable=E1103
r_data = decrypt_data(r.data, enc_key)
if filename_cleaned.endswith('.bin'):
r_data = zlib_decompress(r_data)
# Format v1 file gets uncompressed on download so we just decrypt it
write_to_file(options,
r_data,
filename=filename_patched,
output_path=output_path)
except (HTTPError, SSLError) as e:
error(e)
exit(-1)
def inline_array_events_s3(options, today_log, array_files_list, enc_key, connection):
verbose = options.verbose
to_sort = set()
try:
for index, filename in enumerate(array_files_list):
# Format: 'https://bucket.s3.amazonaws.com/gamefolder/arrayevents/date(seconds)/objectid.bin?
# AWSAccessKeyId=keyid&Expires=timestamp&Signature=signature'
# The objectid doesn't correspond to a database entry but it used for uniqueness and timestamp
filename_cleaned = filename.split('?', 1)[0].rsplit('/', 1)[-1]
event_objectid = filename_cleaned.split('.', 1)[0]
timestamp = get_objectid_timestamp(event_objectid)
formatted_timestamp = strftime('%Y-%m-%d %H:%M:%S', gmtime(timestamp))
if verbose:
log('Requesting array event ' + str(index + 1) + ' occuring at ' + formatted_timestamp)
r = connection.request('GET', filename, redirect=False)
# pylint: disable=E1103
if r.status != 200:
error_msg = 'Couldn\'t download event %d.' % (index + 1)
if r.data.get('msg', None):
error_msg += ' ' + r.data['msg']
error(str(r.status) + error_msg)
exit(-1)
# pylint: enable=E1103
r_data = decrypt_data(r.data, enc_key)
r_data = json_loads(zlib_decompress(r_data))
slug = r_data['slug']
del r_data['slug']
r_data['time'] = formatted_timestamp
if slug not in today_log:
today_log[slug] = { 'playEvents': [], 'customEvents': [] }
today_log[slug]['customEvents'].append(r_data)
# Maintaining a list of slugs to sort the customEvents by date for so that added array events appear in
# order but we do not unneccesarily sort large lists if an array event wasn't added to it
to_sort.add(slug)
for slug in to_sort:
today_log[slug]['customEvents'].sort(key=lambda k: k['time'])
return today_log
except (HTTPError, SSLError) as e:
error(e)
exit(-1)
def get_log_files_local(options, files_list, enc_key):
verbose = options.verbose
silent = options.silent
overwrite = options.overwrite
output_dir = options.outputdir
filename_prefix = options.project + '-'
try:
for filename in files_list:
if filename.startswith('http'):
error('Unexpected file to retrieve')
exit(-1)
# Format v1: 'eventlogspath/gamefolder/events-yyyy-mm-dd.json.gz'
# Format v2: 'eventlogspath/gamefolder/events-yyyy-mm-dd.bin'
# Convert to 'gameslug-events-yyyy-mm-dd.json'
filename_patched = filename_prefix + filename.rsplit(
'/', 1)[-1].split('.', 1)[0] + '.json'
output_path = normpath(path_join(output_dir, filename_patched))
if not overwrite and path_exists(output_path):
if not silent:
warning('Skipping existing file: %s' % output_path)
continue
if verbose:
log('Retrieving file: %s' % filename_patched)
if filename.endswith('.bin'):
with open(filename, 'rb') as fin:
file_content = fin.read()
file_content = decrypt_data(file_content, enc_key)
file_content = zlib_decompress(file_content)
else: # if filename.endswith('.json.gz'):
gzip_file = GzipFile(filename=filename, mode='rb')
file_content = gzip_file.read()
gzip_file.close()
file_content = decrypt_data(file_content, enc_key)
write_to_file(options,
file_content,
filename=filename_patched,
output_path=output_path)
except (IOError, OSError) as e:
error(e)
exit(-1)
def get_log_files_s3(options, files_list, enc_key, connection):
verbose = options.verbose
silent = options.silent
overwrite = options.overwrite
output_dir = options.outputdir
filename_prefix = options.project + "-"
try:
for filename in files_list:
# Format v1: 'https://bucket.s3.amazonaws.com/gamefolder/events-yyyy-mm-dd.json?AWSAccessKeyId=keyid
# &Expires=timestamp&Signature=signature'
# Format v2: 'https://bucket.s3.amazonaws.com/gamefolder/events-yyyy-mm-dd.bin?AWSAccessKeyId=keyid
# &Expires=timestamp&Signature=signature'
# Convert to 'gameslug-events-yyyy-mm-dd.json'
filename_cleaned = filename.split("?", 1)[0].rsplit("/", 1)[-1]
filename_patched = filename_prefix + filename_cleaned.split(".", 1)[0] + ".json"
output_path = normpath(path_join(output_dir, filename_patched))
if not overwrite and path_exists(output_path):
if not silent:
warning("Skipping existing file: %s" % output_path)
continue
if verbose:
log("Requesting file: %s" % filename_patched)
r = connection.request("GET", filename, redirect=False)
# pylint: disable=E1103
if r.status != 200:
error_msg = "Couldn't download %s." % filename_patched
if r.data.get("msg", None):
error_msg += " " + r.data["msg"]
error(str(r.status) + error_msg)
exit(-1)
# pylint: enable=E1103
r_data = decrypt_data(r.data, enc_key)
if filename_cleaned.endswith(".bin"):
r_data = zlib_decompress(r_data)
# Format v1 file gets uncompressed on download so we just decrypt it
write_to_file(options, r_data, filename=filename_patched, output_path=output_path)
except (HTTPError, SSLError) as e:
error(e)
exit(-1)
def inline_array_events_local(options, today_log, array_files_list, enc_key):
verbose = options.verbose
to_sort = set()
try:
index = 0
for index, filename in enumerate(array_files_list):
# Format: 'eventlogspath/gamefolder/arrayevents/date(seconds)/objectid.bin'
# The objectid doesn't correspond to a database entry but is used for uniqueness and timestamp
filename = filename.replace("\\", "/")
event_objectid = filename.rsplit("/", 1)[-1].split(".", 1)[0]
timestamp = get_objectid_timestamp(event_objectid)
formatted_timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime(timestamp))
if verbose:
log("Retrieving events file " + str(index + 1) + " submitted at " + formatted_timestamp)
with open(filename, "rb") as fin:
file_content = fin.read()
file_content = decrypt_data(file_content, enc_key)
file_content = json_loads(zlib_decompress(file_content))
if not isinstance(file_content, list):
file_content = [file_content]
for event in file_content:
slug = event["slug"]
del event["slug"]
event["time"] = strftime("%Y-%m-%d %H:%M:%S", gmtime(event["time"]))
if slug not in today_log:
today_log[slug] = {"playEvents": [], "customEvents": []}
today_log[slug]["customEvents"].append(event)
# Maintaining a list of slugs to sort the customEvents by date for so that added array events appear in
# order but we do not unneccesarily sort large lists if an array event wasn't added to it
to_sort.add(slug)
for slug in to_sort:
today_log[slug]["customEvents"].sort(key=lambda k: k["time"])
return today_log
except (IOError, OSError) as e:
error(e)
exit(-1)
def _decompressfcn(compression, objlen, debug=False):
algo, level = compression
if algo == "zlib":
# skip 9-byte header for ROOT's custom frame:
# https://github.com/root-project/root/blob/master/core/zip/src/Bits.h#L646
if debug:
def out(x):
print("decompressing {0} bytes".format(len(x) - 9))
return zlib_decompress(x[9:])
return out
else:
return lambda x: zlib_decompress(x[9:])
elif algo == "lzma":
# skip 9-byte header for LZMA, too:
# https://github.com/root-project/root/blob/master/core/lzma/src/ZipLZMA.c#L81
if debug:
def out(x):
print("decompressing {0} bytes".format(len(x) - 9))
return lzma_decompress(x[9:])
return out
else:
return lambda x: lzma_decompress(x[9:])
elif algo == "lz4":
# skip 9-byte header plus 8-byte hash: are there any official ROOT versions without the hash?
# https://github.com/root-project/root/blob/master/core/lz4/src/ZipLZ4.cxx#L38
if debug:
def out(x):
print("decompressing {0} bytes".format(len(x) - 9 - 8))
return lz4_decompress(x[9 + 8:], uncompressed_size=objlen)
return out
else:
return lambda x: lz4_decompress(x[9 + 8:],
uncompressed_size=objlen)
else:
raise NotImplementedError("cannot decompress \"{0}\"".format(algo))
def get_log_files_local(options, files_list, enc_key):
verbose = options.verbose
silent = options.silent
overwrite = options.overwrite
output_dir = options.outputdir
filename_prefix = options.project + "-"
try:
for filename in files_list:
if filename.startswith("http"):
error("Unexpected file to retrieve")
exit(-1)
# Format v1: 'eventlogspath/gamefolder/events-yyyy-mm-dd.json.gz'
# Format v2: 'eventlogspath/gamefolder/events-yyyy-mm-dd.bin'
# Convert to 'gameslug-events-yyyy-mm-dd.json'
filename_patched = filename_prefix + filename.rsplit("/", 1)[-1].split(".", 1)[0] + ".json"
output_path = normpath(path_join(output_dir, filename_patched))
if not overwrite and path_exists(output_path):
if not silent:
warning("Skipping existing file: %s" % output_path)
continue
if verbose:
log("Retrieving file: %s" % filename_patched)
if filename.endswith(".bin"):
with open(filename, "rb") as fin:
file_content = fin.read()
file_content = decrypt_data(file_content, enc_key)
file_content = zlib_decompress(file_content)
else: # if filename.endswith('.json.gz'):
gzip_file = GzipFile(filename=filename, mode="rb")
file_content = gzip_file.read()
gzip_file.close()
file_content = decrypt_data(file_content, enc_key)
write_to_file(options, file_content, filename=filename_patched, output_path=output_path)
except (IOError, OSError) as e:
error(e)
exit(-1)
def get_url_content(url: str):
try:
req = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
page_open = urlopen(req)
except HTTPError as error:
__print__(
"Webpage({}) not found, Error code: {}"
.format(url, error.code))
return b""
try:
is_encoded = page_open.info()["Content-Encoding"]
except:
is_encoded = None
if is_encoded == "gzip":
return zlib_decompress(page_open.read(), 16 + MAX_WBITS)
elif is_encoded is None:
return page_open.read()
else:
print(is_encoded)
exit(-1)
def extract_mod_data(mod_info, progress, skip_tes4=True,
__unpacker=structs_cache[u'I'].unpack):
"""Reads the headers and EDIDs of every record in the specified mod,
returning them as a dict, mapping record signature to a dict mapping
FormIDs to a list of tuples containing the headers and EDIDs of every
record with that signature. Note that the flags are not processed
either - if you need that, manually call MreRecord.flags1_() on them.
:rtype: defaultdict[bytes, defaultdict[int, tuple[RecHeader, unicode]]]"""
# This method is *heavily* optimized for performance. Inlines and other
# ugly code ahead
progress = progress or bolt.Progress()
# Store a bunch of repeatedly used constants/methods/etc. because
# accessing via dot is slow
wanted_encoding = bolt.pluginEncoding
avoided_encodings = (u'utf8', u'utf-8')
minf_size = mod_info.fsize
minf_ci_key = mod_info.ci_key
sh_unpack = Subrecord.sub_header_unpack
sh_size = Subrecord.sub_header_size
main_progress_msg = _(u'Loading: %s') % minf_ci_key
# Where we'll store all the collected record data
group_records = defaultdict(lambda: defaultdict(tuple))
# The current top GRUP label - starts out as TES4/TES3
tg_label = bush.game.Esp.plugin_header_sig
# The dict we'll use to store records from the current top GRUP
records = group_records[tg_label]
##: Uncomment these variables and the block below that uses them once
# all of FO4's record classes have been written
# The record types that can even contain EDIDs
#records_with_eids = MreRecord.subrec_sig_to_record_sig[b'EDID']
# Whether or not we can skip looking for EDIDs for the current record
# type because it doesn't even have any
#skip_eids = tg_label not in records_with_eids
with mod_info.abs_path.open(u'rb') as ins:
initial_bytes = ins.read()
with FastModReader(minf_ci_key, initial_bytes) as ins:
# More local methods to avoid repeated dot access
ins_tell = ins.tell
ins_seek = ins.seek
ins_read = ins.read
ins_size = ins.size
while ins_tell() != ins_size:
# Unpack the headers - these can be either GRUPs or regular
# records
header = unpack_header(ins)
_rsig = header.recType
if _rsig == b'GRUP':
# Nothing special to do for non-top GRUPs
if not header.is_top_group_header: continue
tg_label = header.label
progress(ins_tell() / minf_size, u'%s\n%s' % (
main_progress_msg, tg_label.decode(u'ascii')))
records = group_records[tg_label]
# skip_eids = tg_label not in records_with_eids
# elif skip_eids:
# # This record type has no EDIDs, skip directly to the next
# # record (can't use skip_blob because that passes
# # debug_strs to seek()...)
# records[header.fid] = (header, u'')
# ins_seek(ins_tell() + header.blob_size())
else:
# This is a regular record, look for the EDID subrecord
eid = u''
blob_siz = header.blob_size()
next_record = ins_tell() + blob_siz
if header.flags1 & 0x00040000: # 'compressed' flag
size_check = __unpacker(ins_read(4))[0]
try:
new_rec_data = zlib_decompress(ins_read(
blob_siz - 4))
except zlib_error:
if minf_ci_key == u'FalloutNV.esm':
# Yep, FalloutNV.esm has a record with broken
# zlib data. Just skip it.
ins_seek(next_record)
continue
raise
if len(new_rec_data) != size_check:
raise ModError(ins.inName,
u'Mis-sized compressed data. Expected %d, got '
u'%d.' % (blob_siz, len(new_rec_data)))
else:
new_rec_data = ins_read(blob_siz)
recs = FastModReader(minf_ci_key, new_rec_data)
recs_seek = recs.seek
recs_read = recs.read
recs_tell = recs.tell
recs_size = recs.size
recs_unpack = recs.unpack
while recs_tell() != recs_size:
# Inlined from unpackSubHeader & FastModReader.unpack
read_data = recs_read(sh_size)
if len(read_data) != sh_size:
raise ModReadError(
minf_ci_key, [_rsig, u'SUB_HEAD'],
recs_tell() - len(read_data), recs_size)
mel_sig, mel_size = sh_unpack(read_data)
# Extended storage - very rare, so don't optimize
# inlines etc. for it
if mel_sig == b'XXXX':
# Throw away size here (always == 0)
mel_size = recs_unpack(__unpacker, 4, _rsig,
u'XXXX.SIZE')[0]
mel_sig = recs_unpack(sh_unpack, sh_size,
_rsig, u'XXXX.TYPE')[0]
if mel_sig == b'EDID':
# No need to worry about newlines, these are Editor
# IDs and so won't contain any
eid = decoder(recs_read(mel_size).rstrip(null1),
wanted_encoding, avoided_encodings)
break
else:
recs_seek(mel_size, 1)
records[header.fid] = (header, eid)
ins_seek(next_record) # we may have break'd at EDID
if skip_tes4:
del group_records[bush.game.Esp.plugin_header_sig]
return group_records
def decompress(stream):
"""
Decompress stream using zlib.
"""
return zlib_decompress(stream)
def read_index(index_path: Path, rsa_priv_key_path: Path = None) -> dict:
if index_path is None or not index_path.is_file():
raise RuntimeError(
f"Unable to read non-existant index file \"{index_path}\"")
encryption_flag = None
compression_flag = None
session_key = None
data_size = None
to_read_buffer = None
with open(index_path, "rb") as index_stream:
magic = str(index_stream.read(7))
if magic != "TINFOIL":
raise RuntimeError(
"Invalid tinfoil index magic.\n\nExpected Magic = " +
f"\"TINFOIL\"\nMagic in index file = \"{magic}\"")
flags = index_stream.read(1)[0]
encryption_flag = flags & 0xF0
key_available = rsa_priv_key_path is not None and \
rsa_priv_key_path.is_file()
if encryption_flag == EncryptionFlag.ENCRYPT and not key_available:
raise RuntimeError(
"Unable to decrypt encrypted index without private key.")
compression_flag = flags & 0x0F
if compression_flag not in CompressionFlag:
raise RuntimeError(
"Unimplemented compression method encountered while reading " +
"index header.")
session_key = index_stream.read(0x100)
data_size = int.from_bytes(index_stream.read(8), byteorder="little")
to_read_buffer = index_stream.read()
if encryption_flag == EncryptionFlag.ENCRYPT:
rsa_priv_key = import_rsa_key(open(rsa_priv_key_path).read())
pkcs1_oaep_ctx = new_pkcs1_oaep_ctx(rsa_priv_key,
hashAlgo=SHA256,
label=b"")
aes_key = pkcs1_oaep_ctx.decrypt(session_key)
aes_ctx = new_aes_ctx(aes_key, MODE_ECB)
to_read_buffer = aes_ctx.decrypt(to_read_buffer)
if compression_flag == CompressionFlag.ZSTD_COMPRESSION:
to_read_buffer = ZstdDecompressor().decompress(
to_read_buffer[:data_size])
elif compression_flag == CompressionFlag.ZLIB_COMPRESSION:
to_read_buffer = zlib_decompress(to_read_buffer[:data_size])
elif compression_flag == CompressionFlag.NO_COMPRESSION:
to_read_buffer = to_read_buffer[:data_size]
try:
return json_deserialize(to_read_buffer)
except JSONDecodeError:
raise RuntimeError("Unable to deserialize index data.")
def out(x):
print("decompressing {0} bytes".format(len(x) - 9))
return zlib_decompress(x[9:])
def test_get_head_object(client: flask.testing.FlaskClient):
master_tree: shahex = _get_master_tree_hex(client)
master_tree_loose = _get_object(client, master_tree)
treedata: bytes = zlib_decompress(master_tree_loose)
assert treedata.find(b"tree") != -1 and treedata.find(b"a.txt") != -1