def _check_decompress(self, frame):
if self.lz4_compress:
from lz4.block import decompress
return np.frombuffer(decompress(frame),
dtype=self.dtype).reshape(self.frame_shape)
return frame
def frames(self):
"""Iterate over all frames in file.
Yields
------
DAQRecord
"""
offset = FILE_HEADER_STRUCT.size
for _ in range(self.num_containers):
(
record_count,
size_compressed,
size_uncompressed,
) = CONTAINER_HEADER_STRUCT.unpack(self.get(offset, CONTAINER_HEADER_STRUCT.size))
offset += CONTAINER_HEADER_STRUCT.size
uncompressed_data = memoryview(lz4block.decompress(self.get(offset, size_compressed)))
frame_offset = 0
for _ in range(record_count):
category, counter, timestamp, frame_length = DAQ_RECORD_STRUCT.unpack(
uncompressed_data[frame_offset : frame_offset + DAQ_RECORD_STRUCT.size]
)
frame_offset += DAQ_RECORD_STRUCT.size
frame_data = uncompressed_data[frame_offset : frame_offset + frame_length] # .tobytes()
frame_offset += len(frame_data)
frame = DAQRecord(category, counter, timestamp, frame_data)
yield frame
offset += size_compressed
def decompress_lz4_buffer(lz4_buffer: BytesIO):
assert lz4_buffer.read(4) == (0x184C2102).to_bytes(
4, 'little') # Check the legacy magic
MAX_LEGACY_BLOCK_SIZE = 8 * 1024 * 1024 # 8 MB
uncompressed_stream = b''
while True:
compressed_block_size_raw = lz4_buffer.read(4)
if len(compressed_block_size_raw) < 4:
break
compressed_block_size = int.from_bytes(compressed_block_size_raw,
'little')
compressed_block = lz4_buffer.read(compressed_block_size)
if len(compressed_block
) < compressed_block_size or not compressed_block_size:
break
uncompressed_block = decompress(compressed_block,
MAX_LEGACY_BLOCK_SIZE)
uncompressed_stream += uncompressed_block
if len(uncompressed_block) < MAX_LEGACY_BLOCK_SIZE:
break
return uncompressed_stream
def get_current_opened_tab_information() -> typ.List[typ.Dict[str, typ.Any]]:
"""Recovers the currently opened tabs.
:return: a list containing all the opened tabs of all the opened Firefox instances.
"""
# First find the "recovery.jsonlz4" file by playing with paths.
home_directory = pathlib.Path.home()
firefox_directory = home_directory / ".mozilla" / "firefox"
profiles_files = firefox_directory / "profiles.ini"
profiles_config = configparser.ConfigParser()
profiles_config.read(profiles_files)
profile0_directory_name = profiles_config["Profile0"]["Path"]
profile0_directory = firefox_directory / profile0_directory_name
profile0_session_backup_directory = profile0_directory / "sessionstore-backups"
compressed_recovery_file = profile0_session_backup_directory / "recovery.jsonlz4"
# Read the "recovery.jsonlz4" by:
# 1. Reading the header and checking its validity.
# 2. Reading the leftover, decompressing, and interpreting it as a JSON text file.
with open(compressed_recovery_file, "rb") as compf:
assert compf.read(8) == b"mozLz40\0"
compressed_recovery_json = compf.read()
uncompressed_recovery_json = lz4.decompress(compressed_recovery_json)
recovery_data = json.loads(uncompressed_recovery_json)
tabs = _get_opened_tabs_from_json_recovery(recovery_data)
return tabs
def read_file(self):
if self.compressed:
if self.file_handle.read(8) != b"mozLz40\0":
raise Exception("Ungueltiger Datenheader")
else:
return block.decompress(self.file_handle.read())
return self.file_handle.read()
def lz4stringtoarr(string, dtype=np.float32, shape=None):
"""
Converts lz4 compressed string to 1d array. Moved here to circumvent
a syconn dependency.
Parameters
----------
string : str
dtype : np.dtype
shape : tuple
Returns
-------
np.array
1d array
"""
if string == "":
return np.zeros((0, ), dtype=dtype)
try:
arr_1d = np.frombuffer(decompress(string), dtype=dtype)
except Exception as e:
print(str(e) + "\nString length:" + str(len(string)))
return np.zeros((0, ), dtype=dtype)
if shape is not None:
arr_1d = arr_1d.reshape(shape)
return arr_1d
def lz4_decompress(data, size):
try:
from lz4.block import decompress
except ImportError:
raise RuntimeError("python-lz4 >= 0.9 is required to read UnityFS files")
return decompress(data, size)
def data():
from_ = request.args.get('from')
if from_ is None:
from_ = '-inf'
else:
from_ = _parse_relative_date(from_)
until = request.args.get('until')
if until is None:
until = '+inf'
else:
until = _parse_relative_date(until)
threshold = float(request.args.get('threshold', 0))
try:
request_data = json.load(six.BytesIO(request.data))
except json.decoder.JSONDecodeError:
request_data = {}
root = Node('root')
with get_db(app.config['DB_URL']) as db:
for key in request_data.get('keys', []):
for el in db.zrangebyscore(key, from_, until):
with six.StringIO(decompress(el).decode()) as data:
for line in data:
root.add_raw(line)
return jsonify(root.serialize(threshold * root.value))
def lz4_decompress(data):
size = LZ4_HEADER.unpack_from(data[:4])[0]
#it would be better to use the max_size we have in protocol,
#but this hardcoded value will do for now
if size>MAX_SIZE:
sizemb = size//1024//1024
maxmb = MAX_SIZE//1024//1024
raise Exception("uncompressed data is too large: %iMB, limit is %iMB" % (sizemb, maxmb))
return decompress(data)
async def get_decompressed_data(self, method_byte, compressed_hash,
extra_header_size):
compressed = await super(Decompressor, self).get_decompressed_data(
method_byte, compressed_hash, extra_header_size)
uncompressed_size = await self.reader.read_uint32()
return block.decompress(compressed,
uncompressed_size=uncompressed_size)
def _ParseDataObject(self, file_object, file_offset):
"""Parses a data object.
Args:
file_object (dfvfs.FileIO): a file-like object.
file_offset (int): offset of the data object relative to the start
of the file-like object.
Returns:
bytes: data.
Raises:
ParseError: if the data object cannot be parsed.
"""
data_object_map = self._GetDataTypeMap('systemd_journal_data_object')
try:
data_object, _ = self._ReadStructureFromFileObject(
file_object, file_offset, data_object_map)
except (ValueError, errors.ParseError) as exception:
raise errors.ParseError((
'Unable to parse data object at offset: 0x{0:08x} with error: '
'{1!s}').format(file_offset, exception))
if data_object.object_type != self._OBJECT_TYPE_DATA:
raise errors.ParseError('Unsupported object type: {0:d}.'.format(
data_object.object_type))
if data_object.object_flags not in (0, self._OBJECT_COMPRESSED_FLAG_XZ,
self._OBJECT_COMPRESSED_FLAG_LZ4):
raise errors.ParseError(
'Unsupported object flags: 0x{0:02x}.'.format(
data_object.object_flags))
# The data is read separately for performance reasons.
data_size = data_object.data_size - 64
data = file_object.read(data_size)
if data_object.object_flags & self._OBJECT_COMPRESSED_FLAG_XZ:
data = lzma.decompress(data)
elif data_object.object_flags & self._OBJECT_COMPRESSED_FLAG_LZ4:
uncompressed_size_map = self._GetDataTypeMap('uint32le')
try:
uncompressed_size = self._ReadStructureFromByteStream(
data, file_offset + 64, uncompressed_size_map)
except (ValueError, errors.ParseError) as exception:
raise errors.ParseError((
'Unable to parse LZ4 uncompressed size at offset: 0x{0:08x} with '
'error: {1!s}').format(file_offset + 64, exception))
data = lz4_block.decompress(data[8:],
uncompressed_size=uncompressed_size)
return data
def get_data(self, prefix, doc_id):
latent_user_factors = np.array([])
doc = {'id': int(doc_id)}
collection = self.mongo_recsys_storage.get_collection(prefix)
mongo_doc = collection.find_one(doc)
if mongo_doc is None:
print("Пользователя с id %(id)d не существует" % mongo_doc)
else:
latent_user_factors = unpackb(decompress(mongo_doc['value']), object_hook=decode)
return latent_user_factors
def __array__(self, dtype=None):
if self.lz4_compress:
from lz4.block import decompress
frames = [np.frombuffer(decompress(frame), dtype=self.dtype).reshape(self.shape) for frame in self._frames]
else:
frames = self._frames
out = np.stack(frames, axis=0)
if dtype is not None:
out = out.astype(dtype)
return out
def is_meeting_active():
with io.open(filePath, "rb") as fd:
fd.read(8) # b"mozLz40\0"
jdata = json.loads(lz4.decompress(fd.read()).decode("utf-8"))
for win in jdata.get("windows"):
for tab in win.get("tabs"):
if not "index" in tab:
continue
i = tab["index"] - 1
url = tab["entries"][i]["url"]
if any(url.startswith(meeting) for meeting in MEETING_URLS):
return True
return False
def _read_one_frame_16bit_linear(self, frame_index, resize_max_side):
if frame_index<0 or frame_index>=self._frame_count:
raise Exception('Invalid frame index %s' % frame_index)
if self._raise_error_on_missing_frame and not self._frame_indices[frame_index]:
raise Exception('Missing frame index %s' % frame_index)
compressed_buffer = self._read_frame(frame_index)
buffer = lz4block.decompress(compressed_buffer, uncompressed_size=self._img_data_size)
raw_img = np.fromstring(buffer, np.uint8 if self.bitcount==8 else np.uint16).reshape((self.height,self.width))
return raw_processing_to_16bit_linear(raw_img, self.bayer, self.blacklevel, self.bitcount, self.kB, self.kG, self.kR, resize_max_side=resize_max_side)
def lz4stringtoarr(string: bytes, dtype: np.dtype = np.float32,
shape: Optional[Tuple[int]] = None):
"""
Converts lz4 compressed string to 1d array.
Args:
string: Serialized array.
dtype: Data type of original array.
shape: Shape of original array.
Returns:
N-dimensional numpy array.
"""
if len(string) == 0:
return np.zeros((0, ), dtype=dtype)
try:
arr_1d = np.frombuffer(decompress(string), dtype=dtype)
except TypeError: # python3 compatibility
arr_1d = np.frombuffer(decompress(str.encode(string)), dtype=dtype)
if shape is not None:
arr_1d = arr_1d.reshape(shape)
return arr_1d
def dump_example(dataset_name):
print("Converting {:}.h5 ...".format(dataset_name))
file = h5py.File(os.path.join(path, "traindata", "{:}.h5".format(dataset_name)), "r")
for (seq_idx, seq_name) in enumerate(file):
if dataset_name == 'scenes11_train':
scale = 0.4
else:
scale = 1
if ((dataset_name == 'sun3d_train_1.6m_to_infm' and seq_idx == 7) or \
(dataset_name == 'sun3d_train_0.4m_to_0.8m' and seq_idx == 15) or \
(dataset_name == 'scenes11_train' and (seq_idx == 2758 or seq_idx == 4691 or seq_idx == 7023 or seq_idx == 11157 or seq_idx == 17168 or seq_idx == 19595))):
continue # Skip error files
print("Processing sequence {:d}/{:d}".format(seq_idx, len(file)))
dump_dir = os.path.join(path, '../train', dataset_name + "_" + "{:05d}".format(seq_idx))
if not os.path.isdir(dump_dir):
os.mkdir(dump_dir)
dump_dir = Path(dump_dir)
sequence = file[seq_name]["frames"]["t0"]
poses = []
for (f_idx, f_name) in enumerate(sequence):
frame = sequence[f_name]
for dt_type in frame:
dataset = frame[dt_type]
img = dataset[...]
if dt_type == "camera":
if f_idx == 0:
intrinsics = np.array([[img[0], 0, img[3]], [0, img[1], img[4]], [0, 0, 1]])
pose = np.array([[img[5],img[8],img[11],img[14]*scale], [img[6],img[9],img[12],img[15]*scale], [img[7],img[10],img[13],img[16]*scale]])
poses.append(pose.tolist())
elif dt_type == "depth":
dimension = dataset.attrs["extents"]
depth = np.array(np.frombuffer(decompress(img.tobytes(), dimension[0] * dimension[1] * 2), dtype = np.float16)).astype(np.float32)
depth = depth.reshape(dimension[0], dimension[1])*scale
dump_depth_file = dump_dir/'{:04d}.npy'.format(f_idx)
np.save(dump_depth_file, depth)
elif dt_type == "image":
img = imageio.imread(img.tobytes())
dump_img_file = dump_dir/'{:04d}.jpg'.format(f_idx)
scipy.misc.imsave(dump_img_file, img)
dump_cam_file = dump_dir/'cam.txt'
np.savetxt(dump_cam_file, intrinsics)
poses_file = dump_dir/'poses.txt'
np.savetxt(poses_file, np.array(poses).reshape(-1, 12), fmt='%.6e')
if len(dump_dir.files('*.jpg')) < 2:
dump_dir.rmtree()
def open_lz4(file: Union[str, Path]) -> BinaryIO:
"""Reads a mozilla lz4 file and decompresses it in memory while returning it
as `BytesIO`
"""
with open(file, 'rb') as fp:
# read mozilla header
# thanks to https://github.com/jscher2000/Firefox-File-Utilities
header = fp.read(8)
if header != b'mozLz40\0':
raise RuntimeError(
f'invalid header {header!r} for mozilla lz4 file')
data = fp.read()
return BytesIO(decompress(data))
def _force(self):
if self._out is None:
if self.lz4_compress:
from lz4.block import decompress
frames = [
numpy.frombuffer(decompress(frame),
dtype=self.dtype).reshape(self.shape)
for frame in self._frames
]
else:
frames = self._frames
self._out = numpy.concatenate(frames, axis=-1)
self._frames = None
return self._out
def lz4stringtoarr(string, dtype=np.float32, shape=None):
"""
Converts lz4 compressed string to 1d array.
Parameters
----------
string : byte
dtype : type
shape : tuple
Returns
-------
np.array
1d array
"""
if len(string) == 0:
return np.zeros((0, ), dtype=dtype)
try:
arr_1d = np.frombuffer(decompress(string), dtype=dtype)
except TypeError: # python3 compatibility
arr_1d = np.frombuffer(decompress(str.encode(string)), dtype=dtype)
if shape is not None:
arr_1d = arr_1d.reshape(shape)
return arr_1d
def read(self, input):
inputData = bytes(self.compressedChunkSize)
outputData = bytes(self.decompressedChunkSize)
count = 0
uncompressedSize = 0
for eachByte in inputData:
count += 1
for eachByte in outputData:
uncompressedSize += 1
inputData = bytes(input.read(count))
outputData = block.decompress(inputData,uncompressedSize,return_bytearray = True)
return outputData
def open_firefox_session():
# hardcoded for PoC
session="/home/thomas/.mozilla/firefox.backup/p0i4j91j.default-backup-crashrecovery-20190418_144722/sessionstore-backups/previous.jsonlz4"
# firefox format:
# windows->tabs->entries (can have sub windows->tabs->entries)
# qutebrowser format:
# {'windows': tabs: { history:{active:true, pinned:false, title:Foo, url:
# Bar, zoom: 1.0}
# history: original_url, url, url, ...
sessf = open(session, 'rb')
assert sessf.read(8) == b'mozLz40\0'
# sessf.seek(8)
# print(sessf.read())
fsession = json.loads(lz4.decompress(sessf.read()))
qute = parse_firefox_windows(fsession["windows"])
# print(fsession)
print(yaml.dump(qute, default_flow_style=False))
def __array__(self) -> np.ndarray:
"""
Makes the LazyFrames object convertible to a NumPy array
"""
if self.compress:
from lz4.block import decompress
frames = [
np.frombuffer(decompress(frame),
dtype=self._frames[0].dtype).reshape(
self._frames[0].shape)
for frame in self._frames
]
else:
frames = self._frames
return np.stack(frames, axis=0)
def _read_msg(self):
"""Reads a message from peer BEP node"""
header = protocol.Header()
header_size = unpack_short(self._read(2))
header.ParseFromString(self._read(header_size))
response = ResponsesByTypeValue[header.type]()
msg_size = unpack_int(self._read(4))
res = self._read(msg_size)
if header.compression == protocol.LZ4:
res = res[3::-1] + res[
4:] # message int size header must be converted to little endian
res = lz4.decompress(res)
response.ParseFromString(res)
return response
def saver():
location = "/home/kush/.mozilla/firefox/56s5hya4.default/sessionstore-backups/recovery.jsonlz4"
try:
import lz4.block as lz4
except ImportError:
import lz4
stdin = open(location, 'rb')
# name of file here
stdout = open(
'/home/kush/.mozilla/firefox/56s5hya4.default/sessionstore-backups/decrypted_state.json',
'wb')
assert stdin.read(8) == b'mozLz40\0'
stdout.write(lz4.decompress(stdin.read()))
stdin.close()
stdout.close()
def get_decompressed_data(self, method_byte, compressed_hash,
extra_header_size):
size_with_header = read_binary_uint32(self.stream)
compressed_size = size_with_header - extra_header_size - 4
compressed = BytesIO(self.stream.read(compressed_size))
block_check = BytesIO()
write_binary_uint8(method_byte, block_check)
write_binary_uint32(size_with_header, block_check)
block_check.write(compressed.getvalue())
self.check_hash(block_check.getvalue(), compressed_hash)
uncompressed_size = read_binary_uint32(compressed)
compressed = compressed.read(compressed_size - 4)
return block.decompress(compressed,
uncompressed_size=uncompressed_size)
def decompress(file_obj):
if file_obj.read(8) != b"mozLz40\0":
raise InvalidHeader("Invalid magic number")
return lz4.decompress(file_obj.read())
def decompress(file_obj):
if file_obj.read(8) != b"mozLz40\0":
raise InvalidHeader("Invalid magic number")
return lz4.decompress(file_obj.read())
def decodeMii(data):
return decompress(b64decode(data.encode()), uncompressed_size=76)
def lz4_decompress(byts):
# flip from big-endian to little-endian
return lz4_block.decompress(byts[3::-1] + byts[4:])
def decompress_by_name(data, algo):
assert algo in NAME_TO_FLAG, "invalid compression algorithm: %s" % algo
flag = NAME_TO_FLAG[algo]
return decompress(data, flag)
def content(self) -> str:
content = decompress(self.data['content']).decode('utf-8')
return content
