def write_files(self, f: BinaryIO) -> None:
"""Writes file data to f.
Args:
f (BinaryIO): file-like output stream
"""
# TODO: name bstring Full path and name of the file. Only present if Bit 9 of archiveFlags is set.
self.i = 0
total = self.files_count
for folder in self.sorted_folders:
for file in folder.sorted_files:
p = f"{file.folder.name}\\{file.name}"
# logging.info(f"Writing {p:100s}[{(i * 100) / total:2.2f}%]")
data_start = f.tell()
with open(os.path.join(self.data_path, folder.name, file.name), "rb") as o:
if not self.compress:
f.write(o.read())
else:
uncompressed_data = o.read()
compressed_data = lz4.frame.compress(uncompressed_data, compression_level=self.compression_level)
f.write(struct.pack("<L", len(uncompressed_data)))
f.write(compressed_data)
size = f.tell() - data_start
f.seek(file.record_offset + 8)
f.write(struct.pack("<LL", size + (4 if self.compress else 0), data_start))
f.seek(0, os.SEEK_END)
self.i += 1
def write_file_records(self, f: BinaryIO) -> None:
"""Writes the file records block to f
Args:
f (BinaryIO): file-like output stream
"""
logging.debug("Writing file records")
for folder in self.sorted_folders:
logging.debug(f"Processing file records for folder {folder.name}")
offset = f.tell()
f.seek(folder.record_offset + 8 + 4 + 4)
f.write(struct.pack("<Q", offset + self.total_file_name_length))
f.seek(0, os.SEEK_END)
if (self.flags & Flags.INCLUDE_DIRECTORY_NAMES) > 0:
f.write(pack_str(folder.name))
logging.debug(f"Sorted files in {folder.name}: {[x.tes_hash for x in folder.sorted_files]}")
for file in folder.sorted_files:
file.record_offset = f.tell()
f.write(
struct.pack(
"<QLL",
file.tes_hash,
0,
0
)
)
def read(self, file_input: BinaryIO, file_size: int) -> Flags:
"""populate header data"""
first_dg = (file_input.tell() == 0)
chunk = file_input.read(16)
hdr_data = struct.unpack("<IBBHII", chunk)
self.length = hdr_data[0]
self.stx = hdr_data[1]
self.id = hdr_data[2]
self.model = hdr_data[3]
self.date = hdr_data[4]
self.time = hdr_data[5]
if first_dg and self.stx != 2:
if self.verbose:
logger.warning("invalid Kongberg file > STX: %s" % self.stx)
return self.Flags.MISSING_FIRST_STX
if (self.stx != 2) or (self.id == 0):
if self.verbose:
logger.warning("corrupted datagram")
return self.Flags.CORRUPTED_START_DATAGRAM
# try to read ETX
# Make sure we don't try to read beyond the EOF (-13 since 16 for header and 3 for ender)
if (file_input.tell() + (self.length - 13)) >= file_size:
if self.verbose:
logger.warning("unexpected EOF > current pos: %s, datagram length: %s, file size: %s"
% (file_input.tell(), self.length, file_size))
return self.Flags.UNEXPECTED_EOF
# move file cursor to the end of the datagram
file_input.seek(self.length - 15, 1)
chunk = file_input.read(3)
footer_data = struct.unpack("<BH", chunk)
self.etx = footer_data[0]
self.checksum = footer_data[1]
if self.etx != 3:
# print 'ETX not found, trying next datagram at position',file.tell()-(length+3)
return self.Flags.CORRUPTED_END_DATAGRAM
return self.Flags.VALID
def read(self, file_input: BinaryIO, file_size: int) -> Flags:
"""populate header data"""
chunk = file_input.read(20)
hdr_data = struct.unpack("<I4cBBHII", chunk)
self.length = hdr_data[0]
# logger.debug('length: %s' % self.length)
self.id = b''.join(hdr_data[1:5])
# logger.debug('type: %s -> %s' % (self.type, self.kmall_datagrams[self.type]))
self.version = hdr_data[5]
# logger.debug('version: %s' % self.version)
self.system_id = hdr_data[6]
# logger.debug('system id: %s' % self.system_id)
self.sounder_id = hdr_data[7]
# logger.debug('sounder id: %s' % self.sounder_id)
self.time_sec = hdr_data[8]
# logger.debug('time sec: %s' % self.time_sec)
self.time_nanosec = hdr_data[9]
# logger.debug('time nanosec: %s' % self.time_nanosec)
self.dg_time = self.kmall_datetime(self.time_sec, self.time_nanosec)
# logger.debug('datetime: %s' % self.dg_time.strftime('%Y-%m-%d %H:%M:%S.%f'))
# try to read ETX
# Make sure we don't try to read beyond the EOF (-13 since 16 for header and 3 for ender)
if (file_input.tell() + (self.length - 20)) >= file_size:
if self.verbose:
logger.warning("unexpected EOF > current pos: %s, datagram length: %s, file size: %s"
% (file_input.tell(), self.length, file_size))
return self.Flags.UNEXPECTED_EOF
# move file cursor to the end of the datagram
file_input.seek(self.length - 24, 1)
chunk = file_input.read(4)
footer_length = struct.unpack("<I", chunk)[0]
if footer_length != self.length:
logger.info("datagram length mismatch: %s vs. %s" % (self.length, footer_length))
return self.Flags.CORRUPTED_END_DATAGRAM
return self.Flags.VALID
def read(self, file_input: BinaryIO, file_size: int) -> Flags:
"""populate header data"""
chunk = file_input.read(20)
hdr_data = struct.unpack("<I4cBBHII", chunk)
self.length = hdr_data[0]
# logger.debug('length: %s' % self.length)
self.type = b''.join(hdr_data[1:5])
# logger.debug('type: %s -> %s' % (self.type, self.kmall_datagrams[self.type]))
self.version = hdr_data[5]
# logger.debug('version: %s' % self.version)
self.system_id = hdr_data[6]
# logger.debug('system id: %s' % self.system_id)
self.sounder_id = hdr_data[7]
# logger.debug('sounder id: %s' % self.sounder_id)
self.time_sec = hdr_data[8]
# logger.debug('time sec: %s' % self.time_sec)
self.time_microsec = hdr_data[9]
# logger.debug('time microsec: %s' % self.time_microsec)
self.datetime = datetime.utcfromtimestamp(self.time_sec) + timedelta(
microseconds=(self.time_microsec * 10e-3))
# logger.debug('datetime: %s' % self.datetime.strftime('%Y-%m-%d %H:%M:%S.%f'))
# Make sure we don't try to read beyond the EOF (-13 since 16 for header and 3 for ender)
if (file_input.tell() + (self.length - 20)) > file_size:
if self.verbose:
logging.warning(
"unexpected EOF > current pos: %s, datagram length: %s, file size: %s"
% (file_input.tell(), self.length, file_size))
return self.Flags.UNEXPECTED_EOF
# move file cursor to almost the end of the datagram (just minus the length field)
file_input.seek(self.length - 24, 1) # 1 -> current file position
chunk = file_input.read(4)
footer_length = struct.unpack("<I", chunk)[0]
if self.length != footer_length:
logging.warning(
"mismatch between initial and end datagram length: %s != %s" %
(self.length, footer_length))
return self.Flags.CORRUPTED_END_DATAGRAM
return self.Flags.VALID
def __send_file(self, file: BinaryIO, filename: str):
"""
Sends a file in the socket.
:param file: file to be sent
"""
self.__send_string(filename)
# get file size
file.seek(0, io.SEEK_END)
file_len = file.tell()
file.seek(0)
self.__send_int(file_len)
self.socket.sendfile(file)
def write_folder_records(self, f: BinaryIO) -> None:
"""Writes the folder records block to f
Args:
f (BinaryIO): file-like output stream
"""
# And write their info
logging.debug("Writing folder records")
logging.debug(f"Sorted folder hashes: {[x.tes_hash for x in self.sorted_folders]}")
for folder in self.sorted_folders:
folder.record_offset = f.tell()
f.write(
struct.pack(
"<QLLQ",
folder.tes_hash,
len(folder.files),
0,
0
)
)
