def _create_log_pass():
ld = File.LogicalData(BYTES_CHANNEL)
channels = EFLR.ExplicitlyFormattedLogicalRecord(3, ld)
ld = File.LogicalData(BYTES_FRAME)
frame = EFLR.ExplicitlyFormattedLogicalRecord(4, ld)
log_pass = LogPass.log_pass_from_RP66V1(frame, channels)
return log_pass
def test_logical_record_segment_encryption_packet_ctor():
ld = File.LogicalData(ENCRYPTION_EXAMPLE_BYTES)
lrsep = Encryption.LogicalRecordSegmentEncryptionPacket(ld)
assert lrsep.size == 4
assert lrsep.producer_code == 440
assert lrsep.encryption_information == b'\x00\x00\x00\x00'
assert lrsep.bytes == b'\x00\x00'
assert ld.remain == 0
def index_a_single_file(path_in: str, path_out: str, read_back: bool,
validate: bool) -> IndexResult:
"""Read a single file and return an IndexResult."""
file_type = bin_file_type.binary_file_type_from_path(path_in)
if file_type == 'RP66V1':
if path_out:
out_dir = os.path.dirname(path_out)
if not os.path.exists(out_dir):
logger.info(f'Making directory: {out_dir}')
os.makedirs(out_dir, exist_ok=True)
logger.info(f'Indexing {path_in}')
try:
t_start = time.perf_counter()
# Index the file
# process.add_message_to_queue(f'Start {os.path.basename(path_in)}')
with Index.LogicalRecordIndex(path_in) as logical_record_index:
index_time = time.perf_counter() - t_start
if validate:
try:
logical_record_index.validate()
logger.info(f'Validation OK for {path_in}')
except File.ExceptionFileReadPositionsInconsistent(
) as err:
logger.exception(
f'Validation of {path_in} failed with {err}')
t_start = time.perf_counter()
# process.add_message_to_queue(f'Pickle {os.path.basename(path_in)}')
# Pickle the index
if path_out:
pickle_path = path_out + '.pkl'
with open(pickle_path, 'wb') as pickle_file:
pickle.dump(logical_record_index, pickle_file)
len_pickled_index = os.path.getsize(pickle_path)
write_time = time.perf_counter() - t_start
if read_back:
t_start = time.perf_counter()
_read_index = unpickle(pickle_path)
read_back_time = time.perf_counter() - t_start
else:
read_back_time = 0.0
else:
pickled_index = pickle.dumps(logical_record_index)
write_time = time.perf_counter() - t_start
len_pickled_index = len(pickled_index)
read_back_time = 0.0
result = IndexResult(path_in, os.path.getsize(path_in),
len_pickled_index, index_time, write_time,
read_back_time, False, False)
return result
except ExceptionTotalDepthRP66V1: # pragma: no cover
logger.exception(
f'Failed to index with ExceptionTotalDepthRP66V1: {path_in}')
except Exception: # pragma: no cover
logger.exception(f'Failed to index with Exception: {path_in}')
return IndexResult(path_in, os.path.getsize(path_in), 0, 0.0, 0.0, 0.0,
True, False) # pragma: no cover
return IndexResult(path_in, os.path.getsize(path_in), 0, 0.0, 0.0, 0.0,
False, True) # pragma: no cover
def test_iflr_ctor_zero_not_empty():
ld = File.LogicalData(IFLR_ZERO_NOT_EMPTY)
iflr = IFLR.IndirectlyFormattedLogicalRecord(1, ld)
assert iflr.lr_type == 1
assert iflr.object_name == RepCode.ObjectName(O=11, C=0, I=b'0B')
assert iflr.frame_number == 0
assert iflr.preamble_length == 6
assert iflr.remain == 36
assert iflr.remain == ld.remain
def test_iflr_ctor(frame_number, by):
ld = File.LogicalData(by)
iflr = IFLR.IndirectlyFormattedLogicalRecord(1, ld)
assert iflr.lr_type == 1
assert iflr.object_name == RepCode.ObjectName(O=11, C=0, I=b'0B')
assert iflr.frame_number == frame_number
assert iflr.preamble_length == 6
assert iflr.remain == 36
assert iflr.remain == ld.remain
def test_eflr_channel_attributes():
ld = File.LogicalData(BYTES_CHANNEL)
eflr = EFLR.ExplicitlyFormattedLogicalRecord(
Types.EFLR_PUBLIC_SET_TYPE_TO_CODE_MAP[b'CHANNEL'], ld)
print(eflr)
print(eflr.str_long())
assert str(
eflr
) == "<ExplicitlyFormattedLogicalRecord EFLR Set type: b'CHANNEL' name: b'59'>"
assert eflr.str_long(
) == """<ExplicitlyFormattedLogicalRecord EFLR Set type: b'CHANNEL' name: b'59'>
def test_eflr_frame_attributes():
ld = File.LogicalData(BYTES_FRAME)
frame = EFLR.ExplicitlyFormattedLogicalRecord(
Types.EFLR_PUBLIC_SET_TYPE_TO_CODE_MAP[b'FRAME'], ld)
assert len(frame.objects) == 1
# print()
# print(frame)
assert str(
frame
) == "<ExplicitlyFormattedLogicalRecord EFLR Set type: b'FRAME' name: b'60'>"
# print(frame.str_long())
assert frame.str_long(
) == """<ExplicitlyFormattedLogicalRecord EFLR Set type: b'FRAME' name: b'60'>
def test_XAxis_getitem(position_xvalues, expected):
x_axis = XAxis.XAxis(ident=b'A', long_name=b'B', units=b'C')
x_values = []
for frame_number, (vr_postion, lrsh_position,
x_value) in enumerate(position_xvalues):
x_axis.append(
File.LogicalRecordPositionBase(vr_postion, lrsh_position),
frame_number + 1, x_value)
x_values.append(x_value)
assert len(x_axis) == len(expected)
# print()
for i in range(len(x_axis)):
# print(x_axis[i])
assert x_axis[i] == expected[i]
def test_XAxis_append_summary(position_xvalues, exp_summary):
xaxis = XAxis.XAxis(ident=b'A', long_name=b'B', units=b'C')
x_values = []
for frame_number, (vr_postion, lrsh_position,
x_value) in enumerate(position_xvalues):
xaxis.append(File.LogicalRecordPositionBase(vr_postion, lrsh_position),
frame_number + 1, x_value)
x_values.append(x_value)
result = xaxis.summary
# print(result)
assert result.count == len(x_values)
assert result.min == min(x_values)
assert result.max == max(x_values)
array = np.array(x_values)
spacing_summary = XAxis.compute_spacing(array)
assert result.spacing == spacing_summary
def test_eflr_channel_obnames():
ld = File.LogicalData(BYTES_CHANNEL)
channels = EFLR.ExplicitlyFormattedLogicalRecord(
Types.EFLR_PUBLIC_SET_TYPE_TO_CODE_MAP[b'CHANNEL'], ld)
channel_obnames = [obj.name for obj in channels.objects]
# print(channel_obnames)
assert channel_obnames == [
RepCode.ObjectName(O=11, C=0, I=b'DEPT'),
RepCode.ObjectName(O=11, C=0, I=b'INC'),
RepCode.ObjectName(O=11, C=0, I=b'AZI'),
RepCode.ObjectName(O=11, C=0, I=b'MTTVD'),
RepCode.ObjectName(O=11, C=0, I=b'SECT'),
RepCode.ObjectName(O=11, C=0, I=b'RCN'),
RepCode.ObjectName(O=11, C=0, I=b'RCE'),
RepCode.ObjectName(O=11, C=0, I=b'DLSEV'),
RepCode.ObjectName(O=11, C=0, I=b'TLTS'),
]
def dump_RP66V1_test_data(fobj: typing.BinaryIO, fout: typing.TextIO, **kwargs) -> None:
"""Scans the file reporting Visible Records, optionally Logical Record Segments as well."""
with _output_section_header_trailer('File as Raw Test Data', '*', os=fout):
with File.FileRead(fobj) as rp66_file:
count_vr = 0
count_lrsh = 0
count_lrsh_first = 0
fout.write(f'{rp66_file.sul.as_bytes()} # Storage Unit Label\n')
for visible_record in rp66_file.iter_visible_records():
fout.write(
f'{visible_record.as_bytes()}'
f' # Visible record [{count_vr}]'
f' at 0x{visible_record.position:x}'
f' length 0x{visible_record.length:x}'
f' version 0x{visible_record.version:x}\n'
)
for lrsh, by in rp66_file.iter_LRSHs_for_visible_record_and_logical_data_fragment(visible_record):
record_type = 'E' if lrsh.attributes.is_eflr else 'I'
fout.write(
f' {lrsh.as_bytes()}'
f' # LRSH [{count_lrsh_first}/{count_lrsh}] 0x{lrsh.position:x} {record_type} len: {lrsh.length}'
f' first: {lrsh.attributes.is_first} last: {lrsh.attributes.is_last}\n'
)
# fout.write(f' {by} # Logical data length {len(by)} 0x{len(by):x}\n')
WIDTH = 20
# str_list = []
# for i in range(0, len(by), WIDTH):
# str_list.append(f'{by[i:i+WIDTH]}')
str_list = [f'{by[i:i+WIDTH]}' for i in range(0, len(by), WIDTH)]
if len(str_list) > 1:
fout.write(f' # Logical data length {len(by)} 0x{len(by):x}\n')
FW = max(len(s) for s in str_list)
for i, s in enumerate(str_list):
fout.write(
f' {s:{FW}} # Chunk from {i * WIDTH}\n'
)
else:
fout.write(f' {by} # Logical data length {len(by)} 0x{len(by):x}\n')
if lrsh.attributes.is_first:
count_lrsh_first += 1
elif lrsh.attributes.is_last:
pass
else:
pass
count_lrsh += 1
count_vr += 1
def test_eflr_frame_channels():
ld = File.LogicalData(BYTES_FRAME)
frame = EFLR.ExplicitlyFormattedLogicalRecord(
Types.EFLR_PUBLIC_SET_TYPE_TO_CODE_MAP[b'FRAME'], ld)
obj = frame[0]
channels = obj[b'CHANNELS'].value
assert len(channels) == 9
assert channels == [
RepCode.ObjectName(O=11, C=0, I=b'DEPT'),
RepCode.ObjectName(O=11, C=0, I=b'INC'),
RepCode.ObjectName(O=11, C=0, I=b'AZI'),
RepCode.ObjectName(O=11, C=0, I=b'MTTVD'),
RepCode.ObjectName(O=11, C=0, I=b'SECT'),
RepCode.ObjectName(O=11, C=0, I=b'RCN'),
RepCode.ObjectName(O=11, C=0, I=b'RCE'),
RepCode.ObjectName(O=11, C=0, I=b'DLSEV'),
RepCode.ObjectName(O=11, C=0, I=b'TLTS')
]
def scan_RP66V1_file_EFLR_IFLR(fobj: typing.BinaryIO, fout: typing.TextIO, **kwargs) -> None:
"""Scans the file reporting the individual EFLR and IFLR."""
verbose = kwargs.get('verbose', 0)
if not verbose:
fout.write(colorama.Fore.YELLOW + 'Use -v to see individual logical data.\n')
# TODO: eflr_dump is never present
dump_eflr = kwargs.get('eflr_dump', 0)
eflr_set_type = kwargs.get('eflr_set_type', [])
iflr_dump = kwargs.get('iflr_dump', 0)
iflr_set_type = kwargs.get('iflr_set_type', [])
# TODO: Use both of these
# dump_bytes = kwargs.get('dump_bytes', 0)
# dump_raw_bytes = kwargs.get('dump_raw_bytes', 0)
# if not dump_bytes:
# fout.write(colorama.Fore.YELLOW + 'Use -v and --dump-bytes to see actual first n bytes.\n')
with _output_section_header_trailer('RP66V1 EFLR and IFLR Data Summary', '*', os=fout):
with File.FileRead(fobj) as rp66_file:
# TODO: use data_table.format_table
vr_position = 0
header = [
f'{"Visible R":10}',
f'{"LRSH":10}',
f'{"Typ":3}',
f'{" "}',
f'{" "}',
f'{"Length":8}',
]
underline = ['-' * len(h) for h in header]
if verbose:
fout.write(' '.join(header) + '\n')
fout.write(' '.join(underline) + '\n')
for file_logical_data in rp66_file.iter_logical_records():
if file_logical_data.lr_is_eflr:
if file_logical_data.lr_is_encrypted:
if kwargs['encrypted']:
if verbose:
fout.write(colorama.Fore.MAGENTA + f'Encrypted EFLR: {file_logical_data}' + colorama.Style.RESET_ALL)
else:
fout.write(colorama.Fore.MAGENTA + f'Encrypted EFLR: {file_logical_data.position}' + colorama.Style.RESET_ALL)
fout.write('\n')
else:
eflr = EFLR.ExplicitlyFormattedLogicalRecord(file_logical_data.lr_type, file_logical_data.logical_data)
if dump_eflr and len(eflr_set_type) == 0 or eflr.set.type in eflr_set_type:
lines = str(eflr).split('\n')
for i, line in enumerate(lines):
if i == 0:
fout.write(colorama.Fore.MAGENTA + line + colorama.Style.RESET_ALL)
else:
fout.write(line)
fout.write('\n')
else:
# IFLR
if iflr_dump and verbose:
if file_logical_data.lr_is_encrypted:
if kwargs['encrypted']:
if verbose:
fout.write(colorama.Fore.MAGENTA + f'Encrypted IFLR: {file_logical_data}' + colorama.Style.RESET_ALL)
else:
fout.write(colorama.Fore.MAGENTA + f'Encrypted IFLR: {file_logical_data.position}' + colorama.Style.RESET_ALL)
fout.write('\n')
else:
iflr = IFLR.IndirectlyFormattedLogicalRecord(file_logical_data.lr_type, file_logical_data.logical_data)
if len(iflr_set_type) == 0 or iflr.object_name.I in iflr_set_type:
fout.write(str(iflr))
fout.write('\n')
def test_eflr_frame_ctor():
ld = File.LogicalData(BYTES_FRAME)
EFLR.ExplicitlyFormattedLogicalRecord(
Types.EFLR_PUBLIC_SET_TYPE_TO_CODE_MAP[b'FRAME'], ld)
assert result.min == min(x_values)
assert result.max == max(x_values)
array = np.array(x_values)
spacing_summary = XAxis.compute_spacing(array)
assert result.spacing == spacing_summary
@pytest.mark.parametrize(
'position_xvalues, expected',
(
(
[
(0x0, 0xff, 1.0),
],
[
XAxis.IFLRReference(logical_record_position=File.LogicalRecordPositionBase(0x0, 0xff),
frame_number=1, x_axis=1.0),
],
),
(
[
(0x0, 0xff, 1.0),
(0x2, 0x2ff, 2.0),
],
[
XAxis.IFLRReference(logical_record_position=File.LogicalRecordPositionBase(0x0, 0xff),
frame_number=1, x_axis=1.0),
XAxis.IFLRReference(logical_record_position=File.LogicalRecordPositionBase(0x2, 0x2ff),
frame_number=2, x_axis=2.0),
],
),
def _iflr_and_logical_data_from_bytes(by: bytes) -> typing.Tuple[IFLR.IndirectlyFormattedLogicalRecord, File.LogicalData]:
ld = File.LogicalData(by)
iflr = IFLR.IndirectlyFormattedLogicalRecord(1, ld)
return iflr, ld
def _eflr_frame() -> EFLR.ExplicitlyFormattedLogicalRecord:
ld = File.LogicalData(BYTES_EFLR_FRAME)
return EFLR.ExplicitlyFormattedLogicalRecord(4, ld)
def _eflr_channel() -> EFLR.ExplicitlyFormattedLogicalRecord:
ld = File.LogicalData(BYTES_EFLR_CHANNEL)
return EFLR.ExplicitlyFormattedLogicalRecord(3, ld)
def scan_RP66V1_file_logical_data(fobj: typing.BinaryIO, fout: typing.TextIO, **kwargs) -> None:
"""Scans the file reporting the raw Logical Data."""
verbose = kwargs.get('verbose', 0)
if not verbose:
fout.write(colorama.Fore.YELLOW + 'Use -v to see individual logical data.\n')
dump_bytes = kwargs.get('dump_bytes', 0)
dump_raw_bytes = kwargs.get('dump_raw_bytes', 0)
if not dump_bytes:
fout.write(colorama.Fore.YELLOW + 'Use -v and --dump-bytes to see actual first n bytes.\n')
# Both a dict of {record_type : collections.Counter(length)
count_eflr_type_length_count = {}
count_iflr_type_length_count = {}
with _output_section_header_trailer('RP66V1 Logical Data Summary', '*', os=fout):
with File.FileRead(fobj) as rp66_file:
vr_position = 0
header = [
f'{"Visible R":10}',
f'{"LRSH":10}',
f'{"Typ":3}',
f'{" "}',
f'{" "}',
f'{"Length":8}',
]
underline = ['-' * len(h) for h in header]
if verbose:
fout.write(' '.join(header) + '\n')
fout.write(' '.join(underline) + '\n')
for logical_data in rp66_file.iter_logical_records():
if logical_data.lr_is_eflr:
if logical_data.lr_type not in count_eflr_type_length_count:
count_eflr_type_length_count[logical_data.lr_type] = collections.Counter()
count_eflr_type_length_count[logical_data.lr_type].update([len(logical_data)])
else:
if logical_data.lr_type not in count_iflr_type_length_count:
count_iflr_type_length_count[logical_data.lr_type] = collections.Counter()
count_iflr_type_length_count[logical_data.lr_type].update([len(logical_data)])
if verbose:
messages = [
f'0x{logical_data.position.vr_position:08x}' if logical_data.position.vr_position != vr_position else ' ' * 10,
f'0x{logical_data.position.lrsh_position:08x}',
f'{logical_data.lr_type:3d}',
f'{"E" if logical_data.lr_is_eflr else "I"}',
f'{"Crypt" if logical_data.lr_is_encrypted else "Plain"}',
f'{len(logical_data.logical_data):8,d}',
]
if dump_bytes:
if dump_bytes == -1:
if dump_raw_bytes:
messages.append(str(logical_data.logical_data.bytes))
else:
messages.append(format_bytes(logical_data.logical_data.bytes))
else:
if dump_raw_bytes:
messages.append(str(logical_data.logical_data.bytes[:dump_bytes]))
else:
messages.append(format_bytes(logical_data.logical_data.bytes[:dump_bytes]))
fout.write(' '.join(messages))
fout.write('\n')
vr_position = logical_data.position.vr_position
with _output_section_header_trailer('RP66V1 Logical Data EFLR Summary', '=', os=fout):
count_total = sum(
[
sum(count_eflr_type_length_count[record_type].values())
for record_type in count_eflr_type_length_count
]
)
fout.write(f'Total number of EFLR records: {count_total:,d}\n')
length_total_eflr = 0
for record_type in count_eflr_type_length_count:
for length, count in count_eflr_type_length_count[record_type].items():
length_total_eflr += length * count
fout.write(f'Total length of EFLR records: {length_total_eflr:,d}\n')
for record_type in sorted(count_eflr_type_length_count.keys()):
fout.write(
f'EFLR record type {record_type} lengths and count [{len(count_eflr_type_length_count[record_type])}]:\n'
)
for length in sorted(count_eflr_type_length_count[record_type]):
fout.write(f'{length:10,d}: {count_eflr_type_length_count[record_type][length]:10,d}\n')
with _output_section_header_trailer('RP66V1 Logical Data IFLR Summary', '=', os=fout):
count_total = sum(
[
sum(count_iflr_type_length_count[record_type].values())
for record_type in count_iflr_type_length_count
]
)
fout.write(f'Total number of IFLR records: {count_total:,d}\n')
length_total_iflr = 0
for record_type in count_iflr_type_length_count:
for length, count in count_iflr_type_length_count[record_type].items():
length_total_iflr += length * count
fout.write(f'Total length of IFLR records: {length_total_iflr:,d}\n')
for record_type in sorted(count_iflr_type_length_count.keys()):
fout.write(
f'IFLR record type {record_type} lengths and count [{len(count_iflr_type_length_count[record_type])}]:\n'
)
for length in sorted(count_iflr_type_length_count[record_type]):
fout.write(f'{length:10,d}: {count_iflr_type_length_count[record_type][length]:10,d}\n')
fout.write(f'Total length EFLR/IFLR: {length_total_eflr/length_total_iflr:.3%}\n')
assert result.count == len(x_values)
assert result.min == min(x_values)
assert result.max == max(x_values)
array = np.array(x_values)
spacing_summary = XAxis.compute_spacing(array)
assert result.spacing == spacing_summary
@pytest.mark.parametrize('position_xvalues, expected', (
(
[
(0x0, 0xff, 1.0),
],
[
XAxis.IFLRReference(
logical_record_position=File.LogicalRecordPositionBase(
0x0, 0xff),
frame_number=1,
x_axis=1.0),
],
),
(
[
(0x0, 0xff, 1.0),
(0x2, 0x2ff, 2.0),
],
[
XAxis.IFLRReference(
logical_record_position=File.LogicalRecordPositionBase(
0x0, 0xff),
frame_number=1,
x_axis=1.0),
def test_example_iflr_process():
log_pass: LogPass.LogPass = _create_log_pass()
# ObjectName(O=11, C=0, I=b'0B')
frame_array: LogPass.FrameArray = log_pass[RepCode.ObjectName(
11, 0, b'0B')]
frame_array.init_arrays(len(BYTES_IFLR))
# print()
for frame_number, by in enumerate(BYTES_IFLR):
ld = File.LogicalData(by)
_iflr = IFLR.IndirectlyFormattedLogicalRecord(0, ld)
# frame_array.read_x_axis(ld, frame_number=0)
frame_array.read(ld, frame_number=frame_number)
# print(frame_array)
# print()
expected = [
# X axis
np.array([
[0.],
[75197.],
[154724.],
[234606.],
[311024.],
[381102.],
[386839.],
[428193.],
[447720.],
[466339.],
[489547.],
[500559.],
[523772.],
[538638.],
[542417.],
[550409.],
]),
np.array([
[0.],
[0.50002027],
[0.50002027],
[0.7500017],
[0.50002027],
[0.99998325],
[0.9699998],
[0.9699998],
[0.69999975],
[1.0600001],
[0.9699998],
[0.8800001],
[0.78999996],
[1.7599998],
[2.2000003],
[2.9],
], ),
np.array([
[0.],
[0.],
[0.],
[0.],
[0.],
[0.],
[200.44],
[205.45],
[206.18],
[208.69],
[202.7],
[200.93],
[255.77002],
[243.87],
[241.15],
[240.7],
]),
np.array([
[0.],
[190.99757],
[392.98987],
[595.8777],
[789.96594],
[967.95013],
[982.51935],
[1087.5443],
[1137.139],
[1184.4233],
[1243.3641],
[1271.3306],
[1330.2852],
[1368.0354],
[1377.6296],
[1397.9094],
]),
np.array([
[0.],
[0.833423],
[2.596255],
[4.809544],
[6.92684],
[9.256786],
[9.268364],
[7.632412],
[6.981416],
[6.338459],
[5.399805],
[4.980779],
[4.457969],
[4.138596],
[3.98476],
[3.54544],
]),
np.array([
[0.],
[0.833423],
[2.596255],
[4.809544],
[6.92684],
[9.256786],
[9.268364],
[7.632412],
[6.981416],
[6.338459],
[5.399805],
[4.980779],
[4.457969],
[4.138596],
[3.98476],
[3.54544],
]),
np.array([
[0.],
[0.],
[0.],
[0.],
[0.],
[0.],
[-0.043073],
[-0.735641],
[-1.049728],
[-1.387169],
[-1.841496],
[-2.009587],
[-2.565323],
[-3.338264],
[-3.632012],
[-4.421124],
]),
np.array([
[-9.9925000e+02],
[7.9068176e-02],
[0.0000000e+00],
[3.7815217e-02],
[3.7815217e-02],
[8.4224798e-02],
[3.9912241e+00],
[2.4064228e-02],
[1.6329297e-01],
[2.3032904e-01],
[7.0473805e-02],
[1.0141353e-01],
[3.9362201e-01],
[7.9411954e-01],
[1.4060384e+00],
[1.0347618e+00],
]),
np.array([
[-999.25],
[-999.25],
[-999.25],
[-999.25],
[-999.25],
[-999.25],
[50.3937],
[56.4173],
[58.4252],
[58.4252],
[62.4409],
[62.4409],
[62.4409],
[62.4409],
[62.4409],
[58.4252],
]),
]
for c, channel in enumerate(frame_array.channels):
# print(channel.array)
# np.testing.assert_array_almost_equal(channel.array, expected[c])
assert str(channel.array) == str(expected[c])
def test_iflr_str(by, s):
ld = File.LogicalData(by)
iflr = IFLR.IndirectlyFormattedLogicalRecord(1, ld)
assert str(iflr) == s
def test_logical_record_segment_encryption_packet_str():
ld = File.LogicalData(ENCRYPTION_EXAMPLE_BYTES)
lrsep = Encryption.LogicalRecordSegmentEncryptionPacket(ld)
assert str(
lrsep
) == "EncryptionPacket: size: 0x0004 producer: 440 code length: 4 data length: 2"
def scan_RP66V1_file_visible_records(fobj: typing.BinaryIO, fout: typing.TextIO, **kwargs) -> None:
"""Scans the file reporting Visible Records, optionally Logical Record Segments as well."""
verbose = kwargs.get('verbose', 0)
if not verbose:
fout.write(
colorama.Fore.YELLOW
+ 'Use -v to see individual records, RLE of LRSH positions and length distribution of LRS.\n'
)
with _output_section_header_trailer('RP66V1 Visible and LRSH Records', '*', os=fout):
lrsh_dump = kwargs['lrsh_dump']
with File.FileRead(fobj) as rp66_file:
vr_position = lr_position = 0
count_vr = 0
count_lrsh = 0
count_lrsh_first = 0
count_lrsh_type = {
'EFLR': collections.Counter(),
'IFLR': collections.Counter(),
}
count_lrsh_length = statistics.LengthDict()
lrsh_first_last = {
(False, False): 0,
(False, True): 0,
(True, False): 0,
(True, True): 0,
}
rle_visible_record_positions = Rle.RLE()
rle_lrsh_positions = Rle.RLE()
for visible_record in rp66_file.iter_visible_records():
vr_stride = visible_record.position - vr_position
rle_visible_record_positions.add(visible_record.position)
if verbose:
fout.write(f'{visible_record} Stride: 0x{vr_stride:08x} {vr_stride:6,d}\n')
if lrsh_dump:
for lrsh in rp66_file.iter_LRSHs_for_visible_record(visible_record):
count_lrsh_length.add(lrsh.length)
if lrsh.attributes.is_first:
rle_lrsh_positions.add(lrsh.position)
count_lrsh_first += 1
if lrsh.attributes.is_eflr:
count_lrsh_type['EFLR'].update([lrsh.record_type])
else:
count_lrsh_type['IFLR'].update([lrsh.record_type])
output = colorama.Fore.GREEN + f' {lrsh}'
elif lrsh.attributes.is_last:
output = colorama.Fore.RED + f' --{lrsh}'
else:
output = colorama.Fore.YELLOW + f' ..{lrsh}'
if verbose:
lr_stride = lrsh.position - lr_position
fout.write(f' {output} Stride: 0x{lr_stride:08x} {lr_stride:6,d}\n')
lr_position = lrsh.position
lrsh_first_last[(lrsh.attributes.is_first, lrsh.attributes.is_last)] += 1
count_lrsh += 1
vr_position = visible_record.position
count_vr += 1
with _output_section_header_trailer('Summary of Visible Records', '=', os=fout):
fout.write(f'Visible records: {count_vr:,d}\n')
with _output_section_header_trailer('RLE Visible Record Position', '-', os=fout):
_write_position_rle(rle_visible_record_positions, fout)
if lrsh_dump:
with _output_section_header_trailer('Summary of LRSH', '=', os=fout):
fout.write(f'LRSH: total={count_lrsh:,d} is_first={count_lrsh_first}\n')
with _output_section_header_trailer('Summary of Logical Record Types', '-', os=fout):
fout.write(f'LRSH: record types and counts (first segments only):\n')
for flr_type in ('EFLR', 'IFLR'):
fout.write(f'Count of Logical Record types for "{flr_type}" [{len(count_lrsh_type[flr_type])}]:\n')
for record_type in sorted(count_lrsh_type[flr_type].keys()):
fout.write(f'{record_type:3d} : {count_lrsh_type[flr_type][record_type]:8,d}\n')
with _output_section_header_trailer('Summary of LRSH Lengths', '-', os=fout):
fout.write(
f'LRSH: record lengths and counts (all segments)'
f' [{len(count_lrsh_length)}]'
)
if len(count_lrsh_length):
fout.write(f' range: {min(count_lrsh_length.keys())}...{max(count_lrsh_length.keys())}')
fout.write(f'\n')
fout.write('\n'.join(count_lrsh_length.histogram_power_of_2()))
fout.write(f'\n')
with _output_section_header_trailer('Summary of LRSH First/last', '-', os=fout):
fout.write(f'{"(First, Last)":16} : {"Count":8}\n')
for k in sorted(lrsh_first_last.keys()):
fout.write(f'{str(k):16} : {lrsh_first_last[k]:8d}\n')
if verbose:
with _output_section_header_trailer('RLE LRSH Position', '-', os=fout):
_write_position_rle(rle_lrsh_positions, fout)
for length in sorted(count_lrsh_length.keys()):
fout.write(f'{length:3d} : {count_lrsh_length[length]:8,d}\n')
def test_example_iflr():
print()
for by in BYTES_IFLR:
ld = File.LogicalData(by)
iflr = IFLR.IndirectlyFormattedLogicalRecord(0, ld)
def __setstate__(self, state):
self.__dict__.update(state)
self.rp66v1_file = File.FileRead(self.path)
def test_eflr_channel_ctor():
ld = File.LogicalData(BYTES_CHANNEL)
EFLR.ExplicitlyFormattedLogicalRecord(
Types.EFLR_PUBLIC_SET_TYPE_TO_CODE_MAP[b'CHANNEL'], ld)
def test_logical_data_constructs(by):
File.LogicalData(by)
def test_eflr_constructs(lr_type, by):
ld = File.LogicalData(by)
eflr = EFLR.ExplicitlyFormattedLogicalRecord(lr_type, ld)
assert eflr is not None
def __init__(self, path_or_file: typing.Union[str, io.BytesIO]):
self.lr_pos_desc: typing.List[File.LRPosDesc] = []
self.rp66v1_file: File.FileRead = File.FileRead(path_or_file)
self.path = self.rp66v1_file.path
