def test_ExplicitlyFormattedLogicalRecord_eq():
ld = LogicalData(LOGICAL_BYTES_FROM_STANDARD)
eflr_a = EFLR.ExplicitlyFormattedLogicalRecord(3, ld)
ld.rewind()
eflr_b = EFLR.ExplicitlyFormattedLogicalRecord(3, ld)
assert eflr_a == eflr_b
assert eflr_a != 1
def __init__(self, lr_type: int, ld: LogicalData):
self.lr_type: int = lr_type
ld.rewind()
self.set: Set = Set(ld)
self.template: Template = Template()
# This object list contains all objects not including duplicates.
self.objects: typing.List[Object] = []
# This is the final object name map after de-duplication depending on the de-duplication strategy.
# TODO: Perfomance. Use self.object_name_map throughout then don't need to rebuild it if there are duplicates
# to remove.
self.object_name_map: typing.Dict[RepCode.ObjectName, int] = {}
temp_object_name_map: typing.Dict[RepCode.ObjectName, int] = {}
dupes_to_remove: typing.List[int] = []
if ld:
self.template.read(ld)
while ld:
obj = Object(ld, self.template)
if obj.name not in temp_object_name_map:
temp_object_name_map[obj.name] = len(self.objects)
self.objects.append(obj)
else:
self._handle_duplicate_object(obj, temp_object_name_map, dupes_to_remove)
# Clear out any duplicates then index those remaining.
dupes_to_remove.sort()
for i in reversed(dupes_to_remove):
self.DUPE_OBJECT_LOGGER(f'Cleaning table by removing duplicate object:\n{self.objects[i]}')
del self.objects[i]
assert len(self.object_name_map) == 0
for i, obj in enumerate(self.objects):
self.object_name_map[obj.name] = i
self.logical_data_consumed = ld.index
def UNORM(ld: LogicalData) -> int:
"""
Representation code 16, Unsigned 2-byte integer.
[RP66V1 Appendix B Section B.16]
"""
ret = ld.read()
ret <<= 8
ret |= ld.read()
return ret
def __init__(self, lr_type: int, ld: File.LogicalData):
self.lr_type: int = lr_type
ld.rewind()
# [RP66V1 Section 3.3 Indirectly Formatted Logical Record]
self.object_name: RepCode.ObjectName = RepCode.OBNAME(ld)
# [RP66V1 Section 5.6.1 Frames]
self.frame_number = RepCode.UVARI(ld)
self.preamble_length = ld.index
self.remain = ld.remain
# Frame numbers start from 1 but there are many observed cases of IFLRs that have a 0 frame number and zero
# remaining data. Here we only warn if the frame number is zero and the remaining data is non-zero.
# We warn rather than raising in the spirit of optimism.
if self.frame_number == 0 and self.remain != 0:
logger.warning(
f'Frame number needs to be >= 1, not {self.frame_number} [RP66V1 Section 5.6.1 Frames] (there is data remaining)'
)
def ASCII(ld: LogicalData) -> bytes:
"""
Representation code 20, Variable length identifier. Length up to 2**30-1 bytes.
[RP66V1 Appendix B Section B.20]
"""
size: int = UVARI(ld)
return ld.chunk(size)
def test_ExplicitlyFormattedLogicalRecord_key_values(sort_order, expected):
ld = LogicalData(LOGICAL_BYTES_FROM_STANDARD_SINGLE_OBJECT)
eflr = EFLR.ExplicitlyFormattedLogicalRecord(3, ld)
result = eflr.key_values(
stringify_function=stringify.stringify_object_by_type, sort=sort_order)
# print(result)
assert result == expected
def __init__(self, ld: LogicalData, template: Template):
component_descriptor = ComponentDescriptor(ld.read())
if not component_descriptor.is_object:
raise ExceptionEFLRObject(
f'Component Descriptor does not represent a object but a {component_descriptor.type}.'
)
self.name: RepCode.ObjectName = RepCode.OBNAME(ld)
self.attrs: typing.List[typing.Union[AttributeBase, None]] = []
self.attr_label_map: typing.Dict[bytes, int] = {}
index: int = 0
while True:
component_descriptor = ComponentDescriptor(ld.read())
if not component_descriptor.is_attribute_group:
raise ExceptionEFLRObject(
f'Component Descriptor does not represent a attribute but a {component_descriptor.type}.'
)
if template[index].component_descriptor.is_invariant_attribute:
self.attrs.append(template[index])
elif template[index].component_descriptor.is_absent_attribute:
self.attrs.append(None)
else:
# TODO: Check the attribute label is the same as the template. Reference [RP66V1 Section 4.5]
self.attrs.append(
Attribute(component_descriptor, ld, template[index]))
if ld.remain == 0 or ComponentDescriptor(ld.peek()).is_object:
break
# next_component_descriptor = ComponentDescriptor(ld.peek())
# if next_component_descriptor.is_object:
# break
index += 1
while len(self.attrs) < len(template):
self.attrs.append(template[len(self.attrs)])
if len(template) != len(self.attrs):
raise ExceptionEFLRObject(
f'Template specifies {len(template)} attributes but Logical Data has {len(self.attrs)}'
)
# Now populate self.attr_label_map
for a, attr in enumerate(self.attrs):
if attr is None:
label = template.attrs[a].label
else:
label = attr.label
# TODO: Assert that the attribute label is the same as the template. Reference [RP66V1 Section 4.5]
if label in self.attr_label_map:
raise ExceptionEFLRObjectDuplicateLabel(
f'Duplicate Attribute label {label}')
self.attr_label_map[label] = a
def test_ExplicitlyFormattedLogicalRecord_key_value_raises():
ld = LogicalData(LOGICAL_BYTES_FROM_STANDARD)
eflr = EFLR.ExplicitlyFormattedLogicalRecord(3, ld)
assert not eflr.is_key_value()
with pytest.raises(EFLR.ExceptionEFLR) as err:
eflr.key_values(stringify_function=stringify.stringify_object_by_type,
sort=True)
assert err.value.args[0] == 'Can not represent EFLR as key->value table.'
def SNORM(ld: LogicalData) -> int:
"""
Representation code 13, Signed 2-byte integer.
[RP66V1 Appendix B Section B.13]
"""
by = ld.chunk(2)
value = struct.unpack('>h', by)
return value[0]
def test_ExplicitlyFormattedLogicalRecord_table_as_string(
sort_order, expected):
ld = LogicalData(LOGICAL_BYTES_FROM_STANDARD)
eflr = EFLR.ExplicitlyFormattedLogicalRecord(3, ld)
result = eflr.table_as_strings(
stringify_function=stringify.stringify_object_by_type, sort=sort_order)
# print(result)
assert result == expected
def ULONG(ld: LogicalData) -> int:
"""
Representation code 16, Unsigned 4-byte integer.
[RP66V1 Appendix B Section B.17]
"""
by = ld.chunk(4)
value = struct.unpack('>I', by)
return value[0]
def SSHORT(ld: LogicalData) -> int:
"""
SSHORT Representation code 12, Signed 1-byte integer.
[RP66V1 Appendix B Section B.12]
"""
r = ld.read()
if r > 127:
r -= 256
return r
def __init__(self, ld: LogicalData):
ld_index = ld.index
component_descriptor = ComponentDescriptor(ld.read())
if not component_descriptor.is_set_group:
raise ExceptionEFLRSet(f'Component Descriptor does not represent a set but a {component_descriptor.type}.')
self.type: bytes = RepCode.IDENT(ld)
self.name: bytes = ComponentDescriptor.CHARACTERISTICS_AND_COMPONENT_FORMAT_SET_MAP['N'].global_default
if component_descriptor.has_set_N:
self.name = RepCode.IDENT(ld)
self.logical_data_consumed = ld.index - ld_index
def read(self, ld: LogicalData):
"""Populate the template with the Logical Data."""
while True:
component_descriptor = ComponentDescriptor(ld.read())
if not component_descriptor.is_attribute_group:
raise ExceptionEFLRTemplate(
f'Component Descriptor does not represent a attribute but a {component_descriptor.type}.'
)
template_attribute = TemplateAttribute(component_descriptor, ld)
if template_attribute.label in self.attr_label_map:
raise ExceptionEFLRTemplateDuplicateLabel(f'Duplicate template label {template_attribute.label}')
self.attr_label_map[template_attribute.label] = len(self.attrs)
self.attrs.append(template_attribute)
if ld.remain == 0:
# This is kind of unusual, it is an EFLR with a template but no objects.
break
next_component_descriptor = ComponentDescriptor(ld.peek())
if next_component_descriptor.is_object:
break
def __init__(self, ld: LogicalData):
# TODO: Check ranges
self.year: int = USHORT(ld) + 1900
v: int = ld.read()
self.tz: int = (v >> 4) & 0xf
self.month: int = v & 0xf
self.day: int = USHORT(ld)
self.hour: int = USHORT(ld)
self.minute: int = USHORT(ld)
self.second: int = USHORT(ld)
self.millisecond: int = UNORM(ld)
def VSINGL(ld: LogicalData) -> float:
"""Representation code 6, VAX single precision floating point."""
by = ld.chunk(4)
s = (by[1] & 0x80)
m = ((by[0] & 0x7f) << 16) | (by[3] << 8) | by[2]
e = ((by[1] & 0x7f) << 1) | ((by[0] & 0x80) >> 7)
if e == 0 and s == 0:
# m is arbitrary
return 0.0
m = float(m) / (1 << 23)
value = (0.5 + m) * 2**(e - 128)
if s:
return -value
return value
def UVARI(ld: LogicalData) -> int:
"""
Representation code 18, Variable-length unsigned integer.
[RP66V1 Appendix B Section B.18]
"""
value: int = ld.read()
if value & 0xc0 == 0x80:
# Two bytes
value &= 0x7f
value <<= 8
value |= ld.read()
# TODO: Raise if < 2**7
elif value & 0xc0 == 0xc0:
# Four bytes
value &= 0x3f
value <<= 8
value |= ld.read()
value <<= 8
value |= ld.read()
value <<= 8
value |= ld.read()
# TODO: Raise if < 2**14
return value
def seek(self, ld: LogicalData) -> None:
"""Increments the logical data without reading any values into the array."""
if len(self.array) != 0:
raise ExceptionFrameChannel('seek() on empty array. This seems like a logical error.')
ld.seek(RepCode.rep_code_fixed_length(self.rep_code) * self.count)
def test_OBNAME_len_raises():
ld = LogicalData(b'\x00' + b'\x01' + b'\x03')
with pytest.raises(RepCode.ExceptionRepCode) as err:
RepCode.OBNAME_len(ld.bytes, -1)
assert err.value.args[0] == 'Index can not be negative.'
import contextlib
import typing
import pytest
import TotalDepth.RP66V1.core.LogicalRecord.Duplicates
from TotalDepth.RP66V1.core.File import LogicalData
from TotalDepth.RP66V1.core.LogicalRecord import EFLR
from TotalDepth.RP66V1.core.LogicalRecord.ComponentDescriptor import ComponentDescriptor
from TotalDepth.RP66V1.core import RepCode, stringify
from TotalDepth.RP66V1.core.RepCode import ObjectName
@pytest.mark.parametrize('ld, expected_type, expected_name', (
(LogicalData(b'\xf0\x07CHANNEL'), b'CHANNEL', b''),
(LogicalData(b'\xf8\x07CHANNEL\x01\x30'), b'CHANNEL', b'0'),
))
def test_Set(ld, expected_type, expected_name):
result = EFLR.Set(ld)
assert result.type == expected_type
assert result.name == expected_name
assert ld.remain == 0
@pytest.mark.parametrize('ld, expected_type, expected_name', (
(LogicalData(b'\xf0\x07CHANNEL'), b'CHANNEL', b''),
(LogicalData(b'\xf8\x07CHANNEL\x01\x30'), b'CHANNEL', b'0'),
))
def test_Set_eq(ld, expected_type, expected_name):
result = EFLR.Set(ld)
assert result == result
def FSINGL(ld: LogicalData) -> float:
"""Representation code 2, IEEE single precision floating point."""
by = ld.chunk(4)
value = struct.unpack('>f', by)
return value[0]
def USHORT(ld: LogicalData) -> int:
"""
USHORT Representation code 15, Unsigned 1-byte integer.
[RP66V1 Appendix B Section B.15]
"""
return ld.read()
def test_ExplicitlyFormattedLogicalRecord_table_shape():
ld = LogicalData(LOGICAL_BYTES_FROM_STANDARD)
eflr = EFLR.ExplicitlyFormattedLogicalRecord(3, ld)
assert eflr.shape == (3, 5)
def test_ExplicitlyFormattedLogicalRecord_reduced_object_map():
ld = LogicalData(LOGICAL_BYTES_FROM_STANDARD)
eflr = EFLR.ExplicitlyFormattedLogicalRecord(3, ld)
result = EFLR.reduced_object_map(eflr)
# print(result)
assert result == {b'TIME': 0, b'PRESSURE': 1, b'PAD-ARRAY': 2}
def _pascal_string(ld: LogicalData) -> bytes:
"""Reads a Pascal like string from the LogicalData."""
siz: int = ld.read()
return ld.chunk(siz)
def FDOUBL(ld: LogicalData) -> float:
"""Representation code 7, IEEE double precision floating point."""
by = ld.chunk(8)
value = struct.unpack('>d', by)
return value[0]
def test_ExplicitlyFormattedLogicalRecord_key_value():
ld = LogicalData(LOGICAL_BYTES_FROM_STANDARD_SINGLE_OBJECT)
eflr = EFLR.ExplicitlyFormattedLogicalRecord(3, ld)
assert eflr.is_key_value()
def test_DTIME_invalid_tz_description():
ld = LogicalData(b'\x57\x34\x13\x15\x14\x0f\x02\x6c')
# ^
result = RepCode.DTIME(ld)
assert result.tz_description == ''
def test_ExplicitlyFormattedLogicalRecord_len():
ld = LogicalData(LOGICAL_BYTES_FROM_STANDARD)
eflr = EFLR.ExplicitlyFormattedLogicalRecord(3, ld)
result = eflr.str_long()
# print(result)
assert len(eflr) == 3
def __init__(self, ld: LogicalData):
self.size = UNORM(ld)
self.producer_code = UNORM(ld)
self.encryption_information = ld.chunk(self.size)
self.bytes = ld.chunk(ld.remain)