def demo_numpy_access_partial_test_data():
fobj = io.BytesIO(test_data.BASIC_FILE)
with LogicalFile.LogicalIndex(fobj) as logical_index:
for logical_file in logical_index.logical_files:
if logical_file.has_log_pass:
for frame_array in logical_file.log_pass:
# print(frame_array)
# print('TRACE:', [c.ident for c in frame_array.channels])
frame_count = logical_file.populate_frame_array(
frame_array,
frame_slice=Slice.Slice(0, None, 64),
# frame_slice=Slice.Sample(64),
channels={
frame_array.channels[1].ident,
frame_array.channels[2].ident
})
print(
f'Loaded {frame_count} frames'
f' from {frame_array.ident} using {frame_array.sizeof_array} bytes.'
)
for channel in frame_array.channels:
if len(channel.array):
print(channel.ident, channel.long_name,
channel.units)
print(
f'Min: {channel.array.min():12.3f} Max: {channel.array.max():12.3f}'
)
# else:
# print('N/A')
print()
def demo_logical_files_test_data():
file_object = io.BytesIO(test_data.BASIC_FILE)
with LogicalFile.LogicalIndex(file_object) as logical_index:
# print(f'LogicalIndex: {logical_index} with {len(logical_index)} Logical Files')
for logical_file in logical_index.logical_files:
# print(f'LogicalFile [{l}]: {logical_file}')
print(f'***** logical_file.file_header_logical_record.str_long():')
print(logical_file.file_header_logical_record.str_long())
print()
print(f'***** logical_file.origin_logical_record.str_long():')
print(logical_file.origin_logical_record.str_long())
print()
print(f'***** logical_file.defining_origin:')
print(logical_file.defining_origin)
print()
if logical_file.channel is not None:
print(f'***** logical_file.channel.str_long():')
print(logical_file.channel.str_long())
print()
if logical_file.frame is not None:
print(f'***** logical_file.frame.str_long():')
print(logical_file.frame.str_long())
print()
print(f'***** logical_file.has_log_pass:'******'***** logical_file.log_pass:')
print(logical_file.log_pass)
print()
def test_logical_index_logical_file_iflr_position_map():
fobj = io.BytesIO(test_data.BASIC_FILE)
with LogicalFile.LogicalIndex(fobj) as logical_index:
assert len(logical_index) == 1
logical_file = logical_index.logical_files[0]
# pprint.pprint(logical_file.iflr_position_map)
expected_key = RepCode.ObjectName(O=2, C=0, I=b'50')
assert list(logical_file.iflr_position_map.keys()) == [expected_key]
def test_logical_index_has_log_pass():
fobj = io.BytesIO(test_data.BASIC_FILE)
with LogicalFile.LogicalIndex(fobj) as logical_index:
assert len(logical_index) == 1
logical_file = logical_index.logical_files[0]
assert logical_file.channel is not None
assert logical_file.frame is not None
assert logical_file.has_log_pass
assert len(logical_file.log_pass) == 1
def demo_eflr():
with LogicalFile.LogicalIndex(path_in) as logical_index:
# print(logical_index)
for logical_file in logical_index.logical_files:
# print(logical_file)
for position, eflr in logical_file.eflrs:
# print(position_eflr.eflr)
# print(eflr.shape)
print(eflr.str_long())
def demo_logical_files():
with LogicalFile.LogicalIndex(path_in) as logical_index:
print(
f'logicalIndex: {logical_index} with {len(logical_index)} Logical Files'
)
for logical_file in logical_index.logical_files:
print(f'logicalFile: {logical_file}')
print(logical_file.file_header_logical_record.str_long())
print(logical_file.origin_logical_record.str_long())
def test_logical_file_populate_frame_array_partial_split():
fobj = io.BytesIO(test_data.BASIC_FILE)
with LogicalFile.LogicalIndex(fobj) as logical_index:
assert len(logical_index) == 1
logical_file = logical_index.logical_files[0]
assert logical_file.has_log_pass
assert len(logical_file.log_pass) == 1
frame_array = logical_file.log_pass[0]
frame_slice = Slice.Sample(64)
frame_count = logical_file.populate_frame_array(
frame_array, frame_slice)
assert frame_count == 64
def demo_eflr_contents_test_data():
file_object = io.BytesIO(test_data.BASIC_FILE)
with LogicalFile.LogicalIndex(file_object) as logical_index:
for logical_file in logical_index.logical_files:
for position, eflr in logical_file.eflrs:
# eflr is a TotalDepth.RP66V1.core.LogicalRecord.EFLR.ExplicitlyFormattedLogicalRecord
# print(eflr.set.type)
if eflr.set.type == b'PARAMETER':
# print(eflr)
print(eflr[0])
print()
print(eflr[0][0])
def test_logical_file_populate_frame_array_raises_if_frame_array_not_member():
fobj = io.BytesIO(test_data.BASIC_FILE)
with LogicalFile.LogicalIndex(fobj) as logical_index:
assert len(logical_index) == 1
logical_file = logical_index.logical_files[0]
assert logical_file.has_log_pass
assert len(logical_file.log_pass) == 1
frame_array = logical_file.log_pass[0]
frame_array_copy = copy.copy(frame_array)
with pytest.raises(LogicalFile.ExceptionLogicalFile) as err:
logical_file.populate_frame_array(frame_array_copy)
assert err.value.args[
0] == 'populate_frame_array(): given FrameArray is not in Log Pass'
def test_logical_index_logical_file_iflr_position_map_x_axis_summary():
fobj = io.BytesIO(test_data.BASIC_FILE)
with LogicalFile.LogicalIndex(fobj) as logical_index:
assert len(logical_index) == 1
logical_file = logical_index.logical_files[0]
# pprint.pprint(logical_file.iflr_position_map)
expected_key = RepCode.ObjectName(O=2, C=0, I=b'50')
assert list(logical_file.iflr_position_map.keys()) == [expected_key]
x_axis = logical_file.iflr_position_map[expected_key]
assert len(x_axis) == 649
assert x_axis.summary.min == 2889.4
assert x_axis.summary.max == 2954.199999999941
assert x_axis.summary.count == 649
def test_logical_index_logical_file_iflr_position_map_x_axis_summary_spacing():
fobj = io.BytesIO(test_data.BASIC_FILE)
with LogicalFile.LogicalIndex(fobj) as logical_index:
assert len(logical_index) == 1
logical_file = logical_index.logical_files[0]
# pprint.pprint(logical_file.iflr_position_map)
expected_key = RepCode.ObjectName(O=2, C=0, I=b'50')
assert list(logical_file.iflr_position_map.keys()) == [expected_key]
x_axis = logical_file.iflr_position_map[expected_key]
assert x_axis.summary.spacing.min == 0.09999999999990905
assert x_axis.summary.spacing.max == 0.09999999999990905
assert x_axis.summary.spacing.mean == 0.09999999999990905
assert x_axis.summary.spacing.median == 0.09999999999990905
assert x_axis.summary.spacing.std == 0.0
def demo_eflr_contents():
with LogicalFile.LogicalIndex(path_in) as logical_index:
for logical_file in logical_index.logical_files:
for position, eflr in logical_file.eflrs:
# eflr is a TotalDepth.RP66V1.core.LogicalRecord.EFLR.ExplicitlyFormattedLogicalRecord
if eflr.set.type == b'EQUIPMENT':
for row in eflr.objects:
# row is a TotalDepth.RP66V1.core.LogicalRecord.EFLR.Object
print(f' Row: {row.name.I}')
for attr in row.attrs:
# attr is a TotalDepth.RP66V1.core.LogicalRecord.EFLR.Attribute
print(
f' {attr.label}={attr.value} ({attr.units})'
)
def test_logical_index_logical_file_iflr_position_map_x_axis_summary_spacing_counts(
):
fobj = io.BytesIO(test_data.BASIC_FILE)
with LogicalFile.LogicalIndex(fobj) as logical_index:
assert len(logical_index) == 1
logical_file = logical_index.logical_files[0]
# pprint.pprint(logical_file.iflr_position_map)
expected_key = RepCode.ObjectName(O=2, C=0, I=b'50')
assert list(logical_file.iflr_position_map.keys()) == [expected_key]
x_axis = logical_file.iflr_position_map[expected_key]
assert x_axis.summary.spacing.counts.norm == 648
assert x_axis.summary.spacing.counts.dupe == 0
assert x_axis.summary.spacing.counts.skip == 0
assert x_axis.summary.spacing.counts.back == 0
def test_logical_file_populate_frame_array():
fobj = io.BytesIO(test_data.BASIC_FILE)
with LogicalFile.LogicalIndex(fobj) as logical_index:
assert len(logical_index) == 1
logical_file = logical_index.logical_files[0]
assert logical_file.has_log_pass
assert len(logical_file.log_pass) == 1
frame_array = logical_file.log_pass[0]
frame_count = logical_file.populate_frame_array(frame_array)
assert frame_count == 649
assert len(frame_array) == 5
assert frame_array.shape == [(649, 1), (649, 1), (649, 1), (649, 1),
(649, 1)]
names = [c.ident.I for c in frame_array.channels]
assert names == [b'DEPT', b'TENS', b'ETIM', b'DHTN', b'GR']
def index_a_single_file(path_in: str, path_out: str,
private: bool) -> IndexResult:
# logging.info(f'index_a_single_file(): "{path_in}" to "{path_out}"')
bin_file_type = binary_file_type_from_path(path_in)
if bin_file_type == 'RP66V1':
if path_out:
out_dir = os.path.dirname(path_out)
if not os.path.exists(out_dir):
logger.info(f'Making directory: {out_dir}')
os.makedirs(out_dir, exist_ok=True)
logger.info(f'Indexing {path_in} to {path_out}')
try:
t_start = time.perf_counter()
with LogicalFile.LogicalIndex(path_in) as logical_index:
if path_out:
with open(path_out + '.xml', 'w') as f_out:
write_logical_file_sequence_to_xml(
logical_index, f_out, private)
index_size = os.path.getsize(path_out + '.xml')
else:
xml_fobj = io.StringIO()
write_logical_file_sequence_to_xml(logical_index, xml_fobj,
private)
index_output = xml_fobj.getvalue()
index_size = len(index_output)
print(index_output)
result = IndexResult(
path_in,
os.path.getsize(path_in),
index_size,
time.perf_counter() - t_start,
False,
False,
)
logger.info(f'Length of XML: {index_size}')
return result
except ExceptionTotalDepthRP66V1: # pragma: no cover
logger.exception(
f'Failed to index with ExceptionTotalDepthRP66V1: {path_in}')
return IndexResult(path_in, os.path.getsize(path_in), 0, 0.0, True,
False)
except Exception: # pragma: no cover
logger.exception(f'Failed to index with Exception: {path_in}')
return IndexResult(path_in, os.path.getsize(path_in), 0, 0.0, True,
False)
logger.info(f'Ignoring file type "{bin_file_type}" at {path_in}')
return IndexResult(path_in, 0, 0, 0.0, False, True)
def demo_numpy_describe():
with LogicalFile.LogicalIndex(path_in) as logical_index:
for logical_file in logical_index.logical_files:
if logical_file.has_log_pass:
for frame_array in logical_file.log_pass:
print(frame_array)
frame_count = logical_file.populate_frame_array(
frame_array)
print(
f'Loaded {frame_count} frames and {len(frame_array)} channels'
f' from {frame_array.ident} using {frame_array.sizeof_array} bytes.'
)
for channel in frame_array.channels:
print(channel)
# channel.array is a numpy array
np.info(channel.array)
print()
def demo_numpy_access():
with LogicalFile.LogicalIndex(path_in) as logical_index:
for logical_file in logical_index.logical_files:
if logical_file.has_log_pass:
for frame_array in logical_file.log_pass:
print(frame_array)
frame_count = logical_file.populate_frame_array(
frame_array)
print(
f'Loaded {frame_count} frames and {len(frame_array)} channels'
f' from {frame_array.ident} using {frame_array.sizeof_array} bytes.'
)
for channel in frame_array.channels:
print(channel.ident, channel.long_name, channel.units)
# channel.array is a numpy array
print(
f'Min: {channel.array.min():12.3f} Max: {channel.array.max():12.3f}'
)
def demo_numpy_describe_test_data():
fobj = io.BytesIO(test_data.BASIC_FILE)
with LogicalFile.LogicalIndex(fobj) as logical_index:
for logical_file in logical_index.logical_files:
if logical_file.has_log_pass:
for frame_array in logical_file.log_pass:
print(frame_array)
frame_count = logical_file.populate_frame_array(
frame_array)
print(
f'Loaded {frame_count} frames and {len(frame_array)} channels'
f' from {frame_array.ident} using {frame_array.sizeof_array} bytes.'
)
for channel in frame_array.channels:
print(f'Channel: {channel}')
# channel.array is a numpy array
np.info(channel.array)
print()
def scan_RP66V1_file_data_content(fobj: typing.BinaryIO, fout: typing.TextIO,
*, rp66v1_path: str, frame_slice: Slice.Slice, eflr_as_table: bool) -> None:
"""
Scans all of every EFLR and IFLR in the file using a ScanFile object.
"""
with LogicalFile.LogicalIndex(fobj) as logical_index:
with _output_section_header_trailer('RP66V1 File Data Summary', '*', os=fout):
fout.write(str(logical_index.storage_unit_label))
fout.write('\n')
logical_file: LogicalFile.LogicalFile
for lf, logical_file in enumerate(logical_index.logical_files):
with _output_section_header_trailer(f'Logical File [{lf}/{len(logical_index.logical_files)}]', '=', os=fout):
fout.write(str(logical_file))
fout.write('\n')
eflr_position: LogicalFile.PositionEFLR
for e, eflr_position in enumerate(logical_file.eflrs):
header = f'EFLR [{e}/{len(logical_file.eflrs)}] at {str(eflr_position.lrsh_position)}'
with _output_section_header_trailer(header, '-', os=fout):
# fout.write(str(eflr_position.eflr))
# fout.write('\n')
if eflr_as_table:
if eflr_position.eflr.is_key_value():
eflr_str_table = eflr_position.eflr.key_values(
stringify_function=stringify.stringify_object_by_type,
sort=True
)
else:
eflr_str_table = eflr_position.eflr.table_as_strings(
stringify_function=stringify.stringify_object_by_type,
sort=True
)
fout.write('\n'.join(data_table.format_table(eflr_str_table, heading_underline='-')))
fout.write('\n')
else:
fout.write(eflr_position.eflr.str_long())
fout.write('\n')
# Now the LogPass(s)
if logical_file.has_log_pass:
with _output_section_header_trailer('Log Pass', '-', os=fout):
_scan_log_pass_content(logical_index, lf, fout, frame_slice=frame_slice)
else:
fout.write('NO Log Pass for this Logical Record\n')
def single_rp66v1_file_to_las(
path_in: str,
array_reduction: str,
path_out: str,
frame_slice: Slice.Slice,
channels: typing.Set[str],
field_width: int,
float_format: str,
) -> LASWriteResult:
"""Convert a single RP66V1 file to a set of LAS files."""
# logging.info(f'index_a_single_file(): "{path_in}" to "{path_out}"')
assert array_reduction in ARRAY_REDUCTIONS
binary_file_type = bin_file_type.binary_file_type_from_path(path_in)
if binary_file_type == 'RP66V1':
logger.info(f'Converting RP66V1 {path_in} to LAS {os.path.splitext(path_out)[0]}*')
try:
t_start = time.perf_counter()
with LogicalFile.LogicalIndex(path_in) as logical_index:
las_files_written = write_logical_index_to_las(
logical_index, array_reduction, path_out, frame_slice, channels, field_width, float_format
)
output_size = sum(os.path.getsize(f) for f in las_files_written)
result = LASWriteResult(
path_in,
os.path.getsize(path_in),
output_size,
len(las_files_written),
time.perf_counter() - t_start,
False,
False,
)
logger.info(f'{len(las_files_written)} LAS file written with total size: {output_size:,d} bytes')
return result
except ExceptionTotalDepthRP66V1: # pragma: no cover
logger.exception(f'Failed to index with ExceptionTotalDepthRP66V1: {path_in}')
return LASWriteResult(path_in, os.path.getsize(path_in), 0, 0, 0.0, True, False)
except Exception: # pragma: no cover
logger.exception(f'Failed to index with Exception: {path_in}')
return LASWriteResult(path_in, os.path.getsize(path_in), 0, 0, 0.0, True, False)
logger.debug(f'Ignoring file type "{binary_file_type}" at {path_in}')
return LASWriteResult(path_in, 0, 0, 0, 0.0, False, True)
def index_a_single_file(path_in: str, path_out: str, read_back: bool) -> IndexResult:
bin_file_type = binary_file_type_from_path(path_in)
if bin_file_type == 'RP66V1':
if path_out:
out_dir = os.path.dirname(path_out)
if not os.path.exists(out_dir):
logger.info(f'Making directory: {out_dir}')
os.makedirs(out_dir, exist_ok=True)
logger.info(f'Indexing {path_in} to pickle {path_out}')
try:
t_start = time.perf_counter()
with LogicalFile.LogicalIndex(path_in) as logical_index:
pickled_index = pickle.dumps(logical_index)
index_time = time.perf_counter() - t_start
# logger.info(f'Length of pickled index: {len(pickled_index)}')
if path_out:
pickle_path = path_out + '.pkl'
t_start = time.perf_counter()
with open(pickle_path, 'wb') as out_stream:
out_stream.write(pickled_index)
write_time = time.perf_counter() - t_start
if read_back:
t_start = time.perf_counter()
_read_index = unpickle(pickle_path)
read_time = time.perf_counter() - t_start
else:
read_time = 0.0
else:
write_time = read_time = 0.0
result = IndexResult(
path_in, os.path.getsize(path_in), len(pickled_index),
index_time, write_time, read_time, False, False
)
return result
except ExceptionTotalDepthRP66V1: # pragma: no cover
logger.exception(f'Failed to index with ExceptionTotalDepthRP66V1: {path_in}')
except Exception: # pragma: no cover
logger.exception(f'Failed to index with Exception: {path_in}')
return IndexResult(path_in, os.path.getsize(path_in), 0, 0.0, 0.0, 0.0, True, False) # pragma: no cover
return IndexResult(path_in, os.path.getsize(path_in), 0, 0.0, 0.0, 0.0, False, True) # pragma: no cover
def html_scan_RP66V1_file_data_content(
path_in: str, fout: typing.TextIO, label_process: bool,
frame_slice: Slice.Slice) -> HTMLBodySummary:
"""
Scans all of every EFLR and IFLR in the file and writes to HTML.
Similar to TotalDepth.RP66V1.core.Scan.scan_RP66V1_file_data_content
Returns the text to use as a link.
"""
with LogicalFile.LogicalIndex(path_in) as logical_index:
if label_process:
process.add_message_to_queue(os.path.basename(path_in))
logger.info(
f'html_scan_RP66V1_file_data_content(): Creating File.FileRead() from "{os.path.basename(path_in)}"'
)
logger.info(
f'html_scan_RP66V1_file_data_content(): Creating LogicalFile.LogicalIndex()'
f' from "{os.path.basename(path_in)}"')
logger.info(f'html_scan_RP66V1_file_data_content(): Writing HTML')
if label_process:
process.add_message_to_queue('Writing HTML')
with XmlWrite.XhtmlStream(fout) as xhtml_stream:
with XmlWrite.Element(xhtml_stream, 'head'):
with XmlWrite.Element(
xhtml_stream, 'meta', {
'charset': "UTF-8",
'name': "viewport",
'content': "width=device-width, initial-scale=1",
}):
pass
with XmlWrite.Element(xhtml_stream, 'title'):
xhtml_stream.charactersWithBr(f'RP66V1 Scan of {path_in}')
with XmlWrite.Element(xhtml_stream, 'style'):
xhtml_stream.literal(CSS_RP66V1)
with XmlWrite.Element(xhtml_stream, 'body'):
ret = html_write_body(logical_index, frame_slice, xhtml_stream)
logger.info(
f'html_scan_RP66V1_file_data_content(): Done "{os.path.basename(path_in)}"'
)
return ret
def dump_frames_and_or_channels_single_rp66v1_file(path_in: str, frame_slices, channels) -> None:
"""Write a summary of frames and channels."""
binary_file_type = bin_file_type.binary_file_type_from_path(path_in)
if binary_file_type == 'RP66V1':
logger.info(f'Reading RP66V1 {path_in}')
try:
with _output_section_header_trailer(f'File {path_in}', '=', os=sys.stdout):
with LogicalFile.LogicalIndex(path_in) as logical_index:
for l, logical_file in enumerate(logical_index.logical_files):
with _output_section_header_trailer(f'Logical file [{l:04d}]: {logical_file}', '-', os=sys.stdout):
# print(f'Logical file: {logical_file.file_header_logical_record.set.type}')
if logical_file.has_log_pass:
for frame_array in logical_file.log_pass.frame_arrays:
print(f' Frame Array: {frame_array.ident.I}')
if channels:
channel_text = b','.join(c.ident.I for c in frame_array.channels)
print(f' Channels: {channel_text}')
if frame_slices:
print(f' X axis: {frame_array.x_axis}')
iflr_refs = logical_file.iflr_position_map[frame_array.ident]
# print(f'TRACE: Frames: {len(iflr_refs)} from {iflr_refs[0]} to {iflr_refs[-1]}')
x_spacing = (iflr_refs[-1].x_axis - iflr_refs[0].x_axis) / (len(iflr_refs) -1)
print(
f' Frames: {len(iflr_refs)}'
f' from {iflr_refs[0].x_axis}'
f' to {iflr_refs[-1].x_axis}'
f' interval {x_spacing}'
f' [{frame_array.x_axis.units}]'
)
print()
else:
print('No log pass')
# print()
except ExceptionTotalDepthRP66V1: # pragma: no cover
logger.exception(f'Failed to index with ExceptionTotalDepthRP66V1: {path_in}')
except Exception: # pragma: no cover
logger.exception(f'Failed to index with Exception: {path_in}')
def test_logical_index_number_of_logical_files(bytes_name, expected):
by = getattr(test_data, bytes_name)
fobj = io.BytesIO(by)
with LogicalFile.LogicalIndex(fobj) as log_index:
assert len(log_index) == expected
def test_logical_index_number_of_logical_files_no_with(by, expected):
fobj = io.BytesIO(by)
index = LogicalFile.LogicalIndex(fobj)
assert len(index) == expected