def test_format_table_simple_raises():
with pytest.raises(ValueError) as err:
data_table.format_table([['Col 1', 'Col 2'], [
1,
]])
assert err.value.args[
0] == 'Rows not of equal length but lengths of {1, 2}'
def write_parameter_section_to_las(
logical_file: LogicalFile.LogicalFile,
ostream: typing.TextIO,
) -> None:
"""Write the ``PARAMETER`` tables to LAS."""
las_mnem_map: typing.Dict[str, UnitValueDescription] = collections.OrderedDict()
for position_eflr in logical_file.eflrs:
if position_eflr.eflr.set.type == b'PARAMETER':
# print('TRACE:', position_eflr.eflr.str_long())
for obj in position_eflr.eflr.objects:
units = obj[b'VALUES'].units.decode('ascii')
value = stringify.stringify_object_by_type(obj[b'VALUES'].value).strip()
descr = stringify.stringify_object_by_type(obj[b'LONG-NAME'].value).strip()
# NOTE: Overwriting is possible here.
las_mnem_map[obj.name.I.decode('ascii')] = UnitValueDescription(units, value, descr)
table = [
['#MNEM.UNIT', 'Value', 'Description'],
['#---------', '-----', '-----------'],
]
# pprint.pprint(las_mnem_map)
for k in las_mnem_map:
table.append(
[f'{k:<4}.{las_mnem_map[k].unit:<4}', las_mnem_map[k].value, f': {las_mnem_map[k].description}']
)
rows = data_table.format_table(table, pad=' ', left_flush=True)
ostream.write('~Parameter Information Section\n')
for row in rows:
ostream.write(row)
ostream.write('\n')
def _scan_log_pass_content(
logical_index: LogicalFile.LogicalIndex,
logical_file_index: int,
fout: typing.TextIO,
*,
frame_slice: Slice.Slice) -> None:
"""Scan the LogPass."""
assert logical_index[logical_file_index].has_log_pass
logical_file = logical_index[logical_file_index]
lp: LogPass.LogPass = logical_file.log_pass
frame_array: LogPass.FrameArray
for fa, frame_array in enumerate(lp.frame_arrays):
with _output_section_header_trailer(f'Frame Array [{fa}/{len(lp.frame_arrays)}]', '^', os=fout):
fout.write(str(frame_array))
fout.write('\n')
num_frames = logical_file.populate_frame_array(
frame_array,
frame_slice,
)
if num_frames > 0:
x_axis: XAxis.XAxis = logical_index[logical_file_index].iflr_position_map[frame_array.ident]
_write_x_axis_summary(x_axis, fout)
if x_axis.summary.spacing is not None:
interval = f'{x_axis.summary.spacing.median:0.3f}'
else:
interval = 'N/A'
fout.write(
f'Frames [{len(x_axis)}]'
f' from: {float(x_axis[0].x_axis):0.3f}'
f' to {float(x_axis[-1].x_axis):0.3f}'
f' Interval: {interval}'
f' {frame_array.x_axis.units}'
)
fout.write('\n')
fout.write(
f'Frame spacing: {frame_slice.long_str(len(x_axis))}'
f' number of frames: {num_frames}'
f' numpy size: {frame_array.sizeof_array:,d} bytes'
)
fout.write('\n')
frame_table = [['Channel', 'Size', 'Absent', 'Min', 'Mean', 'Std.Dev.', 'Max', 'Units', 'dtype']]
for channel in frame_array.channels:
channel_ident = channel.ident.I.decode("ascii")
# arr = channel.array
arr = AbsentValue.mask_absent_values(channel.array)
frame_table.append(
[channel_ident, arr.size,
# NOTE: Not the masked array!
AbsentValue.count_of_absent_values(channel.array),
arr.min(), arr.mean(),
arr.std(), arr.max(), channel.units, arr.dtype]
)
fout.write('\n'.join(data_table.format_table(frame_table, heading_underline='-', pad=' ')))
fout.write('\n')
else:
fout.write('No frames.')
fout.write('\n')
def test_format_table_simple():
formatted_table = data_table.format_table(SIMPLE_TABLE)
expected = """Col 1 Col 2
1 2
3 4
5 6"""
# print(formatted_table)
assert formatted_table == [
'Col 1 Col 2', ' 1 2', ' 3 4', ' 5 6'
]
assert '\n'.join(formatted_table) == expected
def test_format_table_simple_pad():
formatted_table = data_table.format_table(SIMPLE_TABLE, pad=' | ')
expected = """Col 1 | Col 2
1 | 2
3 | 4
5 | 6"""
# print(formatted_table)
assert formatted_table == [
'Col 1 | Col 2', ' 1 | 2', ' 3 | 4', ' 5 | 6'
]
assert '\n'.join(formatted_table) == expected
def test_format_table_simple_floats():
formatted_table = data_table.format_table(SIMPLE_TABLE_WITH_FLOATS)
expected = """Col 1 Col 2
1.000 2.000
3.000 4.000
5.000 6.000"""
# print(formatted_table)
assert formatted_table == [
'Col 1 Col 2', '1.000 2.000', '3.000 4.000', '5.000 6.000'
]
# print('\n'.join(formatted_table))
assert '\n'.join(formatted_table) == expected
def test_format_table_simple_sphinx():
formatted_table = data_table.format_table(SIMPLE_TABLE,
heading_underline='=')
expected = """Col 1 Col 2
===== =====
1 2
3 4
5 6"""
# print(formatted_table)
assert formatted_table == [
'Col 1 Col 2', '===== =====', ' 1 2', ' 3 4',
' 5 6'
]
assert '\n'.join(formatted_table) == expected
def _write_las_header(input_file: str,
logical_file: LogicalFile.LogicalFile,
logical_file_number: int,
frame_array_ident: str,
ostream: typing.TextIO) -> None:
"""Writes the LAS opening such as:
~Version Information Section
VERS. 2.0 : CWLS Log ASCII Standard - VERSION 2.0
WRAP. NO : One Line per depth step
PROD. TotalDepth : LAS Producer
PROG. TotalDepth.RP66V1.ToLAS 0.1.1 : LAS Program name and version
CREA. 2012-11-14 10:50 : LAS Creation date [YYYY-MMM-DD hh:mm]
DLIS_CREA. 2012-11-10 22:06 : DLIS Creation date and time [YYYY-MMM-DD hh:mm]
SOURCE. SOME-FILE-NAME.dlis : DLIS File Name
FILE-ID. SOME-FILE-ID : File Identification from the FILE-HEADER Logical Record
LOGICAL-FILE. 3 : Logical File number in the DLIS file
FRAME-ARRAY. 60B : Identity of the Frame Array in the Logical File
Reference: [LAS2.0 Las2_Update_Feb2017.pdf Section 5.3 ~V (Version Information)]
"""
now = datetime.datetime.utcnow()
date_time = logical_file.defining_origin[b'CREATION-TIME'].value[0]
dt = date_time.as_datetime()
fhlr: EFLR.ExplicitlyFormattedLogicalRecord = logical_file.file_header_logical_record
file_id = fhlr.objects[0][b'ID'].value[0].decode('ascii').strip()
table: typing.List[typing.List[str]] = [
['VERS.', '2.0', ': CWLS Log ASCII Standard - VERSION 2.0'],
['WRAP.', 'NO', ': One Line per depth step'],
['PROD.', 'TotalDepth', ': LAS Producer'],
['PROG.', f'TotalDepth.RP66V1.ToLAS {LAS_PRODUCER_VERSION}', ': LAS Program name and version'],
['CREA.', f'{now.strftime(LAS_DATETIME_FORMAT_UTC)}', f': LAS Creation date [{LAS_DATE_FORMAT_TEXT}]'],
[
f'DLIS_CREA.',
f'{dt.strftime(LAS_DATETIME_FORMAT_UTC)}', f': DLIS Creation date and time [{LAS_DATE_FORMAT_TEXT}]'
],
['SOURCE.', f'{os.path.basename(input_file)}', ': DLIS File Name'],
['FILE-ID.', f'{file_id}', ': File Identification Number'],
['LOGICAL-FILE.', f'{logical_file_number:d}', ': Logical File number in the DLIS file'],
]
if frame_array_ident:
table.append(
['FRAME-ARRAY.', f'{frame_array_ident}', ': Identity of the Frame Array in the Logical File'],
)
rows = data_table.format_table(table, pad=' ', left_flush=True)
ostream.write('~Version Information Section\n')
for row in rows:
ostream.write(row)
ostream.write('\n')
def write_well_information_to_las(
logical_file: LogicalFile.LogicalFile,
frame_array: typing.Union[LogPass.FrameArray, None],
frame_slice: typing.Union[Slice.Slice, Slice.Sample],
ostream: typing.TextIO,
) -> None:
"""Writes the well information section.
Reference: ``[LAS2.0 Las2_Update_Feb2017.pdf Section 5.4 ~W (Well Information)]``
"""
# Tuple of (units, value, description)
las_map: typing.Dict[str, UnitValueDescription] = extract_well_information_from_origin(logical_file)
_add_start_stop_step_to_dictionary(logical_file, frame_array, frame_slice, las_map)
eflr: EFLR.ExplicitlyFormattedLogicalRecord
for _lrsh_position, eflr in logical_file.eflrs:
if eflr.set.type in DLIS_TO_WELL_INFORMATION_LAS_EFLR_MAPPING:
bytes_index_map: typing.Dict[bytes, int] = EFLR.reduced_object_map(eflr)
for row_key in DLIS_TO_WELL_INFORMATION_LAS_EFLR_MAPPING[eflr.set.type]:
if row_key in bytes_index_map:
obj = eflr[bytes_index_map[row_key]]
# ORIGIN is only key/value so does not have LONG-NAME
# PARAMETER does have LONG-NAME
units = obj[b'VALUES'].units.decode('ascii')
value = stringify.stringify_object_by_type(obj[b'VALUES'].value).strip()
descr = stringify.stringify_object_by_type(obj[b'LONG-NAME'].value).strip()
# NOTE: Overwriting is possible here.
las_map[row_key.decode('ascii')] = UnitValueDescription(units, value, descr)
table = [
['#MNEM.UNIT', 'DATA', 'DESCRIPTION',],
['#----.----', '----', '-----------',],
]
for k in WELL_INFORMATION_KEYS:
if k in las_map:
row = [f'{k:4}.{las_map[k].unit:4}', f'{las_map[k].value}', f': {las_map[k].description}',]
else:
row = [f'{k:4}.{"":4}', '', ':']
table.append(row)
rows = data_table.format_table(table, pad=' ', left_flush=True)
ostream.write('~Well Information Section\n')
for row in rows:
ostream.write(row)
ostream.write('\n')
def scan_RP66V1_file_data_content(fobj: typing.BinaryIO, fout: typing.TextIO,
*, rp66v1_path: str, frame_slice: Slice.Slice, eflr_as_table: bool) -> None:
"""
Scans all of every EFLR and IFLR in the file using a ScanFile object.
"""
with LogicalFile.LogicalIndex(fobj) as logical_index:
with _output_section_header_trailer('RP66V1 File Data Summary', '*', os=fout):
fout.write(str(logical_index.storage_unit_label))
fout.write('\n')
logical_file: LogicalFile.LogicalFile
for lf, logical_file in enumerate(logical_index.logical_files):
with _output_section_header_trailer(f'Logical File [{lf}/{len(logical_index.logical_files)}]', '=', os=fout):
fout.write(str(logical_file))
fout.write('\n')
eflr_position: LogicalFile.PositionEFLR
for e, eflr_position in enumerate(logical_file.eflrs):
header = f'EFLR [{e}/{len(logical_file.eflrs)}] at {str(eflr_position.lrsh_position)}'
with _output_section_header_trailer(header, '-', os=fout):
# fout.write(str(eflr_position.eflr))
# fout.write('\n')
if eflr_as_table:
if eflr_position.eflr.is_key_value():
eflr_str_table = eflr_position.eflr.key_values(
stringify_function=stringify.stringify_object_by_type,
sort=True
)
else:
eflr_str_table = eflr_position.eflr.table_as_strings(
stringify_function=stringify.stringify_object_by_type,
sort=True
)
fout.write('\n'.join(data_table.format_table(eflr_str_table, heading_underline='-')))
fout.write('\n')
else:
fout.write(eflr_position.eflr.str_long())
fout.write('\n')
# Now the LogPass(s)
if logical_file.has_log_pass:
with _output_section_header_trailer('Log Pass', '-', os=fout):
_scan_log_pass_content(logical_index, lf, fout, frame_slice=frame_slice)
else:
fout.write('NO Log Pass for this Logical Record\n')
def write_curve_section_to_las(
frame_array: LogPass.FrameArray,
channels: typing.Set[str],
ostream: typing.TextIO,
) -> None:
"""Write the ``~Curve Information Section`` to the LAS file."""
ostream.write('~Curve Information Section\n')
table = [
['#MNEM.UNIT', 'Curve Description'],
['#---------', '-----------------'],
]
for c, channel in enumerate(frame_array.channels):
if len(channels) == 0 or c == 0 or channel.ident.I.decode("ascii") in channels:
desc = ' '.join([
f': {channel.long_name.decode("ascii")}',
f'Rep Code: {RepCode.REP_CODE_INT_TO_STR[channel.rep_code]}',
f'Dimensions: {channel.dimensions}'
])
table.append([f'{channel.ident.I.decode("ascii"):<4}.{channel.units.decode("ascii"):<4}', desc])
rows = data_table.format_table(table, pad=' ', left_flush=True)
for row in rows:
ostream.write(row)
ostream.write('\n')
def main() -> int:
"""Main entry point."""
description = """usage: %(prog)s [options] file
Reads RP66V1 file(s) and writes them out as LAS files."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = cmn_cmd_opts.path_in_out(
description, prog='TotalDepth.RP66V1.ToLAS.main', version=__version__, epilog=__rights__
)
cmn_cmd_opts.add_log_level(parser, level=20)
cmn_cmd_opts.add_multiprocessing(parser)
Slice.add_frame_slice_to_argument_parser(parser, use_what=True)
process.add_process_logger_to_argument_parser(parser)
gnuplot.add_gnuplot_to_argument_parser(parser)
parser.add_argument(
'--array-reduction', type=str,
help='Method to reduce multidimensional channel data to a single value. [default: %(default)s]',
default='first',
choices=list(sorted(ARRAY_REDUCTIONS)),
)
parser.add_argument(
'--channels', type=str,
help='Comma separated list of channels to write out (X axis is always included).'
' Use \'?\' to see what channels exist without writing anything. [default: "%(default)s"]',
default='',
)
parser.add_argument('--field-width', type=int,
help='Field width for array data [default: %(default)s].', default=16)
parser.add_argument('--float-format', type=str,
help='Floating point format for array data [default: "%(default)s"].', default='.3f')
args = parser.parse_args()
cmn_cmd_opts.set_log_level(args)
# print('args:', args)
# return 0
# Your code here
clk_start = time.perf_counter()
ret_val = 0
result: typing.Dict[str, LASWriteResult] = {}
# if os.path.isfile(args.path_in) and (args.frame_slice.strip() == '?' or args.channels.strip() == '?'):
if args.frame_slice.strip() == '?' or args.channels.strip() == '?':
dump_frames_and_or_channels(args.path_in, args.recurse, args.frame_slice.strip(), args.channels.strip())
else:
channel_set = set()
for ch in args.channels.strip().split(','):
if ch.strip() != '':
channel_set.add(ch.strip())
if cmn_cmd_opts.multiprocessing_requested(args) and os.path.isdir(args.path_in):
result = convert_rp66v1_dir_or_file_to_las_multiprocessing(
args.path_in,
args.path_out,
args.recurse,
args.array_reduction,
Slice.create_slice_or_sample(args.frame_slice),
channel_set,
args.field_width,
args.float_format,
args.jobs,
)
else:
if args.log_process > 0.0:
with process.log_process(args.log_process):
result = convert_rp66v1_dir_or_file_to_las(
args.path_in,
args.path_out,
args.recurse,
args.array_reduction,
Slice.create_slice_or_sample(args.frame_slice),
channel_set,
args.field_width,
args.float_format,
)
else:
result = convert_rp66v1_dir_or_file_to_las(
args.path_in,
args.path_out,
args.recurse,
args.array_reduction,
Slice.create_slice_or_sample(args.frame_slice),
channel_set,
args.field_width,
args.float_format,
)
clk_exec = time.perf_counter() - clk_start
# Report output
if result:
size_index = size_input = 0
files_processed = 0
table = [
['Input', 'Output', 'LAS Count', 'Time', 'Ratio', 'ms/Mb', 'Exception', 'Path']
]
for path in sorted(result.keys()):
las_result = result[path]
# print('TRACE: las_result', las_result)
if las_result.size_input > 0:
ms_mb = las_result.time * 1000 / (las_result.size_input / 1024 ** 2)
ratio = las_result.size_output / las_result.size_input
out = [
f'{las_result.size_input:,d}',
f'{las_result.size_output:,d}',
f'{las_result.las_count:,d}',
f'{las_result.time:.3f}',
f'{ratio:.1%}',
f'{ms_mb:.1f}',
f'{str(las_result.exception)}',
f'"{path}"',
]
table.append(out)
# print(' '.join(out))
size_input += result[path].size_input
size_index += result[path].size_output
files_processed += 1
if las_result.exception:
ret_val = 1
for row in data_table.format_table(table, pad=' ', heading_underline='-'):
print(row)
try:
if args.gnuplot:
plot_gnuplot(result, args.gnuplot)
except Exception as err: # pragma: no cover
logger.exception(str(err))
ret_val = 2
print('Execution time = %8.3f (S)' % clk_exec)
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input/ 1024**2)
ratio = size_index / size_input
else:
ms_mb = 0.0
ratio = 0.0
print(f'Out of {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes')
print(f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {ms_mb:.1f} ms/Mb')
else:
print(f'Execution time: {clk_exec:.3f} (s)')
print('Bye, bye!')
return ret_val
def main() -> int:
description = """usage: %(prog)s [options] file
Scans a RP66V1 file or directory and saves the index as a pickled file."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = cmn_cmd_opts.path_in_out(
description, prog='TotalDepth.RP66V1.IndexPickle.main', version=__version__, epilog=__rights__
)
cmn_cmd_opts.add_log_level(parser, level=20)
cmn_cmd_opts.add_multiprocessing(parser)
parser.add_argument('--read-back', action='store_true', help='Read and time the output. [default: %(default)s]')
process.add_process_logger_to_argument_parser(parser)
gnuplot.add_gnuplot_to_argument_parser(parser)
args = parser.parse_args()
# print('args:', args)
# return 0
cmn_cmd_opts.set_log_level(args)
# Your code here
clk_start = time.perf_counter()
ret_val = 0
if cmn_cmd_opts.multiprocessing_requested(args) and os.path.isdir(args.path_in):
result: typing.Dict[str, IndexResult] = index_dir_multiprocessing(
args.path_in,
args.path_out,
args.jobs,
args.recurse,
args.read_back,
)
else:
if args.log_process > 0.0:
with process.log_process(args.log_process):
result: typing.Dict[str, IndexResult] = index_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
args.read_back,
)
else:
result: typing.Dict[str, IndexResult] = index_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
args.read_back,
)
clk_exec = time.perf_counter() - clk_start
size_index = size_input = 0
files_processed = 0
try:
path_prefix = os.path.commonpath(result.keys())
len_path_prefix = len(path_prefix)
table: typing.List[typing.List[str]] = [
[
'Size (b)', 'Index (b)', 'Ratio (%)',
'Index (s)', 'Index (ms/Mb)',
'Write (s)', 'Write (ms/Mb)',
'Read (s)', 'Read (ms/Mb)',
'Except',
'Path',
]
]
for path in sorted(result.keys()):
idx_result = result[path]
if not idx_result.ignored and idx_result.size_input > 0:
ms_mb_index = idx_result.time_index * 1000 / (idx_result.size_input / 1024 ** 2)
ms_mb_write = idx_result.time_write * 1000 / (idx_result.size_input / 1024 ** 2)
ms_mb_read = idx_result.time_read * 1000 / (idx_result.size_input / 1024 ** 2)
ratio = idx_result.size_index / idx_result.size_input
table.append(
[
f'{idx_result.size_input:,d}', f'{idx_result.size_index:,d}', f'{ratio:.3%}',
f'{idx_result.time_index:.3f}', f'{ms_mb_index:.1f}',
f'{idx_result.time_write:.3f}', f'{ms_mb_write:.1f}',
f'{idx_result.time_read:.3f}', f'{ms_mb_read:.2f}',
f'{str(idx_result.exception):5}',
f'{path[len_path_prefix+1:]}',
]
)
size_input += result[path].size_input
size_index += result[path].size_index
files_processed += 1
if idx_result.exception: # pragma: no cover
ret_val = 1
print(f'Common path prefix: {path_prefix}')
print('\n'.join(data_table.format_table(table, pad=' | ', heading_underline='-')))
if args.gnuplot:
try:
plot_gnuplot(result, args.gnuplot)
except IOError: # pragma: no cover
logger.exception('Plotting with gnuplot failed.')
ret_val = 2
except Exception as err: # pragma: no cover
logger.exception(str(err))
ret_val = 3
print('Execution time = %8.3f (S)' % clk_exec)
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input/ 1024**2)
ratio = size_index / size_input
else: # pragma: no cover
ms_mb = 0.0
ratio = 0.0
print(f'Out of {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes')
print(f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {ms_mb:.1f} ms/Mb')
print('Bye, bye!')
return ret_val