def main() -> int:
"""Main entry point."""
description = """usage: %(prog)s [options] file
Reads RP66V1 file(s) and writes them out as LAS files."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = cmn_cmd_opts.path_in_out(
description, prog='TotalDepth.RP66V1.ToLAS.main', version=__version__, epilog=__rights__
)
cmn_cmd_opts.add_log_level(parser, level=20)
cmn_cmd_opts.add_multiprocessing(parser)
Slice.add_frame_slice_to_argument_parser(parser, use_what=True)
process.add_process_logger_to_argument_parser(parser)
gnuplot.add_gnuplot_to_argument_parser(parser)
parser.add_argument(
'--array-reduction', type=str,
help='Method to reduce multidimensional channel data to a single value. [default: %(default)s]',
default='first',
choices=list(sorted(ARRAY_REDUCTIONS)),
)
parser.add_argument(
'--channels', type=str,
help='Comma separated list of channels to write out (X axis is always included).'
' Use \'?\' to see what channels exist without writing anything. [default: "%(default)s"]',
default='',
)
parser.add_argument('--field-width', type=int,
help='Field width for array data [default: %(default)s].', default=16)
parser.add_argument('--float-format', type=str,
help='Floating point format for array data [default: "%(default)s"].', default='.3f')
args = parser.parse_args()
cmn_cmd_opts.set_log_level(args)
# print('args:', args)
# return 0
# Your code here
clk_start = time.perf_counter()
ret_val = 0
result: typing.Dict[str, LASWriteResult] = {}
# if os.path.isfile(args.path_in) and (args.frame_slice.strip() == '?' or args.channels.strip() == '?'):
if args.frame_slice.strip() == '?' or args.channels.strip() == '?':
dump_frames_and_or_channels(args.path_in, args.recurse, args.frame_slice.strip(), args.channels.strip())
else:
channel_set = set()
for ch in args.channels.strip().split(','):
if ch.strip() != '':
channel_set.add(ch.strip())
if cmn_cmd_opts.multiprocessing_requested(args) and os.path.isdir(args.path_in):
result = convert_rp66v1_dir_or_file_to_las_multiprocessing(
args.path_in,
args.path_out,
args.recurse,
args.array_reduction,
Slice.create_slice_or_sample(args.frame_slice),
channel_set,
args.field_width,
args.float_format,
args.jobs,
)
else:
if args.log_process > 0.0:
with process.log_process(args.log_process):
result = convert_rp66v1_dir_or_file_to_las(
args.path_in,
args.path_out,
args.recurse,
args.array_reduction,
Slice.create_slice_or_sample(args.frame_slice),
channel_set,
args.field_width,
args.float_format,
)
else:
result = convert_rp66v1_dir_or_file_to_las(
args.path_in,
args.path_out,
args.recurse,
args.array_reduction,
Slice.create_slice_or_sample(args.frame_slice),
channel_set,
args.field_width,
args.float_format,
)
clk_exec = time.perf_counter() - clk_start
# Report output
if result:
size_index = size_input = 0
files_processed = 0
table = [
['Input', 'Output', 'LAS Count', 'Time', 'Ratio', 'ms/Mb', 'Exception', 'Path']
]
for path in sorted(result.keys()):
las_result = result[path]
# print('TRACE: las_result', las_result)
if las_result.size_input > 0:
ms_mb = las_result.time * 1000 / (las_result.size_input / 1024 ** 2)
ratio = las_result.size_output / las_result.size_input
out = [
f'{las_result.size_input:,d}',
f'{las_result.size_output:,d}',
f'{las_result.las_count:,d}',
f'{las_result.time:.3f}',
f'{ratio:.1%}',
f'{ms_mb:.1f}',
f'{str(las_result.exception)}',
f'"{path}"',
]
table.append(out)
# print(' '.join(out))
size_input += result[path].size_input
size_index += result[path].size_output
files_processed += 1
if las_result.exception:
ret_val = 1
for row in data_table.format_table(table, pad=' ', heading_underline='-'):
print(row)
try:
if args.gnuplot:
plot_gnuplot(result, args.gnuplot)
except Exception as err: # pragma: no cover
logger.exception(str(err))
ret_val = 2
print('Execution time = %8.3f (S)' % clk_exec)
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input/ 1024**2)
ratio = size_index / size_input
else:
ms_mb = 0.0
ratio = 0.0
print(f'Out of {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes')
print(f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {ms_mb:.1f} ms/Mb')
else:
print(f'Execution time: {clk_exec:.3f} (s)')
print('Bye, bye!')
return ret_val
def test_create_slice_or_sample_raises(init, expected):
with pytest.raises(ValueError) as err:
Slice.create_slice_or_sample(init)
assert err.value.args[0] == expected
def main() -> int:
description = """Scans a RP66V1 file or directory and writes HTML version of the data."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = cmn_cmd_opts.path_in_out(description,
prog='TotalDepth.RP66V1.ScanHTML.main',
version=__version__,
epilog=__rights__)
cmn_cmd_opts.add_log_level(parser, level=20)
cmn_cmd_opts.add_multiprocessing(parser)
parser.add_argument(
'-e',
'--encrypted',
action='store_true',
help='Output encrypted Logical Records as well. [default: %(default)s]',
)
Slice.add_frame_slice_to_argument_parser(parser)
process.add_process_logger_to_argument_parser(parser)
gnuplot.add_gnuplot_to_argument_parser(parser)
args = parser.parse_args()
cmn_cmd_opts.set_log_level(args)
# print('args:', args)
# return 0
clk_start = time.perf_counter()
# Your code here
if args.log_process > 0.0:
with process.log_process(args.log_process):
result: typing.Dict[str, HTMLResult] = scan_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
label_process=True,
frame_slice=Slice.create_slice_or_sample(args.frame_slice),
)
else:
if cmn_cmd_opts.multiprocessing_requested(args) and os.path.isdir(
args.path_in):
result: typing.Dict[str, HTMLResult] = scan_dir_multiprocessing(
args.path_in,
args.path_out,
args.jobs,
frame_slice=Slice.create_slice_or_sample(args.frame_slice),
)
else:
result: typing.Dict[str, HTMLResult] = scan_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
label_process=False,
frame_slice=Slice.create_slice_or_sample(args.frame_slice),
)
if args.log_process > 0.0:
process.add_message_to_queue('Processing HTML Complete.')
clk_exec = time.perf_counter() - clk_start
# print('Execution time = %8.3f (S)' % clk_exec)
size_scan = size_input = 0
files_processed = 0
header = (
f'{"Size In":>16}',
f'{"Size Out":>10}',
f'{"Time":>8}',
f'{"Ratio %":>8}',
f'{"ms/Mb":>8}',
f'{"Fail?":5}',
f'Path',
)
print(' '.join(header))
print(' '.join('-' * len(v) for v in header))
for path in sorted(result.keys()):
idx_result = result[path]
if idx_result.size_input > 0:
ms_mb = idx_result.time * 1000 / (idx_result.size_input / 1024**2)
ratio = idx_result.size_output / idx_result.size_input
print(
f'{idx_result.size_input:16,d} {idx_result.size_output:10,d}'
f' {idx_result.time:8.3f} {ratio:8.3%} {ms_mb:8.1f} {str(idx_result.exception):5}'
f' "{path}"')
size_input += result[path].size_input
size_scan += result[path].size_output
files_processed += 1
if args.gnuplot:
try:
plot_gnuplot(result, args.gnuplot)
except IOError:
logger.exception('Plotting with gnuplot failed.')
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input / 1024**2)
else:
ms_mb = 0.0
print(
f'Processed {len(result):,d} files and {size_input:,d} bytes in {clk_exec:.3f} s, {ms_mb:.1f} ms/Mb'
)
print('Bye, bye!')
return 0
def test_create_slice_or_sample(init, expected):
s = Slice.create_slice_or_sample(init)
assert s == expected
def main() -> int:
description = 'usage: %(prog)s [options] file'
description = """Scans a RP66V1 file and dumps data about the file to stdout.
This is useful for examining the details of RP66V1 files and can dump data at various levels of encapsulation,
from the lowest level upwards:
--VR ~ Visible Records only.
--LRSH ~ Logical Record segments.
--LD ~ Logical data i.e. all Logical Record segments concatenated for each Logical Record.
--EFLR ~ Explicitly Formatted Logical Records.
--IFLR ~ Implicitly Formatted Logical Records.
--LR ~ All data, including the numerical analysis of frame data.
If these are combined then the input is scanned multiple times.
"""
print('Cmd: %s' % ' '.join(sys.argv))
# TODO: Use cmn_cmd_opts
parser = argparse.ArgumentParser(
description=description,
epilog=__rights__,
prog=sys.argv[0],
)
parser.add_argument('path_in', type=str, help='Path to the input file or directory.')
parser.add_argument(
'path_out', type=str, default='', nargs='?', help='Path to the output scan to write [default: stdout].'
)
parser.add_argument(
'-V', '--VR', action='store_true',
help='Dump the Visible Records. [default: %(default)s]',
)
parser.add_argument(
'-L', '--LRSH', action='store_true',
help='Summarise the Visible Records and the Logical Record'
' Segment Headers, use -v to dump records. [default: %(default)s]',
)
parser.add_argument(
'-D', '--LD', action='store_true',
help='Summarise logical data, use -v to dump records.'
' See also --dump-bytes, --dump-raw-bytes. [default: %(default)s]',
)
parser.add_argument(
'-E', '--EFLR', action='store_true',
help='Dump EFLR Set. [default: %(default)s]',
)
parser.add_argument(
"--eflr-set-type", action='append', default=[],
help="List of EFLR Set Types to output, additive, if absent then dump all. [default: %(default)s]",
)
parser.add_argument(
'-I', '--IFLR', action='store_true',
help='Dump IFLRs. [default: %(default)s]',
)
parser.add_argument(
"--iflr-set-type", action='append', default=[],
help="List of IFLR Set Types to output, additive, if absent then dump all. [default: %(default)s]",
)
parser.add_argument(
'-R', '--LR', action='store_true',
help='Dump all data, including frame data from Logical Records. [default: %(default)s]',
)
parser.add_argument(
'-d', '--dump-bytes', type=int, default=0,
help='Dump X leading raw bytes for certain options, if -1 all bytes are dumped. [default: %(default)s]',
)
parser.add_argument(
'--dump-raw-bytes', action='store_true',
help='Dump the raw bytes for certain options in raw format,'
' otherwise Hex format is used. [default: %(default)s]',
)
parser.add_argument(
'-r', '--recurse', action='store_true',
help='Process files recursively. [default: %(default)s]',
)
parser.add_argument(
'-e', '--encrypted', action='store_true',
help='Output encrypted Logical Records as well. [default: %(default)s]',
)
parser.add_argument(
'-k', '--keep-going', action='store_true',
help='Keep going as far as sensible. [default: %(default)s]',
)
Slice.add_frame_slice_to_argument_parser(parser, help_prefix='NOTE: Requires -R, --LR.')
parser.add_argument(
'--eflr-as-table', action='store_true',
help='When with --LR and not --html then dump EFLRs as tables, otherwise every EFLR object.'
' [default: %(default)s]',
)
log_level_help_mapping = ', '.join(
['{:d}<->{:s}'.format(level, logging._levelToName[level]) for level in sorted(logging._levelToName.keys())]
)
log_level_help = f'Log Level as an integer or symbol. ({log_level_help_mapping}) [default: %(default)s]'
parser.add_argument(
"-l", "--log-level",
# type=int,
# dest="loglevel",
default=30,
help=log_level_help
)
parser.add_argument(
"-v", "--verbose", action='count', default=0,
help="Increase verbosity, additive [default: %(default)s]",
)
gnuplot.add_gnuplot_to_argument_parser(parser)
parser.add_argument(
'-T', '--test-data', action='store_true',
help='Dump the file as annotated bytes, useful for creating test data. [default: %(default)s]',
)
args = parser.parse_args()
print('args:', args)
# Extract log level
if args.log_level in logging._nameToLevel:
log_level = logging._nameToLevel[args.log_level]
else:
log_level = int(args.log_level)
# print('Log level:', log_level)
# Initialise logging etc.
logging.basicConfig(level=log_level,
format='%(asctime)s %(levelname)-8s %(message)s',
#datefmt='%y-%m-%d % %H:%M:%S',
stream=sys.stdout)
clk_start = time.perf_counter()
# return 0
# Your code here
result: typing.Dict[str, IndexResult] = {}
output_extension = '.txt'
if args.VR or args.LRSH:
result = scan_dir_or_file(
args.path_in,
args.path_out,
scan_RP66V1_file_visible_records,
args.recurse,
output_extension,
# kwargs passed to scanning function
lrsh_dump=args.LRSH,
verbose=args.verbose,
)
if args.LD:
result = scan_dir_or_file(
args.path_in,
args.path_out,
scan_RP66V1_file_logical_data,
args.recurse,
output_extension,
# kwargs passed to scanning function
dump_bytes=args.dump_bytes,
dump_raw_bytes=args.dump_raw_bytes,
verbose=args.verbose,
)
if args.EFLR or args.IFLR:
result = scan_dir_or_file(
args.path_in,
args.path_out,
scan_RP66V1_file_EFLR_IFLR,
args.recurse,
output_extension,
# kwargs passed to scanning function
verbose=args.verbose,
encrypted=args.encrypted,
keep_going=args.keep_going,
eflr_set_type=[bytes(v, 'ascii') for v in args.eflr_set_type],
iflr_set_type=[bytes(v, 'ascii') for v in args.iflr_set_type],
iflr_dump=args.IFLR,
eflr_dump=args.EFLR,
rp66v1_path=args.path_in,
)
if args.LR:
result = scan_dir_or_file(
args.path_in,
args.path_out,
scan_RP66V1_file_data_content,
args.recurse,
output_extension,
rp66v1_path=args.path_in,
frame_slice=Slice.create_slice_or_sample(args.frame_slice),
eflr_as_table=args.eflr_as_table,
)
if args.test_data:
result = scan_dir_or_file(
args.path_in,
args.path_out,
dump_RP66V1_test_data,
args.recurse,
output_extension,
verbose=args.verbose,
)
clk_exec = time.perf_counter() - clk_start
size_scan = size_input = 0
failures = 0
files_processed = 0
for path in sorted(result.keys()):
idx_result = result[path]
if idx_result.size_input > 0:
ms_mb = idx_result.time * 1000 / (idx_result.size_input / 1024 ** 2)
ratio = idx_result.size_output / idx_result.size_input
print(
f'{idx_result.size_input:16,d} {idx_result.size_output:10,d}'
f' {idx_result.time:8.3f} {ratio:8.3%} {ms_mb:8.1f} {str(idx_result.exception):5}'
f' "{path}"'
)
size_input += result[path].size_input
size_scan += result[path].size_output
files_processed += 1
if idx_result.exception:
failures += 1
if args.gnuplot:
plot_gnuplot(result, args.gnuplot)
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input / 1024**2)
else:
ms_mb = 0.0
print('Execution time = %8.3f (S)' % clk_exec)
print(f'Processed {len(result):,d} files and {size_input:,d} bytes, {ms_mb:.1f} ms/Mb')
print('Bye, bye!')
return failures