def main() -> int:
"""Main entry point."""
description = """usage: %(prog)s [options] file
Scans a RP66V1 file or directory and indexes the files with a LogicalRecordIndex."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = cmn_cmd_opts.path_in_out(
description,
prog='TotalDepth.RP66V1.LogRecIndex.main',
version=__version__,
epilog=__rights__)
cmn_cmd_opts.add_log_level(parser, level=20)
cmn_cmd_opts.add_multiprocessing(parser)
process.add_process_logger_to_argument_parser(parser)
gnuplot.add_gnuplot_to_argument_parser(parser)
parser.add_argument(
'--read-back',
action='store_true',
help=
'Read the pickled index and time it (requires path_out). [default: %(default)s]',
)
parser.add_argument(
'--validate',
action='store_true',
help='Perform validation checks on the index.. [default: %(default)s]',
)
args = parser.parse_args()
# print('args:', args)
# return 0
cmn_cmd_opts.set_log_level(args)
# Your code here
exec_timer = ExecTimer.Timer('LogRecIndex')
if os.path.isdir(
args.path_in) and cmn_cmd_opts.multiprocessing_requested(args):
result: typing.Dict[str, IndexResult] = index_dir_multiprocessing(
args.path_in,
args.path_out,
args.jobs,
args.recurse,
args.read_back,
args.validate,
)
else:
if args.log_process > 0.0:
with process.log_process(args.log_process):
result: typing.Dict[str, IndexResult] = index_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
args.read_back,
args.validate,
)
else:
result: typing.Dict[str, IndexResult] = index_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
args.read_back,
args.validate,
)
size_index = size_input = 0
files_processed = 0
if os.path.isdir(args.path_in):
len_path = len(args.path_in)
if not args.path_in.endswith(os.sep):
len_path += 1
else:
len_path = 0
ret_value = 0
try:
header = (
f'{"Size In":>16}',
f'{"Size Out":>16}',
f'{"Time (ms)":>10}',
f'{"Ratio %":>8}',
f'{"ms/Mb":>8}',
f'{"Fail?":5}',
f'{"Write (ms)":>10}',
f'{"Read (ms)":>10}',
f'Path',
)
print(' '.join(header))
print(' '.join('-' * len(v) for v in header))
for path in sorted(result.keys()):
idx_result = result[path]
if idx_result.size_input > 0:
ms_mb = idx_result.index_time * 1000 / (idx_result.size_input /
1024**2)
ratio = idx_result.size_index / idx_result.size_input
print(
f'{idx_result.size_input:16,d} {idx_result.size_index:16,d}'
f' {1000*idx_result.index_time:10.1f} {ratio:8.3%} {ms_mb:8.1f} {str(idx_result.exception):5}'
f' {1000*idx_result.time_write:10.1f} {1000*idx_result.time_read_back:10.1f}'
f' "{path[len_path:]}"')
size_input += result[path].size_input
size_index += result[path].size_index
files_processed += 1
exec_timer.add_work_done(idx_result.size_input)
if args.gnuplot:
try:
plot_gnuplot(result, args.gnuplot)
except Exception: # pragma: no cover
logger.exception('gunplot failed')
ret_value = 1
except Exception as err: # pragma: no cover
logger.exception(str(err))
ret_value = 2
print(f'Number of jobs: {args.jobs}')
print('Execution:')
print(exec_timer.long_str)
if size_input > 0:
ratio = size_index / size_input
else:
ratio = 0.0
print(
f'Out of {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes'
)
print(
f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {exec_timer.ms_mb:.1f} ms/Mb'
)
print(f'Processed: {size_input:,d} bytes at {exec_timer.ms_mb:.1f} ms/Mb')
print('Bye, bye!')
return ret_value
def main() -> int:
"""Main entry point."""
description = """usage: %(prog)s [options] file
Reads RP66V1 file(s) and writes them out as LAS files."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = cmn_cmd_opts.path_in_out(
description, prog='TotalDepth.RP66V1.ToLAS.main', version=__version__, epilog=__rights__
)
cmn_cmd_opts.add_log_level(parser, level=20)
cmn_cmd_opts.add_multiprocessing(parser)
Slice.add_frame_slice_to_argument_parser(parser, use_what=True)
process.add_process_logger_to_argument_parser(parser)
gnuplot.add_gnuplot_to_argument_parser(parser)
parser.add_argument(
'--array-reduction', type=str,
help='Method to reduce multidimensional channel data to a single value. [default: %(default)s]',
default='first',
choices=list(sorted(ARRAY_REDUCTIONS)),
)
parser.add_argument(
'--channels', type=str,
help='Comma separated list of channels to write out (X axis is always included).'
' Use \'?\' to see what channels exist without writing anything. [default: "%(default)s"]',
default='',
)
parser.add_argument('--field-width', type=int,
help='Field width for array data [default: %(default)s].', default=16)
parser.add_argument('--float-format', type=str,
help='Floating point format for array data [default: "%(default)s"].', default='.3f')
args = parser.parse_args()
cmn_cmd_opts.set_log_level(args)
# print('args:', args)
# return 0
# Your code here
clk_start = time.perf_counter()
ret_val = 0
result: typing.Dict[str, LASWriteResult] = {}
# if os.path.isfile(args.path_in) and (args.frame_slice.strip() == '?' or args.channels.strip() == '?'):
if args.frame_slice.strip() == '?' or args.channels.strip() == '?':
dump_frames_and_or_channels(args.path_in, args.recurse, args.frame_slice.strip(), args.channels.strip())
else:
channel_set = set()
for ch in args.channels.strip().split(','):
if ch.strip() != '':
channel_set.add(ch.strip())
if cmn_cmd_opts.multiprocessing_requested(args) and os.path.isdir(args.path_in):
result = convert_rp66v1_dir_or_file_to_las_multiprocessing(
args.path_in,
args.path_out,
args.recurse,
args.array_reduction,
Slice.create_slice_or_sample(args.frame_slice),
channel_set,
args.field_width,
args.float_format,
args.jobs,
)
else:
if args.log_process > 0.0:
with process.log_process(args.log_process):
result = convert_rp66v1_dir_or_file_to_las(
args.path_in,
args.path_out,
args.recurse,
args.array_reduction,
Slice.create_slice_or_sample(args.frame_slice),
channel_set,
args.field_width,
args.float_format,
)
else:
result = convert_rp66v1_dir_or_file_to_las(
args.path_in,
args.path_out,
args.recurse,
args.array_reduction,
Slice.create_slice_or_sample(args.frame_slice),
channel_set,
args.field_width,
args.float_format,
)
clk_exec = time.perf_counter() - clk_start
# Report output
if result:
size_index = size_input = 0
files_processed = 0
table = [
['Input', 'Output', 'LAS Count', 'Time', 'Ratio', 'ms/Mb', 'Exception', 'Path']
]
for path in sorted(result.keys()):
las_result = result[path]
# print('TRACE: las_result', las_result)
if las_result.size_input > 0:
ms_mb = las_result.time * 1000 / (las_result.size_input / 1024 ** 2)
ratio = las_result.size_output / las_result.size_input
out = [
f'{las_result.size_input:,d}',
f'{las_result.size_output:,d}',
f'{las_result.las_count:,d}',
f'{las_result.time:.3f}',
f'{ratio:.1%}',
f'{ms_mb:.1f}',
f'{str(las_result.exception)}',
f'"{path}"',
]
table.append(out)
# print(' '.join(out))
size_input += result[path].size_input
size_index += result[path].size_output
files_processed += 1
if las_result.exception:
ret_val = 1
for row in data_table.format_table(table, pad=' ', heading_underline='-'):
print(row)
try:
if args.gnuplot:
plot_gnuplot(result, args.gnuplot)
except Exception as err: # pragma: no cover
logger.exception(str(err))
ret_val = 2
print('Execution time = %8.3f (S)' % clk_exec)
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input/ 1024**2)
ratio = size_index / size_input
else:
ms_mb = 0.0
ratio = 0.0
print(f'Out of {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes')
print(f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {ms_mb:.1f} ms/Mb')
else:
print(f'Execution time: {clk_exec:.3f} (s)')
print('Bye, bye!')
return ret_val
def main() -> int:
description = """usage: %(prog)s [options] file
Scans a RP66V1 file or directory and writes out the index(es) in XML."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = cmn_cmd_opts.path_in_out(description,
prog='TotalDepth.RP66V1.IndexXML.main',
version=__version__,
epilog=__rights__)
cmn_cmd_opts.add_log_level(parser, level=20)
cmn_cmd_opts.add_multiprocessing(parser)
parser.add_argument(
'-e',
'--encrypted',
action='store_true',
help='Output encrypted Logical Records as well. [default: %(default)s]',
)
process.add_process_logger_to_argument_parser(parser)
gnuplot.add_gnuplot_to_argument_parser(parser)
parser.add_argument(
'-p',
'--private',
action='store_true',
help='Also write out private EFLRs. [default: %(default)s]',
)
args = parser.parse_args()
# print('args:', args)
# return 0
cmn_cmd_opts.set_log_level(args)
# Your code here
clk_start = time.perf_counter()
ret_val = 0
if os.path.isdir(
args.path_in) and cmn_cmd_opts.multiprocessing_requested(args):
result: typing.Dict[str, IndexResult] = index_dir_multiprocessing(
args.path_in,
args.path_out,
args.private,
args.jobs,
)
else:
if args.log_process > 0.0:
with process.log_process(args.log_process):
result: typing.Dict[str, IndexResult] = index_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
args.private,
)
else:
result: typing.Dict[str, IndexResult] = index_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
args.private,
)
clk_exec = time.perf_counter() - clk_start
size_index = size_input = 0
files_processed = 0
try:
header = (
f'{"Size In":>16}',
f'{"Size Out":>16}',
f'{"Time":>8}',
f'{"Ratio %":>8}',
f'{"ms/Mb":>8}',
f'{"Fail?":5}',
f'Path',
)
print(' '.join(header))
print(' '.join('-' * len(v) for v in header))
for path in sorted(result.keys()):
idx_result = result[path]
if idx_result.size_input > 0:
ms_mb = idx_result.time * 1000 / (idx_result.size_input /
1024**2)
ratio = idx_result.size_index / idx_result.size_input
print(
f'{idx_result.size_input:16,d} {idx_result.size_index:16,d}'
f' {idx_result.time:8.3f} {ratio:8.3%} {ms_mb:8.1f} {str(idx_result.exception):5}'
f' "{path}"')
size_input += result[path].size_input
size_index += result[path].size_index
files_processed += 1
if args.gnuplot:
try:
plot_gnuplot(result, args.gnuplot)
except Exception: # pragma: no cover
logger.exception('gunplot failed')
ret_val = 1
except Exception as err: # pragma: no cover
logger.exception(str(err))
ret_val = 2
print('Execution time = %8.3f (S)' % clk_exec)
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input / 1024**2)
ratio = size_index / size_input
else: # pragma: no cover
ms_mb = 0.0
ratio = 0.0
print(
f'Out of {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes'
)
print(
f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {ms_mb:.1f} ms/Mb'
)
print('Bye, bye!')
return ret_val
def main() -> int:
description = """usage: %(prog)s [options] file
Scans a RP66V1 file or directory and saves the index as a pickled file."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = cmn_cmd_opts.path_in_out(
description, prog='TotalDepth.RP66V1.IndexPickle.main', version=__version__, epilog=__rights__
)
cmn_cmd_opts.add_log_level(parser, level=20)
cmn_cmd_opts.add_multiprocessing(parser)
parser.add_argument('--read-back', action='store_true', help='Read and time the output. [default: %(default)s]')
process.add_process_logger_to_argument_parser(parser)
gnuplot.add_gnuplot_to_argument_parser(parser)
args = parser.parse_args()
# print('args:', args)
# return 0
cmn_cmd_opts.set_log_level(args)
# Your code here
clk_start = time.perf_counter()
ret_val = 0
if cmn_cmd_opts.multiprocessing_requested(args) and os.path.isdir(args.path_in):
result: typing.Dict[str, IndexResult] = index_dir_multiprocessing(
args.path_in,
args.path_out,
args.jobs,
args.recurse,
args.read_back,
)
else:
if args.log_process > 0.0:
with process.log_process(args.log_process):
result: typing.Dict[str, IndexResult] = index_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
args.read_back,
)
else:
result: typing.Dict[str, IndexResult] = index_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
args.read_back,
)
clk_exec = time.perf_counter() - clk_start
size_index = size_input = 0
files_processed = 0
try:
path_prefix = os.path.commonpath(result.keys())
len_path_prefix = len(path_prefix)
table: typing.List[typing.List[str]] = [
[
'Size (b)', 'Index (b)', 'Ratio (%)',
'Index (s)', 'Index (ms/Mb)',
'Write (s)', 'Write (ms/Mb)',
'Read (s)', 'Read (ms/Mb)',
'Except',
'Path',
]
]
for path in sorted(result.keys()):
idx_result = result[path]
if not idx_result.ignored and idx_result.size_input > 0:
ms_mb_index = idx_result.time_index * 1000 / (idx_result.size_input / 1024 ** 2)
ms_mb_write = idx_result.time_write * 1000 / (idx_result.size_input / 1024 ** 2)
ms_mb_read = idx_result.time_read * 1000 / (idx_result.size_input / 1024 ** 2)
ratio = idx_result.size_index / idx_result.size_input
table.append(
[
f'{idx_result.size_input:,d}', f'{idx_result.size_index:,d}', f'{ratio:.3%}',
f'{idx_result.time_index:.3f}', f'{ms_mb_index:.1f}',
f'{idx_result.time_write:.3f}', f'{ms_mb_write:.1f}',
f'{idx_result.time_read:.3f}', f'{ms_mb_read:.2f}',
f'{str(idx_result.exception):5}',
f'{path[len_path_prefix+1:]}',
]
)
size_input += result[path].size_input
size_index += result[path].size_index
files_processed += 1
if idx_result.exception: # pragma: no cover
ret_val = 1
print(f'Common path prefix: {path_prefix}')
print('\n'.join(data_table.format_table(table, pad=' | ', heading_underline='-')))
if args.gnuplot:
try:
plot_gnuplot(result, args.gnuplot)
except IOError: # pragma: no cover
logger.exception('Plotting with gnuplot failed.')
ret_val = 2
except Exception as err: # pragma: no cover
logger.exception(str(err))
ret_val = 3
print('Execution time = %8.3f (S)' % clk_exec)
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input/ 1024**2)
ratio = size_index / size_input
else: # pragma: no cover
ms_mb = 0.0
ratio = 0.0
print(f'Out of {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes')
print(f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {ms_mb:.1f} ms/Mb')
print('Bye, bye!')
return ret_val
def main() -> int:
description = """usage: %(prog)s [options] file
Reads a RP66V1 index XML file and all the data."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = argparse.ArgumentParser(description=description,
epilog=__rights__,
prog=sys.argv[0])
parser.add_argument('path_in', type=str, help='Path to the input.')
parser.add_argument(
'archive',
type=str,
help='Path to the root directory of the archive.',
default='',
# nargs='?',
)
parser.add_argument(
'-r',
'--recurse',
action='store_true',
help='Process files recursively. [default: %(default)s]',
)
log_level_help_mapping = ', '.join([
'{:d}<->{:s}'.format(level, logging._levelToName[level])
for level in sorted(logging._levelToName.keys())
])
log_level_help = f'Log Level as an integer or symbol. ({log_level_help_mapping}) [default: %(default)s]'
parser.add_argument("-l", "--log-level", default=30, help=log_level_help)
parser.add_argument(
"-v",
"--verbose",
action='count',
default=0,
help="Increase verbosity, additive [default: %(default)s]",
)
gnuplot.add_gnuplot_to_argument_parser(parser)
args = parser.parse_args()
print('args:', args)
# return 0
# Extract log level
if args.log_level in logging._nameToLevel:
log_level = logging._nameToLevel[args.log_level]
else:
log_level = int(args.log_level)
# print('Log level:', log_level)
# Initialise logging etc.
logging.basicConfig(
level=log_level,
format='%(asctime)s %(levelname)-8s %(message)s',
#datefmt='%y-%m-%d % %H:%M:%S',
stream=sys.stdout)
# return 0
# Your code here
clk_start = time.perf_counter()
result: typing.Dict[str, IndexResult] = read_index_dir_or_file(
args.path_in,
args.archive,
args.recurse,
)
clk_exec = time.perf_counter() - clk_start
size_index = size_input = 0
files_processed = 0
try:
for path in sorted(result.keys()):
idx_result = result[path]
if idx_result.size_input > 0:
ms_mb = idx_result.time * 1000 / (idx_result.size_input /
1024**2)
ratio = idx_result.size_index / idx_result.size_input
print(
f'{idx_result.size_input:16,d} {idx_result.size_index:10,d}'
f' {idx_result.time:8.3f} {ratio:8.3%} {ms_mb:8.1f} {str(idx_result.exception):5}'
f' "{path}"')
size_input += result[path].size_input
size_index += result[path].size_index
files_processed += 1
if args.gnuplot:
plot_gnuplot(result, args.gnuplot)
except Exception as err:
logger.exception(str(err))
print('Execution time = %8.3f (S)' % clk_exec)
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input / 1024**2)
ratio = size_index / size_input
else:
ms_mb = 0.0
ratio = 0.0
print(
f'Out of {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes'
)
print(
f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {ms_mb:.1f} ms/Mb'
)
print('Bye, bye!')
return 0
def main() -> int:
description = """Scans a RP66V1 file or directory and writes HTML version of the data."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = cmn_cmd_opts.path_in_out(description,
prog='TotalDepth.RP66V1.ScanHTML.main',
version=__version__,
epilog=__rights__)
cmn_cmd_opts.add_log_level(parser, level=20)
cmn_cmd_opts.add_multiprocessing(parser)
parser.add_argument(
'-e',
'--encrypted',
action='store_true',
help='Output encrypted Logical Records as well. [default: %(default)s]',
)
Slice.add_frame_slice_to_argument_parser(parser)
process.add_process_logger_to_argument_parser(parser)
gnuplot.add_gnuplot_to_argument_parser(parser)
args = parser.parse_args()
cmn_cmd_opts.set_log_level(args)
# print('args:', args)
# return 0
clk_start = time.perf_counter()
# Your code here
if args.log_process > 0.0:
with process.log_process(args.log_process):
result: typing.Dict[str, HTMLResult] = scan_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
label_process=True,
frame_slice=Slice.create_slice_or_sample(args.frame_slice),
)
else:
if cmn_cmd_opts.multiprocessing_requested(args) and os.path.isdir(
args.path_in):
result: typing.Dict[str, HTMLResult] = scan_dir_multiprocessing(
args.path_in,
args.path_out,
args.jobs,
frame_slice=Slice.create_slice_or_sample(args.frame_slice),
)
else:
result: typing.Dict[str, HTMLResult] = scan_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
label_process=False,
frame_slice=Slice.create_slice_or_sample(args.frame_slice),
)
if args.log_process > 0.0:
process.add_message_to_queue('Processing HTML Complete.')
clk_exec = time.perf_counter() - clk_start
# print('Execution time = %8.3f (S)' % clk_exec)
size_scan = size_input = 0
files_processed = 0
header = (
f'{"Size In":>16}',
f'{"Size Out":>10}',
f'{"Time":>8}',
f'{"Ratio %":>8}',
f'{"ms/Mb":>8}',
f'{"Fail?":5}',
f'Path',
)
print(' '.join(header))
print(' '.join('-' * len(v) for v in header))
for path in sorted(result.keys()):
idx_result = result[path]
if idx_result.size_input > 0:
ms_mb = idx_result.time * 1000 / (idx_result.size_input / 1024**2)
ratio = idx_result.size_output / idx_result.size_input
print(
f'{idx_result.size_input:16,d} {idx_result.size_output:10,d}'
f' {idx_result.time:8.3f} {ratio:8.3%} {ms_mb:8.1f} {str(idx_result.exception):5}'
f' "{path}"')
size_input += result[path].size_input
size_scan += result[path].size_output
files_processed += 1
if args.gnuplot:
try:
plot_gnuplot(result, args.gnuplot)
except IOError:
logger.exception('Plotting with gnuplot failed.')
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input / 1024**2)
else:
ms_mb = 0.0
print(
f'Processed {len(result):,d} files and {size_input:,d} bytes in {clk_exec:.3f} s, {ms_mb:.1f} ms/Mb'
)
print('Bye, bye!')
return 0
def main() -> int:
description = 'usage: %(prog)s [options] file'
description = """Scans a RP66V1 file and dumps data about the file to stdout.
This is useful for examining the details of RP66V1 files and can dump data at various levels of encapsulation,
from the lowest level upwards:
--VR ~ Visible Records only.
--LRSH ~ Logical Record segments.
--LD ~ Logical data i.e. all Logical Record segments concatenated for each Logical Record.
--EFLR ~ Explicitly Formatted Logical Records.
--IFLR ~ Implicitly Formatted Logical Records.
--LR ~ All data, including the numerical analysis of frame data.
If these are combined then the input is scanned multiple times.
"""
print('Cmd: %s' % ' '.join(sys.argv))
# TODO: Use cmn_cmd_opts
parser = argparse.ArgumentParser(
description=description,
epilog=__rights__,
prog=sys.argv[0],
)
parser.add_argument('path_in', type=str, help='Path to the input file or directory.')
parser.add_argument(
'path_out', type=str, default='', nargs='?', help='Path to the output scan to write [default: stdout].'
)
parser.add_argument(
'-V', '--VR', action='store_true',
help='Dump the Visible Records. [default: %(default)s]',
)
parser.add_argument(
'-L', '--LRSH', action='store_true',
help='Summarise the Visible Records and the Logical Record'
' Segment Headers, use -v to dump records. [default: %(default)s]',
)
parser.add_argument(
'-D', '--LD', action='store_true',
help='Summarise logical data, use -v to dump records.'
' See also --dump-bytes, --dump-raw-bytes. [default: %(default)s]',
)
parser.add_argument(
'-E', '--EFLR', action='store_true',
help='Dump EFLR Set. [default: %(default)s]',
)
parser.add_argument(
"--eflr-set-type", action='append', default=[],
help="List of EFLR Set Types to output, additive, if absent then dump all. [default: %(default)s]",
)
parser.add_argument(
'-I', '--IFLR', action='store_true',
help='Dump IFLRs. [default: %(default)s]',
)
parser.add_argument(
"--iflr-set-type", action='append', default=[],
help="List of IFLR Set Types to output, additive, if absent then dump all. [default: %(default)s]",
)
parser.add_argument(
'-R', '--LR', action='store_true',
help='Dump all data, including frame data from Logical Records. [default: %(default)s]',
)
parser.add_argument(
'-d', '--dump-bytes', type=int, default=0,
help='Dump X leading raw bytes for certain options, if -1 all bytes are dumped. [default: %(default)s]',
)
parser.add_argument(
'--dump-raw-bytes', action='store_true',
help='Dump the raw bytes for certain options in raw format,'
' otherwise Hex format is used. [default: %(default)s]',
)
parser.add_argument(
'-r', '--recurse', action='store_true',
help='Process files recursively. [default: %(default)s]',
)
parser.add_argument(
'-e', '--encrypted', action='store_true',
help='Output encrypted Logical Records as well. [default: %(default)s]',
)
parser.add_argument(
'-k', '--keep-going', action='store_true',
help='Keep going as far as sensible. [default: %(default)s]',
)
Slice.add_frame_slice_to_argument_parser(parser, help_prefix='NOTE: Requires -R, --LR.')
parser.add_argument(
'--eflr-as-table', action='store_true',
help='When with --LR and not --html then dump EFLRs as tables, otherwise every EFLR object.'
' [default: %(default)s]',
)
log_level_help_mapping = ', '.join(
['{:d}<->{:s}'.format(level, logging._levelToName[level]) for level in sorted(logging._levelToName.keys())]
)
log_level_help = f'Log Level as an integer or symbol. ({log_level_help_mapping}) [default: %(default)s]'
parser.add_argument(
"-l", "--log-level",
# type=int,
# dest="loglevel",
default=30,
help=log_level_help
)
parser.add_argument(
"-v", "--verbose", action='count', default=0,
help="Increase verbosity, additive [default: %(default)s]",
)
gnuplot.add_gnuplot_to_argument_parser(parser)
parser.add_argument(
'-T', '--test-data', action='store_true',
help='Dump the file as annotated bytes, useful for creating test data. [default: %(default)s]',
)
args = parser.parse_args()
print('args:', args)
# Extract log level
if args.log_level in logging._nameToLevel:
log_level = logging._nameToLevel[args.log_level]
else:
log_level = int(args.log_level)
# print('Log level:', log_level)
# Initialise logging etc.
logging.basicConfig(level=log_level,
format='%(asctime)s %(levelname)-8s %(message)s',
#datefmt='%y-%m-%d % %H:%M:%S',
stream=sys.stdout)
clk_start = time.perf_counter()
# return 0
# Your code here
result: typing.Dict[str, IndexResult] = {}
output_extension = '.txt'
if args.VR or args.LRSH:
result = scan_dir_or_file(
args.path_in,
args.path_out,
scan_RP66V1_file_visible_records,
args.recurse,
output_extension,
# kwargs passed to scanning function
lrsh_dump=args.LRSH,
verbose=args.verbose,
)
if args.LD:
result = scan_dir_or_file(
args.path_in,
args.path_out,
scan_RP66V1_file_logical_data,
args.recurse,
output_extension,
# kwargs passed to scanning function
dump_bytes=args.dump_bytes,
dump_raw_bytes=args.dump_raw_bytes,
verbose=args.verbose,
)
if args.EFLR or args.IFLR:
result = scan_dir_or_file(
args.path_in,
args.path_out,
scan_RP66V1_file_EFLR_IFLR,
args.recurse,
output_extension,
# kwargs passed to scanning function
verbose=args.verbose,
encrypted=args.encrypted,
keep_going=args.keep_going,
eflr_set_type=[bytes(v, 'ascii') for v in args.eflr_set_type],
iflr_set_type=[bytes(v, 'ascii') for v in args.iflr_set_type],
iflr_dump=args.IFLR,
eflr_dump=args.EFLR,
rp66v1_path=args.path_in,
)
if args.LR:
result = scan_dir_or_file(
args.path_in,
args.path_out,
scan_RP66V1_file_data_content,
args.recurse,
output_extension,
rp66v1_path=args.path_in,
frame_slice=Slice.create_slice_or_sample(args.frame_slice),
eflr_as_table=args.eflr_as_table,
)
if args.test_data:
result = scan_dir_or_file(
args.path_in,
args.path_out,
dump_RP66V1_test_data,
args.recurse,
output_extension,
verbose=args.verbose,
)
clk_exec = time.perf_counter() - clk_start
size_scan = size_input = 0
failures = 0
files_processed = 0
for path in sorted(result.keys()):
idx_result = result[path]
if idx_result.size_input > 0:
ms_mb = idx_result.time * 1000 / (idx_result.size_input / 1024 ** 2)
ratio = idx_result.size_output / idx_result.size_input
print(
f'{idx_result.size_input:16,d} {idx_result.size_output:10,d}'
f' {idx_result.time:8.3f} {ratio:8.3%} {ms_mb:8.1f} {str(idx_result.exception):5}'
f' "{path}"'
)
size_input += result[path].size_input
size_scan += result[path].size_output
files_processed += 1
if idx_result.exception:
failures += 1
if args.gnuplot:
plot_gnuplot(result, args.gnuplot)
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input / 1024**2)
else:
ms_mb = 0.0
print('Execution time = %8.3f (S)' % clk_exec)
print(f'Processed {len(result):,d} files and {size_input:,d} bytes, {ms_mb:.1f} ms/Mb')
print('Bye, bye!')
return failures