def main() -> int:
"""Main entry point."""
description = """usage: %(prog)s [options] file
Reads RP66V1 file(s) and writes them out as LAS files."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = cmn_cmd_opts.path_in_out(
description, prog='TotalDepth.RP66V1.ToLAS.main', version=__version__, epilog=__rights__
)
cmn_cmd_opts.add_log_level(parser, level=20)
cmn_cmd_opts.add_multiprocessing(parser)
Slice.add_frame_slice_to_argument_parser(parser, use_what=True)
process.add_process_logger_to_argument_parser(parser)
gnuplot.add_gnuplot_to_argument_parser(parser)
parser.add_argument(
'--array-reduction', type=str,
help='Method to reduce multidimensional channel data to a single value. [default: %(default)s]',
default='first',
choices=list(sorted(ARRAY_REDUCTIONS)),
)
parser.add_argument(
'--channels', type=str,
help='Comma separated list of channels to write out (X axis is always included).'
' Use \'?\' to see what channels exist without writing anything. [default: "%(default)s"]',
default='',
)
parser.add_argument('--field-width', type=int,
help='Field width for array data [default: %(default)s].', default=16)
parser.add_argument('--float-format', type=str,
help='Floating point format for array data [default: "%(default)s"].', default='.3f')
args = parser.parse_args()
cmn_cmd_opts.set_log_level(args)
# print('args:', args)
# return 0
# Your code here
clk_start = time.perf_counter()
ret_val = 0
result: typing.Dict[str, LASWriteResult] = {}
# if os.path.isfile(args.path_in) and (args.frame_slice.strip() == '?' or args.channels.strip() == '?'):
if args.frame_slice.strip() == '?' or args.channels.strip() == '?':
dump_frames_and_or_channels(args.path_in, args.recurse, args.frame_slice.strip(), args.channels.strip())
else:
channel_set = set()
for ch in args.channels.strip().split(','):
if ch.strip() != '':
channel_set.add(ch.strip())
if cmn_cmd_opts.multiprocessing_requested(args) and os.path.isdir(args.path_in):
result = convert_rp66v1_dir_or_file_to_las_multiprocessing(
args.path_in,
args.path_out,
args.recurse,
args.array_reduction,
Slice.create_slice_or_sample(args.frame_slice),
channel_set,
args.field_width,
args.float_format,
args.jobs,
)
else:
if args.log_process > 0.0:
with process.log_process(args.log_process):
result = convert_rp66v1_dir_or_file_to_las(
args.path_in,
args.path_out,
args.recurse,
args.array_reduction,
Slice.create_slice_or_sample(args.frame_slice),
channel_set,
args.field_width,
args.float_format,
)
else:
result = convert_rp66v1_dir_or_file_to_las(
args.path_in,
args.path_out,
args.recurse,
args.array_reduction,
Slice.create_slice_or_sample(args.frame_slice),
channel_set,
args.field_width,
args.float_format,
)
clk_exec = time.perf_counter() - clk_start
# Report output
if result:
size_index = size_input = 0
files_processed = 0
table = [
['Input', 'Output', 'LAS Count', 'Time', 'Ratio', 'ms/Mb', 'Exception', 'Path']
]
for path in sorted(result.keys()):
las_result = result[path]
# print('TRACE: las_result', las_result)
if las_result.size_input > 0:
ms_mb = las_result.time * 1000 / (las_result.size_input / 1024 ** 2)
ratio = las_result.size_output / las_result.size_input
out = [
f'{las_result.size_input:,d}',
f'{las_result.size_output:,d}',
f'{las_result.las_count:,d}',
f'{las_result.time:.3f}',
f'{ratio:.1%}',
f'{ms_mb:.1f}',
f'{str(las_result.exception)}',
f'"{path}"',
]
table.append(out)
# print(' '.join(out))
size_input += result[path].size_input
size_index += result[path].size_output
files_processed += 1
if las_result.exception:
ret_val = 1
for row in data_table.format_table(table, pad=' ', heading_underline='-'):
print(row)
try:
if args.gnuplot:
plot_gnuplot(result, args.gnuplot)
except Exception as err: # pragma: no cover
logger.exception(str(err))
ret_val = 2
print('Execution time = %8.3f (S)' % clk_exec)
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input/ 1024**2)
ratio = size_index / size_input
else:
ms_mb = 0.0
ratio = 0.0
print(f'Out of {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes')
print(f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {ms_mb:.1f} ms/Mb')
else:
print(f'Execution time: {clk_exec:.3f} (s)')
print('Bye, bye!')
return ret_val
def main() -> int:
"""Main entry point."""
description = """usage: %(prog)s [options] file
Scans a RP66V1 file or directory and indexes the files with a LogicalRecordIndex."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = cmn_cmd_opts.path_in_out(
description,
prog='TotalDepth.RP66V1.LogRecIndex.main',
version=__version__,
epilog=__rights__)
cmn_cmd_opts.add_log_level(parser, level=20)
cmn_cmd_opts.add_multiprocessing(parser)
process.add_process_logger_to_argument_parser(parser)
gnuplot.add_gnuplot_to_argument_parser(parser)
parser.add_argument(
'--read-back',
action='store_true',
help=
'Read the pickled index and time it (requires path_out). [default: %(default)s]',
)
parser.add_argument(
'--validate',
action='store_true',
help='Perform validation checks on the index.. [default: %(default)s]',
)
args = parser.parse_args()
# print('args:', args)
# return 0
cmn_cmd_opts.set_log_level(args)
# Your code here
exec_timer = ExecTimer.Timer('LogRecIndex')
if os.path.isdir(
args.path_in) and cmn_cmd_opts.multiprocessing_requested(args):
result: typing.Dict[str, IndexResult] = index_dir_multiprocessing(
args.path_in,
args.path_out,
args.jobs,
args.recurse,
args.read_back,
args.validate,
)
else:
if args.log_process > 0.0:
with process.log_process(args.log_process):
result: typing.Dict[str, IndexResult] = index_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
args.read_back,
args.validate,
)
else:
result: typing.Dict[str, IndexResult] = index_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
args.read_back,
args.validate,
)
size_index = size_input = 0
files_processed = 0
if os.path.isdir(args.path_in):
len_path = len(args.path_in)
if not args.path_in.endswith(os.sep):
len_path += 1
else:
len_path = 0
ret_value = 0
try:
header = (
f'{"Size In":>16}',
f'{"Size Out":>16}',
f'{"Time (ms)":>10}',
f'{"Ratio %":>8}',
f'{"ms/Mb":>8}',
f'{"Fail?":5}',
f'{"Write (ms)":>10}',
f'{"Read (ms)":>10}',
f'Path',
)
print(' '.join(header))
print(' '.join('-' * len(v) for v in header))
for path in sorted(result.keys()):
idx_result = result[path]
if idx_result.size_input > 0:
ms_mb = idx_result.index_time * 1000 / (idx_result.size_input /
1024**2)
ratio = idx_result.size_index / idx_result.size_input
print(
f'{idx_result.size_input:16,d} {idx_result.size_index:16,d}'
f' {1000*idx_result.index_time:10.1f} {ratio:8.3%} {ms_mb:8.1f} {str(idx_result.exception):5}'
f' {1000*idx_result.time_write:10.1f} {1000*idx_result.time_read_back:10.1f}'
f' "{path[len_path:]}"')
size_input += result[path].size_input
size_index += result[path].size_index
files_processed += 1
exec_timer.add_work_done(idx_result.size_input)
if args.gnuplot:
try:
plot_gnuplot(result, args.gnuplot)
except Exception: # pragma: no cover
logger.exception('gunplot failed')
ret_value = 1
except Exception as err: # pragma: no cover
logger.exception(str(err))
ret_value = 2
print(f'Number of jobs: {args.jobs}')
print('Execution:')
print(exec_timer.long_str)
if size_input > 0:
ratio = size_index / size_input
else:
ratio = 0.0
print(
f'Out of {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes'
)
print(
f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {exec_timer.ms_mb:.1f} ms/Mb'
)
print(f'Processed: {size_input:,d} bytes at {exec_timer.ms_mb:.1f} ms/Mb')
print('Bye, bye!')
return ret_value
def main() -> int:
description = """usage: %(prog)s [options] file
Scans a RP66V1 file or directory and writes out the index(es) in XML."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = cmn_cmd_opts.path_in_out(description,
prog='TotalDepth.RP66V1.IndexXML.main',
version=__version__,
epilog=__rights__)
cmn_cmd_opts.add_log_level(parser, level=20)
cmn_cmd_opts.add_multiprocessing(parser)
parser.add_argument(
'-e',
'--encrypted',
action='store_true',
help='Output encrypted Logical Records as well. [default: %(default)s]',
)
process.add_process_logger_to_argument_parser(parser)
gnuplot.add_gnuplot_to_argument_parser(parser)
parser.add_argument(
'-p',
'--private',
action='store_true',
help='Also write out private EFLRs. [default: %(default)s]',
)
args = parser.parse_args()
# print('args:', args)
# return 0
cmn_cmd_opts.set_log_level(args)
# Your code here
clk_start = time.perf_counter()
ret_val = 0
if os.path.isdir(
args.path_in) and cmn_cmd_opts.multiprocessing_requested(args):
result: typing.Dict[str, IndexResult] = index_dir_multiprocessing(
args.path_in,
args.path_out,
args.private,
args.jobs,
)
else:
if args.log_process > 0.0:
with process.log_process(args.log_process):
result: typing.Dict[str, IndexResult] = index_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
args.private,
)
else:
result: typing.Dict[str, IndexResult] = index_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
args.private,
)
clk_exec = time.perf_counter() - clk_start
size_index = size_input = 0
files_processed = 0
try:
header = (
f'{"Size In":>16}',
f'{"Size Out":>16}',
f'{"Time":>8}',
f'{"Ratio %":>8}',
f'{"ms/Mb":>8}',
f'{"Fail?":5}',
f'Path',
)
print(' '.join(header))
print(' '.join('-' * len(v) for v in header))
for path in sorted(result.keys()):
idx_result = result[path]
if idx_result.size_input > 0:
ms_mb = idx_result.time * 1000 / (idx_result.size_input /
1024**2)
ratio = idx_result.size_index / idx_result.size_input
print(
f'{idx_result.size_input:16,d} {idx_result.size_index:16,d}'
f' {idx_result.time:8.3f} {ratio:8.3%} {ms_mb:8.1f} {str(idx_result.exception):5}'
f' "{path}"')
size_input += result[path].size_input
size_index += result[path].size_index
files_processed += 1
if args.gnuplot:
try:
plot_gnuplot(result, args.gnuplot)
except Exception: # pragma: no cover
logger.exception('gunplot failed')
ret_val = 1
except Exception as err: # pragma: no cover
logger.exception(str(err))
ret_val = 2
print('Execution time = %8.3f (S)' % clk_exec)
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input / 1024**2)
ratio = size_index / size_input
else: # pragma: no cover
ms_mb = 0.0
ratio = 0.0
print(
f'Out of {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes'
)
print(
f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {ms_mb:.1f} ms/Mb'
)
print('Bye, bye!')
return ret_val
def main() -> int:
description = """usage: %(prog)s [options] file
Scans a RP66V1 file or directory and saves the index as a pickled file."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = cmn_cmd_opts.path_in_out(
description, prog='TotalDepth.RP66V1.IndexPickle.main', version=__version__, epilog=__rights__
)
cmn_cmd_opts.add_log_level(parser, level=20)
cmn_cmd_opts.add_multiprocessing(parser)
parser.add_argument('--read-back', action='store_true', help='Read and time the output. [default: %(default)s]')
process.add_process_logger_to_argument_parser(parser)
gnuplot.add_gnuplot_to_argument_parser(parser)
args = parser.parse_args()
# print('args:', args)
# return 0
cmn_cmd_opts.set_log_level(args)
# Your code here
clk_start = time.perf_counter()
ret_val = 0
if cmn_cmd_opts.multiprocessing_requested(args) and os.path.isdir(args.path_in):
result: typing.Dict[str, IndexResult] = index_dir_multiprocessing(
args.path_in,
args.path_out,
args.jobs,
args.recurse,
args.read_back,
)
else:
if args.log_process > 0.0:
with process.log_process(args.log_process):
result: typing.Dict[str, IndexResult] = index_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
args.read_back,
)
else:
result: typing.Dict[str, IndexResult] = index_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
args.read_back,
)
clk_exec = time.perf_counter() - clk_start
size_index = size_input = 0
files_processed = 0
try:
path_prefix = os.path.commonpath(result.keys())
len_path_prefix = len(path_prefix)
table: typing.List[typing.List[str]] = [
[
'Size (b)', 'Index (b)', 'Ratio (%)',
'Index (s)', 'Index (ms/Mb)',
'Write (s)', 'Write (ms/Mb)',
'Read (s)', 'Read (ms/Mb)',
'Except',
'Path',
]
]
for path in sorted(result.keys()):
idx_result = result[path]
if not idx_result.ignored and idx_result.size_input > 0:
ms_mb_index = idx_result.time_index * 1000 / (idx_result.size_input / 1024 ** 2)
ms_mb_write = idx_result.time_write * 1000 / (idx_result.size_input / 1024 ** 2)
ms_mb_read = idx_result.time_read * 1000 / (idx_result.size_input / 1024 ** 2)
ratio = idx_result.size_index / idx_result.size_input
table.append(
[
f'{idx_result.size_input:,d}', f'{idx_result.size_index:,d}', f'{ratio:.3%}',
f'{idx_result.time_index:.3f}', f'{ms_mb_index:.1f}',
f'{idx_result.time_write:.3f}', f'{ms_mb_write:.1f}',
f'{idx_result.time_read:.3f}', f'{ms_mb_read:.2f}',
f'{str(idx_result.exception):5}',
f'{path[len_path_prefix+1:]}',
]
)
size_input += result[path].size_input
size_index += result[path].size_index
files_processed += 1
if idx_result.exception: # pragma: no cover
ret_val = 1
print(f'Common path prefix: {path_prefix}')
print('\n'.join(data_table.format_table(table, pad=' | ', heading_underline='-')))
if args.gnuplot:
try:
plot_gnuplot(result, args.gnuplot)
except IOError: # pragma: no cover
logger.exception('Plotting with gnuplot failed.')
ret_val = 2
except Exception as err: # pragma: no cover
logger.exception(str(err))
ret_val = 3
print('Execution time = %8.3f (S)' % clk_exec)
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input/ 1024**2)
ratio = size_index / size_input
else: # pragma: no cover
ms_mb = 0.0
ratio = 0.0
print(f'Out of {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes')
print(f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {ms_mb:.1f} ms/Mb')
print('Bye, bye!')
return ret_val
def main() -> int:
description = """Scans a RP66V1 file or directory and writes HTML version of the data."""
print('Cmd: %s' % ' '.join(sys.argv))
parser = cmn_cmd_opts.path_in_out(description,
prog='TotalDepth.RP66V1.ScanHTML.main',
version=__version__,
epilog=__rights__)
cmn_cmd_opts.add_log_level(parser, level=20)
cmn_cmd_opts.add_multiprocessing(parser)
parser.add_argument(
'-e',
'--encrypted',
action='store_true',
help='Output encrypted Logical Records as well. [default: %(default)s]',
)
Slice.add_frame_slice_to_argument_parser(parser)
process.add_process_logger_to_argument_parser(parser)
gnuplot.add_gnuplot_to_argument_parser(parser)
args = parser.parse_args()
cmn_cmd_opts.set_log_level(args)
# print('args:', args)
# return 0
clk_start = time.perf_counter()
# Your code here
if args.log_process > 0.0:
with process.log_process(args.log_process):
result: typing.Dict[str, HTMLResult] = scan_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
label_process=True,
frame_slice=Slice.create_slice_or_sample(args.frame_slice),
)
else:
if cmn_cmd_opts.multiprocessing_requested(args) and os.path.isdir(
args.path_in):
result: typing.Dict[str, HTMLResult] = scan_dir_multiprocessing(
args.path_in,
args.path_out,
args.jobs,
frame_slice=Slice.create_slice_or_sample(args.frame_slice),
)
else:
result: typing.Dict[str, HTMLResult] = scan_dir_or_file(
args.path_in,
args.path_out,
args.recurse,
label_process=False,
frame_slice=Slice.create_slice_or_sample(args.frame_slice),
)
if args.log_process > 0.0:
process.add_message_to_queue('Processing HTML Complete.')
clk_exec = time.perf_counter() - clk_start
# print('Execution time = %8.3f (S)' % clk_exec)
size_scan = size_input = 0
files_processed = 0
header = (
f'{"Size In":>16}',
f'{"Size Out":>10}',
f'{"Time":>8}',
f'{"Ratio %":>8}',
f'{"ms/Mb":>8}',
f'{"Fail?":5}',
f'Path',
)
print(' '.join(header))
print(' '.join('-' * len(v) for v in header))
for path in sorted(result.keys()):
idx_result = result[path]
if idx_result.size_input > 0:
ms_mb = idx_result.time * 1000 / (idx_result.size_input / 1024**2)
ratio = idx_result.size_output / idx_result.size_input
print(
f'{idx_result.size_input:16,d} {idx_result.size_output:10,d}'
f' {idx_result.time:8.3f} {ratio:8.3%} {ms_mb:8.1f} {str(idx_result.exception):5}'
f' "{path}"')
size_input += result[path].size_input
size_scan += result[path].size_output
files_processed += 1
if args.gnuplot:
try:
plot_gnuplot(result, args.gnuplot)
except IOError:
logger.exception('Plotting with gnuplot failed.')
if size_input > 0:
ms_mb = clk_exec * 1000 / (size_input / 1024**2)
else:
ms_mb = 0.0
print(
f'Processed {len(result):,d} files and {size_input:,d} bytes in {clk_exec:.3f} s, {ms_mb:.1f} ms/Mb'
)
print('Bye, bye!')
return 0
def main():
print('Cmd: %s' % ' '.join(sys.argv))
# TODO: Option to treat files with -f, --format as LAS, LIS, AUTO
# TODO: Depth scale overrides -s, --scale ?
parser = cmn_cmd_opts.path_in_out(
'Generates SVG plot(s) from input LIS & LAS file/directory to an output file/directory.',
prog=None,
version=__version__,
)
cmn_cmd_opts.add_log_level(parser)
cmn_cmd_opts.add_multiprocessing(parser)
parser.add_argument(
"-A",
"--API",
action="store_true",
dest="apiHeader",
default=False,
help="Put an API header on each plot. [default: False]")
parser.add_argument("-x", "--xml", action="append", dest="LgFormat", default=[],
help="Use XML LgFormat UniqueId to use for plotting (additive)." \
+" Use -x? to see what LgFormats (UniqueID+Description) are available." \
+" Use -x?? to see what curves each format can plot. See also -X. [default: []]")
parser.add_argument(
"-X",
"--XML",
type=int,
dest="LgFormat_min",
default=0,
help=
"Use any LgFormat XML plots that use n or more outputs. If -x option present limited by those LgFormats [default: 0]"
)
parser.add_argument("-g",
"--glob",
action="store_true",
dest="glob",
default=None,
help="File match pattern. Default: %(default)s.")
# parser.add_argument("-f", "--file-type", choices=['LAS', 'LIS', 'AUTO'],
# help="File format to assume for the input, AUTO will do it's best. [default: \"AUTO\"].")
parser.add_argument(
"-s",
"--scale",
action="append",
type=int,
dest="scale",
default=0,
help="Scale of X axis to use (an integer). [default: 0].")
args = parser.parse_args()
# Initialise logging etc.
cmn_cmd_opts.set_log_level(args)
# print('args', args)
# return 0
start_clock = time.clock()
start_time = time.time()
# Your code here
if '?' in ''.join(args.LgFormat):
# Handle -x? here and exit
myFg = FILMCfgXML.FilmCfgXMLRead()
print('XML LgFormats available: [{:d}]'.format(len(myFg.keys())))
print(myFg.longStr(''.join(args.LgFormat).count('?')))
return 1
if cmn_cmd_opts.multiprocessing_requested(args):
myPlp = PlotLogPasses(
args.path_in,
args.path_out,
args,
)
myResult = myPlp.plotLogInfo
else:
myResult = plotLogPassesMP(
args.path_in,
args.path_out,
args,
)
if os.path.isdir(args.path_out):
myResult.writeHTML(os.path.join(args.path_out, 'index.html'),
args.path_in)
print('plotLogInfo', str(myResult))
print(' CPU time = %8.3f (S)' % (time.clock() - start_clock))
print('Exec. time = %8.3f (S)' % (time.time() - start_time))
print('Bye, bye!')
return 0