コード例 #1
0
def main() -> int:
    """Main entry point."""
    description = """usage: %(prog)s [options] file
Reads RP66V1 file(s) and writes them out as LAS files."""
    print('Cmd: %s' % ' '.join(sys.argv))

    parser = cmn_cmd_opts.path_in_out(
        description, prog='TotalDepth.RP66V1.ToLAS.main', version=__version__, epilog=__rights__
    )
    cmn_cmd_opts.add_log_level(parser, level=20)
    cmn_cmd_opts.add_multiprocessing(parser)
    Slice.add_frame_slice_to_argument_parser(parser, use_what=True)
    process.add_process_logger_to_argument_parser(parser)
    gnuplot.add_gnuplot_to_argument_parser(parser)
    parser.add_argument(
        '--array-reduction', type=str,
        help='Method to reduce multidimensional channel data to a single value. [default: %(default)s]',
        default='first',
        choices=list(sorted(ARRAY_REDUCTIONS)),
        )
    parser.add_argument(
        '--channels', type=str,
        help='Comma separated list of channels to write out (X axis is always included).'
             ' Use \'?\' to see what channels exist without writing anything. [default: "%(default)s"]',
        default='',
        )
    parser.add_argument('--field-width', type=int,
                        help='Field width for array data [default: %(default)s].', default=16)
    parser.add_argument('--float-format', type=str,
                        help='Floating point format for array data [default: "%(default)s"].', default='.3f')
    args = parser.parse_args()
    cmn_cmd_opts.set_log_level(args)
    # print('args:', args)
    # return 0
    # Your code here
    clk_start = time.perf_counter()
    ret_val = 0
    result: typing.Dict[str, LASWriteResult] = {}
    # if os.path.isfile(args.path_in) and (args.frame_slice.strip() == '?' or args.channels.strip() == '?'):
    if args.frame_slice.strip() == '?' or args.channels.strip() == '?':
        dump_frames_and_or_channels(args.path_in, args.recurse, args.frame_slice.strip(), args.channels.strip())
    else:
        channel_set = set()
        for ch in args.channels.strip().split(','):
            if ch.strip() != '':
                channel_set.add(ch.strip())
        if cmn_cmd_opts.multiprocessing_requested(args) and os.path.isdir(args.path_in):
            result = convert_rp66v1_dir_or_file_to_las_multiprocessing(
                args.path_in,
                args.path_out,
                args.recurse,
                args.array_reduction,
                Slice.create_slice_or_sample(args.frame_slice),
                channel_set,
                args.field_width,
                args.float_format,
                args.jobs,
            )
        else:
            if args.log_process > 0.0:
                with process.log_process(args.log_process):
                    result = convert_rp66v1_dir_or_file_to_las(
                        args.path_in,
                        args.path_out,
                        args.recurse,
                        args.array_reduction,
                        Slice.create_slice_or_sample(args.frame_slice),
                        channel_set,
                        args.field_width,
                        args.float_format,
                    )
            else:
                result = convert_rp66v1_dir_or_file_to_las(
                    args.path_in,
                    args.path_out,
                    args.recurse,
                    args.array_reduction,
                    Slice.create_slice_or_sample(args.frame_slice),
                    channel_set,
                    args.field_width,
                    args.float_format,
                )
    clk_exec = time.perf_counter() - clk_start
    # Report output
    if result:
        size_index = size_input = 0
        files_processed = 0
        table = [
            ['Input', 'Output', 'LAS Count', 'Time', 'Ratio', 'ms/Mb', 'Exception', 'Path']
        ]
        for path in sorted(result.keys()):
            las_result = result[path]
            # print('TRACE: las_result', las_result)
            if las_result.size_input > 0:
                ms_mb = las_result.time * 1000 / (las_result.size_input / 1024 ** 2)
                ratio = las_result.size_output / las_result.size_input
                out = [
                    f'{las_result.size_input:,d}',
                    f'{las_result.size_output:,d}',
                    f'{las_result.las_count:,d}',
                    f'{las_result.time:.3f}',
                    f'{ratio:.1%}',
                    f'{ms_mb:.1f}',
                    f'{str(las_result.exception)}',
                    f'"{path}"',
                ]
                table.append(out)
                # print(' '.join(out))
                size_input += result[path].size_input
                size_index += result[path].size_output
                files_processed += 1
                if las_result.exception:
                    ret_val = 1
        for row in data_table.format_table(table, pad=' ', heading_underline='-'):
            print(row)
        try:
            if args.gnuplot:
                plot_gnuplot(result, args.gnuplot)
        except Exception as err:  # pragma: no cover
            logger.exception(str(err))
            ret_val = 2
        print('Execution time = %8.3f (S)' % clk_exec)
        if size_input > 0:
            ms_mb = clk_exec * 1000 / (size_input/ 1024**2)
            ratio = size_index / size_input
        else:
            ms_mb = 0.0
            ratio = 0.0
        print(f'Out of  {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes')
        print(f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {ms_mb:.1f} ms/Mb')
    else:
        print(f'Execution time: {clk_exec:.3f} (s)')
    print('Bye, bye!')
    return ret_val
コード例 #2
0
ファイル: LogRecIndex.py プロジェクト: ojsindher/TotalDepth
def main() -> int:
    """Main entry point."""
    description = """usage: %(prog)s [options] file
    Scans a RP66V1 file or directory and indexes the files with a LogicalRecordIndex."""
    print('Cmd: %s' % ' '.join(sys.argv))
    parser = cmn_cmd_opts.path_in_out(
        description,
        prog='TotalDepth.RP66V1.LogRecIndex.main',
        version=__version__,
        epilog=__rights__)
    cmn_cmd_opts.add_log_level(parser, level=20)
    cmn_cmd_opts.add_multiprocessing(parser)
    process.add_process_logger_to_argument_parser(parser)
    gnuplot.add_gnuplot_to_argument_parser(parser)
    parser.add_argument(
        '--read-back',
        action='store_true',
        help=
        'Read the pickled index and time it (requires path_out). [default: %(default)s]',
    )
    parser.add_argument(
        '--validate',
        action='store_true',
        help='Perform validation checks on the index.. [default: %(default)s]',
    )
    args = parser.parse_args()
    # print('args:', args)
    # return 0
    cmn_cmd_opts.set_log_level(args)
    # Your code here
    exec_timer = ExecTimer.Timer('LogRecIndex')
    if os.path.isdir(
            args.path_in) and cmn_cmd_opts.multiprocessing_requested(args):
        result: typing.Dict[str, IndexResult] = index_dir_multiprocessing(
            args.path_in,
            args.path_out,
            args.jobs,
            args.recurse,
            args.read_back,
            args.validate,
        )
    else:
        if args.log_process > 0.0:
            with process.log_process(args.log_process):
                result: typing.Dict[str, IndexResult] = index_dir_or_file(
                    args.path_in,
                    args.path_out,
                    args.recurse,
                    args.read_back,
                    args.validate,
                )
        else:
            result: typing.Dict[str, IndexResult] = index_dir_or_file(
                args.path_in,
                args.path_out,
                args.recurse,
                args.read_back,
                args.validate,
            )
    size_index = size_input = 0
    files_processed = 0
    if os.path.isdir(args.path_in):
        len_path = len(args.path_in)
        if not args.path_in.endswith(os.sep):
            len_path += 1
    else:
        len_path = 0
    ret_value = 0
    try:
        header = (
            f'{"Size In":>16}',
            f'{"Size Out":>16}',
            f'{"Time (ms)":>10}',
            f'{"Ratio %":>8}',
            f'{"ms/Mb":>8}',
            f'{"Fail?":5}',
            f'{"Write (ms)":>10}',
            f'{"Read (ms)":>10}',
            f'Path',
        )
        print(' '.join(header))
        print(' '.join('-' * len(v) for v in header))
        for path in sorted(result.keys()):
            idx_result = result[path]
            if idx_result.size_input > 0:
                ms_mb = idx_result.index_time * 1000 / (idx_result.size_input /
                                                        1024**2)
                ratio = idx_result.size_index / idx_result.size_input
                print(
                    f'{idx_result.size_input:16,d} {idx_result.size_index:16,d}'
                    f' {1000*idx_result.index_time:10.1f} {ratio:8.3%} {ms_mb:8.1f} {str(idx_result.exception):5}'
                    f' {1000*idx_result.time_write:10.1f} {1000*idx_result.time_read_back:10.1f}'
                    f' "{path[len_path:]}"')
                size_input += result[path].size_input
                size_index += result[path].size_index
                files_processed += 1
                exec_timer.add_work_done(idx_result.size_input)
        if args.gnuplot:
            try:
                plot_gnuplot(result, args.gnuplot)
            except Exception:  # pragma: no cover
                logger.exception('gunplot failed')
                ret_value = 1
    except Exception as err:  # pragma: no cover
        logger.exception(str(err))
        ret_value = 2
    print(f'Number of jobs: {args.jobs}')
    print('Execution:')
    print(exec_timer.long_str)
    if size_input > 0:
        ratio = size_index / size_input
    else:
        ratio = 0.0
    print(
        f'Out of  {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes'
    )
    print(
        f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {exec_timer.ms_mb:.1f} ms/Mb'
    )
    print(f'Processed: {size_input:,d} bytes at {exec_timer.ms_mb:.1f} ms/Mb')
    print('Bye, bye!')
    return ret_value
コード例 #3
0
def main() -> int:
    description = """usage: %(prog)s [options] file
    Scans a RP66V1 file or directory and writes out the index(es) in XML."""
    print('Cmd: %s' % ' '.join(sys.argv))
    parser = cmn_cmd_opts.path_in_out(description,
                                      prog='TotalDepth.RP66V1.IndexXML.main',
                                      version=__version__,
                                      epilog=__rights__)
    cmn_cmd_opts.add_log_level(parser, level=20)
    cmn_cmd_opts.add_multiprocessing(parser)
    parser.add_argument(
        '-e',
        '--encrypted',
        action='store_true',
        help='Output encrypted Logical Records as well. [default: %(default)s]',
    )
    process.add_process_logger_to_argument_parser(parser)
    gnuplot.add_gnuplot_to_argument_parser(parser)
    parser.add_argument(
        '-p',
        '--private',
        action='store_true',
        help='Also write out private EFLRs. [default: %(default)s]',
    )
    args = parser.parse_args()
    # print('args:', args)
    # return 0
    cmn_cmd_opts.set_log_level(args)
    # Your code here
    clk_start = time.perf_counter()
    ret_val = 0
    if os.path.isdir(
            args.path_in) and cmn_cmd_opts.multiprocessing_requested(args):
        result: typing.Dict[str, IndexResult] = index_dir_multiprocessing(
            args.path_in,
            args.path_out,
            args.private,
            args.jobs,
        )
    else:
        if args.log_process > 0.0:
            with process.log_process(args.log_process):
                result: typing.Dict[str, IndexResult] = index_dir_or_file(
                    args.path_in,
                    args.path_out,
                    args.recurse,
                    args.private,
                )
        else:
            result: typing.Dict[str, IndexResult] = index_dir_or_file(
                args.path_in,
                args.path_out,
                args.recurse,
                args.private,
            )
    clk_exec = time.perf_counter() - clk_start
    size_index = size_input = 0
    files_processed = 0
    try:
        header = (
            f'{"Size In":>16}',
            f'{"Size Out":>16}',
            f'{"Time":>8}',
            f'{"Ratio %":>8}',
            f'{"ms/Mb":>8}',
            f'{"Fail?":5}',
            f'Path',
        )
        print(' '.join(header))
        print(' '.join('-' * len(v) for v in header))
        for path in sorted(result.keys()):
            idx_result = result[path]
            if idx_result.size_input > 0:
                ms_mb = idx_result.time * 1000 / (idx_result.size_input /
                                                  1024**2)
                ratio = idx_result.size_index / idx_result.size_input
                print(
                    f'{idx_result.size_input:16,d} {idx_result.size_index:16,d}'
                    f' {idx_result.time:8.3f} {ratio:8.3%} {ms_mb:8.1f} {str(idx_result.exception):5}'
                    f' "{path}"')
                size_input += result[path].size_input
                size_index += result[path].size_index
                files_processed += 1
        if args.gnuplot:
            try:
                plot_gnuplot(result, args.gnuplot)
            except Exception:  # pragma: no cover
                logger.exception('gunplot failed')
                ret_val = 1
    except Exception as err:  # pragma: no cover
        logger.exception(str(err))
        ret_val = 2
    print('Execution time = %8.3f (S)' % clk_exec)
    if size_input > 0:
        ms_mb = clk_exec * 1000 / (size_input / 1024**2)
        ratio = size_index / size_input
    else:  # pragma: no cover
        ms_mb = 0.0
        ratio = 0.0
    print(
        f'Out of  {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes'
    )
    print(
        f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {ms_mb:.1f} ms/Mb'
    )
    print('Bye, bye!')
    return ret_val
コード例 #4
0
ファイル: IndexPickle.py プロジェクト: ojsindher/TotalDepth
def main() -> int:
    description = """usage: %(prog)s [options] file
Scans a RP66V1 file or directory and saves the index as a pickled file."""
    print('Cmd: %s' % ' '.join(sys.argv))
    parser = cmn_cmd_opts.path_in_out(
        description, prog='TotalDepth.RP66V1.IndexPickle.main', version=__version__, epilog=__rights__
    )
    cmn_cmd_opts.add_log_level(parser, level=20)
    cmn_cmd_opts.add_multiprocessing(parser)
    parser.add_argument('--read-back', action='store_true', help='Read and time the output. [default: %(default)s]')
    process.add_process_logger_to_argument_parser(parser)
    gnuplot.add_gnuplot_to_argument_parser(parser)
    args = parser.parse_args()
    # print('args:', args)
    # return 0
    cmn_cmd_opts.set_log_level(args)
    # Your code here
    clk_start = time.perf_counter()
    ret_val = 0
    if cmn_cmd_opts.multiprocessing_requested(args) and os.path.isdir(args.path_in):
        result: typing.Dict[str, IndexResult] = index_dir_multiprocessing(
            args.path_in,
            args.path_out,
            args.jobs,
            args.recurse,
            args.read_back,
        )
    else:
        if args.log_process > 0.0:
            with process.log_process(args.log_process):
                result: typing.Dict[str, IndexResult] = index_dir_or_file(
                    args.path_in,
                    args.path_out,
                    args.recurse,
                    args.read_back,
                )
        else:
            result: typing.Dict[str, IndexResult] = index_dir_or_file(
                args.path_in,
                args.path_out,
                args.recurse,
                args.read_back,
            )
    clk_exec = time.perf_counter() - clk_start
    size_index = size_input = 0
    files_processed = 0
    try:
        path_prefix = os.path.commonpath(result.keys())
        len_path_prefix = len(path_prefix)
        table: typing.List[typing.List[str]] = [
            [
                'Size (b)', 'Index (b)', 'Ratio (%)',
                'Index (s)', 'Index (ms/Mb)',
                'Write (s)', 'Write (ms/Mb)',
                'Read (s)', 'Read (ms/Mb)',
                'Except',
                'Path',
            ]
        ]
        for path in sorted(result.keys()):
            idx_result = result[path]
            if not idx_result.ignored and idx_result.size_input > 0:
                ms_mb_index = idx_result.time_index * 1000 / (idx_result.size_input / 1024 ** 2)
                ms_mb_write = idx_result.time_write * 1000 / (idx_result.size_input / 1024 ** 2)
                ms_mb_read = idx_result.time_read * 1000 / (idx_result.size_input / 1024 ** 2)
                ratio = idx_result.size_index / idx_result.size_input
                table.append(
                    [
                        f'{idx_result.size_input:,d}', f'{idx_result.size_index:,d}', f'{ratio:.3%}',
                        f'{idx_result.time_index:.3f}', f'{ms_mb_index:.1f}',
                        f'{idx_result.time_write:.3f}', f'{ms_mb_write:.1f}',
                        f'{idx_result.time_read:.3f}', f'{ms_mb_read:.2f}',
                        f'{str(idx_result.exception):5}',
                        f'{path[len_path_prefix+1:]}',
                    ]
                )
                size_input += result[path].size_input
                size_index += result[path].size_index
                files_processed += 1
                if idx_result.exception:  # pragma: no cover
                    ret_val = 1
        print(f'Common path prefix: {path_prefix}')
        print('\n'.join(data_table.format_table(table, pad=' | ', heading_underline='-')))
        if args.gnuplot:
            try:
                plot_gnuplot(result, args.gnuplot)
            except IOError:  # pragma: no cover
                logger.exception('Plotting with gnuplot failed.')
                ret_val = 2
    except Exception as err:  # pragma: no cover
        logger.exception(str(err))
        ret_val = 3
    print('Execution time = %8.3f (S)' % clk_exec)
    if size_input > 0:
        ms_mb = clk_exec * 1000 / (size_input/ 1024**2)
        ratio = size_index / size_input
    else:  # pragma: no cover
        ms_mb = 0.0
        ratio = 0.0
    print(f'Out of  {len(result):,d} processed {files_processed:,d} files of total size {size_input:,d} input bytes')
    print(f'Wrote {size_index:,d} output bytes, ratio: {ratio:8.3%} at {ms_mb:.1f} ms/Mb')
    print('Bye, bye!')
    return ret_val
コード例 #5
0
ファイル: ScanHTML.py プロジェクト: ojsindher/TotalDepth
def main() -> int:
    description = """Scans a RP66V1 file or directory and writes HTML version of the data."""
    print('Cmd: %s' % ' '.join(sys.argv))
    parser = cmn_cmd_opts.path_in_out(description,
                                      prog='TotalDepth.RP66V1.ScanHTML.main',
                                      version=__version__,
                                      epilog=__rights__)
    cmn_cmd_opts.add_log_level(parser, level=20)
    cmn_cmd_opts.add_multiprocessing(parser)
    parser.add_argument(
        '-e',
        '--encrypted',
        action='store_true',
        help='Output encrypted Logical Records as well. [default: %(default)s]',
    )
    Slice.add_frame_slice_to_argument_parser(parser)
    process.add_process_logger_to_argument_parser(parser)
    gnuplot.add_gnuplot_to_argument_parser(parser)
    args = parser.parse_args()
    cmn_cmd_opts.set_log_level(args)
    # print('args:', args)
    # return 0
    clk_start = time.perf_counter()
    # Your code here
    if args.log_process > 0.0:
        with process.log_process(args.log_process):
            result: typing.Dict[str, HTMLResult] = scan_dir_or_file(
                args.path_in,
                args.path_out,
                args.recurse,
                label_process=True,
                frame_slice=Slice.create_slice_or_sample(args.frame_slice),
            )
    else:
        if cmn_cmd_opts.multiprocessing_requested(args) and os.path.isdir(
                args.path_in):
            result: typing.Dict[str, HTMLResult] = scan_dir_multiprocessing(
                args.path_in,
                args.path_out,
                args.jobs,
                frame_slice=Slice.create_slice_or_sample(args.frame_slice),
            )
        else:
            result: typing.Dict[str, HTMLResult] = scan_dir_or_file(
                args.path_in,
                args.path_out,
                args.recurse,
                label_process=False,
                frame_slice=Slice.create_slice_or_sample(args.frame_slice),
            )
    if args.log_process > 0.0:
        process.add_message_to_queue('Processing HTML Complete.')
    clk_exec = time.perf_counter() - clk_start
    # print('Execution time = %8.3f (S)' % clk_exec)
    size_scan = size_input = 0
    files_processed = 0
    header = (
        f'{"Size In":>16}',
        f'{"Size Out":>10}',
        f'{"Time":>8}',
        f'{"Ratio %":>8}',
        f'{"ms/Mb":>8}',
        f'{"Fail?":5}',
        f'Path',
    )
    print(' '.join(header))
    print(' '.join('-' * len(v) for v in header))
    for path in sorted(result.keys()):
        idx_result = result[path]
        if idx_result.size_input > 0:
            ms_mb = idx_result.time * 1000 / (idx_result.size_input / 1024**2)
            ratio = idx_result.size_output / idx_result.size_input
            print(
                f'{idx_result.size_input:16,d} {idx_result.size_output:10,d}'
                f' {idx_result.time:8.3f} {ratio:8.3%} {ms_mb:8.1f} {str(idx_result.exception):5}'
                f' "{path}"')
            size_input += result[path].size_input
            size_scan += result[path].size_output
            files_processed += 1

    if args.gnuplot:
        try:
            plot_gnuplot(result, args.gnuplot)
        except IOError:
            logger.exception('Plotting with gnuplot failed.')
    if size_input > 0:
        ms_mb = clk_exec * 1000 / (size_input / 1024**2)
    else:
        ms_mb = 0.0
    print(
        f'Processed {len(result):,d} files and {size_input:,d} bytes in {clk_exec:.3f} s, {ms_mb:.1f} ms/Mb'
    )
    print('Bye, bye!')
    return 0
コード例 #6
0
ファイル: PlotLogs.py プロジェクト: ojsindher/TotalDepth
def main():
    print('Cmd: %s' % ' '.join(sys.argv))
    # TODO: Option to treat files with -f, --format as LAS, LIS, AUTO
    # TODO: Depth scale overrides -s, --scale ?
    parser = cmn_cmd_opts.path_in_out(
        'Generates SVG plot(s) from input LIS & LAS file/directory to an output file/directory.',
        prog=None,
        version=__version__,
    )
    cmn_cmd_opts.add_log_level(parser)
    cmn_cmd_opts.add_multiprocessing(parser)
    parser.add_argument(
        "-A",
        "--API",
        action="store_true",
        dest="apiHeader",
        default=False,
        help="Put an API header on each plot. [default: False]")
    parser.add_argument("-x", "--xml", action="append", dest="LgFormat", default=[],
                      help="Use XML LgFormat UniqueId to use for plotting (additive)." \
                      +" Use -x? to see what LgFormats (UniqueID+Description) are available." \
                      +" Use -x?? to see what curves each format can plot. See also -X. [default: []]")
    parser.add_argument(
        "-X",
        "--XML",
        type=int,
        dest="LgFormat_min",
        default=0,
        help=
        "Use any LgFormat XML plots that use n or more outputs. If -x option present limited by those LgFormats [default: 0]"
    )
    parser.add_argument("-g",
                        "--glob",
                        action="store_true",
                        dest="glob",
                        default=None,
                        help="File match pattern. Default: %(default)s.")
    # parser.add_argument("-f", "--file-type", choices=['LAS', 'LIS', 'AUTO'],
    #        help="File format to assume for the input, AUTO will do it's best. [default: \"AUTO\"].")
    parser.add_argument(
        "-s",
        "--scale",
        action="append",
        type=int,
        dest="scale",
        default=0,
        help="Scale of X axis to use (an integer). [default: 0].")
    args = parser.parse_args()
    # Initialise logging etc.
    cmn_cmd_opts.set_log_level(args)
    # print('args', args)
    # return 0
    start_clock = time.clock()
    start_time = time.time()
    # Your code here
    if '?' in ''.join(args.LgFormat):
        # Handle -x? here and exit
        myFg = FILMCfgXML.FilmCfgXMLRead()
        print('XML LgFormats available: [{:d}]'.format(len(myFg.keys())))
        print(myFg.longStr(''.join(args.LgFormat).count('?')))
        return 1
    if cmn_cmd_opts.multiprocessing_requested(args):
        myPlp = PlotLogPasses(
            args.path_in,
            args.path_out,
            args,
        )
        myResult = myPlp.plotLogInfo
    else:
        myResult = plotLogPassesMP(
            args.path_in,
            args.path_out,
            args,
        )
    if os.path.isdir(args.path_out):
        myResult.writeHTML(os.path.join(args.path_out, 'index.html'),
                           args.path_in)
    print('plotLogInfo', str(myResult))
    print('  CPU time = %8.3f (S)' % (time.clock() - start_clock))
    print('Exec. time = %8.3f (S)' % (time.time() - start_time))
    print('Bye, bye!')
    return 0