Exemple #1
0
    def run(self, outputs={}):
        start_time = system.now_raw()
        log.logger.log_json_stage_run(self.name, start_time)

        log.ODM_INFO('Running %s stage' % self.name)

        self.process(self.args, outputs)

        # The tree variable should always be populated at this point
        if outputs.get('tree') is None:
            raise Exception(
                "Assert violation: tree variable is missing from outputs dictionary."
            )

        if self.args.time:
            system.benchmark(start_time, outputs['tree'].benchmarking,
                             self.name)

        log.ODM_INFO('Finished %s stage' % self.name)
        self.update_progress_end()

        # Last stage?
        if self.args.end_with == self.name or self.args.rerun == self.name:
            log.ODM_INFO("No more stages to run")
            return

        # Run next stage?
        elif self.next_stage is not None:
            self.next_stage.run(outputs)
Exemple #2
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Meshing Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        verbose = '-verbose' if self.params.verbose else ''

        # define paths and create working directories
        system.mkdir_p(tree.odm_meshing)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_meshing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_meshing' in args.rerun_from)

        if not io.file_exists(tree.odm_mesh) or rerun_cell:
            log.ODM_DEBUG('Writing ODM Mesh file in: %s' % tree.odm_mesh)

            kwargs = {
                'bin': context.odm_modules_path,
                'outfile': tree.odm_mesh,
                'log': tree.odm_meshing_log,
                'max_vertex': self.params.max_vertex,
                'oct_tree': self.params.oct_tree,
                'samples': self.params.samples,
                'solver': self.params.solver,
                'verbose': verbose
            }
            if not args.use_pmvs:
                kwargs['infile'] = tree.opensfm_model
            else:
                kwargs['infile'] = tree.pmvs_model

            # run meshing binary
            system.run(
                '{bin}/odm_meshing -inputFile {infile} '
                '-outputFile {outfile} -logFile {log} '
                '-maxVertexCount {max_vertex} -octreeDepth {oct_tree} {verbose} '
                '-samplesPerNode {samples} -solverDivide {solver}'.format(
                    **kwargs))
        else:
            log.ODM_WARNING('Found a valid ODM Mesh file in: %s' %
                            tree.odm_mesh)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Meshing')

        log.ODM_INFO('Running ODM Meshing Cell - Finished')
        return ecto.OK if args.end_with != 'odm_meshing' else ecto.QUIT
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Texturing Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree

        # define paths and create working directories
        system.mkdir_p(tree.odm_texturing)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_texturing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_texturing' in args.rerun_from)

        if not io.file_exists(tree.odm_textured_model_obj) or rerun_cell:
            log.ODM_DEBUG('Writing ODM Textured file in: %s' %
                          tree.odm_textured_model_obj)

            # odm_texturing definitions
            kwargs = {
                'bin': context.odm_modules_path,
                'out_dir': tree.odm_texturing,
                'bundle': tree.opensfm_bundle,
                'imgs_path': tree.dataset_resize,
                'imgs_list': tree.opensfm_bundle_list,
                'model': tree.odm_mesh,
                'log': tree.odm_texuring_log,
                'resize': self.params.resize,
                'resolution': self.params.resolution,
                'size': self.params.size
            }

            # run texturing binary
            system.run(
                '{bin}/odm_texturing -bundleFile {bundle} '
                '-imagesPath {imgs_path} -imagesListPath {imgs_list} '
                '-inputModelPath {model} -outputFolder {out_dir}/ '
                '-textureResolution {resolution} -bundleResizedTo {resize} '
                '-textureWithSize {size} -logFile {log}'.format(**kwargs))
        else:
            log.ODM_WARNING('Found a valid ODM Texture file in: %s' %
                            tree.odm_textured_model_obj)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Texturing')

        log.ODM_INFO('Running ODM Texturing Cell - Finished')
        return ecto.OK if args.end_with != 'odm_texturing' else ecto.QUIT
Exemple #4
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Texturing Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree

        # define paths and create working directories
        system.mkdir_p(tree.odm_texturing)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_texturing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_texturing' in args.rerun_from)

        if not io.file_exists(tree.odm_textured_model_obj) or rerun_cell:
            log.ODM_DEBUG('Writing ODM Textured file in: %s'
                          % tree.odm_textured_model_obj)

            # odm_texturing definitions
            kwargs = {
                'bin': context.odm_modules_path,
                'out_dir': tree.odm_texturing,
                'bundle': tree.opensfm_bundle,
                'imgs_path': tree.dataset_resize,
                'imgs_list': tree.opensfm_bundle_list,
                'model': tree.odm_mesh,
                'log': tree.odm_texuring_log,
                'resize': self.params.resize,
                'resolution': self.params.resolution,
                'size': self.params.size
            }

            # run texturing binary
            system.run('{bin}/odm_texturing -bundleFile {bundle} '
                       '-imagesPath {imgs_path} -imagesListPath {imgs_list} '
                       '-inputModelPath {model} -outputFolder {out_dir}/ '
                       '-textureResolution {resolution} -bundleResizedTo {resize} '
                       '-textureWithSize {size} -logFile {log}'.format(**kwargs))
        else:
            log.ODM_WARNING('Found a valid ODM Texture file in: %s'
                            % tree.odm_textured_model_obj)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Texturing')

        log.ODM_INFO('Running ODM Texturing Cell - Finished')
        return ecto.OK if args.end_with != 'odm_texturing' else ecto.QUIT
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Meshing Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        verbose = '-verbose' if self.params.verbose else ''

        # define paths and create working directories
        system.mkdir_p(tree.odm_meshing)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_meshing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_meshing' in args.rerun_from)

        if not io.file_exists(tree.odm_mesh) or rerun_cell:
            log.ODM_DEBUG('Writing ODM Mesh file in: %s' % tree.odm_mesh)

            kwargs = {
                'bin': context.odm_modules_path,
                'outfile': tree.odm_mesh,
                'log': tree.odm_meshing_log,
                'max_vertex': self.params.max_vertex,
                'oct_tree': self.params.oct_tree,
                'samples': self.params.samples,
                'solver': self.params.solver,
                'verbose': verbose
            }
            if not args.use_pmvs:
                kwargs['infile'] = tree.opensfm_model
            else:
                kwargs['infile'] = tree.pmvs_model

            # run meshing binary
            system.run('{bin}/odm_meshing -inputFile {infile} '
                       '-outputFile {outfile} -logFile {log} '
                       '-maxVertexCount {max_vertex} -octreeDepth {oct_tree} {verbose} '
                       '-samplesPerNode {samples} -solverDivide {solver}'.format(**kwargs))
        else:
            log.ODM_WARNING('Found a valid ODM Mesh file in: %s' %
                            tree.odm_mesh)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Meshing')

        log.ODM_INFO('Running ODM Meshing Cell - Finished')
        return ecto.OK if args.end_with != 'odm_meshing' else ecto.QUIT
Exemple #6
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Resize Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        photos = self.inputs.photos

        if not photos:
            log.ODM_ERROR('Not enough photos in photos to resize')
            return ecto.QUIT

        if self.params.resize_to <= 0:
            log.ODM_ERROR('Resize parameter must be greater than 0')
            return ecto.QUIT

        # create working directory
        system.mkdir_p(tree.dataset_resize)

        log.ODM_DEBUG('Resizing dataset to: %s' % tree.dataset_resize)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'resize') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'resize' in args.rerun_from)

        # loop over photos
        if self.params.skip_resize:
            photos = Pool().map(
                partial(no_resize, tree.dataset_raw, tree.dataset_resize,
                        rerun_cell), photos)
            log.ODM_INFO('Copied %s images' % len(photos))
        else:
            photos = Pool().map(
                partial(resize, tree.dataset_raw, tree.dataset_resize,
                        self.params.resize_to, rerun_cell), photos)
            log.ODM_INFO('Resized %s images' % len(photos))

        # append photos to cell output
        self.outputs.photos = photos

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Resizing')

        log.ODM_INFO('Running ODM Resize Cell - Finished')
        return ecto.OK if args.end_with != 'resize' else ecto.QUIT
Exemple #7
0
    def process(self, inputs, outputs):
        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM FilterPoints Cell')

        # get inputs
        tree = inputs.tree
        args = inputs.args
        reconstruction = inputs.reconstruction

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_filterpoints') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_filterpoints' in args.rerun_from)
        if not os.path.exists(tree.odm_filterpoints):
            system.mkdir_p(tree.odm_filterpoints)

        # check if reconstruction was done before
        if not io.file_exists(tree.filtered_point_cloud) or rerun_cell:
            if args.fast_orthophoto:
                inputPointCloud = os.path.join(tree.opensfm,
                                               'reconstruction.ply')
            elif args.use_opensfm_dense:
                inputPointCloud = tree.opensfm_model
            else:
                inputPointCloud = tree.mve_model

            confidence = None
            if not args.use_opensfm_dense and not args.fast_orthophoto:
                confidence = args.mve_confidence

            point_cloud.filter(inputPointCloud,
                               tree.filtered_point_cloud,
                               standard_deviation=args.pc_filter,
                               confidence=confidence,
                               verbose=args.verbose)
        else:
            log.ODM_WARNING('Found a valid point cloud file in: %s' %
                            tree.filtered_point_cloud)

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'MVE')

        log.ODM_INFO('Running ODM FilterPoints Cell - Finished')
        return ecto.OK if args.end_with != 'odm_filterpoints' else ecto.QUIT
Exemple #8
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running OMD PMVS Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'pmvs') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'pmvs' in args.rerun_from)

        if not io.file_exists(tree.pmvs_model) or rerun_cell:
            log.ODM_DEBUG('Creating dense pointcloud in: %s' % tree.pmvs_model)

            kwargs = {
                'bin': context.cmvs_opts_path,
                'prefix': tree.pmvs_rec_path,
                'level': self.params.level,
                'csize': self.params.csize,
                'thresh': self.params.thresh,
                'wsize': self.params.wsize,
                'min_imgs': self.params.min_imgs,
                'cores': self.params.cores
            }

            # generate pmvs2 options
            system.run('{bin} {prefix}/ {level} {csize} {thresh} {wsize} '
                       '{min_imgs} {cores}'.format(**kwargs))

            # run pmvs2
            system.run('%s %s/ option-0000' %
                       (context.pmvs2_path, tree.pmvs_rec_path))

        else:
            log.ODM_WARNING('Found a valid PMVS file in %s' % tree.pmvs_model)

        outputs.reconstruction = inputs.reconstruction
        
        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'PMVS')

        log.ODM_INFO('Running ODM PMVS Cell - Finished')
        return ecto.OK if args.end_with != 'pmvs' else ecto.QUIT
Exemple #9
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running OMD PMVS Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'pmvs') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'pmvs' in args.rerun_from)

        if not io.file_exists(tree.pmvs_model) or rerun_cell:
            log.ODM_DEBUG('Creating dense pointcloud in: %s' % tree.pmvs_model)

            kwargs = {
                'bin': context.cmvs_opts_path,
                'prefix': tree.pmvs_rec_path,
                'level': self.params.level,
                'csize': self.params.csize,
                'thresh': self.params.thresh,
                'wsize': self.params.wsize,
                'min_imgs': self.params.min_imgs,
                'cores': self.params.cores
            }

            # generate pmvs2 options
            system.run('{bin} {prefix}/ {level} {csize} {thresh} {wsize} '
                       '{min_imgs} {cores}'.format(**kwargs))

            # run pmvs2
            system.run('%s %s/ option-0000' %
                       (context.pmvs2_path, tree.pmvs_rec_path))

        else:
            log.ODM_WARNING('Found a valid PMVS file in %s' % tree.pmvs_model)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'PMVS')

        log.ODM_INFO('Running ODM PMVS Cell - Finished')
        return ecto.OK if args.end_with != 'pmvs' else ecto.QUIT
    def process(self, inputs, outputs):
        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Georeferencing Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        gcpfile = io.join_paths(tree.root_path, self.params.gcp_file)

        # define paths and create working directories
        system.mkdir_p(tree.odm_georeferencing)

        # in case a gcp file it's not provided, let's try to generate it using
        # images metadata. Internally calls jhead.
        if not self.params.use_gcp and \
           not io.file_exists(tree.odm_georeferencing_coords):

            log.ODM_WARNING('Warning: No coordinates file. '
                            'Generating coordinates file in: %s' %
                            tree.odm_georeferencing_coords)
            try:
                # odm_georeference definitions
                kwargs = {
                    'bin': context.odm_modules_path,
                    'imgs': tree.dataset_raw,
                    'imgs_list': tree.opensfm_bundle_list,
                    'coords': tree.odm_georeferencing_coords,
                    'log': tree.odm_georeferencing_utm_log
                }

                # run UTM extraction binary
                system.run(
                    '{bin}/odm_extract_utm -imagesPath {imgs}/ '
                    '-imageListFile {imgs_list} -outputCoordFile {coords} '
                    '-logFile {log}'.format(**kwargs))

            except Exception, e:
                log.ODM_ERROR(
                    'Could not generate GCP file from images metadata.'
                    'Consider rerunning with argument --odm_georeferencing-useGcp'
                    ' and provide a proper GCP file')
                log.ODM_ERROR(e)
                return ecto.QUIT
Exemple #11
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM CMVS Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'cmvs') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'cmvs' in args.rerun_from)

        if not io.file_exists(tree.pmvs_bundle) or rerun_cell:
            log.ODM_DEBUG('Writing CMVS vis in: %s' % tree.pmvs_bundle)

            # copy bundle file to pmvs dir
            from shutil import copyfile
            copyfile(tree.opensfm_bundle, tree.pmvs_bundle)

            kwargs = {
                'bin': context.cmvs_path,
                'prefix': self.inputs.tree.pmvs_rec_path,
                'max_images': self.params.max_images,
                'cores': self.params.cores
            }

            # run cmvs
            system.run('{bin} {prefix}/ {max_images} {cores}'.format(**kwargs))
        else:
            log.ODM_WARNING('Found a valid CMVS file in: %s' %
                            tree.pmvs_bundle)

        outputs.reconstruction = inputs.reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'CMVS')

        log.ODM_INFO('Running ODM CMVS Cell - Finished')
        return ecto.OK if args.end_with != 'cmvs' else ecto.QUIT
    def process(self, inputs, outputs):
        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM FilterPoints Cell')

        # get inputs
        tree = inputs.tree
        args = inputs.args
        reconstruction = inputs.reconstruction

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_filterpoints') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_filterpoints' in args.rerun_from)
        if not os.path.exists(tree.odm_filterpoints): system.mkdir_p(tree.odm_filterpoints)

        # check if reconstruction was done before
        if not io.file_exists(tree.filtered_point_cloud) or rerun_cell:
            if args.fast_orthophoto:
                inputPointCloud = os.path.join(tree.opensfm, 'reconstruction.ply')
            elif args.use_opensfm_dense:
                inputPointCloud = tree.opensfm_model
            else:
                inputPointCloud = tree.mve_model

            confidence = None
            if not args.use_opensfm_dense and not args.fast_orthophoto:
                confidence = args.mve_confidence

            point_cloud.filter(inputPointCloud, tree.filtered_point_cloud, standard_deviation=args.pc_filter, confidence=confidence, verbose=args.verbose)
        else:
            log.ODM_WARNING('Found a valid point cloud file in: %s' %
                            tree.filtered_point_cloud)

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'MVE')

        log.ODM_INFO('Running ODM FilterPoints Cell - Finished')
        return ecto.OK if args.end_with != 'odm_filterpoints' else ecto.QUIT
Exemple #13
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()
        
        log.ODM_INFO('Running ODM CMVS Cell')

        # get inputs 
        args = self.inputs.args
        tree = self.inputs.tree

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'cmvs') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'cmvs' in args.rerun_from)

        if not io.file_exists(tree.pmvs_bundle) or rerun_cell:
            log.ODM_DEBUG('Writing CMVS vis in: %s' % tree.pmvs_bundle)

            # copy bundle file to pmvs dir
            from shutil import copyfile
            copyfile(tree.opensfm_bundle, 
                     tree.pmvs_bundle)

            kwargs = {
                'bin': context.cmvs_path,
                'prefix': self.inputs.tree.pmvs_rec_path,
                'max_images': self.params.max_images,
                'cores': self.params.cores
            }

            # run cmvs
            system.run('{bin} {prefix}/ {max_images} {cores}'.format(**kwargs))
        else:
            log.ODM_WARNING('Found a valid CMVS file in: %s' % 
                            tree.pmvs_bundle)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'CMVS')

        log.ODM_INFO('Running ODM CMVS Cell - Finished')
        return ecto.OK if args.end_with != 'cmvs' else ecto.QUIT
    def process(self, inputs, outputs):
        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Georeferencing Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        gcpfile = io.join_paths(tree.root_path, self.params.gcp_file)

        # define paths and create working directories
        system.mkdir_p(tree.odm_georeferencing)

        # in case a gcp file it's not provided, let's try to generate it using
        # images metadata. Internally calls jhead.
        if not self.params.use_gcp and \
           not io.file_exists(tree.odm_georeferencing_coords):
            
            log.ODM_WARNING('Warning: No coordinates file. '
                            'Generating coordinates file in: %s'
                            % tree.odm_georeferencing_coords)
            try:
                # odm_georeference definitions
                kwargs = {
                    'bin': context.odm_modules_path,
                    'imgs': tree.dataset_raw,
                    'imgs_list': tree.opensfm_bundle_list,
                    'coords': tree.odm_georeferencing_coords,
                    'log': tree.odm_georeferencing_utm_log
                }

                # run UTM extraction binary
                system.run('{bin}/odm_extract_utm -imagesPath {imgs}/ '
                           '-imageListFile {imgs_list} -outputCoordFile {coords} '
                           '-logFile {log}'.format(**kwargs))

            except Exception, e:
                log.ODM_ERROR('Could not generate GCP file from images metadata.'
                              'Consider rerunning with argument --odm_georeferencing-useGcp'
                              ' and provide a proper GCP file')
                log.ODM_ERROR(e)
                return ecto.QUIT
Exemple #15
0
    def process(self, inputs, outputs):
        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running MVE Cell')

        # get inputs
        tree = inputs.tree
        args = inputs.args
        reconstruction = inputs.reconstruction
        photos = reconstruction.photos

        if not photos:
            log.ODM_ERROR('Not enough photos in photos array to start MVE')
            return ecto.QUIT

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'mve') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'mve' in args.rerun_from)

        # check if reconstruction was done before
        if not io.file_exists(tree.mve_model) or rerun_cell:
            # cleanup if a rerun
            if io.dir_exists(tree.mve_path) and rerun_cell:
                shutil.rmtree(tree.mve_path)

            # make bundle directory
            if not io.file_exists(tree.mve_bundle):
                system.mkdir_p(tree.mve_path)
                system.mkdir_p(io.join_paths(tree.mve_path, 'bundle'))
                io.copy(tree.opensfm_image_list, tree.mve_image_list)
                io.copy(tree.opensfm_bundle, tree.mve_bundle)

            # mve makescene wants the output directory
            # to not exists before executing it (otherwise it
            # will prompt the user for confirmation)
            if io.dir_exists(tree.mve):
                shutil.rmtree(tree.mve)

            # run mve makescene
            if not io.dir_exists(tree.mve_views):
                system.run('%s %s %s' %
                           (context.makescene_path, tree.mve_path, tree.mve),
                           env_vars={'OMP_NUM_THREADS': args.max_concurrency})

            # Compute mve output scale based on depthmap_resolution
            max_width = 0
            max_height = 0
            for photo in photos:
                max_width = max(photo.width, max_width)
                max_height = max(photo.height, max_height)

            max_pixels = args.depthmap_resolution * args.depthmap_resolution
            if max_width * max_height <= max_pixels:
                mve_output_scale = 0
            else:
                ratio = float(max_width * max_height) / float(max_pixels)
                mve_output_scale = int(
                    math.ceil(math.log(ratio) / math.log(4.0)))

            dmrecon_config = [
                "-s%s" % mve_output_scale,
                "--progress=silent",
                "--local-neighbors=2",
                "--force",
            ]

            # Run MVE's dmrecon
            log.ODM_INFO(
                '                                                                               '
            )
            log.ODM_INFO(
                '                                    ,*/**                                      '
            )
            log.ODM_INFO(
                '                                  ,*@%*/@%*                                    '
            )
            log.ODM_INFO(
                '                                ,/@%******@&*.                                 '
            )
            log.ODM_INFO(
                '                              ,*@&*********/@&*                                '
            )
            log.ODM_INFO(
                '                            ,*@&**************@&*                              '
            )
            log.ODM_INFO(
                '                          ,/@&******************@&*.                           '
            )
            log.ODM_INFO(
                '                        ,*@&*********************/@&*                          '
            )
            log.ODM_INFO(
                '                      ,*@&**************************@&*.                       '
            )
            log.ODM_INFO(
                '                    ,/@&******************************&&*,                     '
            )
            log.ODM_INFO(
                '                  ,*&&**********************************@&*.                   '
            )
            log.ODM_INFO(
                '                ,*@&**************************************@&*.                 '
            )
            log.ODM_INFO(
                '              ,*@&***************#@@@@@@@@@%****************&&*,               '
            )
            log.ODM_INFO(
                '            .*&&***************&@@@@@@@@@@@@@@****************@@*.             '
            )
            log.ODM_INFO(
                '          .*@&***************&@@@@@@@@@@@@@@@@@%****(@@%********@@*.           '
            )
            log.ODM_INFO(
                '        .*@@***************%@@@@@@@@@@@@@@@@@@@@@#****&@@@@%******&@*,         '
            )
            log.ODM_INFO(
                '      .*&@****************@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@/*****@@*.       '
            )
            log.ODM_INFO(
                '    .*@@****************@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@%*************@@*.     '
            )
            log.ODM_INFO(
                '  .*@@****/***********@@@@@&**(@@@@@@@@@@@@@@@@@@@@@@@#*****************%@*,   '
            )
            log.ODM_INFO(
                ' */@*******@*******#@@@@%*******/@@@@@@@@@@@@@@@@@@@@********************/@(,  '
            )
            log.ODM_INFO(
                ' ,*@(********&@@@@@@#**************/@@@@@@@#**(@@&/**********************@&*   '
            )
            log.ODM_INFO(
                '   *#@/*******************************@@@@@***&@&**********************&@*,    '
            )
            log.ODM_INFO(
                '     *#@#******************************&@@@***@#*********************&@*,      '
            )
            log.ODM_INFO(
                '       */@#*****************************@@@************************@@*.        '
            )
            log.ODM_INFO(
                '         *#@/***************************/@@/*********************%@*,          '
            )
            log.ODM_INFO(
                '           *#@#**************************#@@%******************%@*,            '
            )
            log.ODM_INFO(
                '             */@#*************************(@@@@@@@&%/********&@*.              '
            )
            log.ODM_INFO(
                '               *(@(*********************************/%@@%**%@*,                '
            )
            log.ODM_INFO(
                '                 *(@%************************************%@**                  '
            )
            log.ODM_INFO(
                '                   **@%********************************&@*,                    '
            )
            log.ODM_INFO(
                '                     *(@(****************************%@/*                      '
            )
            log.ODM_INFO(
                '                       ,(@%************************#@/*                        '
            )
            log.ODM_INFO(
                '                         ,*@%********************&@/,                          '
            )
            log.ODM_INFO(
                '                           */@#****************#@/*                            '
            )
            log.ODM_INFO(
                '                             ,/@&************#@/*                              '
            )
            log.ODM_INFO(
                '                               ,*@&********%@/,                                '
            )
            log.ODM_INFO(
                '                                 */@#****(@/*                                  '
            )
            log.ODM_INFO(
                '                                   ,/@@@@(*                                    '
            )
            log.ODM_INFO(
                '                                     .**,                                      '
            )
            log.ODM_INFO('')
            log.ODM_INFO(
                "Running dense reconstruction. This might take a while. Please be patient, the process is not dead or hung."
            )
            log.ODM_INFO("                              Process is running")
            system.run(
                '%s %s %s' %
                (context.dmrecon_path, ' '.join(dmrecon_config), tree.mve),
                env_vars={'OMP_NUM_THREADS': args.max_concurrency})

            scene2pset_config = ["-F%s" % mve_output_scale]

            # run scene2pset
            system.run('%s %s "%s" "%s"' %
                       (context.scene2pset_path, ' '.join(scene2pset_config),
                        tree.mve, tree.mve_model),
                       env_vars={'OMP_NUM_THREADS': args.max_concurrency})
        else:
            log.ODM_WARNING('Found a valid MVE reconstruction file in: %s' %
                            tree.mve_model)

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'MVE')

        log.ODM_INFO('Running ODM MVE Cell - Finished')
        return ecto.OK if args.end_with != 'mve' else ecto.QUIT
Exemple #16
0
    def process(self, args, outputs):
        tree = outputs['tree']
        reconstruction = outputs['reconstruction']

        if not os.path.exists(tree.odm_report): system.mkdir_p(tree.odm_report)

        log.ODM_INFO("Exporting shots.geojson")

        shots_geojson = os.path.join(tree.odm_report, "shots.geojson")
        if not io.file_exists(shots_geojson) or self.rerun():
            # Extract geographical camera shots
            if reconstruction.is_georeferenced():
                shots = get_geojson_shots_from_opensfm(
                    tree.opensfm_reconstruction,
                    utm_srs=reconstruction.get_proj_srs(),
                    utm_offset=reconstruction.georef.utm_offset())
            else:
                # Pseudo geo
                shots = get_geojson_shots_from_opensfm(
                    tree.opensfm_reconstruction,
                    pseudo_geotiff=tree.odm_orthophoto_tif)

            if shots:
                with open(shots_geojson, "w") as fout:
                    fout.write(json.dumps(shots))

                log.ODM_INFO("Wrote %s" % shots_geojson)
            else:
                log.ODM_WARNING("Cannot extract shots")
        else:
            log.ODM_WARNING('Found a valid shots file in: %s' % shots_geojson)

        if args.skip_report:
            # Stop right here
            log.ODM_WARNING("Skipping report generation as requested")
            return

        # Augment OpenSfM stats file with our own stats
        odm_stats_json = os.path.join(tree.odm_report, "stats.json")
        octx = OSFMContext(tree.opensfm)
        osfm_stats_json = octx.path("stats", "stats.json")
        odm_stats = None
        point_cloud_file = None
        views_dimension = None

        if not os.path.exists(odm_stats_json) or self.rerun():
            if os.path.exists(osfm_stats_json):
                with open(osfm_stats_json, 'r') as f:
                    odm_stats = json.loads(f.read())

                # Add point cloud stats
                if os.path.exists(tree.odm_georeferencing_model_laz):
                    point_cloud_file = tree.odm_georeferencing_model_laz
                    views_dimension = "UserData"

                    # pc_info_file should have been generated by cropper
                    pc_info_file = os.path.join(
                        tree.odm_georeferencing,
                        "odm_georeferenced_model.info.json")
                    odm_stats[
                        'point_cloud_statistics'] = generate_point_cloud_stats(
                            tree.odm_georeferencing_model_laz, pc_info_file,
                            self.rerun())
                else:
                    ply_pc = os.path.join(tree.odm_filterpoints,
                                          "point_cloud.ply")
                    if os.path.exists(ply_pc):
                        point_cloud_file = ply_pc
                        views_dimension = "views"

                        pc_info_file = os.path.join(tree.odm_filterpoints,
                                                    "point_cloud.info.json")
                        odm_stats[
                            'point_cloud_statistics'] = generate_point_cloud_stats(
                                ply_pc, pc_info_file, self.rerun())
                    else:
                        log.ODM_WARNING("No point cloud found")

                odm_stats['point_cloud_statistics'][
                    'dense'] = not args.fast_orthophoto

                # Add runtime stats
                total_time = (system.now_raw() -
                              outputs['start_time']).total_seconds()
                odm_stats['odm_processing_statistics'] = {
                    'total_time':
                    total_time,
                    'total_time_human':
                    hms(total_time),
                    'average_gsd':
                    gsd.opensfm_reconstruction_average_gsd(
                        octx.recon_file(),
                        use_all_shots=reconstruction.has_gcp()),
                }

                with open(odm_stats_json, 'w') as f:
                    f.write(json.dumps(odm_stats))
            else:
                log.ODM_WARNING(
                    "Cannot generate report, OpenSfM stats are missing")
        else:
            log.ODM_WARNING("Reading existing stats %s" % odm_stats_json)
            with open(odm_stats_json, 'r') as f:
                odm_stats = json.loads(f.read())

        # Generate overlap diagram
        if odm_stats.get('point_cloud_statistics'
                         ) and point_cloud_file and views_dimension:
            bounds = odm_stats['point_cloud_statistics'].get('stats', {}).get(
                'bbox', {}).get('native', {}).get('bbox')
            if bounds:
                image_target_size = 1400  # pixels
                osfm_stats_dir = os.path.join(tree.opensfm, "stats")
                diagram_tiff = os.path.join(osfm_stats_dir, "overlap.tif")
                diagram_png = os.path.join(osfm_stats_dir, "overlap.png")

                width = bounds.get('maxx') - bounds.get('minx')
                height = bounds.get('maxy') - bounds.get('miny')
                max_dim = max(width, height)
                resolution = float(max_dim) / float(image_target_size)
                radius = resolution * math.sqrt(2)

                # Larger radius for sparse point cloud diagram
                if not odm_stats['point_cloud_statistics']['dense']:
                    radius *= 10

                system.run("pdal translate -i \"{}\" "
                           "-o \"{}\" "
                           "--writer gdal "
                           "--writers.gdal.resolution={} "
                           "--writers.gdal.data_type=uint8_t "
                           "--writers.gdal.dimension={} "
                           "--writers.gdal.output_type=max "
                           "--writers.gdal.radius={} ".format(
                               point_cloud_file, diagram_tiff, resolution,
                               views_dimension, radius))
                report_assets = os.path.abspath(
                    os.path.join(os.path.dirname(__file__),
                                 "../opendm/report"))
                overlap_color_map = os.path.join(report_assets,
                                                 "overlap_color_map.txt")

                bounds_file_path = os.path.join(
                    tree.odm_georeferencing,
                    'odm_georeferenced_model.bounds.gpkg')
                if (args.crop > 0
                        or args.boundary) and os.path.isfile(bounds_file_path):
                    Cropper.crop(bounds_file_path,
                                 diagram_tiff,
                                 get_orthophoto_vars(args),
                                 keep_original=False)

                system.run(
                    "gdaldem color-relief \"{}\" \"{}\" \"{}\" -of PNG -alpha".
                    format(diagram_tiff, overlap_color_map, diagram_png))

                # Copy assets
                for asset in [
                        "overlap_diagram_legend.png", "dsm_gradient.png"
                ]:
                    shutil.copy(os.path.join(report_assets, asset),
                                os.path.join(osfm_stats_dir, asset))

                # Generate previews of ortho/dsm
                if os.path.isfile(tree.odm_orthophoto_tif):
                    osfm_ortho = os.path.join(osfm_stats_dir, "ortho.png")
                    generate_png(tree.odm_orthophoto_tif, osfm_ortho,
                                 image_target_size)

                dems = []
                if args.dsm:
                    dems.append("dsm")
                if args.dtm:
                    dems.append("dtm")

                for dem in dems:
                    dem_file = tree.path("odm_dem", "%s.tif" % dem)
                    if os.path.isfile(dem_file):
                        # Resize first (faster)
                        resized_dem_file = io.related_file_path(
                            dem_file, postfix=".preview")
                        system.run(
                            "gdal_translate -outsize {} 0 \"{}\" \"{}\" --config GDAL_CACHEMAX {}%"
                            .format(image_target_size, dem_file,
                                    resized_dem_file, get_max_memory()))

                        log.ODM_INFO("Computing raster stats for %s" %
                                     resized_dem_file)
                        dem_stats = get_raster_stats(resized_dem_file)
                        if len(dem_stats) > 0:
                            odm_stats[dem + '_statistics'] = dem_stats[0]

                        osfm_dem = os.path.join(osfm_stats_dir, "%s.png" % dem)
                        colored_dem, hillshade_dem, colored_hillshade_dem = generate_colored_hillshade(
                            resized_dem_file)
                        system.run(
                            "gdal_translate -outsize {} 0 -of png \"{}\" \"{}\" --config GDAL_CACHEMAX {}%"
                            .format(image_target_size, colored_hillshade_dem,
                                    osfm_dem, get_max_memory()))
                        for f in [
                                resized_dem_file, colored_dem, hillshade_dem,
                                colored_hillshade_dem
                        ]:
                            if os.path.isfile(f):
                                os.remove(f)
            else:
                log.ODM_WARNING(
                    "Cannot generate overlap diagram, cannot compute point cloud bounds"
                )
        else:
            log.ODM_WARNING(
                "Cannot generate overlap diagram, point cloud stats missing")

        octx.export_report(os.path.join(tree.odm_report, "report.pdf"),
                           odm_stats, self.rerun())
Exemple #17
0
    def process(self, args, outputs):
        outputs['start_time'] = system.now_raw()
        tree = types.ODM_Tree(args.project_path, args.gcp, args.geo)
        outputs['tree'] = tree

        if args.time and io.file_exists(tree.benchmarking):
            # Delete the previously made file
            os.remove(tree.benchmarking)
            with open(tree.benchmarking, 'a') as b:
                b.write(
                    'ODM Benchmarking file created %s\nNumber of Cores: %s\n\n'
                    % (system.now(), context.num_cores))

        # check if the image filename is supported
        def valid_image_filename(filename):
            (pathfn, ext) = os.path.splitext(filename)
            return ext.lower(
            ) in context.supported_extensions and pathfn[-5:] != "_mask"

        # Get supported images from dir
        def get_images(in_dir):
            log.ODM_DEBUG(in_dir)
            entries = os.listdir(in_dir)
            valid, rejects = [], []
            for f in entries:
                if valid_image_filename(f):
                    valid.append(f)
                else:
                    rejects.append(f)
            return valid, rejects

        def find_mask(photo_path, masks):
            (pathfn, ext) = os.path.splitext(os.path.basename(photo_path))
            k = "{}_mask".format(pathfn)

            mask = masks.get(k)
            if mask:
                # Spaces are not supported due to OpenSfM's mask_list.txt format reqs
                if not " " in mask:
                    return mask
                else:
                    log.ODM_WARNING(
                        "Image mask {} has a space. Spaces are currently not supported for image masks."
                        .format(mask))

        # get images directory
        images_dir = tree.dataset_raw

        # define paths and create working directories
        system.mkdir_p(tree.odm_georeferencing)

        log.ODM_INFO('Loading dataset from: %s' % images_dir)

        # check if we rerun cell or not
        images_database_file = os.path.join(tree.root_path, 'images.json')
        if not io.file_exists(images_database_file) or self.rerun():
            if not os.path.exists(images_dir):
                raise system.ExitException(
                    "There are no images in %s! Make sure that your project path and dataset name is correct. The current is set to: %s"
                    % (images_dir, args.project_path))

            files, rejects = get_images(images_dir)
            if files:
                # create ODMPhoto list
                path_files = [os.path.join(images_dir, f) for f in files]

                # Lookup table for masks
                masks = {}
                for r in rejects:
                    (p, ext) = os.path.splitext(r)
                    if p[-5:] == "_mask" and ext.lower(
                    ) in context.supported_extensions:
                        masks[p] = r

                photos = []
                with open(tree.dataset_list, 'w') as dataset_list:
                    log.ODM_INFO("Loading %s images" % len(path_files))
                    for f in path_files:
                        try:
                            p = types.ODM_Photo(f)
                            p.set_mask(find_mask(f, masks))
                            photos += [p]
                            dataset_list.write(photos[-1].filename + '\n')
                        except PhotoCorruptedException:
                            log.ODM_WARNING(
                                "%s seems corrupted and will not be used" %
                                os.path.basename(f))

                # Check if a geo file is available
                if tree.odm_geo_file is not None and os.path.isfile(
                        tree.odm_geo_file):
                    log.ODM_INFO("Found image geolocation file")
                    gf = GeoFile(tree.odm_geo_file)
                    updated = 0
                    for p in photos:
                        entry = gf.get_entry(p.filename)
                        if entry:
                            p.update_with_geo_entry(entry)
                            p.compute_opk()
                            updated += 1
                    log.ODM_INFO("Updated %s image positions" % updated)

                # GPSDOP override if we have GPS accuracy information (such as RTK)
                if 'gps_accuracy_is_set' in args:
                    log.ODM_INFO("Forcing GPS DOP to %s for all images" %
                                 args.gps_accuracy)

                    for p in photos:
                        p.override_gps_dop(args.gps_accuracy)

                # Override projection type
                if args.camera_lens != "auto":
                    log.ODM_INFO("Setting camera lens to %s for all images" %
                                 args.camera_lens)

                    for p in photos:
                        p.override_camera_projection(args.camera_lens)

                # Save image database for faster restart
                save_images_database(photos, images_database_file)
            else:
                raise system.ExitException(
                    'Not enough supported images in %s' % images_dir)
        else:
            # We have an images database, just load it
            photos = load_images_database(images_database_file)

        log.ODM_INFO('Found %s usable images' % len(photos))
        log.logger.log_json_images(len(photos))

        # Create reconstruction object
        reconstruction = types.ODM_Reconstruction(photos)

        if tree.odm_georeferencing_gcp and not args.use_exif:
            reconstruction.georeference_with_gcp(
                tree.odm_georeferencing_gcp,
                tree.odm_georeferencing_coords,
                tree.odm_georeferencing_gcp_utm,
                tree.odm_georeferencing_model_txt_geo,
                rerun=self.rerun())
        else:
            reconstruction.georeference_with_gps(
                tree.dataset_raw,
                tree.odm_georeferencing_coords,
                tree.odm_georeferencing_model_txt_geo,
                rerun=self.rerun())

        reconstruction.save_proj_srs(
            os.path.join(tree.odm_georeferencing,
                         tree.odm_georeferencing_proj))
        outputs['reconstruction'] = reconstruction

        # Try to load boundaries
        if args.boundary:
            if reconstruction.is_georeferenced():
                outputs['boundary'] = boundary.load_boundary(
                    args.boundary, reconstruction.get_proj_srs())
            else:
                args.boundary = None
                log.ODM_WARNING(
                    "Reconstruction is not georeferenced, but boundary file provided (will ignore boundary file)"
                )

        # If sfm-algorithm is triangulation, check if photos have OPK
        if args.sfm_algorithm == 'triangulation':
            for p in photos:
                if not p.has_opk():
                    log.ODM_WARNING(
                        "No omega/phi/kappa angles found in input photos (%s), switching sfm-algorithm to incremental"
                        % p.filename)
                    args.sfm_algorithm = 'incremental'
                    break
    def process(self, inputs, outputs):

        # find a file in the root directory
        def find(file, dir):
            for root, dirs, files in os.walk(dir):
                return '/'.join((root, file)) if file in files else None

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Georeferencing Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        gcpfile = io.join_paths(tree.root_path, self.params.gcp_file) \
            if self.params.gcp_file else find('gcp_list.txt', tree.root_path)
        geocreated = True
        verbose = '-verbose' if self.params.verbose else ''

        # define paths and create working directories
        system.mkdir_p(tree.odm_georeferencing)

        # in case a gcp file it's not provided, let's try to generate it using
        # images metadata. Internally calls jhead.
        log.ODM_DEBUG(self.params.gcp_file)
        if not self.params.gcp_file: # and \
        #   not io.file_exists(tree.odm_georeferencing_coords):
            
            log.ODM_WARNING('No coordinates file. '
                            'Generating coordinates file: %s'
                            % tree.odm_georeferencing_coords)

            # odm_georeference definitions
            kwargs = {
                'bin': context.odm_modules_path,
                'imgs': tree.dataset_resize,
                'imgs_list': tree.opensfm_bundle_list,
                'coords': tree.odm_georeferencing_coords,
                'log': tree.odm_georeferencing_utm_log,
                'verbose': verbose
            }

            # run UTM extraction binary
            extract_utm = system.run_and_return('{bin}/odm_extract_utm -imagesPath {imgs}/ '
                       '-imageListFile {imgs_list} -outputCoordFile {coords} {verbose} '
                       '-logFile {log}'.format(**kwargs))

            if extract_utm != '':
                log.ODM_WARNING('Could not generate coordinates file. '
                                'Ignore if there is a GCP file. Error: %s'
                                % extract_utm)


        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_georeferencing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_georeferencing' in args.rerun_from)

        if not io.file_exists(tree.odm_georeferencing_model_obj_geo) or \
           not io.file_exists(tree.odm_georeferencing_model_ply_geo) or rerun_cell:

            # odm_georeference definitions
            kwargs = {
                'bin': context.odm_modules_path,
                'bundle': tree.opensfm_bundle,
                'imgs': tree.dataset_resize,
                'imgs_list': tree.opensfm_bundle_list,
                'model': tree.odm_textured_model_obj,
                'log': tree.odm_georeferencing_log,
                'coords': tree.odm_georeferencing_coords,
                'pc_geo': tree.odm_georeferencing_model_ply_geo,
                'geo_sys': tree.odm_georeferencing_model_txt_geo,
                'model_geo': tree.odm_georeferencing_model_obj_geo,
                'size': self.params.img_size,
                'gcp': gcpfile,
                'verbose': verbose

            }
            if args.use_opensfm_pointcloud:
                kwargs['pc'] = tree.opensfm_model
            else:
                kwargs['pc'] = tree.pmvs_model

            # Check to see if the GCP file exists

            if not self.params.use_exif and (self.params.gcp_file or find('gcp_list.txt', tree.root_path)):
                log.ODM_INFO('Found %s' % gcpfile)
                try:
                    system.run('{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} '
                               '-bundleResizedTo {size} -inputFile {model} -outputFile {model_geo} '
                               '-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo} {verbose} '
                               '-logFile {log} -georefFileOutputPath {geo_sys} -gcpFile {gcp} '
                               '-outputCoordFile {coords}'.format(**kwargs))
                except Exception:
                    log.ODM_EXCEPTION('Georeferencing failed. ')
                    return ecto.QUIT
            elif io.file_exists(tree.odm_georeferencing_coords):
                log.ODM_INFO('Running georeferencing with generated coords file.')
                system.run('{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} '
                           '-inputFile {model} -outputFile {model_geo} '
                           '-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo} {verbose} '
                           '-logFile {log} -georefFileOutputPath {geo_sys}'.format(**kwargs))
            else:
                log.ODM_WARNING('Georeferencing failed. Make sure your '
                                'photos have geotags in the EXIF or you have '
                                'provided a GCP file. ')
                geocreated = False # skip the rest of the georeferencing

            if geocreated:
                # update images metadata
                geo_ref = types.ODM_GeoRef()
                geo_ref.parse_coordinate_system(tree.odm_georeferencing_coords)

                for idx, photo in enumerate(self.inputs.photos):
                    geo_ref.utm_to_latlon(tree.odm_georeferencing_latlon, photo, idx)

                # convert ply model to LAS reference system
                geo_ref.convert_to_las(tree.odm_georeferencing_model_ply_geo,
                                       tree.odm_georeferencing_pdal)

                # XYZ point cloud output
                log.ODM_INFO("Creating geo-referenced CSV file (XYZ format, can be used with GRASS to create DEM)")
                with open(tree.odm_georeferencing_xyz_file, "wb") as csvfile:
                    csvfile_writer = csv.writer(csvfile, delimiter=",")
                    reachedpoints = False
                    with open(tree.odm_georeferencing_model_ply_geo) as f:
                        for lineNumber, line in enumerate(f):
                            if reachedpoints:
                                tokens = line.split(" ")
                                csv_line = [float(tokens[0])+geo_ref.utm_east_offset,
                                            float(tokens[1])+geo_ref.utm_north_offset,
                                            tokens[2]]
                                csvfile_writer.writerow(csv_line)
                            if line.startswith("end_header"):
                                reachedpoints = True
                csvfile.close()

        else:
            log.ODM_WARNING('Found a valid georeferenced model in: %s'
                            % tree.odm_georeferencing_model_ply_geo)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Georeferencing')

        log.ODM_INFO('Running ODM Georeferencing Cell - Finished')
        return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT
Exemple #19
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Meshing Cell')

        # get inputs
        args = inputs.args
        tree = inputs.tree
        reconstruction = inputs.reconstruction

        # define paths and create working directories
        system.mkdir_p(tree.odm_meshing)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_meshing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_meshing' in args.rerun_from)

        infile = tree.smvs_model
        if args.fast_orthophoto:
            infile = os.path.join(tree.opensfm, 'reconstruction.ply')
        elif args.use_opensfm_dense:
            infile = tree.opensfm_model

        # Create full 3D model unless --skip-3dmodel is set
        if not args.skip_3dmodel:
            if not io.file_exists(tree.odm_mesh) or rerun_cell:
                log.ODM_DEBUG('Writing ODM Mesh file in: %s' % tree.odm_mesh)

                mesh.screened_poisson_reconstruction(
                    infile,
                    tree.odm_mesh,
                    depth=self.params.oct_tree,
                    samples=self.params.samples,
                    maxVertexCount=self.params.max_vertex,
                    pointWeight=self.params.point_weight,
                    threads=self.params.max_concurrency,
                    verbose=self.params.verbose)

            else:
                log.ODM_WARNING('Found a valid ODM Mesh file in: %s' %
                                tree.odm_mesh)

        # Always generate a 2.5D mesh
        # unless --use-3dmesh is set.
        if not args.use_3dmesh:
            if not io.file_exists(tree.odm_25dmesh) or rerun_cell:

                log.ODM_DEBUG('Writing ODM 2.5D Mesh file in: %s' %
                              tree.odm_25dmesh)
                dsm_resolution = gsd.cap_resolution(
                    args.orthophoto_resolution,
                    tree.opensfm_reconstruction,
                    ignore_gsd=args.ignore_gsd) / 100.0

                # Create reference DSM at half ortho resolution
                dsm_resolution *= 2

                # Sparse point clouds benefits from using
                # a larger resolution value (more radius interolation, less holes)
                if args.fast_orthophoto:
                    dsm_resolution *= 2

                mesh.create_25dmesh(infile,
                                    tree.odm_25dmesh,
                                    dsm_resolution=dsm_resolution,
                                    depth=self.params.oct_tree,
                                    maxVertexCount=self.params.max_vertex,
                                    samples=self.params.samples,
                                    verbose=self.params.verbose,
                                    max_workers=args.max_concurrency)
            else:
                log.ODM_WARNING('Found a valid ODM 2.5D Mesh file in: %s' %
                                tree.odm_25dmesh)

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Meshing')

        log.ODM_INFO('Running ODM Meshing Cell - Finished')
        return ecto.OK if args.end_with != 'odm_meshing' else ecto.QUIT
Exemple #20
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM OpenSfM Cell')

        # get inputs
        tree = inputs.tree
        args = inputs.args
        reconstruction = inputs.reconstruction
        photos = reconstruction.photos

        if not photos:
            log.ODM_ERROR('Not enough photos in photos array to start OpenSfM')
            return ecto.QUIT

        # create working directories
        system.mkdir_p(tree.opensfm)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'opensfm') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'opensfm' in args.rerun_from)

        if args.fast_orthophoto:
            output_file = io.join_paths(tree.opensfm, 'reconstruction.ply')
        elif args.use_opensfm_dense:
            output_file = tree.opensfm_model
        else:
            output_file = tree.opensfm_reconstruction

        # check if reconstruction was done before
        if not io.file_exists(output_file) or rerun_cell:
            # create file list
            list_path = io.join_paths(tree.opensfm, 'image_list.txt')
            has_alt = True
            with open(list_path, 'w') as fout:
                for photo in photos:
                    if not photo.altitude:
                        has_alt = False
                    fout.write('%s\n' %
                               io.join_paths(tree.dataset_raw, photo.filename))

            # create config file for OpenSfM
            config = [
                "use_exif_size: %s" %
                ('no' if not self.params.use_exif_size else 'yes'),
                "feature_process_size: %s" % self.params.feature_process_size,
                "feature_min_frames: %s" % self.params.feature_min_frames,
                "processes: %s" % self.params.processes,
                "matching_gps_neighbors: %s" %
                self.params.matching_gps_neighbors,
                "depthmap_method: %s" % args.opensfm_depthmap_method,
                "depthmap_resolution: %s" % args.depthmap_resolution,
                "depthmap_min_patch_sd: %s" %
                args.opensfm_depthmap_min_patch_sd,
                "depthmap_min_consistent_views: %s" %
                args.opensfm_depthmap_min_consistent_views,
                "optimize_camera_parameters: %s" %
                ('no' if self.params.fixed_camera_params else 'yes')
            ]

            if has_alt:
                log.ODM_DEBUG(
                    "Altitude data detected, enabling it for GPS alignment")
                config.append("use_altitude_tag: yes")
                config.append("align_method: naive")
            else:
                config.append("align_method: orientation_prior")
                config.append("align_orientation_prior: vertical")

            if args.use_hybrid_bundle_adjustment:
                log.ODM_DEBUG("Enabling hybrid bundle adjustment")
                config.append(
                    "bundle_interval: 100"
                )  # Bundle after adding 'bundle_interval' cameras
                config.append(
                    "bundle_new_points_ratio: 1.2"
                )  # Bundle when (new points) / (bundled points) > bundle_new_points_ratio
                config.append(
                    "local_bundle_radius: 1"
                )  # Max image graph distance for images to be included in local bundle adjustment

            if args.matcher_distance > 0:
                config.append("matching_gps_distance: %s" %
                              self.params.matching_gps_distance)

            if tree.odm_georeferencing_gcp:
                config.append("bundle_use_gcp: yes")
                io.copy(tree.odm_georeferencing_gcp, tree.opensfm)

            # write config file
            log.ODM_DEBUG(config)
            config_filename = io.join_paths(tree.opensfm, 'config.yaml')
            with open(config_filename, 'w') as fout:
                fout.write("\n".join(config))

            # run OpenSfM reconstruction
            matched_done_file = io.join_paths(tree.opensfm,
                                              'matching_done.txt')
            if not io.file_exists(matched_done_file) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm extract_metadata %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
                system.run('PYTHONPATH=%s %s/bin/opensfm detect_features %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
                system.run('PYTHONPATH=%s %s/bin/opensfm match_features %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
                with open(matched_done_file, 'w') as fout:
                    fout.write("Matching done!\n")
            else:
                log.ODM_WARNING(
                    'Found a feature matching done progress file in: %s' %
                    matched_done_file)

            if not io.file_exists(tree.opensfm_tracks) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm create_tracks %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
            else:
                log.ODM_WARNING('Found a valid OpenSfM tracks file in: %s' %
                                tree.opensfm_tracks)

            if not io.file_exists(tree.opensfm_reconstruction) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm reconstruct %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
            else:
                log.ODM_WARNING(
                    'Found a valid OpenSfM reconstruction file in: %s' %
                    tree.opensfm_reconstruction)

            # Check that a reconstruction file has been created
            if not io.file_exists(tree.opensfm_reconstruction):
                log.ODM_ERROR(
                    "The program could not process this dataset using the current settings. "
                    "Check that the images have enough overlap, "
                    "that there are enough recognizable features "
                    "and that the images are in focus. "
                    "You could also try to increase the --min-num-features parameter."
                    "The program will now exit.")
                sys.exit(1)

            # Always export VisualSFM's reconstruction and undistort images
            # as we'll use these for texturing (after GSD estimation and resizing)
            if not args.ignore_gsd:
                image_scale = gsd.image_scale_factor(
                    args.orthophoto_resolution, tree.opensfm_reconstruction)
            else:
                image_scale = 1.0

            if not io.file_exists(
                    tree.opensfm_reconstruction_nvm) or rerun_cell:
                system.run(
                    'PYTHONPATH=%s %s/bin/opensfm export_visualsfm --image_extension png --scale_focal %s %s'
                    % (context.pyopencv_path, context.opensfm_path,
                       image_scale, tree.opensfm))
            else:
                log.ODM_WARNING(
                    'Found a valid OpenSfM NVM reconstruction file in: %s' %
                    tree.opensfm_reconstruction_nvm)

            # These will be used for texturing
            system.run(
                'PYTHONPATH=%s %s/bin/opensfm undistort --image_format png --image_scale %s %s'
                % (context.pyopencv_path, context.opensfm_path, image_scale,
                   tree.opensfm))

            # Skip dense reconstruction if necessary and export
            # sparse reconstruction instead
            if args.fast_orthophoto:
                system.run(
                    'PYTHONPATH=%s %s/bin/opensfm export_ply --no-cameras %s' %
                    (context.pyopencv_path, context.opensfm_path,
                     tree.opensfm))
            elif args.use_opensfm_dense:
                # Undistort images at full scale in JPG
                # (TODO: we could compare the size of the PNGs if they are < than depthmap_resolution
                # and use those instead of re-exporting full resolution JPGs)
                system.run('PYTHONPATH=%s %s/bin/opensfm undistort %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
                system.run(
                    'PYTHONPATH=%s %s/bin/opensfm compute_depthmaps %s' %
                    (context.pyopencv_path, context.opensfm_path,
                     tree.opensfm))
        else:
            log.ODM_WARNING(
                'Found a valid OpenSfM reconstruction file in: %s' %
                tree.opensfm_reconstruction)

        # check if reconstruction was exported to bundler before
        if not io.file_exists(tree.opensfm_bundle_list) or rerun_cell:
            # convert back to bundler's format
            system.run(
                'PYTHONPATH=%s %s/bin/export_bundler %s' %
                (context.pyopencv_path, context.opensfm_path, tree.opensfm))
        else:
            log.ODM_WARNING('Found a valid Bundler file in: %s' %
                            tree.opensfm_reconstruction)

        if reconstruction.georef:
            system.run(
                'PYTHONPATH=%s %s/bin/opensfm export_geocoords %s --transformation --proj \'%s\''
                % (context.pyopencv_path, context.opensfm_path, tree.opensfm,
                   reconstruction.georef.projection.srs))

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'OpenSfM')

        log.ODM_INFO('Running ODM OpenSfM Cell - Finished')
        return ecto.OK if args.end_with != 'opensfm' else ecto.QUIT
Exemple #21
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Resize Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        photos = self.inputs.photos

        if not photos:
            log.ODM_ERROR('Not enough photos in photos to resize')
            return ecto.QUIT

        if self.params.resize_to <= 0:
            log.ODM_ERROR('Resize parameter must be greater than 0')
            return ecto.QUIT

        # create working directory
        system.mkdir_p(tree.dataset_resize)

        log.ODM_DEBUG('Resizing dataset to: %s' % tree.dataset_resize)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'resize') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'resize' in args.rerun_from)

        # loop over photos
        for photo in photos:
            # define image paths
            path_file = photo.path_file
            new_path_file = io.join_paths(tree.dataset_resize, photo.filename)
            # set raw image path in case we want to rerun cell
            if io.file_exists(new_path_file) and rerun_cell:
                path_file = io.join_paths(tree.dataset_raw, photo.filename)

            if not io.file_exists(new_path_file) or rerun_cell:
                # open and resize image with opencv
                img = cv2.imread(path_file)
                # compute new size
                max_side = max(img.shape[0], img.shape[1])
                if max_side <= self.params.resize_to:
                    log.ODM_WARNING('Resize Parameter is greater than the largest side of the image')
                ratio = float(self.params.resize_to) / float(max_side)
                img_r = cv2.resize(img, None, fx=ratio, fy=ratio)
                # write image with opencv
                cv2.imwrite(new_path_file, img_r)
                # read metadata with pyexiv2
                old_meta = pyexiv2.ImageMetadata(path_file)
                new_meta = pyexiv2.ImageMetadata(new_path_file)
                old_meta.read()
                new_meta.read()
                # copy metadata
                old_meta.copy(new_meta)
                # update metadata size
                new_meta['Exif.Photo.PixelXDimension'] = img_r.shape[0]
                new_meta['Exif.Photo.PixelYDimension'] = img_r.shape[1]
                new_meta.write()
                # update photos array with new values
                photo.path_file = new_path_file
                photo.width = img_r.shape[0]
                photo.height = img_r.shape[1]
                photo.update_focal()

                # log message
                log.ODM_DEBUG('Resized %s | dimensions: %s' %
                              (photo.filename, img_r.shape))
            else:
                # log message
                log.ODM_WARNING('Already resized %s | dimensions: %s x %s' %
                                (photo.filename, photo.width, photo.height))

        log.ODM_INFO('Resized %s images' % len(photos))

        # append photos to cell output
        self.outputs.photos = photos

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Resizing')

        log.ODM_INFO('Running ODM Resize Cell - Finished')
        return ecto.OK if args.end_with != 'resize' else ecto.QUIT
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Orthophoto Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        reconstruction = inputs.reconstruction
        verbose = '-verbose' if self.params.verbose else ''

        # define paths and create working directories
        system.mkdir_p(tree.odm_orthophoto)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_orthophoto') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_orthophoto' in args.rerun_from)

        if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell:

            # odm_orthophoto definitions
            kwargs = {
                'bin': context.odm_modules_path,
                'log': tree.odm_orthophoto_log,
                'ortho': tree.odm_orthophoto_file,
                'corners': tree.odm_orthophoto_corners,
                'res': 1.0 / (gsd.cap_resolution(self.params.resolution, tree.opensfm_reconstruction, ignore_gsd=args.ignore_gsd) / 100.0),
                'verbose': verbose
            }

            # Have geo coordinates?
            georef = reconstruction.georef

            # Check if the georef object is initialized
            # (during a --rerun this might not be)
            # TODO: we should move this to a more central
            # location (perhaps during the dataset initialization)
            if georef and not georef.utm_east_offset:
                georeferencing_dir = tree.odm_georeferencing if args.use_3dmesh and not args.skip_3dmodel else tree.odm_25dgeoreferencing
                odm_georeferencing_model_txt_geo_file = os.path.join(georeferencing_dir, tree.odm_georeferencing_model_txt_geo)

                if io.file_exists(odm_georeferencing_model_txt_geo_file):
                    georef.extract_offsets(odm_georeferencing_model_txt_geo_file)
                else:
                    log.ODM_WARNING('Cannot read UTM offset from {}. An orthophoto will not be generated.'.format(odm_georeferencing_model_txt_geo_file))


            if georef:
                if args.use_3dmesh:
                    kwargs['model_geo'] = os.path.join(tree.odm_texturing, tree.odm_georeferencing_model_obj_geo)
                else:
                    kwargs['model_geo'] = os.path.join(tree.odm_25dtexturing, tree.odm_georeferencing_model_obj_geo)
            else:
                if args.use_3dmesh:
                    kwargs['model_geo'] = os.path.join(tree.odm_texturing, tree.odm_textured_model_obj)
                else:
                    kwargs['model_geo'] = os.path.join(tree.odm_25dtexturing, tree.odm_textured_model_obj)

            # run odm_orthophoto
            system.run('{bin}/odm_orthophoto -inputFile {model_geo} '
                       '-logFile {log} -outputFile {ortho} -resolution {res} {verbose} '
                       '-outputCornerFile {corners}'.format(**kwargs))

            # Create georeferenced GeoTiff
            geotiffcreated = False

            if georef and georef.projection and georef.utm_east_offset and georef.utm_north_offset:
                ulx = uly = lrx = lry = 0.0
                with open(tree.odm_orthophoto_corners) as f:
                    for lineNumber, line in enumerate(f):
                        if lineNumber == 0:
                            tokens = line.split(' ')
                            if len(tokens) == 4:
                                ulx = float(tokens[0]) + \
                                    float(georef.utm_east_offset)
                                lry = float(tokens[1]) + \
                                    float(georef.utm_north_offset)
                                lrx = float(tokens[2]) + \
                                    float(georef.utm_east_offset)
                                uly = float(tokens[3]) + \
                                    float(georef.utm_north_offset)
                log.ODM_INFO('Creating GeoTIFF')

                kwargs = {
                    'ulx': ulx,
                    'uly': uly,
                    'lrx': lrx,
                    'lry': lry,
                    'tiled': '' if self.params.no_tiled else '-co TILED=yes ',
                    'compress': self.params.compress,
                    'predictor': '-co PREDICTOR=2 ' if self.params.compress in
                                                       ['LZW', 'DEFLATE'] else '',
                    'proj': georef.projection.srs,
                    'bigtiff': self.params.bigtiff,
                    'png': tree.odm_orthophoto_file,
                    'tiff': tree.odm_orthophoto_tif,
                    'log': tree.odm_orthophoto_tif_log,
                    'max_memory': get_max_memory(),
                    'threads': self.params.max_concurrency
                }

                system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} '
                           '{tiled} '
                           '-co BIGTIFF={bigtiff} '
                           '-co COMPRESS={compress} '
                           '{predictor} '
                           '-co BLOCKXSIZE=512 '
                           '-co BLOCKYSIZE=512 '
                           '-co NUM_THREADS={threads} '
                           '-a_srs \"{proj}\" '
                           '--config GDAL_CACHEMAX {max_memory}% '
                           '{png} {tiff} > {log}'.format(**kwargs))

                if args.crop > 0:
                    shapefile_path = os.path.join(tree.odm_georeferencing, 'odm_georeferenced_model.bounds.shp')
                    Cropper.crop(shapefile_path, tree.odm_orthophoto_tif, {
                            'TILED': 'NO' if self.params.no_tiled else 'YES',
                            'COMPRESS': self.params.compress,
                            'PREDICTOR': '2' if self.params.compress in ['LZW', 'DEFLATE'] else '1',
                            'BIGTIFF': self.params.bigtiff,
                            'BLOCKXSIZE': 512,
                            'BLOCKYSIZE': 512,
                            'NUM_THREADS': self.params.max_concurrency
                        })

                if self.params.build_overviews:
                    log.ODM_DEBUG("Building Overviews")
                    kwargs = {
                        'orthophoto': tree.odm_orthophoto_tif,
                        'log': tree.odm_orthophoto_gdaladdo_log
                    }
                    # Run gdaladdo
                    system.run('gdaladdo -ro -r average '
                               '--config BIGTIFF_OVERVIEW IF_SAFER '
                               '--config COMPRESS_OVERVIEW JPEG '
                               '{orthophoto} 2 4 8 16 > {log}'.format(**kwargs))

                geotiffcreated = True
            if not geotiffcreated:
                log.ODM_WARNING('No geo-referenced orthophoto created due '
                                'to missing geo-referencing or corner coordinates.')

        else:
            log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_file)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Orthophoto')

        log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished')
        return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT
Exemple #23
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running OMD OrthoPhoto Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree

        # define paths and create working directories
        system.mkdir_p(tree.odm_orthophoto)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_orthophoto') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_orthophoto' in args.rerun_from)

        if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell:

            # odm_orthophoto definitions
            kwargs = {
                'bin': context.odm_modules_path,
                'model_geo': tree.odm_georeferencing_model_obj_geo,
                'log': tree.odm_orthophoto_log,
                'ortho': tree.odm_orthophoto_file,
                'corners': tree.odm_orthophoto_corners,
                'res': self.params.resolution
            }

            # run odm_orthophoto
            system.run('{bin}/odm_orthophoto -inputFile {model_geo} '
                       '-logFile {log} -outputFile {ortho} -resolution {res} '
                       '-outputCornerFile {corners}'.format(**kwargs))

            # Create georeferenced GeoTiff
            geotiffcreated = False
            georef = types.ODM_GeoRef()
            # creates the coord refs # TODO I don't want to have to do this twice- after odm_georef
            georef.parse_coordinate_system(tree.odm_georeferencing_coords)

            if georef.epsg and georef.utm_east_offset and georef.utm_north_offset:
                ulx = uly = lrx = lry = 0.0
                with open(tree.odm_orthophoto_corners) as f:
                    for lineNumber, line in enumerate(f):
                        if lineNumber == 0:
                            tokens = line.split(' ')
                            if len(tokens) == 4:
                                ulx = float(tokens[0]) + \
                                    float(georef.utm_east_offset)
                                lry = float(tokens[1]) + \
                                    float(georef.utm_north_offset)
                                lrx = float(tokens[2]) + \
                                    float(georef.utm_east_offset)
                                uly = float(tokens[3]) + \
                                    float(georef.utm_north_offset)
                log.ODM_INFO('Creating GeoTIFF')

                kwargs = {
                    'ulx': ulx,
                    'uly': uly,
                    'lrx': lrx,
                    'lry': lry,
                    'epsg': georef.epsg,
                    'png': tree.odm_orthophoto_file,
                    'tiff': tree.odm_orthophoto_tif,
                    'log': tree.odm_orthophoto_tif_log
                }

                system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} '
                           '-a_srs \"EPSG:{epsg}\" {png} {tiff} > {log}'.format(**kwargs))
                geotiffcreated = True
            if not geotiffcreated:
                log.ODM_WARNING('No geo-referenced orthophoto created due '
                                'to missing geo-referencing or corner coordinates.')

        else:
            log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_file)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Orthophoto')

        log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished')
        return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT
Exemple #24
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running MVS Texturing Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree

        # define paths and create working directories
        system.mkdir_p(tree.odm_texturing)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'mvs_texturing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'mvs_texturing' in args.rerun_from)

        if not io.file_exists(tree.odm_textured_model_obj) or rerun_cell:
            log.ODM_DEBUG('Writing MVS Textured file in: %s'
                          % tree.odm_textured_model_obj)
            
            
            # Format arguments to fit Mvs-Texturing app
            skipGeometricVisibilityTest = ""
            skipGlobalSeamLeveling = ""
            skipLocalSeamLeveling = ""
            skipHoleFilling = ""
            keepUnseenFaces = ""
            
            if (self.params.skip_vis_test):
                skipGeometricVisibilityTest = "--skip_geometric_visibility_test"
            if (self.params.skip_glob_seam_leveling):
                skipGlobalSeamLeveling = "--skip_global_seam_leveling"
            if (self.params.skip_loc_seam_leveling):
                skipLocalSeamLeveling = "--skip_local_seam_leveling"
            if (self.params.skip_hole_fill):
                skipHoleFilling = "--skip_hole_filling"
            if (self.params.keep_unseen_faces):
                keepUnseenFaces = "--keep_unseen_faces"
            
            # mvstex definitions
            kwargs = {
                'bin': context.mvstex_path,
                'out_dir': io.join_paths(tree.odm_texturing, "odm_textured_model"),
                'pmvs_folder': tree.pmvs_rec_path,
                'nvm_file': io.join_paths(tree.pmvs_rec_path, "nvmCams.nvm"),
                'model': tree.odm_mesh,
                'dataTerm': self.params.data_term,
                'outlierRemovalType': self.params.outlier_rem_type,
                'skipGeometricVisibilityTest': skipGeometricVisibilityTest,
                'skipGlobalSeamLeveling': skipGlobalSeamLeveling,
                'skipLocalSeamLeveling': skipLocalSeamLeveling,
                'skipHoleFilling': skipHoleFilling,
                'keepUnseenFaces': keepUnseenFaces
            }

            if args.use_opensfm_pointcloud:
                kwargs['nvm_file'] = io.join_paths(tree.opensfm,
                                                   "reconstruction.nvm")
            else:
                log.ODM_DEBUG('Generating .nvm file from pmvs output: %s'
                              % '{nvm_file}'.format(**kwargs))

                # Create .nvm camera file.
                pmvs2nvmcams.run('{pmvs_folder}'.format(**kwargs),
                                 '{nvm_file}'.format(**kwargs))

            # run texturing binary
            system.run('{bin} {nvm_file} {model} {out_dir} '
                       '-d {dataTerm} -o {outlierRemovalType} '
                       '{skipGeometricVisibilityTest} '
                       '{skipGlobalSeamLeveling} '
                       '{skipLocalSeamLeveling} '
                       '{skipHoleFilling} '
                       '{keepUnseenFaces}'.format(**kwargs))
        else:
            log.ODM_WARNING('Found a valid ODM Texture file in: %s'
                            % tree.odm_textured_model_obj)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Texturing')

        log.ODM_INFO('Running ODM Texturing Cell - Finished')
        return ecto.OK if args.end_with != 'odm_texturing' else ecto.QUIT
Exemple #25
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Meshing Cell')

        # get inputs
        args = inputs.args
        tree = inputs.tree
        reconstruction = inputs.reconstruction

        # define paths and create working directories
        system.mkdir_p(tree.odm_meshing)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_meshing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_meshing' in args.rerun_from)

        infile = tree.smvs_model
        if args.fast_orthophoto:
            infile = os.path.join(tree.opensfm, 'reconstruction.ply')
        elif args.use_opensfm_dense:
            infile = tree.opensfm_model

        # Create full 3D model unless --skip-3dmodel is set
        if not args.skip_3dmodel:
            if not io.file_exists(tree.odm_mesh) or rerun_cell:
                log.ODM_DEBUG('Writing ODM Mesh file in: %s' % tree.odm_mesh)

                mesh.screened_poisson_reconstruction(
                    infile,
                    tree.odm_mesh,
                    depth=self.params.oct_tree,
                    samples=self.params.samples,
                    maxVertexCount=self.params.max_vertex,
                    pointWeight=self.params.point_weight,
                    threads=self.params.max_concurrency,
                    verbose=self.params.verbose)

            else:
                log.ODM_WARNING('Found a valid ODM Mesh file in: %s' %
                                tree.odm_mesh)

        # Always generate a 2.5D mesh
        # unless --use-3dmesh is set.
        if not args.use_3dmesh:
            if not io.file_exists(tree.odm_25dmesh) or rerun_cell:

                log.ODM_DEBUG('Writing ODM 2.5D Mesh file in: %s' %
                              tree.odm_25dmesh)
                ortho_resolution = gsd.cap_resolution(
                    args.orthophoto_resolution,
                    tree.opensfm_reconstruction,
                    ignore_gsd=args.ignore_gsd) / 100.0

                dsm_multiplier = max(
                    1.0,
                    gsd.rounded_gsd(tree.opensfm_reconstruction,
                                    default_value=4,
                                    ndigits=3,
                                    ignore_gsd=args.ignore_gsd))

                # A good DSM size depends on the flight altitude.
                # Flights at low altitude need more details (higher resolution)
                # Flights at higher altitude benefit from smoother surfaces (lower resolution)
                dsm_resolution = ortho_resolution * dsm_multiplier

                dsm_radius = dsm_resolution * math.sqrt(2)

                # Sparse point clouds benefits from using
                # a larger radius interolation --> less holes
                if args.fast_orthophoto:
                    dsm_radius *= 2

                log.ODM_DEBUG('ODM 2.5D DSM resolution: %s' % dsm_resolution)

                mesh.create_25dmesh(
                    infile,
                    tree.odm_25dmesh,
                    dsm_radius=dsm_radius,
                    dsm_resolution=dsm_resolution,
                    depth=self.params.oct_tree,
                    maxVertexCount=self.params.max_vertex,
                    samples=self.params.samples,
                    verbose=self.params.verbose,
                    max_workers=args.max_concurrency,
                    method='poisson' if args.fast_orthophoto else 'gridded')
            else:
                log.ODM_WARNING('Found a valid ODM 2.5D Mesh file in: %s' %
                                tree.odm_25dmesh)

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Meshing')

        log.ODM_INFO('Running ODM Meshing Cell - Finished')
        return ecto.OK if args.end_with != 'odm_meshing' else ecto.QUIT
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Orthophoto Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        verbose = '-verbose' if self.params.verbose else ''

        # define paths and create working directories
        system.mkdir_p(tree.odm_orthophoto)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_orthophoto') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_orthophoto' in args.rerun_from)

        if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell:

            # odm_orthophoto definitions
            kwargs = {
                'bin': context.odm_modules_path,
                'log': tree.odm_orthophoto_log,
                'ortho': tree.odm_orthophoto_file,
                'corners': tree.odm_orthophoto_corners,
                'res': self.params.resolution,
                'verbose': verbose
            }

            kwargs['model_geo'] = tree.odm_georeferencing_model_obj_geo \
                if io.file_exists(tree.odm_georeferencing_coords) \
                else tree.odm_textured_model_obj


            # run odm_orthophoto
            system.run('{bin}/odm_orthophoto -inputFile {model_geo} '
                       '-logFile {log} -outputFile {ortho} -resolution {res} {verbose} '
                       '-outputCornerFile {corners}'.format(**kwargs))

            if not io.file_exists(tree.odm_georeferencing_coords):
                log.ODM_WARNING('No coordinates file. A georeferenced raster '
                                'will not be created')
            else:
                # Create georeferenced GeoTiff
                geotiffcreated = False
                georef = types.ODM_GeoRef()
                # creates the coord refs # TODO I don't want to have to do this twice- after odm_georef
                georef.parse_coordinate_system(tree.odm_georeferencing_coords)

                if georef.epsg and georef.utm_east_offset and georef.utm_north_offset:
                    ulx = uly = lrx = lry = 0.0
                    with open(tree.odm_orthophoto_corners) as f:
                        for lineNumber, line in enumerate(f):
                            if lineNumber == 0:
                                tokens = line.split(' ')
                                if len(tokens) == 4:
                                    ulx = float(tokens[0]) + \
                                        float(georef.utm_east_offset)
                                    lry = float(tokens[1]) + \
                                        float(georef.utm_north_offset)
                                    lrx = float(tokens[2]) + \
                                        float(georef.utm_east_offset)
                                    uly = float(tokens[3]) + \
                                        float(georef.utm_north_offset)
                    log.ODM_INFO('Creating GeoTIFF')

                    kwargs = {
                        'ulx': ulx,
                        'uly': uly,
                        'lrx': lrx,
                        'lry': lry,
                        'tiled': '' if self.params.no_tiled else '-co TILED=yes ',
                        'compress': self.params.compress,
                        'predictor': '-co PREDICTOR=2 ' if self.params.compress in
                                                           ['LZW', 'DEFLATE'] else '',
                        'epsg': georef.epsg,
                        't_srs': self.params.t_srs or "EPSG:{0}".format(georef.epsg),
                        'bigtiff': self.params.bigtiff,
                        'png': tree.odm_orthophoto_file,
                        'tiff': tree.odm_orthophoto_tif,
                        'log': tree.odm_orthophoto_tif_log
                    }

                    system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} '
                               '{tiled} '
                               '-co BIGTIFF={bigtiff} '
                               '-co COMPRESS={compress} '
                               '{predictor} '
                               '-co BLOCKXSIZE=512 '
                               '-co BLOCKYSIZE=512 '
                               '-co NUM_THREADS=ALL_CPUS '
                               '-a_srs \"EPSG:{epsg}\" '
                               '{png} {tiff} > {log}'.format(**kwargs))

                    if self.params.build_overviews:
                        log.ODM_DEBUG("Building Overviews")
                        kwargs = {
                            'orthophoto': tree.odm_orthophoto_tif,
                            'log': tree.odm_orthophoto_gdaladdo_log
                        }
                        # Run gdaladdo
                        system.run('gdaladdo -ro -r average '
                                   '--config BIGTIFF_OVERVIEW IF_SAFER '
                                   '--config COMPRESS_OVERVIEW JPEG '
                                   '{orthophoto} 2 4 8 16 > {log}'.format(**kwargs))

                    geotiffcreated = True
                if not geotiffcreated:
                    log.ODM_WARNING('No geo-referenced orthophoto created due '
                                    'to missing geo-referencing or corner coordinates.')

        else:
            log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_file)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Orthophoto')

        log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished')
        return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT
Exemple #27
0
    def process(self, inputs, outputs):
        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM DEM Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        las_model_found = io.file_exists(tree.odm_georeferencing_model_las)
        env_paths = [context.superbuild_bin_path]

        # Just to make sure
        l2d_module_installed = True
        try:
            system.run('l2d_classify --help > /dev/null', env_paths)
        except:
            log.ODM_WARNING('lidar2dems is not installed properly')
            l2d_module_installed = False

        log.ODM_INFO('Create DSM: ' + str(args.dsm))
        log.ODM_INFO('Create DTM: ' + str(args.dtm))
        log.ODM_INFO('DEM input file {0} found: {1}'.format(tree.odm_georeferencing_model_las, str(las_model_found)))

        # Do we need to process anything here?
        if (args.dsm or args.dtm) and las_model_found and l2d_module_installed:

            # define paths and create working directories
            odm_dem_root = tree.path('odm_dem')
            system.mkdir_p(odm_dem_root)

            dsm_output_filename = os.path.join(odm_dem_root, 'dsm.tif')
            dtm_output_filename = os.path.join(odm_dem_root, 'dtm.tif')

            # check if we rerun cell or not
            rerun_cell = (args.rerun is not None and
                          args.rerun == 'odm_dem') or \
                         (args.rerun_all) or \
                         (args.rerun_from is not None and
                          'odm_dem' in args.rerun_from)

            if (args.dtm and not io.file_exists(dtm_output_filename)) or \
                (args.dsm and not io.file_exists(dsm_output_filename)) or \
                rerun_cell:

                # Extract boundaries and srs of point cloud
                summary_file_path = os.path.join(odm_dem_root, 'odm_georeferenced_model.summary.json')
                boundary_file_path = os.path.join(odm_dem_root, 'odm_georeferenced_model.boundary.json')

                system.run('pdal info --summary {0} > {1}'.format(tree.odm_georeferencing_model_las, summary_file_path), env_paths)
                system.run('pdal info --boundary {0} > {1}'.format(tree.odm_georeferencing_model_las,  boundary_file_path), env_paths)

                pc_proj4 = ""
                pc_geojson_bounds_feature = None

                with open(summary_file_path, 'r') as f:
                    json_f = json.loads(f.read())
                    pc_proj4 = json_f['summary']['srs']['proj4']

                with open(boundary_file_path, 'r') as f:
                    json_f = json.loads(f.read())
                    pc_geojson_boundary_feature = json_f['boundary']['boundary_json']

                # Write bounds to GeoJSON
                bounds_geojson_path = os.path.join(odm_dem_root, 'odm_georeferenced_model.bounds.geojson')
                with open(bounds_geojson_path, "w") as f:
                    f.write(json.dumps({
                        "type": "FeatureCollection",
                        "features": [{
                            "type": "Feature",
                            "geometry": pc_geojson_boundary_feature
                        }]
                    }))

                bounds_shapefile_path = os.path.join(odm_dem_root, 'bounds.shp')

                # Convert bounds to Shapefile
                kwargs = {
                    'input': bounds_geojson_path,
                    'output': bounds_shapefile_path,
                    'proj4': pc_proj4
                }
                system.run('ogr2ogr -overwrite -a_srs "{proj4}" {output} {input}'.format(**kwargs))

                # Process with lidar2dems
                terrain_params_map = {
                    'flatnonforest': (1, 3), 
                    'flatforest': (1, 2), 
                    'complexnonforest': (5, 2), 
                    'complexforest': (10, 2)
                }
                terrain_params = terrain_params_map[args.dem_terrain_type.lower()]             

                kwargs = {
                    'verbose': '-v' if self.params.verbose else '',
                    'slope': terrain_params[0],
                    'cellsize': terrain_params[1],
                    'outdir': odm_dem_root,
                    'site': bounds_shapefile_path
                }

                l2d_params = '--slope {slope} --cellsize {cellsize} ' \
                             '{verbose} ' \
                             '-o -s {site} ' \
                             '--outdir {outdir}'.format(**kwargs)

                approximate = '--approximate' if args.dem_approximate else ''

                # Classify only if we need a DTM
                run_classification = args.dtm

                if run_classification:
                    system.run('l2d_classify {0} --decimation {1} '
                               '{2} --initialDistance {3} {4}'.format(
                        l2d_params, args.dem_decimation, approximate, 
                        args.dem_initial_distance, tree.odm_georeferencing), env_paths)
                else:
                    log.ODM_INFO("Will skip classification, only DSM is needed")
                    copyfile(tree.odm_georeferencing_model_las, os.path.join(odm_dem_root, 'bounds-0_l2d_s{slope}c{cellsize}.las'.format(**kwargs)))

                products = []
                if args.dsm: products.append('dsm') 
                if args.dtm: products.append('dtm')

                radius_steps = [args.dem_resolution]
                for _ in range(args.dem_gapfill_steps - 1):
                    radius_steps.append(radius_steps[-1] * 3) # 3 is arbitrary, maybe there's a better value?

                for product in products:
                    demargs = {
                        'product': product,
                        'indir': odm_dem_root,
                        'l2d_params': l2d_params,
                        'maxsd': args.dem_maxsd,
                        'maxangle': args.dem_maxangle,
                        'resolution': args.dem_resolution,
                        'radius_steps': ' '.join(map(str, radius_steps)),
                        'gapfill': '--gapfill' if args.dem_gapfill_steps > 0 else '',
                        
                        # If we didn't run a classification, we should pass the decimate parameter here
                        'decimation': '--decimation {0}'.format(args.dem_decimation) if not run_classification else ''
                    }

                    system.run('l2d_dems {product} {indir} {l2d_params} '
                               '--maxsd {maxsd} --maxangle {maxangle} '
                               '--resolution {resolution} --radius {radius_steps} '
                               '{decimation} '
                               '{gapfill} '.format(**demargs), env_paths)

                    # Rename final output
                    if product == 'dsm':
                        os.rename(os.path.join(odm_dem_root, 'bounds-0_dsm.idw.tif'), dsm_output_filename)
                    elif product == 'dtm':
                        os.rename(os.path.join(odm_dem_root, 'bounds-0_dtm.idw.tif'), dtm_output_filename)

            else:
                log.ODM_WARNING('Found existing outputs in: %s' % odm_dem_root)
        else:
            log.ODM_WARNING('DEM will not be generated')

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Dem')

        log.ODM_INFO('Running ODM DEM Cell - Finished')
        return ecto.OK if args.end_with != 'odm_dem' else ecto.QUIT
Exemple #28
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Georeferencing Cell')

        # get inputs
        args = inputs.args
        tree = inputs.tree
        reconstruction = inputs.reconstruction
        gcpfile = tree.odm_georeferencing_gcp
        doPointCloudGeo = True
        transformPointCloud = True
        verbose = '-verbose' if self.params.verbose else ''
        geo_ref = reconstruction.georef

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_georeferencing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_georeferencing' in args.rerun_from)

        runs = [{
            'georeferencing_dir':
            tree.odm_georeferencing,
            'texturing_dir':
            tree.odm_texturing,
            'model':
            os.path.join(tree.odm_texturing, tree.odm_textured_model_obj)
        }]

        if args.skip_3dmodel:
            runs = []

        if not args.use_3dmesh:
            # Make sure 2.5D mesh is georeferenced before the 3D mesh
            # Because it will be used to calculate a transform
            # for the point cloud. If we use the 3D model transform,
            # DEMs and orthophoto might not align!
            runs.insert(
                0, {
                    'georeferencing_dir':
                    tree.odm_25dgeoreferencing,
                    'texturing_dir':
                    tree.odm_25dtexturing,
                    'model':
                    os.path.join(tree.odm_25dtexturing,
                                 tree.odm_textured_model_obj)
                })

        for r in runs:
            odm_georeferencing_model_obj_geo = os.path.join(
                r['texturing_dir'], tree.odm_georeferencing_model_obj_geo)
            odm_georeferencing_log = os.path.join(r['georeferencing_dir'],
                                                  tree.odm_georeferencing_log)
            odm_georeferencing_transform_file = os.path.join(
                r['georeferencing_dir'],
                tree.odm_georeferencing_transform_file)
            odm_georeferencing_model_txt_geo_file = os.path.join(
                r['georeferencing_dir'], tree.odm_georeferencing_model_txt_geo)

            if not io.file_exists(odm_georeferencing_model_obj_geo) or \
               not io.file_exists(tree.odm_georeferencing_model_laz) or rerun_cell:

                # odm_georeference definitions
                kwargs = {
                    'bin': context.odm_modules_path,
                    'input_pc_file': tree.filtered_point_cloud,
                    'bundle': tree.opensfm_bundle,
                    'imgs': tree.dataset_raw,
                    'imgs_list': tree.opensfm_bundle_list,
                    'model': r['model'],
                    'log': odm_georeferencing_log,
                    'input_trans_file': tree.opensfm_transformation,
                    'transform_file': odm_georeferencing_transform_file,
                    'coords': tree.odm_georeferencing_coords,
                    'output_pc_file': tree.odm_georeferencing_model_laz,
                    'geo_sys': odm_georeferencing_model_txt_geo_file,
                    'model_geo': odm_georeferencing_model_obj_geo,
                    'gcp': gcpfile,
                    'verbose': verbose
                }

                if transformPointCloud:
                    kwargs[
                        'pc_params'] = '-inputPointCloudFile {input_pc_file} -outputPointCloudFile {output_pc_file}'.format(
                            **kwargs)

                    if geo_ref and geo_ref.projection and geo_ref.projection.srs:
                        kwargs[
                            'pc_params'] += ' -outputPointCloudSrs %s' % pipes.quote(
                                geo_ref.projection.srs)
                    else:
                        log.ODM_WARNING(
                            'NO SRS: The output point cloud will not have a SRS.'
                        )
                else:
                    kwargs['pc_params'] = ''

                # Check to see if the GCP file exists

                if not self.params.use_exif and (self.params.gcp_file or
                                                 tree.odm_georeferencing_gcp):
                    log.ODM_INFO('Found %s' % gcpfile)
                    try:
                        system.run(
                            '{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} '
                            '-inputFile {model} -outputFile {model_geo} '
                            '{pc_params} {verbose} '
                            '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys} -gcpFile {gcp} '
                            '-outputCoordFile {coords}'.format(**kwargs))
                    except Exception:
                        log.ODM_EXCEPTION('Georeferencing failed. ')
                        return ecto.QUIT
                elif io.file_exists(
                        tree.opensfm_transformation) and io.file_exists(
                            tree.odm_georeferencing_coords):
                    log.ODM_INFO(
                        'Running georeferencing with OpenSfM transformation matrix'
                    )
                    system.run(
                        '{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} -inputCoordFile {coords} '
                        '-inputFile {model} -outputFile {model_geo} '
                        '{pc_params} {verbose} '
                        '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'
                        .format(**kwargs))
                elif io.file_exists(tree.odm_georeferencing_coords):
                    log.ODM_INFO(
                        'Running georeferencing with generated coords file.')
                    system.run(
                        '{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} '
                        '-inputFile {model} -outputFile {model_geo} '
                        '{pc_params} {verbose} '
                        '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'
                        .format(**kwargs))
                else:
                    log.ODM_WARNING(
                        'Georeferencing failed. Make sure your '
                        'photos have geotags in the EXIF or you have '
                        'provided a GCP file. ')
                    doPointCloudGeo = False  # skip the rest of the georeferencing

                if doPointCloudGeo:
                    # update images metadata
                    geo_ref.extract_offsets(
                        odm_georeferencing_model_txt_geo_file)
                    reconstruction.georef = geo_ref

                    # XYZ point cloud output
                    if args.pc_csv:
                        log.ODM_INFO(
                            "Creating geo-referenced CSV file (XYZ format)")

                        system.run(
                            "pdal translate -i \"{}\" "
                            "-o \"{}\" "
                            "--writers.text.format=csv "
                            "--writers.text.order=\"X,Y,Z\" "
                            "--writers.text.keep_unspecified=false ".format(
                                tree.odm_georeferencing_model_laz,
                                tree.odm_georeferencing_xyz_file))

                    # LAS point cloud output
                    if args.pc_las:
                        log.ODM_INFO("Creating geo-referenced LAS file")

                        system.run("pdal translate -i \"{}\" "
                                   "-o \"{}\" ".format(
                                       tree.odm_georeferencing_model_laz,
                                       tree.odm_georeferencing_model_las))

                    if args.crop > 0:
                        log.ODM_INFO(
                            "Calculating cropping area and generating bounds shapefile from point cloud"
                        )
                        cropper = Cropper(tree.odm_georeferencing,
                                          'odm_georeferenced_model')

                        decimation_step = 40 if args.fast_orthophoto or args.use_opensfm_dense else 90

                        # More aggressive decimation for large datasets
                        if not args.fast_orthophoto:
                            decimation_step *= int(
                                len(reconstruction.photos) / 1000) + 1

                        cropper.create_bounds_shapefile(
                            tree.odm_georeferencing_model_laz,
                            args.crop,
                            decimation_step=decimation_step,
                            outlier_radius=20 if args.fast_orthophoto else 2)

                    # Do not execute a second time, since
                    # We might be doing georeferencing for
                    # multiple models (3D, 2.5D, ...)
                    doPointCloudGeo = False
                    transformPointCloud = False
            else:
                log.ODM_WARNING('Found a valid georeferenced model in: %s' %
                                tree.odm_georeferencing_model_laz)

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Georeferencing')

        log.ODM_INFO('Running ODM Georeferencing Cell - Finished')
        return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT
Exemple #29
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM OpenSfM Cell')

        # get inputs
        tree = self.inputs.tree
        args = self.inputs.args
        photos = self.inputs.photos

        if not photos:
            log.ODM_ERROR('Not enough photos in photos array to start OpenSfM')
            return ecto.QUIT

        # create working directories
        system.mkdir_p(tree.opensfm)
        system.mkdir_p(tree.pmvs)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'opensfm') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'opensfm' in args.rerun_from)

        if not args.use_pmvs:
            output_file = tree.opensfm_model
        else:
            output_file = tree.opensfm_reconstruction

        # check if reconstruction was done before
        if not io.file_exists(output_file) or rerun_cell:
            # create file list
            list_path = io.join_paths(tree.opensfm, 'image_list.txt')
            with open(list_path, 'w') as fout:
                for photo in photos:
                    fout.write('%s\n' % photo.path_file)

            # create config file for OpenSfM
            config = [
                "use_exif_size: %s" %
                ('no' if not self.params.use_exif_size else 'yes'),
                "feature_process_size: %s" % self.params.feature_process_size,
                "feature_min_frames: %s" % self.params.feature_min_frames,
                "processes: %s" % self.params.processes,
                "matching_gps_neighbors: %s" %
                self.params.matching_gps_neighbors
            ]

            if args.matcher_distance > 0:
                config.append("matching_gps_distance: %s" %
                              self.params.matching_gps_distance)

            # write config file
            config_filename = io.join_paths(tree.opensfm, 'config.yaml')
            with open(config_filename, 'w') as fout:
                fout.write("\n".join(config))

            # run OpenSfM reconstruction
            system.run(
                'PYTHONPATH=%s %s/bin/run_all %s' %
                (context.pyopencv_path, context.opensfm_path, tree.opensfm))
            if not args.use_pmvs:
                system.run('PYTHONPATH=%s %s/bin/opensfm export_visualsfm %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
                system.run('PYTHONPATH=%s %s/bin/opensfm undistort %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
                system.run(
                    'PYTHONPATH=%s %s/bin/opensfm compute_depthmaps %s' %
                    (context.pyopencv_path, context.opensfm_path,
                     tree.opensfm))
        else:
            log.ODM_WARNING('Found a valid OpenSfM file in: %s' %
                            tree.opensfm_reconstruction)

        # check if reconstruction was exported to bundler before
        if not io.file_exists(tree.opensfm_bundle_list) or rerun_cell:
            # convert back to bundler's format
            system.run(
                'PYTHONPATH=%s %s/bin/export_bundler %s' %
                (context.pyopencv_path, context.opensfm_path, tree.opensfm))
        else:
            log.ODM_WARNING('Found a valid Bundler file in: %s' %
                            tree.opensfm_reconstruction)

        if args.use_pmvs:
            # check if reconstruction was exported to pmvs before
            if not io.file_exists(tree.pmvs_visdat) or rerun_cell:
                # run PMVS converter
                system.run('PYTHONPATH=%s %s/bin/export_pmvs %s --output %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm, tree.pmvs))
            else:
                log.ODM_WARNING('Found a valid CMVS file in: %s' %
                                tree.pmvs_visdat)

            if args.time:
                system.benchmark(start_time, tree.benchmarking, 'OpenSfM')

        log.ODM_INFO('Running ODM OpenSfM Cell - Finished')
        return ecto.OK if args.end_with != 'opensfm' else ecto.QUIT
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Texturing Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        verbose = '-verbose' if self.params.verbose else ''

        # define paths and create working directories
        system.mkdir_p(tree.odm_texturing)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_texturing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_texturing' in args.rerun_from)

        # Undistort radial distortion
        if not os.path.isdir(tree.odm_texturing_undistorted_image_path) or rerun_cell:
            system.run(' '.join([
                'cd {} &&'.format(tree.opensfm),
                'PYTHONPATH={}:{}'.format(context.pyopencv_path,
                                          context.opensfm_path),
                'python',
                os.path.join(context.odm_modules_src_path,
                             'odm_slam/src/undistort_radial.py'),
                '--output',
                tree.odm_texturing_undistorted_image_path,
                tree.opensfm,
            ]))

            system.run(
                'PYTHONPATH=%s %s/bin/export_bundler %s' %
                (context.pyopencv_path, context.opensfm_path, tree.opensfm))
        else:
            log.ODM_WARNING(
                'Found a valid Bundler file in: %s' %
                (tree.opensfm_reconstruction))

        if not io.file_exists(tree.odm_textured_model_obj) or rerun_cell:
            log.ODM_DEBUG('Writing ODM Textured file in: %s'
                          % tree.odm_textured_model_obj)

            # odm_texturing definitions
            kwargs = {
                'bin': context.odm_modules_path,
                'out_dir': tree.odm_texturing,
                'bundle': tree.opensfm_bundle,
                'imgs_path': tree.odm_texturing_undistorted_image_path,
                'imgs_list': tree.opensfm_bundle_list,
                'model': tree.odm_mesh,
                'log': tree.odm_texuring_log,
                'resize': self.params.resize,
                'resolution': self.params.resolution,
                'size': self.params.size,
                'verbose': verbose
            }

            # run texturing binary
            system.run('{bin}/odm_texturing -bundleFile {bundle} '
                       '-imagesPath {imgs_path} -imagesListPath {imgs_list} '
                       '-inputModelPath {model} -outputFolder {out_dir}/ '
                       '-textureResolution {resolution} -bundleResizedTo {resize} {verbose} '
                       '-textureWithSize {size} -logFile {log}'.format(**kwargs))
        else:
            log.ODM_WARNING('Found a valid ODM Texture file in: %s'
                            % tree.odm_textured_model_obj)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Texturing')

        log.ODM_INFO('Running ODM Texturing Cell - Finished')
        return ecto.OK if args.end_with != 'odm_texturing' else ecto.QUIT
Exemple #31
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM OpenSfM Cell')

        # get inputs
        tree = self.inputs.tree
        args = self.inputs.args
        photos = self.inputs.photos

        if not photos:
            log.ODM_ERROR('Not enough photos in photos array to start OpenSfM')
            return ecto.QUIT

        # create working directories
        system.mkdir_p(tree.opensfm)
        system.mkdir_p(tree.pmvs)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'opensfm') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'opensfm' in args.rerun_from)

        if not args.use_pmvs:
            output_file = tree.opensfm_model
        else:
            output_file = tree.opensfm_reconstruction

        # check if reconstruction was done before
        if not io.file_exists(output_file) or rerun_cell:
            # create file list
            list_path = io.join_paths(tree.opensfm, 'image_list.txt')
            has_alt = True
            with open(list_path, 'w') as fout:
                for photo in photos:
                    if not photo.altitude:
                        has_alt = False
                    fout.write('%s\n' % photo.path_file)

            # create config file for OpenSfM
            config = [
                "use_exif_size: %s" %
                ('no' if not self.params.use_exif_size else 'yes'),
                "feature_process_size: %s" % self.params.feature_process_size,
                "feature_min_frames: %s" % self.params.feature_min_frames,
                "processes: %s" % self.params.processes,
                "matching_gps_neighbors: %s" %
                self.params.matching_gps_neighbors,
                "optimize_camera_parameters: %s" %
                ('no' if self.params.fixed_camera_params else 'yes')
            ]

            if has_alt:
                log.ODM_DEBUG(
                    "Altitude data detected, enabling it for GPS alignment")
                config.append("use_altitude_tag: True")
                config.append("align_method: naive")

            if args.matcher_distance > 0:
                config.append("matching_gps_distance: %s" %
                              self.params.matching_gps_distance)

            # write config file
            config_filename = io.join_paths(tree.opensfm, 'config.yaml')
            with open(config_filename, 'w') as fout:
                fout.write("\n".join(config))

            # run OpenSfM reconstruction
            matched_done_file = io.join_paths(tree.opensfm,
                                              'matching_done.txt')
            if not io.file_exists(matched_done_file) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm extract_metadata %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
                system.run('PYTHONPATH=%s %s/bin/opensfm detect_features %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
                system.run('PYTHONPATH=%s %s/bin/opensfm match_features %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
                with open(matched_done_file, 'w') as fout:
                    fout.write("Matching done!\n")
            else:
                log.ODM_WARNING(
                    'Found a feature matching done progress file in: %s' %
                    matched_done_file)

            if not io.file_exists(tree.opensfm_tracks) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm create_tracks %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
            else:
                log.ODM_WARNING('Found a valid OpenSfM tracks file in: %s' %
                                tree.opensfm_tracks)

            if not io.file_exists(tree.opensfm_reconstruction) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm reconstruct %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
            else:
                log.ODM_WARNING(
                    'Found a valid OpenSfM reconstruction file in: %s' %
                    tree.opensfm_reconstruction)

            if not io.file_exists(
                    tree.opensfm_reconstruction_meshed) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm mesh %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
            else:
                log.ODM_WARNING(
                    'Found a valid OpenSfM meshed reconstruction file in: %s' %
                    tree.opensfm_reconstruction_meshed)

            if not args.use_pmvs:
                if not io.file_exists(
                        tree.opensfm_reconstruction_nvm) or rerun_cell:
                    system.run(
                        'PYTHONPATH=%s %s/bin/opensfm export_visualsfm %s' %
                        (context.pyopencv_path, context.opensfm_path,
                         tree.opensfm))
                else:
                    log.ODM_WARNING(
                        'Found a valid OpenSfM NVM reconstruction file in: %s'
                        % tree.opensfm_reconstruction_nvm)

                system.run('PYTHONPATH=%s %s/bin/opensfm undistort %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
                system.run(
                    'PYTHONPATH=%s %s/bin/opensfm compute_depthmaps %s' %
                    (context.pyopencv_path, context.opensfm_path,
                     tree.opensfm))
        else:
            log.ODM_WARNING(
                'Found a valid OpenSfM reconstruction file in: %s' %
                tree.opensfm_reconstruction)

        # check if reconstruction was exported to bundler before
        if not io.file_exists(tree.opensfm_bundle_list) or rerun_cell:
            # convert back to bundler's format
            system.run(
                'PYTHONPATH=%s %s/bin/export_bundler %s' %
                (context.pyopencv_path, context.opensfm_path, tree.opensfm))
        else:
            log.ODM_WARNING('Found a valid Bundler file in: %s' %
                            tree.opensfm_reconstruction)

        if args.use_pmvs:
            # check if reconstruction was exported to pmvs before
            if not io.file_exists(tree.pmvs_visdat) or rerun_cell:
                # run PMVS converter
                system.run('PYTHONPATH=%s %s/bin/export_pmvs %s --output %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm, tree.pmvs))
            else:
                log.ODM_WARNING('Found a valid CMVS file in: %s' %
                                tree.pmvs_visdat)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'OpenSfM')

        log.ODM_INFO('Running ODM OpenSfM Cell - Finished')
        return ecto.OK if args.end_with != 'opensfm' else ecto.QUIT
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Orthophoto Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        reconstruction = inputs.reconstruction
        verbose = '-verbose' if self.params.verbose else ''

        # define paths and create working directories
        system.mkdir_p(tree.odm_orthophoto)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_orthophoto') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_orthophoto' in args.rerun_from)

        if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell:

            # odm_orthophoto definitions
            kwargs = {
                'bin':
                context.odm_modules_path,
                'log':
                tree.odm_orthophoto_log,
                'ortho':
                tree.odm_orthophoto_file,
                'corners':
                tree.odm_orthophoto_corners,
                'res':
                1.0 / (gsd.cap_resolution(self.params.resolution,
                                          tree.opensfm_reconstruction,
                                          ignore_gsd=args.ignore_gsd) / 100.0),
                'verbose':
                verbose
            }

            # Have geo coordinates?
            georef = reconstruction.georef

            # Check if the georef object is initialized
            # (during a --rerun this might not be)
            # TODO: we should move this to a more central
            # location (perhaps during the dataset initialization)
            if georef and not georef.utm_east_offset:
                georeferencing_dir = tree.odm_georeferencing if args.use_3dmesh and not args.skip_3dmodel else tree.odm_25dgeoreferencing
                odm_georeferencing_model_txt_geo_file = os.path.join(
                    georeferencing_dir, tree.odm_georeferencing_model_txt_geo)

                if io.file_exists(odm_georeferencing_model_txt_geo_file):
                    georef.extract_offsets(
                        odm_georeferencing_model_txt_geo_file)
                else:
                    log.ODM_WARNING(
                        'Cannot read UTM offset from {}. An orthophoto will not be generated.'
                        .format(odm_georeferencing_model_txt_geo_file))

            if georef:
                if args.use_3dmesh:
                    kwargs['model_geo'] = os.path.join(
                        tree.odm_texturing,
                        tree.odm_georeferencing_model_obj_geo)
                else:
                    kwargs['model_geo'] = os.path.join(
                        tree.odm_25dtexturing,
                        tree.odm_georeferencing_model_obj_geo)
            else:
                if args.use_3dmesh:
                    kwargs['model_geo'] = os.path.join(
                        tree.odm_texturing, tree.odm_textured_model_obj)
                else:
                    kwargs['model_geo'] = os.path.join(
                        tree.odm_25dtexturing, tree.odm_textured_model_obj)

            # run odm_orthophoto
            system.run(
                '{bin}/odm_orthophoto -inputFile {model_geo} '
                '-logFile {log} -outputFile {ortho} -resolution {res} {verbose} '
                '-outputCornerFile {corners}'.format(**kwargs))

            # Create georeferenced GeoTiff
            geotiffcreated = False

            if georef and georef.projection and georef.utm_east_offset and georef.utm_north_offset:
                ulx = uly = lrx = lry = 0.0
                with open(tree.odm_orthophoto_corners) as f:
                    for lineNumber, line in enumerate(f):
                        if lineNumber == 0:
                            tokens = line.split(' ')
                            if len(tokens) == 4:
                                ulx = float(tokens[0]) + \
                                    float(georef.utm_east_offset)
                                lry = float(tokens[1]) + \
                                    float(georef.utm_north_offset)
                                lrx = float(tokens[2]) + \
                                    float(georef.utm_east_offset)
                                uly = float(tokens[3]) + \
                                    float(georef.utm_north_offset)
                log.ODM_INFO('Creating GeoTIFF')

                kwargs = {
                    'ulx':
                    ulx,
                    'uly':
                    uly,
                    'lrx':
                    lrx,
                    'lry':
                    lry,
                    'tiled':
                    '' if self.params.no_tiled else '-co TILED=yes ',
                    'compress':
                    self.params.compress,
                    'predictor':
                    '-co PREDICTOR=2 '
                    if self.params.compress in ['LZW', 'DEFLATE'] else '',
                    'proj':
                    georef.projection.srs,
                    'bigtiff':
                    self.params.bigtiff,
                    'png':
                    tree.odm_orthophoto_file,
                    'tiff':
                    tree.odm_orthophoto_tif,
                    'log':
                    tree.odm_orthophoto_tif_log,
                    'max_memory':
                    get_max_memory(),
                    'threads':
                    self.params.max_concurrency
                }

                system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} '
                           '{tiled} '
                           '-co BIGTIFF={bigtiff} '
                           '-co COMPRESS={compress} '
                           '{predictor} '
                           '-co BLOCKXSIZE=512 '
                           '-co BLOCKYSIZE=512 '
                           '-co NUM_THREADS={threads} '
                           '-a_srs \"{proj}\" '
                           '--config GDAL_CACHEMAX {max_memory}% '
                           '{png} {tiff} > {log}'.format(**kwargs))

                if args.crop > 0:
                    shapefile_path = os.path.join(
                        tree.odm_georeferencing,
                        'odm_georeferenced_model.bounds.shp')
                    Cropper.crop(
                        shapefile_path, tree.odm_orthophoto_tif, {
                            'TILED':
                            'NO' if self.params.no_tiled else 'YES',
                            'COMPRESS':
                            self.params.compress,
                            'PREDICTOR':
                            '2' if self.params.compress in ['LZW', 'DEFLATE']
                            else '1',
                            'BIGTIFF':
                            self.params.bigtiff,
                            'BLOCKXSIZE':
                            512,
                            'BLOCKYSIZE':
                            512,
                            'NUM_THREADS':
                            self.params.max_concurrency
                        })

                if self.params.build_overviews:
                    log.ODM_DEBUG("Building Overviews")
                    kwargs = {
                        'orthophoto': tree.odm_orthophoto_tif,
                        'log': tree.odm_orthophoto_gdaladdo_log
                    }
                    # Run gdaladdo
                    system.run(
                        'gdaladdo -ro -r average '
                        '--config BIGTIFF_OVERVIEW IF_SAFER '
                        '--config COMPRESS_OVERVIEW JPEG '
                        '{orthophoto} 2 4 8 16 > {log}'.format(**kwargs))

                geotiffcreated = True
            if not geotiffcreated:
                log.ODM_WARNING(
                    'No geo-referenced orthophoto created due '
                    'to missing geo-referencing or corner coordinates.')

        else:
            log.ODM_WARNING('Found a valid orthophoto in: %s' %
                            tree.odm_orthophoto_file)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Orthophoto')

        log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished')
        return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT
Exemple #33
0
    def process(self, inputs, outputs):

        # find a file in the root directory
        def find(file, dir):
            for root, dirs, files in os.walk(dir):
                return '/'.join((root, file)) if file in files else None

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Georeferencing Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        gcpfile = io.join_paths(tree.root_path, self.params.gcp_file) \
            if self.params.gcp_file else find('gcp_list.txt', tree.root_path)
        geocreated = True
        verbose = '-verbose' if self.params.verbose else ''

        # define paths and create working directories
        system.mkdir_p(tree.odm_georeferencing)

        # in case a gcp file it's not provided, let's try to generate it using
        # images metadata. Internally calls jhead.
        log.ODM_DEBUG(self.params.gcp_file)
        if not self.params.gcp_file:  # and \
            #   not io.file_exists(tree.odm_georeferencing_coords):

            log.ODM_WARNING('No coordinates file. '
                            'Generating coordinates file: %s' %
                            tree.odm_georeferencing_coords)

            # odm_georeference definitions
            kwargs = {
                'bin': context.odm_modules_path,
                'imgs': tree.dataset_resize,
                'imgs_list': tree.opensfm_bundle_list,
                'coords': tree.odm_georeferencing_coords,
                'log': tree.odm_georeferencing_utm_log,
                'verbose': verbose
            }

            # run UTM extraction binary
            extract_utm = system.run_and_return(
                '{bin}/odm_extract_utm -imagesPath {imgs}/ '
                '-imageListFile {imgs_list} -outputCoordFile {coords} {verbose} '
                '-logFile {log}'.format(**kwargs))

            if extract_utm != '':
                log.ODM_WARNING('Could not generate coordinates file. '
                                'Ignore if there is a GCP file. Error: %s' %
                                extract_utm)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_georeferencing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_georeferencing' in args.rerun_from)

        if not io.file_exists(tree.odm_georeferencing_model_obj_geo) or \
           not io.file_exists(tree.odm_georeferencing_model_ply_geo) or rerun_cell:

            # odm_georeference definitions
            kwargs = {
                'bin': context.odm_modules_path,
                'bundle': tree.opensfm_bundle,
                'imgs': tree.dataset_resize,
                'imgs_list': tree.opensfm_bundle_list,
                'model': tree.odm_textured_model_obj,
                'log': tree.odm_georeferencing_log,
                'coords': tree.odm_georeferencing_coords,
                'pc_geo': tree.odm_georeferencing_model_ply_geo,
                'geo_sys': tree.odm_georeferencing_model_txt_geo,
                'model_geo': tree.odm_georeferencing_model_obj_geo,
                'size': self.params.img_size,
                'gcp': gcpfile,
                'verbose': verbose
            }
            if not args.use_pmvs:
                kwargs['pc'] = tree.opensfm_model
            else:
                kwargs['pc'] = tree.pmvs_model

            # Check to see if the GCP file exists

            if not self.params.use_exif and (self.params.gcp_file or find(
                    'gcp_list.txt', tree.root_path)):
                log.ODM_INFO('Found %s' % gcpfile)
                try:
                    system.run(
                        '{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} '
                        '-bundleResizedTo {size} -inputFile {model} -outputFile {model_geo} '
                        '-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo} {verbose} '
                        '-logFile {log} -georefFileOutputPath {geo_sys} -gcpFile {gcp} '
                        '-outputCoordFile {coords}'.format(**kwargs))
                except Exception:
                    log.ODM_EXCEPTION('Georeferencing failed. ')
                    return ecto.QUIT
            elif io.file_exists(tree.odm_georeferencing_coords):
                log.ODM_INFO(
                    'Running georeferencing with generated coords file.')
                system.run(
                    '{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} '
                    '-inputFile {model} -outputFile {model_geo} '
                    '-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo} {verbose} '
                    '-logFile {log} -georefFileOutputPath {geo_sys}'.format(
                        **kwargs))
            else:
                log.ODM_WARNING('Georeferencing failed. Make sure your '
                                'photos have geotags in the EXIF or you have '
                                'provided a GCP file. ')
                geocreated = False  # skip the rest of the georeferencing

            if geocreated:
                # update images metadata
                geo_ref = types.ODM_GeoRef()
                geo_ref.parse_coordinate_system(tree.odm_georeferencing_coords)

                for idx, photo in enumerate(self.inputs.photos):
                    geo_ref.utm_to_latlon(tree.odm_georeferencing_latlon,
                                          photo, idx)

                # convert ply model to LAS reference system
                geo_ref.convert_to_las(tree.odm_georeferencing_model_ply_geo,
                                       tree.odm_georeferencing_pdal)

                # XYZ point cloud output
                log.ODM_INFO(
                    "Creating geo-referenced CSV file (XYZ format, can be used with GRASS to create DEM)"
                )
                with open(tree.odm_georeferencing_xyz_file, "wb") as csvfile:
                    csvfile_writer = csv.writer(csvfile, delimiter=",")
                    reachedpoints = False
                    with open(tree.odm_georeferencing_model_ply_geo) as f:
                        for lineNumber, line in enumerate(f):
                            if reachedpoints:
                                tokens = line.split(" ")
                                csv_line = [
                                    float(tokens[0]) + geo_ref.utm_east_offset,
                                    float(tokens[1]) +
                                    geo_ref.utm_north_offset, tokens[2]
                                ]
                                csvfile_writer.writerow(csv_line)
                            if line.startswith("end_header"):
                                reachedpoints = True
                csvfile.close()

        else:
            log.ODM_WARNING('Found a valid georeferenced model in: %s' %
                            tree.odm_georeferencing_model_ply_geo)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Georeferencing')

        log.ODM_INFO('Running ODM Georeferencing Cell - Finished')
        return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT
Exemple #34
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running MVS Texturing Cell')

        # get inputs
        args = inputs.args
        tree = inputs.tree
        reconstruction = inputs.reconstruction

        # define paths and create working directories
        system.mkdir_p(tree.odm_texturing)
        if not args.use_3dmesh: system.mkdir_p(tree.odm_25dtexturing)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'mvs_texturing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'mvs_texturing' in args.rerun_from)

        runs = [{
            'out_dir': tree.odm_texturing,
            'model': tree.odm_mesh,
            'nadir': False
        }]

        if args.skip_3dmodel:
            runs = []

        if not args.use_3dmesh:
            runs += [{
                'out_dir': tree.odm_25dtexturing,
                'model': tree.odm_25dmesh,
                'nadir': True
            }]

        for r in runs:
            odm_textured_model_obj = os.path.join(r['out_dir'],
                                                  tree.odm_textured_model_obj)

            if not io.file_exists(odm_textured_model_obj) or rerun_cell:
                log.ODM_DEBUG('Writing MVS Textured file in: %s' %
                              odm_textured_model_obj)

                # Format arguments to fit Mvs-Texturing app
                skipGeometricVisibilityTest = ""
                skipGlobalSeamLeveling = ""
                skipLocalSeamLeveling = ""
                skipHoleFilling = ""
                keepUnseenFaces = ""
                nadir = ""

                if (self.params.skip_vis_test):
                    skipGeometricVisibilityTest = "--skip_geometric_visibility_test"
                if (self.params.skip_glob_seam_leveling):
                    skipGlobalSeamLeveling = "--skip_global_seam_leveling"
                if (self.params.skip_loc_seam_leveling):
                    skipLocalSeamLeveling = "--skip_local_seam_leveling"
                if (self.params.skip_hole_fill):
                    skipHoleFilling = "--skip_hole_filling"
                if (self.params.keep_unseen_faces):
                    keepUnseenFaces = "--keep_unseen_faces"
                if (r['nadir']):
                    nadir = '--nadir_mode'

                # mvstex definitions
                kwargs = {
                    'bin': context.mvstex_path,
                    'out_dir': io.join_paths(r['out_dir'],
                                             "odm_textured_model"),
                    'model': r['model'],
                    'dataTerm': self.params.data_term,
                    'outlierRemovalType': self.params.outlier_rem_type,
                    'skipGeometricVisibilityTest': skipGeometricVisibilityTest,
                    'skipGlobalSeamLeveling': skipGlobalSeamLeveling,
                    'skipLocalSeamLeveling': skipLocalSeamLeveling,
                    'skipHoleFilling': skipHoleFilling,
                    'keepUnseenFaces': keepUnseenFaces,
                    'toneMapping': self.params.tone_mapping,
                    'nadirMode': nadir,
                    'nadirWeight': 2**args.texturing_nadir_weight - 1,
                    'nvm_file': io.join_paths(tree.opensfm,
                                              "reconstruction.nvm")
                }

                # Make sure tmp directory is empty
                mvs_tmp_dir = os.path.join(r['out_dir'], 'tmp')
                if io.dir_exists(mvs_tmp_dir):
                    log.ODM_INFO(
                        "Removing old tmp directory {}".format(mvs_tmp_dir))
                    shutil.rmtree(mvs_tmp_dir)

                # run texturing binary
                system.run('{bin} {nvm_file} {model} {out_dir} '
                           '-d {dataTerm} -o {outlierRemovalType} '
                           '-t {toneMapping} '
                           '{skipGeometricVisibilityTest} '
                           '{skipGlobalSeamLeveling} '
                           '{skipLocalSeamLeveling} '
                           '{skipHoleFilling} '
                           '{keepUnseenFaces} '
                           '{nadirMode} '
                           '-n {nadirWeight}'.format(**kwargs))
            else:
                log.ODM_WARNING('Found a valid ODM Texture file in: %s' %
                                odm_textured_model_obj)

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Texturing')

        log.ODM_INFO('Running ODM Texturing Cell - Finished')
        return ecto.OK if args.end_with != 'mvs_texturing' else ecto.QUIT
Exemple #35
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Orthophoto Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        verbose = '-verbose' if self.params.verbose else ''

        # define paths and create working directories
        system.mkdir_p(tree.odm_orthophoto)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_orthophoto') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_orthophoto' in args.rerun_from)

        if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell:

            # odm_orthophoto definitions
            kwargs = {
                'bin': context.odm_modules_path,
                'log': tree.odm_orthophoto_log,
                'ortho': tree.odm_orthophoto_file,
                'corners': tree.odm_orthophoto_corners,
                'res': self.params.resolution,
                'verbose': verbose
            }

            # Have geo coordinates?
            if io.file_exists(tree.odm_georeferencing_coords):
                if args.use_25dmesh:
                    kwargs['model_geo'] = os.path.join(
                        tree.odm_25dtexturing,
                        tree.odm_georeferencing_model_obj_geo)
                else:
                    kwargs['model_geo'] = os.path.join(
                        tree.odm_texturing,
                        tree.odm_georeferencing_model_obj_geo)
            else:
                if args.use_25dmesh:
                    kwargs['model_geo'] = os.path.join(
                        tree.odm_25dtexturing, tree.odm_textured_model_obj)
                else:
                    kwargs['model_geo'] = os.path.join(
                        tree.odm_texturing, tree.odm_textured_model_obj)

            # run odm_orthophoto
            system.run(
                '{bin}/odm_orthophoto -inputFile {model_geo} '
                '-logFile {log} -outputFile {ortho} -resolution {res} {verbose} '
                '-outputCornerFile {corners}'.format(**kwargs))

            if not io.file_exists(tree.odm_georeferencing_coords):
                log.ODM_WARNING('No coordinates file. A georeferenced raster '
                                'will not be created')
            else:
                # Create georeferenced GeoTiff
                geotiffcreated = False
                georef = types.ODM_GeoRef()
                # creates the coord refs # TODO I don't want to have to do this twice- after odm_georef
                georef.parse_coordinate_system(tree.odm_georeferencing_coords)

                if georef.epsg and georef.utm_east_offset and georef.utm_north_offset:
                    ulx = uly = lrx = lry = 0.0
                    with open(tree.odm_orthophoto_corners) as f:
                        for lineNumber, line in enumerate(f):
                            if lineNumber == 0:
                                tokens = line.split(' ')
                                if len(tokens) == 4:
                                    ulx = float(tokens[0]) + \
                                        float(georef.utm_east_offset)
                                    lry = float(tokens[1]) + \
                                        float(georef.utm_north_offset)
                                    lrx = float(tokens[2]) + \
                                        float(georef.utm_east_offset)
                                    uly = float(tokens[3]) + \
                                        float(georef.utm_north_offset)
                    log.ODM_INFO('Creating GeoTIFF')

                    kwargs = {
                        'ulx':
                        ulx,
                        'uly':
                        uly,
                        'lrx':
                        lrx,
                        'lry':
                        lry,
                        'tiled':
                        '' if self.params.no_tiled else '-co TILED=yes ',
                        'compress':
                        self.params.compress,
                        'predictor':
                        '-co PREDICTOR=2 '
                        if self.params.compress in ['LZW', 'DEFLATE'] else '',
                        'epsg':
                        georef.epsg,
                        't_srs':
                        self.params.t_srs or "EPSG:{0}".format(georef.epsg),
                        'bigtiff':
                        self.params.bigtiff,
                        'png':
                        tree.odm_orthophoto_file,
                        'tiff':
                        tree.odm_orthophoto_tif,
                        'log':
                        tree.odm_orthophoto_tif_log
                    }

                    system.run(
                        'gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} '
                        '{tiled} '
                        '-co BIGTIFF={bigtiff} '
                        '-co COMPRESS={compress} '
                        '{predictor} '
                        '-co BLOCKXSIZE=512 '
                        '-co BLOCKYSIZE=512 '
                        '-co NUM_THREADS=ALL_CPUS '
                        '-a_srs \"EPSG:{epsg}\" '
                        '{png} {tiff} > {log}'.format(**kwargs))

                    if self.params.build_overviews:
                        log.ODM_DEBUG("Building Overviews")
                        kwargs = {
                            'orthophoto': tree.odm_orthophoto_tif,
                            'log': tree.odm_orthophoto_gdaladdo_log
                        }
                        # Run gdaladdo
                        system.run(
                            'gdaladdo -ro -r average '
                            '--config BIGTIFF_OVERVIEW IF_SAFER '
                            '--config COMPRESS_OVERVIEW JPEG '
                            '{orthophoto} 2 4 8 16 > {log}'.format(**kwargs))

                    geotiffcreated = True
                if not geotiffcreated:
                    log.ODM_WARNING(
                        'No geo-referenced orthophoto created due '
                        'to missing geo-referencing or corner coordinates.')

        else:
            log.ODM_WARNING('Found a valid orthophoto in: %s' %
                            tree.odm_orthophoto_file)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Orthophoto')

        log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished')
        return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT
Exemple #36
0
    def process(self, inputs, outputs):
        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running MVE Cell')

        # get inputs
        tree = inputs.tree
        args = inputs.args
        reconstruction = inputs.reconstruction
        photos = reconstruction.photos

        if not photos:
            log.ODM_ERROR('Not enough photos in photos array to start MVE')
            return ecto.QUIT

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'mve') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'mve' in args.rerun_from)

        # check if reconstruction was done before
        if not io.file_exists(tree.mve_model) or rerun_cell:
            # cleanup if a rerun
            if io.dir_exists(tree.mve_path) and rerun_cell:
                shutil.rmtree(tree.mve_path)

            # make bundle directory
            if not io.file_exists(tree.mve_bundle):
                system.mkdir_p(tree.mve_path)
                system.mkdir_p(io.join_paths(tree.mve_path, 'bundle'))
                io.copy(tree.opensfm_image_list, tree.mve_image_list)
                io.copy(tree.opensfm_bundle, tree.mve_bundle)

            # mve makescene wants the output directory
            # to not exists before executing it (otherwise it
            # will prompt the user for confirmation)
            if io.dir_exists(tree.mve):
                shutil.rmtree(tree.mve)

            # run mve makescene
            if not io.dir_exists(tree.mve_views):
                system.run('%s %s %s' % (context.makescene_path, tree.mve_path, tree.mve), env_vars={'OMP_NUM_THREADS': args.max_concurrency})

            # Compute mve output scale based on depthmap_resolution
            max_width = 0
            max_height = 0
            for photo in photos:
                max_width = max(photo.width, max_width)
                max_height = max(photo.height, max_height)

            max_pixels = args.depthmap_resolution * args.depthmap_resolution
            if max_width * max_height <= max_pixels:
                mve_output_scale = 0
            else:
                ratio = float(max_width * max_height) / float(max_pixels)
                mve_output_scale = int(math.ceil(math.log(ratio) / math.log(4.0)))

            dmrecon_config = [
                "-s%s" % mve_output_scale,
	            "--progress=silent",
                "--local-neighbors=2",
                "--force",
            ]

            # Run MVE's dmrecon
            log.ODM_INFO('                                                                               ')
            log.ODM_INFO('                                    ,*/**                                      ')
            log.ODM_INFO('                                  ,*@%*/@%*                                    ')
            log.ODM_INFO('                                ,/@%******@&*.                                 ')
            log.ODM_INFO('                              ,*@&*********/@&*                                ')
            log.ODM_INFO('                            ,*@&**************@&*                              ')
            log.ODM_INFO('                          ,/@&******************@&*.                           ')
            log.ODM_INFO('                        ,*@&*********************/@&*                          ')
            log.ODM_INFO('                      ,*@&**************************@&*.                       ')
            log.ODM_INFO('                    ,/@&******************************&&*,                     ')
            log.ODM_INFO('                  ,*&&**********************************@&*.                   ')
            log.ODM_INFO('                ,*@&**************************************@&*.                 ')
            log.ODM_INFO('              ,*@&***************#@@@@@@@@@%****************&&*,               ')
            log.ODM_INFO('            .*&&***************&@@@@@@@@@@@@@@****************@@*.             ')
            log.ODM_INFO('          .*@&***************&@@@@@@@@@@@@@@@@@%****(@@%********@@*.           ')
            log.ODM_INFO('        .*@@***************%@@@@@@@@@@@@@@@@@@@@@#****&@@@@%******&@*,         ')
            log.ODM_INFO('      .*&@****************@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@/*****@@*.       ')
            log.ODM_INFO('    .*@@****************@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@%*************@@*.     ')
            log.ODM_INFO('  .*@@****/***********@@@@@&**(@@@@@@@@@@@@@@@@@@@@@@@#*****************%@*,   ')
            log.ODM_INFO(' */@*******@*******#@@@@%*******/@@@@@@@@@@@@@@@@@@@@********************/@(,  ')
            log.ODM_INFO(' ,*@(********&@@@@@@#**************/@@@@@@@#**(@@&/**********************@&*   ')
            log.ODM_INFO('   *#@/*******************************@@@@@***&@&**********************&@*,    ')
            log.ODM_INFO('     *#@#******************************&@@@***@#*********************&@*,      ')
            log.ODM_INFO('       */@#*****************************@@@************************@@*.        ')
            log.ODM_INFO('         *#@/***************************/@@/*********************%@*,          ')
            log.ODM_INFO('           *#@#**************************#@@%******************%@*,            ')
            log.ODM_INFO('             */@#*************************(@@@@@@@&%/********&@*.              ')
            log.ODM_INFO('               *(@(*********************************/%@@%**%@*,                ')
            log.ODM_INFO('                 *(@%************************************%@**                  ')
            log.ODM_INFO('                   **@%********************************&@*,                    ')
            log.ODM_INFO('                     *(@(****************************%@/*                      ')
            log.ODM_INFO('                       ,(@%************************#@/*                        ')
            log.ODM_INFO('                         ,*@%********************&@/,                          ')
            log.ODM_INFO('                           */@#****************#@/*                            ')
            log.ODM_INFO('                             ,/@&************#@/*                              ')
            log.ODM_INFO('                               ,*@&********%@/,                                ')
            log.ODM_INFO('                                 */@#****(@/*                                  ')
            log.ODM_INFO('                                   ,/@@@@(*                                    ')
            log.ODM_INFO('                                     .**,                                      ')
            log.ODM_INFO('')
            log.ODM_INFO("Running dense reconstruction. This might take a while. Please be patient, the process is not dead or hung.")
            log.ODM_INFO("                              Process is running")
            system.run('%s %s %s' % (context.dmrecon_path, ' '.join(dmrecon_config), tree.mve), env_vars={'OMP_NUM_THREADS': args.max_concurrency})

            scene2pset_config = [
                "-F%s" % mve_output_scale
            ]

            # run scene2pset
            system.run('%s %s "%s" "%s"' % (context.scene2pset_path, ' '.join(scene2pset_config), tree.mve, tree.mve_model), env_vars={'OMP_NUM_THREADS': args.max_concurrency})
        else:
            log.ODM_WARNING('Found a valid MVE reconstruction file in: %s' %
                            tree.mve_model)

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'MVE')

        log.ODM_INFO('Running ODM MVE Cell - Finished')
        return ecto.OK if args.end_with != 'mve' else ecto.QUIT
Exemple #37
0
    def process(self, args, outputs):
        outputs['start_time'] = system.now_raw()
        tree = types.ODM_Tree(args.project_path, args.gcp, args.geo)
        outputs['tree'] = tree

        if args.time and io.file_exists(tree.benchmarking):
            # Delete the previously made file
            os.remove(tree.benchmarking)
            with open(tree.benchmarking, 'a') as b:
                b.write('ODM Benchmarking file created %s\nNumber of Cores: %s\n\n' % (system.now(), context.num_cores))
    
        # check if the image filename is supported
        def valid_image_filename(filename):
            (pathfn, ext) = os.path.splitext(filename)
            return ext.lower() in context.supported_extensions and pathfn[-5:] != "_mask"

        # Get supported images from dir
        def get_images(in_dir):
            log.ODM_DEBUG(in_dir)
            entries = os.listdir(in_dir)
            valid, rejects = [], []
            for f in entries:
                if valid_image_filename(f):
                    valid.append(f)
                else:
                    rejects.append(f)
            return valid, rejects

        def find_mask(photo_path, masks):
            (pathfn, ext) = os.path.splitext(os.path.basename(photo_path))
            k = "{}_mask".format(pathfn)
            
            mask = masks.get(k)
            if mask:
                # Spaces are not supported due to OpenSfM's mask_list.txt format reqs
                if not " " in mask:
                    return mask
                else:
                    log.ODM_WARNING("Image mask {} has a space. Spaces are currently not supported for image masks.".format(mask))

        # get images directory
        images_dir = tree.dataset_raw

        # define paths and create working directories
        system.mkdir_p(tree.odm_georeferencing)

        log.ODM_INFO('Loading dataset from: %s' % images_dir)

        # check if we rerun cell or not
        images_database_file = os.path.join(tree.root_path, 'images.json')
        if not io.file_exists(images_database_file) or self.rerun():
            if not os.path.exists(images_dir):
                raise system.ExitException("There are no images in %s! Make sure that your project path and dataset name is correct. The current is set to: %s" % (images_dir, args.project_path))

            files, rejects = get_images(images_dir)
            if files:
                # create ODMPhoto list
                path_files = [os.path.join(images_dir, f) for f in files]

                # Lookup table for masks
                masks = {}
                for r in rejects:
                    (p, ext) = os.path.splitext(r)
                    if p[-5:] == "_mask" and ext.lower() in context.supported_extensions:
                        masks[p] = r

                photos = []
                with open(tree.dataset_list, 'w') as dataset_list:
                    log.ODM_INFO("Loading %s images" % len(path_files))
                    for f in path_files:
                        try:
                            p = types.ODM_Photo(f)
                            p.set_mask(find_mask(f, masks))
                            photos.append(p)
                            dataset_list.write(photos[-1].filename + '\n')
                        except PhotoCorruptedException:
                            log.ODM_WARNING("%s seems corrupted and will not be used" % os.path.basename(f))

                # Check if a geo file is available
                if tree.odm_geo_file is not None and os.path.isfile(tree.odm_geo_file):
                    log.ODM_INFO("Found image geolocation file")
                    gf = GeoFile(tree.odm_geo_file)
                    updated = 0
                    for p in photos:
                        entry = gf.get_entry(p.filename)
                        if entry:
                            p.update_with_geo_entry(entry)
                            p.compute_opk()
                            updated += 1
                    log.ODM_INFO("Updated %s image positions" % updated)

                # GPSDOP override if we have GPS accuracy information (such as RTK)
                if 'gps_accuracy_is_set' in args:
                    log.ODM_INFO("Forcing GPS DOP to %s for all images" % args.gps_accuracy)

                    for p in photos:
                        p.override_gps_dop(args.gps_accuracy)
                
                # Override projection type
                if args.camera_lens != "auto":
                    log.ODM_INFO("Setting camera lens to %s for all images" % args.camera_lens)

                    for p in photos:
                        p.override_camera_projection(args.camera_lens)

                # Automatic sky removal
                if args.sky_removal:
                    # For each image that :
                    #  - Doesn't already have a mask, AND
                    #  - Is not nadir (or if orientation info is missing), AND
                    #  - There are no spaces in the image filename (OpenSfM requirement)
                    # Automatically generate a sky mask
                    
                    # Generate list of sky images
                    sky_images = []
                    for p in photos:
                        if p.mask is None and (p.pitch is None or (abs(p.pitch) > 20)) and (not " " in p.filename):
                            sky_images.append({'file': os.path.join(images_dir, p.filename), 'p': p})

                    if len(sky_images) > 0:
                        log.ODM_INFO("Automatically generating sky masks for %s images" % len(sky_images))
                        model = ai.get_model("skyremoval", "https://github.com/OpenDroneMap/SkyRemoval/releases/download/v1.0.5/model.zip", "v1.0.5")
                        if model is not None:
                            sf = SkyFilter(model=model)

                            def parallel_sky_filter(item):
                                try:
                                    mask_file = sf.run_img(item['file'], images_dir)

                                    # Check and set
                                    if mask_file is not None and os.path.isfile(mask_file):
                                        item['p'].set_mask(os.path.basename(mask_file))
                                        log.ODM_INFO("Wrote %s" % os.path.basename(mask_file))
                                    else:
                                        log.ODM_WARNING("Cannot generate mask for %s" % img)
                                except Exception as e:
                                    log.ODM_WARNING("Cannot generate mask for %s: %s" % (img, str(e)))

                            parallel_map(parallel_sky_filter, sky_images, max_workers=args.max_concurrency)

                            log.ODM_INFO("Sky masks generation completed!")
                        else:
                            log.ODM_WARNING("Cannot load AI model (you might need to be connected to the internet?)")
                    else:
                        log.ODM_INFO("No sky masks will be generated (masks already provided, or images are nadir)")

                # End sky removal

                # Save image database for faster restart
                save_images_database(photos, images_database_file)
            else:
                raise system.ExitException('Not enough supported images in %s' % images_dir)
        else:
            # We have an images database, just load it
            photos = load_images_database(images_database_file)

        log.ODM_INFO('Found %s usable images' % len(photos))
        log.logger.log_json_images(len(photos))

        # Create reconstruction object
        reconstruction = types.ODM_Reconstruction(photos)
        
        if tree.odm_georeferencing_gcp and not args.use_exif:
            reconstruction.georeference_with_gcp(tree.odm_georeferencing_gcp,
                                                 tree.odm_georeferencing_coords,
                                                 tree.odm_georeferencing_gcp_utm,
                                                 tree.odm_georeferencing_model_txt_geo,
                                                 rerun=self.rerun())
        else:
            reconstruction.georeference_with_gps(tree.dataset_raw, 
                                                 tree.odm_georeferencing_coords, 
                                                 tree.odm_georeferencing_model_txt_geo,
                                                 rerun=self.rerun())
        
        reconstruction.save_proj_srs(os.path.join(tree.odm_georeferencing, tree.odm_georeferencing_proj))
        outputs['reconstruction'] = reconstruction

        # Try to load boundaries
        if args.boundary:
            if reconstruction.is_georeferenced():
                outputs['boundary'] = boundary.load_boundary(args.boundary, reconstruction.get_proj_srs())
            else:
                args.boundary = None
                log.ODM_WARNING("Reconstruction is not georeferenced, but boundary file provided (will ignore boundary file)")

        # If sfm-algorithm is triangulation, check if photos have OPK
        if args.sfm_algorithm == 'triangulation':
            for p in photos:
                if not p.has_opk():
                    log.ODM_WARNING("No omega/phi/kappa angles found in input photos (%s), switching sfm-algorithm to incremental" % p.filename)
                    args.sfm_algorithm = 'incremental'
                    break
Exemple #38
0
    def process(self, inputs, outputs):
        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM DEM Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        las_model_found = io.file_exists(tree.odm_georeferencing_model_laz)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_dem') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_dem' in args.rerun_from)

        log.ODM_INFO('Classify: ' + str(args.pc_classify != "none"))
        log.ODM_INFO('Create DSM: ' + str(args.dsm))
        log.ODM_INFO('Create DTM: ' + str(args.dtm))
        log.ODM_INFO('DEM input file {0} found: {1}'.format(
            tree.odm_georeferencing_model_laz, str(las_model_found)))

        # Setup terrain parameters
        terrain_params_map = {
            'flatnonforest': (1, 3),
            'flatforest': (1, 2),
            'complexnonforest': (5, 2),
            'complexforest': (10, 2)
        }
        terrain_params = terrain_params_map[args.dem_terrain_type.lower()]
        slope, cellsize = terrain_params

        # define paths and create working directories
        odm_dem_root = tree.path('odm_dem')
        if not io.dir_exists(odm_dem_root):
            system.mkdir_p(odm_dem_root)

        if args.pc_classify != "none" and las_model_found:
            pc_classify_marker = os.path.join(odm_dem_root,
                                              'pc_classify_done.txt')

            if not io.file_exists(pc_classify_marker) or rerun_cell:
                log.ODM_INFO("Classifying {} using {}".format(
                    tree.odm_georeferencing_model_laz, args.pc_classify))
                commands.classify(tree.odm_georeferencing_model_laz,
                                  args.pc_classify == "smrf",
                                  slope,
                                  cellsize,
                                  approximate=args.dem_approximate,
                                  initialDistance=args.dem_initial_distance,
                                  verbose=args.verbose)
                with open(pc_classify_marker, 'w') as f:
                    f.write('Classify: {}\n'.format(args.pc_classify))
                    f.write('Slope: {}\n'.format(slope))
                    f.write('Cellsize: {}\n'.format(cellsize))
                    f.write('Approximate: {}\n'.format(args.dem_approximate))
                    f.write('InitialDistance: {}\n'.format(
                        args.dem_initial_distance))

        # Do we need to process anything here?
        if (args.dsm or args.dtm) and las_model_found:
            dsm_output_filename = os.path.join(odm_dem_root, 'dsm.tif')
            dtm_output_filename = os.path.join(odm_dem_root, 'dtm.tif')

            if (args.dtm and not io.file_exists(dtm_output_filename)) or \
                (args.dsm and not io.file_exists(dsm_output_filename)) or \
                rerun_cell:

                products = []
                if args.dsm: products.append('dsm')
                if args.dtm: products.append('dtm')

                resolution = gsd.cap_resolution(args.dem_resolution,
                                                tree.opensfm_reconstruction,
                                                gsd_error_estimate=-3,
                                                ignore_gsd=args.ignore_gsd)
                radius_steps = [(resolution / 100.0) / 2.0]
                for _ in range(args.dem_gapfill_steps - 1):
                    radius_steps.append(
                        radius_steps[-1] *
                        2)  # 2 is arbitrary, maybe there's a better value?

                for product in products:
                    commands.create_dems(
                        [tree.odm_georeferencing_model_laz],
                        product,
                        radius=map(str, radius_steps),
                        gapfill=True,
                        outdir=odm_dem_root,
                        resolution=resolution / 100.0,
                        maxsd=args.dem_maxsd,
                        maxangle=args.dem_maxangle,
                        decimation=args.dem_decimation,
                        verbose=args.verbose,
                        max_workers=get_max_concurrency_for_dem(
                            args.max_concurrency,
                            tree.odm_georeferencing_model_laz))

                    if args.crop > 0:
                        bounds_shapefile_path = os.path.join(
                            tree.odm_georeferencing,
                            'odm_georeferenced_model.bounds.shp')
                        if os.path.exists(bounds_shapefile_path):
                            Cropper.crop(
                                bounds_shapefile_path,
                                os.path.join(odm_dem_root,
                                             "{}.tif".format(product)),
                                {
                                    'TILED': 'YES',
                                    'COMPRESS': 'LZW',
                                    'BLOCKXSIZE': 512,
                                    'BLOCKYSIZE': 512,
                                    'NUM_THREADS': self.params.max_concurrency
                                })
            else:
                log.ODM_WARNING('Found existing outputs in: %s' % odm_dem_root)
        else:
            log.ODM_WARNING('DEM will not be generated')

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Dem')

        log.ODM_INFO('Running ODM DEM Cell - Finished')
        return ecto.OK if args.end_with != 'odm_dem' else ecto.QUIT
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM OpenSfM Cell')

        # get inputs
        tree = inputs.tree
        args = inputs.args
        reconstruction = inputs.reconstruction
        photos = reconstruction.photos

        if not photos:
            log.ODM_ERROR('Not enough photos in photos array to start OpenSfM')
            return ecto.QUIT

        # create working directories
        system.mkdir_p(tree.opensfm)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'opensfm') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'opensfm' in args.rerun_from)

        if args.fast_orthophoto:
            output_file = io.join_paths(tree.opensfm, 'reconstruction.ply')
        elif args.use_opensfm_dense:
            output_file = tree.opensfm_model
        else:
            output_file = tree.opensfm_reconstruction

        # check if reconstruction was done before
        if not io.file_exists(output_file) or rerun_cell:
            # create file list
            list_path = io.join_paths(tree.opensfm, 'image_list.txt')
            has_alt = True
            with open(list_path, 'w') as fout:
                for photo in photos:
                    if not photo.altitude:
                        has_alt = False
                    fout.write('%s\n' % io.join_paths(tree.dataset_raw, photo.filename))

            # create config file for OpenSfM
            config = [
                "use_exif_size: %s" % ('no' if not self.params.use_exif_size else 'yes'),
                "feature_process_size: %s" % self.params.feature_process_size,
                "feature_min_frames: %s" % self.params.feature_min_frames,
                "processes: %s" % self.params.processes,
                "matching_gps_neighbors: %s" % self.params.matching_gps_neighbors,
                "depthmap_method: %s" % args.opensfm_depthmap_method,
                "depthmap_resolution: %s" % args.depthmap_resolution,
                "depthmap_min_patch_sd: %s" % args.opensfm_depthmap_min_patch_sd,
                "depthmap_min_consistent_views: %s" % args.opensfm_depthmap_min_consistent_views,
                "optimize_camera_parameters: %s" % ('no' if self.params.fixed_camera_params else 'yes')
            ]

            if has_alt:
                log.ODM_DEBUG("Altitude data detected, enabling it for GPS alignment")
                config.append("use_altitude_tag: yes")
                config.append("align_method: naive")
            else:
                config.append("align_method: orientation_prior")
                config.append("align_orientation_prior: vertical")

            if args.use_hybrid_bundle_adjustment:
                log.ODM_DEBUG("Enabling hybrid bundle adjustment")
                config.append("bundle_interval: 100")          # Bundle after adding 'bundle_interval' cameras
                config.append("bundle_new_points_ratio: 1.2")  # Bundle when (new points) / (bundled points) > bundle_new_points_ratio
                config.append("local_bundle_radius: 1")        # Max image graph distance for images to be included in local bundle adjustment

            if args.matcher_distance > 0:
                config.append("matching_gps_distance: %s" % self.params.matching_gps_distance)

            if tree.odm_georeferencing_gcp:
                config.append("bundle_use_gcp: yes")
                io.copy(tree.odm_georeferencing_gcp, tree.opensfm)

            # write config file
            log.ODM_DEBUG(config)
            config_filename = io.join_paths(tree.opensfm, 'config.yaml')
            with open(config_filename, 'w') as fout:
                fout.write("\n".join(config))

            # run OpenSfM reconstruction
            matched_done_file = io.join_paths(tree.opensfm, 'matching_done.txt')
            if not io.file_exists(matched_done_file) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm extract_metadata %s' %
                           (context.pyopencv_path, context.opensfm_path, tree.opensfm))
                system.run('PYTHONPATH=%s %s/bin/opensfm detect_features %s' %
                           (context.pyopencv_path, context.opensfm_path, tree.opensfm))
                system.run('PYTHONPATH=%s %s/bin/opensfm match_features %s' %
                           (context.pyopencv_path, context.opensfm_path, tree.opensfm))
                with open(matched_done_file, 'w') as fout:
                    fout.write("Matching done!\n")
            else:
                log.ODM_WARNING('Found a feature matching done progress file in: %s' %
                                matched_done_file)

            if not io.file_exists(tree.opensfm_tracks) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm create_tracks %s' %
                           (context.pyopencv_path, context.opensfm_path, tree.opensfm))
            else:
                log.ODM_WARNING('Found a valid OpenSfM tracks file in: %s' %
                                tree.opensfm_tracks)

            if not io.file_exists(tree.opensfm_reconstruction) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm reconstruct %s' %
                           (context.pyopencv_path, context.opensfm_path, tree.opensfm))
            else:
                log.ODM_WARNING('Found a valid OpenSfM reconstruction file in: %s' %
                                tree.opensfm_reconstruction)

            # Check that a reconstruction file has been created
            if not io.file_exists(tree.opensfm_reconstruction):
                log.ODM_ERROR("The program could not process this dataset using the current settings. "
                                "Check that the images have enough overlap, "
                                "that there are enough recognizable features "
                                "and that the images are in focus. "
                                "You could also try to increase the --min-num-features parameter."
                                "The program will now exit.")
                sys.exit(1)


            # Always export VisualSFM's reconstruction and undistort images
            # as we'll use these for texturing (after GSD estimation and resizing)
            if not args.ignore_gsd:
                image_scale = gsd.image_scale_factor(args.orthophoto_resolution, tree.opensfm_reconstruction)
            else:
                image_scale = 1.0

            if not io.file_exists(tree.opensfm_reconstruction_nvm) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm export_visualsfm --image_extension png --scale_focal %s %s' %
                            (context.pyopencv_path, context.opensfm_path, image_scale, tree.opensfm))
            else:
                log.ODM_WARNING('Found a valid OpenSfM NVM reconstruction file in: %s' %
                                tree.opensfm_reconstruction_nvm)

            # These will be used for texturing
            system.run('PYTHONPATH=%s %s/bin/opensfm undistort --image_format png --image_scale %s %s' %
                        (context.pyopencv_path, context.opensfm_path, image_scale, tree.opensfm))

            # Skip dense reconstruction if necessary and export
            # sparse reconstruction instead
            if args.fast_orthophoto:
                system.run('PYTHONPATH=%s %s/bin/opensfm export_ply --no-cameras %s' %
                        (context.pyopencv_path, context.opensfm_path, tree.opensfm))
            elif args.use_opensfm_dense:
                # Undistort images at full scale in JPG
                # (TODO: we could compare the size of the PNGs if they are < than depthmap_resolution
                # and use those instead of re-exporting full resolution JPGs)
                system.run('PYTHONPATH=%s %s/bin/opensfm undistort %s' %
                        (context.pyopencv_path, context.opensfm_path, tree.opensfm))
                system.run('PYTHONPATH=%s %s/bin/opensfm compute_depthmaps %s' %
                        (context.pyopencv_path, context.opensfm_path, tree.opensfm))
        else:
            log.ODM_WARNING('Found a valid OpenSfM reconstruction file in: %s' %
                            tree.opensfm_reconstruction)

        # check if reconstruction was exported to bundler before
        if not io.file_exists(tree.opensfm_bundle_list) or rerun_cell:
            # convert back to bundler's format
            system.run('PYTHONPATH=%s %s/bin/export_bundler %s' %
                       (context.pyopencv_path, context.opensfm_path, tree.opensfm))
        else:
            log.ODM_WARNING('Found a valid Bundler file in: %s' %
                            tree.opensfm_reconstruction)

        if reconstruction.georef:
            system.run('PYTHONPATH=%s %s/bin/opensfm export_geocoords %s --transformation --proj \'%s\'' %
                       (context.pyopencv_path, context.opensfm_path, tree.opensfm, reconstruction.georef.projection.srs))

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'OpenSfM')

        log.ODM_INFO('Running ODM OpenSfM Cell - Finished')
        return ecto.OK if args.end_with != 'opensfm' else ecto.QUIT
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Georeferencing Cell')

        # get inputs
        args = inputs.args
        tree = inputs.tree
        reconstruction = inputs.reconstruction
        gcpfile = tree.odm_georeferencing_gcp
        doPointCloudGeo = True
        transformPointCloud = True
        verbose = '-verbose' if self.params.verbose else ''
        geo_ref = reconstruction.georef

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_georeferencing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_georeferencing' in args.rerun_from)

        runs = [{
            'georeferencing_dir': tree.odm_georeferencing,
            'texturing_dir': tree.odm_texturing,
            'model': os.path.join(tree.odm_texturing, tree.odm_textured_model_obj)
        }]

        if args.skip_3dmodel:
            runs = []

        if not args.use_3dmesh:
            # Make sure 2.5D mesh is georeferenced before the 3D mesh
            # Because it will be used to calculate a transform
            # for the point cloud. If we use the 3D model transform,
            # DEMs and orthophoto might not align!
            runs.insert(0, {
                    'georeferencing_dir': tree.odm_25dgeoreferencing,
                    'texturing_dir': tree.odm_25dtexturing,
                    'model': os.path.join(tree.odm_25dtexturing, tree.odm_textured_model_obj)
                })

        for r in runs:
            odm_georeferencing_model_obj_geo = os.path.join(r['texturing_dir'], tree.odm_georeferencing_model_obj_geo)
            odm_georeferencing_log = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_log)
            odm_georeferencing_transform_file = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_transform_file)
            odm_georeferencing_model_txt_geo_file = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_model_txt_geo)

            if not io.file_exists(odm_georeferencing_model_obj_geo) or \
               not io.file_exists(tree.odm_georeferencing_model_laz) or rerun_cell:

                # odm_georeference definitions
                kwargs = {
                    'bin': context.odm_modules_path,
                    'input_pc_file': tree.filtered_point_cloud,
                    'bundle': tree.opensfm_bundle,
                    'imgs': tree.dataset_raw,
                    'imgs_list': tree.opensfm_bundle_list,
                    'model': r['model'],
                    'log': odm_georeferencing_log,
                    'input_trans_file': tree.opensfm_transformation,
                    'transform_file': odm_georeferencing_transform_file,
                    'coords': tree.odm_georeferencing_coords,
                    'output_pc_file': tree.odm_georeferencing_model_laz,
                    'geo_sys': odm_georeferencing_model_txt_geo_file,
                    'model_geo': odm_georeferencing_model_obj_geo,
                    'gcp': gcpfile,
                    'verbose': verbose
                }

                if transformPointCloud:
                    kwargs['pc_params'] = '-inputPointCloudFile {input_pc_file} -outputPointCloudFile {output_pc_file}'.format(**kwargs)

                    if geo_ref and geo_ref.projection and geo_ref.projection.srs:
                        kwargs['pc_params'] += ' -outputPointCloudSrs %s' % pipes.quote(geo_ref.projection.srs)
                    else:
                        log.ODM_WARNING('NO SRS: The output point cloud will not have a SRS.')
                else:
                    kwargs['pc_params'] = ''
 
                # Check to see if the GCP file exists

                if not self.params.use_exif and (self.params.gcp_file or tree.odm_georeferencing_gcp):
                   log.ODM_INFO('Found %s' % gcpfile)
                   try:
                       system.run('{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} '
                                  '-inputFile {model} -outputFile {model_geo} '
                                  '{pc_params} {verbose} '
                                  '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys} -gcpFile {gcp} '
                                  '-outputCoordFile {coords}'.format(**kwargs))
                   except Exception:
                       log.ODM_EXCEPTION('Georeferencing failed. ')
                       return ecto.QUIT
                elif io.file_exists(tree.opensfm_transformation) and io.file_exists(tree.odm_georeferencing_coords):
                    log.ODM_INFO('Running georeferencing with OpenSfM transformation matrix')
                    system.run('{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} -inputCoordFile {coords} '
                               '-inputFile {model} -outputFile {model_geo} '
                               '{pc_params} {verbose} '
                               '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'.format(**kwargs))
                elif io.file_exists(tree.odm_georeferencing_coords):
                    log.ODM_INFO('Running georeferencing with generated coords file.')
                    system.run('{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} '
                               '-inputFile {model} -outputFile {model_geo} '
                               '{pc_params} {verbose} '
                               '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'.format(**kwargs))
                else:
                    log.ODM_WARNING('Georeferencing failed. Make sure your '
                                    'photos have geotags in the EXIF or you have '
                                    'provided a GCP file. ')
                    doPointCloudGeo = False # skip the rest of the georeferencing

                if doPointCloudGeo:
                    # update images metadata
                    geo_ref.extract_offsets(odm_georeferencing_model_txt_geo_file)
                    reconstruction.georef = geo_ref

                    # XYZ point cloud output
                    if args.pc_csv:
                        log.ODM_INFO("Creating geo-referenced CSV file (XYZ format)")
                        
                        system.run("pdal translate -i \"{}\" "
                            "-o \"{}\" "
                            "--writers.text.format=csv "
                            "--writers.text.order=\"X,Y,Z\" "
                            "--writers.text.keep_unspecified=false ".format(
                                tree.odm_georeferencing_model_laz,
                                tree.odm_georeferencing_xyz_file))
                    
                    # LAS point cloud output
                    if args.pc_las:
                        log.ODM_INFO("Creating geo-referenced LAS file")
                        
                        system.run("pdal translate -i \"{}\" "
                            "-o \"{}\" ".format(
                                tree.odm_georeferencing_model_laz,
                                tree.odm_georeferencing_model_las))
                    
                    if args.crop > 0:
                        log.ODM_INFO("Calculating cropping area and generating bounds shapefile from point cloud")
                        cropper = Cropper(tree.odm_georeferencing, 'odm_georeferenced_model')
                        
                        decimation_step = 40 if args.fast_orthophoto or args.use_opensfm_dense else 90
                        
                        # More aggressive decimation for large datasets
                        if not args.fast_orthophoto:
                            decimation_step *= int(len(reconstruction.photos) / 1000) + 1

                        cropper.create_bounds_shapefile(tree.odm_georeferencing_model_laz, args.crop, 
                                                    decimation_step=decimation_step,
                                                    outlier_radius=20 if args.fast_orthophoto else 2)

                    # Do not execute a second time, since
                    # We might be doing georeferencing for
                    # multiple models (3D, 2.5D, ...)
                    doPointCloudGeo = False
                    transformPointCloud = False
            else:
                log.ODM_WARNING('Found a valid georeferenced model in: %s'
                                % tree.odm_georeferencing_model_laz)

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Georeferencing')

        log.ODM_INFO('Running ODM Georeferencing Cell - Finished')
        return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT
Exemple #41
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Georeferencing Cell')

        # get inputs
        args = inputs.args
        tree = inputs.tree
        reconstruction = inputs.reconstruction
        gcpfile = tree.odm_georeferencing_gcp
        doPointCloudGeo = True
        transformPointCloud = True
        verbose = '-verbose' if self.params.verbose else ''

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'odm_georeferencing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'odm_georeferencing' in args.rerun_from)

        runs = [{
            'georeferencing_dir':
            tree.odm_georeferencing,
            'texturing_dir':
            tree.odm_texturing,
            'model':
            os.path.join(tree.odm_texturing, tree.odm_textured_model_obj)
        }]

        if args.skip_3dmodel:
            runs = []

        if not args.use_3dmesh:
            runs += [{
                'georeferencing_dir':
                tree.odm_25dgeoreferencing,
                'texturing_dir':
                tree.odm_25dtexturing,
                'model':
                os.path.join(tree.odm_25dtexturing,
                             tree.odm_textured_model_obj)
            }]

        for r in runs:
            odm_georeferencing_model_obj_geo = os.path.join(
                r['texturing_dir'], tree.odm_georeferencing_model_obj_geo)
            odm_georeferencing_model_ply_geo = os.path.join(
                r['georeferencing_dir'], tree.odm_georeferencing_model_ply_geo)
            odm_georeferencing_log = os.path.join(r['georeferencing_dir'],
                                                  tree.odm_georeferencing_log)
            odm_georeferencing_transform_file = os.path.join(
                r['georeferencing_dir'],
                tree.odm_georeferencing_transform_file)
            odm_georeferencing_model_txt_geo_file = os.path.join(
                r['georeferencing_dir'], tree.odm_georeferencing_model_txt_geo)

            if not io.file_exists(odm_georeferencing_model_obj_geo) or \
               not io.file_exists(odm_georeferencing_model_ply_geo) or rerun_cell:

                # odm_georeference definitions
                kwargs = {
                    'bin': context.odm_modules_path,
                    'bundle': tree.opensfm_bundle,
                    'imgs': tree.dataset_raw,
                    'imgs_list': tree.opensfm_bundle_list,
                    'model': r['model'],
                    'log': odm_georeferencing_log,
                    'input_trans_file': tree.opensfm_transformation,
                    'transform_file': odm_georeferencing_transform_file,
                    'coords': tree.odm_georeferencing_coords,
                    'pc_geo': odm_georeferencing_model_ply_geo,
                    'geo_sys': odm_georeferencing_model_txt_geo_file,
                    'model_geo': odm_georeferencing_model_obj_geo,
                    'gcp': gcpfile,
                    'verbose': verbose
                }

                if args.fast_orthophoto:
                    kwargs['pc'] = os.path.join(tree.opensfm,
                                                'reconstruction.ply')
                elif args.use_opensfm_dense:
                    kwargs['pc'] = tree.opensfm_model
                else:
                    kwargs['pc'] = tree.smvs_model

                if transformPointCloud:
                    kwargs[
                        'pc_params'] = '-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo}'.format(
                            **kwargs)
                else:
                    kwargs['pc_params'] = ''

                # Check to see if the GCP file exists

                if not self.params.use_exif and (self.params.gcp_file or
                                                 tree.odm_georeferencing_gcp):
                    log.ODM_INFO('Found %s' % gcpfile)
                    try:
                        system.run(
                            '{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} '
                            '-inputFile {model} -outputFile {model_geo} '
                            '{pc_params} {verbose} '
                            '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys} -gcpFile {gcp} '
                            '-outputCoordFile {coords}'.format(**kwargs))
                    except Exception:
                        log.ODM_EXCEPTION('Georeferencing failed. ')
                        return ecto.QUIT
                elif io.file_exists(
                        tree.opensfm_transformation) and io.file_exists(
                            tree.odm_georeferencing_coords):
                    log.ODM_INFO(
                        'Running georeferencing with OpenSfM transformation matrix'
                    )
                    system.run(
                        '{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} -inputCoordFile {coords} '
                        '-inputFile {model} -outputFile {model_geo} '
                        '{pc_params} {verbose} '
                        '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'
                        .format(**kwargs))
                elif io.file_exists(tree.odm_georeferencing_coords):
                    log.ODM_INFO(
                        'Running georeferencing with generated coords file.')
                    system.run(
                        '{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} '
                        '-inputFile {model} -outputFile {model_geo} '
                        '{pc_params} {verbose} '
                        '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'
                        .format(**kwargs))
                else:
                    log.ODM_WARNING(
                        'Georeferencing failed. Make sure your '
                        'photos have geotags in the EXIF or you have '
                        'provided a GCP file. ')
                    doPointCloudGeo = False  # skip the rest of the georeferencing

                if doPointCloudGeo:
                    # update images metadata
                    geo_ref = reconstruction.georef
                    geo_ref.extract_offsets(
                        odm_georeferencing_model_txt_geo_file)

                    # convert ply model to LAS reference system
                    geo_ref.convert_to_las(odm_georeferencing_model_ply_geo,
                                           tree.odm_georeferencing_model_laz,
                                           tree.odm_georeferencing_las_json)

                    reconstruction.georef = geo_ref

                    # XYZ point cloud output
                    if args.pc_csv:
                        log.ODM_INFO(
                            "Creating geo-referenced CSV file (XYZ format)")
                        with open(tree.odm_georeferencing_xyz_file,
                                  "wb") as csvfile:
                            csvfile_writer = csv.writer(csvfile, delimiter=",")
                            with open(odm_georeferencing_model_ply_geo) as f:
                                endianess = '<'  # little endian
                                while True:
                                    line = f.readline()
                                    if "binary_big_endian" in line:
                                        endianess = '>'
                                    if line.startswith("end_header"):
                                        break

                                fmt = '{}dddBBB'.format(endianess)
                                while True:
                                    chunk = f.read(27)  # 3 doubles, 3 uints
                                    if len(chunk) < 27:
                                        break
                                    tokens = struct.unpack(fmt, chunk)
                                    csv_line = [
                                        float(tokens[0]),
                                        float(tokens[1]), tokens[2]
                                    ]
                                    csvfile_writer.writerow(csv_line)

                    if args.crop > 0:
                        log.ODM_INFO(
                            "Calculating cropping area and generating bounds shapefile from point cloud"
                        )
                        cropper = Cropper(tree.odm_georeferencing,
                                          'odm_georeferenced_model')
                        cropper.create_bounds_shapefile(
                            tree.odm_georeferencing_model_laz,
                            args.crop,
                            decimation_step=40 if args.fast_orthophoto
                            or args.use_opensfm_dense else 90,
                            outlier_radius=20 if args.fast_orthophoto else 2)

                    # Do not execute a second time, since
                    # We might be doing georeferencing for
                    # multiple models (3D, 2.5D, ...)
                    doPointCloudGeo = False
                    transformPointCloud = False
            else:
                log.ODM_WARNING('Found a valid georeferenced model in: %s' %
                                odm_georeferencing_model_ply_geo)

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Georeferencing')

        log.ODM_INFO('Running ODM Georeferencing Cell - Finished')
        return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT
Exemple #42
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Resize Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        photos = self.inputs.photos

        if not photos:
            log.ODM_ERROR('Not enough photos in photos to resize')
            return ecto.QUIT

        if self.params.resize_to <= 0:
            log.ODM_ERROR('Resize parameter must be greater than 0')
            return ecto.QUIT

        # create working directory
        system.mkdir_p(tree.dataset_resize)

        log.ODM_DEBUG('Resizing dataset to: %s' % tree.dataset_resize)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'resize') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'resize' in args.rerun_from)

        # loop over photos
        for photo in photos:
            # define image paths
            path_file = photo.path_file
            new_path_file = io.join_paths(tree.dataset_resize, photo.filename)
            # set raw image path in case we want to rerun cell
            if io.file_exists(new_path_file) and rerun_cell:
                path_file = io.join_paths(tree.dataset_raw, photo.filename)

            if not io.file_exists(new_path_file) or rerun_cell:
                # open and resize image with opencv
                img = cv2.imread(path_file)
                # compute new size
                max_side = max(img.shape[0], img.shape[1])
                if max_side <= self.params.resize_to:
                    log.ODM_WARNING(
                        'Resize Parameter is greater than the largest side of the image'
                    )
                ratio = float(self.params.resize_to) / float(max_side)
                img_r = cv2.resize(img, None, fx=ratio, fy=ratio)
                # write image with opencv
                cv2.imwrite(new_path_file, img_r)
                # read metadata with pyexiv2
                old_meta = pyexiv2.ImageMetadata(path_file)
                new_meta = pyexiv2.ImageMetadata(new_path_file)
                old_meta.read()
                new_meta.read()
                # copy metadata
                old_meta.copy(new_meta)
                # update metadata size
                new_meta['Exif.Photo.PixelXDimension'] = img_r.shape[0]
                new_meta['Exif.Photo.PixelYDimension'] = img_r.shape[1]
                new_meta.write()
                # update photos array with new values
                photo.path_file = new_path_file
                photo.width = img_r.shape[0]
                photo.height = img_r.shape[1]
                photo.update_focal()

                # log message
                log.ODM_DEBUG('Resized %s | dimensions: %s' %
                              (photo.filename, img_r.shape))
            else:
                # log message
                log.ODM_WARNING('Already resized %s | dimensions: %s x %s' %
                                (photo.filename, photo.width, photo.height))

        log.ODM_INFO('Resized %s images' % len(photos))

        # append photos to cell output
        self.outputs.photos = photos

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Resizing')

        log.ODM_INFO('Running ODM Resize Cell - Finished')
        return ecto.OK if args.end_with != 'resize' else ecto.QUIT
Exemple #43
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running MVS Texturing Cell')

        # get inputs
        args = inputs.args
        tree = inputs.tree
        reconstruction = inputs.reconstruction

        # define paths and create working directories
        system.mkdir_p(tree.odm_texturing)
        if not args.use_3dmesh: system.mkdir_p(tree.odm_25dtexturing)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'mvs_texturing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'mvs_texturing' in args.rerun_from)

        runs = [{
            'out_dir': tree.odm_texturing,
            'model': tree.odm_mesh,
            'nadir': False
        }]

        if args.skip_3dmodel:
            runs = []

        if not args.use_3dmesh:
            runs += [{
                    'out_dir': tree.odm_25dtexturing,
                    'model': tree.odm_25dmesh,
                    'nadir': True
                }]

        for r in runs:
            odm_textured_model_obj = os.path.join(r['out_dir'], tree.odm_textured_model_obj)

            if not io.file_exists(odm_textured_model_obj) or rerun_cell:
                log.ODM_DEBUG('Writing MVS Textured file in: %s'
                              % odm_textured_model_obj)

                # Format arguments to fit Mvs-Texturing app
                skipGeometricVisibilityTest = ""
                skipGlobalSeamLeveling = ""
                skipLocalSeamLeveling = ""
                skipHoleFilling = ""
                keepUnseenFaces = ""
                nadir = ""

                if (self.params.skip_vis_test):
                    skipGeometricVisibilityTest = "--skip_geometric_visibility_test"
                if (self.params.skip_glob_seam_leveling):
                    skipGlobalSeamLeveling = "--skip_global_seam_leveling"
                if (self.params.skip_loc_seam_leveling):
                    skipLocalSeamLeveling = "--skip_local_seam_leveling"
                if (self.params.skip_hole_fill):
                    skipHoleFilling = "--skip_hole_filling"
                if (self.params.keep_unseen_faces):
                    keepUnseenFaces = "--keep_unseen_faces"
                if (r['nadir']):
                    nadir = '--nadir_mode'

                # mvstex definitions
                kwargs = {
                    'bin': context.mvstex_path,
                    'out_dir': io.join_paths(r['out_dir'], "odm_textured_model"),
                    'model': r['model'],
                    'dataTerm': self.params.data_term,
                    'outlierRemovalType': self.params.outlier_rem_type,
                    'skipGeometricVisibilityTest': skipGeometricVisibilityTest,
                    'skipGlobalSeamLeveling': skipGlobalSeamLeveling,
                    'skipLocalSeamLeveling': skipLocalSeamLeveling,
                    'skipHoleFilling': skipHoleFilling,
                    'keepUnseenFaces': keepUnseenFaces,
                    'toneMapping': self.params.tone_mapping,
                    'nadirMode': nadir,
                    'nadirWeight': 2 ** args.texturing_nadir_weight - 1,
                    'nvm_file': io.join_paths(tree.opensfm, "reconstruction.nvm")
                }

                # Make sure tmp directory is empty
                mvs_tmp_dir = os.path.join(r['out_dir'], 'tmp')
                if io.dir_exists(mvs_tmp_dir):
                    log.ODM_INFO("Removing old tmp directory {}".format(mvs_tmp_dir))
                    shutil.rmtree(mvs_tmp_dir)

                # run texturing binary
                system.run('{bin} {nvm_file} {model} {out_dir} '
                           '-d {dataTerm} -o {outlierRemovalType} '
                           '-t {toneMapping} '
                           '{skipGeometricVisibilityTest} '
                           '{skipGlobalSeamLeveling} '
                           '{skipLocalSeamLeveling} '
                           '{skipHoleFilling} '
                           '{keepUnseenFaces} '
                           '{nadirMode} '
                           '-n {nadirWeight}'.format(**kwargs))
            else:
                log.ODM_WARNING('Found a valid ODM Texture file in: %s'
                                % odm_textured_model_obj)

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Texturing')

        log.ODM_INFO('Running ODM Texturing Cell - Finished')
        return ecto.OK if args.end_with != 'mvs_texturing' else ecto.QUIT
Exemple #44
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running MVS Texturing Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree

        # define paths and create working directories
        system.mkdir_p(tree.odm_texturing)
        if args.use_25dmesh: system.mkdir_p(tree.odm_25dtexturing) 

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'mvs_texturing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'mvs_texturing' in args.rerun_from)

        runs = [{
            'out_dir': tree.odm_texturing,
            'model': tree.odm_mesh,
            'force_skip_vis_test': False
        }]

        if args.fast_orthophoto:
            runs = []

        if args.use_25dmesh:
            runs += [{
                    'out_dir': tree.odm_25dtexturing,
                    'model': tree.odm_25dmesh,

                    # We always skip the visibility test when using the 2.5D mesh
                    # because many faces end up being narrow, and almost perpendicular 
                    # to the ground plane. The visibility test improperly classifies
                    # them as "not seen" since the test is done on a single triangle vertex,
                    # and while one vertex might be occluded, the other two might not.
                    'force_skip_vis_test': True 
                }]

        for r in runs:
            odm_textured_model_obj = os.path.join(r['out_dir'], tree.odm_textured_model_obj)

            if not io.file_exists(odm_textured_model_obj) or rerun_cell:
                log.ODM_DEBUG('Writing MVS Textured file in: %s'
                              % odm_textured_model_obj)
                
                
                # Format arguments to fit Mvs-Texturing app
                skipGeometricVisibilityTest = ""
                skipGlobalSeamLeveling = ""
                skipLocalSeamLeveling = ""
                skipHoleFilling = ""
                keepUnseenFaces = ""
                
                if (self.params.skip_vis_test or r['force_skip_vis_test']):
                    skipGeometricVisibilityTest = "--skip_geometric_visibility_test"
                if (self.params.skip_glob_seam_leveling):
                    skipGlobalSeamLeveling = "--skip_global_seam_leveling"
                if (self.params.skip_loc_seam_leveling):
                    skipLocalSeamLeveling = "--skip_local_seam_leveling"
                if (self.params.skip_hole_fill):
                    skipHoleFilling = "--skip_hole_filling"
                if (self.params.keep_unseen_faces):
                    keepUnseenFaces = "--keep_unseen_faces"

                # mvstex definitions
                kwargs = {
                    'bin': context.mvstex_path,
                    'out_dir': io.join_paths(r['out_dir'], "odm_textured_model"),
                    'pmvs_folder': tree.pmvs_rec_path,
                    'nvm_file': io.join_paths(tree.pmvs_rec_path, "nvmCams.nvm"),
                    'model': r['model'],
                    'dataTerm': self.params.data_term,
                    'outlierRemovalType': self.params.outlier_rem_type,
                    'skipGeometricVisibilityTest': skipGeometricVisibilityTest,
                    'skipGlobalSeamLeveling': skipGlobalSeamLeveling,
                    'skipLocalSeamLeveling': skipLocalSeamLeveling,
                    'skipHoleFilling': skipHoleFilling,
                    'keepUnseenFaces': keepUnseenFaces,
                    'toneMapping': self.params.tone_mapping
                }

                if not args.use_pmvs:
                    kwargs['nvm_file'] = io.join_paths(tree.opensfm,
                                                       "reconstruction.nvm")
                else:
                    log.ODM_DEBUG('Generating .nvm file from pmvs output: %s'
                                  % '{nvm_file}'.format(**kwargs))

                    # Create .nvm camera file.
                    pmvs2nvmcams.run('{pmvs_folder}'.format(**kwargs),
                                     '{nvm_file}'.format(**kwargs))

                # run texturing binary
                system.run('{bin} {nvm_file} {model} {out_dir} '
                           '-d {dataTerm} -o {outlierRemovalType} '
                           '-t {toneMapping} '
                           '{skipGeometricVisibilityTest} '
                           '{skipGlobalSeamLeveling} '
                           '{skipLocalSeamLeveling} '
                           '{skipHoleFilling} '
                           '{keepUnseenFaces}'.format(**kwargs))
            else:
                log.ODM_WARNING('Found a valid ODM Texture file in: %s'
                                % odm_textured_model_obj)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Texturing')

        log.ODM_INFO('Running ODM Texturing Cell - Finished')
        return ecto.OK if args.end_with != 'mvs_texturing' else ecto.QUIT
Exemple #45
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM OpenSfM Cell')

        # get inputs
        tree = inputs.tree
        args = inputs.args
        reconstruction = inputs.reconstruction
        photos = reconstruction.photos

        if not photos:
            log.ODM_ERROR('Not enough photos in photos array to start OpenSfM')
            return ecto.QUIT

        # create working directories
        system.mkdir_p(tree.opensfm)
        system.mkdir_p(tree.pmvs)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'opensfm') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'opensfm' in args.rerun_from)

        if not args.use_pmvs:
            output_file = tree.opensfm_model
            if args.fast_orthophoto:
                output_file = io.join_paths(tree.opensfm, 'reconstruction.ply')
        else:
            output_file = tree.opensfm_reconstruction

        # check if reconstruction was done before
        if not io.file_exists(output_file) or rerun_cell:
            # create file list
            list_path = io.join_paths(tree.opensfm, 'image_list.txt')
            has_alt = True
            with open(list_path, 'w') as fout:
                for photo in photos:
                    if not photo.altitude:
                        has_alt = False
                    fout.write('%s\n' % photo.path_file)

            # create config file for OpenSfM
            config = [
                "use_exif_size: %s" %
                ('no' if not self.params.use_exif_size else 'yes'),
                "feature_process_size: %s" % self.params.feature_process_size,
                "feature_min_frames: %s" % self.params.feature_min_frames,
                "processes: %s" % self.params.processes,
                "matching_gps_neighbors: %s" %
                self.params.matching_gps_neighbors,
                "depthmap_method: %s" % args.opensfm_depthmap_method,
                "depthmap_resolution: %s" % args.opensfm_depthmap_resolution,
                "depthmap_min_patch_sd: %s" %
                args.opensfm_depthmap_min_patch_sd,
                "depthmap_min_consistent_views: %s" %
                args.opensfm_depthmap_min_consistent_views,
                "optimize_camera_parameters: %s" %
                ('no' if self.params.fixed_camera_params else 'yes')
            ]

            if has_alt:
                log.ODM_DEBUG(
                    "Altitude data detected, enabling it for GPS alignment")
                config.append("use_altitude_tag: True")
                config.append("align_method: naive")

            if args.use_hybrid_bundle_adjustment:
                log.ODM_DEBUG("Enabling hybrid bundle adjustment")
                config.append(
                    "bundle_interval: 100"
                )  # Bundle after adding 'bundle_interval' cameras
                config.append(
                    "bundle_new_points_ratio: 1.2"
                )  # Bundle when (new points) / (bundled points) > bundle_new_points_ratio
                config.append(
                    "local_bundle_radius: 1"
                )  # Max image graph distance for images to be included in local bundle adjustment

            if args.matcher_distance > 0:
                config.append("matching_gps_distance: %s" %
                              self.params.matching_gps_distance)

            if tree.odm_georeferencing_gcp:
                config.append("bundle_use_gcp: yes")
                io.copy(tree.odm_georeferencing_gcp, tree.opensfm)

            # write config file
            log.ODM_DEBUG(config)
            config_filename = io.join_paths(tree.opensfm, 'config.yaml')
            with open(config_filename, 'w') as fout:
                fout.write("\n".join(config))

            # run OpenSfM reconstruction
            matched_done_file = io.join_paths(tree.opensfm,
                                              'matching_done.txt')
            if not io.file_exists(matched_done_file) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm extract_metadata %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
                system.run('PYTHONPATH=%s %s/bin/opensfm detect_features %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
                system.run('PYTHONPATH=%s %s/bin/opensfm match_features %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
                with open(matched_done_file, 'w') as fout:
                    fout.write("Matching done!\n")
            else:
                log.ODM_WARNING(
                    'Found a feature matching done progress file in: %s' %
                    matched_done_file)

            if not io.file_exists(tree.opensfm_tracks) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm create_tracks %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
            else:
                log.ODM_WARNING('Found a valid OpenSfM tracks file in: %s' %
                                tree.opensfm_tracks)

            if not io.file_exists(tree.opensfm_reconstruction) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm reconstruct %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
            else:
                log.ODM_WARNING(
                    'Found a valid OpenSfM reconstruction file in: %s' %
                    tree.opensfm_reconstruction)

            if not io.file_exists(
                    tree.opensfm_reconstruction_meshed) or rerun_cell:
                system.run('PYTHONPATH=%s %s/bin/opensfm mesh %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))
            else:
                log.ODM_WARNING(
                    'Found a valid OpenSfM meshed reconstruction file in: %s' %
                    tree.opensfm_reconstruction_meshed)

            if not args.use_pmvs:
                if not io.file_exists(
                        tree.opensfm_reconstruction_nvm) or rerun_cell:
                    system.run(
                        'PYTHONPATH=%s %s/bin/opensfm export_visualsfm %s' %
                        (context.pyopencv_path, context.opensfm_path,
                         tree.opensfm))
                else:
                    log.ODM_WARNING(
                        'Found a valid OpenSfM NVM reconstruction file in: %s'
                        % tree.opensfm_reconstruction_nvm)

                system.run('PYTHONPATH=%s %s/bin/opensfm undistort %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm))

                # Skip dense reconstruction if necessary and export
                # sparse reconstruction instead
                if args.fast_orthophoto:
                    system.run(
                        'PYTHONPATH=%s %s/bin/opensfm export_ply --no-cameras %s'
                        % (context.pyopencv_path, context.opensfm_path,
                           tree.opensfm))
                else:
                    system.run(
                        'PYTHONPATH=%s %s/bin/opensfm compute_depthmaps %s' %
                        (context.pyopencv_path, context.opensfm_path,
                         tree.opensfm))

        else:
            log.ODM_WARNING(
                'Found a valid OpenSfM reconstruction file in: %s' %
                tree.opensfm_reconstruction)

        # check if reconstruction was exported to bundler before
        if not io.file_exists(tree.opensfm_bundle_list) or rerun_cell:
            # convert back to bundler's format
            system.run(
                'PYTHONPATH=%s %s/bin/export_bundler %s' %
                (context.pyopencv_path, context.opensfm_path, tree.opensfm))
        else:
            log.ODM_WARNING('Found a valid Bundler file in: %s' %
                            tree.opensfm_reconstruction)

        if args.use_pmvs:
            # check if reconstruction was exported to pmvs before
            if not io.file_exists(tree.pmvs_visdat) or rerun_cell:
                # run PMVS converter
                system.run('PYTHONPATH=%s %s/bin/export_pmvs %s --output %s' %
                           (context.pyopencv_path, context.opensfm_path,
                            tree.opensfm, tree.pmvs))
            else:
                log.ODM_WARNING('Found a valid CMVS file in: %s' %
                                tree.pmvs_visdat)

        if reconstruction.georef:
            system.run(
                'PYTHONPATH=%s %s/bin/opensfm export_geocoords %s --transformation --proj \'%s\''
                % (context.pyopencv_path, context.opensfm_path, tree.opensfm,
                   reconstruction.georef.projection.srs))

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'OpenSfM')

        log.ODM_INFO('Running ODM OpenSfM Cell - Finished')
        return ecto.OK if args.end_with != 'opensfm' else ecto.QUIT
Exemple #46
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running MVS Texturing Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree

        # define paths and create working directories
        system.mkdir_p(tree.odm_texturing)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'mvs_texturing') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'mvs_texturing' in args.rerun_from)

        if not io.file_exists(tree.odm_textured_model_obj) or rerun_cell:
            log.ODM_DEBUG('Writing MVS Textured file in: %s'
                          % tree.odm_textured_model_obj)
            
            
            # Format arguments to fit Mvs-Texturing app
            skipGeometricVisibilityTest = ""
            skipGlobalSeamLeveling = ""
            skipLocalSeamLeveling = ""
            skipHoleFilling = ""
            keepUnseenFaces = ""
            
            if (self.params.skip_vis_test):
                skipGeometricVisibilityTest = "--skip_geometric_visibility_test"
            if (self.params.skip_glob_seam_leveling):
                skipGlobalSeamLeveling = "--skip_global_seam_leveling"
            if (self.params.skip_loc_seam_leveling):
                skipLocalSeamLeveling = "--skip_local_seam_leveling"
            if (self.params.skip_hole_fill):
                skipHoleFilling = "--skip_hole_filling"
            if (self.params.keep_unseen_faces):
                keepUnseenFaces = "--keep_unseen_faces"

            # mvstex definitions
            kwargs = {
                'bin': context.mvstex_path,
                'out_dir': io.join_paths(tree.odm_texturing, "odm_textured_model"),
                'pmvs_folder': tree.pmvs_rec_path,
                'nvm_file': io.join_paths(tree.pmvs_rec_path, "nvmCams.nvm"),
                'model': tree.odm_mesh,
                'dataTerm': self.params.data_term,
                'outlierRemovalType': self.params.outlier_rem_type,
                'skipGeometricVisibilityTest': skipGeometricVisibilityTest,
                'skipGlobalSeamLeveling': skipGlobalSeamLeveling,
                'skipLocalSeamLeveling': skipLocalSeamLeveling,
                'skipHoleFilling': skipHoleFilling,
                'keepUnseenFaces': keepUnseenFaces,
                'toneMapping': self.params.tone_mapping
            }

            if not args.use_pmvs:
                kwargs['nvm_file'] = io.join_paths(tree.opensfm,
                                                   "reconstruction.nvm")
            else:
                log.ODM_DEBUG('Generating .nvm file from pmvs output: %s'
                              % '{nvm_file}'.format(**kwargs))

                # Create .nvm camera file.
                pmvs2nvmcams.run('{pmvs_folder}'.format(**kwargs),
                                 '{nvm_file}'.format(**kwargs))

            # run texturing binary
            system.run('{bin} {nvm_file} {model} {out_dir} '
                       '-d {dataTerm} -o {outlierRemovalType} '
                       '-t {toneMapping} '
                       '{skipGeometricVisibilityTest} '
                       '{skipGlobalSeamLeveling} '
                       '{skipLocalSeamLeveling} '
                       '{skipHoleFilling} '
                       '{keepUnseenFaces}'.format(**kwargs))
        else:
            log.ODM_WARNING('Found a valid ODM Texture file in: %s'
                            % tree.odm_textured_model_obj)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Texturing')

        log.ODM_INFO('Running ODM Texturing Cell - Finished')
        return ecto.OK if args.end_with != 'odm_texturing' else ecto.QUIT
Exemple #47
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running SMVS Cell')

        # get inputs
        tree = inputs.tree
        args = inputs.args
        reconstruction = inputs.reconstruction
        photos = reconstruction.photos

        if not photos:
            log.ODM_ERROR('Not enough photos in photos array to start SMVS')
            return ecto.QUIT

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'smvs') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'smvs' in args.rerun_from)

        # check if reconstruction was done before
        if not io.file_exists(tree.smvs_model) or rerun_cell:
            # cleanup if a rerun
            if io.dir_exists(tree.mve_path) and rerun_cell:
                shutil.rmtree(tree.mve_path)

            # make bundle directory
            if not io.file_exists(tree.mve_bundle):
                system.mkdir_p(tree.mve_path)
                system.mkdir_p(io.join_paths(tree.mve_path, 'bundle'))
                io.copy(tree.opensfm_image_list, tree.mve_image_list)
                io.copy(tree.opensfm_bundle, tree.mve_bundle)

            # mve makescene wants the output directory
            # to not exists before executing it (otherwise it
            # will prompt the user for confirmation)
            if io.dir_exists(tree.smvs):
                shutil.rmtree(tree.smvs)

            # run mve makescene
            if not io.dir_exists(tree.mve_views):
                system.run('%s %s %s' % (context.makescene_path, tree.mve_path, tree.smvs))

            # config
            config = [
                "-t%s" % self.params.threads,
                "-a%s" % self.params.alpha,
                "--max-pixels=%s" % int(self.params.max_pixels),
                "-o%s" % self.params.output_scale,
                "--debug-lvl=%s" % ('1' if self.params.verbose else '0'),
                "%s" % '-S' if self.params.shading else '',
                "%s" % '-g' if self.params.gamma_srgb and self.params.shading else '',
                "--force" if rerun_cell else ''
            ]

            # run smvs
            system.run('%s %s %s' % (context.smvs_path, ' '.join(config), tree.smvs))
            
            # find and rename the output file for simplicity
            smvs_files = glob.glob(os.path.join(tree.smvs, 'smvs-*'))
            smvs_files.sort(key=os.path.getmtime) # sort by last modified date
            if len(smvs_files) > 0:
                old_file = smvs_files[-1]
                if not (io.rename_file(old_file, tree.smvs_model)):
                    log.ODM_WARNING("File %s does not exist, cannot be renamed. " % old_file)

                # Filter
                point_cloud.filter(tree.smvs_model, standard_deviation=args.pc_filter, verbose=args.verbose)
            else:
                log.ODM_WARNING("Cannot find a valid point cloud (smvs-XX.ply) in %s. Check the console output for errors." % tree.smvs)
        else:
            log.ODM_WARNING('Found a valid SMVS reconstruction file in: %s' %
                            tree.smvs_model)

        outputs.reconstruction = reconstruction

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'SMVS')

        log.ODM_INFO('Running ODM SMVS Cell - Finished')
        return ecto.OK if args.end_with != 'smvs' else ecto.QUIT
Exemple #48
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM OpenSfM Cell')

        # get inputs
        tree = self.inputs.tree
        args = self.inputs.args
        photos = self.inputs.photos

        if not photos:
            log.ODM_ERROR('Not enough photos in photos array to start OpenSfM')
            return ecto.QUIT

        # create working directories     
        system.mkdir_p(tree.opensfm)
        system.mkdir_p(tree.pmvs)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'opensfm') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'opensfm' in args.rerun_from)

        # check if reconstruction was done before

        if not io.file_exists(tree.opensfm_reconstruction) or rerun_cell:
            # create file list
            list_path = io.join_paths(tree.opensfm, 'image_list.txt')
            with open(list_path, 'w') as fout:
                for photo in photos:
                    fout.write('%s\n' % photo.path_file)

            # create config file for OpenSfM
            config = [
                "use_exif_size: %s" % ('no' if not self.params.use_exif_size else 'yes'),
                "feature_process_size: %s" % self.params.feature_process_size,
                "feature_min_frames: %s" % self.params.feature_min_frames,
                "processes: %s" % self.params.processes,
                "matching_gps_neighbors: %s" % self.params.matching_gps_neighbors
            ]

            if args.matcher_distance > 0:
                config.append("matching_gps_distance: %s" % self.params.matching_gps_distance)

            # write config file
            config_filename = io.join_paths(tree.opensfm, 'config.yaml')
            with open(config_filename, 'w') as fout:
                fout.write("\n".join(config))

            # run OpenSfM reconstruction
            system.run('PYTHONPATH=%s %s/bin/run_all %s' %
                       (context.pyopencv_path, context.opensfm_path, tree.opensfm))
        else:
            log.ODM_WARNING('Found a valid OpenSfM file in: %s' %
                            tree.opensfm_reconstruction)

        # check if reconstruction was exported to bundler before

        if not io.file_exists(tree.opensfm_bundle_list) or rerun_cell:
            # convert back to bundler's format
            system.run('PYTHONPATH=%s %s/bin/export_bundler %s' %
                       (context.pyopencv_path, context.opensfm_path, tree.opensfm))
        else:
            log.ODM_WARNING('Found a valid Bundler file in: %s' %
                            tree.opensfm_reconstruction)

        # check if reconstruction was exported to pmvs before

        if not io.file_exists(tree.pmvs_visdat) or rerun_cell:
            # run PMVS converter
            system.run('PYTHONPATH=%s %s/bin/export_pmvs %s --output %s' %
                       (context.pyopencv_path, context.opensfm_path, tree.opensfm, tree.pmvs))
        else:
            log.ODM_WARNING('Found a valid CMVS file in: %s' % tree.pmvs_visdat)

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'OpenSfM')

        log.ODM_INFO('Running ODM OpenSfM Cell - Finished')
        return ecto.OK if args.end_with != 'opensfm' else ecto.QUIT
Exemple #49
0
    def process(self, args, outputs):
        outputs['start_time'] = system.now_raw()
        tree = types.ODM_Tree(args.project_path, args.gcp, args.geo)
        outputs['tree'] = tree

        if args.time and io.file_exists(tree.benchmarking):
            # Delete the previously made file
            os.remove(tree.benchmarking)
            with open(tree.benchmarking, 'a') as b:
                b.write(
                    'ODM Benchmarking file created %s\nNumber of Cores: %s\n\n'
                    % (system.now(), context.num_cores))

        # check if the image filename is supported
        def valid_image_filename(filename):
            (pathfn, ext) = os.path.splitext(filename)
            return ext.lower(
            ) in context.supported_extensions and pathfn[-5:] != "_mask"

        # Get supported images from dir
        def get_images(in_dir):
            log.ODM_DEBUG(in_dir)
            entries = os.listdir(in_dir)
            valid, rejects = [], []
            for f in entries:
                if valid_image_filename(f):
                    valid.append(f)
                else:
                    rejects.append(f)
            return valid, rejects

        def find_mask(photo_path, masks):
            (pathfn, ext) = os.path.splitext(os.path.basename(photo_path))
            k = "{}_mask".format(pathfn)

            mask = masks.get(k)
            if mask:
                # Spaces are not supported due to OpenSfM's mask_list.txt format reqs
                if not " " in mask:
                    return mask
                else:
                    log.ODM_WARNING(
                        "Image mask {} has a space. Spaces are currently not supported for image masks."
                        .format(mask))

        # get images directory
        images_dir = tree.dataset_raw

        # define paths and create working directories
        system.mkdir_p(tree.odm_georeferencing)

        log.ODM_INFO('Loading dataset from: %s' % images_dir)

        # check if we rerun cell or not
        images_database_file = os.path.join(tree.root_path, 'images.json')
        if not io.file_exists(images_database_file) or self.rerun():
            if not os.path.exists(images_dir):
                log.ODM_ERROR(
                    "There are no images in %s! Make sure that your project path and dataset name is correct. The current is set to: %s"
                    % (images_dir, args.project_path))
                exit(1)

            files, rejects = get_images(images_dir)
            if files:
                # create ODMPhoto list
                path_files = [os.path.join(images_dir, f) for f in files]

                # Lookup table for masks
                masks = {}
                for r in rejects:
                    (p, ext) = os.path.splitext(r)
                    if p[-5:] == "_mask" and ext.lower(
                    ) in context.supported_extensions:
                        masks[p] = r

                photos = []
                with open(tree.dataset_list, 'w') as dataset_list:
                    log.ODM_INFO("Loading %s images" % len(path_files))
                    for f in path_files:
                        p = types.ODM_Photo(f)
                        p.set_mask(find_mask(f, masks))
                        photos += [p]
                        dataset_list.write(photos[-1].filename + '\n')

                # Check if a geo file is available
                if tree.odm_geo_file is not None and os.path.exists(
                        tree.odm_geo_file):
                    log.ODM_INFO("Found image geolocation file")
                    gf = GeoFile(tree.odm_geo_file)
                    updated = 0
                    for p in photos:
                        entry = gf.get_entry(p.filename)
                        if entry:
                            p.update_with_geo_entry(entry)
                            updated += 1
                    log.ODM_INFO("Updated %s image positions" % updated)

                # Save image database for faster restart
                save_images_database(photos, images_database_file)
            else:
                log.ODM_ERROR('Not enough supported images in %s' % images_dir)
                exit(1)
        else:
            # We have an images database, just load it
            photos = load_images_database(images_database_file)

        log.ODM_INFO('Found %s usable images' % len(photos))

        # Create reconstruction object
        reconstruction = types.ODM_Reconstruction(photos)

        if tree.odm_georeferencing_gcp and not args.use_exif:
            reconstruction.georeference_with_gcp(
                tree.odm_georeferencing_gcp,
                tree.odm_georeferencing_coords,
                tree.odm_georeferencing_gcp_utm,
                tree.odm_georeferencing_model_txt_geo,
                rerun=self.rerun())
        else:
            reconstruction.georeference_with_gps(
                tree.dataset_raw,
                tree.odm_georeferencing_coords,
                tree.odm_georeferencing_model_txt_geo,
                rerun=self.rerun())

        reconstruction.save_proj_srs(
            os.path.join(tree.odm_georeferencing,
                         tree.odm_georeferencing_proj))
        outputs['reconstruction'] = reconstruction
Exemple #50
0
    def process(self, inputs, outputs):

        # Benchmarking
        start_time = system.now_raw()

        log.ODM_INFO('Running ODM Resize Cell')

        # get inputs
        args = self.inputs.args
        tree = self.inputs.tree
        photos = self.inputs.photos

        if not photos:
            log.ODM_ERROR('Not enough photos in photos to resize')
            return ecto.QUIT

        if self.params.resize_to <= 0:
            log.ODM_ERROR('Resize parameter must be greater than 0')
            return ecto.QUIT

        # create working directory
        system.mkdir_p(tree.dataset_resize)

        log.ODM_DEBUG('Resizing dataset to: %s' % tree.dataset_resize)

        # check if we rerun cell or not
        rerun_cell = (args.rerun is not None and
                      args.rerun == 'resize') or \
                     (args.rerun_all) or \
                     (args.rerun_from is not None and
                      'resize' in args.rerun_from)

        # loop over photos
        if self.params.skip_resize:
            photos = Pool().map(
                partial(no_resize,
                        tree.dataset_raw,
                        tree.dataset_resize,
                        rerun_cell),
                photos
            )
            log.ODM_INFO('Copied %s images' % len(photos))
        else:
            photos = Pool().map(
                partial(resize,
                        tree.dataset_raw,
                        tree.dataset_resize,
                        self.params.resize_to,
                        rerun_cell),
                photos
            )
            log.ODM_INFO('Resized %s images' % len(photos))

        # append photos to cell output
        self.outputs.photos = photos

        if args.time:
            system.benchmark(start_time, tree.benchmarking, 'Resizing')

        log.ODM_INFO('Running ODM Resize Cell - Finished')
        return ecto.OK if args.end_with != 'resize' else ecto.QUIT