def test_filtered_copy(self): gcp = GCPFile('tests/assets/gcp_latlon_valid.txt') self.assertTrue(gcp.exists()) self.assertEqual(gcp.entries_count(), 2) copy = GCPFile(gcp.make_filtered_copy('tests/assets/output/filtered_copy.txt', 'tests/assets/images', min_images=1)) self.assertTrue(copy.exists()) self.assertEqual(copy.entries_count(), 1)
def process(self, args, outputs): tree = outputs['tree'] reconstruction = outputs['reconstruction'] photos = reconstruction.photos outputs['large'] = len(photos) > args.split if outputs['large']: # If we have a cluster address, we'll use a distributed workflow local_workflow = not bool(args.sm_cluster) octx = OSFMContext(tree.opensfm) split_done_file = octx.path("split_done.txt") if not io.file_exists(split_done_file) or self.rerun(): orig_max_concurrency = args.max_concurrency if not local_workflow: args.max_concurrency = max(1, args.max_concurrency - 1) log.ODM_INFO( "Setting max-concurrency to %s to better handle remote splits" % args.max_concurrency) log.ODM_INFO( "Large dataset detected (%s photos) and split set at %s. Preparing split merge." % (len(photos), args.split)) config = [ "submodels_relpath: ../submodels/opensfm", "submodel_relpath_template: ../submodels/submodel_%04d/opensfm", "submodel_images_relpath_template: ../submodels/submodel_%04d/images", "submodel_size: %s" % args.split, "submodel_overlap: %s" % args.split_overlap, ] octx.setup(args, tree.dataset_raw, photos, gcp_path=tree.odm_georeferencing_gcp, append_config=config, rerun=self.rerun()) octx.extract_metadata(self.rerun()) self.update_progress(5) if local_workflow: octx.feature_matching(self.rerun()) self.update_progress(20) # Create submodels if not io.dir_exists(tree.submodels_path) or self.rerun(): if io.dir_exists(tree.submodels_path): log.ODM_WARNING( "Removing existing submodels directory: %s" % tree.submodels_path) shutil.rmtree(tree.submodels_path) octx.run("create_submodels") else: log.ODM_WARNING( "Submodels directory already exist at: %s" % tree.submodels_path) # Find paths of all submodels mds = metadataset.MetaDataSet(tree.opensfm) submodel_paths = [ os.path.abspath(p) for p in mds.get_submodel_paths() ] gcp_file = GCPFile(tree.odm_georeferencing_gcp) for sp in submodel_paths: sp_octx = OSFMContext(sp) # Copy filtered GCP file if needed # One in OpenSfM's directory, one in the submodel project directory if gcp_file.exists(): submodel_gcp_file = os.path.abspath( sp_octx.path("..", "gcp_list.txt")) submodel_images_dir = os.path.abspath( sp_octx.path("..", "images")) if gcp_file.make_filtered_copy(submodel_gcp_file, submodel_images_dir): log.ODM_DEBUG("Copied filtered GCP file to %s" % submodel_gcp_file) io.copy( submodel_gcp_file, os.path.abspath(sp_octx.path("gcp_list.txt"))) else: log.ODM_DEBUG( "No GCP will be copied for %s, not enough images in the submodel are referenced by the GCP" % sp_octx.name()) # Reconstruct each submodel log.ODM_INFO( "Dataset has been split into %s submodels. Reconstructing each submodel..." % len(submodel_paths)) self.update_progress(25) if local_workflow: for sp in submodel_paths: log.ODM_INFO("Reconstructing %s" % sp) OSFMContext(sp).reconstruct(self.rerun()) else: lre = LocalRemoteExecutor(args.sm_cluster) lre.set_projects([ os.path.abspath(os.path.join(p, "..")) for p in submodel_paths ]) lre.run_reconstruction() self.update_progress(50) # Align octx.align_reconstructions(self.rerun()) self.update_progress(55) # Aligned reconstruction is in reconstruction.aligned.json # We need to rename it to reconstruction.json remove_paths = [] for sp in submodel_paths: sp_octx = OSFMContext(sp) aligned_recon = sp_octx.path('reconstruction.aligned.json') main_recon = sp_octx.path('reconstruction.json') if not io.file_exists(aligned_recon): log.ODM_WARNING( "Submodel %s does not have an aligned reconstruction (%s). " "This could mean that the submodel could not be reconstructed " " (are there enough features to reconstruct it?). Skipping." % (sp_octx.name(), aligned_recon)) remove_paths.append(sp) continue if io.file_exists(main_recon): os.remove(main_recon) shutil.move(aligned_recon, main_recon) log.ODM_DEBUG("%s is now %s" % (aligned_recon, main_recon)) # Remove invalid submodels submodel_paths = [ p for p in submodel_paths if not p in remove_paths ] # Run ODM toolchain for each submodel if local_workflow: for sp in submodel_paths: sp_octx = OSFMContext(sp) log.ODM_INFO("========================") log.ODM_INFO("Processing %s" % sp_octx.name()) log.ODM_INFO("========================") argv = get_submodel_argv(args.name, tree.submodels_path, sp_octx.name()) # Re-run the ODM toolchain on the submodel system.run(" ".join(map(quote, argv)), env_vars=os.environ.copy()) else: lre.set_projects([ os.path.abspath(os.path.join(p, "..")) for p in submodel_paths ]) lre.run_toolchain() # Restore max_concurrency value args.max_concurrency = orig_max_concurrency with open(split_done_file, 'w') as fout: fout.write("Split done!\n") else: log.ODM_WARNING('Found a split done file in: %s' % split_done_file) else: log.ODM_INFO("Normal dataset, will process all at once.") self.progress = 0.0