def test_copy_asp_scene_exclude_drg_and_err(self): files_to_move = [ "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-DEM.tif", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-PC.las", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-PC.laz", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-GoodPixelMap.tif", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-DEM.prj", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001.geojson", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-stereo.default", "WV01_20120422_102001001AE38C00_102001001B0AAD00.shp", "WV01_20120422_102001001AE38C00_102001001B0AAD00.dbf", "WV01_20120422_102001001AE38C00_102001001B0AAD00.shx", "WV01_20120422_102001001AE38C00_102001001B0AAD00.prj", ] self.args.exclude_drg = True self.args.exclude_err = True overlap_prefix = "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001" for f in self.asp_scene_files: move_file = utils.check_file_inclusion(f, self.asp_pairname, overlap_prefix, self.args) if f in files_to_move: self.assertTrue(move_file) else: self.assertFalse(move_file)
def test_copy_asp_strip(self): files_to_move = [ "WV01_20120422_102001001AE38C00_102001001B0AAD00-DEM.tif", "WV01_20120422_102001001AE38C00_102001001B0AAD00-PC.las", "WV01_20120422_102001001AE38C00_102001001B0AAD00-PC.laz", "WV01_20120422_102001001AE38C00_102001001B0AAD00-GoodPixelMap.tif", "WV01_20120422_102001001AE38C00_102001001B0AAD00-DEM.prj", "WV01_20120422_102001001AE38C00_102001001B0AAD00-DRG.tif", "WV01_20120422_102001001AE38C00_102001001B0AAD00-IntersectionErr.tif", "WV01_20120422_102001001AE38C00_102001001B0AAD00.geojson", "WV01_20120422_102001001AE38C00_102001001B0AAD00-stereo.default", "WV01_20120422_102001001AE38C00_102001001B0AAD00.shp", "WV01_20120422_102001001AE38C00_102001001B0AAD00.dbf", "WV01_20120422_102001001AE38C00_102001001B0AAD00.shx", "WV01_20120422_102001001AE38C00_102001001B0AAD00.prj", ] overlap_prefix = self.asp_pairname for f in self.asp_strip_files: move_file = utils.check_file_inclusion(f, self.asp_pairname, overlap_prefix, self.args) if f in files_to_move: self.assertTrue(move_file) else: self.assertFalse(move_file)
def test_copy_asp_scene_include_all(self): files_to_move = [ "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-DEM.tif", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-PC.tif", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-PC-center.txt", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-PC.las", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-PC.laz", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-GoodPixelMap.tif", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-DEM.prj", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-DRG.tif", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-IntersectionErr.tif", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001_fltr-DEM.tif", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001_fltr-DEM.prj", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001.geojson", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-stereo.default", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-log-stereo_corr", "WV01_20120422_102001001AE38C00_102001001B0AAD00.shp", "WV01_20120422_102001001AE38C00_102001001B0AAD00.dbf", "WV01_20120422_102001001AE38C00_102001001B0AAD00.shx", "WV01_20120422_102001001AE38C00_102001001B0AAD00.prj", ] self.args.include_fltr = True self.args.include_logs = True self.args.include_pc = True overlap_prefix = "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001" for f in self.asp_scene_files: move_file = utils.check_file_inclusion(f, self.asp_pairname, overlap_prefix, self.args) if f in files_to_move: self.assertTrue(move_file) else: self.assertFalse(move_file)
def test_copy_setsm_dems_only(self): files_to_move = [ "SETSM_WV01_20080830_1020010003561200_1020010004712300_seg1_8m_dem.tif", "SETSM_WV01_20080830_1020010003561200_1020010004712300_seg1_8m_meta.txt", ] self.args.dems_only = True overlap_prefix = "SETSM_WV01_20080830_1020010003561200_1020010004712300_seg1_8m" for f in self.setsm_strip_files: move_file = utils.check_file_inclusion(f, self.setsm_pairname, overlap_prefix, self.args) if f in files_to_move: self.assertTrue(move_file) else: self.assertFalse(move_file)
def test_copy_asp_scene_dems_only(self): files_to_move = [ "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-DEM.tif", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001-DEM.prj", "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001.geojson", ] self.args.dems_only = True overlap_prefix = "WV01_20120422153705_102001001AE38C00_12APR22153705-P1BS-052895840020_01_P001_WV01_20120422153618_102001001B0AAD00_12APR22153618-P1BS-052895834010_01_P001" for f in self.asp_scene_files: move_file = utils.check_file_inclusion(f, self.asp_pairname, overlap_prefix, self.args) if f in files_to_move: self.assertTrue(move_file) else: self.assertFalse(move_file)
def test_copy_asp_strip_dems_only(self): files_to_move = [ "WV01_20120422_102001001AE38C00_102001001B0AAD00-DEM.tif", "WV01_20120422_102001001AE38C00_102001001B0AAD00-DEM.prj", "WV01_20120422_102001001AE38C00_102001001B0AAD00.geojson", ] self.args.dems_only = True overlap_prefix = self.asp_pairname for f in self.asp_strip_files: move_file = utils.check_file_inclusion(f, self.asp_pairname, overlap_prefix, self.args) if f in files_to_move: self.assertTrue(move_file) else: self.assertFalse(move_file)
def main(): #### Set Up Arguments parser = argparse.ArgumentParser(description="move/copy ASP deliverable files") #### Positional Arguments parser.add_argument('src', help="source dir/shp of ASP dems") parser.add_argument('dstdir', help="destination directory") #### Optional Arguments parser.add_argument('-m', '--move', action='store_true', default=False, help='move files instead of copy') parser.add_argument('--dryrun', action='store_true', default=False, help='print action but do not alter files\n') parser.add_argument('--exclude-drg', action='store_true', default=False, help='exclude DRG/Ortho') parser.add_argument('--dems-only', action='store_true', default=False, help='copy DEMs only - overrides --exclude and --include options, except --include-fltr') parser.add_argument('--no-dirs', action='store_true', default=False, help='do not make pairname subdirs for overlaps\n') parser.add_argument('--tar-only', action='store_true', default=False, help='copy only tar archive, overrides --exclude and --include options') parser.add_argument('--exclude-err', action='store_true', default=False, help='ASP: exclude intersectionErr raster') parser.add_argument('--include-pc', action='store_true', default=False, help='ASP: include point cloud') parser.add_argument('--include-fltr', action='store_true', default=False, help='ASP: include non-interpolated DEM') parser.add_argument('--include-logs', action='store_true', default=False, help='ASP: include stereo logs') #### Parse Arguments args = parser.parse_args() src = os.path.abspath(args.src) if args.dems_only and args.tar_only: parser.error("options --tar-only and --dems-only are not not compatible") #### Validate args if os.path.isdir(src): srctype = 'dir' elif os.path.isfile(src) and src.endswith(".shp"): srctype = "shp" else: parser.error("Src is not a valid directory or shapefile: %s" %src) print("Collecting DEMs from source...") #### ID all dems, pairname is dirname overlaps = [] if srctype == 'dir': for root, dirs, files in os.walk(src): for f in files: if (f.endswith(('-DEM.tif','_dem.tif')) and not 'fltr' in f): overlaps.append(os.path.join(root,f)) elif srctype == 'shp': #### open shp flds = ("FILEPATH","WIN_PATH") dem_fld = "DEM_NAME" ds = ogr.Open(src) if ds is not None: lyr = ds.GetLayerByName(os.path.splitext(os.path.basename(src))[0]) lyr.ResetReading() src_srs = lyr.GetSpatialRef() featDefn = lyr.GetLayerDefn() for feat in lyr: path = None paths = [] try: i = feat.GetFieldIndex(dem_fld) dem_name = feat.GetField(i) except ValueError as e: print("Cannot locate DEM name field (%s)" %(dem_fld)) if not dem_name: print("Cannot locate DEM name field (%s)" %(dem_fld)) else: for fld in flds: try: i = feat.GetFieldIndex(fld) attrib = feat.GetField(i) except ValueError as e: print("Cannot locate candidate field (%s) in source feature class" %(fld)) else: if attrib: attrib_path = os.path.join(attrib,dem_name) paths.append(attrib) paths.append(attrib_path) if os.path.isfile(attrib): path = attrib elif os.path.isfile(attrib_path): path = attrib_path if path: print(path) overlaps.append(path) else: if len(paths) > 0: print("Cannot locate path for DEM in any of the following locations: \n%s" %('\n\t'.join(paths))) else: print("Cannot get valid values from candidate fields (%s) in source feature class" %(', '.join(flds))) ds = None overlaps = list(set(overlaps)) ##### Iterate through dems and copy/move files total = len(overlaps) i = 0 for overlap in overlaps: i+=1 print('\n[%d of %d]\t %s' %(i,total,os.path.basename(overlap))) #### Check that path is not terranova storage location if args.move is True and (path.startswith(r'V:\pgc\data\elev\dem') or path.startswith(r'V:/pgc/data/elev/dem') or path.startswith(r'/mnt/pgc/data/elev/dem/asp')): print("Cannot use --move flag on DEMs located in /pgc/data/elev/dem/") else: srcpairdir = os.path.dirname(overlap) pairname = os.path.basename(srcpairdir) if args.no_dirs: file_dstdir = args.dstdir elif "SETSM" in os.path.basename(overlap): file_dstdir = args.dstdir else: file_dstdir = os.path.join(args.dstdir,pairname) overlap_prefix = os.path.basename(overlap)[:-8] #### Copy all files with overlap prefix and Copy pair shp for f in os.listdir(srcpairdir): move_file = utils.check_file_inclusion(f, pairname, overlap_prefix, args) #### Copy/Move if move_file is True: if not args.dryrun: if not os.path.isdir(file_dstdir): os.makedirs(file_dstdir) ifp = os.path.join(srcpairdir,f) ofp = os.path.join(file_dstdir,f) if args.move is True: if not os.path.isfile(ofp): print("Moving %s --> %s" %(ifp,ofp)) if not args.dryrun: os.rename(ifp,ofp) else: if not os.path.isfile(ofp): print("Copying %s --> %s" %(ifp,ofp)) if not args.dryrun: shutil.copy2(ifp,ofp)
pairname = os.path.basename(srcpairdir) if args.no_dirs: file_dstdir = args.dstdir elif "SETSM" in os.path.basename(overlap): file_dstdir = args.dstdir else: file_dstdir = os.path.join(args.dstdir, pairname) overlap_prefix = os.path.basename(overlap)[:-8] #### Copy all files with overlap prefix and Copy pair shp for f in os.listdir(srcpairdir): move_file = utils.check_file_inclusion(f, pairname, overlap_prefix, args) #### Copy/Move if move_file is True: if not args.dryrun: if not os.path.isdir(file_dstdir): os.makedirs(file_dstdir) ifp = os.path.join(srcpairdir, f) ofp = os.path.join(file_dstdir, f) if args.move is True: if not os.path.isfile(ofp): print "Moving %s --> %s" % (ifp, ofp) if not args.dryrun: os.rename(ifp, ofp) else:
pairname = os.path.basename(srcpairdir) if args.no_dirs: file_dstdir = args.dstdir elif "SETSM" in os.path.basename(overlap): file_dstdir = args.dstdir else: file_dstdir = os.path.join(args.dstdir,pairname) overlap_prefix = os.path.basename(overlap)[:-8] #### Copy all files with overlap prefix and Copy pair shp for f in os.listdir(srcpairdir): move_file = utils.check_file_inclusion(f, pairname, overlap_prefix, args) #### Copy/Move if move_file is True: if not args.dryrun: if not os.path.isdir(file_dstdir): os.makedirs(file_dstdir) ifp = os.path.join(srcpairdir,f) ofp = os.path.join(file_dstdir,f) if args.move is True: if not os.path.isfile(ofp): print "Moving %s --> %s" %(ifp,ofp) if not args.dryrun: os.rename(ifp,ofp) else: