def main(): title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser = parser0.add_default_parser() group = parser.add_argument_group('archive options') group.add_argument('--keep', help='Keep files after adding to archive', default=False, action='store_true') group.add_argument('--recursive', help='Iterate through subdirectories', default=False, action='store_true') group.add_argument( '--update', help='Update asset if newer version available, (must call gips_process to regenerate products', default=False, action='store_true' ) args = parser0.parse_args() try: print title cls = import_data_class(args.command) cls.Asset.archive(**vars(args)) except Exception, e: import traceback VerboseOut(traceback.format_exc(), 4) print 'Data archive error: %s' % e
def main(): title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser = parser0.add_default_parser() group = parser.add_argument_group('archive options') group.add_argument('--keep', help='Keep files after adding to archive', default=False, action='store_true') group.add_argument('--recursive', help='Iterate through subdirectories', default=False, action='store_true') group.add_argument( '--update', help= 'Update asset if newer version available, (must call gips_process to regenerate products', default=False, action='store_true') args = parser0.parse_args() try: print title cls = import_data_class(args.command) cls.Asset.archive(**vars(args)) except Exception, e: import traceback VerboseOut(traceback.format_exc(), 4) print 'Data archive error: %s' % e
def main(): title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser = parser0.add_inventory_parser() group = parser.add_argument_group('additional inventory options') group.add_argument('--md', help='Show dates using MM-DD', action='store_true', default=False) group.add_argument( '--rectify', help= 'Instead of displaying or fetching inventory, rectify the inventory ' 'database by comparing it against the present state of the data repos.', action='store_true', default=False) args = parser0.parse_args() cls = utils.gips_script_setup(args.command, args.stop_on_error) with utils.error_handler(): print(title) if args.rectify: if not orm.use_orm(): raise ValueError("--rectify can only be used if" " GIPS_ORM = True.") for k, v in vars(args).items(): # Let the user know not to expect other options to effect rectify if v and k not in ('rectify', 'verbose', 'command'): msg = "INFO: Option '--{}' is has no effect on --rectify." utils.verbose_out(msg.format(k), 1) print("Rectifying inventory DB with filesystem archive:") print("Rectifying assets:") dbinv.rectify_assets(cls.Asset) print("Rectifying products:") dbinv.rectify_products(cls) return extents = SpatialExtent.factory(cls, site=args.site, rastermask=args.rastermask, key=args.key, where=args.where, tiles=args.tiles, pcov=args.pcov, ptile=args.ptile) for extent in extents: inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args)) inv.pprint(md=args.md, size=args.size) utils.gips_exit( ) # produce a summary error report then quit with a proper exit status
def main(): import gips title = 'GIPS Configuration Utility (v%s)' % (version) try: parser = GIPSParser(description=title, datasources=False) subparser = parser.add_subparsers(dest='command') subparser.add_parser('print', help='Print current settings') p = subparser.add_parser( 'env', help='Configure GIPS repositories in this environment') p.add_argument('-r', '--repos', help='Top level directory for repositories', default='/data/repos') p.add_argument( '-e', '--email', help='Set email address (used for anonymous FTP sources)', default='') p = subparser.add_parser( 'user', help= 'Configure GIPS repositories for this user (for per user customizations)' ) #p.add_argument('-e', '--email', help='Set email address (used for anonymous FTP sources)') #h = 'Install full configuration file without inheriting from environment settings' #p.add_argument('-f', '--full', help=h, default=False, action='store_true') args = parser.parse_args() print title if args.command == 'print': try: from gips.utils import settings s = settings() for v in dir(s): if not v.startswith('__') and v != 'gips': print print v exec('pprint.pprint(s.%s)' % v) except Exception, e: #print traceback.format_exc() print 'Unable to access settings. Run `gips_config env`' elif args.command == 'env': try: cfgfile = create_environment_settings(args.repos, email=args.email) print 'Environment settings file: %s' % cfgfile print 'Creating repository directories' create_repos() except Exception, e: print traceback.format_exc() print 'Could not create environment settings: %s' % e
def main(): title = Colors.BOLD + 'GIPS Data Repositories (v%s)' % (gipsversion) + Colors.OFF # argument parsing try: parser = GIPSParser(description=title) args = parser.parse_args() print title cls = import_data_class(args.command) cls.print_products() except Exception, e: import traceback VerboseOut(traceback.format_exc(), 4) print 'GIPS Data Repository error: %s' % e
def main(): title = Colors.BOLD + "GIPS Data Project (v%s)" % __version__ + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser0.add_inventory_parser(site_required=True) parser0.add_process_parser() parser0.add_project_parser() parser0.add_warp_parser() args = parser0.parse_args() try: print title cls = import_data_class(args.command) extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile) # create tld: SITENAME--KEY_DATATYPE_SUFFIX if args.notld: tld = args.outdir else: key = "" if args.key == "" else "--" + args.key suffix = "" if args.suffix == "" else "_" + args.suffix res = "" if args.res is None else "_%sx%s" % (args.res[0], args.res[1]) bname = extents[0].site.LayerName() + key + res + "_" + args.command + suffix tld = os.path.join(args.outdir, bname) for extent in extents: t_extent = TemporalExtent(args.dates, args.days) inv = DataInventory(cls, extent, t_extent, **vars(args)) datadir = os.path.join(tld, extent.site.Value()) if inv.numfiles > 0: inv.mosaic( datadir=datadir, tree=args.tree, overwrite=args.overwrite, res=args.res, interpolation=args.interpolation, crop=args.crop, alltouch=args.alltouch, ) inv = ProjectInventory(datadir) inv.pprint() else: VerboseOut("No data found for {} within temporal extent {}".format(str(t_extent), str(t_extent)), 2) except Exception as e: import traceback VerboseOut(traceback.format_exc(), 4) print "Data Project error: %s" % e
def main(): title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF # argument parsing parser = GIPSParser(description=title) group = parser.add_argument_group('archive options') group.add_argument('--keep', help='Keep files after adding to archive', default=False, action='store_true') group.add_argument('--recursive', help='Iterate through subdirectories', default=False, action='store_true') group.add_argument( '--update', help= 'Update asset if newer version available, (must call gips_process to regenerate products', default=False, action='store_true') group.add_argument( '--path', default='.', help='Path to search for files to archive, defaults to `.`') args = parser.parse_args() utils.gips_script_setup(None, args.stop_on_error) with utils.error_handler('Data archive error'): print title cls = import_data_class(args.command) orm.setup() # set up DB orm in case it's needed for Asset.archive() archived_assets = cls.archive_assets(args.path, args.recursive, args.keep, args.update) # if DB inventory is enabled, update it to contain the newly archived assets if orm.use_orm(): for a in archived_assets: dbinv.update_or_add_asset(asset=a.asset, sensor=a.sensor, tile=a.tile, date=a.date, name=a.archived_filename, driver=cls.name.lower()) utils.gips_exit()
def main(): title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser0.add_inventory_parser() parser0.add_process_parser() args = parser0.parse_args() try: print title cls = import_data_class(args.command) extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile) for extent in extents: inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args)) inv.process(overwrite=args.overwrite) except Exception, e: import traceback VerboseOut(traceback.format_exc(), 4) print 'Data processing error: %s' % e
def main(): title = Colors.BOLD + 'GIPS Data Repositories (v%s)' % ( gipsversion) + Colors.OFF # argument parsing parser = GIPSParser(description=title) args = parser.parse_args() cls = utils.gips_script_setup(args.command, args.stop_on_error) print title with utils.error_handler(): cls.print_products() utils.gips_exit( ) # produce a summary error report then quit with a proper exit status
def main(): title = Colors.BOLD + 'GIPS Image Statistics (v%s)' % __version__ + Colors.OFF parser0 = GIPSParser(datasources=False, description=title) parser0.add_default_parser() parser0.add_projdir_parser() group = parser0.add_argument_group('masking options') args = parser0.parse_args() # TODO - check that at least 1 of filemask or pmask is supplied try: print title header = ['min', 'max', 'mean', 'sd', 'skew', 'count'] for projdir in args.projdir: VerboseOut('Stats for Project directory: %s' % projdir, 1) inv = ProjectInventory(projdir, args.products) files = {} for date in inv.dates: VerboseOut('Calculating statistics for %s' % date) for p in inv.products(date): img = inv[date].open(p) if p not in files.keys(): files[p] = open( os.path.join(projdir, p + '_stats.txt'), 'w') # write header files[p].write('date ') if img.NumBands() == 1: files[p].write(' '.join(header)) else: for band in img: h = [ band.Description() + "-" + a for a in header ] files[p].write(' '.join(h) + ' ') files[p].write('\n') # print date and stats files[p].write(date.strftime('%Y-%j')) for band in img: stats = band.Stats() [files[p].write(' ' + str(s)) for s in stats] files[p].write(' ') files[p].write('\n') img = None for f in files: files[f].close() except Exception, e: import traceback VerboseOut(traceback.format_exc(), 4) print 'Error: %s' % e
def main(): title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser0.add_inventory_parser() parser0.add_process_parser() args = parser0.parse_args() try: print title cls = import_data_class(args.command) extents = SpatialExtent.factory( cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile ) batchargs = None if args.batchout: tdl = [] batchargs = '--chunksize ' + str(args.chunksize) batchargs += ' --format ' + str(args.format) if args.overwrite: batchargs += ' --overwrite ' if args.products: batchargs += ' -p ' + ' '.join(args.products) for extent in extents: inv = DataInventory( cls, extent, TemporalExtent(args.dates, args.days), **vars(args) ) if args.batchout: tdl = reduce( list.__add__, map( lambda tiles: [ args.command + ' -t ' + str(tile) + ' -d ' + str(tiles.date) + ' ' + batchargs + '\n' for tile in tiles.tiles.keys() ], inv.data.values(), ), tdl ) else: inv.process(overwrite=args.overwrite) if args.batchout: with open(args.batchout, 'w') as ofile: ofile.writelines(tdl) except Exception, e: import traceback VerboseOut(traceback.format_exc(), 4) print 'Data processing error: %s' % e
def main(): title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser0.add_inventory_parser() parser0.add_process_parser() parser0.add_project_parser() parser0.add_warp_parser() args = parser0.parse_args() try: print title cls = data_class(args.command) # create tld: DATATYPE_tiles_RESOLUTION_SUFFIX if args.notld: tld = args.outdir else: tld = os.path.join(args.outdir, '%s_tiles' % args.command) if args.res is not None: tld = tld + '_%sx%s' % (args.res[0], args.res[1]) if args.suffix != '': tld = tld + '_' + args.suffix mkdir(tld) extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile) for extent in extents: inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args)) for date in inv.dates: for tid in inv[date].tiles: # make sure back-end tiles are processed inv[date].tiles[tid].process(args.products, overwrite=False) # warp the tiles inv[date].tiles[tid].copy(tld, args.products, inv.spatial.site, args.res, args.interpolation, args.crop, args.overwrite, args.tree) except Exception, e: import traceback VerboseOut(traceback.format_exc(), 4) print 'Warp Tiles error: %s' % e
def main(): import gips title = 'GIPS Configuration Utility (v%s)' % (version) parser = GIPSParser(description=title, datasources=False) subparser = parser.add_subparsers(dest='command') subparser.add_parser('print', help='Print current settings') p = subparser.add_parser('env', help='Configure GIPS repositories in this environment') p.add_argument('-r', '--repos', help='Top level directory for repositories', default='/data/repos') p.add_argument('-e', '--email', help='Set email address (used for anonymous FTP sources)', default='') p = subparser.add_parser('user', help='Configure GIPS repositories for this user (for per user customizations)') #p.add_argument('-e', '--email', help='Set email address (used for anonymous FTP sources)') #h = 'Install full configuration file without inheriting from environment settings' #p.add_argument('-f', '--full', help=h, default=False, action='store_true') args = parser.parse_args() print title if args.command == 'print': try: from gips.utils import settings s = settings() for v in dir(s): if not v.startswith('__') and v != 'gips': print print v exec('pprint.pprint(s.%s)' % v) except Exception as e: # print traceback.format_exc() print 'Unable to access settings: {}'.format(e) sys.exit(1) elif args.command == 'env': try: cfgfile = create_environment_settings(args.repos, email=args.email) print 'Environment settings file: %s' % cfgfile print 'Creating repository directories' create_repos() except Exception, e: print traceback.format_exc() print 'Could not create environment settings: %s' % e
def main(): title = Colors.BOLD + "GIPS Image Statistics (v%s)" % __version__ + Colors.OFF parser0 = GIPSParser(datasources=False, description=title) parser0.add_default_parser() parser0.add_projdir_parser() group = parser0.add_argument_group("masking options") args = parser0.parse_args() # TODO - check that at least 1 of filemask or pmask is supplied try: print title header = ["min", "max", "mean", "sd", "skew", "count"] for projdir in args.projdir: VerboseOut("Stats for Project directory: %s" % projdir, 1) inv = ProjectInventory(projdir, args.products) files = {} for date in inv.dates: VerboseOut("Calculating statistics for %s" % date) for p in inv.products(date): img = inv[date].open(p) if p not in files.keys(): files[p] = open(os.path.join(projdir, p + "_stats.txt"), "w") # write header files[p].write("date ") if img.NumBands() == 1: files[p].write(" ".join(header)) else: for band in img: h = [band.Description() + "-" + a for a in header] files[p].write(" ".join(h) + " ") files[p].write("\n") # print date and stats files[p].write(date.strftime("%Y-%j")) for band in img: stats = band.Stats() [files[p].write(" " + str(s)) for s in stats] files[p].write(" ") files[p].write("\n") img = None for f in files: files[f].close() except Exception, e: import traceback VerboseOut(traceback.format_exc(), 4) print "Error: %s" % e
def main(): title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser = parser0.add_inventory_parser() group = parser.add_argument_group('inventory display') group.add_argument('--md', help='Show dates using MM-DD', action='store_true', default=False) args = parser0.parse_args() try: print title cls = data_class(args.command) extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile) for extent in extents: inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args)) inv.pprint(md=args.md) except Exception, e: import traceback VerboseOut(traceback.format_exc(), 4) print 'Data inventory error: %s' % e
def main(): title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser = parser0.add_inventory_parser() group = parser.add_argument_group('inventory display') group.add_argument('--md', help='Show dates using MM-DD', action='store_true', default=False) args = parser0.parse_args() try: print title cls = import_data_class(args.command) extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile) for extent in extents: inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args)) inv.pprint(md=args.md) except Exception, e: import traceback VerboseOut(traceback.format_exc(), 4) print 'Data inventory error: %s' % e
def main(): title = Colors.BOLD + 'GIPS Image Statistics (v%s)' % __version__ + Colors.OFF parser0 = GIPSParser(datasources=False, description=title) parser0.add_projdir_parser() group = parser0.add_argument_group('masking options') args = parser0.parse_args() utils.gips_script_setup(stop_on_error=args.stop_on_error) print title # TODO - check that at least 1 of filemask or pmask is supplied header = ['date', 'band', 'min', 'max', 'mean', 'sd', 'skew', 'count'] with utils.error_handler(): for projdir in args.projdir: VerboseOut('Stats for Project directory: %s' % projdir, 1) inv = ProjectInventory(projdir, args.products) p_dates = {} # map each product to its list of valid dates for date in inv.dates: for p in inv.products(date): p_dates.setdefault(p, []).append(date) p_dates = {p: sorted(dl) for p, dl in p_dates.items()} for p_type, valid_dates in p_dates.items(): stats_fn = os.path.join(projdir, p_type + '_stats.txt') with open(stats_fn, 'w') as stats_fo: sf = getattr(utils.settings(), 'STATS_FORMAT', {}) writer = csv.writer(stats_fo, **sf) writer.writerow(header) # print date, band description, and stats for date in valid_dates: img = inv[date].open(p_type) date_str = date.strftime('%Y-%j') utils.verbose_out('Computing stats for {} {}'.format( p_type, date_str), 2) for b in img: stats = [str(s) for s in b.Stats()] writer.writerow( [date_str, b.Description()] + stats) img = None utils.gips_exit() # produce a summary error report then quit with a proper exit status
def main(): title = Colors.BOLD + 'GIPS Project Masking (v%s)' % __version__ + Colors.OFF parser = GIPSParser(datasources=False, description=title) parser.add_projdir_parser() group = parser.add_argument_group('masking options') group.add_argument('--filemask', help='Mask all files with this static mask', default=None) group.add_argument('--pmask', help='Mask files with this corresponding product', nargs='*', default=[]) group.add_argument('--invert', help='Invert the masks from corresponding products', nargs='*', default=[]) h = 'Write mask to original image instead of creating new image' group.add_argument('--original', help=h, default=False, action='store_true') h = 'Overwrite existing files when creating new' group.add_argument('--overwrite', help=h, default=False, action='store_true') h = 'Suffix to apply to masked file (not compatible with --original)' group.add_argument('--suffix', help=h, default='-masked') args = parser.parse_args() # TODO - check that at least 1 of filemask or pmask is supplied utils.gips_script_setup(None, args.stop_on_error) with utils.error_handler('Masking error'): VerboseOut(title) for projdir in args.projdir: if args.filemask is not None: mask_file = gippy.GeoImage(args.filemask) inv = ProjectInventory(projdir, args.products) for date in inv.dates: VerboseOut('Masking files from %s' % date) if args.filemask is None and args.pmask == []: available_masks = inv[date].masks() else: available_masks = inv[date].masks(args.pmask) for p in inv.products(date): # don't mask any masks if p in available_masks: continue meta = '' update = True if args.original else False img = inv[date].open(p, update=update) if args.filemask is not None: img.AddMask(mask_file[0]) meta = basename(args.filemask) + ' ' for mask in available_masks: mask_img = inv[date].open(mask)[0] if mask in args.invert: mask_img.SetNoData(utils.np.nan) mask_img = mask_img.BXOR(1) meta += 'inverted-' img.AddMask(mask_img) meta = meta + basename(inv[date][mask]) + ' ' if meta != '': if args.original: VerboseOut(' %s' % (img.Basename()), 2) img.Process() img.SetMeta('MASKS', meta) else: fout = os.path.splitext( img.Filename())[0] + args.suffix + '.tif' if not os.path.exists(fout) or args.overwrite: VerboseOut( ' %s -> %s' % (img.Basename(), basename(fout)), 2) imgout = img.Process(fout) imgout.SetMeta('MASKS', meta) imgout = None img = None mask_file = None utils.gips_exit()
def main(): title = Colors.BOLD + 'GIPS Project Masking (v%s)' % __version__ + Colors.OFF parser0 = GIPSParser(datasources=False, description=title) parser0.add_default_parser() parser0.add_projdir_parser() group = parser0.add_argument_group('masking options') group.add_argument('--filemask', help='Mask all files with this static mask', default=None) group.add_argument('--pmask', help='Mask files with this corresponding product', nargs='*', default=[]) h = 'Write mask to original image instead of creating new image' group.add_argument('--original', help=h, default=False, action='store_true') h = 'Overwrite existing files when creating new' group.add_argument('--overwrite', help=h, default=False, action='store_true') h = 'Suffix to apply to masked file (not compatible with --original)' group.add_argument('--suffix', help=h, default='-masked') #parser0.add_argument('-i', '--invert', help='Invert mask (0->1, 1->0)', default=False, action='store_true') #parser0.add_argument('--value', help='Mask == val', default=1) args = parser0.parse_args() # TODO - check that at least 1 of filemask or pmask is supplied try: print title for projdir in args.projdir: if args.filemask is not None: mask_file = gippy.GeoImage(args.filemask) inv = ProjectInventory(projdir, args.products) for date in inv.dates: VerboseOut('Masking files from %s' % date) available_masks = inv[date].masks(args.pmask) for p in inv.products(date): # don't mask any masks if p in available_masks: continue meta = '' update = True if args.original else False img = inv[date].open(p, update=update) if args.filemask is not None: img.AddMask(mask_file[0]) meta = basename(args.filemask) + ' ' for mask in available_masks: img.AddMask(inv[date].open(mask)[0]) meta = meta + basename(inv[date][mask]) + ' ' if meta != '': if args.original: VerboseOut(' %s' % (img.Basename()), 2) img.Process() img.SetMeta('MASKS', meta) else: fout = os.path.splitext( img.Filename())[0] + args.suffix + '.tif' if not os.path.exists(fout) or args.overwrite: VerboseOut( ' %s -> %s' % (img.Basename(), basename(fout)), 2) imgout = img.Process(fout) imgout.SetMeta('MASKS', meta) imgout = None img = None mask_file = None except Exception, e: import traceback VerboseOut(traceback.format_exc(), 4) print 'Masking error: %s' % e
def main(): title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser0.add_inventory_parser() parser0.add_process_parser() args = parser0.parse_args() cls = utils.gips_script_setup(args.command, args.stop_on_error) print title with utils.error_handler(): extents = SpatialExtent.factory( cls, site=args.site, rastermask=args.rastermask, key=args.key, where=args.where, tiles=args.tiles, pcov=args.pcov, ptile=args.ptile ) batchargs = None if args.batchout: tdl = [] batchargs = '--chunksize ' + str(args.chunksize) batchargs += ' --format ' + str(args.format) batchargs += ' --numprocs ' + str(args.numprocs) batchargs += ' --verbose ' + str(args.verbose) if args.overwrite: batchargs += ' --overwrite ' if args.products: batchargs += ' -p ' + ' '.join(args.products) for extent in extents: inv = DataInventory( cls, extent, TemporalExtent(args.dates, args.days), **vars(args) ) if args.batchout: def get_commands(tiles_obj): commands = [] for tile in tiles_obj.tiles.keys(): needed = any([p not in [k for sen, k in tiles_obj.tiles[tile].filenames.keys()] for p in args.products]) if not needed: continue commands.append(args.command + ' -t ' + str(tile) + ' -d ' + str(tiles_obj.date) + ' ' + batchargs + '\n') return commands tdl = reduce( list.__add__, map( get_commands, inv.data.values() ), tdl ) else: inv.process(overwrite=args.overwrite) if args.batchout: with open(args.batchout, 'w') as ofile: ofile.writelines(tdl) utils.gips_exit() # produce a summary error report then quit with a proper exit status
def main(): title = Colors.BOLD + 'GIPS Data Export (v%s)' % __version__ + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser0.add_inventory_parser(site_required=True) parser0.add_process_parser() parser0.add_project_parser() parser0.add_warp_parser() args = parser0.parse_args() cls = utils.gips_script_setup(args.command, args.stop_on_error) print title with utils.error_handler(): extents = SpatialExtent.factory(cls, site=args.site, rastermask=args.rastermask, key=args.key, where=args.where, tiles=args.tiles, pcov=args.pcov, ptile=args.ptile) # create tld: SITENAME--KEY_DATATYPE_SUFFIX if args.notld: tld = args.outdir else: key = '' if args.key == '' else '--' + args.key suffix = '' if args.suffix == '' else '_' + args.suffix res = '' if args.res is None else '_%sx%s' % (args.res[0], args.res[1]) bname = (extents[0].site.LayerName() + key + res + '_' + args.command + suffix) tld = os.path.join(args.outdir, bname) for extent in extents: t_extent = TemporalExtent(args.dates, args.days) inv = DataInventory(cls, extent, t_extent, **vars(args)) datadir = os.path.join(tld, extent.site.Value()) if inv.numfiles > 0: inv.mosaic( datadir=datadir, tree=args.tree, overwrite=args.overwrite, res=args.res, interpolation=args.interpolation, crop=args.crop, alltouch=args.alltouch, process=(not args.dont_process), ) inv = ProjectInventory(datadir) inv.pprint() else: VerboseOut( 'No data found for {} within temporal extent {}'.format( str(t_extent), str(t_extent)), 2, ) utils.gips_exit( ) # produce a summary error report then quit with a proper exit status
def main(): title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser0.add_inventory_parser() parser0.add_process_parser() parser0.add_project_parser() parser0.add_warp_parser() args = parser0.parse_args() cls = utils.gips_script_setup(args.command, args.stop_on_error) print title with utils.error_handler(): # create output directory if needed # tld is "{}_tiles_{}_{}".format(DATATYPE, RESOLUTION, SUFFIX) if args.notld: tld = args.outdir else: tld = os.path.join(args.outdir, '%s_tiles' % args.command) if args.res is not None: tld = tld + '_%sx%s' % (args.res[0], args.res[1]) if args.suffix != '': tld = tld + '_' + args.suffix mkdir(tld) extents = SpatialExtent.factory(cls, site=args.site, rastermask=args.rastermask, key=args.key, where=args.where, tiles=args.tiles, pcov=args.pcov, ptile=args.ptile) for extent in extents: inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args)) for date in inv.dates: for tid in inv[date].tiles: # make sure back-end tiles are processed inv[date].tiles[tid].process(args.products, overwrite=False) # warp the tiles & copy into place in the output dir inv[date].tiles[tid].copy(tld, args.products, inv.spatial.site, args.res, args.interpolation, args.crop, args.overwrite, args.tree) utils.gips_exit( ) # produce a summary error report then quit with a proper exit status
def main(): title = Colors.BOLD + 'GIPS Data Project (v%s)' % __version__ + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser0.add_inventory_parser(site_required=True) parser0.add_process_parser() parser0.add_project_parser() parser0.add_warp_parser() args = parser0.parse_args() try: print title cls = import_data_class(args.command) extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile) # create tld: SITENAME--KEY_DATATYPE_SUFFIX if args.notld: tld = args.outdir else: key = '' if args.key == '' else '--' + args.key suffix = '' if args.suffix == '' else '_' + args.suffix res = '' if args.res is None else '_%sx%s' % (args.res[0], args.res[1]) bname = extents[0].site.LayerName() + key + res + '_' + args.command + suffix tld = os.path.join(args.outdir, bname) for extent in extents: inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args)) datadir = os.path.join(tld, extent.site.Value()) if inv.numfiles > 0: inv.mosaic(datadir=datadir, tree=args.tree, overwrite=args.overwrite, res=args.res, interpolation=args.interpolation, crop=args.crop) if not args.tree: inv = ProjectInventory(datadir) inv.pprint() except Exception, e: import traceback VerboseOut(traceback.format_exc(), 4) print 'Data Project error: %s' % e
def main(): title = Colors.BOLD + 'GIPS Project Masking (v%s)' % __version__ + Colors.OFF parser0 = GIPSParser(datasources=False, description=title) parser0.add_default_parser() parser0.add_projdir_parser() group = parser0.add_argument_group('masking options') group.add_argument('--filemask', help='Mask all files with this static mask', default=None) group.add_argument('--pmask', help='Mask files with this corresponding product', nargs='*', default=[]) h = 'Write mask to original image instead of creating new image' group.add_argument('--original', help=h, default=False, action='store_true') h = 'Overwrite existing files when creating new' group.add_argument('--overwrite', help=h, default=False, action='store_true') h = 'Suffix to apply to masked file (not compatible with --original)' group.add_argument('--suffix', help=h, default='-masked') #parser0.add_argument('-i', '--invert', help='Invert mask (0->1, 1->0)', default=False, action='store_true') #parser0.add_argument('--value', help='Mask == val', default=1) args = parser0.parse_args() # TODO - check that at least 1 of filemask or pmask is supplied try: VerboseOut(title) for projdir in args.projdir: if args.filemask is not None: mask_file = gippy.GeoImage(args.filemask) inv = ProjectInventory(projdir, args.products) for date in inv.dates: VerboseOut('Masking files from %s' % date) if args.filemask is None and args.pmask == []: available_masks = inv[date].masks() else: available_masks = inv[date].masks(args.pmask) for p in inv.products(date): # don't mask any masks if p in available_masks: continue meta = '' update = True if args.original else False img = inv[date].open(p, update=update) if args.filemask is not None: img.AddMask(mask_file[0]) meta = basename(args.filemask) + ' ' for mask in available_masks: img.AddMask(inv[date].open(mask)[0]) meta = meta + basename(inv[date][mask]) + ' ' if meta != '': if args.original: VerboseOut(' %s' % (img.Basename()), 2) img.Process() img.SetMeta('MASKS', meta) else: fout = os.path.splitext(img.Filename())[0] + args.suffix + '.tif' if not os.path.exists(fout) or args.overwrite: VerboseOut(' %s -> %s' % (img.Basename(), basename(fout)), 2) imgout = img.Process(fout) imgout.SetMeta('MASKS', meta) imgout = None img = None mask_file = None except Exception, e: import traceback VerboseOut(traceback.format_exc(), 4) print 'Masking error: %s' % e
def main(): title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser0.add_inventory_parser() parser0.add_process_parser() parser0.add_project_parser() parser0.add_warp_parser() args = parser0.parse_args() try: print title cls = import_data_class(args.command) # create tld: DATATYPE_tiles_RESOLUTION_SUFFIX if args.notld: tld = args.outdir else: tld = os.path.join(args.outdir, '%s_tiles' % args.command) if args.res is not None: tld = tld + '_%sx%s' % (args.res[0], args.res[1]) if args.suffix != '': tld = tld + '_' + args.suffix mkdir(tld) extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile) for extent in extents: inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args)) for date in inv.dates: for tid in inv[date].tiles: # make sure back-end tiles are processed inv[date].tiles[tid].process(args.products, overwrite=False) # warp the tiles inv[date].tiles[tid].copy(tld, args.products, inv.spatial.site, args.res, args.interpolation, args.crop, args.overwrite, args.tree) except Exception, e: import traceback VerboseOut(traceback.format_exc(), 4) print 'Warp Tiles error: %s' % e
def main(): import gips title = 'GIPS Configuration Utility (v%s)' % (version) parser = GIPSParser(description=title, datasources=False) subparser = parser.add_subparsers(dest='command') subparser.add_parser('print', help='Print current settings') p = subparser.add_parser( 'env', help='Configure GIPS repositories in this environment') p.add_argument('-r', '--repos', help='Top level directory for repositories', default='/data/repos') p.add_argument('-e', '--email', help='Set email address (used for anonymous FTP sources)', default='') p = subparser.add_parser( 'user', help= 'Configure GIPS repositories for this user (for per user customizations)' ) #p.add_argument('-e', '--email', help='Set email address (used for anonymous FTP sources)') #h = 'Install full configuration file without inheriting from environment settings' #p.add_argument('-f', '--full', help=h, default=False, action='store_true') args = parser.parse_args() print title utils.gips_script_setup( driver_string=None, # NOTE: no driver string for gips_config stop_on_error=args.stop_on_error, setup_orm=False, # NOTE: ORM cannot be setup before `gips_config env` ) # has been run if args.command == 'print': with utils.error_handler('Unable to access settings'): from gips.utils import settings s = settings() for v in dir(s): if not v.startswith('__') and v != 'gips': print print v exec('pprint.pprint(s.%s)' % v) elif args.command == 'env': with utils.error_handler('Could not create environment settings'): created_cf, cfgfile = create_environment_settings(args.repos, email=args.email) elif args.command == 'user': with utils.error_handler('Could not create user settings'): # first try importing environment settings import gips.settings created_cf, cfgfile = create_user_settings() if args.command in ('user', 'env'): msg = ('Wrote new config file: {}.' if created_cf else 'Found existing config, left unmodified: {}.') print msg.format(cfgfile) with utils.error_handler('Could not create repos'): print 'Creating repository directories, if needed.' try: create_repos() except: if created_cf: print( 'Error; removing (likely broken) config file:' ' {}.'.format(cfgfile)) os.remove(cfgfile) raise with utils.error_handler('Could not migrate database'): migrate_database() utils.gips_exit()