def add_data_sources(self):
""" Adds data sources to parser """
subparser = self.add_subparsers(dest='command')
sources = data_sources()
if len(sources) == 0:
verbose_out("There are no available data sources!", 1, sys.stderr)
for src, desc in sources.items():
p = subparser.add_parser(src,
help=desc,
parents=self.parent_parsers)
utils.import_data_class(src).add_filter_args(p)
def main():
title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser = parser0.add_default_parser()
group = parser.add_argument_group('archive options')
group.add_argument('--keep',
help='Keep files after adding to archive',
default=False,
action='store_true')
group.add_argument('--recursive',
help='Iterate through subdirectories',
default=False,
action='store_true')
group.add_argument(
'--update',
help=
'Update asset if newer version available, (must call gips_process to regenerate products',
default=False,
action='store_true')
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
cls.Asset.archive(**vars(args))
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data archive error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser = parser0.add_default_parser()
group = parser.add_argument_group('archive options')
group.add_argument('--keep', help='Keep files after adding to archive', default=False, action='store_true')
group.add_argument('--recursive', help='Iterate through subdirectories', default=False, action='store_true')
group.add_argument(
'--update',
help='Update asset if newer version available, (must call gips_process to regenerate products',
default=False,
action='store_true'
)
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
cls.Asset.archive(**vars(args))
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data archive error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
args.tiles, args.pcov, args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
inv.process(overwrite=args.overwrite)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data processing error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(
cls, args.site, args.key, args.where, args.tiles, args.pcov,
args.ptile
)
batchargs = None
if args.batchout:
tdl = []
batchargs = '--chunksize ' + str(args.chunksize)
batchargs += ' --format ' + str(args.format)
if args.overwrite:
batchargs += ' --overwrite '
if args.products:
batchargs += ' -p ' + ' '.join(args.products)
for extent in extents:
inv = DataInventory(
cls, extent,
TemporalExtent(args.dates, args.days), **vars(args)
)
if args.batchout:
tdl = reduce(
list.__add__,
map(
lambda tiles: [
args.command + ' -t ' + str(tile) +
' -d ' + str(tiles.date) + ' ' +
batchargs + '\n'
for tile in tiles.tiles.keys()
],
inv.data.values(),
),
tdl
)
else:
inv.process(overwrite=args.overwrite)
if args.batchout:
with open(args.batchout, 'w') as ofile:
ofile.writelines(tdl)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data processing error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Project (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser(site_required=True)
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
args.tiles, args.pcov, args.ptile)
# create tld: SITENAME--KEY_DATATYPE_SUFFIX
if args.notld:
tld = args.outdir
else:
key = '' if args.key == '' else '--' + args.key
suffix = '' if args.suffix == '' else '_' + args.suffix
res = '' if args.res is None else '_%sx%s' % (args.res[0],
args.res[1])
bname = (extents[0].site.LayerName() + key + res + '_' +
args.command + suffix)
tld = os.path.join(args.outdir, bname)
for extent in extents:
t_extent = TemporalExtent(args.dates, args.days)
inv = DataInventory(cls, extent, t_extent, **vars(args))
datadir = os.path.join(tld, extent.site.Value())
if inv.numfiles > 0:
inv.mosaic(
datadir=datadir,
tree=args.tree,
overwrite=args.overwrite,
res=args.res,
interpolation=args.interpolation,
crop=args.crop,
alltouch=args.alltouch,
)
inv = ProjectInventory(datadir)
inv.pprint()
else:
VerboseOut(
'No data found for {} within temporal extent {}'.format(
str(t_extent), str(t_extent)),
2,
)
except Exception as e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data Project error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Repositories (v%s)' % (gipsversion) + Colors.OFF
# argument parsing
try:
parser = GIPSParser(description=title)
args = parser.parse_args()
print title
cls = import_data_class(args.command)
cls.print_products()
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'GIPS Data Repository error: %s' % e
def main():
title = Colors.BOLD + "GIPS Data Project (v%s)" % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser(site_required=True)
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)
# create tld: SITENAME--KEY_DATATYPE_SUFFIX
if args.notld:
tld = args.outdir
else:
key = "" if args.key == "" else "--" + args.key
suffix = "" if args.suffix == "" else "_" + args.suffix
res = "" if args.res is None else "_%sx%s" % (args.res[0], args.res[1])
bname = extents[0].site.LayerName() + key + res + "_" + args.command + suffix
tld = os.path.join(args.outdir, bname)
for extent in extents:
t_extent = TemporalExtent(args.dates, args.days)
inv = DataInventory(cls, extent, t_extent, **vars(args))
datadir = os.path.join(tld, extent.site.Value())
if inv.numfiles > 0:
inv.mosaic(
datadir=datadir,
tree=args.tree,
overwrite=args.overwrite,
res=args.res,
interpolation=args.interpolation,
crop=args.crop,
alltouch=args.alltouch,
)
inv = ProjectInventory(datadir)
inv.pprint()
else:
VerboseOut("No data found for {} within temporal extent {}".format(str(t_extent), str(t_extent)), 2)
except Exception as e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print "Data Project error: %s" % e
def main():
title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
# create tld: DATATYPE_tiles_RESOLUTION_SUFFIX
if args.notld:
tld = args.outdir
else:
tld = os.path.join(args.outdir, '%s_tiles' % args.command)
if args.res is not None:
tld = tld + '_%sx%s' % (args.res[0], args.res[1])
if args.suffix != '':
tld = tld + '_' + args.suffix
mkdir(tld)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
args.tiles, args.pcov, args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
for date in inv.dates:
for tid in inv[date].tiles:
# make sure back-end tiles are processed
inv[date].tiles[tid].process(args.products,
overwrite=False)
# warp the tiles
inv[date].tiles[tid].copy(tld, args.products,
inv.spatial.site, args.res,
args.interpolation, args.crop,
args.overwrite, args.tree)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Warp Tiles error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser = GIPSParser(description=title)
group = parser.add_argument_group('archive options')
group.add_argument('--keep',
help='Keep files after adding to archive',
default=False,
action='store_true')
group.add_argument('--recursive',
help='Iterate through subdirectories',
default=False,
action='store_true')
group.add_argument(
'--update',
help=
'Update asset if newer version available, (must call gips_process to regenerate products',
default=False,
action='store_true')
group.add_argument(
'--path',
default='.',
help='Path to search for files to archive, defaults to `.`')
args = parser.parse_args()
utils.gips_script_setup(None, args.stop_on_error)
with utils.error_handler('Data archive error'):
print title
cls = import_data_class(args.command)
orm.setup() # set up DB orm in case it's needed for Asset.archive()
archived_assets = cls.archive_assets(args.path, args.recursive,
args.keep, args.update)
# if DB inventory is enabled, update it to contain the newly archived assets
if orm.use_orm():
for a in archived_assets:
dbinv.update_or_add_asset(asset=a.asset,
sensor=a.sensor,
tile=a.tile,
date=a.date,
name=a.archived_filename,
driver=cls.name.lower())
utils.gips_exit()
def t_raster_spatialspec():
"""Test that shapefile and rastermask return the same tile list"""
with utils.make_temp_dir(prefix='t_gridded_inventory') as td:
rastermask = os.path.join(td, 'nhseacost_mask.tif')
rasterize = ('gdal_rasterize -burn 1 -ot Byte -tr 30 30 {} {}'.format(
NH_SHP_PATH, rastermask))
status, output = commands.getstatusoutput(rasterize)
assert status == 0
cls = utils.import_data_class('modis')
s_extents = SpatialExtent.factory(cls, site=NH_SHP_PATH)
r_extents = SpatialExtent.factory(cls, rastermask=rastermask)
assert len(s_extents) == len(r_extents)
for i in range(len(r_extents)):
assert s_extents[i].tiles == r_extents[i].tiles
def main():
title = Colors.BOLD + 'GIPS Data Project (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser(site_required=True)
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)
# create tld: SITENAME--KEY_DATATYPE_SUFFIX
if args.notld:
tld = args.outdir
else:
key = '' if args.key == '' else '--' + args.key
suffix = '' if args.suffix == '' else '_' + args.suffix
res = '' if args.res is None else '_%sx%s' % (args.res[0], args.res[1])
bname = extents[0].site.LayerName() + key + res + '_' + args.command + suffix
tld = os.path.join(args.outdir, bname)
for extent in extents:
inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
datadir = os.path.join(tld, extent.site.Value())
if inv.numfiles > 0:
inv.mosaic(datadir=datadir, tree=args.tree, overwrite=args.overwrite,
res=args.res, interpolation=args.interpolation, crop=args.crop)
if not args.tree:
inv = ProjectInventory(datadir)
inv.pprint()
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data Project error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
# create tld: DATATYPE_tiles_RESOLUTION_SUFFIX
if args.notld:
tld = args.outdir
else:
tld = os.path.join(args.outdir, '%s_tiles' % args.command)
if args.res is not None:
tld = tld + '_%sx%s' % (args.res[0], args.res[1])
if args.suffix != '':
tld = tld + '_' + args.suffix
mkdir(tld)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)
for extent in extents:
inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
for date in inv.dates:
for tid in inv[date].tiles:
# make sure back-end tiles are processed
inv[date].tiles[tid].process(args.products, overwrite=False)
# warp the tiles
inv[date].tiles[tid].copy(tld, args.products, inv.spatial.site,
args.res, args.interpolation, args.crop, args.overwrite, args.tree)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Warp Tiles error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)
for extent in extents:
inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
inv.process(overwrite=args.overwrite)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data processing error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser = parser0.add_inventory_parser()
group = parser.add_argument_group('inventory display')
group.add_argument('--md', help='Show dates using MM-DD', action='store_true', default=False)
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
args.tiles, args.pcov, args.ptile)
for extent in extents:
inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
inv.pprint(md=args.md)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data inventory error: %s' % e