def main():
title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser = parser0.add_default_parser()
group = parser.add_argument_group('archive options')
group.add_argument('--keep', help='Keep files after adding to archive', default=False, action='store_true')
group.add_argument('--recursive', help='Iterate through subdirectories', default=False, action='store_true')
group.add_argument(
'--update',
help='Update asset if newer version available, (must call gips_process to regenerate products',
default=False,
action='store_true'
)
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
cls.Asset.archive(**vars(args))
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data archive error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser = parser0.add_default_parser()
group = parser.add_argument_group('archive options')
group.add_argument('--keep',
help='Keep files after adding to archive',
default=False,
action='store_true')
group.add_argument('--recursive',
help='Iterate through subdirectories',
default=False,
action='store_true')
group.add_argument(
'--update',
help=
'Update asset if newer version available, (must call gips_process to regenerate products',
default=False,
action='store_true')
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
cls.Asset.archive(**vars(args))
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data archive error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser = parser0.add_inventory_parser()
group = parser.add_argument_group('additional inventory options')
group.add_argument('--md',
help='Show dates using MM-DD',
action='store_true',
default=False)
group.add_argument(
'--rectify',
help=
'Instead of displaying or fetching inventory, rectify the inventory '
'database by comparing it against the present state of the data repos.',
action='store_true',
default=False)
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
with utils.error_handler():
print(title)
if args.rectify:
if not orm.use_orm():
raise ValueError("--rectify can only be used if"
" GIPS_ORM = True.")
for k, v in vars(args).items():
# Let the user know not to expect other options to effect rectify
if v and k not in ('rectify', 'verbose', 'command'):
msg = "INFO: Option '--{}' is has no effect on --rectify."
utils.verbose_out(msg.format(k), 1)
print("Rectifying inventory DB with filesystem archive:")
print("Rectifying assets:")
dbinv.rectify_assets(cls.Asset)
print("Rectifying products:")
dbinv.rectify_products(cls)
return
extents = SpatialExtent.factory(cls,
site=args.site,
rastermask=args.rastermask,
key=args.key,
where=args.where,
tiles=args.tiles,
pcov=args.pcov,
ptile=args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
inv.pprint(md=args.md, size=args.size)
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def main():
import gips
title = 'GIPS Configuration Utility (v%s)' % (version)
try:
parser = GIPSParser(description=title, datasources=False)
subparser = parser.add_subparsers(dest='command')
subparser.add_parser('print', help='Print current settings')
p = subparser.add_parser(
'env', help='Configure GIPS repositories in this environment')
p.add_argument('-r',
'--repos',
help='Top level directory for repositories',
default='/data/repos')
p.add_argument(
'-e',
'--email',
help='Set email address (used for anonymous FTP sources)',
default='')
p = subparser.add_parser(
'user',
help=
'Configure GIPS repositories for this user (for per user customizations)'
)
#p.add_argument('-e', '--email', help='Set email address (used for anonymous FTP sources)')
#h = 'Install full configuration file without inheriting from environment settings'
#p.add_argument('-f', '--full', help=h, default=False, action='store_true')
args = parser.parse_args()
print title
if args.command == 'print':
try:
from gips.utils import settings
s = settings()
for v in dir(s):
if not v.startswith('__') and v != 'gips':
print
print v
exec('pprint.pprint(s.%s)' % v)
except Exception, e:
#print traceback.format_exc()
print 'Unable to access settings. Run `gips_config env`'
elif args.command == 'env':
try:
cfgfile = create_environment_settings(args.repos,
email=args.email)
print 'Environment settings file: %s' % cfgfile
print 'Creating repository directories'
create_repos()
except Exception, e:
print traceback.format_exc()
print 'Could not create environment settings: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Repositories (v%s)' % (gipsversion) + Colors.OFF
# argument parsing
try:
parser = GIPSParser(description=title)
args = parser.parse_args()
print title
cls = import_data_class(args.command)
cls.print_products()
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'GIPS Data Repository error: %s' % e
def main():
title = Colors.BOLD + "GIPS Data Project (v%s)" % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser(site_required=True)
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)
# create tld: SITENAME--KEY_DATATYPE_SUFFIX
if args.notld:
tld = args.outdir
else:
key = "" if args.key == "" else "--" + args.key
suffix = "" if args.suffix == "" else "_" + args.suffix
res = "" if args.res is None else "_%sx%s" % (args.res[0], args.res[1])
bname = extents[0].site.LayerName() + key + res + "_" + args.command + suffix
tld = os.path.join(args.outdir, bname)
for extent in extents:
t_extent = TemporalExtent(args.dates, args.days)
inv = DataInventory(cls, extent, t_extent, **vars(args))
datadir = os.path.join(tld, extent.site.Value())
if inv.numfiles > 0:
inv.mosaic(
datadir=datadir,
tree=args.tree,
overwrite=args.overwrite,
res=args.res,
interpolation=args.interpolation,
crop=args.crop,
alltouch=args.alltouch,
)
inv = ProjectInventory(datadir)
inv.pprint()
else:
VerboseOut("No data found for {} within temporal extent {}".format(str(t_extent), str(t_extent)), 2)
except Exception as e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print "Data Project error: %s" % e
def main():
title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser = GIPSParser(description=title)
group = parser.add_argument_group('archive options')
group.add_argument('--keep',
help='Keep files after adding to archive',
default=False,
action='store_true')
group.add_argument('--recursive',
help='Iterate through subdirectories',
default=False,
action='store_true')
group.add_argument(
'--update',
help=
'Update asset if newer version available, (must call gips_process to regenerate products',
default=False,
action='store_true')
group.add_argument(
'--path',
default='.',
help='Path to search for files to archive, defaults to `.`')
args = parser.parse_args()
utils.gips_script_setup(None, args.stop_on_error)
with utils.error_handler('Data archive error'):
print title
cls = import_data_class(args.command)
orm.setup() # set up DB orm in case it's needed for Asset.archive()
archived_assets = cls.archive_assets(args.path, args.recursive,
args.keep, args.update)
# if DB inventory is enabled, update it to contain the newly archived assets
if orm.use_orm():
for a in archived_assets:
dbinv.update_or_add_asset(asset=a.asset,
sensor=a.sensor,
tile=a.tile,
date=a.date,
name=a.archived_filename,
driver=cls.name.lower())
utils.gips_exit()
def main():
title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
args.tiles, args.pcov, args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
inv.process(overwrite=args.overwrite)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data processing error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Repositories (v%s)' % (
gipsversion) + Colors.OFF
# argument parsing
parser = GIPSParser(description=title)
args = parser.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
print title
with utils.error_handler():
cls.print_products()
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def main():
title = Colors.BOLD + 'GIPS Image Statistics (v%s)' % __version__ + Colors.OFF
parser0 = GIPSParser(datasources=False, description=title)
parser0.add_default_parser()
parser0.add_projdir_parser()
group = parser0.add_argument_group('masking options')
args = parser0.parse_args()
# TODO - check that at least 1 of filemask or pmask is supplied
try:
print title
header = ['min', 'max', 'mean', 'sd', 'skew', 'count']
for projdir in args.projdir:
VerboseOut('Stats for Project directory: %s' % projdir, 1)
inv = ProjectInventory(projdir, args.products)
files = {}
for date in inv.dates:
VerboseOut('Calculating statistics for %s' % date)
for p in inv.products(date):
img = inv[date].open(p)
if p not in files.keys():
files[p] = open(
os.path.join(projdir, p + '_stats.txt'), 'w')
# write header
files[p].write('date ')
if img.NumBands() == 1:
files[p].write(' '.join(header))
else:
for band in img:
h = [
band.Description() + "-" + a
for a in header
]
files[p].write(' '.join(h) + ' ')
files[p].write('\n')
# print date and stats
files[p].write(date.strftime('%Y-%j'))
for band in img:
stats = band.Stats()
[files[p].write(' ' + str(s)) for s in stats]
files[p].write(' ')
files[p].write('\n')
img = None
for f in files:
files[f].close()
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(
cls, args.site, args.key, args.where, args.tiles, args.pcov,
args.ptile
)
batchargs = None
if args.batchout:
tdl = []
batchargs = '--chunksize ' + str(args.chunksize)
batchargs += ' --format ' + str(args.format)
if args.overwrite:
batchargs += ' --overwrite '
if args.products:
batchargs += ' -p ' + ' '.join(args.products)
for extent in extents:
inv = DataInventory(
cls, extent,
TemporalExtent(args.dates, args.days), **vars(args)
)
if args.batchout:
tdl = reduce(
list.__add__,
map(
lambda tiles: [
args.command + ' -t ' + str(tile) +
' -d ' + str(tiles.date) + ' ' +
batchargs + '\n'
for tile in tiles.tiles.keys()
],
inv.data.values(),
),
tdl
)
else:
inv.process(overwrite=args.overwrite)
if args.batchout:
with open(args.batchout, 'w') as ofile:
ofile.writelines(tdl)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data processing error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
try:
print title
cls = data_class(args.command)
# create tld: DATATYPE_tiles_RESOLUTION_SUFFIX
if args.notld:
tld = args.outdir
else:
tld = os.path.join(args.outdir, '%s_tiles' % args.command)
if args.res is not None:
tld = tld + '_%sx%s' % (args.res[0], args.res[1])
if args.suffix != '':
tld = tld + '_' + args.suffix
mkdir(tld)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)
for extent in extents:
inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
for date in inv.dates:
for tid in inv[date].tiles:
# make sure back-end tiles are processed
inv[date].tiles[tid].process(args.products, overwrite=False)
# warp the tiles
inv[date].tiles[tid].copy(tld, args.products, inv.spatial.site,
args.res, args.interpolation, args.crop, args.overwrite, args.tree)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Warp Tiles error: %s' % e
def main():
import gips
title = 'GIPS Configuration Utility (v%s)' % (version)
parser = GIPSParser(description=title, datasources=False)
subparser = parser.add_subparsers(dest='command')
subparser.add_parser('print', help='Print current settings')
p = subparser.add_parser('env', help='Configure GIPS repositories in this environment')
p.add_argument('-r', '--repos', help='Top level directory for repositories', default='/data/repos')
p.add_argument('-e', '--email', help='Set email address (used for anonymous FTP sources)', default='')
p = subparser.add_parser('user', help='Configure GIPS repositories for this user (for per user customizations)')
#p.add_argument('-e', '--email', help='Set email address (used for anonymous FTP sources)')
#h = 'Install full configuration file without inheriting from environment settings'
#p.add_argument('-f', '--full', help=h, default=False, action='store_true')
args = parser.parse_args()
print title
if args.command == 'print':
try:
from gips.utils import settings
s = settings()
for v in dir(s):
if not v.startswith('__') and v != 'gips':
print
print v
exec('pprint.pprint(s.%s)' % v)
except Exception as e:
# print traceback.format_exc()
print 'Unable to access settings: {}'.format(e)
sys.exit(1)
elif args.command == 'env':
try:
cfgfile = create_environment_settings(args.repos, email=args.email)
print 'Environment settings file: %s' % cfgfile
print 'Creating repository directories'
create_repos()
except Exception, e:
print traceback.format_exc()
print 'Could not create environment settings: %s' % e
def main():
title = Colors.BOLD + "GIPS Image Statistics (v%s)" % __version__ + Colors.OFF
parser0 = GIPSParser(datasources=False, description=title)
parser0.add_default_parser()
parser0.add_projdir_parser()
group = parser0.add_argument_group("masking options")
args = parser0.parse_args()
# TODO - check that at least 1 of filemask or pmask is supplied
try:
print title
header = ["min", "max", "mean", "sd", "skew", "count"]
for projdir in args.projdir:
VerboseOut("Stats for Project directory: %s" % projdir, 1)
inv = ProjectInventory(projdir, args.products)
files = {}
for date in inv.dates:
VerboseOut("Calculating statistics for %s" % date)
for p in inv.products(date):
img = inv[date].open(p)
if p not in files.keys():
files[p] = open(os.path.join(projdir, p + "_stats.txt"), "w")
# write header
files[p].write("date ")
if img.NumBands() == 1:
files[p].write(" ".join(header))
else:
for band in img:
h = [band.Description() + "-" + a for a in header]
files[p].write(" ".join(h) + " ")
files[p].write("\n")
# print date and stats
files[p].write(date.strftime("%Y-%j"))
for band in img:
stats = band.Stats()
[files[p].write(" " + str(s)) for s in stats]
files[p].write(" ")
files[p].write("\n")
img = None
for f in files:
files[f].close()
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print "Error: %s" % e
def main():
title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser = parser0.add_inventory_parser()
group = parser.add_argument_group('inventory display')
group.add_argument('--md', help='Show dates using MM-DD', action='store_true', default=False)
args = parser0.parse_args()
try:
print title
cls = data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
args.tiles, args.pcov, args.ptile)
for extent in extents:
inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
inv.pprint(md=args.md)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data inventory error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser = parser0.add_inventory_parser()
group = parser.add_argument_group('inventory display')
group.add_argument('--md', help='Show dates using MM-DD', action='store_true', default=False)
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
args.tiles, args.pcov, args.ptile)
for extent in extents:
inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
inv.pprint(md=args.md)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data inventory error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Image Statistics (v%s)' % __version__ + Colors.OFF
parser0 = GIPSParser(datasources=False, description=title)
parser0.add_projdir_parser()
group = parser0.add_argument_group('masking options')
args = parser0.parse_args()
utils.gips_script_setup(stop_on_error=args.stop_on_error)
print title
# TODO - check that at least 1 of filemask or pmask is supplied
header = ['date', 'band', 'min', 'max', 'mean', 'sd', 'skew', 'count']
with utils.error_handler():
for projdir in args.projdir:
VerboseOut('Stats for Project directory: %s' % projdir, 1)
inv = ProjectInventory(projdir, args.products)
p_dates = {} # map each product to its list of valid dates
for date in inv.dates:
for p in inv.products(date):
p_dates.setdefault(p, []).append(date)
p_dates = {p: sorted(dl) for p, dl in p_dates.items()}
for p_type, valid_dates in p_dates.items():
stats_fn = os.path.join(projdir, p_type + '_stats.txt')
with open(stats_fn, 'w') as stats_fo:
sf = getattr(utils.settings(), 'STATS_FORMAT', {})
writer = csv.writer(stats_fo, **sf)
writer.writerow(header)
# print date, band description, and stats
for date in valid_dates:
img = inv[date].open(p_type)
date_str = date.strftime('%Y-%j')
utils.verbose_out('Computing stats for {} {}'.format(
p_type, date_str), 2)
for b in img:
stats = [str(s) for s in b.Stats()]
writer.writerow(
[date_str, b.Description()] + stats)
img = None
utils.gips_exit() # produce a summary error report then quit with a proper exit status
def main():
title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)
for extent in extents:
inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
inv.process(overwrite=args.overwrite)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data processing error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Project Masking (v%s)' % __version__ + Colors.OFF
parser = GIPSParser(datasources=False, description=title)
parser.add_projdir_parser()
group = parser.add_argument_group('masking options')
group.add_argument('--filemask',
help='Mask all files with this static mask',
default=None)
group.add_argument('--pmask',
help='Mask files with this corresponding product',
nargs='*',
default=[])
group.add_argument('--invert',
help='Invert the masks from corresponding products',
nargs='*',
default=[])
h = 'Write mask to original image instead of creating new image'
group.add_argument('--original',
help=h,
default=False,
action='store_true')
h = 'Overwrite existing files when creating new'
group.add_argument('--overwrite',
help=h,
default=False,
action='store_true')
h = 'Suffix to apply to masked file (not compatible with --original)'
group.add_argument('--suffix', help=h, default='-masked')
args = parser.parse_args()
# TODO - check that at least 1 of filemask or pmask is supplied
utils.gips_script_setup(None, args.stop_on_error)
with utils.error_handler('Masking error'):
VerboseOut(title)
for projdir in args.projdir:
if args.filemask is not None:
mask_file = gippy.GeoImage(args.filemask)
inv = ProjectInventory(projdir, args.products)
for date in inv.dates:
VerboseOut('Masking files from %s' % date)
if args.filemask is None and args.pmask == []:
available_masks = inv[date].masks()
else:
available_masks = inv[date].masks(args.pmask)
for p in inv.products(date):
# don't mask any masks
if p in available_masks:
continue
meta = ''
update = True if args.original else False
img = inv[date].open(p, update=update)
if args.filemask is not None:
img.AddMask(mask_file[0])
meta = basename(args.filemask) + ' '
for mask in available_masks:
mask_img = inv[date].open(mask)[0]
if mask in args.invert:
mask_img.SetNoData(utils.np.nan)
mask_img = mask_img.BXOR(1)
meta += 'inverted-'
img.AddMask(mask_img)
meta = meta + basename(inv[date][mask]) + ' '
if meta != '':
if args.original:
VerboseOut(' %s' % (img.Basename()), 2)
img.Process()
img.SetMeta('MASKS', meta)
else:
fout = os.path.splitext(
img.Filename())[0] + args.suffix + '.tif'
if not os.path.exists(fout) or args.overwrite:
VerboseOut(
' %s -> %s' %
(img.Basename(), basename(fout)), 2)
imgout = img.Process(fout)
imgout.SetMeta('MASKS', meta)
imgout = None
img = None
mask_file = None
utils.gips_exit()
def main():
title = Colors.BOLD + 'GIPS Project Masking (v%s)' % __version__ + Colors.OFF
parser0 = GIPSParser(datasources=False, description=title)
parser0.add_default_parser()
parser0.add_projdir_parser()
group = parser0.add_argument_group('masking options')
group.add_argument('--filemask',
help='Mask all files with this static mask',
default=None)
group.add_argument('--pmask',
help='Mask files with this corresponding product',
nargs='*',
default=[])
h = 'Write mask to original image instead of creating new image'
group.add_argument('--original',
help=h,
default=False,
action='store_true')
h = 'Overwrite existing files when creating new'
group.add_argument('--overwrite',
help=h,
default=False,
action='store_true')
h = 'Suffix to apply to masked file (not compatible with --original)'
group.add_argument('--suffix', help=h, default='-masked')
#parser0.add_argument('-i', '--invert', help='Invert mask (0->1, 1->0)', default=False, action='store_true')
#parser0.add_argument('--value', help='Mask == val', default=1)
args = parser0.parse_args()
# TODO - check that at least 1 of filemask or pmask is supplied
try:
print title
for projdir in args.projdir:
if args.filemask is not None:
mask_file = gippy.GeoImage(args.filemask)
inv = ProjectInventory(projdir, args.products)
for date in inv.dates:
VerboseOut('Masking files from %s' % date)
available_masks = inv[date].masks(args.pmask)
for p in inv.products(date):
# don't mask any masks
if p in available_masks:
continue
meta = ''
update = True if args.original else False
img = inv[date].open(p, update=update)
if args.filemask is not None:
img.AddMask(mask_file[0])
meta = basename(args.filemask) + ' '
for mask in available_masks:
img.AddMask(inv[date].open(mask)[0])
meta = meta + basename(inv[date][mask]) + ' '
if meta != '':
if args.original:
VerboseOut(' %s' % (img.Basename()), 2)
img.Process()
img.SetMeta('MASKS', meta)
else:
fout = os.path.splitext(
img.Filename())[0] + args.suffix + '.tif'
if not os.path.exists(fout) or args.overwrite:
VerboseOut(
' %s -> %s' %
(img.Basename(), basename(fout)), 2)
imgout = img.Process(fout)
imgout.SetMeta('MASKS', meta)
imgout = None
img = None
mask_file = None
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Masking error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
print title
with utils.error_handler():
extents = SpatialExtent.factory(
cls, site=args.site, rastermask=args.rastermask,
key=args.key, where=args.where, tiles=args.tiles,
pcov=args.pcov, ptile=args.ptile
)
batchargs = None
if args.batchout:
tdl = []
batchargs = '--chunksize ' + str(args.chunksize)
batchargs += ' --format ' + str(args.format)
batchargs += ' --numprocs ' + str(args.numprocs)
batchargs += ' --verbose ' + str(args.verbose)
if args.overwrite:
batchargs += ' --overwrite '
if args.products:
batchargs += ' -p ' + ' '.join(args.products)
for extent in extents:
inv = DataInventory(
cls, extent,
TemporalExtent(args.dates, args.days), **vars(args)
)
if args.batchout:
def get_commands(tiles_obj):
commands = []
for tile in tiles_obj.tiles.keys():
needed = any([p not in [k for sen, k in tiles_obj.tiles[tile].filenames.keys()]
for p in args.products])
if not needed:
continue
commands.append(args.command + ' -t ' + str(tile) +
' -d ' + str(tiles_obj.date) + ' ' +
batchargs + '\n')
return commands
tdl = reduce(
list.__add__,
map(
get_commands,
inv.data.values()
),
tdl
)
else:
inv.process(overwrite=args.overwrite)
if args.batchout:
with open(args.batchout, 'w') as ofile:
ofile.writelines(tdl)
utils.gips_exit() # produce a summary error report then quit with a proper exit status
def main():
title = Colors.BOLD + 'GIPS Data Export (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser(site_required=True)
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
print title
with utils.error_handler():
extents = SpatialExtent.factory(cls,
site=args.site,
rastermask=args.rastermask,
key=args.key,
where=args.where,
tiles=args.tiles,
pcov=args.pcov,
ptile=args.ptile)
# create tld: SITENAME--KEY_DATATYPE_SUFFIX
if args.notld:
tld = args.outdir
else:
key = '' if args.key == '' else '--' + args.key
suffix = '' if args.suffix == '' else '_' + args.suffix
res = '' if args.res is None else '_%sx%s' % (args.res[0],
args.res[1])
bname = (extents[0].site.LayerName() + key + res + '_' +
args.command + suffix)
tld = os.path.join(args.outdir, bname)
for extent in extents:
t_extent = TemporalExtent(args.dates, args.days)
inv = DataInventory(cls, extent, t_extent, **vars(args))
datadir = os.path.join(tld, extent.site.Value())
if inv.numfiles > 0:
inv.mosaic(
datadir=datadir,
tree=args.tree,
overwrite=args.overwrite,
res=args.res,
interpolation=args.interpolation,
crop=args.crop,
alltouch=args.alltouch,
process=(not args.dont_process),
)
inv = ProjectInventory(datadir)
inv.pprint()
else:
VerboseOut(
'No data found for {} within temporal extent {}'.format(
str(t_extent), str(t_extent)),
2,
)
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def main():
title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
print title
with utils.error_handler():
# create output directory if needed
# tld is "{}_tiles_{}_{}".format(DATATYPE, RESOLUTION, SUFFIX)
if args.notld:
tld = args.outdir
else:
tld = os.path.join(args.outdir, '%s_tiles' % args.command)
if args.res is not None:
tld = tld + '_%sx%s' % (args.res[0], args.res[1])
if args.suffix != '':
tld = tld + '_' + args.suffix
mkdir(tld)
extents = SpatialExtent.factory(cls,
site=args.site,
rastermask=args.rastermask,
key=args.key,
where=args.where,
tiles=args.tiles,
pcov=args.pcov,
ptile=args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
for date in inv.dates:
for tid in inv[date].tiles:
# make sure back-end tiles are processed
inv[date].tiles[tid].process(args.products,
overwrite=False)
# warp the tiles & copy into place in the output dir
inv[date].tiles[tid].copy(tld, args.products,
inv.spatial.site, args.res,
args.interpolation, args.crop,
args.overwrite, args.tree)
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def main():
title = Colors.BOLD + 'GIPS Data Project (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser(site_required=True)
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)
# create tld: SITENAME--KEY_DATATYPE_SUFFIX
if args.notld:
tld = args.outdir
else:
key = '' if args.key == '' else '--' + args.key
suffix = '' if args.suffix == '' else '_' + args.suffix
res = '' if args.res is None else '_%sx%s' % (args.res[0], args.res[1])
bname = extents[0].site.LayerName() + key + res + '_' + args.command + suffix
tld = os.path.join(args.outdir, bname)
for extent in extents:
inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
datadir = os.path.join(tld, extent.site.Value())
if inv.numfiles > 0:
inv.mosaic(datadir=datadir, tree=args.tree, overwrite=args.overwrite,
res=args.res, interpolation=args.interpolation, crop=args.crop)
if not args.tree:
inv = ProjectInventory(datadir)
inv.pprint()
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data Project error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Project Masking (v%s)' % __version__ + Colors.OFF
parser0 = GIPSParser(datasources=False, description=title)
parser0.add_default_parser()
parser0.add_projdir_parser()
group = parser0.add_argument_group('masking options')
group.add_argument('--filemask', help='Mask all files with this static mask', default=None)
group.add_argument('--pmask', help='Mask files with this corresponding product', nargs='*', default=[])
h = 'Write mask to original image instead of creating new image'
group.add_argument('--original', help=h, default=False, action='store_true')
h = 'Overwrite existing files when creating new'
group.add_argument('--overwrite', help=h, default=False, action='store_true')
h = 'Suffix to apply to masked file (not compatible with --original)'
group.add_argument('--suffix', help=h, default='-masked')
#parser0.add_argument('-i', '--invert', help='Invert mask (0->1, 1->0)', default=False, action='store_true')
#parser0.add_argument('--value', help='Mask == val', default=1)
args = parser0.parse_args()
# TODO - check that at least 1 of filemask or pmask is supplied
try:
VerboseOut(title)
for projdir in args.projdir:
if args.filemask is not None:
mask_file = gippy.GeoImage(args.filemask)
inv = ProjectInventory(projdir, args.products)
for date in inv.dates:
VerboseOut('Masking files from %s' % date)
if args.filemask is None and args.pmask == []:
available_masks = inv[date].masks()
else:
available_masks = inv[date].masks(args.pmask)
for p in inv.products(date):
# don't mask any masks
if p in available_masks:
continue
meta = ''
update = True if args.original else False
img = inv[date].open(p, update=update)
if args.filemask is not None:
img.AddMask(mask_file[0])
meta = basename(args.filemask) + ' '
for mask in available_masks:
img.AddMask(inv[date].open(mask)[0])
meta = meta + basename(inv[date][mask]) + ' '
if meta != '':
if args.original:
VerboseOut(' %s' % (img.Basename()), 2)
img.Process()
img.SetMeta('MASKS', meta)
else:
fout = os.path.splitext(img.Filename())[0] + args.suffix + '.tif'
if not os.path.exists(fout) or args.overwrite:
VerboseOut(' %s -> %s' % (img.Basename(), basename(fout)), 2)
imgout = img.Process(fout)
imgout.SetMeta('MASKS', meta)
imgout = None
img = None
mask_file = None
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Masking error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
# create tld: DATATYPE_tiles_RESOLUTION_SUFFIX
if args.notld:
tld = args.outdir
else:
tld = os.path.join(args.outdir, '%s_tiles' % args.command)
if args.res is not None:
tld = tld + '_%sx%s' % (args.res[0], args.res[1])
if args.suffix != '':
tld = tld + '_' + args.suffix
mkdir(tld)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
args.tiles, args.pcov, args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
for date in inv.dates:
for tid in inv[date].tiles:
# make sure back-end tiles are processed
inv[date].tiles[tid].process(args.products,
overwrite=False)
# warp the tiles
inv[date].tiles[tid].copy(tld, args.products,
inv.spatial.site, args.res,
args.interpolation, args.crop,
args.overwrite, args.tree)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Warp Tiles error: %s' % e
def main():
import gips
title = 'GIPS Configuration Utility (v%s)' % (version)
parser = GIPSParser(description=title, datasources=False)
subparser = parser.add_subparsers(dest='command')
subparser.add_parser('print', help='Print current settings')
p = subparser.add_parser(
'env', help='Configure GIPS repositories in this environment')
p.add_argument('-r',
'--repos',
help='Top level directory for repositories',
default='/data/repos')
p.add_argument('-e',
'--email',
help='Set email address (used for anonymous FTP sources)',
default='')
p = subparser.add_parser(
'user',
help=
'Configure GIPS repositories for this user (for per user customizations)'
)
#p.add_argument('-e', '--email', help='Set email address (used for anonymous FTP sources)')
#h = 'Install full configuration file without inheriting from environment settings'
#p.add_argument('-f', '--full', help=h, default=False, action='store_true')
args = parser.parse_args()
print title
utils.gips_script_setup(
driver_string=None, # NOTE: no driver string for gips_config
stop_on_error=args.stop_on_error,
setup_orm=False, # NOTE: ORM cannot be setup before `gips_config env`
) # has been run
if args.command == 'print':
with utils.error_handler('Unable to access settings'):
from gips.utils import settings
s = settings()
for v in dir(s):
if not v.startswith('__') and v != 'gips':
print
print v
exec('pprint.pprint(s.%s)' % v)
elif args.command == 'env':
with utils.error_handler('Could not create environment settings'):
created_cf, cfgfile = create_environment_settings(args.repos,
email=args.email)
elif args.command == 'user':
with utils.error_handler('Could not create user settings'):
# first try importing environment settings
import gips.settings
created_cf, cfgfile = create_user_settings()
if args.command in ('user', 'env'):
msg = ('Wrote new config file: {}.' if created_cf else
'Found existing config, left unmodified: {}.')
print msg.format(cfgfile)
with utils.error_handler('Could not create repos'):
print 'Creating repository directories, if needed.'
try:
create_repos()
except:
if created_cf:
print(
'Error; removing (likely broken) config file:'
' {}.'.format(cfgfile))
os.remove(cfgfile)
raise
with utils.error_handler('Could not migrate database'):
migrate_database()
utils.gips_exit()
