def main(cls):
""" Main for algorithm classes """
dhf = argparse.ArgumentDefaultsHelpFormatter
# Top level parser
parser = argparse.ArgumentParser(formatter_class=dhf,
description=cls.info())
parser.add_argument('-v',
'--verbose',
help='Verbosity - 0: quiet, 1: normal, 2+: debug',
default=1,
type=int)
parser = cls.parser(parser)
args = parser.parse_args()
gippy.Options.SetVerbose(args.verbose)
VerboseOut(cls.info())
utils.gips_script_setup(driver_string=None, setup_orm=False)
with utils.error_handler('Error in {}'.format(cls.name)):
alg = cls(**vars(args))
alg.run_command(**vars(args))
utils.gips_exit()
def main():
title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser = parser0.add_inventory_parser()
group = parser.add_argument_group('additional inventory options')
group.add_argument('--md',
help='Show dates using MM-DD',
action='store_true',
default=False)
group.add_argument(
'--rectify',
help=
'Instead of displaying or fetching inventory, rectify the inventory '
'database by comparing it against the present state of the data repos.',
action='store_true',
default=False)
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
with utils.error_handler():
print(title)
if args.rectify:
if not orm.use_orm():
raise ValueError("--rectify can only be used if"
" GIPS_ORM = True.")
for k, v in vars(args).items():
# Let the user know not to expect other options to effect rectify
if v and k not in ('rectify', 'verbose', 'command'):
msg = "INFO: Option '--{}' is has no effect on --rectify."
utils.verbose_out(msg.format(k), 1)
print("Rectifying inventory DB with filesystem archive:")
print("Rectifying assets:")
dbinv.rectify_assets(cls.Asset)
print("Rectifying products:")
dbinv.rectify_products(cls)
return
extents = SpatialExtent.factory(cls,
site=args.site,
rastermask=args.rastermask,
key=args.key,
where=args.where,
tiles=args.tiles,
pcov=args.pcov,
ptile=args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
inv.pprint(md=args.md, size=args.size)
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def main():
title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
print title
with utils.error_handler():
# create output directory if needed
# tld is "{}_tiles_{}_{}".format(DATATYPE, RESOLUTION, SUFFIX)
if args.notld:
tld = args.outdir
else:
tld = os.path.join(args.outdir, '%s_tiles' % args.command)
if args.res is not None:
tld = tld + '_%sx%s' % (args.res[0], args.res[1])
if args.suffix != '':
tld = tld + '_' + args.suffix
mkdir(tld)
extents = SpatialExtent.factory(cls,
site=args.site,
rastermask=args.rastermask,
key=args.key,
where=args.where,
tiles=args.tiles,
pcov=args.pcov,
ptile=args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
for date in inv.dates:
for tid in inv[date].tiles:
# make sure back-end tiles are processed
inv[date].tiles[tid].process(args.products,
overwrite=False)
# warp the tiles & copy into place in the output dir
inv[date].tiles[tid].copy(tld, args.products,
inv.spatial.site, args.res,
args.interpolation, args.crop,
args.overwrite, args.tree)
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def t_gips_exit(mocker, accum_errors):
"""Test GIPS' exit function."""
m_report_error = mocker.patch.object(utils, 'report_error')
m_sys_exit = mocker.patch.object(utils.sys, 'exit')
m_sys_exit.side_effect = RuntimeError('real one raises SystemExit')
mocker.patch.object(utils, '_accumulated_errors', new=accum_errors)
with pytest.raises(RuntimeError):
utils.gips_exit()
# confirm exit status is 0 if no errors, 1 otherwise:
m_sys_exit.assert_called_once_with(0 if len(accum_errors) == 0 else 1)
# confirm an error report is generated for each error:
assert m_report_error.call_count == len(accum_errors)
def main():
title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser = GIPSParser(description=title)
group = parser.add_argument_group('archive options')
group.add_argument('--keep',
help='Keep files after adding to archive',
default=False,
action='store_true')
group.add_argument('--recursive',
help='Iterate through subdirectories',
default=False,
action='store_true')
group.add_argument(
'--update',
help=
'Update asset if newer version available, (must call gips_process to regenerate products',
default=False,
action='store_true')
group.add_argument(
'--path',
default='.',
help='Path to search for files to archive, defaults to `.`')
args = parser.parse_args()
utils.gips_script_setup(None, args.stop_on_error)
with utils.error_handler('Data archive error'):
print title
cls = import_data_class(args.command)
orm.setup() # set up DB orm in case it's needed for Asset.archive()
archived_assets = cls.archive_assets(args.path, args.recursive,
args.keep, args.update)
# if DB inventory is enabled, update it to contain the newly archived assets
if orm.use_orm():
for a in archived_assets:
dbinv.update_or_add_asset(asset=a.asset,
sensor=a.sensor,
tile=a.tile,
date=a.date,
name=a.archived_filename,
driver=cls.name.lower())
utils.gips_exit()
def main():
title = Colors.BOLD + 'GIPS Image Statistics (v%s)' % __version__ + Colors.OFF
parser0 = GIPSParser(datasources=False, description=title)
parser0.add_projdir_parser()
group = parser0.add_argument_group('masking options')
args = parser0.parse_args()
utils.gips_script_setup(stop_on_error=args.stop_on_error)
print title
# TODO - check that at least 1 of filemask or pmask is supplied
header = ['date', 'band', 'min', 'max', 'mean', 'sd', 'skew', 'count']
with utils.error_handler():
for projdir in args.projdir:
VerboseOut('Stats for Project directory: %s' % projdir, 1)
inv = ProjectInventory(projdir, args.products)
p_dates = {} # map each product to its list of valid dates
for date in inv.dates:
for p in inv.products(date):
p_dates.setdefault(p, []).append(date)
p_dates = {p: sorted(dl) for p, dl in p_dates.items()}
for p_type, valid_dates in p_dates.items():
stats_fn = os.path.join(projdir, p_type + '_stats.txt')
with open(stats_fn, 'w') as stats_fo:
sf = getattr(utils.settings(), 'STATS_FORMAT', {})
writer = csv.writer(stats_fo, **sf)
writer.writerow(header)
# print date, band description, and stats
for date in valid_dates:
img = inv[date].open(p_type)
date_str = date.strftime('%Y-%j')
utils.verbose_out('Computing stats for {} {}'.format(
p_type, date_str), 2)
for b in img:
stats = [str(s) for s in b.Stats()]
writer.writerow(
[date_str, b.Description()] + stats)
img = None
utils.gips_exit() # produce a summary error report then quit with a proper exit status
def main():
title = Colors.BOLD + 'GIPS Data Repositories (v%s)' % (
gipsversion) + Colors.OFF
# argument parsing
parser = GIPSParser(description=title)
args = parser.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
print title
with utils.error_handler():
cls.print_products()
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def main():
title = Colors.BOLD + 'GIPS Data Export (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser(site_required=True)
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
print title
with utils.error_handler():
extents = SpatialExtent.factory(cls,
site=args.site,
rastermask=args.rastermask,
key=args.key,
where=args.where,
tiles=args.tiles,
pcov=args.pcov,
ptile=args.ptile)
# create tld: SITENAME--KEY_DATATYPE_SUFFIX
if args.notld:
tld = args.outdir
else:
key = '' if args.key == '' else '--' + args.key
suffix = '' if args.suffix == '' else '_' + args.suffix
res = '' if args.res is None else '_%sx%s' % (args.res[0],
args.res[1])
bname = (extents[0].site.LayerName() + key + res + '_' +
args.command + suffix)
tld = os.path.join(args.outdir, bname)
for extent in extents:
t_extent = TemporalExtent(args.dates, args.days)
inv = DataInventory(cls, extent, t_extent, **vars(args))
datadir = os.path.join(tld, extent.site.Value())
if inv.numfiles > 0:
inv.mosaic(
datadir=datadir,
tree=args.tree,
overwrite=args.overwrite,
res=args.res,
interpolation=args.interpolation,
crop=args.crop,
alltouch=args.alltouch,
process=(not args.dont_process),
)
inv = ProjectInventory(datadir)
inv.pprint()
else:
VerboseOut(
'No data found for {} within temporal extent {}'.format(
str(t_extent), str(t_extent)),
2,
)
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def main():
import gips
title = 'GIPS Configuration Utility (v%s)' % (version)
parser = GIPSParser(description=title, datasources=False)
subparser = parser.add_subparsers(dest='command')
subparser.add_parser('print', help='Print current settings')
p = subparser.add_parser(
'env', help='Configure GIPS repositories in this environment')
p.add_argument('-r',
'--repos',
help='Top level directory for repositories',
default='/data/repos')
p.add_argument('-e',
'--email',
help='Set email address (used for anonymous FTP sources)',
default='')
p = subparser.add_parser(
'user',
help=
'Configure GIPS repositories for this user (for per user customizations)'
)
#p.add_argument('-e', '--email', help='Set email address (used for anonymous FTP sources)')
#h = 'Install full configuration file without inheriting from environment settings'
#p.add_argument('-f', '--full', help=h, default=False, action='store_true')
args = parser.parse_args()
print title
utils.gips_script_setup(
driver_string=None, # NOTE: no driver string for gips_config
stop_on_error=args.stop_on_error,
setup_orm=False, # NOTE: ORM cannot be setup before `gips_config env`
) # has been run
if args.command == 'print':
with utils.error_handler('Unable to access settings'):
from gips.utils import settings
s = settings()
for v in dir(s):
if not v.startswith('__') and v != 'gips':
print
print v
exec('pprint.pprint(s.%s)' % v)
elif args.command == 'env':
with utils.error_handler('Could not create environment settings'):
created_cf, cfgfile = create_environment_settings(args.repos,
email=args.email)
elif args.command == 'user':
with utils.error_handler('Could not create user settings'):
# first try importing environment settings
import gips.settings
created_cf, cfgfile = create_user_settings()
if args.command in ('user', 'env'):
msg = ('Wrote new config file: {}.' if created_cf else
'Found existing config, left unmodified: {}.')
print msg.format(cfgfile)
with utils.error_handler('Could not create repos'):
print 'Creating repository directories, if needed.'
try:
create_repos()
except:
if created_cf:
print(
'Error; removing (likely broken) config file:'
' {}.'.format(cfgfile))
os.remove(cfgfile)
raise
with utils.error_handler('Could not migrate database'):
migrate_database()
utils.gips_exit()
def main():
title = Colors.BOLD + 'GIPS Project Masking (v%s)' % __version__ + Colors.OFF
parser = GIPSParser(datasources=False, description=title)
parser.add_projdir_parser()
group = parser.add_argument_group('masking options')
group.add_argument('--filemask',
help='Mask all files with this static mask',
default=None)
group.add_argument('--pmask',
help='Mask files with this corresponding product',
nargs='*',
default=[])
group.add_argument('--invert',
help='Invert the masks from corresponding products',
nargs='*',
default=[])
h = 'Write mask to original image instead of creating new image'
group.add_argument('--original',
help=h,
default=False,
action='store_true')
h = 'Overwrite existing files when creating new'
group.add_argument('--overwrite',
help=h,
default=False,
action='store_true')
h = 'Suffix to apply to masked file (not compatible with --original)'
group.add_argument('--suffix', help=h, default='-masked')
args = parser.parse_args()
# TODO - check that at least 1 of filemask or pmask is supplied
utils.gips_script_setup(None, args.stop_on_error)
with utils.error_handler('Masking error'):
VerboseOut(title)
for projdir in args.projdir:
if args.filemask is not None:
mask_file = gippy.GeoImage(args.filemask)
inv = ProjectInventory(projdir, args.products)
for date in inv.dates:
VerboseOut('Masking files from %s' % date)
if args.filemask is None and args.pmask == []:
available_masks = inv[date].masks()
else:
available_masks = inv[date].masks(args.pmask)
for p in inv.products(date):
# don't mask any masks
if p in available_masks:
continue
meta = ''
update = True if args.original else False
img = inv[date].open(p, update=update)
if args.filemask is not None:
img.AddMask(mask_file[0])
meta = basename(args.filemask) + ' '
for mask in available_masks:
mask_img = inv[date].open(mask)[0]
if mask in args.invert:
mask_img.SetNoData(utils.np.nan)
mask_img = mask_img.BXOR(1)
meta += 'inverted-'
img.AddMask(mask_img)
meta = meta + basename(inv[date][mask]) + ' '
if meta != '':
if args.original:
VerboseOut(' %s' % (img.Basename()), 2)
img.Process()
img.SetMeta('MASKS', meta)
else:
fout = os.path.splitext(
img.Filename())[0] + args.suffix + '.tif'
if not os.path.exists(fout) or args.overwrite:
VerboseOut(
' %s -> %s' %
(img.Basename(), basename(fout)), 2)
imgout = img.Process(fout)
imgout.SetMeta('MASKS', meta)
imgout = None
img = None
mask_file = None
utils.gips_exit()
def main():
title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
print title
with utils.error_handler():
extents = SpatialExtent.factory(
cls, site=args.site, rastermask=args.rastermask,
key=args.key, where=args.where, tiles=args.tiles,
pcov=args.pcov, ptile=args.ptile
)
batchargs = None
if args.batchout:
tdl = []
batchargs = '--chunksize ' + str(args.chunksize)
batchargs += ' --format ' + str(args.format)
batchargs += ' --numprocs ' + str(args.numprocs)
batchargs += ' --verbose ' + str(args.verbose)
if args.overwrite:
batchargs += ' --overwrite '
if args.products:
batchargs += ' -p ' + ' '.join(args.products)
for extent in extents:
inv = DataInventory(
cls, extent,
TemporalExtent(args.dates, args.days), **vars(args)
)
if args.batchout:
def get_commands(tiles_obj):
commands = []
for tile in tiles_obj.tiles.keys():
needed = any([p not in [k for sen, k in tiles_obj.tiles[tile].filenames.keys()]
for p in args.products])
if not needed:
continue
commands.append(args.command + ' -t ' + str(tile) +
' -d ' + str(tiles_obj.date) + ' ' +
batchargs + '\n')
return commands
tdl = reduce(
list.__add__,
map(
get_commands,
inv.data.values()
),
tdl
)
else:
inv.process(overwrite=args.overwrite)
if args.batchout:
with open(args.batchout, 'w') as ofile:
ofile.writelines(tdl)
utils.gips_exit() # produce a summary error report then quit with a proper exit status