def t_data_inventory_db_save(mpo, orm):
"""Confirm DataInventory() saves to the inventory DB on fetch."""
from gips.inventory.dbinv import models
m_fetch = mpo(modisAsset, 'fetch')
m_need_to_fetch = mpo(modisData, 'need_to_fetch')
m_need_to_fetch.return_value = True
# need some filenames . . .
aol = [modisAsset(fn) for fn in asset_filenames]
# normally _archivefile would set this attrib:
[setattr(ao, 'archived_filename', ao.filename) for ao in aol]
m_fetch.return_value = aol
# specify spatial & temporal
tiles = ['h12v04', 'h13v05', 'h12v05', 'h13v04']
se = SpatialExtent(modisData, tiles, 0.0, 0.0)
te = TemporalExtent('2012-12-01,2012-12-03')
# can ignore rest of **kwargs in constructor due to mocking
di = DataInventory(modisData, se, te, fetch=True)
# confirm DB has correct entries in it now
rows = [model_to_dict(ao) for ao in models.Asset.objects.all()]
[a.pop('id') for a in rows] # don't care what the keys are
actual = {r['asset']: r
for r in rows
} # enforce an order so whims of hashing doesn't ruin test
assert expected_assets == actual
def main():
title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
args.tiles, args.pcov, args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
inv.process(overwrite=args.overwrite)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data processing error: %s' % e
def inventory(cls, site=None, key='', where='', tiles=None, pcov=0.0,
ptile=0.0, dates=None, days=None, **kwargs):
""" Return list of inventories (size 1 if not looping through geometries) """
from gips.inventory import DataInventory
from gips.core import SpatialExtent, TemporalExtent
spatial = SpatialExtent.factory(cls, site=site, key=key, where=where, tiles=tiles, pcov=pcov, ptile=ptile)
temporal = TemporalExtent(dates, days)
return DataInventory(cls, spatial[0], temporal, **kwargs)
def main():
title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser = parser0.add_inventory_parser()
group = parser.add_argument_group('additional inventory options')
group.add_argument('--md',
help='Show dates using MM-DD',
action='store_true',
default=False)
group.add_argument(
'--rectify',
help=
'Instead of displaying or fetching inventory, rectify the inventory '
'database by comparing it against the present state of the data repos.',
action='store_true',
default=False)
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
with utils.error_handler():
print(title)
if args.rectify:
if not orm.use_orm():
raise ValueError("--rectify can only be used if"
" GIPS_ORM = True.")
for k, v in vars(args).items():
# Let the user know not to expect other options to effect rectify
if v and k not in ('rectify', 'verbose', 'command'):
msg = "INFO: Option '--{}' is has no effect on --rectify."
utils.verbose_out(msg.format(k), 1)
print("Rectifying inventory DB with filesystem archive:")
print("Rectifying assets:")
dbinv.rectify_assets(cls.Asset)
print("Rectifying products:")
dbinv.rectify_products(cls)
return
extents = SpatialExtent.factory(cls,
site=args.site,
rastermask=args.rastermask,
key=args.key,
where=args.where,
tiles=args.tiles,
pcov=args.pcov,
ptile=args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
inv.pprint(md=args.md, size=args.size)
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def main():
title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(
cls, args.site, args.key, args.where, args.tiles, args.pcov,
args.ptile
)
batchargs = None
if args.batchout:
tdl = []
batchargs = '--chunksize ' + str(args.chunksize)
batchargs += ' --format ' + str(args.format)
if args.overwrite:
batchargs += ' --overwrite '
if args.products:
batchargs += ' -p ' + ' '.join(args.products)
for extent in extents:
inv = DataInventory(
cls, extent,
TemporalExtent(args.dates, args.days), **vars(args)
)
if args.batchout:
tdl = reduce(
list.__add__,
map(
lambda tiles: [
args.command + ' -t ' + str(tile) +
' -d ' + str(tiles.date) + ' ' +
batchargs + '\n'
for tile in tiles.tiles.keys()
],
inv.data.values(),
),
tdl
)
else:
inv.process(overwrite=args.overwrite)
if args.batchout:
with open(args.batchout, 'w') as ofile:
ofile.writelines(tdl)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data processing error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Project (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser(site_required=True)
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
args.tiles, args.pcov, args.ptile)
# create tld: SITENAME--KEY_DATATYPE_SUFFIX
if args.notld:
tld = args.outdir
else:
key = '' if args.key == '' else '--' + args.key
suffix = '' if args.suffix == '' else '_' + args.suffix
res = '' if args.res is None else '_%sx%s' % (args.res[0],
args.res[1])
bname = (extents[0].site.LayerName() + key + res + '_' +
args.command + suffix)
tld = os.path.join(args.outdir, bname)
for extent in extents:
t_extent = TemporalExtent(args.dates, args.days)
inv = DataInventory(cls, extent, t_extent, **vars(args))
datadir = os.path.join(tld, extent.site.Value())
if inv.numfiles > 0:
inv.mosaic(
datadir=datadir,
tree=args.tree,
overwrite=args.overwrite,
res=args.res,
interpolation=args.interpolation,
crop=args.crop,
alltouch=args.alltouch,
)
inv = ProjectInventory(datadir)
inv.pprint()
else:
VerboseOut(
'No data found for {} within temporal extent {}'.format(
str(t_extent), str(t_extent)),
2,
)
except Exception as e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data Project error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
print title
with utils.error_handler():
# create output directory if needed
# tld is "{}_tiles_{}_{}".format(DATATYPE, RESOLUTION, SUFFIX)
if args.notld:
tld = args.outdir
else:
tld = os.path.join(args.outdir, '%s_tiles' % args.command)
if args.res is not None:
tld = tld + '_%sx%s' % (args.res[0], args.res[1])
if args.suffix != '':
tld = tld + '_' + args.suffix
mkdir(tld)
extents = SpatialExtent.factory(cls,
site=args.site,
rastermask=args.rastermask,
key=args.key,
where=args.where,
tiles=args.tiles,
pcov=args.pcov,
ptile=args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
for date in inv.dates:
for tid in inv[date].tiles:
# make sure back-end tiles are processed
inv[date].tiles[tid].process(args.products,
overwrite=False)
# warp the tiles & copy into place in the output dir
inv[date].tiles[tid].copy(tld, args.products,
inv.spatial.site, args.res,
args.interpolation, args.crop,
args.overwrite, args.tree)
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def t_data_inventory_load(orm):
"""Confirm DataInventory() loads assets and products from the inventory DB.
In particular check the resulting hierarchy of objects for correctness."""
# first load DB with data - start with values that should make it through to assertions
for asset_fields in expected_assets.values():
dbinv.add_asset(**asset_fields)
for product_fields in expected_products.values():
dbinv.add_product(**product_fields)
# load more data, intentionally putting some 'misses' into the DB:
# wrong driver
dbinv.add_asset(**dict(expected_assets['MYD11A1'], driver='prism'))
dbinv.add_product(**dict(expected_products['quality'], driver='merra'))
# wrong date
wrong_date = datetime.datetime(2012, 12, 4)
dbinv.add_asset(**dict(expected_assets['MYD11A1'], date=wrong_date))
dbinv.add_product(**dict(expected_products['quality'], date=wrong_date))
# wrong tile
dbinv.add_asset(**dict(expected_assets['MYD11A1'], tile='h12v03'))
dbinv.add_product(**dict(expected_products['quality'], tile='h12v03'))
# wrong product (DataInventory(products=foo))
dbinv.add_product(**dict(expected_products['quality'], product='clouds'))
# instantiate the class under test
tiles = ['h12v04', 'h12v05', 'h13v04', 'h13v05']
products = ['temp8td', 'quality', 'landcover']
se = SpatialExtent(modisData, tiles, 0.0, 0.0)
te = TemporalExtent('2012-12-01,2012-12-03')
di = DataInventory(modisData, se, te, products)
# confirm DataInventory object is correct
assert len(di.data) == 3
for tiles_obj in di.data.values():
assert len(tiles_obj.tiles) == 1
data_obj = tiles_obj.tiles['h12v04']
assert len(data_obj.assets) == 1
# compare the asset object
asset_obj = data_obj.assets.values()[0]
ea = expected_assets[asset_obj.asset]
for field in ('asset', 'date', 'sensor', 'tile'):
assert ea[field] == getattr(asset_obj, field)
assert ea['name'] == asset_obj.filename
# compare the stored product
assert len(data_obj.filenames) == 1
((sensor, product), fname) = (data_obj.filenames.items())[0]
ep = expected_products[product]
assert (ep['sensor'] == sensor and ep['product'] == product
and ep['name'] == fname)
def main():
title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
# create tld: DATATYPE_tiles_RESOLUTION_SUFFIX
if args.notld:
tld = args.outdir
else:
tld = os.path.join(args.outdir, '%s_tiles' % args.command)
if args.res is not None:
tld = tld + '_%sx%s' % (args.res[0], args.res[1])
if args.suffix != '':
tld = tld + '_' + args.suffix
mkdir(tld)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
args.tiles, args.pcov, args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
for date in inv.dates:
for tid in inv[date].tiles:
# make sure back-end tiles are processed
inv[date].tiles[tid].process(args.products,
overwrite=False)
# warp the tiles
inv[date].tiles[tid].copy(tld, args.products,
inv.spatial.site, args.res,
args.interpolation, args.crop,
args.overwrite, args.tree)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Warp Tiles error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser = parser0.add_inventory_parser()
group = parser.add_argument_group('inventory display')
group.add_argument('--md', help='Show dates using MM-DD', action='store_true', default=False)
args = parser0.parse_args()
try:
print title
cls = import_data_class(args.command)
extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
args.tiles, args.pcov, args.ptile)
for extent in extents:
inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
inv.pprint(md=args.md)
except Exception, e:
import traceback
VerboseOut(traceback.format_exc(), 4)
print 'Data inventory error: %s' % e
def main():
title = Colors.BOLD + 'GIPS Data Export (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser(site_required=True)
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
print title
with utils.error_handler():
extents = SpatialExtent.factory(cls,
site=args.site,
rastermask=args.rastermask,
key=args.key,
where=args.where,
tiles=args.tiles,
pcov=args.pcov,
ptile=args.ptile)
# create tld: SITENAME--KEY_DATATYPE_SUFFIX
if args.notld:
tld = args.outdir
else:
key = '' if args.key == '' else '--' + args.key
suffix = '' if args.suffix == '' else '_' + args.suffix
res = '' if args.res is None else '_%sx%s' % (args.res[0],
args.res[1])
bname = (extents[0].site.LayerName() + key + res + '_' +
args.command + suffix)
tld = os.path.join(args.outdir, bname)
for extent in extents:
t_extent = TemporalExtent(args.dates, args.days)
inv = DataInventory(cls, extent, t_extent, **vars(args))
datadir = os.path.join(tld, extent.site.Value())
if inv.numfiles > 0:
inv.mosaic(
datadir=datadir,
tree=args.tree,
overwrite=args.overwrite,
res=args.res,
interpolation=args.interpolation,
crop=args.crop,
alltouch=args.alltouch,
process=(not args.dont_process),
)
inv = ProjectInventory(datadir)
inv.pprint()
else:
VerboseOut(
'No data found for {} within temporal extent {}'.format(
str(t_extent), str(t_extent)),
2,
)
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def main():
title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
print title
with utils.error_handler():
extents = SpatialExtent.factory(
cls, site=args.site, rastermask=args.rastermask,
key=args.key, where=args.where, tiles=args.tiles,
pcov=args.pcov, ptile=args.ptile
)
batchargs = None
if args.batchout:
tdl = []
batchargs = '--chunksize ' + str(args.chunksize)
batchargs += ' --format ' + str(args.format)
batchargs += ' --numprocs ' + str(args.numprocs)
batchargs += ' --verbose ' + str(args.verbose)
if args.overwrite:
batchargs += ' --overwrite '
if args.products:
batchargs += ' -p ' + ' '.join(args.products)
for extent in extents:
inv = DataInventory(
cls, extent,
TemporalExtent(args.dates, args.days), **vars(args)
)
if args.batchout:
def get_commands(tiles_obj):
commands = []
for tile in tiles_obj.tiles.keys():
needed = any([p not in [k for sen, k in tiles_obj.tiles[tile].filenames.keys()]
for p in args.products])
if not needed:
continue
commands.append(args.command + ' -t ' + str(tile) +
' -d ' + str(tiles_obj.date) + ' ' +
batchargs + '\n')
return commands
tdl = reduce(
list.__add__,
map(
get_commands,
inv.data.values()
),
tdl
)
else:
inv.process(overwrite=args.overwrite)
if args.batchout:
with open(args.batchout, 'w') as ofile:
ofile.writelines(tdl)
utils.gips_exit() # produce a summary error report then quit with a proper exit status