Esempio n. 1
0
def main():
    title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser()
    parser0.add_process_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
                                        args.tiles, args.pcov, args.ptile)
        for extent in extents:
            inv = DataInventory(cls, extent,
                                TemporalExtent(args.dates, args.days),
                                **vars(args))
            inv.process(overwrite=args.overwrite)

    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data processing error: %s' % e
Esempio n. 2
0
def main():
    title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser = parser0.add_inventory_parser()
    group = parser.add_argument_group('additional inventory options')
    group.add_argument('--md',
                       help='Show dates using MM-DD',
                       action='store_true',
                       default=False)
    group.add_argument(
        '--rectify',
        help=
        'Instead of displaying or fetching inventory, rectify the inventory '
        'database by comparing it against the present state of the data repos.',
        action='store_true',
        default=False)
    args = parser0.parse_args()

    cls = utils.gips_script_setup(args.command, args.stop_on_error)

    with utils.error_handler():
        print(title)

        if args.rectify:
            if not orm.use_orm():
                raise ValueError("--rectify can only be used if"
                                 " GIPS_ORM = True.")
            for k, v in vars(args).items():
                # Let the user know not to expect other options to effect rectify
                if v and k not in ('rectify', 'verbose', 'command'):
                    msg = "INFO: Option '--{}' is has no effect on --rectify."
                    utils.verbose_out(msg.format(k), 1)
            print("Rectifying inventory DB with filesystem archive:")
            print("Rectifying assets:")
            dbinv.rectify_assets(cls.Asset)
            print("Rectifying products:")
            dbinv.rectify_products(cls)
            return

        extents = SpatialExtent.factory(cls,
                                        site=args.site,
                                        rastermask=args.rastermask,
                                        key=args.key,
                                        where=args.where,
                                        tiles=args.tiles,
                                        pcov=args.pcov,
                                        ptile=args.ptile)
        for extent in extents:
            inv = DataInventory(cls, extent,
                                TemporalExtent(args.dates, args.days),
                                **vars(args))
            inv.pprint(md=args.md, size=args.size)

    utils.gips_exit(
    )  # produce a summary error report then quit with a proper exit status
Esempio n. 3
0
def main():
    title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser()
    parser0.add_process_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(
            cls, args.site, args.key, args.where, args.tiles, args.pcov,
            args.ptile
        )
        batchargs = None
        if args.batchout:
            tdl = []
            batchargs = '--chunksize ' + str(args.chunksize)
            batchargs += ' --format ' + str(args.format)
            if args.overwrite:
                batchargs += ' --overwrite '
            if args.products:
                batchargs += ' -p ' + ' '.join(args.products)

        for extent in extents:
            inv = DataInventory(
                cls, extent,
                TemporalExtent(args.dates, args.days), **vars(args)
            )
            if args.batchout:
                tdl = reduce(
                    list.__add__,
                    map(
                        lambda tiles: [
                            args.command + ' -t ' + str(tile) +
                            ' -d ' + str(tiles.date) + ' ' +
                            batchargs + '\n'
                            for tile in tiles.tiles.keys()
                        ],
                        inv.data.values(),
                    ),
                    tdl
                )

            else:
                inv.process(overwrite=args.overwrite)
        if args.batchout:
            with open(args.batchout, 'w') as ofile:
                ofile.writelines(tdl)

    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data processing error: %s' % e
Esempio n. 4
0
def t_data_inventory_db_save(mpo, orm):
    """Confirm DataInventory() saves to the inventory DB on fetch."""
    from gips.inventory.dbinv import models
    m_fetch = mpo(modisAsset, 'fetch')
    m_need_to_fetch = mpo(modisData, 'need_to_fetch')
    m_need_to_fetch.return_value = True
    # need some filenames . . .
    aol = [modisAsset(fn) for fn in asset_filenames]
    # normally _archivefile would set this attrib:
    [setattr(ao, 'archived_filename', ao.filename) for ao in aol]
    m_fetch.return_value = aol

    # specify spatial & temporal
    tiles = ['h12v04', 'h13v05', 'h12v05', 'h13v04']
    se = SpatialExtent(modisData, tiles, 0.0, 0.0)
    te = TemporalExtent('2012-12-01,2012-12-03')
    # can ignore rest of **kwargs in constructor due to mocking
    di = DataInventory(modisData, se, te, fetch=True)

    # confirm DB has correct entries in it now
    rows = [model_to_dict(ao) for ao in models.Asset.objects.all()]
    [a.pop('id') for a in rows]  # don't care what the keys are
    actual = {r['asset']: r
              for r in rows
              }  # enforce an order so whims of hashing doesn't ruin test

    assert expected_assets == actual
Esempio n. 5
0
File: core.py Progetto: aniucd/gips
 def inventory(cls, site=None, key='', where='', tiles=None, pcov=0.0,
               ptile=0.0, dates=None, days=None, **kwargs):
     """ Return list of inventories (size 1 if not looping through geometries) """
     from gips.inventory import DataInventory
     from gips.core import SpatialExtent, TemporalExtent
     spatial = SpatialExtent.factory(cls, site=site, key=key, where=where, tiles=tiles, pcov=pcov, ptile=ptile)
     temporal = TemporalExtent(dates, days)
     return DataInventory(cls, spatial[0], temporal, **kwargs)
Esempio n. 6
0
def main():
    title = Colors.BOLD + 'GIPS Data Project (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser(site_required=True)
    parser0.add_process_parser()
    parser0.add_project_parser()
    parser0.add_warp_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
                                        args.tiles, args.pcov, args.ptile)

        # create tld: SITENAME--KEY_DATATYPE_SUFFIX
        if args.notld:
            tld = args.outdir
        else:
            key = '' if args.key == '' else '--' + args.key
            suffix = '' if args.suffix == '' else '_' + args.suffix
            res = '' if args.res is None else '_%sx%s' % (args.res[0],
                                                          args.res[1])
            bname = (extents[0].site.LayerName() + key + res + '_' +
                     args.command + suffix)
            tld = os.path.join(args.outdir, bname)

        for extent in extents:
            t_extent = TemporalExtent(args.dates, args.days)
            inv = DataInventory(cls, extent, t_extent, **vars(args))
            datadir = os.path.join(tld, extent.site.Value())
            if inv.numfiles > 0:
                inv.mosaic(
                    datadir=datadir,
                    tree=args.tree,
                    overwrite=args.overwrite,
                    res=args.res,
                    interpolation=args.interpolation,
                    crop=args.crop,
                    alltouch=args.alltouch,
                )
                inv = ProjectInventory(datadir)
                inv.pprint()
            else:
                VerboseOut(
                    'No data found for {} within temporal extent {}'.format(
                        str(t_extent), str(t_extent)),
                    2,
                )
    except Exception as e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data Project error: %s' % e
Esempio n. 7
0
def main():
    title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser()
    parser0.add_process_parser()
    parser0.add_project_parser()
    parser0.add_warp_parser()
    args = parser0.parse_args()

    cls = utils.gips_script_setup(args.command, args.stop_on_error)
    print title

    with utils.error_handler():
        # create output directory if needed
        # tld is "{}_tiles_{}_{}".format(DATATYPE, RESOLUTION, SUFFIX)
        if args.notld:
            tld = args.outdir
        else:
            tld = os.path.join(args.outdir, '%s_tiles' % args.command)
            if args.res is not None:
                tld = tld + '_%sx%s' % (args.res[0], args.res[1])
            if args.suffix != '':
                tld = tld + '_' + args.suffix
        mkdir(tld)

        extents = SpatialExtent.factory(cls,
                                        site=args.site,
                                        rastermask=args.rastermask,
                                        key=args.key,
                                        where=args.where,
                                        tiles=args.tiles,
                                        pcov=args.pcov,
                                        ptile=args.ptile)
        for extent in extents:
            inv = DataInventory(cls, extent,
                                TemporalExtent(args.dates, args.days),
                                **vars(args))
            for date in inv.dates:
                for tid in inv[date].tiles:
                    # make sure back-end tiles are processed
                    inv[date].tiles[tid].process(args.products,
                                                 overwrite=False)
                    # warp the tiles & copy into place in the output dir
                    inv[date].tiles[tid].copy(tld, args.products,
                                              inv.spatial.site, args.res,
                                              args.interpolation, args.crop,
                                              args.overwrite, args.tree)

    utils.gips_exit(
    )  # produce a summary error report then quit with a proper exit status
Esempio n. 8
0
def t_data_inventory_load(orm):
    """Confirm DataInventory() loads assets and products from the inventory DB.

    In particular check the resulting hierarchy of objects for correctness."""

    # first load DB with data - start with values that should make it through to assertions
    for asset_fields in expected_assets.values():
        dbinv.add_asset(**asset_fields)
    for product_fields in expected_products.values():
        dbinv.add_product(**product_fields)

    # load more data, intentionally putting some 'misses' into the DB:
    # wrong driver
    dbinv.add_asset(**dict(expected_assets['MYD11A1'], driver='prism'))
    dbinv.add_product(**dict(expected_products['quality'], driver='merra'))
    # wrong date
    wrong_date = datetime.datetime(2012, 12, 4)
    dbinv.add_asset(**dict(expected_assets['MYD11A1'], date=wrong_date))
    dbinv.add_product(**dict(expected_products['quality'], date=wrong_date))
    # wrong tile
    dbinv.add_asset(**dict(expected_assets['MYD11A1'], tile='h12v03'))
    dbinv.add_product(**dict(expected_products['quality'], tile='h12v03'))
    # wrong product (DataInventory(products=foo))
    dbinv.add_product(**dict(expected_products['quality'], product='clouds'))

    # instantiate the class under test
    tiles = ['h12v04', 'h12v05', 'h13v04', 'h13v05']
    products = ['temp8td', 'quality', 'landcover']
    se = SpatialExtent(modisData, tiles, 0.0, 0.0)
    te = TemporalExtent('2012-12-01,2012-12-03')
    di = DataInventory(modisData, se, te, products)

    # confirm DataInventory object is correct
    assert len(di.data) == 3
    for tiles_obj in di.data.values():
        assert len(tiles_obj.tiles) == 1
        data_obj = tiles_obj.tiles['h12v04']
        assert len(data_obj.assets) == 1

        # compare the asset object
        asset_obj = data_obj.assets.values()[0]
        ea = expected_assets[asset_obj.asset]
        for field in ('asset', 'date', 'sensor', 'tile'):
            assert ea[field] == getattr(asset_obj, field)
        assert ea['name'] == asset_obj.filename

        # compare the stored product
        assert len(data_obj.filenames) == 1
        ((sensor, product), fname) = (data_obj.filenames.items())[0]
        ep = expected_products[product]
        assert (ep['sensor'] == sensor and ep['product'] == product
                and ep['name'] == fname)
Esempio n. 9
0
def main():
    title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser()
    parser0.add_process_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)
        for extent in extents:
            inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
            inv.process(overwrite=args.overwrite)

    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data processing error: %s' % e
Esempio n. 10
0
def main():
    title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser = parser0.add_inventory_parser()
    group = parser.add_argument_group('inventory display')
    group.add_argument('--md', help='Show dates using MM-DD', action='store_true', default=False)
    args = parser0.parse_args()

    try:
        print title
        cls = data_class(args.command)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where, 
                                        args.tiles, args.pcov, args.ptile)
        for extent in extents:
            inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
            inv.pprint(md=args.md)            
           
    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data inventory error: %s' % e
Esempio n. 11
0
def main():
    title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser = parser0.add_inventory_parser()
    group = parser.add_argument_group('inventory display')
    group.add_argument('--md', help='Show dates using MM-DD', action='store_true', default=False)
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where, 
                                        args.tiles, args.pcov, args.ptile)
        for extent in extents:
            inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
            inv.pprint(md=args.md)            
           
    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data inventory error: %s' % e
Esempio n. 12
0
def main():
    title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser()
    parser0.add_process_parser()
    parser0.add_project_parser()
    parser0.add_warp_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        # create tld: DATATYPE_tiles_RESOLUTION_SUFFIX
        if args.notld:
            tld = args.outdir
        else:
            tld = os.path.join(args.outdir, '%s_tiles' % args.command)
            if args.res is not None:
                tld = tld + '_%sx%s' % (args.res[0], args.res[1])
            if args.suffix != '':
                tld = tld + '_' + args.suffix
        mkdir(tld)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
                                        args.tiles, args.pcov, args.ptile)
        for extent in extents:
            inv = DataInventory(cls, extent,
                                TemporalExtent(args.dates, args.days),
                                **vars(args))
            for date in inv.dates:
                for tid in inv[date].tiles:
                    # make sure back-end tiles are processed
                    inv[date].tiles[tid].process(args.products,
                                                 overwrite=False)
                    # warp the tiles
                    inv[date].tiles[tid].copy(tld, args.products,
                                              inv.spatial.site, args.res,
                                              args.interpolation, args.crop,
                                              args.overwrite, args.tree)

    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Warp Tiles error: %s' % e
Esempio n. 13
0
def main():
    title = Colors.BOLD + "GIPS Data Project (v%s)" % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser(site_required=True)
    parser0.add_process_parser()
    parser0.add_project_parser()
    parser0.add_warp_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)

        # create tld: SITENAME--KEY_DATATYPE_SUFFIX
        if args.notld:
            tld = args.outdir
        else:
            key = "" if args.key == "" else "--" + args.key
            suffix = "" if args.suffix == "" else "_" + args.suffix
            res = "" if args.res is None else "_%sx%s" % (args.res[0], args.res[1])
            bname = extents[0].site.LayerName() + key + res + "_" + args.command + suffix
            tld = os.path.join(args.outdir, bname)

        for extent in extents:
            t_extent = TemporalExtent(args.dates, args.days)
            inv = DataInventory(cls, extent, t_extent, **vars(args))
            datadir = os.path.join(tld, extent.site.Value())
            if inv.numfiles > 0:
                inv.mosaic(
                    datadir=datadir,
                    tree=args.tree,
                    overwrite=args.overwrite,
                    res=args.res,
                    interpolation=args.interpolation,
                    crop=args.crop,
                    alltouch=args.alltouch,
                )
                inv = ProjectInventory(datadir)
                inv.pprint()
            else:
                VerboseOut("No data found for {} within temporal extent {}".format(str(t_extent), str(t_extent)), 2)
    except Exception as e:
        import traceback

        VerboseOut(traceback.format_exc(), 4)
        print "Data Project error: %s" % e
Esempio n. 14
0
def main():
    title = Colors.BOLD + 'GIPS Data Project (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser(site_required=True)
    parser0.add_process_parser()
    parser0.add_project_parser()
    parser0.add_warp_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)

        # create tld: SITENAME--KEY_DATATYPE_SUFFIX
        if args.notld:
            tld = args.outdir
        else:
            key = '' if args.key == '' else '--' + args.key
            suffix = '' if args.suffix == '' else '_' + args.suffix
            res = '' if args.res is None else '_%sx%s' % (args.res[0], args.res[1])
            bname = extents[0].site.LayerName() + key + res + '_' + args.command + suffix
            tld = os.path.join(args.outdir, bname)

        for extent in extents:
            inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
            datadir = os.path.join(tld, extent.site.Value())
            if inv.numfiles > 0:
                inv.mosaic(datadir=datadir, tree=args.tree, overwrite=args.overwrite,
                           res=args.res, interpolation=args.interpolation, crop=args.crop)
            if not args.tree:
                inv = ProjectInventory(datadir)
                inv.pprint()

    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data Project error: %s' % e
Esempio n. 15
0
def main():
    title = Colors.BOLD + 'GIPS Data Export (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser(site_required=True)
    parser0.add_process_parser()
    parser0.add_project_parser()
    parser0.add_warp_parser()
    args = parser0.parse_args()

    cls = utils.gips_script_setup(args.command, args.stop_on_error)
    print title

    with utils.error_handler():
        extents = SpatialExtent.factory(cls,
                                        site=args.site,
                                        rastermask=args.rastermask,
                                        key=args.key,
                                        where=args.where,
                                        tiles=args.tiles,
                                        pcov=args.pcov,
                                        ptile=args.ptile)

        # create tld: SITENAME--KEY_DATATYPE_SUFFIX
        if args.notld:
            tld = args.outdir
        else:
            key = '' if args.key == '' else '--' + args.key
            suffix = '' if args.suffix == '' else '_' + args.suffix
            res = '' if args.res is None else '_%sx%s' % (args.res[0],
                                                          args.res[1])
            bname = (extents[0].site.LayerName() + key + res + '_' +
                     args.command + suffix)
            tld = os.path.join(args.outdir, bname)

        for extent in extents:
            t_extent = TemporalExtent(args.dates, args.days)
            inv = DataInventory(cls, extent, t_extent, **vars(args))
            datadir = os.path.join(tld, extent.site.Value())
            if inv.numfiles > 0:
                inv.mosaic(
                    datadir=datadir,
                    tree=args.tree,
                    overwrite=args.overwrite,
                    res=args.res,
                    interpolation=args.interpolation,
                    crop=args.crop,
                    alltouch=args.alltouch,
                    process=(not args.dont_process),
                )
                inv = ProjectInventory(datadir)
                inv.pprint()
            else:
                VerboseOut(
                    'No data found for {} within temporal extent {}'.format(
                        str(t_extent), str(t_extent)),
                    2,
                )

    utils.gips_exit(
    )  # produce a summary error report then quit with a proper exit status
Esempio n. 16
0
def main():
    title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser()
    parser0.add_process_parser()
    args = parser0.parse_args()

    cls = utils.gips_script_setup(args.command, args.stop_on_error)
    print title

    with utils.error_handler():
        extents = SpatialExtent.factory(
            cls, site=args.site, rastermask=args.rastermask,
            key=args.key, where=args.where, tiles=args.tiles,
            pcov=args.pcov, ptile=args.ptile
        )
        batchargs = None
        if args.batchout:
            tdl = []
            batchargs = '--chunksize ' + str(args.chunksize)
            batchargs += ' --format ' + str(args.format)
            batchargs += ' --numprocs ' + str(args.numprocs)
            batchargs += ' --verbose ' + str(args.verbose)
            if args.overwrite:
                batchargs += ' --overwrite '
            if args.products:
                batchargs += ' -p ' + ' '.join(args.products)

        for extent in extents:
            inv = DataInventory(
                cls, extent,
                TemporalExtent(args.dates, args.days), **vars(args)
            )
            if args.batchout:
                def get_commands(tiles_obj):
                    commands = []
                    for tile in tiles_obj.tiles.keys():
                        needed = any([p not in [k for sen, k in tiles_obj.tiles[tile].filenames.keys()]
                                      for p in args.products])
                        if not needed:
                            continue
                        commands.append(args.command + ' -t ' + str(tile) +
                                    ' -d ' + str(tiles_obj.date) + ' ' +
                                    batchargs + '\n')
                    return commands


                tdl = reduce(
                    list.__add__,
                    map(
                        get_commands,
                        inv.data.values()
                    ),
                    tdl
                )

            else:
                inv.process(overwrite=args.overwrite)
        if args.batchout:
            with open(args.batchout, 'w') as ofile:
                ofile.writelines(tdl)

    utils.gips_exit() # produce a summary error report then quit with a proper exit status