Exemplo n.º 1
0
 def add_data_sources(self):
     """ Adds data sources to parser """
     subparser = self.add_subparsers(dest='command')
     sources = data_sources()
     if len(sources) == 0:
         verbose_out("There are no available data sources!", 1, sys.stderr)
     for src, desc in sources.items():
         p = subparser.add_parser(src,
                                  help=desc,
                                  parents=self.parent_parsers)
         utils.import_data_class(src).add_filter_args(p)
Exemplo n.º 2
0
def main():
    title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser = parser0.add_default_parser()
    group = parser.add_argument_group('archive options')
    group.add_argument('--keep',
                       help='Keep files after adding to archive',
                       default=False,
                       action='store_true')
    group.add_argument('--recursive',
                       help='Iterate through subdirectories',
                       default=False,
                       action='store_true')
    group.add_argument(
        '--update',
        help=
        'Update asset if newer version available, (must call gips_process to regenerate products',
        default=False,
        action='store_true')
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)
        cls.Asset.archive(**vars(args))
    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data archive error: %s' % e
Exemplo n.º 3
0
def main():
    title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser = parser0.add_default_parser()
    group = parser.add_argument_group('archive options')
    group.add_argument('--keep', help='Keep files after adding to archive', default=False, action='store_true')
    group.add_argument('--recursive', help='Iterate through subdirectories', default=False, action='store_true')
    group.add_argument(
        '--update',
        help='Update asset if newer version available, (must call gips_process to regenerate products',
        default=False,
        action='store_true'
    )
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)
        cls.Asset.archive(**vars(args))
    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data archive error: %s' % e
Exemplo n.º 4
0
def main():
    title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser()
    parser0.add_process_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
                                        args.tiles, args.pcov, args.ptile)
        for extent in extents:
            inv = DataInventory(cls, extent,
                                TemporalExtent(args.dates, args.days),
                                **vars(args))
            inv.process(overwrite=args.overwrite)

    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data processing error: %s' % e
Exemplo n.º 5
0
def main():
    title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser()
    parser0.add_process_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(
            cls, args.site, args.key, args.where, args.tiles, args.pcov,
            args.ptile
        )
        batchargs = None
        if args.batchout:
            tdl = []
            batchargs = '--chunksize ' + str(args.chunksize)
            batchargs += ' --format ' + str(args.format)
            if args.overwrite:
                batchargs += ' --overwrite '
            if args.products:
                batchargs += ' -p ' + ' '.join(args.products)

        for extent in extents:
            inv = DataInventory(
                cls, extent,
                TemporalExtent(args.dates, args.days), **vars(args)
            )
            if args.batchout:
                tdl = reduce(
                    list.__add__,
                    map(
                        lambda tiles: [
                            args.command + ' -t ' + str(tile) +
                            ' -d ' + str(tiles.date) + ' ' +
                            batchargs + '\n'
                            for tile in tiles.tiles.keys()
                        ],
                        inv.data.values(),
                    ),
                    tdl
                )

            else:
                inv.process(overwrite=args.overwrite)
        if args.batchout:
            with open(args.batchout, 'w') as ofile:
                ofile.writelines(tdl)

    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data processing error: %s' % e
Exemplo n.º 6
0
def main():
    title = Colors.BOLD + 'GIPS Data Project (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser(site_required=True)
    parser0.add_process_parser()
    parser0.add_project_parser()
    parser0.add_warp_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
                                        args.tiles, args.pcov, args.ptile)

        # create tld: SITENAME--KEY_DATATYPE_SUFFIX
        if args.notld:
            tld = args.outdir
        else:
            key = '' if args.key == '' else '--' + args.key
            suffix = '' if args.suffix == '' else '_' + args.suffix
            res = '' if args.res is None else '_%sx%s' % (args.res[0],
                                                          args.res[1])
            bname = (extents[0].site.LayerName() + key + res + '_' +
                     args.command + suffix)
            tld = os.path.join(args.outdir, bname)

        for extent in extents:
            t_extent = TemporalExtent(args.dates, args.days)
            inv = DataInventory(cls, extent, t_extent, **vars(args))
            datadir = os.path.join(tld, extent.site.Value())
            if inv.numfiles > 0:
                inv.mosaic(
                    datadir=datadir,
                    tree=args.tree,
                    overwrite=args.overwrite,
                    res=args.res,
                    interpolation=args.interpolation,
                    crop=args.crop,
                    alltouch=args.alltouch,
                )
                inv = ProjectInventory(datadir)
                inv.pprint()
            else:
                VerboseOut(
                    'No data found for {} within temporal extent {}'.format(
                        str(t_extent), str(t_extent)),
                    2,
                )
    except Exception as e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data Project error: %s' % e
Exemplo n.º 7
0
def main():
    title = Colors.BOLD + 'GIPS Data Repositories (v%s)' % (gipsversion) + Colors.OFF

    # argument parsing
    try:
        parser = GIPSParser(description=title)
        args = parser.parse_args()
        print title
        cls = import_data_class(args.command)
        cls.print_products()
    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'GIPS Data Repository error: %s' % e
Exemplo n.º 8
0
def main():
    title = Colors.BOLD + "GIPS Data Project (v%s)" % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser(site_required=True)
    parser0.add_process_parser()
    parser0.add_project_parser()
    parser0.add_warp_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)

        # create tld: SITENAME--KEY_DATATYPE_SUFFIX
        if args.notld:
            tld = args.outdir
        else:
            key = "" if args.key == "" else "--" + args.key
            suffix = "" if args.suffix == "" else "_" + args.suffix
            res = "" if args.res is None else "_%sx%s" % (args.res[0], args.res[1])
            bname = extents[0].site.LayerName() + key + res + "_" + args.command + suffix
            tld = os.path.join(args.outdir, bname)

        for extent in extents:
            t_extent = TemporalExtent(args.dates, args.days)
            inv = DataInventory(cls, extent, t_extent, **vars(args))
            datadir = os.path.join(tld, extent.site.Value())
            if inv.numfiles > 0:
                inv.mosaic(
                    datadir=datadir,
                    tree=args.tree,
                    overwrite=args.overwrite,
                    res=args.res,
                    interpolation=args.interpolation,
                    crop=args.crop,
                    alltouch=args.alltouch,
                )
                inv = ProjectInventory(datadir)
                inv.pprint()
            else:
                VerboseOut("No data found for {} within temporal extent {}".format(str(t_extent), str(t_extent)), 2)
    except Exception as e:
        import traceback

        VerboseOut(traceback.format_exc(), 4)
        print "Data Project error: %s" % e
Exemplo n.º 9
0
def main():
    title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser()
    parser0.add_process_parser()
    parser0.add_project_parser()
    parser0.add_warp_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        # create tld: DATATYPE_tiles_RESOLUTION_SUFFIX
        if args.notld:
            tld = args.outdir
        else:
            tld = os.path.join(args.outdir, '%s_tiles' % args.command)
            if args.res is not None:
                tld = tld + '_%sx%s' % (args.res[0], args.res[1])
            if args.suffix != '':
                tld = tld + '_' + args.suffix
        mkdir(tld)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
                                        args.tiles, args.pcov, args.ptile)
        for extent in extents:
            inv = DataInventory(cls, extent,
                                TemporalExtent(args.dates, args.days),
                                **vars(args))
            for date in inv.dates:
                for tid in inv[date].tiles:
                    # make sure back-end tiles are processed
                    inv[date].tiles[tid].process(args.products,
                                                 overwrite=False)
                    # warp the tiles
                    inv[date].tiles[tid].copy(tld, args.products,
                                              inv.spatial.site, args.res,
                                              args.interpolation, args.crop,
                                              args.overwrite, args.tree)

    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Warp Tiles error: %s' % e
Exemplo n.º 10
0
def main():
    title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF

    # argument parsing
    parser = GIPSParser(description=title)
    group = parser.add_argument_group('archive options')
    group.add_argument('--keep',
                       help='Keep files after adding to archive',
                       default=False,
                       action='store_true')
    group.add_argument('--recursive',
                       help='Iterate through subdirectories',
                       default=False,
                       action='store_true')
    group.add_argument(
        '--update',
        help=
        'Update asset if newer version available, (must call gips_process to regenerate products',
        default=False,
        action='store_true')
    group.add_argument(
        '--path',
        default='.',
        help='Path to search for files to archive, defaults to `.`')
    args = parser.parse_args()

    utils.gips_script_setup(None, args.stop_on_error)

    with utils.error_handler('Data archive error'):
        print title
        cls = import_data_class(args.command)
        orm.setup()  # set up DB orm in case it's needed for Asset.archive()
        archived_assets = cls.archive_assets(args.path, args.recursive,
                                             args.keep, args.update)

        # if DB inventory is enabled, update it to contain the newly archived assets
        if orm.use_orm():
            for a in archived_assets:
                dbinv.update_or_add_asset(asset=a.asset,
                                          sensor=a.sensor,
                                          tile=a.tile,
                                          date=a.date,
                                          name=a.archived_filename,
                                          driver=cls.name.lower())

    utils.gips_exit()
Exemplo n.º 11
0
def t_raster_spatialspec():
    """Test that shapefile and rastermask return the same tile list"""
    with utils.make_temp_dir(prefix='t_gridded_inventory') as td:
        rastermask = os.path.join(td, 'nhseacost_mask.tif')

        rasterize = ('gdal_rasterize -burn 1 -ot Byte -tr 30 30 {} {}'.format(
            NH_SHP_PATH, rastermask))
        status, output = commands.getstatusoutput(rasterize)
        assert status == 0

        cls = utils.import_data_class('modis')
        s_extents = SpatialExtent.factory(cls, site=NH_SHP_PATH)
        r_extents = SpatialExtent.factory(cls, rastermask=rastermask)

        assert len(s_extents) == len(r_extents)
        for i in range(len(r_extents)):
            assert s_extents[i].tiles == r_extents[i].tiles
Exemplo n.º 12
0
def main():
    title = Colors.BOLD + 'GIPS Data Project (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser(site_required=True)
    parser0.add_process_parser()
    parser0.add_project_parser()
    parser0.add_warp_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)

        # create tld: SITENAME--KEY_DATATYPE_SUFFIX
        if args.notld:
            tld = args.outdir
        else:
            key = '' if args.key == '' else '--' + args.key
            suffix = '' if args.suffix == '' else '_' + args.suffix
            res = '' if args.res is None else '_%sx%s' % (args.res[0], args.res[1])
            bname = extents[0].site.LayerName() + key + res + '_' + args.command + suffix
            tld = os.path.join(args.outdir, bname)

        for extent in extents:
            inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
            datadir = os.path.join(tld, extent.site.Value())
            if inv.numfiles > 0:
                inv.mosaic(datadir=datadir, tree=args.tree, overwrite=args.overwrite,
                           res=args.res, interpolation=args.interpolation, crop=args.crop)
            if not args.tree:
                inv = ProjectInventory(datadir)
                inv.pprint()

    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data Project error: %s' % e
Exemplo n.º 13
0
def main():
    title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser()
    parser0.add_process_parser()
    parser0.add_project_parser()
    parser0.add_warp_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        # create tld: DATATYPE_tiles_RESOLUTION_SUFFIX
        if args.notld:
            tld = args.outdir
        else:
            tld = os.path.join(args.outdir, '%s_tiles' % args.command)
            if args.res is not None:
                tld = tld + '_%sx%s' % (args.res[0], args.res[1])
            if args.suffix != '':
                tld = tld + '_' + args.suffix
        mkdir(tld)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)
        for extent in extents:
            inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
            for date in inv.dates:
                for tid in inv[date].tiles:
                    # make sure back-end tiles are processed
                    inv[date].tiles[tid].process(args.products, overwrite=False)
                    # warp the tiles
                    inv[date].tiles[tid].copy(tld, args.products, inv.spatial.site,
                                              args.res, args.interpolation, args.crop, args.overwrite, args.tree)

    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Warp Tiles error: %s' % e
Exemplo n.º 14
0
def main():
    title = Colors.BOLD + 'GIPS Data Processing (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser()
    parser0.add_process_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where, args.tiles, args.pcov, args.ptile)
        for extent in extents:
            inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
            inv.process(overwrite=args.overwrite)

    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data processing error: %s' % e
Exemplo n.º 15
0
def main():
    title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser = parser0.add_inventory_parser()
    group = parser.add_argument_group('inventory display')
    group.add_argument('--md', help='Show dates using MM-DD', action='store_true', default=False)
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where, 
                                        args.tiles, args.pcov, args.ptile)
        for extent in extents:
            inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args))
            inv.pprint(md=args.md)            
           
    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data inventory error: %s' % e