Example #1
0
def main():
    title = Colors.BOLD + 'GIPS Image Statistics (v%s)' % __version__ + Colors.OFF

    parser0 = GIPSParser(datasources=False, description=title)
    parser0.add_default_parser()
    parser0.add_projdir_parser()
    group = parser0.add_argument_group('masking options')
    args = parser0.parse_args()

    # TODO - check that at least 1 of filemask or pmask is supplied

    try:
        print title
        header = ['min', 'max', 'mean', 'sd', 'skew', 'count']

        for projdir in args.projdir:
            VerboseOut('Stats for Project directory: %s' % projdir, 1)
            inv = ProjectInventory(projdir, args.products)

            files = {}
            for date in inv.dates:
                VerboseOut('Calculating statistics for %s' % date)
                for p in inv.products(date):
                    img = inv[date].open(p)
                    if p not in files.keys():
                        files[p] = open(
                            os.path.join(projdir, p + '_stats.txt'), 'w')
                        # write header
                        files[p].write('date ')
                        if img.NumBands() == 1:
                            files[p].write(' '.join(header))
                        else:
                            for band in img:
                                h = [
                                    band.Description() + "-" + a
                                    for a in header
                                ]
                                files[p].write(' '.join(h) + ' ')
                        files[p].write('\n')
                    # print date and stats
                    files[p].write(date.strftime('%Y-%j'))
                    for band in img:
                        stats = band.Stats()
                        [files[p].write(' ' + str(s)) for s in stats]
                        files[p].write(' ')
                    files[p].write('\n')
                    img = None
            for f in files:
                files[f].close()

    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Error: %s' % e
Example #2
0
def main():
    title = Colors.BOLD + "GIPS Image Statistics (v%s)" % __version__ + Colors.OFF

    parser0 = GIPSParser(datasources=False, description=title)
    parser0.add_default_parser()
    parser0.add_projdir_parser()
    group = parser0.add_argument_group("masking options")
    args = parser0.parse_args()

    # TODO - check that at least 1 of filemask or pmask is supplied

    try:
        print title
        header = ["min", "max", "mean", "sd", "skew", "count"]

        for projdir in args.projdir:
            VerboseOut("Stats for Project directory: %s" % projdir, 1)
            inv = ProjectInventory(projdir, args.products)

            files = {}
            for date in inv.dates:
                VerboseOut("Calculating statistics for %s" % date)
                for p in inv.products(date):
                    img = inv[date].open(p)
                    if p not in files.keys():
                        files[p] = open(os.path.join(projdir, p + "_stats.txt"), "w")
                        # write header
                        files[p].write("date ")
                        if img.NumBands() == 1:
                            files[p].write(" ".join(header))
                        else:
                            for band in img:
                                h = [band.Description() + "-" + a for a in header]
                                files[p].write(" ".join(h) + " ")
                        files[p].write("\n")
                    # print date and stats
                    files[p].write(date.strftime("%Y-%j"))
                    for band in img:
                        stats = band.Stats()
                        [files[p].write(" " + str(s)) for s in stats]
                        files[p].write(" ")
                    files[p].write("\n")
                    img = None
            for f in files:
                files[f].close()

    except Exception, e:
        import traceback

        VerboseOut(traceback.format_exc(), 4)
        print "Error: %s" % e
Example #3
0
def main():
    title = Colors.BOLD + 'GIPS Image Statistics (v%s)' % __version__ + Colors.OFF

    parser0 = GIPSParser(datasources=False, description=title)
    parser0.add_projdir_parser()
    group = parser0.add_argument_group('masking options')
    args = parser0.parse_args()

    utils.gips_script_setup(stop_on_error=args.stop_on_error)
    print title

    # TODO - check that at least 1 of filemask or pmask is supplied
    header = ['date', 'band', 'min', 'max', 'mean', 'sd', 'skew', 'count']

    with utils.error_handler():
        for projdir in args.projdir:
            VerboseOut('Stats for Project directory: %s' % projdir, 1)
            inv = ProjectInventory(projdir, args.products)

            p_dates = {} # map each product to its list of valid dates
            for date in inv.dates:
                for p in inv.products(date):
                    p_dates.setdefault(p, []).append(date)
            p_dates = {p: sorted(dl) for p, dl in p_dates.items()}

            for p_type, valid_dates in p_dates.items():
                stats_fn = os.path.join(projdir, p_type + '_stats.txt')
                with open(stats_fn, 'w') as stats_fo:
                    sf = getattr(utils.settings(), 'STATS_FORMAT', {})
                    writer = csv.writer(stats_fo, **sf)
                    writer.writerow(header)

                    # print date, band description, and stats
                    for date in valid_dates:
                        img = inv[date].open(p_type)
                        date_str = date.strftime('%Y-%j')
                        utils.verbose_out('Computing stats for {} {}'.format(
                                p_type, date_str), 2)
                        for b in img:
                            stats = [str(s) for s in b.Stats()]
                            writer.writerow(
                                    [date_str, b.Description()] + stats)
                        img = None

    utils.gips_exit() # produce a summary error report then quit with a proper exit status
Example #4
0
def main():
    title = Colors.BOLD + 'GIPS Data Project (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser(site_required=True)
    parser0.add_process_parser()
    parser0.add_project_parser()
    parser0.add_warp_parser()
    args = parser0.parse_args()

    try:
        print title
        cls = import_data_class(args.command)

        extents = SpatialExtent.factory(cls, args.site, args.key, args.where,
                                        args.tiles, args.pcov, args.ptile)

        # create tld: SITENAME--KEY_DATATYPE_SUFFIX
        if args.notld:
            tld = args.outdir
        else:
            key = '' if args.key == '' else '--' + args.key
            suffix = '' if args.suffix == '' else '_' + args.suffix
            res = '' if args.res is None else '_%sx%s' % (args.res[0],
                                                          args.res[1])
            bname = (extents[0].site.LayerName() + key + res + '_' +
                     args.command + suffix)
            tld = os.path.join(args.outdir, bname)

        for extent in extents:
            t_extent = TemporalExtent(args.dates, args.days)
            inv = DataInventory(cls, extent, t_extent, **vars(args))
            datadir = os.path.join(tld, extent.site.Value())
            if inv.numfiles > 0:
                inv.mosaic(
                    datadir=datadir,
                    tree=args.tree,
                    overwrite=args.overwrite,
                    res=args.res,
                    interpolation=args.interpolation,
                    crop=args.crop,
                    alltouch=args.alltouch,
                )
                inv = ProjectInventory(datadir)
                inv.pprint()
            else:
                VerboseOut(
                    'No data found for {} within temporal extent {}'.format(
                        str(t_extent), str(t_extent)),
                    2,
                )
    except Exception as e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Data Project error: %s' % e
Example #5
0
 def __init__(self, **kwargs):
     if 'projdir' in kwargs:
         self.inv = ProjectInventory(kwargs['projdir'],
                                     kwargs.get('products'))
     if 'nproc' in kwargs:
         self.nproc = kwargs['nproc']
Example #6
0
def main():
    title = Colors.BOLD + 'GIPS Data Export (v%s)' % __version__ + Colors.OFF

    # argument parsing
    parser0 = GIPSParser(description=title)
    parser0.add_inventory_parser(site_required=True)
    parser0.add_process_parser()
    parser0.add_project_parser()
    parser0.add_warp_parser()
    args = parser0.parse_args()

    cls = utils.gips_script_setup(args.command, args.stop_on_error)
    print title

    with utils.error_handler():
        extents = SpatialExtent.factory(cls,
                                        site=args.site,
                                        rastermask=args.rastermask,
                                        key=args.key,
                                        where=args.where,
                                        tiles=args.tiles,
                                        pcov=args.pcov,
                                        ptile=args.ptile)

        # create tld: SITENAME--KEY_DATATYPE_SUFFIX
        if args.notld:
            tld = args.outdir
        else:
            key = '' if args.key == '' else '--' + args.key
            suffix = '' if args.suffix == '' else '_' + args.suffix
            res = '' if args.res is None else '_%sx%s' % (args.res[0],
                                                          args.res[1])
            bname = (extents[0].site.LayerName() + key + res + '_' +
                     args.command + suffix)
            tld = os.path.join(args.outdir, bname)

        for extent in extents:
            t_extent = TemporalExtent(args.dates, args.days)
            inv = DataInventory(cls, extent, t_extent, **vars(args))
            datadir = os.path.join(tld, extent.site.Value())
            if inv.numfiles > 0:
                inv.mosaic(
                    datadir=datadir,
                    tree=args.tree,
                    overwrite=args.overwrite,
                    res=args.res,
                    interpolation=args.interpolation,
                    crop=args.crop,
                    alltouch=args.alltouch,
                    process=(not args.dont_process),
                )
                inv = ProjectInventory(datadir)
                inv.pprint()
            else:
                VerboseOut(
                    'No data found for {} within temporal extent {}'.format(
                        str(t_extent), str(t_extent)),
                    2,
                )

    utils.gips_exit(
    )  # produce a summary error report then quit with a proper exit status
Example #7
0
def main():
    title = Colors.BOLD + 'GIPS Project Masking (v%s)' % __version__ + Colors.OFF

    parser0 = GIPSParser(datasources=False, description=title)
    parser0.add_default_parser()
    parser0.add_projdir_parser()
    group = parser0.add_argument_group('masking options')
    group.add_argument('--filemask',
                       help='Mask all files with this static mask',
                       default=None)
    group.add_argument('--pmask',
                       help='Mask files with this corresponding product',
                       nargs='*',
                       default=[])
    h = 'Write mask to original image instead of creating new image'
    group.add_argument('--original',
                       help=h,
                       default=False,
                       action='store_true')
    h = 'Overwrite existing files when creating new'
    group.add_argument('--overwrite',
                       help=h,
                       default=False,
                       action='store_true')
    h = 'Suffix to apply to masked file (not compatible with --original)'
    group.add_argument('--suffix', help=h, default='-masked')
    #parser0.add_argument('-i', '--invert', help='Invert mask (0->1, 1->0)', default=False, action='store_true')
    #parser0.add_argument('--value', help='Mask == val', default=1)
    args = parser0.parse_args()

    # TODO - check that at least 1 of filemask or pmask is supplied

    try:
        print title
        for projdir in args.projdir:

            if args.filemask is not None:
                mask_file = gippy.GeoImage(args.filemask)

            inv = ProjectInventory(projdir, args.products)
            for date in inv.dates:
                VerboseOut('Masking files from %s' % date)
                available_masks = inv[date].masks(args.pmask)
                for p in inv.products(date):
                    # don't mask any masks
                    if p in available_masks:
                        continue
                    meta = ''
                    update = True if args.original else False
                    img = inv[date].open(p, update=update)
                    if args.filemask is not None:
                        img.AddMask(mask_file[0])
                        meta = basename(args.filemask) + ' '
                    for mask in available_masks:
                        img.AddMask(inv[date].open(mask)[0])
                        meta = meta + basename(inv[date][mask]) + ' '
                    if meta != '':
                        if args.original:
                            VerboseOut('  %s' % (img.Basename()), 2)
                            img.Process()
                            img.SetMeta('MASKS', meta)
                        else:
                            fout = os.path.splitext(
                                img.Filename())[0] + args.suffix + '.tif'
                            if not os.path.exists(fout) or args.overwrite:
                                VerboseOut(
                                    '  %s -> %s' %
                                    (img.Basename(), basename(fout)), 2)
                                imgout = img.Process(fout)
                                imgout.SetMeta('MASKS', meta)
                                imgout = None
                    img = None
            mask_file = None

    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Masking error: %s' % e
Example #8
0
def main():
    title = Colors.BOLD + 'GIPS Project Masking (v%s)' % __version__ + Colors.OFF

    parser = GIPSParser(datasources=False, description=title)
    parser.add_projdir_parser()
    group = parser.add_argument_group('masking options')
    group.add_argument('--filemask',
                       help='Mask all files with this static mask',
                       default=None)
    group.add_argument('--pmask',
                       help='Mask files with this corresponding product',
                       nargs='*',
                       default=[])
    group.add_argument('--invert',
                       help='Invert the masks from corresponding products',
                       nargs='*',
                       default=[])
    h = 'Write mask to original image instead of creating new image'
    group.add_argument('--original',
                       help=h,
                       default=False,
                       action='store_true')
    h = 'Overwrite existing files when creating new'
    group.add_argument('--overwrite',
                       help=h,
                       default=False,
                       action='store_true')
    h = 'Suffix to apply to masked file (not compatible with --original)'
    group.add_argument('--suffix', help=h, default='-masked')
    args = parser.parse_args()

    # TODO - check that at least 1 of filemask or pmask is supplied

    utils.gips_script_setup(None, args.stop_on_error)

    with utils.error_handler('Masking error'):
        VerboseOut(title)
        for projdir in args.projdir:

            if args.filemask is not None:
                mask_file = gippy.GeoImage(args.filemask)

            inv = ProjectInventory(projdir, args.products)
            for date in inv.dates:
                VerboseOut('Masking files from %s' % date)
                if args.filemask is None and args.pmask == []:
                    available_masks = inv[date].masks()
                else:
                    available_masks = inv[date].masks(args.pmask)
                for p in inv.products(date):
                    # don't mask any masks
                    if p in available_masks:
                        continue
                    meta = ''
                    update = True if args.original else False
                    img = inv[date].open(p, update=update)
                    if args.filemask is not None:
                        img.AddMask(mask_file[0])
                        meta = basename(args.filemask) + ' '
                    for mask in available_masks:
                        mask_img = inv[date].open(mask)[0]
                        if mask in args.invert:
                            mask_img.SetNoData(utils.np.nan)
                            mask_img = mask_img.BXOR(1)
                            meta += 'inverted-'
                        img.AddMask(mask_img)
                        meta = meta + basename(inv[date][mask]) + ' '
                    if meta != '':
                        if args.original:
                            VerboseOut('  %s' % (img.Basename()), 2)
                            img.Process()
                            img.SetMeta('MASKS', meta)
                        else:
                            fout = os.path.splitext(
                                img.Filename())[0] + args.suffix + '.tif'
                            if not os.path.exists(fout) or args.overwrite:
                                VerboseOut(
                                    '  %s -> %s' %
                                    (img.Basename(), basename(fout)), 2)
                                imgout = img.Process(fout)
                                imgout.SetMeta('MASKS', meta)
                                imgout = None
                    img = None
            mask_file = None

    utils.gips_exit()
Example #9
0
def main():
    title = Colors.BOLD + 'GIPS Project Masking (v%s)' % __version__ + Colors.OFF

    parser0 = GIPSParser(datasources=False, description=title)
    parser0.add_default_parser()
    parser0.add_projdir_parser()
    group = parser0.add_argument_group('masking options')
    group.add_argument('--filemask', help='Mask all files with this static mask', default=None)
    group.add_argument('--pmask', help='Mask files with this corresponding product', nargs='*', default=[])
    h = 'Write mask to original image instead of creating new image'
    group.add_argument('--original', help=h, default=False, action='store_true')
    h = 'Overwrite existing files when creating new'
    group.add_argument('--overwrite', help=h, default=False, action='store_true')
    h = 'Suffix to apply to masked file (not compatible with --original)'
    group.add_argument('--suffix', help=h, default='-masked')
    #parser0.add_argument('-i', '--invert', help='Invert mask (0->1, 1->0)', default=False, action='store_true')
    #parser0.add_argument('--value', help='Mask == val', default=1)
    args = parser0.parse_args()

    # TODO - check that at least 1 of filemask or pmask is supplied

    try:
        VerboseOut(title)
        for projdir in args.projdir:

            if args.filemask is not None:
                mask_file = gippy.GeoImage(args.filemask)

            inv = ProjectInventory(projdir, args.products)
            for date in inv.dates:
                VerboseOut('Masking files from %s' % date)
                if args.filemask is None and args.pmask == []:
                    available_masks = inv[date].masks()
                else:
                    available_masks = inv[date].masks(args.pmask)
                for p in inv.products(date):
                    # don't mask any masks
                    if p in available_masks:
                        continue
                    meta = ''
                    update = True if args.original else False
                    img = inv[date].open(p, update=update)
                    if args.filemask is not None:
                        img.AddMask(mask_file[0])
                        meta = basename(args.filemask) + ' '
                    for mask in available_masks:
                        img.AddMask(inv[date].open(mask)[0])
                        meta = meta + basename(inv[date][mask]) + ' '
                    if meta != '':
                        if args.original:
                            VerboseOut('  %s' % (img.Basename()), 2)
                            img.Process()
                            img.SetMeta('MASKS', meta)
                        else:
                            fout = os.path.splitext(img.Filename())[0] + args.suffix + '.tif'
                            if not os.path.exists(fout) or args.overwrite:
                                VerboseOut('  %s -> %s' % (img.Basename(), basename(fout)), 2)
                                imgout = img.Process(fout)
                                imgout.SetMeta('MASKS', meta)
                                imgout = None
                    img = None
            mask_file = None

    except Exception, e:
        import traceback
        VerboseOut(traceback.format_exc(), 4)
        print 'Masking error: %s' % e