Beispiel #1
0
# -*- coding: utf-8 -*-

import sys

import math
from pymrgeo import MrGeo
from pymrgeo.rastermapop import RasterMapOp

if __name__ == "__main__":

    # mrgeo = MrGeo()  # forked mrgeo
    mrgeo = MrGeo(host="localhost", port=12345)  # already running, remote mrgeo

    # sys.exit(1)

    mrgeo.usedebug()

    # images = mrgeo.list_images()

    mrgeo.start()

    ones = mrgeo.load_image("all-ones-save")

    # test error handling
    # foo = 1 / 0
    sl = ones.slope()
    # RasterMapOp.slope(ones, 1)

    # hundreds = mrgeo.load_image("all-hundreds")
    # aspect = hundreds.aspect()
Beispiel #2
0
def main(argv):
    root = ''
    month = -1
    year = -1
    output_postfix = ''
    try:
        opts, args = getopt.getopt(argv, "hr:m:y:o:",
                                   ["root=", "month=,year=,output-postfix="])
    except getopt.GetoptError:
        print 'multi-landsat.py -r <rootdir> -m <monthnum> -y <year> -o <output-postfix>'
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            print 'multi-landsat.py -r <rootdir> -m <monthnum> -y <year> -o <output-postfix>'
            sys.exit()
        elif opt in ("-r", "--root"):
            root = arg
        elif opt in ("-m", "--month"):
            month = int(arg)
        elif opt in ("-y", "--year"):
            year = int(arg)
        elif opt in ("-o", "--output-postfix"):
            output_postfix = arg

    print 'Root is %s' % (root)
    print 'Month is %d' % (month)
    print 'Year is %d' % (year)

    metadata = load_all_metadata(root)

    # Filter the metadata to find the most recent imagery for each scene.
    # It is assumed that the root directory only contains scenes we are
    # interested in.
    filtered_metadata = {}
    for md in metadata:
        acqdate = parse(
            md["L1_METADATA_FILE"]["PRODUCT_METADATA"]["DATE_ACQUIRED"])
        if acqdate.month == month and acqdate.year == year:
            wrs_path = int(
                md["L1_METADATA_FILE"]["PRODUCT_METADATA"]["WRS_PATH"])
            wrs_row = int(
                md["L1_METADATA_FILE"]["PRODUCT_METADATA"]["WRS_ROW"])
            key = "%03d%03d" % (wrs_path, wrs_row)
            if filtered_metadata.get(key) is None:
                filtered_metadata[key] = md
            else:
                existing_acq_date = parse(
                    filtered_metadata[key]["L1_METADATA_FILE"]
                    ["PRODUCT_METADATA"]["DATE_ACQUIRED"])
                if acqdate > existing_acq_date:
                    filtered_metadata[key] = md
    for key in filtered_metadata:
        print "Processing scene at path %s, row %s, acquired %s, at %s" % (
            filtered_metadata[key]["L1_METADATA_FILE"]["PRODUCT_METADATA"]
            ["WRS_PATH"], filtered_metadata[key]["L1_METADATA_FILE"]
            ["PRODUCT_METADATA"]["WRS_ROW"], filtered_metadata[key]
            ["L1_METADATA_FILE"]["PRODUCT_METADATA"]["DATE_ACQUIRED"],
            filtered_metadata[key]["PARENT_DIR"])

    # being a separate image
    mrgeo = MrGeo()
    mrgeo.usedebug()

    mrgeo.start()

    # Let's go ahead and ingest those images into MrGeo, with each band
    red_refl = ingest_reflectance_image(mrgeo, filtered_metadata, 4,
                                        "landsat-red-refl" + output_postfix)
    nir_refl = ingest_reflectance_image(mrgeo, filtered_metadata, 5,
                                        "landsat-nir-refl" + output_postfix)

    ndvi = (nir_refl - red_refl) / (nir_refl + red_refl)
    ndvi.save('landsat-ndvi' + output_postfix)

    mrgeo.stop()
    print("***** Done *****")
Beispiel #3
0
                    else:
                        # strip off the "B"
                        band = int(b[1:])
                        landsat[band] = pathname
                    pass
                elif ext == '.json':
                    landsat[ext[1:]] = pathname
                    pass

    if not landsat.has_key('json'):
        raise Exception('No JSON metadata file in ' + root)

    with open(landsat['json']) as metafile:
        metadata = json.load(metafile)

    mrgeo = MrGeo()
    # mrgeo.usedebug()

    mrgeo.start()

    bqa = mrgeo.ingest_image(landsat['bqa'])
    # bqa.export('/data/export/landsat-bqa.tif', singleFile=True)

    cloud_mask = bqa < 32768  # 0 where clouds, 1 where no clouds
    # cloud_mask.export('/data/export/landsat-clouds.tif', singleFile=True)

    red = cloud_mask.con(positiveRaster=mrgeo.ingest_image(landsat[4]),
                         negativeConst=RasterMapOp.nan())
    green = cloud_mask.con(positiveRaster=mrgeo.ingest_image(landsat[3]),
                           negativeConst=RasterMapOp.nan())
    blue = cloud_mask.con(positiveRaster=mrgeo.ingest_image(landsat[2]),
Beispiel #4
0
import sys


from pymrgeo import MrGeo

if __name__ == "__main__":
    mrgeo = MrGeo()

    # sys.exit(1)

    mrgeo.usedebug()

    images = mrgeo.list_images()

    mrgeo.start()

    ones = mrgeo.load_image("all-ones-save")

    # slope = ones.slope()

    # hundreds = mrgeo.load_resource("all-hundreds")
    # aspect = hundreds.aspect()

    # slope.save("slope-test")
    # aspect.save("aspect-test")

    print("***** Starting *****")
    # small_elevation = mrgeo.load_resource("small-elevation")
    # slope = small_elevation.slope()
    # slope.save("slope-test")
    # print("***** Finished Slope 1 *****")
Beispiel #5
0
# -*- coding: utf-8 -*-

import sys

import math
from pymrgeo import MrGeo

if __name__ == "__main__":

    mrgeo = MrGeo()

    # sys.exit(1)

    mrgeo.usedebug()

    # images = mrgeo.list_images()

    mrgeo.start()

    # ones = mrgeo.load_image("all-ones-save")

    # slope = ones.slope()

    # hundreds = mrgeo.load_image("all-hundreds")
    # aspect = hundreds.aspect()

    # slope.save("slope-test")
    # aspect.save("aspect-test")

    print("***** Starting *****")
    elevation = mrgeo.load_image("aster-30m-xsmall")
Beispiel #6
0
    def setUpClass(cls):
        cls.classname = cls.__name__

        # print(cls.classname + " setup")

        cls.mrgeo = MrGeo()
        cls.gateway = cls.mrgeo.gateway

        jvm = cls.gateway.jvm
        java_import(jvm, "org.mrgeo.core.MrGeoConstants")
        java_import(jvm, "org.mrgeo.core.MrGeoProperties")
        java_import(jvm, "org.mrgeo.hdfs.utils.HadoopFileUtils")
        java_import(jvm, "org.apache.hadoop.fs.Path")
        java_import(jvm, "org.mrgeo.utils.LoggingUtils")

        fs = jvm.HadoopFileUtils.getFileSystem()
        p = jvm.Path(cls._INPUT_BASE).makeQualified(fs)
        cls._INPUT_HDFS = p

        p = jvm.Path(cls._OUTPUT_BASE).makeQualified(fs)
        cls._OUTPUT_HDFS = p

        basedir = os.getenv('BASEDIR', '.')
        dirname = os.path.abspath(basedir)
        try:
            while True:
                names = os.listdir(dirname)
                if cls._INPUT in names:
                    break
                dirname = os.path.abspath(os.path.join(dirname, os.pardir))
        except:
            pass

        basedir = os.path.abspath(dirname)

        cls.inputdir = os.path.abspath(basedir + '/' + cls._INPUT + "/" +
                                       cls.classname) + '/'
        cls.outputdir = os.path.abspath(basedir + '/' + cls._INPUT + '/' +
                                        cls._OUTPUT + "/" +
                                        cls.classname) + '/'

        cls.inputhdfs = jvm.Path(
            cls._INPUT_HDFS,
            "python/" + cls.classname).makeQualified(fs).toString() + '/'
        cls.outputhdfs = jvm.Path(
            cls._OUTPUT_HDFS,
            "python/" + cls.classname).makeQualified(fs).toString() + '/'

        if not os.path.exists(cls.inputdir):
            os.makedirs(cls.inputdir)

        if os.path.exists(cls.outputdir):
            shutil.rmtree(cls.outputdir, ignore_errors=True)

        if not os.path.exists(cls.outputdir):
            os.makedirs(cls.outputdir)

        jvm.HadoopFileUtils.create(cls.inputhdfs)

        if jvm.HadoopFileUtils.exists(cls.outputhdfs):
            jvm.HadoopFileUtils.cleanDirectory(cls.outputhdfs)

        jvm.HadoopFileUtils.create(cls.outputhdfs)

        jvm.MrGeoProperties.getInstance().setProperty(
            jvm.MrGeoConstants.MRGEO_HDFS_IMAGE, cls.inputhdfs)
        jvm.MrGeoProperties.getInstance().setProperty(
            jvm.MrGeoConstants.MRGEO_HDFS_VECTOR, cls.inputhdfs)

        jvm.LoggingUtils.setDefaultLogLevel(jvm.LoggingUtils.ERROR)
Beispiel #7
0
                    else:
                        # strip off the "B"
                        band = int(b[1:])
                        landsat[band] = pathname
                    pass
                elif ext == ".json":
                    landsat[ext[1:]] = pathname
                    pass

    if not landsat.has_key("json"):
        raise Exception("No JSON metadata file in " + root)

    with open(landsat["json"]) as metafile:
        metadata = json.load(metafile)

    mrgeo = MrGeo()
    # mrgeo.usedebug()

    mrgeo.start()

    bqa = mrgeo.ingest_image(landsat["bqa"])
    # bqa.export('/data/export/landsat-bqa.tif', singleFile=True)

    cloud_mask = bqa < 32768  # 0 where clouds, 1 where no clouds
    # cloud_mask.export('/data/export/landsat-clouds.tif', singleFile=True)

    red = cloud_mask.con(positiveRaster=mrgeo.ingest_image(landsat[4]), negativeConst=RasterMapOp.nan())
    green = cloud_mask.con(positiveRaster=mrgeo.ingest_image(landsat[3]), negativeConst=RasterMapOp.nan())
    blue = cloud_mask.con(positiveRaster=mrgeo.ingest_image(landsat[2]), negativeConst=RasterMapOp.nan())
    # red = mrgeo.ingest_image(landsat[4])
    # green = mrgeo.ingest_image(landsat[3])
Beispiel #8
0
def main(argv):
    root = ''
    month = -1
    year = -1
    output_postfix = ''
    try:
        opts, args = getopt.getopt(argv,"hr:m:y:o:",["root=","month=,year=,output-postfix="])
    except getopt.GetoptError:
        print 'multi-landsat.py -r <rootdir> -m <monthnum> -y <year> -o <output-postfix>'
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            print 'multi-landsat.py -r <rootdir> -m <monthnum> -y <year> -o <output-postfix>'
            sys.exit()
        elif opt in ("-r", "--root"):
            root = arg
        elif opt in ("-m", "--month"):
            month = int(arg)
        elif opt in ("-y", "--year"):
            year = int(arg)
        elif opt in ("-o", "--output-postfix"):
            output_postfix = arg

    print 'Root is %s' % (root)
    print 'Month is %d' % (month)
    print 'Year is %d' % (year)

    metadata = load_all_metadata(root)

    # Filter the metadata to find the most recent imagery for each scene.
    # It is assumed that the root directory only contains scenes we are
    # interested in.
    filtered_metadata = {}
    for md in metadata:
        acqdate = parse(md["L1_METADATA_FILE"]["PRODUCT_METADATA"]["DATE_ACQUIRED"])
        if acqdate.month == month and acqdate.year == year:
            wrs_path = int(md["L1_METADATA_FILE"]["PRODUCT_METADATA"]["WRS_PATH"])
            wrs_row = int(md["L1_METADATA_FILE"]["PRODUCT_METADATA"]["WRS_ROW"])
            key = "%03d%03d" % (wrs_path, wrs_row)
            if filtered_metadata.get(key) is None:
                filtered_metadata[key] = md
            else:
                existing_acq_date = parse(filtered_metadata[key]["L1_METADATA_FILE"]["PRODUCT_METADATA"]["DATE_ACQUIRED"])
                if acqdate > existing_acq_date:
                    filtered_metadata[key] = md
    for key in filtered_metadata:
        print "Processing scene at path %s, row %s, acquired %s, at %s" % (filtered_metadata[key]["L1_METADATA_FILE"]["PRODUCT_METADATA"]["WRS_PATH"],
                                                                    filtered_metadata[key]["L1_METADATA_FILE"]["PRODUCT_METADATA"]["WRS_ROW"],
                                                                    filtered_metadata[key]["L1_METADATA_FILE"]["PRODUCT_METADATA"]["DATE_ACQUIRED"],
                                                                    filtered_metadata[key]["PARENT_DIR"])

    # being a separate image
    mrgeo = MrGeo()
    mrgeo.usedebug()

    mrgeo.start()

    # Let's go ahead and ingest those images into MrGeo, with each band
    red_refl = ingest_reflectance_image(mrgeo, filtered_metadata, 4, "landsat-red-refl" + output_postfix)
    nir_refl = ingest_reflectance_image(mrgeo, filtered_metadata, 5, "landsat-nir-refl" + output_postfix)

    ndvi = (nir_refl - red_refl) / (nir_refl + nir_refl)
    ndvi.save('landsat-ndvi' + output_postfix)

    mrgeo.stop()
    print("***** Done *****")
Beispiel #9
0
# -*- coding: utf-8 -*-

import sys
from pymrgeo import MrGeo

if __name__ == "__main__":
    mrgeo = MrGeo()  # forked mrgeo
    # mrgeo = MrGeo(host="localhost", port=12345)  # already running, remote mrgeo

    # sys.exit(1)

    # mrgeo.usedebug()

    # images = mrgeo.list_images()

    mrgeo.start()

    # ones = mrgeo.ingest_image(name="gis-data/aster-30m-xsmall")
    # ones.save("aster-small-python-test")

    # ones.export("/data/export/aster-small-python-test", singleFile=True)

    ones = mrgeo.load_image("all-ones-save")

    rdd = ones.rdd()
    print(rdd)

    list = rdd.collect()
    print(list)

    for tile in list:
Beispiel #10
0
# -*- coding: utf-8 -*-

import sys

import numpy
import zlib
import struct
from pymrgeo import MrGeo

if __name__ == "__main__":

    mrgeo = MrGeo()

    # sys.exit(1)

    mrgeo.usedebug()

    # images = mrgeo.list_images()

    mrgeo.start()

    # ones = mrgeo.load_image("all-ones-save")

    # slope = ones.slope()

    # hundreds = mrgeo.load_resource("all-hundreds")
    # aspect = hundreds.aspect()

    # slope.save("slope-test")
    # aspect.save("aspect-test")