# -*- coding: utf-8 -*-
import sys
import math
from pymrgeo import MrGeo
from pymrgeo.rastermapop import RasterMapOp
if __name__ == "__main__":
# mrgeo = MrGeo() # forked mrgeo
mrgeo = MrGeo(host="localhost", port=12345) # already running, remote mrgeo
# sys.exit(1)
mrgeo.usedebug()
# images = mrgeo.list_images()
mrgeo.start()
ones = mrgeo.load_image("all-ones-save")
# test error handling
# foo = 1 / 0
sl = ones.slope()
# RasterMapOp.slope(ones, 1)
# hundreds = mrgeo.load_image("all-hundreds")
# aspect = hundreds.aspect()
def main(argv):
root = ''
month = -1
year = -1
output_postfix = ''
try:
opts, args = getopt.getopt(argv, "hr:m:y:o:",
["root=", "month=,year=,output-postfix="])
except getopt.GetoptError:
print 'multi-landsat.py -r <rootdir> -m <monthnum> -y <year> -o <output-postfix>'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print 'multi-landsat.py -r <rootdir> -m <monthnum> -y <year> -o <output-postfix>'
sys.exit()
elif opt in ("-r", "--root"):
root = arg
elif opt in ("-m", "--month"):
month = int(arg)
elif opt in ("-y", "--year"):
year = int(arg)
elif opt in ("-o", "--output-postfix"):
output_postfix = arg
print 'Root is %s' % (root)
print 'Month is %d' % (month)
print 'Year is %d' % (year)
metadata = load_all_metadata(root)
# Filter the metadata to find the most recent imagery for each scene.
# It is assumed that the root directory only contains scenes we are
# interested in.
filtered_metadata = {}
for md in metadata:
acqdate = parse(
md["L1_METADATA_FILE"]["PRODUCT_METADATA"]["DATE_ACQUIRED"])
if acqdate.month == month and acqdate.year == year:
wrs_path = int(
md["L1_METADATA_FILE"]["PRODUCT_METADATA"]["WRS_PATH"])
wrs_row = int(
md["L1_METADATA_FILE"]["PRODUCT_METADATA"]["WRS_ROW"])
key = "%03d%03d" % (wrs_path, wrs_row)
if filtered_metadata.get(key) is None:
filtered_metadata[key] = md
else:
existing_acq_date = parse(
filtered_metadata[key]["L1_METADATA_FILE"]
["PRODUCT_METADATA"]["DATE_ACQUIRED"])
if acqdate > existing_acq_date:
filtered_metadata[key] = md
for key in filtered_metadata:
print "Processing scene at path %s, row %s, acquired %s, at %s" % (
filtered_metadata[key]["L1_METADATA_FILE"]["PRODUCT_METADATA"]
["WRS_PATH"], filtered_metadata[key]["L1_METADATA_FILE"]
["PRODUCT_METADATA"]["WRS_ROW"], filtered_metadata[key]
["L1_METADATA_FILE"]["PRODUCT_METADATA"]["DATE_ACQUIRED"],
filtered_metadata[key]["PARENT_DIR"])
# being a separate image
mrgeo = MrGeo()
mrgeo.usedebug()
mrgeo.start()
# Let's go ahead and ingest those images into MrGeo, with each band
red_refl = ingest_reflectance_image(mrgeo, filtered_metadata, 4,
"landsat-red-refl" + output_postfix)
nir_refl = ingest_reflectance_image(mrgeo, filtered_metadata, 5,
"landsat-nir-refl" + output_postfix)
ndvi = (nir_refl - red_refl) / (nir_refl + red_refl)
ndvi.save('landsat-ndvi' + output_postfix)
mrgeo.stop()
print("***** Done *****")
else:
# strip off the "B"
band = int(b[1:])
landsat[band] = pathname
pass
elif ext == '.json':
landsat[ext[1:]] = pathname
pass
if not landsat.has_key('json'):
raise Exception('No JSON metadata file in ' + root)
with open(landsat['json']) as metafile:
metadata = json.load(metafile)
mrgeo = MrGeo()
# mrgeo.usedebug()
mrgeo.start()
bqa = mrgeo.ingest_image(landsat['bqa'])
# bqa.export('/data/export/landsat-bqa.tif', singleFile=True)
cloud_mask = bqa < 32768 # 0 where clouds, 1 where no clouds
# cloud_mask.export('/data/export/landsat-clouds.tif', singleFile=True)
red = cloud_mask.con(positiveRaster=mrgeo.ingest_image(landsat[4]),
negativeConst=RasterMapOp.nan())
green = cloud_mask.con(positiveRaster=mrgeo.ingest_image(landsat[3]),
negativeConst=RasterMapOp.nan())
blue = cloud_mask.con(positiveRaster=mrgeo.ingest_image(landsat[2]),
import sys
from pymrgeo import MrGeo
if __name__ == "__main__":
mrgeo = MrGeo()
# sys.exit(1)
mrgeo.usedebug()
images = mrgeo.list_images()
mrgeo.start()
ones = mrgeo.load_image("all-ones-save")
# slope = ones.slope()
# hundreds = mrgeo.load_resource("all-hundreds")
# aspect = hundreds.aspect()
# slope.save("slope-test")
# aspect.save("aspect-test")
print("***** Starting *****")
# small_elevation = mrgeo.load_resource("small-elevation")
# slope = small_elevation.slope()
# slope.save("slope-test")
# print("***** Finished Slope 1 *****")
# -*- coding: utf-8 -*-
import sys
import math
from pymrgeo import MrGeo
if __name__ == "__main__":
mrgeo = MrGeo()
# sys.exit(1)
mrgeo.usedebug()
# images = mrgeo.list_images()
mrgeo.start()
# ones = mrgeo.load_image("all-ones-save")
# slope = ones.slope()
# hundreds = mrgeo.load_image("all-hundreds")
# aspect = hundreds.aspect()
# slope.save("slope-test")
# aspect.save("aspect-test")
print("***** Starting *****")
elevation = mrgeo.load_image("aster-30m-xsmall")
def setUpClass(cls):
cls.classname = cls.__name__
# print(cls.classname + " setup")
cls.mrgeo = MrGeo()
cls.gateway = cls.mrgeo.gateway
jvm = cls.gateway.jvm
java_import(jvm, "org.mrgeo.core.MrGeoConstants")
java_import(jvm, "org.mrgeo.core.MrGeoProperties")
java_import(jvm, "org.mrgeo.hdfs.utils.HadoopFileUtils")
java_import(jvm, "org.apache.hadoop.fs.Path")
java_import(jvm, "org.mrgeo.utils.LoggingUtils")
fs = jvm.HadoopFileUtils.getFileSystem()
p = jvm.Path(cls._INPUT_BASE).makeQualified(fs)
cls._INPUT_HDFS = p
p = jvm.Path(cls._OUTPUT_BASE).makeQualified(fs)
cls._OUTPUT_HDFS = p
basedir = os.getenv('BASEDIR', '.')
dirname = os.path.abspath(basedir)
try:
while True:
names = os.listdir(dirname)
if cls._INPUT in names:
break
dirname = os.path.abspath(os.path.join(dirname, os.pardir))
except:
pass
basedir = os.path.abspath(dirname)
cls.inputdir = os.path.abspath(basedir + '/' + cls._INPUT + "/" +
cls.classname) + '/'
cls.outputdir = os.path.abspath(basedir + '/' + cls._INPUT + '/' +
cls._OUTPUT + "/" +
cls.classname) + '/'
cls.inputhdfs = jvm.Path(
cls._INPUT_HDFS,
"python/" + cls.classname).makeQualified(fs).toString() + '/'
cls.outputhdfs = jvm.Path(
cls._OUTPUT_HDFS,
"python/" + cls.classname).makeQualified(fs).toString() + '/'
if not os.path.exists(cls.inputdir):
os.makedirs(cls.inputdir)
if os.path.exists(cls.outputdir):
shutil.rmtree(cls.outputdir, ignore_errors=True)
if not os.path.exists(cls.outputdir):
os.makedirs(cls.outputdir)
jvm.HadoopFileUtils.create(cls.inputhdfs)
if jvm.HadoopFileUtils.exists(cls.outputhdfs):
jvm.HadoopFileUtils.cleanDirectory(cls.outputhdfs)
jvm.HadoopFileUtils.create(cls.outputhdfs)
jvm.MrGeoProperties.getInstance().setProperty(
jvm.MrGeoConstants.MRGEO_HDFS_IMAGE, cls.inputhdfs)
jvm.MrGeoProperties.getInstance().setProperty(
jvm.MrGeoConstants.MRGEO_HDFS_VECTOR, cls.inputhdfs)
jvm.LoggingUtils.setDefaultLogLevel(jvm.LoggingUtils.ERROR)
else:
# strip off the "B"
band = int(b[1:])
landsat[band] = pathname
pass
elif ext == ".json":
landsat[ext[1:]] = pathname
pass
if not landsat.has_key("json"):
raise Exception("No JSON metadata file in " + root)
with open(landsat["json"]) as metafile:
metadata = json.load(metafile)
mrgeo = MrGeo()
# mrgeo.usedebug()
mrgeo.start()
bqa = mrgeo.ingest_image(landsat["bqa"])
# bqa.export('/data/export/landsat-bqa.tif', singleFile=True)
cloud_mask = bqa < 32768 # 0 where clouds, 1 where no clouds
# cloud_mask.export('/data/export/landsat-clouds.tif', singleFile=True)
red = cloud_mask.con(positiveRaster=mrgeo.ingest_image(landsat[4]), negativeConst=RasterMapOp.nan())
green = cloud_mask.con(positiveRaster=mrgeo.ingest_image(landsat[3]), negativeConst=RasterMapOp.nan())
blue = cloud_mask.con(positiveRaster=mrgeo.ingest_image(landsat[2]), negativeConst=RasterMapOp.nan())
# red = mrgeo.ingest_image(landsat[4])
# green = mrgeo.ingest_image(landsat[3])
def main(argv):
root = ''
month = -1
year = -1
output_postfix = ''
try:
opts, args = getopt.getopt(argv,"hr:m:y:o:",["root=","month=,year=,output-postfix="])
except getopt.GetoptError:
print 'multi-landsat.py -r <rootdir> -m <monthnum> -y <year> -o <output-postfix>'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print 'multi-landsat.py -r <rootdir> -m <monthnum> -y <year> -o <output-postfix>'
sys.exit()
elif opt in ("-r", "--root"):
root = arg
elif opt in ("-m", "--month"):
month = int(arg)
elif opt in ("-y", "--year"):
year = int(arg)
elif opt in ("-o", "--output-postfix"):
output_postfix = arg
print 'Root is %s' % (root)
print 'Month is %d' % (month)
print 'Year is %d' % (year)
metadata = load_all_metadata(root)
# Filter the metadata to find the most recent imagery for each scene.
# It is assumed that the root directory only contains scenes we are
# interested in.
filtered_metadata = {}
for md in metadata:
acqdate = parse(md["L1_METADATA_FILE"]["PRODUCT_METADATA"]["DATE_ACQUIRED"])
if acqdate.month == month and acqdate.year == year:
wrs_path = int(md["L1_METADATA_FILE"]["PRODUCT_METADATA"]["WRS_PATH"])
wrs_row = int(md["L1_METADATA_FILE"]["PRODUCT_METADATA"]["WRS_ROW"])
key = "%03d%03d" % (wrs_path, wrs_row)
if filtered_metadata.get(key) is None:
filtered_metadata[key] = md
else:
existing_acq_date = parse(filtered_metadata[key]["L1_METADATA_FILE"]["PRODUCT_METADATA"]["DATE_ACQUIRED"])
if acqdate > existing_acq_date:
filtered_metadata[key] = md
for key in filtered_metadata:
print "Processing scene at path %s, row %s, acquired %s, at %s" % (filtered_metadata[key]["L1_METADATA_FILE"]["PRODUCT_METADATA"]["WRS_PATH"],
filtered_metadata[key]["L1_METADATA_FILE"]["PRODUCT_METADATA"]["WRS_ROW"],
filtered_metadata[key]["L1_METADATA_FILE"]["PRODUCT_METADATA"]["DATE_ACQUIRED"],
filtered_metadata[key]["PARENT_DIR"])
# being a separate image
mrgeo = MrGeo()
mrgeo.usedebug()
mrgeo.start()
# Let's go ahead and ingest those images into MrGeo, with each band
red_refl = ingest_reflectance_image(mrgeo, filtered_metadata, 4, "landsat-red-refl" + output_postfix)
nir_refl = ingest_reflectance_image(mrgeo, filtered_metadata, 5, "landsat-nir-refl" + output_postfix)
ndvi = (nir_refl - red_refl) / (nir_refl + nir_refl)
ndvi.save('landsat-ndvi' + output_postfix)
mrgeo.stop()
print("***** Done *****")
# -*- coding: utf-8 -*-
import sys
from pymrgeo import MrGeo
if __name__ == "__main__":
mrgeo = MrGeo() # forked mrgeo
# mrgeo = MrGeo(host="localhost", port=12345) # already running, remote mrgeo
# sys.exit(1)
# mrgeo.usedebug()
# images = mrgeo.list_images()
mrgeo.start()
# ones = mrgeo.ingest_image(name="gis-data/aster-30m-xsmall")
# ones.save("aster-small-python-test")
# ones.export("/data/export/aster-small-python-test", singleFile=True)
ones = mrgeo.load_image("all-ones-save")
rdd = ones.rdd()
print(rdd)
list = rdd.collect()
print(list)
for tile in list:
# -*- coding: utf-8 -*-
import sys
import numpy
import zlib
import struct
from pymrgeo import MrGeo
if __name__ == "__main__":
mrgeo = MrGeo()
# sys.exit(1)
mrgeo.usedebug()
# images = mrgeo.list_images()
mrgeo.start()
# ones = mrgeo.load_image("all-ones-save")
# slope = ones.slope()
# hundreds = mrgeo.load_resource("all-hundreds")
# aspect = hundreds.aspect()
# slope.save("slope-test")
# aspect.save("aspect-test")
