def data_search(catalog_path, json_path, dem_tif_path, dir_tif_path,
acc_tif_path):
catalog_path = "file://" + catalog_path
conf = gps.geopyspark_conf(master="local[*]", appName="master")
pysc = SparkContext(conf=conf)
polys = None
# Creates a MultiPolygon from Geojson
with open(json_path) as f:
js = json.load(f)
FeatureObj = None
if js['type'] == 'FeatureCollection':
FeatureObj = js['features'][0]
elif js['type'] == 'Feature':
FeatureObj = js
if FeatureObj['geometry']['type'] == 'MultiPolygon':
polygons = FeatureObj['geometry']['coordinates']
polygons_array = []
for polygon in polygons:
input_array = []
for point in polygon[0]:
input_array.append(tuple(point))
polygons_array.append(Polygon(input_array))
polys = MultiPolygon(polygons_array)
elif FeatureObj['geometry']['type'] == 'Polygon':
polygon = FeatureObj['geometry']['coordinates']
points = polygon[0]
input_array = []
for point in points:
input_array.append(tuple(point))
polys = Polygon(input_array)
print("Get DEM")
tiled_raster_layer = gps.query(uri=catalog_path,
layer_name="dem",
layer_zoom=0,
query_geom=polys)
print(tiled_raster_layer.count())
print(tiled_raster_layer.layer_metadata.extent)
tiled_raster_layer.save_stitched(dem_tif_path)
print("Get Direction")
tiled_raster_layer = gps.query(uri=catalog_path,
layer_name="direction",
layer_zoom=0,
query_geom=polys)
# tiled_raster_layer = gps.query(uri=catalog_path, layer_name="dir", layer_zoom=0, query_geom=polys)
print(tiled_raster_layer.count())
print(tiled_raster_layer.layer_metadata.extent)
tiled_raster_layer.save_stitched(dir_tif_path)
print("Get Accumulation")
tiled_raster_layer = gps.query(uri=catalog_path,
layer_name="accumulation",
layer_zoom=0,
query_geom=polys)
# tiled_raster_layer = gps.query(uri=catalog_path, layer_name="acc", layer_zoom=0, query_geom=polys)
print(tiled_raster_layer.count())
print(tiled_raster_layer.layer_metadata.extent)
tiled_raster_layer.save_stitched(acc_tif_path)
def test():
conf = gps.geopyspark_conf(master="local[*]", appName="master")
pysc = SparkContext(conf=conf)
layer_metadata = gps.read_layer_metadata(uri="file:///usr/local/large_scale_hydro/catalog", layer_name="demo-dem", layer_zoom=0)
layer_extent = layer_metadata.extent
print(layer_extent)
poly = None
# Creates a Polygon from Geojson
json_path = '/usr/local/large_scale_hydro/result/polygon.geojson'
with open(json_path) as f:
js = json.load(f)
features = js['features']
if features[0]['geometry']['type'] == 'Polygon':
polygon = features[0]['geometry']['coordinates']
points = polygon[0]
input_array = []
for point in points:
input_array.append(tuple(point))
poly = Polygon(input_array)
tiled_raster_layer = gps.query(uri="file:///usr/local/large_scale_hydro/catalog", layer_name="demo-dem", layer_zoom=0, query_geom=poly)
print(tiled_raster_layer.count())
print(tiled_raster_layer.layer_metadata.extent)
tiled_raster_layer.save_stitched('/usr/local/large_scale_hydro/result/result_geojson.tif')
def test(work_path):
process_path = work_path + "/process"
if not os.path.exists(process_path):
os.makedirs(process_path)
conf = gps.geopyspark_conf(master="local[*]", appName="master")
pysc = SparkContext(conf=conf)
poly = None
# Creates a Polygon from Geojson
json_path = '/usr/local/large_scale_hydro/result/polygon.geojson'
with open(json_path) as f:
js = json.load(f)
FeatureObj = None
if js['type'] == 'FeatureCollection':
FeatureObj = js['features'][0]
elif js['type'] == 'Feature':
FeatureObj = js
if FeatureObj['geometry']['type'] == 'MultiPolygon':
polygons = FeatureObj['geometry']['coordinates']
polygons_array = []
for polygon in polygons:
input_array = []
for point in polygon[0]:
input_array.append(tuple(point))
polygons_array.append(Polygon(input_array))
polys = MultiPolygon(polygons_array)
elif FeatureObj['geometry']['type'] == 'Polygon':
polygon = FeatureObj['geometry']['coordinates']
points = polygon[0]
input_array = []
for point in points:
input_array.append(tuple(point))
polys = Polygon(input_array)
dem_tif_path = process_path + '/dem.tif'
print("Get DEM")
tiled_raster_layer = gps.query(
uri="file:///usr/local/large_scale_hydro/catalog",
layer_name="dem",
layer_zoom=0,
query_geom=polys)
print(tiled_raster_layer.count())
print(tiled_raster_layer.layer_metadata.extent)
tiled_raster_layer.save_stitched(dem_tif_path)
dir_tif_path = process_path + '/dir.tif'
print("Get Direction")
tiled_raster_layer = gps.query(
uri="file:///usr/local/large_scale_hydro/catalog",
layer_name="direction",
layer_zoom=0,
query_geom=poly)
print(tiled_raster_layer.count())
print(tiled_raster_layer.layer_metadata.extent)
tiled_raster_layer.save_stitched(dir_tif_path)
acc_tif_path = process_path + '/acc.tif'
print("Get Accumulation")
tiled_raster_layer = gps.query(
uri="file:///usr/local/large_scale_hydro/catalog",
layer_name="accumulation",
layer_zoom=0,
query_geom=poly)
print(tiled_raster_layer.count())
print(tiled_raster_layer.layer_metadata.extent)
tiled_raster_layer.save_stitched(acc_tif_path)
lake_tif_path = process_path + '/lakes.tif'
print("Get Lakes")
tiled_raster_layer = gps.query(
uri="file:///usr/local/large_scale_hydro/catalog",
layer_name="lakes",
layer_zoom=0,
query_geom=poly)
print(tiled_raster_layer.count())
print(tiled_raster_layer.layer_metadata.extent)
tiled_raster_layer.save_stitched(lake_tif_path)
from shapely.geometry import MultiPolygon, box
conf = gps.geopyspark_conf(master="local[*]", appName="layers")
pysc = SparkContext(conf=conf)
uri = "file:/data/workspace/geotrellis-landsat-tutorial/data/1k_tiles/"
layer_name = "landsat1K"
metadata = gps.read_layer_metadata(uri=uri,
layer_name=layer_name,
layer_zoom=0)
print(metadata)
# Get list of tiles
print(metadata.bounds)
# Read the first tile
tile = gps.read_value(uri=uri,
layer_name=layer_name,
layer_zoom=0,
col=metadata.bounds.minKey.col,
row=metadata.bounds.minKey.row)
print(tile)
# Read the layer
layer = gps.query(uri=uri, layer_name=layer_name, layer_zoom=0)
from pyrasterframes import *
from pyspark.sql import *
import geopyspark
import numpy
from pyspark.sql import SparkSession
from pyrasterframes import *
from pyrasterframes.rasterfunctions import *
conf = geopyspark.geopyspark_conf(appName="POC")
session = SparkSession.builder.config(
conf=conf).getOrCreate().withRasterFrames()
uri = "file:/data/workspace/geotrellis-landsat-tutorial/data/1k_tiles/"
layer_name = "landsat1K"
layer = geopyspark.query(uri=uri, layer_name=layer_name, layer_zoom=0)
rf = layer.to_rasterframe(3)
rf.show(2)
# Show CRS
rf.tileLayerMetadata()['crs']
# Convert Tile data to array
rf.select(tileToDoubleArray("tile_1")).show(10, 80)
# Global aggregation statistics
rf.agg(aggNoDataCells("tile_1"), aggDataCells("tile_1"),
aggMean("tile_1")).show(5, False)
# Tile aggregation statistics
rf.select(tileMean("tile_1"), tileMin("tile_1"), tileMax("tile_1")).show(5)