def generate_sample_grid(self):
print('Generating samples...')
# Compute sample size seed
num_samples = self.target_sample_size / (self.x_cuts * self.y_cuts * 8)
# Project the geometry into the target projection
bounding_box = self.study_area.bounds(ee.ErrorMargin(1, 'projected'), self.projection) \
.transform(self.projection) \
.bounds(ee.ErrorMargin(1, 'projected'), self.projection)
# Get the bounding box partitions
partitions = self.__partition_bounding_box(bounding_box)
# Get sample centroids
sample_points = self.__get_sample_points(partitions, num_samples,
self.projection)
# Compute the foot-prints for visualization purposes
sample_tiles = self.__get_sample_tiles(sample_points)
# Export all of the various ee.FeatureCollesctions to the input asset directory
self.__export_compute_feature_collections(partitions, sample_points,
sample_tiles)
print('Output number of Samples:', sample_points.size().getInfo())
return None
def create_dataset_image(study_area, projection, x_cuts, y_cuts):
# Project the geometry into the target projection
bounding_box = study_area.bounds(ee.ErrorMargin(0.0001, 'projected'), projection) \
.transform(projection) \
.bounds(ee.ErrorMargin(1, 'projected'), projection)
# Get the bounding box partitions
partitions = partition_bounding_box(bounding_box, x_cuts, y_cuts,
projection)
# Randomly assign each partition to a set
partitions = partitions.randomColumn()
train = partitions.filterMetadata('random', 'less_than', 0.70)
validation = partitions.filterMetadata('random', 'greater_than', 0.70) \
.filterMetadata('random', 'less_than', 0.90)
test = partitions.filterMetadata('random', 'greater_than', 0.90)
# Adds the a property called "dataset_subset" to each feature
train = add_model_set_property(train, 1)
validation = add_model_set_property(validation, 2)
test = add_model_set_property(test, 3)
# Recombine the datasets
combined = train.merge(validation).merge(test)
# Create the binary image to sample
dataset_image = ee.Image.constant(0).toByte().paint(combined, 'dataset_subset') \
.rename('dataset_subset')
return dataset_image
def calc_validation_score(
mask,
val_poly_ir_crops,
val_poly_ir_trees,
):
"""Calculates the validation score for a mask layer using validation polygons that were uploaded as assets to GEE
beforehand"""
def validate_irrigated_area(feature):
total_pixels = mask.reduceRegion(reducer=ee.Reducer.count(),
geometry=feature.geometry(),
scale=30)
irrigated_pixels = mask.reduceRegion(reducer=ee.Reducer.sum(),
geometry=feature.geometry(),
scale=30)
score = ee.Number(irrigated_pixels.get('b1')).round().divide(
ee.Number(total_pixels.get('b1')))
score = ee.Algorithms.If(score.gt(1), 1, score)
feature = feature.set({
'total_pixels': total_pixels.get('b1'),
'irrigated_pixels': irrigated_pixels.get('b1'),
'score': score
})
return feature
validation_areas_ir_crops = ee.FeatureCollection(
val_poly_ir_crops.geometry().intersection(CdC.geometry(),
ee.ErrorMargin(1)))
validation_areas_ir_trees = ee.FeatureCollection(
val_poly_ir_trees.geometry().intersection(CdC.geometry(),
ee.ErrorMargin(1)))
validation_areas_ir_crops = validation_areas_ir_crops.map(
convert_to_polygons).flatten()
validation_areas_ir_trees = validation_areas_ir_trees.map(
convert_to_polygons).flatten()
validation_areas_ir_crops = validation_areas_ir_crops.map(
validate_irrigated_area)
validation_areas_ir_trees = validation_areas_ir_trees.map(
validate_irrigated_area)
final_validation_score_ir_crops = validation_areas_ir_crops.reduceColumns(
selectors=ee.List(['score']), reducer=ee.Reducer.mean())
final_validation_score_ir_trees = validation_areas_ir_trees.reduceColumns(
selectors=ee.List(['score']), reducer=ee.Reducer.mean())
return final_validation_score_ir_crops.getInfo(
)["mean"], final_validation_score_ir_trees.getInfo()["mean"]
def generate_voronoi_polygons(points, scale, aoi):
"""
Generates Voronoi polygons
:param points:
:param scale:
:param aoi:
:return:
"""
error = ee.ErrorMargin(1, 'projected')
# proj = ee.Projection('EPSG:3857').atScale(scale)
proj = ee.Projection('EPSG:4326').atScale(scale)
distance = ee.Image(0).float().paint(points, 1) \
.fastDistanceTransform().sqrt().clip(aoi) \
.reproject(proj)
concavity = distance.convolve(ee.Kernel.laplacian8()) \
.reproject(proj)
concavity = concavity.multiply(distance)
concavityTh = 0
edges = concavity.lt(concavityTh)
# label connected components
connected = edges.Not() \
.connectedComponents(ee.Kernel.circle(1), 256) \
.clip(aoi) \
.focal_max(scale * 3, 'circle', 'meters') \
.focal_min(scale * 3, 'circle', 'meters') \
.focal_mode(scale * 5, 'circle', 'meters') \
.reproject(proj)
# fixing reduceToVectors() bug, remap to smaller int
def fixOverflowError(i):
hist = i.reduceRegion(ee.Reducer.frequencyHistogram(), aoi, scale)
uniqueLabels = ee.Dictionary(ee.Dictionary(hist).get('labels')).keys() \
.map(lambda o: ee.Number.parse(o))
labels = ee.List.sequence(0, uniqueLabels.size().subtract(1))
return i.remap(uniqueLabels, labels).rename('labels').int()
connected = fixOverflowError(connected).reproject(proj)
polygons = connected.select('labels').reduceToVectors(
**{
"scale": scale,
"crs": proj,
"geometry": aoi,
"eightConnected": True,
"labelProperty": 'labels',
"tileScale": 4
})
# polygons = polygons.map(lambda o: o.snap(error, proj))
return {"polygons": polygons, "distance": distance}
def testDynamicConstructorCasting(self):
"""Test the behavior of casting with dynamic classes."""
self.InitializeApi()
result = ee.Geometry.Rectangle(1, 1, 2, 2).bounds(0, 'EPSG:4326')
expected = (ee.Geometry.Polygon([[1, 2], [1, 1], [2, 1], [2, 2]])
.bounds(ee.ErrorMargin(0), ee.Projection('EPSG:4326')))
self.assertEqual(expected, result)
def getChips(request):
pt = ee.Geometry.Point(float(request.get('lon')), float(request.get('lat')))
cloud_pcent = int(request.get('cloudpcent'))
deltad = int(request.get('deltad'))
end_date = ee.Date(request.get('end_period'))
start_date = end_date.advance(-deltad, 'day')
s2 = ee.ImageCollection('COPERNICUS/S2').filterBounds(pt).\
filterMetadata('CLOUDY_PIXEL_PERCENTAGE', 'less_than', cloud_pcent).\
filterDate(start_date, end_date).sort('system:time_start')
chips_date = ee.List(s2.aggregate_array('system:time_start')).map(lambda t: ee.Date(t).format("YYYY-MM-dd"))
projection = ee.Image(s2.first()).select('B2').projection()
region = pt.transform(projection,1.0).buffer(640.0, ee.ErrorMargin(1.0)).bounds(1.0, projection)
s2 = s2.map(lambda img: ee.Image(img).reproject(projection).clip(region).visualize(bands=['B8', 'B11', 'B4'], max=[6000, 6000, 4000]))
#print s2.size().getInfo()
url = ee.data.makeThumbUrl(ee.data.getThumbId({'image':s2.serialize(), 'dimensions': "128x128", 'format': 'png'}))
#print chips_date.getInfo()
chips= {'chips_dates': chips_date.getInfo(), 'thurl': url}
return json.dumps(chips)
def partitions_pairs_to_partitions(pair):
# Get the parameters from the input array
x_coord = ee.Number(ee.List(pair).get(0)).toInt16()
y_coord = ee.Number(ee.List(pair).get(1)).toInt16()
id_str = ee.Number(ee.List(pair).get(2)).toInt16()
# Retrieve the 4 verticies needed
llh = ee.Feature(grid_vertices.filterMetadata('grid_x', 'equals', x_coord) \
.filterMetadata('grid_y', 'equals', y_coord).first()).geometry()
lrh = ee.Feature(grid_vertices.filterMetadata('grid_x', 'equals', x_coord.add(1)) \
.filterMetadata('grid_y', 'equals', y_coord).first()).geometry()
urh = ee.Feature(grid_vertices.filterMetadata('grid_x', 'equals', x_coord.add(1)) \
.filterMetadata('grid_y', 'equals', y_coord.add(1)).first()).geometry()
ulh = ee.Feature(grid_vertices.filterMetadata('grid_x', 'equals', x_coord) \
.filterMetadata('grid_y', 'equals', y_coord.add(1)).first()).geometry()
# Convert into a geometry
partition_geo = ee.Geometry.Polygon([[llh.coordinates(), lrh.coordinates(), urh.coordinates(), ulh.coordinates()]], self.projection, False) \
.buffer(-1 * erosion_dist/2, ee.ErrorMargin(1, 'projected'), self.projection)
# Append the feature to the output
partition = ee.Feature(
partition_geo, {
'partition_id': id_str,
'partition_x': x_coord,
'partition_y': y_coord
})
return partition
def _create_grid(self, ee_feature, grid_size_degrees):
"""
Breaks down a feature into an N x N grid and returns a new list of ee.Features.
:param ee_feature: ee.Feature
:param grid_size_degrees: Grid size in arc degrees
:return: list of ee.Features
"""
# Arc grid JS equivalent here https://code.earthengine.google.com/bdb4f409515d1fda0592a8330a0f6528
# Get bounds of grid
bounds = ee_feature.bounds().geometry().bounds().getInfo()
x_coords = [b[0] for b in bounds["coordinates"][0]]
y_coords = [b[1] for b in bounds["coordinates"][0]]
lon_start = min(x_coords)
lon_end = max(x_coords)
lat_start = min(y_coords)
lat_end = max(y_coords)
lon_width = lon_end - lon_start
lat_width = lat_end - lat_start
# test grid size against bounding box
if grid_size_degrees > lon_width and grid_size_degrees > lat_width:
logger.info("Grid larger than feature. Skipping.")
return [ee_feature]
elif grid_size_degrees > lon_width / 2 and grid_size_degrees > lat_width / 2:
logger.warning(
"Expecting less than 4 grids. Consider using a smaller grid_size_degrees or larger area_threshold."
)
# Generate grid over ee_feature
polys = []
lon = lon_start
while lon < lon_end:
x1 = lon
x2 = lon + grid_size_degrees
lon += grid_size_degrees
lat = lat_start
while lat < lat_end:
y1 = lat
y2 = lat + grid_size_degrees
lat += grid_size_degrees
polys.append(
ee.Feature(ee.Geometry.Rectangle(x1, y1, x2, y2), {}))
# Intersects grid against ee_feature
intersected_feats = []
for p in polys:
intersection = p.intersection(ee_feature, ee.ErrorMargin(1))
intersected_feats.append(
ee.Feature(intersection).set(
{"area": intersection.area().divide(1000 * 1000).floor()}))
return intersected_feats
def calcCloudStats(img):
imgPoly = ee.Algorithms.GeometryConstructors.Polygon(
ee.Geometry(img.get('system:footprint')).coordinates())
roi = ee.Geometry(img.get('ROI'))
intersection = roi.intersection(imgPoly, ee.ErrorMargin(0.5))
cloudMask = img.select(['cloudScore'
]).gt(cloudThresh).clip(roi).rename('cloudMask')
cloudAreaImg = cloudMask.multiply(ee.Image.pixelArea())
stats = cloudAreaImg.reduceRegion(reducer=ee.Reducer.sum(),
geometry=roi,
scale=10,
maxPixels=1e12)
cloudPercent = ee.Number(stats.get('cloudMask')).divide(
imgPoly.area(0.001)).multiply(100)
coveragePercent = ee.Number(intersection.area()).divide(
roi.area(0.001)).multiply(100)
cloudPercentROI = ee.Number(stats.get('cloudMask')).divide(
roi.area(0.001)).multiply(100)
img = img.set('CLOUDY_PERCENTAGE', cloudPercent)
img = img.set('ROI_COVERAGE_PERCENT', coveragePercent)
img = img.set('CLOUDY_PERCENTAGE_ROI', cloudPercentROI)
return img
def calcCloudStats(img):
imgPoly = ee.Algorithms.GeometryConstructors.Polygon(
ee.Geometry(img.get("system:footprint")).coordinates())
roi = ee.Geometry(img.get("ROI"))
intersection = roi.intersection(imgPoly, ee.ErrorMargin(0.5))
cloudMask = img.select(["cloudScore"
]).gt(cloudThresh).clip(roi).rename("cloudMask")
cloudAreaImg = cloudMask.multiply(ee.Image.pixelArea())
stats = cloudAreaImg.reduceRegion(
**{
"reducer": ee.Reducer.sum(),
"geometry": roi,
"scale": 10,
"maxPixels": 1e12
})
cloudPercent = ee.Number(stats.get("cloudMask")).divide(
imgPoly.area()).multiply(100)
coveragePercent = ee.Number(intersection.area()).divide(
roi.area()).multiply(100)
cloudPercentROI = ee.Number(stats.get("cloudMask")).divide(
roi.area()).multiply(100)
img = img.set("CLOUDY_PERCENTAGE", cloudPercent)
img = img.set("ROI_COVERAGE_PERCENT", coveragePercent)
img = img.set("CLOUDY_PERCENTAGE_ROI", cloudPercentROI)
return img
def add_coverage(image):
footprint = ee.Feature(image.get('footprint'))
image_footprint = footprint.geometry()
aoi_area = aoi_footprint.area()
intersect = aoi_footprint.intersection(image_footprint, ee.ErrorMargin(1))
intersect_area = intersect.area()
coverage = ee.Number(intersect_area.divide(aoi_area))
return image.set('AOI_COVERAGE', coverage)
def generate_sample_grid(self):
print('Generating samples...')
# Project the geometry into the target projection
bounding_box = self.study_area.bounds(ee.ErrorMargin(1, 'projected'), self.projection) \
.transform(self.projection) \
.bounds(ee.ErrorMargin(1, 'projected'), self.projection)
# Get the bounding box partitions
partitions = self.__partition_bounding_box(bounding_box)
# Export all of the various ee.FeatureCollesctions to the input asset directory
self.__export_compute_feature_collections(partitions, sample_points,
sample_tiles)
print('Output number of Samples:', sample_points.size().getInfo())
return None
def generate_perimeter_points(geom, step):
"""
Generates points along interiors and exteriors
:param geom:
:param step:
:return:
"""
error = ee.ErrorMargin(1, 'meters')
p = geom.perimeter(error)
n = p.divide(step).int()
step = p.divide(n)
# map over exterior and interiors
def wrap_ring(coords):
ring = ee.Geometry.LineString(coords)
distances = ee.List.sequence(0, ring.length(error), step)
return ee.Feature(ring) \
.set({"distances": distances}) \
.set({"distancesCount": distances.length()})
rings = geom.coordinates().map(wrap_ring)
rings = ee.FeatureCollection(rings)
def generate_points(ring):
distances = ring.get('distances')
segments = ring.geometry().cutLines(distances).geometries()
segment_points = \
segments.map(lambda g: ee.Feature(ee.Geometry(g).centroid(1)))
return ee.FeatureCollection(segment_points)
points = rings \
.filter(ee.Filter.gt('distancesCount', 2)) \
.map(generate_points) \
.flatten()
return ee.FeatureCollection(points)
def calcCloudCoverage(img, cloudThresh=0.2):
imgPoly = ee.Algorithms.GeometryConstructors.Polygon(
ee.Geometry(img.get('system:footprint')).coordinates())
roi = ee.Geometry(img.get('ROI'))
#line below to used debug issue with export tile pipeline
# roi = img.geometry()
intersection = roi.intersection(imgPoly, ee.ErrorMargin(0.5))
cloudMask = img.select(['cloudScore'
]).gt(cloudThresh).clip(roi).rename('cloudMask')
cloudAreaImg = cloudMask.multiply(ee.Image.pixelArea())
stats = cloudAreaImg.reduceRegion(
reducer=ee.Reducer.sum(),
geometry=roi,
scale=10,
maxPixels=1e12,
## bottom two not in the javascript version
bestEffort=True,
tileScale=16)
## maxAreaError not in the javascript version, which uses the default
## for the .area function calls
maxAreaError = 10
cloudPercent = ee.Number(stats.get('cloudMask')).divide(
imgPoly.area(maxAreaError)).multiply(100)
coveragePercent = ee.Number(intersection.area(maxAreaError)).divide(
roi.area(maxAreaError)).multiply(100)
cloudPercentROI = ee.Number(stats.get('cloudMask')).divide(
roi.area(maxAreaError)).multiply(100)
img = img.set('CLOUDY_PERCENTAGE', cloudPercent)
img = img.set('ROI_COVERAGE_PERCENT', coveragePercent)
img = img.set('CLOUDY_PERCENTAGE_ROI', cloudPercentROI)
print("calculated cloud coverage values")
return img
def addStats(img: ee.Image) -> ee.Image:
patch = ee.Geometry(img.get('patch'))
patch_area = patch.area(0.001)
img_footprint = ee.Algorithms.GeometryConstructors.Polygon(
ee.Geometry(img.get('system:footprint')).coordinates())
intersection = patch.intersection(img_footprint, ee.ErrorMargin(0.001))
intersection_area = intersection.area(0.001)
coverage = ee.Number(intersection_area).divide(patch_area)
cloud_area_img = img.select('clouds').multiply(ee.Image.pixelArea())
stats = cloud_area_img.reduceRegion(reducer=ee.Reducer.sum(),
geometry=patch,
scale=10,
maxPixels=1e12)
cloud_area = stats.get('clouds')
cloud_coverage = ee.Number(cloud_area).divide(patch_area)
img = img.set('patchCoverage', coverage)
img = img.set('patchCloudCoverage', cloud_coverage)
score = coverage.multiply(cloud_coverage)
img = img.set('patchScore', score)
return img
def main():
# Define the username
username = "******"
# Define the output location
output_dir = "SERVIR/real_time_monitoring"
# Define kernel size
kernel_size = 64
image_kernel = get_kernel(kernel_size)
# Cloud Storage Parameters
cloud_bucket = "kilbride_bucket_1"
cloud_folder = "glad_alert_records/"
# Get the projection that is needed for the study area
projection = ee.Projection('EPSG:32648')
# Load in the GLAD Alert Images
glad_alerts = ee.Image(
'users/JohnBKilbride/SERVIR/real_time_monitoring/glad_alerts_2019_to_2020'
)
# Load in the sample locations
sample_locations = ee.FeatureCollection("users/JohnBKilbride/SERVIR/real_time_monitoring/sample_locations_2019_2020_train_val_test_50k") \
.randomColumn(None, 43214).sort('random').limit(5)
#################################### Ignore stuff below #######################
# Load in the topographic metrics
topography = load_topographic()
# Loop through the points
num_samples = sample_locations.size().getInfo()
sample_list = sample_locations.toList(num_samples)
print(
'\nInitiating {num_exports} exports.'.format(num_exports=num_samples))
for i in range(0, num_samples):
print('\nExporting record {i}'.format(i=i))
# Get the next feature by index and cast
current_location = ee.Feature(sample_list.get(i))
# Get the geometry from the current feature
location_geo = current_location.geometry()
# Get the dataset type
dataset_subset = ee.Number(
current_location.get('dataset_subset')).toInt8().getInfo()
if dataset_subset == 1:
dataset_name = 'train'
elif dataset_subset == 2:
dataset_name = 'validation'
elif dataset_subset == 3:
dataset_name = 'test'
# Get the day and year of each of the dates
before_day = ee.Number(current_location.get('start_day'))
before_year = ee.Number(current_location.get('start_year'))
after_day = ee.Number(current_location.get('alert_day'))
after_year = ee.Number(current_location.get('alert_year'))
# Get the Before and After Images
sentinel = generate_before_after_image(before_day, after_day,
before_year, after_year,
location_geo)
# Get the Labels
glad_label = calculate_glad_label(glad_alerts, before_day, after_day,
before_year, after_year)
# Combine the labels
all_bands = ee.Image.cat(sentinel, topography, glad_label).toFloat()
# # Sample with point and neighborhoodToArray()
# arrays = all_bands.neighborhoodToArray(image_kernel)
# output = arrays.sampleRegions(
# collection = ee.FeatureCollection([current_location]),
# properties = ["VV_before","VH_before","VV_after","VH_after","glad_alert"],
# scale = 10,
# projection = projection,
# geometries = False
# )
# # Export the example to Google Cloud Storage
# task_1 = ee.batch.Export.table.toCloudStorage(
# collection = output,
# description = 'Export-GLAD-' + str(i),
# bucket = cloud_bucket,
# fileNamePrefix = cloud_folder + '/glad_alert_' + str(i),
# fileFormat = 'TFRecord',
# selectors = ["VV_before","VH_before","VV_after","VH_after","glad_alert"]
# )
# task_1.start()
# For debugging
# if i < 10:
export_geometry = location_geo.buffer(ee.Number(kernel_size/2).multiply(10), ee.ErrorMargin(0.0001, "projected"), projection) \
.bounds(ee.ErrorMargin(0.0001, "projected"), projection)
task_2 = ee.batch.Export.image.toDrive(
image=all_bands,
description='GLAD-Image-Export-' + str(i),
folder='alert_' + str(kernel_size) + '_' + str(kernel_size) +
'_before_after_delta_topo',
fileNamePrefix='glad_' + dataset_name + '_' + str(i),
region=export_geometry,
scale=10,
crs=projection,
maxPixels=1e6)
task_2.start()
# Check for completed exports every 250 iterations
if i % 250 == 0:
check_for_monitor_capacity()
EXPORT_MONITOR.add_task('export_' + str(i), task_2)
# Run the monitoring of the exports
EXPORT_MONITOR.monitor_tasks()
EXPORT_MONITOR.reset_monitor()
print('...export completed.')
return None
def get_water_network_properties():
"""
Generates variables along water skeleton network polylines.
"""
j = request.json
region = ee.Geometry(j['region'])
start = j['start']
stop = j['stop']
scale = j['scale']
step = j['step']
crs = j['crs']
error = ee.ErrorMargin(scale / 2, 'meters')
if 'network' in j:
raise Exception(
'TODO: re-using existing networks is not supported yet')
# get water mask
water_vector = get_water_mask_vector(region, scale, start, stop)
# skeletonize
output = river_functions.generate_skeleton_from_voronoi(
scale, water_vector)
centerline = ee.FeatureCollection(output["centerline"])
distance = ee.Image(output["distance"])
# generate width at every offset
centerline = centerline.map(
lambda line: line.set("length", line.length(error)))
short_lines = centerline.filter(ee.Filter.lte('length', step))
long_lines = centerline.filter(ee.Filter.gt('length', step))
def process_long_line(line):
line_length = line.length(error)
distances = ee.List.sequence(0, line_length, step)
segments = line.geometry().cutLines(distances, error)
def generate_line_middle_point(pair):
pair = ee.List(pair)
s = ee.Geometry(pair.get(0))
offset = ee.Number(pair.get(1))
centroid = ee.Geometry.Point(s.coordinates().get(0))
return ee.Feature(centroid) \
.set("lineId", line.id()) \
.set("offset", offset)
segments = segments.geometries().zip(distances) \
.map(generate_line_middle_point)
return ee.FeatureCollection(segments)
long_line_points = long_lines.map(process_long_line).flatten()
def process_short_line(line):
geom = line.geometry(error, 'EPSG:4326')
geom = ee.Geometry.Point(geom.coordinates().get(0), 'EPSG:4326')
return ee.Feature(geom) \
.set("lineId", line.id()) \
.set("offset", 0)
short_line_points = short_lines.map(process_short_line)
points = long_line_points.merge(short_line_points)
fa = ee.Image('WWF/HydroSHEDS/15ACC')
dem = ee.Image('JAXA/ALOS/AW3D30_V1_1').select('MED')
def add_flow_accumulation(pt):
flow_accumulation = fa.reduceRegion(reducer=ee.Reducer.max(),
geometry=pt.geometry().buffer(
scale * 10),
scale=scale).values().get(0)
return pt \
.set("flow_accumulation", ee.Number(flow_accumulation))
def add_width(pt):
width = distance.reduceRegion(reducer=ee.Reducer.max(),
geometry=pt.geometry(),
scale=scale).values().get(0)
return pt \
.set("width", ee.Number(width).multiply(scale).multiply(2))
def add_elevation(pt):
elevation = dem.reduceRegion(reducer=ee.Reducer.median(),
geometry=pt.geometry().buffer(scale * 10),
scale=scale).values().get(0)
return pt \
.set("elevation", ee.Number(elevation))
points = points.map(add_width)
points = points.map(add_elevation)
points = points.map(add_flow_accumulation)
if crs and crs != 'EPSG:4326':
points = points.map(transform_feature(crs, scale))
# create response
data = points.getInfo()
return Response(json.dumps(data), status=200, mimetype='application/json')
continue
saveFolder_ = saveFolder + "/" + feature['properties']['title']
if not os.path.exists(saveFolder_):
os.mkdir(saveFolder_)
else:
continue
cor = feature['geometry']['coordinates']
aoi = ee.Geometry.Polygon([
cor[0][0][0:2], cor[0][1][0:2], cor[0][2][0:2], cor[0][3][0:2],
cor[0][4][0:2]
], None, False)
#buffer aoi
aoi = aoi.buffer(100).bounds().simplify(ee.ErrorMargin(1, "meters"))
epsg = get_utm_epsg(cor[0][0][0], cor[0][0][1])
for year in [2018, 2019]:
for m in range(1, 12):
start_date = ee.Date.fromYMD(year, m, 1)
end_date = ee.Date.fromYMD(year, m + 1, 1)
s2_all = ee.ImageCollection('COPERNICUS/S2') \
.filterDate(start_date, end_date).filterBounds(aoi)
print("the available s2 data for this roi and time: ",
s2_all.size().getInfo())
def inner_map(feat):
return ee.Feature(feat).buffer(erosion_distance/2, ee.ErrorMargin(1,'projected'), self.projection) \
.bounds(ee.ErrorMargin(1, 'projected'), self.projection)
def generate_skeleton_from_voronoi(scale, water_vector):
# step between points along perimeter
step = scale * 10
simplify_centerline_factor = 15
error = ee.ErrorMargin(1, 'meters')
# proj = ee.Projection('EPSG:3857').atScale(scale)
proj = ee.Projection('EPSG:4326').atScale(scale)
# turn water mask into a skeleton
def add_coords_count(o):
return ee.Feature(None, {"count": ee.List(o).length(), "values": o})
c = water_vector.geometry().coordinates()
exterior = c.get(0)
interior = c.slice(1).map(add_coords_count)
interior = ee.FeatureCollection(interior)
interior = interior.filter(ee.Filter.gt('count', 5))
interior = interior.toList(10000).map(
lambda o: ee.Feature(o).get('values'))
water_vector = ee.Feature(
ee.Geometry.Polygon(ee.List([exterior]).cat(interior)))
geometry = water_vector.geometry()
geometry_buffer = geometry.buffer(scale * 4, error)
perimeter_geometry = geometry_buffer \
.difference(geometry_buffer.buffer(-scale * 2, error), error)
geometry = geometry_buffer
points = generate_perimeter_points(geometry, step)
output = generate_voronoi_polygons(points, scale, geometry)
polygons = output["polygons"]
distance = output["distance"]
dist_filter = ee.Filter.And(
ee.Filter.intersects(
**{"leftField": ".geo", "rightField": ".geo", "maxError": error}),
ee.Filter.equals(
**{"leftField": "labels", "rightField": "labels"}).Not()
)
dist_save_all = ee.Join.saveAll(**{"matchesKey": 'matches'})
features = dist_save_all.apply(polygons, polygons, dist_filter)
# find intersection with neighbouring polygons
def find_neighbours(ff1):
matches = ee.FeatureCollection(ee.List(ff1.get('matches')))
def find_neighbours2(ff2):
i = ff2.intersection(ff1, error, proj)
t = i.intersects(perimeter_geometry, error, proj)
m = i.intersects(geometry, error, proj)
return i.set({"touchesPerimeter": t}).set(
{"intersectsWithMask": m})
return matches.map(find_neighbours2)
features = features.map(find_neighbours).flatten()
# find a centerline
f = ee.Filter.And(ee.Filter.eq('touchesPerimeter', False),
ee.Filter.eq('intersectsWithMask', True))
centerline = features.filter(f)
centerline = centerline.geometry().dissolve(scale, proj) \
.simplify(scale * simplify_centerline_factor, proj)
centerline = centerline.geometries().map(
lambda g: ee.Feature(ee.Geometry(g)))
centerline = ee.FeatureCollection(centerline)
centerline = centerline \
.map(lambda o: o.set({"type": o.geometry().type()})) \
.filter(ee.Filter.eq('type', 'LineString')) \
# .map(lambda o: o.transform(ee.Projection('EPSG:4326').atScale(scale)), error)
return {"centerline": centerline, "distance": distance}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.error_margin = ee.ErrorMargin(1)
self.area_proj = "EPSG:5070"
import geemap
# Create a map centered at (lat, lon).
Map = geemap.Map(center=[40, -100], zoom=4)
# Create two circular geometries.
poly1 = ee.Geometry.Point([-50, 30]).buffer(1e6)
poly2 = ee.Geometry.Point([-40, 30]).buffer(1e6)
# Display polygon 1 in red and polygon 2 in blue.
Map.setCenter(-45, 30)
Map.addLayer(poly1, {'color': 'FF0000'}, 'poly1')
Map.addLayer(poly2, {'color': '0000FF'}, 'poly2')
# Compute the intersection, display it in blue.
intersection = poly1.intersection(poly2, ee.ErrorMargin(1))
Map.addLayer(intersection, {'color': '00FF00'}, 'intersection')
# Compute the union, display it in magenta.
union = poly1.union(poly2, ee.ErrorMargin(1))
Map.addLayer(union, {'color': 'FF00FF'}, 'union')
# Compute the difference, display in yellow.
diff1 = poly1.difference(poly2, ee.ErrorMargin(1))
Map.addLayer(diff1, {'color': 'FFFF00'}, 'diff1')
# Compute symmetric difference, display in black.
symDiff = poly1.symmetricDifference(poly2, ee.ErrorMargin(1))
Map.addLayer(symDiff, {'color': '000000'}, 'symmetric difference')
def f_clip_ranges(feature):
return feature.intersection(aoi, ee.ErrorMargin(1))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.error_margin = ee.ErrorMargin(1)
self.countries = ee.FeatureCollection(self.inputs["countries"]["ee_path"])
self.ecoregions = ee.FeatureCollection(self.inputs["ecoregions"]["ee_path"])
self.pas = ee.FeatureCollection(self.inputs["pas"]["ee_path"])
def __export_dataset_sample (self, sample, sample_num, num_exports):
"""
Function exports an individual sample to google drive as a TFRecord.
These can be combined as a TensorFlow Dataset.
Parameters
----------
sample : ee.Feature
A feature containing the following propertioes:
partition_id: a numerical ID indicating the sample's partition
id_list: get
sample_num : integer
A number which is appended to the file path to identify a sample uniquely.
num_exports: integer
The total number of exports. This is used when printing info about the export status
Returns
-------
None.
"""
# Cast the sample
sample = ee.Feature(sample)
# Get the partition coordinates from the sample
# partition_id = str(ee.Number(sample.get('partition_id')).toInt16().getInfo())
# Get all of the info needed for export
sample_ids = ee.List(sample.get("id_list")).getInfo()
model_set = ee.String(sample.get("model_set")).getInfo()
if model_set == 'test':
pass
else:
print(model_set)
# Get rid of the leading characters introduced by previous processing steps
sample_ids_trimmed = []
for sentinel_id in sample_ids:
sample_ids_trimmed.append(sentinel_id)
# Convert the IDs to images
scenes = []
alert_date = None
for i in range(0, len(sample_ids_trimmed)):
# Get the id from the list of ids
scene = ee.Image("COPERNICUS/S1_GRD/"+sample_ids_trimmed[i])
# Append the scene to the list
scenes.append(scene)
# Get the alert date
if i == 0:
alert_date = scene.date()
# Convert the list of images to an ee.ImageCollection and select the correct bands
scenes = ee.ImageCollection.fromImages(scenes) \
.select(self.output_bands) \
.sort('system:time_start')
# Generate the features
features = scenes.toBands().rename(self.model_feature_names) \
.unmask(-50) .unitScale(-50, 1)
# Load the GLAD Alert for the scene
print(alert_date.get('year').getInfo(), alert_date.get('month').getInfo(), alert_date.get('day').getInfo())
label = self.__retrieve_label(alert_date)
# Stack the outputs
labels_and_features = ee.Image.cat([features, label]).toFloat()
# Run the sampling
output = self.__sample_model_data(labels_and_features, sample.geometry())
# Create the export filename
file_name = 'alert_record' + '_' + str(sample_num+1)
# Export an image
clip_geometry = sample.geometry().buffer(self.kernel_size / 2, ee.ErrorMargin(1, 'projected'), self.projection.atScale(10)) \
.bounds(ee.ErrorMargin(1, 'projected'), self.projection.atScale(10))
image_task = ee.batch.Export.image.toDrive(
image = labels_and_features.clip(clip_geometry).toFloat(),
description = 'Export-GLAD-Label-' + str(sample_num+1),
folder = 'GLAD_TEST',
fileNamePrefix = file_name,
scale = 10,
maxPixels = 1e13
)
image_task.start()
# Initiate the export
task = ee.batch.Export.table.toCloudStorage(
collection = ee.FeatureCollection([output]),
description = "Export-Mekong-Alert-" + str(sample_num),
bucket = self.gcs_bucket,
fileNamePrefix = self.gcs_export_folder + '/' + model_set + '/' + file_name,
fileFormat = "TFRecord",
selectors = labels_and_features.bandNames().getInfo()
)
# Check if the number of output features is correct
# Number should be: self.model_feature_names + 2
target_num_properties = len(self.model_feature_names) + 2
output_num_properties = output.propertyNames().length().getInfo()
if target_num_properties == output_num_properties:
print('Initating '+str(sample_num+1)+' of '+str(num_exports))
# task.start()
else:
print('Export for index ' + str(sample_num+1) + ' had too few features.')
# Log the info in the exporter
self.export_monitor.add_task('export_'+str(sample_num), task)
return None
