# locations = pd.read_csv('locations_remedy.csv')
locations = pd.read_csv('world_locations.csv', header=None)
# county_region = ee.FeatureCollection('ft:18Ayj5e7JxxtTPm1BdMnnzWbZMrxMB49eqGDTsaSp')
world_region = ee.FeatureCollection(
'ft:1tdSwUL7MVpOauSgRzqVTOwdfy17KDbw-1d9omPw')
imgcoll = ee.ImageCollection('MODIS/MOD09A1') \
.filterBounds(ee.Geometry.Rectangle(-106.5, 50,-64, 23))\
.filterDate('2001-12-31','2015-12-31')
img = imgcoll.iterate(appendBand_6)
img = ee.Image(img)
img_0 = ee.Image(ee.Number(0))
img_5000 = ee.Image(ee.Number(5000))
img = img.min(img_5000)
img = img.max(img_0)
# img=ee.Image(ee.Number(100))
# img=ee.ImageCollection('LC8_L1T').mosaic()
for country, index in locations.values:
fname = 'index' + '{}'.format(int(index))
# offset = 0.11
scale = 500
crs = 'EPSG:4326'
def ozone(geom, date):
"""
returns ozone measurement from merged TOMS/OMI dataset
OR
uses our fill value (which is mean value for that latlon and day-of-year)
"""
# Point geometry required
centroid = geom.centroid()
def ozone_measurement(centroid, O3_date):
# filtered ozone collection
ozone_ic = ee.ImageCollection('TOMS/MERGED').filterDate(
O3_date, O3_date.advance(1, 'month'))
# ozone image
ozone_img = ee.Image(ozone_ic.first())
# ozone value IF TOMS/OMI image exists ELSE use fill value
ozone = ee.Algorithms.If(ozone_img,\
ozone_img.reduceRegion(reducer=ee.Reducer.mean(), geometry=centroid).get('ozone'),\
ozone_fill(centroid,O3_date))
return ozone
def ozone_fill(centroid, O3_date):
"""
Gets our ozone fill value (i.e. mean value for that doy and latlon)
you can see it
1) compared to LEDAPS: https://code.earthengine.google.com/8e62a5a66e4920e701813e43c0ecb83e
2) as a video: https://www.youtube.com/watch?v=rgqwvMRVguI&feature=youtu.be
"""
# ozone fills (i.e. one band per doy)
ozone_fills = ee.ImageCollection(
'users/samsammurphy/public/ozone_fill').toList(366)
# day of year index
jan01 = ee.Date.fromYMD(O3_date.get('year'), 1, 1)
doy_index = date.difference(jan01, 'day').toInt(
) # (NB. index is one less than doy, so no need to +1)
# day of year image
fill_image = ee.Image(ozone_fills.get(doy_index))
# return scalar fill value
return fill_image.reduceRegion(reducer=ee.Reducer.mean(),
geometry=centroid).get('ozone')
# O3 datetime in 24 hour intervals
O3_date = Atmospheric.round_date(date, 24)
# TOMS temporal gap
TOMS_gap = ee.DateRange('1994-11-01', '1996-08-01')
# avoid TOMS gap entirely
ozone = ee.Algorithms.If(TOMS_gap.contains(O3_date),
ozone_fill(centroid, O3_date),
ozone_measurement(centroid, O3_date))
# fix other data gaps (e.g. spatial, missing images, etc..)
ozone = ee.Algorithms.If(ozone, ozone, ozone_fill(centroid, O3_date))
#convert to Py6S units
ozone_Py6S_units = ee.Number(ozone).divide(
1000) # (i.e. Dobson units are milli-atm-cm )
return ozone_Py6S_units
return feature.set('diff', diff.pow(2))
# Load watersheds from a data table and filter to the continental US.
sheds = ee.FeatureCollection('USGS/WBD/2017/HUC06') \
.filterBounds(ee.Geometry.Rectangle(-127.18, 19.39, -62.75, 51.29))
# This function computes the squared difference between an area property
# and area computed directly from the feature's geometry.
# areaDiff = function(feature) {
# # Compute area in sq. km directly from the geometry.
# area = feature.geometry().area().divide(1000 * 1000)
# # Compute the differece between computed area and the area property.
# diff = area.subtract(ee.Number.parse(feature.get('areasqkm')))
# # Return the feature with the squared difference set to the 'diff' property.
# return feature.set('diff', diff.pow(2))
# }
# Calculate RMSE for population of difference pairs.
rmse = ee.Number(
# Map the difference function over the collection.
sheds.map(areaDiff)
# Reduce to get the mean squared difference. \
.reduceColumns(ee.Reducer.mean(), ['diff']) \
.get('mean')
) \
.sqrt()
# Print the result.
print('RMSE=', rmse.getInfo())
def AddAllLags(imgCollection, num_lags=1, filtering_property=None):
"""
Given a imgCollection it returns a list with the same number of images of the imageCollection
where each image contains its bands together with the bands of num_lags images where filtering_property
does NOT hold
It also add to the metadata of the image the lagged time stamp ("system:time_start_lag_X")
:param imgCollection: each img must have system:time_start property
:param num_lags: number of lagged elements of each image
:param filtering_property: if None no filtering will be done
:return: list with images
:rtype ee.List
"""
lags_client = range(1, num_lags + 1)
bands_server = ee.Image(imgCollection.first()).bandNames()
number_of_bands_server = ee.Number(bands_server.size())
total_number_bands = number_of_bands_server.multiply(num_lags + 1)
# create zero image with number_of_bands_server x num_lags + 1 bands
zeros = ee.List.repeat(0, total_number_bands)
bands_for_select = ee.List.sequence(0, total_number_bands.subtract(1))
# sufix_lag_server = ["_lag_1",....,"_lag_n-1","_lag_n"]
sufix_lag_server = ee.List(
list(map(lambda lg: "_lag_" + str(lg), lags_client)))
# sufix_lag_server_initial = ["","_lag_1",....,"_lag_n-1","_lag_n"]
sufix_lag_server_initial = ee.List([ee.String("")]).cat(sufix_lag_server)
# sufix_lag_server_shifted = ["","_lag_1",....,"_lag_n-1"]
sufix_lag_server_shifted = sufix_lag_server_initial.slice(0, num_lags)
all_bands = GenerateBandLags(bands_server, sufix_lag_server_initial)
# Important constant otherwise doesn't work
image_primera = ee.Image.constant(zeros).select(bands_for_select,
all_bands)
# Set time property
name_time_server = sufix_lag_server_initial.map(
lambda sufix: ee.String("system:time_start").cat(ee.String(sufix)))
zeros_time_start_server = ee.List.repeat(0, num_lags + 1)
dictio = ee.Dictionary.fromLists(name_time_server, zeros_time_start_server)
image_primera = image_primera.set(dictio)
if filtering_property is not None:
image_primera = image_primera.set(filtering_property, 1)
# Create first element for iterate
lista_ee = ee.List([image_primera])
def accumulate(image, lista_rec):
lista_recibida = ee.List(lista_rec)
previous = ee.Image(lista_recibida.get(-1))
bands_with_lags_server = GenerateBandLags(bands_server,
sufix_lag_server)
sufix_lag_server_shifted_iteration = sufix_lag_server_shifted
# if cloud > X bands_with_lags_shifted_server = bands_with_lags_server
if filtering_property is not None:
sufix_lag_server_shifted_iteration = ee.Algorithms.If(
ee.Number(previous.get(filtering_property)), sufix_lag_server,
sufix_lag_server_shifted)
sufix_lag_server_shifted_iteration = ee.List(
sufix_lag_server_shifted_iteration)
bands_with_lags_shifted_server = GenerateBandLags(
bands_server, sufix_lag_server_shifted_iteration)
previous_add = previous.select(bands_with_lags_shifted_server,
bands_with_lags_server)
image = image.addBands(previous_add)
name_time_server_shifted = sufix_lag_server_shifted_iteration.map(
lambda sufix: ee.String("system:time_start").cat(ee.String(sufix)))
values_time_server_select = name_time_server_shifted.map(
lambda field: previous.get(field))
name_time_server_select = sufix_lag_server.map(
lambda sufix: ee.String("system:time_start").cat(ee.String(sufix)))
dictio_set = ee.Dictionary.fromLists(name_time_server_select,
values_time_server_select)
image = image.set(dictio_set)
return lista_recibida.add(image)
lista_retorno = ee.List(imgCollection.iterate(accumulate,
lista_ee)).slice(1)
return lista_retorno
def ee_year(self, year):
return ee.Number(year)
def toiterate(element, ini):
ini = ee.Number(ini)
bit = ee.Number(2).pow(ee.Number(element))
return ini.add(bit)
def addCentroid(feat):
feat = ee.Feature(feat)
centroid = feat.centroid().geometry()
coords = ee.List(centroid.coordinates())
return feat.set('longitude', ee.Number(coords.get(0)),
'latitude', ee.Number(coords.get(1)))
def filterby_pathrow(p, r, wrsfetcoll):
p = ee.Number(p)
r = ee.Number(r)
f = ee.Filter.And(ee.Filter.eq('WRS_ROW', r), ee.Filter.eq('WRS_PATH', p))
return wrsfetcoll.filter(f)
def chi2cdf(chi2, df):
''' Chi square cumulative distribution function '''
return ee.Image(chi2.divide(2)).gammainc(ee.Number(df).divide(2))
def get_region(feature):
coll = LS_COLL.filterBounds(ee.Feature(feature).geometry())
ncoll = coll.size().getInfo()
# feature to be extracted
feat = ee.Algorithms.If(
ee.Number(BUFFER_DIST).gt(0),
ee.Feature(feature).buffer(BUFFER_DIST).bounds(), ee.Feature(feature))
feat_dict = feat.getInfo()
feat_props = feat_dict['properties']
pt_list = list()
if ncoll > 0:
# list of properties of each image in the collection as dictionary
coll_dicts = get_coll_dict(coll).getInfo()
# extract pixel values from the collection
# the output is a list or lists with the first row as column names
temp_list = coll.getRegion(ee.Feature(feat).geometry(),
SCALE).getInfo()
n = len(temp_list)
elem_names = temp_list[0]
id_column = elem_names.index('id')
for j in range(1, n):
pt_dict = dict()
for k in range(0, len(elem_names)):
pt_dict[elem_names[k]] = temp_list[j][k]
id = temp_list[j][id_column]
scene_prop_list = list(coll_dict for coll_dict in coll_dicts
if coll_dict['id'] == id)
scene_prop = scene_prop_list[0]
for prop in scene_properties:
pt_dict[prop] = scene_prop[prop]
for prop in feat_properties:
pt_dict[prop] = feat_props[prop]
pt_list.append(pt_dict)
else:
pt_dict = dict()
pt_dict['id'] = NULL_VALUE
pt_dict['time'] = NULL_VALUE
for prop in feat_properties:
pt_dict[prop] = feat_props[prop]
for prop in scene_properties:
pt_dict[prop] = NULL_VALUE
for band in bands:
pt_dict[band] = NULL_VALUE
pt_list.append(pt_dict)
return pt_list
def _get_mean_corrfactor(refimg, destimg, interspoly):
refmean = _get_mean(ee.Image(refimg), interspoly)
destmean = _get_mean(ee.Image(destimg), interspoly)
deltamean = ee.Algorithms.If(
refmean, ee.Algorithms.If(destmean, refmean.subtract(destmean), 0), 0)
return ee.Number(deltamean)
def iter_cols(i):
i = ee.String(i)
v = ee.Number(feature.get(i))
minv = ee.Number(min_max_dict.get(i.cat("_min")))
maxv = ee.Number(min_max_dict.get(i.cat("_max")))
return v.subtract(minv).divide(maxv.subtract(minv))
def __init__(self,
img,
bands,
cloud_mask,
max_lags,
region,
seed=None,
beta=None):
"""
Class which implements linear and kernel models on Google Earth Engine
for image forecasting as described in XXX paper
:param img: to apply the model
:type img: ee.Image
:param bands: list with the names of the bands
:type bands: list[str]
:param cloud_mask: cloud_mask image to mask pixels for "estimation"
:type cloud_mask: ee.Image
:param max_lags: number of lags of the model
:param beta: beta of the model (ponderate estimation vs prediction)
:param region: geom to fit the model (model will be fitted only with pixels of this geom)
:type region: ee.Geometry
"""
self.img = img
self.bands = bands
self.cloud_mask = cloud_mask
self.max_lags = max_lags
self.region = region
bands_modeling_estimation = list(self.bands)
for lag in range(1, self.max_lags):
bands_lag = list(map(lambda x: x + "_lag_" + str(lag), self.bands))
bands_modeling_estimation.extend(bands_lag)
self.bands_modeling_estimation = bands_modeling_estimation
bands_modeling_prediction = []
for lag in range(1, self.max_lags + 1):
bands_lag = list(map(lambda x: x + "_lag_" + str(lag), self.bands))
bands_modeling_prediction.extend(bands_lag)
self.bands_modeling_prediction = bands_modeling_prediction
self.bands_modeling_estimation_input = list(
filter(lambda band: band.find("lag") != -1,
self.bands_modeling_estimation))
self.bands_modeling_estimation_output = list(
filter(lambda band: band.find("lag") == -1,
self.bands_modeling_estimation))
self.bands_modeling_prediction_input = list(
filter(lambda band: band.find("lag_1") == -1,
self.bands_modeling_prediction))
self.bands_modeling_prediction_output = list(
filter(lambda band: band.find("lag_1") != -1,
self.bands_modeling_prediction))
if seed is None:
self.seed = random.randint(0, 36000)
else:
self.seed = seed
# Calculamos CC de la imagen actual
dictio = self.cloud_mask.select([0], ["cloud"]).reduceRegion(
reducer=ee.Reducer.mean(),
geometry=region,
bestEffort=True,
scale=10.)
self.cc_image = ee.Number(dictio.get("cloud")).getInfo()
self.img_est = self.img.updateMask(self.cloud_mask.eq(0))
self.img_est = self.img_est.select(self.bands_modeling_estimation)
self.img_pred = self.img.select(self.bands_modeling_prediction)
# Decrease beta linearly as the amount of cc_image increases
if beta is None:
self.beta = .5 - .5 * self.cc_image / CC_IMAGE_TOP
self.beta = self.beta if self.beta > 0 else 0
else:
self.beta = beta
self.alpha = None
self.kernel_rbf = None
self.gamma = None
self.lmbda = None
self.omega = None
self.intercept = None
def _BuildDataSet(self, sampling_factor, normalize, numPixels=None):
estimation_set = self.img_est.sample(region=self.region,
factor=sampling_factor,
numPixels=numPixels,
seed=self.seed)
prediction_set = self.img_pred.sample(region=self.region,
factor=sampling_factor,
numPixels=numPixels,
seed=self.seed)
# Add weights
estimation_set_size = ee.Number(estimation_set.size())
prediction_set_size = ee.Number(prediction_set.size())
peso_estimacion = ee.Number(self.beta).divide(estimation_set_size)
peso_prediccion = ee.Number(1 - self.beta).divide(prediction_set_size)
bands_modeling_estimation_input_weight = list(
self.bands_modeling_estimation_input)
bmp = list(self.bands_modeling_prediction)
bmp.append("weight")
bme = list(self.bands_modeling_estimation)
bme.append("weight")
estimation_set = estimation_set.map(
lambda ft: ee.Feature(ft).set("weight", peso_estimacion))
prediction_set = prediction_set.map(
lambda ft: ee.Feature(ft).set("weight", peso_prediccion))
bands_modeling_estimation_input_weight.append("weight")
self.estimation_set = estimation_set
self.prediction_set = prediction_set
if (self.beta > 0) and (self.cc_image < CC_IMAGE_TOP):
self.datos = estimation_set.merge(prediction_set.select(bmp, bme))
else:
logger.info("Using only prediction")
self.datos = prediction_set.select(bmp, bme)
if normalize:
self.datos, self.inputs_mean, self.inputs_std = normalization.ComputeNormalizationFeatureCollection(
self.datos,
self.bands_modeling_estimation_input,
only_center_data=False,
weight="weight")
self.datos, self.outputs_mean, self.outputs_std = normalization.ComputeNormalizationFeatureCollection(
self.datos,
self.bands_modeling_estimation_output,
only_center_data=True,
weight="weight")
self.inputs_mean = self.inputs_mean.toArray(
self.bands_modeling_estimation_input)
self.inputs_std = self.inputs_std.toArray(
self.bands_modeling_estimation_input)
self.outputs_mean = self.outputs_mean.toArray(
self.bands_modeling_estimation_output)
#if "B10" in self.bands_modeling_estimation_output:
# self.datos.select("B10").divide(100)
#if "B11" in self.bands_modeling_estimation_output:
# output_dataset["B11"] /= 100
self.inputs = self.datos.select(bands_modeling_estimation_input_weight)
self.outputs = self.datos.select(self.bands_modeling_estimation_output)
return
def compareIndexArray(indexArray, number, sequence):
Bi_index = ee.Algorithms.If(
ee.Number(sequence).eq(0), indexArray.gte(ee.Number(number)),
indexArray.lte(ee.Number(number)))
return ee.Array(Bi_index)
# %%
Map = folium.Map(location=[40, -100], zoom_start=4)
Map.setOptions('HYBRID')
# %%
'''
## Add Earth Engine Python script
'''
# %%
p1 = ee.Geometry.Point([103.521, 13.028])
p2 = ee.Geometry.Point([105.622, 13.050])
Date_Start = ee.Date('2000-05-01')
Date_End = ee.Date('2007-12-01')
Date_window = ee.Number(30)
# Create list of dates for time series
n_months = Date_End.difference(Date_Start, 'month').round()
print("Number of months:", n_months.getInfo())
dates = ee.List.sequence(0, n_months, 1)
print(dates.getInfo())
def make_datelist(n):
return Date_Start.advance(n, 'month')
dates = dates.map(make_datelist)
print(dates.getInfo())
def wrap(name, i):
i = ee.Number(i)
scale = ee.Number(image.select([name]).projection().nominalScale())
condition = scale.lte(i)
newscale = ee.Algorithms.If(condition, scale, i)
return newscale
legger_classes = {
'Water': 1,
'Verhard oppervlak': 2,
'Gras en Akker': 3,
'Riet en Ruigte': 4,
'Bos': 5,
'Struweel': 6,
'': 0
}
class_names = list(legger_classes.keys())
legger_classes = ee.Dictionary(legger_classes)
classes_legger = ee.Dictionary.fromLists(
legger_classes.values().map(lambda o: ee.Number(o).format('%d')),
legger_classes.keys())
def to_date_time_string(millis):
return ee.Date(millis).format('YYYY-MM-dd HH:mm')
def get_satellite_images(region, date_begin, date_end, cloud_filtering):
images = ee.ImageCollection('COPERNICUS/S2') \
.select(band_names['s2'], band_names['readable']) \
.filterBounds(region)
if date_begin:
if not date_end:
date_end = date_begin.advance(1, 'day')
def addField(feature):
sum = ee.Number(feature.get('property1')).add(feature.get('property2'))
return feature.set({'sum': sum})
def prepare_voorspel_data(image, ecotop_features, grass_pct, herb_pct,
willow_pct, feature, start_date, years):
"""
:param image:
:param ecotop_features:
:param grass_pct:
:param herb_pct:
:param willow_pct:
:param feature:
:param start_date:
:param years:
:return:
"""
class_values = [1, 2, 3, 4, 5, 6]
class_roughness = [0.0, 0.15, 0.39, 1.45, 12.84, 24.41]
# Convert image from classes to roughness
image = image.remap(class_values, class_roughness)
# Specify roughness value image for each class
k_water = ee.Number(class_roughness[0])
k_bare = ee.Number(class_roughness[1])
k_grass = ee.Number(class_roughness[2])
k_herbaceous = ee.Number(class_roughness[3])
k_reed = ee.Number(20.73)
k_forest = ee.Number(class_roughness[4])
k_willow = ee.Number(class_roughness[5])
k_water_im = ee.Image(k_water)
k_bare_im = ee.Image(k_bare)
k_grass_im = ee.Image(k_grass)
k_herbaceous_im = ee.Image(k_herbaceous)
k_reed_im = ee.Image(k_reed)
k_forest_im = ee.Image(k_forest)
k_willow_im = ee.Image(k_willow)
# Rates for progression from one class to another.
# All rates take 10 yr to change class
bare_to_reed_rate = k_reed.subtract(k_bare).divide(5)
grass_to_herb_rate = k_herbaceous.subtract(k_grass).divide(10)
herb_to_willow_rate = k_willow.subtract(k_herbaceous).divide(10)
willow_to_forest_rate = k_forest.subtract(k_willow).divide(10)
# Shoreline roughness image @ t=0
first_roughness_image = k_water_im.updateMask(image.eq(k_water)) \
.addBands(k_bare_im.updateMask(image.eq(k_bare))) \
.addBands(k_grass_im.updateMask(image.eq(k_grass))) \
.addBands(k_herbaceous_im.updateMask(image.eq(k_herbaceous))) \
.addBands(k_forest_im.updateMask(image.eq(k_forest))) \
.addBands(k_willow_im.updateMask(image.eq(k_willow))) \
.rename(['waterRoughness', 'bareRoughness', 'grassRoughness',
'herbaceousRoughness', 'forestRoughness', 'willowRoughness']) \
.set('system:time_start', start_date)
#
moisture_masks = wet_moist_masks(start_date, feature)
wet_mask = moisture_masks.select('wetMask')
moist_mask = moisture_masks.select('moistMask')
# Mechanical dynamics from ecotoop layers
mech_dyn_list = [
'Onbekend', 'Gering dynamisch', 'Matig/gering dynamisch',
'Sterk/matig dynamisch', 'Sterk dynamisch'
]
ecotoop_mech_dyn = ecotop_features.remap(mech_dyn_list, [0, 1, 2, 3, 4],
'MECH_DYN')
mech_dyn_im = ee.Image().int().paint(ecotoop_mech_dyn, 'MECH_DYN')
weak_dynamics = mech_dyn_im.lte(1)
moderate_dynamics = mech_dyn_im.eq(2)
strong_dynamics = mech_dyn_im.gte(3)
# bare remain bare with strong dynamics
remain_bare = image.eq(k_bare).multiply(strong_dynamics)
# bare will change to reed in 2 yr if it is wet and weak dynamics
bare_to_reed_mask = image.eq(k_bare).multiply(wet_mask).multiply(
weak_dynamics)
# bare will change to willow if moist and moderate dynamics
bare_to_willow_mask = image.eq(k_bare).multiply(moist_mask).multiply(
moderate_dynamics)
# Grazing/management will affect the rate at which bare progresses
bare_grazing_im = bare_grazing_variables(ecotop_features,
bare_to_reed_mask,
bare_to_willow_mask)
bare_to_reed_mask = bare_to_reed_mask.multiply(
bare_grazing_im.select('bareToReedGrazing'))
max_reed_im = k_reed_im.multiply(bare_to_reed_mask)
# Create random images
random_a = ee.Image.random(0)
random_b = ee.Image.random(1)
random_c = ee.Image.random(2)
# % grass will change into herb in 10 yr
grass_to_herb_mask = image.eq(k_grass).And(random_a.lte(grass_pct))
# % herbaceous veg will change to shrubs in 10 yr
herb_to_willow_mask = image.eq(k_herbaceous).And(random_b.lte(herb_pct))
# % of willows/shrubs will change into forest in 10 yr
willow_to_forest_mask = image.eq(k_willow).And(random_c.lte(willow_pct))
def accumulate_roughness(year, prev):
"""
"""
# Get the previous calculated image
date_now = start_date.advance(ee.Number(year), 'year')
prev_im = ee.Image(ee.List(prev).get(-1))
water_now = prev_im.select('waterRoughness')
bare_to_willow_prev = prev_im.select('bareRoughness').multiply(
bare_to_willow_mask)
bare_to_willow_now = k_willow_im.min(
willow_function(bare_grazing_im.select('bareToWillowGrazingA'),
bare_grazing_im.select('bareToWillowGrazingB'),
year))
bare_now = prev_im.select('bareRoughness') \
.add(max_reed_im.min(ee.Image(bare_to_reed_rate).multiply(bare_to_reed_mask)).unmask()) \
.add(bare_to_willow_now.subtract(bare_to_willow_prev).unmask())
grass_now = prev_im.select('grassRoughness') \
.add(ee.Image(grass_to_herb_rate).multiply(grass_to_herb_mask))
herb_now = prev_im.select('herbaceousRoughness') \
.add(ee.Image(herb_to_willow_rate).multiply(herb_to_willow_mask))
forest_now = prev_im.select('forestRoughness')
willow_now = prev_im.select('willowRoughness') \
.add(ee.Image(willow_to_forest_rate).multiply(willow_to_forest_mask))
image_now = water_now \
.addBands(bare_now) \
.addBands(grass_now) \
.addBands(herb_now) \
.addBands(forest_now) \
.addBands(willow_now) \
.rename(['waterRoughness', 'bareRoughness', 'grassRoughness',
'herbaceousRoughness', 'forestRoughness', 'willowRoughness']) \
.set('system:time_start', date_now)
# return ee.ImageCollection(prev_im).merge(ee.ImageCollection([image_now]))
return ee.List(prev).add(image_now)
year_array = ee.List.sequence(1, years, 1)
# Create an ImageCollection of images by iterating.
# blank_collection = ee.ImageCollection([first_roughness_image])
blank_collection = ee.List([first_roughness_image])
# cumulative_images = year_array.iterate(accumulate_roughness, blank_collection)
cumulative_images = ee.ImageCollection(
ee.List(year_array.iterate(accumulate_roughness, blank_collection)))
return cumulative_images
# coding=utf-8
''' Functions for calculation indices '''
from __future__ import print_function
import ee
import ee.data
if not ee.data._initialized: ee.Initialize()
'''
NDVI_EXP = "(NIR-RED)/(NIR+RED)"
EVI_EXP = "G*((NIR-RED)/(NIR+(C1*RED)-(C2*BLUE)+L))"
NBR_EXP = "(NIR-SWIR)/(NIR+SWIR)"
'''
true = ee.Number(1)
false = ee.Number(0)
FORMULAS = {
'NDVI': '(NIR-RED)/(NIR+RED)',
'EVI': 'G*((NIR-RED)/(NIR+(C1*RED)-(C2*BLUE)+L))',
'NBR': '(NIR-SWIR2)/(NIR+SWIR2)',
'NBR2': '(SWIR-SWIR2)/(SWIR+SWIR2)'
}
AVAILABLE = FORMULAS.keys()
def compute(index, band_params, extra_params=None, addBand=True):
if index not in AVAILABLE:
raise ValueError('Index not available')
addBandEE = true if addBand else false
def predict_roughness(region, start_date, num_years):
# start_date = ee.Date(startYearString + '-11-01')
feature = ee.FeatureCollection(region["features"]).first()
ecotop_features = ee.FeatureCollection(
"users/gertjang/succession/ecotopen_cyclus_drie_rijntakken_utm31n")
classified_images = ee.ImageCollection(yearly_collections['landuse'])
start_year = ee.Date(start_date).get('year')
lookback = start_date.advance(-1, 'year')
# Hydrology from ecotop layers
ecotop_hydrology = ecotop_features.remap([
'Onbekend', 'Overstromingsvrij', 'Periodiek tot zelden overstroomd',
'Oever - vochtig', 'Oever - drassig/vochtig', 'Oever - drassig',
'Oever - nat/drassig/vochtig', 'Oever - nat', 'Ondiep', 'Matig diep',
'Diep', 'Zeer diep/diep'
], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], 'HYDROLOGIE')
hydrologie_image = ee.Image().int().paint(ecotop_hydrology, 'HYDROLOGIE')
shoreline = hydrologie_image.gte(3).And(hydrologie_image.lte(7))
terrestrial = hydrologie_image.lte(2)
no_succession = hydrologie_image.gte(8)
# Prepare starting Image
classified_image = ee.Image(
classified_images.filterDate(ee.Date.fromYMD(start_year, 1, 1),
ee.Date.fromYMD(start_year, 12,
31)).first())
roughness_image = classified_image.remap(
[1, 2, 3, 4, 5, 6],
[0.0, 0.15, 0.39, 1.45, 12.84, 24.41]).rename("predicted")
classified_image_shore = classified_image.updateMask(shoreline)
classified_image_terrestrial = classified_image.updateMask(terrestrial)
classified_image_no_succession = classified_image.updateMask(no_succession)
roughness_no_succession = roughness_image.updateMask(no_succession)
# Shoreline Evolution Rates
# 10% grass will change into herb in 10 yr
grass_to_herb = ee.Number(0.1)
# 40% herbaceous veg will change to shrubs in 10 yr
herb_to_willow = ee.Number(0.4)
# 10% of shrubs will change into forest in 10 yr
willow_to_forest = ee.Number(0.1)
cumulative_shore = prepare_voorspel_data(classified_image_shore,
ecotop_features, grass_to_herb,
herb_to_willow, willow_to_forest,
feature, start_date, num_years)
# Terrestrial Evolution Rates
# 10% grass will change into herb in 10 yr (same as shoreline)
# 60% herbaceous veg will change to shrubs in 10 yr
herb_to_willow = ee.Number(0.6)
# 20% of shrubs will change into forest in 10 yr
willow_to_forest = ee.Number(0.2)
# Forrest remains forest
cumulative_terrestrial = prepare_voorspel_data(
classified_image_terrestrial, ecotop_features, grass_to_herb,
herb_to_willow, willow_to_forest, feature, start_date, num_years)
total_roughness_images = merge_roughness_regions(classified_image,
cumulative_shore,
cumulative_terrestrial,
roughness_no_succession)
return total_roughness_images
def last_day(self, year):
return ee.Date.fromYMD(ee.Number(year - 1), 12, 31).getRelative(
"day", "year")
def TOC_Feature_coor(featurecollection_input: ee.FeatureCollection,
QCname: str,
Indexname: str,
thresholdList: ee.List,
Classname: str = None,
ClassList: dict = None,
exportCoor: str = False,
exportVariable: str = False) -> None:
'''
This function is to export the coordinates of the TOC curve. The input format is the ee.FeatureCollection.
:param featurecollection_input: (ee.FeatureCollection) featurecollection that contains reference and index properties
:param QCname: (string) The name of reference property
:param Indexname: (string) The name of index property
:param thresholdList: (ee.List/list) The list of thresholds. (from high to low or from low to high)
:param Classname: (string) The name of class property when applying stratified sampling
:param ClassList: (disctionary) The size of stratums {classname: classsize, classname: classsize ...}
:param exportCoor: (string / default is False) The default parameter is False (not export the coordinates), string should be output path of coordinates. (extension should be txt)
:param exportVariable: (string / default is False) The default parameter is False (not export the coordinates), string should be output of variables. (extension should be txt)
:return: (ee.list) coordinates of the TOC curve. The first column is x, the second is y, the third is threshold of the point.
'''
featurecollection_list = featurecollection_input.toList(
featurecollection_input.size())
def getPropertyIndex(x):
return ee.Feature(x).getNumber(Indexname)
def getPropertyQC(x):
return ee.Feature(x).getNumber(QCname)
Index_1 = featurecollection_list.map(getPropertyIndex)
Index_2 = ee.Array(Index_1)
QC_1 = featurecollection_list.map(getPropertyQC)
QC_2 = ee.Array(QC_1)
if ((Classname is not None) and (ClassList is not None)):
if (type(Classname) is not str):
Classname = Classname.getInfo()
if (type(ClassList) is not ee.Dictionary):
ClassList = ee.Dictionary(ClassList)
classProperty = featurecollection_input.aggregate_histogram(Classname)
def classWeight(key, value):
presenceInClass = classProperty.get(key)
return ee.Number(value).divide(presenceInClass)
ClassWeights = ClassList.map(classWeight)
print(ClassWeights.getInfo())
def setWeight(x):
x = ee.Feature(x)
return ClassWeights.get(x.get(Classname))
weight = ee.Array(featurecollection_list.map(setWeight))
else:
weight = ee.Array(ee.List.repeat(1, QC_1.size()))
presenceInY = sumArray(QC_2.multiply(weight))
totalNumber = sumArray(weight)
sequence = ee.Algorithms.If(
ee.Number(thresholdList.get(0)).lt(thresholdList.get(-1)), 1, 0)
def coordinatelist(x):
index1 = compareIndexArray(Index_2, x, sequence)
index1_weight = index1.multiply(weight)
index2 = ee.Array((index1.add(QC_2).gte(2)))
index2_weight = index2.multiply(weight)
x_1 = sumArray(index1_weight)
y_1 = sumArray(index2_weight)
coor = ee.List([x_1, y_1, ee.Number(x)])
return coor
result = thresholdList.map(coordinatelist)
result_output = result
if (exportCoor):
result = np.array(result.getInfo()).transpose()
exportCoorFunc(exportCoor, np.array([result[0].tolist()]),
np.array([result[1].tolist()]), result[2].tolist())
if (exportVariable):
pInY = presenceInY.getInfo()
tNumber = totalNumber.getInfo()
AUC = calculate_AUC(np.array([result[0].tolist()]),
np.array([result[1].tolist()]), pInY, tNumber)
ccorner = correctCorner(np.array([result[0].tolist()]),
np.array([result[1].tolist()]), pInY)
exportVariableFunc(exportVariable, tNumber, pInY, ccorner, AUC)
return result_output
def create_number(value):
if isinstance(value, int) or isinstance(value, float):
return ee.Number(int(value))
elif isinstance(value, str):
conversion = int(value)
return ee.Number(conversion)
def classWeight(key, value):
presenceInClass = classProperty.get(key)
return ee.Number(value).divide(presenceInClass)
def conditional(image):
return ee.Algorithms.If(
ee.Number(image.get('SUN_ELEVATION')).gt(40), image, ee.Image(0))
def sumArray(arr1):
sumArr = arr1.reduce(**{'reducer': ee.Reducer.sum(), 'axes': [0]})
value = sumArr.get([0])
return ee.Number(value)
def testProfilePrinting(self):
out = StringIO.StringIO()
with ee.profilePrinting(destination=out):
self.assertEquals('hooked=True getProfiles=False',
ee.Number(1).getInfo())
self.assertEquals('hooked=False getProfiles=True', out.getvalue())
def main():
# Load in the pre-processed GLAD alerts
glad_alerts = ee.Image(
'users/JohnBKilbride/SERVIR/real_time_monitoring/glad_alerts_2019_to_2020'
)
# Get the projection that is needed for the study area
projection = ee.Projection('EPSG:32648')
# Define the username
username = "******"
# Define the output location
output_dir = "SERVIR/real_time_monitoring"
# Kernel size (# of pixels)
kernel_size = 64
# Compute the kernel radius
kernel_radius = ee.Number(kernel_size).divide(2)
# Get the study area
study_area = ee.Geometry.Polygon(
[[[104.0311, 14.3134], [104.0311, 12.5128], [106.0416, 12.5128],
[106.0416, 14.3134]]], None, False)
# Seperate the 2019 and 2020 glad data
glad_2019 = glad_alerts.select(['alertBinary19', 'alertDate19']) \
.addBands(ee.Image.constant(2019).rename('year')) \
.select(["alertBinary19","alertDate19", "year"],["binary","alert_day", "alert_year"]) \
.toInt16()
glad_2020 = glad_alerts.select(['alertBinary20', 'alertDate20']) \
.addBands(ee.Image.constant(2020).rename('year')) \
.select(["alertBinary20","alertDate20", "year"],["binary","alert_day", "alert_year"]) \
.toInt16()
# Take a stratified random sample of the 2019 layer
sample_2019 = get_sample_of_disturbances(glad_2019, projection, study_area)
sample_2020 = get_sample_of_disturbances(glad_2020, projection, study_area)
# Merge the two different samples
combined_samples = sample_2019.merge(sample_2020)
# Add the "start date" to each of the images
# This represents the first pre-disturbance observation that was actually valid (uses Landsat QA bands)
output = ee.FeatureCollection(add_start_date(combined_samples)) \
.select(['alert_day','alert_year','start_day','start_year'])
# Apply a random displacement to each of the point locations
output = apply_displacement(output, projection, kernel_radius)
# Export the sample locations with the julian date of the disturbance to google drive
task = ee.batch.Export.table.toAsset(collection=output,
description="Sample-Points-GLAD",
assetId="users/" + username + "/" +
output_dir +
"/sample_locations_2019_2020_50k")
task.start()
return None
