def run(params, logger):
"""."""
logger.debug("Loading parameters.")
un_adju = params.get('un_adju', None)
isi_thr = float(params.get('isi_thr', None))
ntl_thr = float(params.get('ntl_thr', None))
wat_thr = float(params.get('wat_thr', None))
cap_ope = float(params.get('cap_ope', None))
pct_suburban = float(params.get('pct_suburban', None))
pct_urban = float(params.get('pct_urban', None))
geojsons = json.loads(params.get('geojsons', None))
crs = params.get('crs', None)
# Check the ENV. Are we running this locally or in prod?
if params.get('ENV') == 'dev':
EXECUTION_ID = str(random.randint(1000000, 99999999))
else:
EXECUTION_ID = params.get('EXECUTION_ID', None)
logger.debug("Checking total area of supplied geojsons:")
area = 0
for geojson in geojsons:
aoi = ee.Geometry.MultiPolygon(get_coords(geojson))
area += aoi.area().getInfo() / (1000*1000)
# QGIS code limits area of bounding box to 25,000 sq km, so we shouldn't
# ever have bounding boxes exceeding that area, but add an additional check
# here (with an error margin of 10,000 sq km...) just in case.
if area > 35000:
logger.debug("Area ({:.6n} km sq) is too large - failing task".format(area))
raise Exception
else:
logger.debug("Processing total area of {:.6n} km sq".format(area))
logger.debug("Running main script.")
out = urban(isi_thr, ntl_thr, wat_thr, cap_ope, pct_suburban, pct_urban,
un_adju, crs, geojsons, EXECUTION_ID, logger)
schema = CloudResultsSchema()
logger.debug("Deserializing")
final_output = schema.load(out[0])
for o in out[1:]:
this_out = schema.load(o)
final_output.urls.extend(this_out.urls)
logger.debug("Serializing")
# Now serialize the output again and return it
return schema.dump(final_output)
def restrend_pointwise(year_start, year_end, geojson, EXECUTION_ID, logger):
"""Calculate temporal NDVI analysis.
Calculates the trend of temporal NDVI using NDVI data from the
MODIS Collection 6 MOD13Q1 dataset. Areas where changes are not significant
are masked out using a Mann-Kendall test.
Args:
year_start: The starting year (to define the period the trend is
calculated over).
year_end: The ending year (to define the period the trend is
calculated over).
geojson: A polygon defining the area of interest.
Returns:
Output of google earth engine task.
"""
# Function to integrate NDVI dataset from 15d to 1yr
def int_15d_1yr_clim(img_stack):
img_coll = ee.List([])
for k in range(1, 34):
ndvi_lyr = img_stack.select(
ee.List.sequence((k - 1) * 24, (k * 24) - 1)).reduce(
ee.Reducer.mean()).rename(['ndvi']).set({'year': 1981 + k})
img_coll = img_coll.add(ndvi_lyr)
return ee.ImageCollection(img_coll)
# Function to compute differences between observed and predicted NDVI and comilation in an image collection
def stack(year_start, year_end):
img_coll = ee.List([])
for k in range(year_start, year_end):
ndvi = ndvi_1yr_o.filter(ee.Filter.eq('year',
k)).select('ndvi').median()
clim = clim_1yr_o.filter(ee.Filter.eq('year',
k)).select('ndvi').median()
img = ndvi.addBands(clim.addBands(ee.Image(k).float())).rename(
['ndvi', 'clim', 'year']).set({'year': k})
img_coll = img_coll.add(img)
return ee.ImageCollection(img_coll)
# Function to predict NDVI from climate
first = ee.List([])
def ndvi_clim_p(image, list):
ndvi = lf_clim_ndvi.select('offset').add(
(lf_clim_ndvi.select('scale').multiply(image))).set(
{'year': image.get('year')})
return ee.List(list).add(ndvi)
# Create image collection of residuals
def ndvi_res(year_start, year_end):
img_coll = ee.List([])
for k in range(year_start, year_end):
ndvi_o = coll_1yr_o.filter(ee.Filter.eq(
'year', k)).select('ndvi').median()
ndvi_p = ndvi_1yr_p.filter(ee.Filter.eq('year', k)).median()
ndvi_r = ee.Image(k).float().addBands(ndvi_o.subtract(ndvi_p))
img_coll = img_coll.add(ndvi_r.rename(['year', 'ndvi_res']))
return ee.ImageCollection(img_coll)
ndvi_1yr_o = preproc.modis_ndvi_annual_integral(year_start, year_end)
# TODO: define clim_15d_o which is the merra-2 soil moisture data. For now,
# forcing use of Senegal data for testing.
clim_15d_o = ee.Image(
'users/geflanddegradation/soil/sen_soilm_merra2_15d_1982_2015')
# Apply function to compute climate annual integrals from 15d observed data
clim_1yr_o = int_15d_1yr_clim(clim_15d_o.divide(10000))
# Apply function to create image collection with stack of NDVI int, climate int and year
coll_1yr_o = stack(year_start, year_end)
# Reduce the collection with the linear fit reducer (independent var are followed by dependent var)
lf_clim_ndvi = coll_1yr_o.select(['clim',
'ndvi']).reduce(ee.Reducer.linearFit())
# Apply function to predict NDVI based on climate
ndvi_1yr_p = ee.ImageCollection(
ee.List(coll_1yr_o.select('clim').iterate(ndvi_clim_p, first)))
# Apply function to compute NDVI annual residuals
ndvi_1yr_r = ndvi_res(year_start, year_end)
# Fit a linear regression to the NDVI residuals
lf_prest = ndvi_1yr_r.select(['year',
'ndvi_res']).reduce(ee.Reducer.linearFit())
# Compute Kendall statistics
mk_prest = stats.mann_kendall(ndvi_1yr_r.select('ndvi_res'))
# Define Kendall parameter values for a significance of 0.05
period = year_end - year_start + 1
coefficients = ee.Array([
4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 31, 33, 36, 40, 43, 47, 50, 54,
59, 63, 66, 70, 75, 79, 84, 88, 93, 97, 102, 106, 111, 115, 120, 126,
131, 137, 142
])
kendall = coefficients.get([period - 4])
# Create export function
export = {
'image':
lf_prest.select('scale').where(
mk_prest.abs().lte(kendall), -99999).where(
lf_prest.select('scale').abs().lte(0.000001),
-99999).unmask(-99999),
'description':
EXECUTION_ID,
'fileNamePrefix':
EXECUTION_ID,
'bucket':
BUCKET,
'maxPixels':
10000000000,
'scale':
250,
'region':
util.get_coords(geojson)
}
# Export final mosaic to assets
task = ee.batch.Export.image.toCloudStorage(**export)
task.start()
task_state = task.status().get('state')
while task_state == 'READY' or task_state == 'RUNNING':
task_progress = task.status().get('progress', 0.0)
# update GEF-EXECUTION progress
logger.send_progress(task_progress)
# update variable to check the condition
task_state = task.status().get('state')
sleep(5)
return "https://{}.storage.googleapis.com/{}.tif".format(
BUCKET, EXECUTION_ID)
def restrend_system(year_start, year_end, geojson, EXECUTION_ID, logger):
"""Calculate temporal NDVI analysis.
Calculates the trend of temporal NDVI using NDVI data from the
MODIS Collection 6 MOD13Q1 dataset. Areas where changes are not significant
are masked out using a Mann-Kendall test.
Args:
year_start: The starting year (to define the period the trend is
calculated over).
year_end: The ending year (to define the period the trend is
calculated over).
geojson: A polygon defining the area of interest.
Returns:
Output of google earth engine task.
"""
# Function to integrate NDVI dataset from 15d to 1yr
def int_15d_1yr_p(img_stack):
img_coll = ee.List([])
for k in range(1, 34):
ndvi_lyr = img_stack.select(
ee.List.sequence((k - 1) * 24, (k * 24) - 1)).reduce(
ee.Reducer.mean()).rename(['ndvi']).set({'year': 1981 + k})
img_coll = img_coll.add(ndvi_lyr)
return ee.ImageCollection(img_coll)
# Create image collection of residuals
def ndvi_res(year_start, year_end):
img_coll = ee.List([])
for k in range(year_start, year_end):
ndvi_o = coll_1yr_o.filter(ee.Filter.eq(
'year', k)).select('ndvi').median()
ndvi_p = ndvi_1yr_p.filter(ee.Filter.eq('year', k)).median()
ndvi_r = ee.Image(k).float().addBands(ndvi_o.subtract(ndvi_p))
img_coll = img_coll.add(ndvi_r.rename(['year', 'ndvi_res']))
return ee.ImageCollection(img_coll)
# Conversion of soil moisture to NDVI using equations developed by NASA presented in the report 1
# TODO: define clim_15d_o which is the merra-2 soil moisture data. For now,
# forcing use of Senegal data for testing.
clim_15d_o = ee.Image(
'users/geflanddegradation/soil/sen_soilm_merra2_15d_1982_2015')
ndvi_p1 = clim_15d_o.divide(10000).multiply(0.11).add(0.19)
ndvi_p2 = clim_15d_o.divide(10000).multiply(6.63).add(-2.22)
ndvi_p3 = clim_15d_o.divide(10000).pow(3).multiply(1.38).add(
clim_15d_o.divide(10000).pow(2).multiply(-3.83)).add(
clim_15d_o.divide(10000).multiply(3.83)).add(-0.63)
# Conditional statement to combine the 3 NDVI estimates into one image
ndvi_15d_p = ndvi_p2.where(clim_15d_o.lte(0.37 * 10000),
ndvi_p1).where(clim_15d_o.gte(0.39 * 10000),
ndvi_p3)
# Apply function to compute predicted NDVI annual integrals from 15d predicted NDVI data
ndvi_1yr_p = int_15d_1yr_p(ndvi_15d_p)
coll_1yr_o = preproc.modis_ndvi_annual_integral(year_start, year_end)
# Compute differences between observed and predicted NDVI annual integrals
ndvi_1yr_r = ndvi_res(year_start, year_end)
# Fit a linear regression to the NDVI differences
lf_srest = ndvi_1yr_r.select(['year',
'ndvi_res']).reduce(ee.Reducer.linearFit())
# Compute Kendall statistics
mk_srest = stats.mann_kendall(ndvi_1yr_r.select('ndvi_res'))
# Define Kendall parameter values for a significance of 0.05
period = year_end - year_start + 1
coefficients = ee.Array([
4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 31, 33, 36, 40, 43, 47, 50, 54,
59, 63, 66, 70, 75, 79, 84, 88, 93, 97, 102, 106, 111, 115, 120, 126,
131, 137, 142
])
kendall = coefficients.get([period - 4])
# Create export function
export = {
'image':
lf_srest.select('scale').where(
mk_srest.abs().lte(kendall), -99999).where(
lf_srest.select('scale').abs().lte(0.000001),
-99999).unmask(-99999),
'description':
EXECUTION_ID,
'fileNamePrefix':
EXECUTION_ID,
'bucket':
BUCKET,
'maxPixels':
10000000000,
'scale':
250,
'region':
util.get_coords(geojson)
}
# Export final mosaic to assets
task = ee.batch.Export.image.toCloudStorage(**export)
task.start()
task_state = task.status().get('state')
while task_state == 'READY' or task_state == 'RUNNING':
task_progress = task.status().get('progress', 0.0)
# update GEF-EXECUTION progress
logger.send_progress(task_progress)
# update variable to check the condition
task_state = task.status().get('state')
sleep(5)
return "https://{}.storage.googleapis.com/{}.tif".format(
BUCKET, EXECUTION_ID)
def integral_trend(year_start, year_end, geojson, EXECUTION_ID, logger):
"""Calculate annual trend of integrated NDVI.
Calculates the trend of annual integrated NDVI using NDVI data from the
MODIS Collection 6 MOD13Q1 dataset. Areas where changes are not significant
are masked out using a Mann-Kendall test.
Args:
year_start: The starting year (to define the period the trend is
calculated over).
year_end: The ending year (to define the period the trend is
calculated over).
geojson: A polygon defining the area of interest.
EXECUTION_ID: String identifying this process, used in naming the
results.
Returns:
Location of output on google cloud storage.
"""
# Compute NDVI annual integrals from 15d observed NDVI data
ndvi_1yr_o = preproc.modis_ndvi_annual_integral(year_start, year_end)
# Compute linear trend function to predict ndvi based on year (ndvi trend)
lf_trend = ndvi_1yr_o.select(['year',
'ndvi']).reduce(ee.Reducer.linearFit())
# Define Kendall parameter values for a significance of 0.05
period = year_end - year_start + 1
coefficients = ee.Array([
4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 31, 33, 36, 40, 43, 47, 50, 54,
59, 63, 66, 70, 75, 79, 84, 88, 93, 97, 102, 106, 111, 115, 120, 126,
131, 137, 142
])
kendall = coefficients.get([period - 4])
# Compute Kendall statistics
mk_trend = stats.mann_kendall(ndvi_1yr_o.select('ndvi'))
export = {
'image':
lf_trend.select('scale').where(
mk_trend.abs().lte(kendall), -99999).where(
lf_trend.select('scale').abs().lte(0.000001),
-99999).unmask(-99999),
'description':
EXECUTION_ID,
'fileNamePrefix':
EXECUTION_ID,
'bucket':
BUCKET,
'maxPixels':
10000000000,
'scale':
250,
'region':
util.get_coords(geojson)
}
# Export final mosaic to assets
task = ee.batch.Export.image.toCloudStorage(**export)
# Task -> READY
task.start()
task_state = task.status().get('state')
while task_state == 'READY' or task_state == 'RUNNING':
task_progress = task.status().get('progress', 0.0)
# update GEF-EXECUTION progress
logger.send_progress(task_progress)
# update variable to check the condition
task_state = task.status().get('state')
sleep(5)
return "https://{}.storage.googleapis.com/{}.tif".format(
BUCKET, EXECUTION_ID)