def run(params, logger): """.""" logger.debug("Loading parameters.") un_adju = params.get('un_adju', None) isi_thr = float(params.get('isi_thr', None)) ntl_thr = float(params.get('ntl_thr', None)) wat_thr = float(params.get('wat_thr', None)) cap_ope = float(params.get('cap_ope', None)) pct_suburban = float(params.get('pct_suburban', None)) pct_urban = float(params.get('pct_urban', None)) geojsons = json.loads(params.get('geojsons', None)) crs = params.get('crs', None) # Check the ENV. Are we running this locally or in prod? if params.get('ENV') == 'dev': EXECUTION_ID = str(random.randint(1000000, 99999999)) else: EXECUTION_ID = params.get('EXECUTION_ID', None) logger.debug("Checking total area of supplied geojsons:") area = 0 for geojson in geojsons: aoi = ee.Geometry.MultiPolygon(get_coords(geojson)) area += aoi.area().getInfo() / (1000*1000) # QGIS code limits area of bounding box to 25,000 sq km, so we shouldn't # ever have bounding boxes exceeding that area, but add an additional check # here (with an error margin of 10,000 sq km...) just in case. if area > 35000: logger.debug("Area ({:.6n} km sq) is too large - failing task".format(area)) raise Exception else: logger.debug("Processing total area of {:.6n} km sq".format(area)) logger.debug("Running main script.") out = urban(isi_thr, ntl_thr, wat_thr, cap_ope, pct_suburban, pct_urban, un_adju, crs, geojsons, EXECUTION_ID, logger) schema = CloudResultsSchema() logger.debug("Deserializing") final_output = schema.load(out[0]) for o in out[1:]: this_out = schema.load(o) final_output.urls.extend(this_out.urls) logger.debug("Serializing") # Now serialize the output again and return it return schema.dump(final_output)
def restrend_pointwise(year_start, year_end, geojson, EXECUTION_ID, logger): """Calculate temporal NDVI analysis. Calculates the trend of temporal NDVI using NDVI data from the MODIS Collection 6 MOD13Q1 dataset. Areas where changes are not significant are masked out using a Mann-Kendall test. Args: year_start: The starting year (to define the period the trend is calculated over). year_end: The ending year (to define the period the trend is calculated over). geojson: A polygon defining the area of interest. Returns: Output of google earth engine task. """ # Function to integrate NDVI dataset from 15d to 1yr def int_15d_1yr_clim(img_stack): img_coll = ee.List([]) for k in range(1, 34): ndvi_lyr = img_stack.select( ee.List.sequence((k - 1) * 24, (k * 24) - 1)).reduce( ee.Reducer.mean()).rename(['ndvi']).set({'year': 1981 + k}) img_coll = img_coll.add(ndvi_lyr) return ee.ImageCollection(img_coll) # Function to compute differences between observed and predicted NDVI and comilation in an image collection def stack(year_start, year_end): img_coll = ee.List([]) for k in range(year_start, year_end): ndvi = ndvi_1yr_o.filter(ee.Filter.eq('year', k)).select('ndvi').median() clim = clim_1yr_o.filter(ee.Filter.eq('year', k)).select('ndvi').median() img = ndvi.addBands(clim.addBands(ee.Image(k).float())).rename( ['ndvi', 'clim', 'year']).set({'year': k}) img_coll = img_coll.add(img) return ee.ImageCollection(img_coll) # Function to predict NDVI from climate first = ee.List([]) def ndvi_clim_p(image, list): ndvi = lf_clim_ndvi.select('offset').add( (lf_clim_ndvi.select('scale').multiply(image))).set( {'year': image.get('year')}) return ee.List(list).add(ndvi) # Create image collection of residuals def ndvi_res(year_start, year_end): img_coll = ee.List([]) for k in range(year_start, year_end): ndvi_o = coll_1yr_o.filter(ee.Filter.eq( 'year', k)).select('ndvi').median() ndvi_p = ndvi_1yr_p.filter(ee.Filter.eq('year', k)).median() ndvi_r = ee.Image(k).float().addBands(ndvi_o.subtract(ndvi_p)) img_coll = img_coll.add(ndvi_r.rename(['year', 'ndvi_res'])) return ee.ImageCollection(img_coll) ndvi_1yr_o = preproc.modis_ndvi_annual_integral(year_start, year_end) # TODO: define clim_15d_o which is the merra-2 soil moisture data. For now, # forcing use of Senegal data for testing. clim_15d_o = ee.Image( 'users/geflanddegradation/soil/sen_soilm_merra2_15d_1982_2015') # Apply function to compute climate annual integrals from 15d observed data clim_1yr_o = int_15d_1yr_clim(clim_15d_o.divide(10000)) # Apply function to create image collection with stack of NDVI int, climate int and year coll_1yr_o = stack(year_start, year_end) # Reduce the collection with the linear fit reducer (independent var are followed by dependent var) lf_clim_ndvi = coll_1yr_o.select(['clim', 'ndvi']).reduce(ee.Reducer.linearFit()) # Apply function to predict NDVI based on climate ndvi_1yr_p = ee.ImageCollection( ee.List(coll_1yr_o.select('clim').iterate(ndvi_clim_p, first))) # Apply function to compute NDVI annual residuals ndvi_1yr_r = ndvi_res(year_start, year_end) # Fit a linear regression to the NDVI residuals lf_prest = ndvi_1yr_r.select(['year', 'ndvi_res']).reduce(ee.Reducer.linearFit()) # Compute Kendall statistics mk_prest = stats.mann_kendall(ndvi_1yr_r.select('ndvi_res')) # Define Kendall parameter values for a significance of 0.05 period = year_end - year_start + 1 coefficients = ee.Array([ 4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 31, 33, 36, 40, 43, 47, 50, 54, 59, 63, 66, 70, 75, 79, 84, 88, 93, 97, 102, 106, 111, 115, 120, 126, 131, 137, 142 ]) kendall = coefficients.get([period - 4]) # Create export function export = { 'image': lf_prest.select('scale').where( mk_prest.abs().lte(kendall), -99999).where( lf_prest.select('scale').abs().lte(0.000001), -99999).unmask(-99999), 'description': EXECUTION_ID, 'fileNamePrefix': EXECUTION_ID, 'bucket': BUCKET, 'maxPixels': 10000000000, 'scale': 250, 'region': util.get_coords(geojson) } # Export final mosaic to assets task = ee.batch.Export.image.toCloudStorage(**export) task.start() task_state = task.status().get('state') while task_state == 'READY' or task_state == 'RUNNING': task_progress = task.status().get('progress', 0.0) # update GEF-EXECUTION progress logger.send_progress(task_progress) # update variable to check the condition task_state = task.status().get('state') sleep(5) return "https://{}.storage.googleapis.com/{}.tif".format( BUCKET, EXECUTION_ID)
def restrend_system(year_start, year_end, geojson, EXECUTION_ID, logger): """Calculate temporal NDVI analysis. Calculates the trend of temporal NDVI using NDVI data from the MODIS Collection 6 MOD13Q1 dataset. Areas where changes are not significant are masked out using a Mann-Kendall test. Args: year_start: The starting year (to define the period the trend is calculated over). year_end: The ending year (to define the period the trend is calculated over). geojson: A polygon defining the area of interest. Returns: Output of google earth engine task. """ # Function to integrate NDVI dataset from 15d to 1yr def int_15d_1yr_p(img_stack): img_coll = ee.List([]) for k in range(1, 34): ndvi_lyr = img_stack.select( ee.List.sequence((k - 1) * 24, (k * 24) - 1)).reduce( ee.Reducer.mean()).rename(['ndvi']).set({'year': 1981 + k}) img_coll = img_coll.add(ndvi_lyr) return ee.ImageCollection(img_coll) # Create image collection of residuals def ndvi_res(year_start, year_end): img_coll = ee.List([]) for k in range(year_start, year_end): ndvi_o = coll_1yr_o.filter(ee.Filter.eq( 'year', k)).select('ndvi').median() ndvi_p = ndvi_1yr_p.filter(ee.Filter.eq('year', k)).median() ndvi_r = ee.Image(k).float().addBands(ndvi_o.subtract(ndvi_p)) img_coll = img_coll.add(ndvi_r.rename(['year', 'ndvi_res'])) return ee.ImageCollection(img_coll) # Conversion of soil moisture to NDVI using equations developed by NASA presented in the report 1 # TODO: define clim_15d_o which is the merra-2 soil moisture data. For now, # forcing use of Senegal data for testing. clim_15d_o = ee.Image( 'users/geflanddegradation/soil/sen_soilm_merra2_15d_1982_2015') ndvi_p1 = clim_15d_o.divide(10000).multiply(0.11).add(0.19) ndvi_p2 = clim_15d_o.divide(10000).multiply(6.63).add(-2.22) ndvi_p3 = clim_15d_o.divide(10000).pow(3).multiply(1.38).add( clim_15d_o.divide(10000).pow(2).multiply(-3.83)).add( clim_15d_o.divide(10000).multiply(3.83)).add(-0.63) # Conditional statement to combine the 3 NDVI estimates into one image ndvi_15d_p = ndvi_p2.where(clim_15d_o.lte(0.37 * 10000), ndvi_p1).where(clim_15d_o.gte(0.39 * 10000), ndvi_p3) # Apply function to compute predicted NDVI annual integrals from 15d predicted NDVI data ndvi_1yr_p = int_15d_1yr_p(ndvi_15d_p) coll_1yr_o = preproc.modis_ndvi_annual_integral(year_start, year_end) # Compute differences between observed and predicted NDVI annual integrals ndvi_1yr_r = ndvi_res(year_start, year_end) # Fit a linear regression to the NDVI differences lf_srest = ndvi_1yr_r.select(['year', 'ndvi_res']).reduce(ee.Reducer.linearFit()) # Compute Kendall statistics mk_srest = stats.mann_kendall(ndvi_1yr_r.select('ndvi_res')) # Define Kendall parameter values for a significance of 0.05 period = year_end - year_start + 1 coefficients = ee.Array([ 4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 31, 33, 36, 40, 43, 47, 50, 54, 59, 63, 66, 70, 75, 79, 84, 88, 93, 97, 102, 106, 111, 115, 120, 126, 131, 137, 142 ]) kendall = coefficients.get([period - 4]) # Create export function export = { 'image': lf_srest.select('scale').where( mk_srest.abs().lte(kendall), -99999).where( lf_srest.select('scale').abs().lte(0.000001), -99999).unmask(-99999), 'description': EXECUTION_ID, 'fileNamePrefix': EXECUTION_ID, 'bucket': BUCKET, 'maxPixels': 10000000000, 'scale': 250, 'region': util.get_coords(geojson) } # Export final mosaic to assets task = ee.batch.Export.image.toCloudStorage(**export) task.start() task_state = task.status().get('state') while task_state == 'READY' or task_state == 'RUNNING': task_progress = task.status().get('progress', 0.0) # update GEF-EXECUTION progress logger.send_progress(task_progress) # update variable to check the condition task_state = task.status().get('state') sleep(5) return "https://{}.storage.googleapis.com/{}.tif".format( BUCKET, EXECUTION_ID)
def integral_trend(year_start, year_end, geojson, EXECUTION_ID, logger): """Calculate annual trend of integrated NDVI. Calculates the trend of annual integrated NDVI using NDVI data from the MODIS Collection 6 MOD13Q1 dataset. Areas where changes are not significant are masked out using a Mann-Kendall test. Args: year_start: The starting year (to define the period the trend is calculated over). year_end: The ending year (to define the period the trend is calculated over). geojson: A polygon defining the area of interest. EXECUTION_ID: String identifying this process, used in naming the results. Returns: Location of output on google cloud storage. """ # Compute NDVI annual integrals from 15d observed NDVI data ndvi_1yr_o = preproc.modis_ndvi_annual_integral(year_start, year_end) # Compute linear trend function to predict ndvi based on year (ndvi trend) lf_trend = ndvi_1yr_o.select(['year', 'ndvi']).reduce(ee.Reducer.linearFit()) # Define Kendall parameter values for a significance of 0.05 period = year_end - year_start + 1 coefficients = ee.Array([ 4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 31, 33, 36, 40, 43, 47, 50, 54, 59, 63, 66, 70, 75, 79, 84, 88, 93, 97, 102, 106, 111, 115, 120, 126, 131, 137, 142 ]) kendall = coefficients.get([period - 4]) # Compute Kendall statistics mk_trend = stats.mann_kendall(ndvi_1yr_o.select('ndvi')) export = { 'image': lf_trend.select('scale').where( mk_trend.abs().lte(kendall), -99999).where( lf_trend.select('scale').abs().lte(0.000001), -99999).unmask(-99999), 'description': EXECUTION_ID, 'fileNamePrefix': EXECUTION_ID, 'bucket': BUCKET, 'maxPixels': 10000000000, 'scale': 250, 'region': util.get_coords(geojson) } # Export final mosaic to assets task = ee.batch.Export.image.toCloudStorage(**export) # Task -> READY task.start() task_state = task.status().get('state') while task_state == 'READY' or task_state == 'RUNNING': task_progress = task.status().get('progress', 0.0) # update GEF-EXECUTION progress logger.send_progress(task_progress) # update variable to check the condition task_state = task.status().get('state') sleep(5) return "https://{}.storage.googleapis.com/{}.tif".format( BUCKET, EXECUTION_ID)