def wrap(n, ini):
ini = ee.List(ini)
last_range = ee.DateRange(ini.get(-1))
last_date = last_range.end()
next_date = last_date.advance(1, 'second')
next_interval = next_date.advance(interval, unit)
return ini.add(ee.DateRange(next_date, next_interval))
def S2_loc(m):
start = ee.Date(m)
end = ee.Date(m).advance(15, 'days')
date_range = ee.DateRange(start,end)
sentime = ee.ImageCollection(S2resample) \
.filterDate(date_range)
return(sentime.min()).toFloat()
def newDate(m):
start = ee.Date(m)
end = ee.Date(m).advance(15, 'days')
date_range = ee.DateRange(start, end)
#newa = ee.Date(date_range).get('system:time_start')
#return(m.addBands(end))
return(end)
def wrap(dr, l):
l = ee.List(l)
dr = ee.DateRange(dr)
middle = ee.Number(dr.end().difference(dr.start(),
'day')).divide(2).floor()
filtered = collection.filterDate(dr.start(),
dr.end().advance(1, 'day'))
dates = ee.List(filtered.aggregate_array('system:time_start')).map(
lambda d: ee.Date(d).format())
def true(filt, ll):
if not composite_args and not composite_kwargs:
comp = composite_function(filtered)
elif composite_args and not composite_kwargs:
comp = composite_function(filtered, *composite_args)
elif composite_kwargs and not composite_args:
comp = composite_function(filtered, **composite_kwargs)
else:
comp = composite_function(filtered, *composite_args,
**composite_kwargs)
comp = comp.set('system:time_start',
dr.start().advance(middle, 'day').millis())
comp = comp.set('dates', dates)\
.set('composite:time_start', dr.start().format())\
.set('composite:time_end', dr.end().format())
return ll.add(comp)
return ee.Algorithms.If(filtered.size(), true(filtered, l), l)
def build_img_coll(self):
dmsp_original = ee.FeatureCollection(
"NOAA/DMSP-OLS/CALIBRATED_LIGHTS_V4")
center_date = get_center_date(self.start_date, self.end_date)
dmsp_img = get_closest_to_date(dmsp_original, center_date)
dmsp_img = dmsp_img.select(['avg_vis'], ['DMSP'])
dmsp = ee.ImageCollection([dmsp_img])
zeros_image = ee.Image(0).reproject(
ee.Image(dmsp_original.first()).projection()).select([0], ['DMSP'])
zeros_image = zeros_image.set('system:time_start',
ee.Date('2014-01-01').millis())
zeros_image = zeros_image.set('system:time_end',
ee.Date('3000-01-01').millis())
zeros_image_coll = ee.ImageCollection([zeros_image])
requested_daterange = ee.DateRange(self.start_date, self.end_date)
viirs_start = ee.Date('2014-1-1')
use_zeros_image = requested_daterange.contains(viirs_start)
start_year = ee.Number(ee.Date(self.start_date).get('year'))
viirs_start_year = ee.Number(viirs_start.get('year'))
use_zeros_image2 = start_year.gte(viirs_start_year)
dmsp = ee.Algorithms.If(use_zeros_image, zeros_image_coll, dmsp)
dmsp = ee.Algorithms.If(use_zeros_image2, zeros_image_coll, dmsp)
self.dmsp = ee.ImageCollection(dmsp)
def getDate(m):
start = ee.Date(m)
end = ee.Date(m).advance(15, 'days')
date_range = ee.DateRange(start, end)
#sentime = ee.ImageCollection(S2)
#.filterDate(date_range)
return(date_range)
def mixed(m):
start = ee.Date(m)
end = ee.Date(m).advance(15, 'days')
date_range = ee.DateRange(start,end)
sentime = ee.ImageCollection(S2L8TS) \
.filterDate(date_range)
return(sentime.min())
def MODIS_loc(m):
start = ee.Date(m)
end = ee.Date(m).advance(15, 'days')
date_range = ee.DateRange(start,end)
sentime = ee.ImageCollection(MODIScoll) \
.filterDate(date_range) \
.map(addTime)
return(sentime.min())
def over_ranges(drange, ini):
ini = ee.List(ini)
drange = ee.DateRange(drange)
start = drange.start()
end = drange.end()
filtered = collection.filterDate(start, end)
condition = ee.Number(filtered.size()).gt(0)
return ee.List(ee.Algorithms.If(condition, ini.add(filtered), ini))
def test_daterange_list():
start_date = ee.Date('2010-01-01')
end_date = ee.Date('2015-01-01')
unit = 'year'
interval = 1
expected = ee.List([
ee.DateRange('2010-01-01', '2011-01-01'),
ee.DateRange('2011-01-01', '2012-01-01'),
ee.DateRange('2012-01-01', '2013-01-01'),
ee.DateRange('2013-01-01', '2014-01-01'),
ee.DateRange('2014-01-01', '2015-01-01')
])
daterange_list = tools.date.daterangeList(start_date, end_date,
interval, unit)
assert expected == daterange_list
def prep_export(period_range):
_t_start = ee.DateRange(period_range).start()
export_path = ee.String("ecopath/").cat(
ee.String(model).cat(_t_start.format()))
im = get_map(model, period_range)
im = im.set('path', export_path)
im = im.set('tStart', _t_start)
im = im.set('system:time_start', _t_start.millis())
im = im.set('period', period_range)
return im
def get_single_image(region: ee.Geometry, start_date: date,
end_date: date) -> ee.Image:
dates = ee.DateRange(
date_to_string(start_date),
date_to_string(end_date),
)
startDate = ee.DateRange(dates).start()
endDate = ee.DateRange(dates).end()
imgC = ee.ImageCollection(image_collection).filterDate(
startDate, endDate).filterBounds(region)
imgC = (imgC.map(lambda x: x.clip(region)).map(lambda x: x.set(
"ROI", region)).map(computeS2CloudScore).map(calcCloudStats).map(
projectShadows).map(computeQualityScore).sort("CLOUDY_PERCENTAGE"))
cloudFree = mergeCollection(imgC)
return cloudFree
def wrap(dr, l):
l = ee.List(l)
dr = ee.DateRange(dr)
filtered = collection.filterDate(dr.start(), dr.end())
def true(filt, ll):
comp = composite_function(filtered)
comp = comp.set('system:time_start',
dr.start().advance(composite_date, 'day').millis())
return ll.add(comp)
return ee.Algorithms.If(filtered.size(), true(filtered, l), l)
def create_date_range(datelist,monthwindow=2):
'''
Input:
datelist : <list or tuple> of date in (YY,MM,DD) format
monthwindow : <int> month before and after <datelist> to search around
Output:
returns Earth Engine DateRange object.
'''
datecenter = ee.Date.fromYMD(datelist[0],datelist[1],datelist[2])
window = ee.DateRange(datecenter.advance(-monthwindow,unit='month'),\
datecenter.advance(monthwindow,unit='month'))
return window
def timeImage(img):
timestamp = ee.Number(img.get('system:time_start'))
date = ee.Date(timestamp)
dayrange = ee.DateRange(ee.Date.fromYMD(year, 10, 1),
ee.Date.fromYMD(wyend, 1, 1))
day = ee.Algorithms.If(
dayrange.contains(date),
ee.Number(date.getRelative('day', 'year')).double().subtract(
ee.Date.fromYMD(year, 10,
1).difference(ee.Date.fromYMD(year, 1, 1), 'day')),
ee.Number(date.getRelative('day', 'year')).double().add(92))
doy = ee.Image.constant(day).mask(img.select(0).mask())
return img.addBands(doy)
def build_img_coll(self):
viirs = self.init_coll("NOAA/VIIRS/DNB/MONTHLY_V1/VCMSLCFG")
dmsp = self.init_coll("NOAA/DMSP-OLS/CALIBRATED_LIGHTS_V4")
# Carefull not to communicate to the server
requested_daterange = ee.DateRange(self.start_date, self.end_date)
viirs_start = ee.Date('2014-1-1')
use_viirs = requested_daterange.contains(viirs_start)
start_year = ee.Number(ee.Date(self.start_date).get('year'))
viirs_start_year = ee.Number(viirs_start.get('year'))
use_viirs2 = start_year.gte(viirs_start_year)
self.nl = ee.Algorithms.If(use_viirs, viirs, dmsp)
self.nl = ee.Algorithms.If(use_viirs2, viirs, self.nl)
self.nl = ee.ImageCollection(self.nl)
def over_ranges(drange, ini):
ini = ee.List(ini)
drange = ee.DateRange(drange)
start = drange.start()
end = drange.end()
filtered = collection.filterDate(start, end)
condition = ee.Number(filtered.size()).gt(0)
def true():
image = apply_reducer(reducer, filtered)\
.set('system:time_start', end.millis())\
.set('reduced_from', start.format())\
.set('reduced_to', end.format())
# rename to original names
image = image.select(image.bandNames(), bands)
result = ini.add(image)
return result
return ee.List(ee.Algorithms.If(condition, true(), ini))
def get_map(model, period_range):
models = {"HYCOM": hycom_currents, "glossis": glossis_currents}
selected = models[model]
period_range = ee.DateRange(period_range)
selected = selected.filterDate(period_range.start(),
period_range.end())
image_count = selected.size()
selected = selected.mean()
selected = selected.set('model', model)
selected = selected.set('tStart', period_range.start())
selected = selected.set('tStop', period_range.end())
selected = selected.set('imageCount', image_count)
selected = selected.set('units', 'm/s')
return selected
def add_year(self, year):
year = ee.Number(year)
if self.over_end:
start_year = year.subtract(1)
else:
start_year = ee.Number(year)
end_year = ee.Number(year)
sday = self.start.day
eday = self.end.day
# look for feb 29h in non leap
if not is_leap(year):
if self.start.month == 2 and sday == 29:
sday = 28
if self.end.month == 2 and eday == 29:
eday = 28
start = ee.Date.fromYMD(start_year, self.start.month, sday)
end = ee.Date.fromYMD(end_year, self.end.month, eday)
daterange = ee.DateRange(ee.Date(start), ee.Date(end))
return daterange
def daterange_list(start_date, end_date, interval=1, unit='month'):
""" Divide a range that goes from start_date to end_date into many
ee.DateRange, each one holding as many units as the interval.
For example, for a range from
:param start_date: the start date. For the second DateRange and the
following, it'll be one second after the end of the previus DateRange
:param end_date: the end date
:param interval: range of the DateRange in units
:param unit: can be 'year', 'month' 'week', 'day', 'hour', 'minute',
or 'second'
:return: a list holding ee.DateRange
:rtype: list
"""
units = ['year', 'month' 'week', 'day', 'hour', 'minute', 'second']
if unit not in units:
raise ValueError('unit param must be one of {}'.format(units))
def callback(interval, unit):
def wrap(n, ini):
ini = ee.List(ini)
last_range = ee.DateRange(ini.get(-1))
last_date = last_range.end()
next_date = last_date.advance(1, 'second')
next_interval = next_date.advance(interval, unit)
return ini.add(ee.DateRange(next_date, next_interval))
return wrap
total_days = end_date.difference(start_date, unit)
total_dateranges = total_days.divide(interval).toInt()
dateranges_list = ee.List.sequence(1, total_dateranges)
# first daterange
first = ee.DateRange(start_date, start_date.advance(interval, unit))
return ee.List(
dateranges_list.iterate(callback(interval, unit), ee.List([first])))
def add_year(self, year):
""" Create the beginning and end of a season with the given year.
If param year is a ee.Number, it returns a ee.DateRange
:param year: season's year
:type year: int or ee.Number
:return: season's beginning and end
:rtype: tuple
"""
if isinstance(year, int) or isinstance(year, float):
a = int(year)
ini = str(a - self.year_factor) + "-" + self.ini
end = str(a) + "-" + self.end
# print "add_year", ini, end
return ini, end
elif isinstance(year, ee.Number):
factor = ee.Number(self.year_factor)
y = year.subtract(factor).format()
temp_ini = ee.String(self.ini)
ini = y.cat('-').cat(temp_ini)
temp_end = ee.String(self.end)
end = year.format().cat('-').cat(temp_end)
r = ee.DateRange(ee.Date(ini), ee.Date(end))
return r
def get_climate_data(bounds, datelist, offset_years=29, verbose=False):
'''
Collects all climate images from <offset_years> previous of the <datelist>,
and aggregates them into means and standard deviations
Inputs:
bounds: <ee.bounds object>
datelist : <list or tuple> of date in (YY,MM,DD) format
offset_years : <int> years back to go from datelist[YY]
verbose : <bool> specifies to print output
Output:
returns an Earth Engine ImageCollection object
'''
year_start = datelist[0]-offset_years
date_start = ee.Date.fromYMD(year_start,1,1)
date_end = ee.Date.fromYMD(datelist[0],1,1)
date_range = ee.DateRange(date_start, date_end)
climate_collection = ee.ImageCollection('OREGONSTATE/PRISM/AN81m').filterDate(date_range).filterBounds(bounds)
stddev = climate_collection.reduce(ee.Reducer.stdDev())
mean = climate_collection.reduce(ee.Reducer.mean())
out = mean.addBands(stddev)
if verbose:
print(f'Collected CLIMATE data from 01-01-{year_start} to 01-01-{datelist[0]}.\n')
return out
# // Load Landsat 5 data, filter by dates and bounds.
# l5_collection = ee.ImageCollection('LANDSAT/LT05/C01/T2').filterDate('1987-01-01','1987-12-31').filterBounds(bound_geom);
l5_collection = ee.ImageCollection('LANDSAT/LT05/C01/T2').filterBounds(bound_geom)
# // Convert the collection to a list and get the number of images.
size = l5_collection.toList(100000).length()
size = size.getInfo()
print('Number of images: ', size) # 12598
# // Get the number of images.
count = l5_collection.size()
count = count.getInfo()
print('Count: ', count)
# // Get the date range of images in the collection.
dates = ee.List(l5_collection.get('date_range'))
dateRange = ee.DateRange(dates.get(0), dates.get(1))
dateRange = dateRange.getInfo()
print('Date range: ', dateRange) #
# // Get statistics for a property of the images in the collection.
sunStats = l5_collection.aggregate_stats('SUN_ELEVATION');
print('Sun elevation statistics: ', sunStats);
cloudCover = l5_collection.aggregate_stats('CLOUD_COVER').getInfo()
'''
Aggregates over a given property of the objects in a collection, calculating the
sum, min, max, mean, sample standard deviation, sample variance, total standard deviation and total variance of the selected property.
max: 100
mean: 75.43951420860454
min: -1
sample_sd: 29.319729461053385
def ozone(geom, date):
"""
returns ozone measurement from merged TOMS/OMI dataset
OR
uses our fill value (which is mean value for that latlon and day-of-year)
"""
# Point geometry required
centroid = geom.centroid()
def ozone_measurement(centroid, O3_date):
# filtered ozone collection
ozone_ic = ee.ImageCollection('TOMS/MERGED').filterDate(
O3_date, O3_date.advance(1, 'month'))
# ozone image
ozone_img = ee.Image(ozone_ic.first())
# ozone value IF TOMS/OMI image exists ELSE use fill value
ozone = ee.Algorithms.If(ozone_img,\
ozone_img.reduceRegion(reducer=ee.Reducer.mean(), geometry=centroid).get('ozone'),\
ozone_fill(centroid,O3_date))
return ozone
def ozone_fill(centroid, O3_date):
"""
Gets our ozone fill value (i.e. mean value for that doy and latlon)
you can see it
1) compared to LEDAPS: https://code.earthengine.google.com/8e62a5a66e4920e701813e43c0ecb83e
2) as a video: https://www.youtube.com/watch?v=rgqwvMRVguI&feature=youtu.be
"""
# ozone fills (i.e. one band per doy)
ozone_fills = ee.ImageCollection(
'users/samsammurphy/public/ozone_fill').toList(366)
# day of year index
jan01 = ee.Date.fromYMD(O3_date.get('year'), 1, 1)
doy_index = date.difference(jan01, 'day').toInt(
) # (NB. index is one less than doy, so no need to +1)
# day of year image
fill_image = ee.Image(ozone_fills.get(doy_index))
# return scalar fill value
return fill_image.reduceRegion(reducer=ee.Reducer.mean(),
geometry=centroid).get('ozone')
# O3 datetime in 24 hour intervals
O3_date = Atmospheric.round_date(date, 24)
# TOMS temporal gap
TOMS_gap = ee.DateRange('1994-11-01', '1996-08-01')
# avoid TOMS gap entirely
ozone = ee.Algorithms.If(TOMS_gap.contains(O3_date),
ozone_fill(centroid, O3_date),
ozone_measurement(centroid, O3_date))
# fix other data gaps (e.g. spatial, missing images, etc..)
ozone = ee.Algorithms.If(ozone, ozone, ozone_fill(centroid, O3_date))
#convert to Py6S units
ozone_Py6S_units = ee.Number(ozone).divide(
1000) # (i.e. Dobson units are milli-atm-cm )
return ozone_Py6S_units
def scriptEE(self, parameters):
print('Inicializando Conexao...')
ee.Initialize()
print('Conectado!')
ee.data.setDeadline(0)
sentinelTeste = ee.Image(
'COPERNICUS/S2/20180715T134211_20180715T134212_T21JYN').select(
['B4', 'B3', 'B2'])
dtInicio = str(parameters.anoInicial.year()) + "-" + str(
parameters.anoInicial.month()) + "-" + "01"
dtFim = str(parameters.anoFinal.year()) + "-" + str(
parameters.anoFinal.month()) + "-" + "01"
data = ee.DateRange(ee.Date(dtInicio),
ee.Date(dtFim).advance(1, 'month'))
ocoi = ee.Feature(
ee.FeatureCollection(
"users/odraitaipu/bracos/ocoi").union().first()).set(
'nome', 'Ocoí').geometry()
def mapMacrofitas(image):
ndvi = image.normalizedDifference().clip(ocoi)
soma = ndvi.reproject(
image.select('B4').projection()).reduceResolution(
ee.Reducer.sum())
porcentagem = soma.divide(ndvi).multiply(100).round()
clippedNdvi = ndvi.updateMask(ndvi.where(porcentagem.lte(99), 0))
output = ee.Image(0).where(clippedNdvi.gte(0.2), 1)
result = output.updateMask(output).rename('macrofitas').reproject(
image.select('B4').projection())
return result
def renderImage(imag):
img = ee.Image(imag)
'''
rgb visualize
minMax = img.reduceRegion(ee.Reducer.percentile([5,95]), ocoi, 100, None, None, True).values().sort()
visualise = img.visualize(['B4','B3','B2'], None, None, minMax.get(0), minMax.get(-1))
'''
minMax = img.reduceRegion(ee.Reducer.minMax(), ocoi, 30, None,
None, True)
visualize = img.visualize(
['macrofitas'], None, None, 1, minMax.get('macrofitas_max'),
None, None, [
'CE7E45', 'DF923D', 'F1B555', 'FCD163', '99B718', '74A901',
'66A000', '529400', '3E8601', '207401', '056201', '004C00',
'023B01', '012E01', '011D01', '011301'
])
mapa = visualize.getMapId()
mapid = mapa.get('mapid')
token = mapa.get('token')
url = 'type=xyz&url=https://earthengine.googleapis.com/map/' + mapid + '/{z}/{x}/{y}' + '?token=' + token
iface.addRasterLayer(url, 'macrofitas', 'wms')
print('Adicionado camada!')
if parameters.collection == 'Landsat 8':
collect = ee.ImageCollection('LANDSAT/LC08/C01/T1').filterDate(
data).filterBounds(ocoi).sort('system:time_start').set(
'nomeColecao', parameters.collection, 'satelite',
'LANDSAT/LC08/C01/T1_TOA').select(['B5', 'B4', 'B6'])
collectMacrofitas = ee.ImageCollection(collect.map(mapMacrofitas))
macrofitas = ee.Image(collectMacrofitas.sum())
# renderImage(macrofitas)
minMax = macrofitas.reduceRegion(ee.Reducer.minMax(), ocoi, 30,
None, None, True)
visualize = macrofitas.visualize(
['macrofitas'], None, None, 1, minMax.get('macrofitas_max'),
None, None, [
'CE7E45', 'DF923D', 'F1B555', 'FCD163', '99B718', '74A901',
'66A000', '529400', '3E8601', '207401', '056201', '004C00',
'023B01', '012E01', '011D01', '011301'
])
url = visualize.getDownloadURL({
'name':
'teste',
'scale':
30,
'region':
ee.String("[").cat(
ee.List(ocoi.convexHull().coordinates().get(0)).join(
",")).cat("]").getInfo()
})
print('Link: ')
print(url)
folder = "/home/laboratorio/.qgis2/python/plugins/earthengine3/downloads/"
name = "teste"
import requests
r = requests.get(url)
with open(folder + "teste.zip", "wb") as file:
file.write(r.content)
import zipfile
with zipfile.ZipFile(folder + "teste.zip", "r") as z:
z.extractall(folder)
iface.addRasterLayer(folder + "teste.vis-blue.tif",
"macrofitasBlue")
'''
def extract_date_range(cfg):
date_range = ee.DateRange(*cfg.SATELLITE_DATA.DATE_RANGE)
return date_range
def create_period_step(num_period):
return ee.DateRange(
ee.Date(t_start).advance(num_period, period), # take i-th period
ee.Date(t_start).advance(ee.Number(num_period).add(1),
period) # until i+1th period
)
def dfo(roi, began, ended, threshold, my_comp='3Day', get_max=False):
# Get rectangular bounds because it works faster than complex geometries.
# Clip to actual geometry at the end.
roi_bounds = roi.bounds()
# Get dates as ee.Date()
# The "Began" and "End" dates are taken in this case and buffered at the
# start and end by 2 days. This is so the first day of the flood, defined
# by the "Began" date is included in a 3-day composite with the images from
# two days prior. Then we move step-wise through the each day building
# composites for 2 or 3-day windows.
date_range = ee.DateRange(
ee.Date(began).advance(-2, "day"),
ee.Date(ended).advance(3, "day"))
def region_clip(img):
return img.clip(roi_bounds)
# STEP 2 - LOAD IMPORTANT MODIS DATA BASED ON DATES
# Collect Terra and Aqua satellites
terra = modis_toolbox.get_terra(roi_bounds, date_range).map(region_clip)
aqua = modis_toolbox.get_aqua(roi_bounds, date_range).map(region_clip)
# Apply Pan-sharpen function to aqua and terra images
terra_sharp = terra.map(modis_toolbox.pan_sharpen)
aqua_sharp = aqua.map(modis_toolbox.pan_sharpen)
# add NIR/RED ratio to all images
terra_ratio = terra_sharp.map(modis_toolbox.b1b2_ratio)
aqua_ratio = aqua_sharp.map(modis_toolbox.b1b2_ratio)
# Apply QA Band Extract to Terra & Aqua
terra_final = terra_ratio.map(modis_toolbox.add_qa_bands)
aqua_final = aqua_ratio.map(modis_toolbox.add_qa_bands)
# Finally, the Terra and Aqua products are combined into one image
# collection so they can be accessed in together in the DFO algorithm.
modis = ee.ImageCollection(terra_final.merge(aqua_final)\
.sort("system:time_start", True))
print "Collected and pre-processed MODIS Images"
# STEP 3 - APPLY THE DFO WATER DETECTION & COMPOSITING ALGORITHMS
# Okay - this is where it starts to get exciting. The portion of the code
# applies the DFO algorithm by first optimizing the thresholds applied to
# water detection, then applying those thresholds, and finally compositing
# images based on 2 or 3-day windows. The "otsu" mode uses bimodal histogram
# splitting on the bands to select thresholds, or the "standard" mode uses
# the static thresholds from DFO
# STEP 3.1 - SELECT thresholds
if threshold == "standard":
thresh_dict = {"b1b2": 0.70, "b7": 675.00, 'base_res': None}
elif threshold == "otsu":
# Apply the qa_mask to each modis image. We then make a composite across
# all the flood images to one image. This is done so to increase the
# sampling space as well as represent variation within the flood event
# itself. Clip to the roi to exclude ocean area in the sample.
modis_masked = modis.map(modis_toolbox.qa_mask)
sample_frame = modis_masked.median().clip(roi)
# Get a watermask that can be used to define strata for sampling
began_img = ee.Image().set({"Began": began})
strata = misc.get_jrc_yearly_perm(began, roi_bounds)\
.updateMask(sample_frame.select("red_250m").mask()).int8()\
.clip(roi)
# Otsu histrograms require a "bi-modal histogram". We need to constrain
# the reflectance range that can be used in the histogram as it may
# include high-reflectance features (e.g. missed clouds) that will make
# the histogram "multi-modal". Below are the steps to constrain the
# histograms into a reasonable range that one might expect water/ land
swir_mask = sample_frame.select("swir").gt(-500)\
.And(sample_frame.select("swir").lt(3000))
cleaned_swir = sample_frame.select("swir").updateMask(swir_mask)
# Put it all together in a final sample image
sample_img = sample_frame.addBands(strata)\
.addBands(cleaned_swir, overwrite=True)
# Collect Sample using stratifiedSample() function
sample_bands = ["b1b2_ratio", "swir", "jrc_perm_yearly"]
base_res = ee.Image(modis.first()).select("red_250m")\
.projection().nominalScale().multiply(1).getInfo()
base_res = round(base_res, 2)
sample = sample_img.select(sample_bands)\
.stratifiedSample(numPoints=2500,
classBand="jrc_perm_yearly",
region=roi, scale=base_res,
dropNulls=True)
# Convert the sample to a histogram
b1b2_hist = sample.reduceColumns(ee.Reducer.histogram(),
["b1b2_ratio"]).get("histogram")
swir_hist = sample.reduceColumns(ee.Reducer.histogram(),
["swir"]).get("histogram")
# Calculate histogram, run otsu, and collect into a dictionary
b1b2_thresh = otsu.get_threshold(b1b2_hist)
swir_thresh = otsu.get_threshold(swir_hist)
thresh_dict = {
'b1b2': b1b2_thresh.getInfo(),
'b7': swir_thresh.getInfo(),
'base_res': base_res
}
print "Calculated thresholds for Otsu: {0}".format(thresh_dict)
else:
raise ValueError("'threshold' options are 'standard' or 'otsu'")
# STEP 3.2 - APPLY THRESHOLDS TO MODIS IMAGES
# The following function cycles through each MODIS image prepared above
# and applies the thresholds to distinguish land from water. To do this
# Bands 1, 2, and 7 (Red, NIR, and SWIR) are used. Within the function a
# Band 2/ Band 1 ratio is defined. The thresholds are combined and where a
# pixel passes all three thresholds it is flagged as water.
def dfo_water_detection(modis_collection, thresh_b1b2, thresh_b7):
def water_flag(img):
# Apply thresholds to each ratio/ band
b1b2_ratio = ee.Image(img.select('b1b2_ratio'))
b1b2_sliced = b1b2_ratio.lt(
ee.Image.constant(thresh_b1b2)) # Band 1/Band 2
b1_sliced = img.select(["red_250m"],["b1_thresh"])\
.lt(ee.Image.constant(2027)) # Band 1 Threshold
b7_sliced = img.select(["swir"],["b7_thresh"])\
.lt(ee.Image.constant(thresh_b7)) # Band 7 Threshold
# Add all the thresholds to one image and then sum()
thresholds = b1b2_sliced.addBands(b1_sliced).addBands(b7_sliced)
thresholds_count = thresholds.reduce(ee.Reducer.sum())
# Apply water_flage threshold to final image
water_flag = thresholds_count.gte(ee.Image.constant(3))
return water_flag.copyProperties(img).set(
"system:time_start", img.get("system:time_start"))
# Apply the 'water_flag' function over the modis collection
dfo_water_collection = modis_collection.map(water_flag)
return dfo_water_collection.set({
'threshold_b1b2':
round(thresh_b1b2, 3),
'threshold_b7':
round(thresh_b7, 2),
'otsu_sample_res':
thresh_dict['base_res']
})
# The dfoWaterDetection() function is mapped over the MODIS collection
modis_dfo_water_detection = dfo_water_detection(modis, thresh_dict["b1b2"],
thresh_dict["b7"])
# STEP 3.2 - DFO COMPOSITES
# The following functions create 2 or 3-day composites. IMPORTANT:
# This is based on the variable defined above as "my_comp". This is
# done by using a join where a lag period is defined (in milliseconds)
# where images 2 or 3 days prior are joined to the current image as a
# property. This is later extracted in a function to access the images
# and create a composite.
def join_previous_days(left_collection, right_collection, lag_days):
filt = ee.Filter.And(
ee.Filter.maxDifference(1000 * 60 * 60 * 24 * lag_days,
leftField="system:time_start",
rightField="system:time_start"),
ee.Filter.greaterThanOrEquals(leftField="system:time_start",
rightField="system:time_start"))
return ee.Join.saveAll(matchesKey='dfo_images',
measureKey='delta_t',
ordering="system:time_start",
ascending=True).apply(left_collection,
right_collection, filt)
# The join_previous_days() function is then used to calculate "2Day" or
# "3Day" composites. The separate image collections can be accessed by
# defining the variable my_comp above.
lag_days = {"3Day": 2, "2Day": 1}
modis_join_previous = join_previous_days(modis_dfo_water_detection,
modis_dfo_water_detection,
lag_days[my_comp])
# The next function takes the join_previous_days results and combines the
# images in order to create the actual composite. This is done by
# accessing the images stored in the properties of each image (i.e. the
# images 1 or 2 days prior). Each composite has 2x the number of
# images as it does days since we use both Terra and Aqua. Where at
# least half of those days are flagged as water (i.e. 3 days for 3-day
# composites and 2-days for 2-day composites) a pixel is marked as
# water. This step help avoid marking cloud shadows, that move between
# images, as water. A common misclassificatin in these types of
# algorithms.
def dfo_flood_water(composite_collection, comp_days):
def apply_comp_day(image):
dfo_composite = ee.ImageCollection.fromImages(
image.get("dfo_images")).sum()
stable_water_thresh = dfo_composite.gte(comp_days)
return stable_water_thresh.select(
["sum"], ["flood_water"]).copyProperties(image).set(
{"system:time_start": image.get("system:time_start")})
stable_water_thresh = composite_collection.map(apply_comp_day)
return stable_water_thresh.set(
{"composite_type": ee.String(str(comp_days)).cat("Day")})
# If the began date is before Aqua started, change the critera for flooded pixels
# Since there will be half the images available
#
# Terra & Aqua (post 2002-07-04)
# DFO Threshold for flood water is 3 for 3-day composites and 2 for
# 2-day composites
#
# Terra Only (pre 2002-07-04)
# DFO Threshold for flood water is 2 for 3-day composites and 1 for
# 2-day composites.
if (ee.Date(began).difference(ee.Date("2002-07-04"), "day")).gte(0):
dfo_comp = {"3Day": 3, "2Day": 2}
elif (ee.Date(began).difference(ee.Date("2002-07-04"), "day")).lt(0):
dfo_comp = {"3Day": 2, "2Day": 1}
dfo_flood_coll = dfo_flood_water(modis_join_previous, dfo_comp[my_comp])
# STEP 3.3 COLLAPSE COMPOSITES INTO A FINAL FLOOD MAP
# The following function is the last step in the DFO algorithm. Here we
# take the resulting image collection of composites and collapse it into a
# final flood extent and flood frequency image. Flood extent is defined by
# all pixels that were identified as flood in any composite. Flood
# frequency is the number of times a pixel was flagged as a flood pixel.
def flood_extent_freq(img_coll):
freq = ee.ImageCollection(img_coll).sum().divide(
ee.Image.constant(2)).toUint16()
flooded = freq.gte(ee.Image.constant(1))
return ee.Image(
flooded.select(["flood_water"], ["flooded"]).addBands(
freq.select(["flood_water"],
["duration"])).copyProperties(img_coll))
dfo_flood_img = flood_extent_freq(dfo_flood_coll)
# STEP 3.4 CALCULATE CLEAR DAYS
# The following function takes an imageCollection that was previously run
# through qaBandExtract (i.e. the input band names match with those output
# by qaBandExtract) and returns an image that calculates the number of clear
# days for each pixel during the flood period.
def get_clear_views(img_coll):
def get_cloud_mask(img):
clouds = img.select("cloud_state").eq(0)
shadows = img.select("cloud_shadow").eq(0)
return clouds.add(shadows).gt(0)
clear_views = img_coll.map(get_cloud_mask)
number_clear_views = ee.Image(clear_views.sum()).select(
["cloud_state"], ["clear_views"]).toUint16()
def add_obs(img):
obs = img.select(["cloud_state"], ["observation"]).gte(0)
return img.addBands(obs)
observations = img_coll.map(add_obs)
total_obs = observations.select('observation').sum()
clear_perc = number_clear_views.divide(ee.Image(total_obs))\
.select(["clear_views"], ["clear_perc"])
return number_clear_views.addBands(clear_perc)
dfo_clear_days = get_clear_views(modis)
# STEP 3.4a ADD MAX IMG
# For the validation we want to use the image with the maximum flood extent.
# Calculate the maxImg with the function below that runs a reduceRegion() and
# then selects the image with the max value.
if get_max == True:
def get_max_img(img_coll):
# Function to calculate the flood extent of each image
def calc_extent(img):
img_extent = ee.Image(img).reduceRegion(
reducer=ee.Reducer.sum(),
geometry=roi,
scale=1000,
maxPixels=10e9)
return img.set({'extent': img_extent.get('flood_water')})
# Apply calcExtent() function to each image
extent = img_coll.map(calc_extent)
max_val = extent.aggregate_max('extent')
max_img = ee.Image(
extent.filterMetadata('extent', 'equals', max_val).first())
date = ee.Date(max_img.get('system:time_start'))
return max_img.select(['flood_water'],
['max_img']).set({'max_img_date': date})
max_img = get_max_img(dfo_flood_coll)
max_img_date = ee.Date(
max_img.get("max_img_date")).format('yyyy-MM-dd')
# STEP 3.5_TRUE: PREP FINAL IMAGES
# Add all the prepared bands together
dfo_final = ee.Image(dfo_flood_img).addBands([dfo_clear_days, max_img])\
.clip(roi)\
.set({"began": ee.Date(began).format("yyyy-MM-dd"),
"ended": ee.Date(ended).format("yyyy-MM-dd"),
"threshold_type": threshold,
"max_img_date": max_img_date})
elif get_max == False:
# STEP 3.5_FALSE: PREP FINAL IMAGES
# Add all the prepared bands together
dfo_final = ee.Image(dfo_flood_img).addBands(dfo_clear_days).clip(roi)\
.set({"began": ee.Date(began).format("yyyy-MM-dd"),
"ended": ee.Date(ended).format("yyyy-MM-dd"),
"threshold_type": threshold})
else:
raise ValueError("'max_img' options are 'True' or 'False'")
print "DFO Flood Dectection Complete"
return dfo_final
def make_drange(date):
date = ee.Date(date)
return ee.DateRange(date.advance(-date_range[0], date_range_unit),
date.advance(date_range[1], date_range_unit))
def getImageCollectionRange(imageCollection,image):
imageCollection = ee.ImageCollection(imageCollection)
image = ee.Image(image)
return imageCollection.map(lambda image: image.set('daterange',ee.DateRange(ee.Date(image.get('system:time_start')),ee.Date(image.get('system:time_end'))))) \
.filter(ee.Filter.dateRangeContains('daterange', ee.Date(image.get('system:time_start'))))
