def masker(image: Image):
cloudShadowBitMask = 1 << 3
cloudsBitMask = 1 << 5
qa: Image = image.select('pixel_qa')
mask = qa.bitwiseAnd(cloudShadowBitMask).eq(0).And(qa.bitwiseAnd(cloudsBitMask).eq(0))
return image.updateMask(mask)
def unit(img: ee.Image, epsilon: ee.Image):
t_max = img.reduce(ee.Reducer.max())
t_min = img.reduce(ee.Reducer.min())
united = img.subtract(t_min).divide((t_max.subtract(t_min)).add(epsilon))
return united
def cast_shadows(image: ee.Image, cloud: ee.Image) -> ee.Image:
"""
Calculate potential locations of cloud shadows.
Adapted from the 3_Sentinel2_CloudAndShadowMask notebook in:
https://github.com/rdandrimont/AGREE/
"""
# solar geometry (radians)
azimuth = (ee.Number(image.get("MEAN_SOLAR_AZIMUTH_ANGLE")).multiply(
np.pi / 180.0).add(0.5 * np.pi))
zenith = (ee.Number(0.5 * np.pi).subtract(
ee.Number(image.get("MEAN_SOLAR_ZENITH_ANGLE")).multiply(np.pi /
180.0)))
# find where cloud shadows should be based on solar geometry
nominalScale = cloud.projection().nominalScale()
def change_cloud_projection(cloudHeight):
shadowVector = zenith.tan().multiply(cloudHeight)
x = azimuth.cos().multiply(shadowVector).divide(nominalScale).round()
y = azimuth.sin().multiply(shadowVector).divide(nominalScale).round()
return cloud.changeProj(cloud.projection(),
cloud.projection().translate(x, y))
cloudHeights = ee.List.sequence(200, 10000, 500)
shadows = cloudHeights.map(change_cloud_projection)
shadows = ee.ImageCollection.fromImages(shadows).max()
# (modified by Sam Murphy) dark pixel detection
# B3 = green
# B12 = swir2
dark = image.normalizedDifference(["B3", "B12"]).gt(0.25)
shadows = shadows.And(dark)
return shadows
def addLayer(image: ee.Image,
visParams=None,
name=None,
shown=True,
opacity=1.0):
"""
Mimique addLayer GEE function
https://developers.google.com/earth-engine/api_docs#map.addlayer
Uses:
>>> from ee_plugin import Map
>>> Map.addLayer(.....)
"""
if not isinstance(image, ee.Image):
err_str = "\n\nThe image argument in 'addLayer' function must be a 'ee.Image' instance."
raise AttributeError(err_str)
if visParams:
image = image.visualize(**visParams)
if name is None:
# extract name from id
try:
name = json.loads(
image.id().serialize())["scope"][0][1]["arguments"]["id"]
except:
name = "untitled"
ee_plugin.utils.add_or_update_ee_image_layer(image, name, shown, opacity)
def add_map_layer(
self: folium.Map,
image: ee.Image,
vis_params: dict = None,
name: str = None,
show: bool = True,
opacity: float = 1,
min_zoom: int = 0,
):
"""Add an image layer to a :class:`folium.Map`."""
import folium
if vis_params is None:
vis_params = vis(image)
if name is None:
name = ', '.join(image.bandNames().getInfo())
attribution = ('Map Data © <a href="https://earthengine.google.com/">'
'Google Earth Engine</a>')
tiles = image.getMapId(vis_params)['tile_fetcher']
folium.raster_layers.TileLayer(
tiles=tiles.url_format,
attr=attribution,
name=name,
show=show,
opacity=opacity,
min_zoom=min_zoom,
overlay=True,
control=True,
).add_to(self)
def print_projection_info(cls, image: ee.Image ):
bandNames = image.bandNames();
print(f'Band names: {bandNames}')
b1proj = image.select('B1').projection()
print(f'Band 1 projection: {b1proj}' )
b1scale = image.select('B1').projection().nominalScale()
print(f'Band 1 scale: {b1scale}')
def addCloudAndShadowBand(cls, image: ee.Image) -> ee.Image:
qa = image.select('pixel_qa')
cloudBitMask = ee.Number(2).pow(5).int()
cloudShadowBitMask = ee.Number(2).pow(3).int()
cloud = qa.bitwiseAnd(cloudBitMask).neq(0)
cloudShadow = qa.bitwiseAnd(cloudShadowBitMask).neq(0)
mask = ee.Image(0).where(cloud.eq(1), 1).where(cloudShadow.eq(1), 1).rename('cloud_mask')
return image.addBands(mask)
def add_evi_ndvi(composite: ee.Image) -> ee.Image:
red = composite.select("B3")
blue = composite.select("B1")
infrared = composite.select("B4")
return composite.addBands(
infrared.subtract(red).multiply(2.5).divide(
infrared.add(red.multiply(6)).subtract(
blue.multiply(7.5)).add(1))).addBands(
composite.normalizedDifference(bandNames=["B4", "B3"]))
def cloud_mask(img: ee.Image) -> ee.Image:
img = ee.Image(img)
qa_band = img.select('QA')
# Bits 10 and 11 are clouds and cirrus, respectively
bit_mask_cloud = 1 << 10
bit_mask_cirrus = 1 << 11
no_clouds = qa_band.bitwiseAnd(bit_mask_cloud).eq(0).And(
qa_band.bitwiseAnd(bit_mask_cirrus).eq(0))
return img.updateMask(no_clouds)
def vis(img: ee.Image, bands: list = None, region: ee.Geometry = None) -> dict:
"""Return visualization parameters for the given image."""
region = region or img.geometry()
bands = bands or img.bandNames().getInfo()
reducer = ee.Reducer.percentile([1, 99], ['min', 'max'])
quantiles = img.reduceRegion(reducer, scale=90, bestEffort=True).getInfo()
return {
'min': [quantiles[b + '_min'] for b in bands],
'max': [quantiles[b + '_max'] for b in bands],
}
def indices(cls, image: ee.Image) -> ee.Image:
bands = ee.Image([
cls.mndwi(image),
cls.mbsr(image),
cls.ndvi(image),
cls.awesh(image),
image.select("B"),
image.select("NIR"),
image.select("SWIR1"),
image.select("SWIR2"),
image.select("cloud_mask")
])
return bands.set('system:time_start', image.get('system:time_start'))
def add_latlon(img: ee.Image) -> ee.Image:
'''Creates a new ee.Image with 2 added bands of longitude and latitude
coordinates named 'LON' and 'LAT', respectively
'''
latlon = ee.Image.pixelLonLat().select(
opt_selectors=['longitude', 'latitude'], opt_names=['LON', 'LAT'])
return img.addBands(latlon)
def edge_effect(img: ee.Image) -> ee.Image:
ones = ee.Image(1)
resolutionOut = 30
# Set some constants
distance = min(255 * resolutionOut, 10000)
halfLife = 500
HMeeMax = ee.Image(0)
seq = arange(0.1, 1.05, 0.05)
for i in seq:
HMt = img.gt(i)
# dt: filter out small "salt" patches
# JA: keeping this at 90 for now
HMt = HMt.focal_mean(90, "circle", "meters")
HMt = HMt.gt(0.5)
# dt: calculate Euc distance away from 1s
HMtdistance = HMt.distance(ee.Kernel.euclidean(distance, "meters"),
False)
HMee = (ee.Image(i).multiply(
ee.Image(0.5).pow(HMtdistance.divide(
ee.Image(halfLife)))).unmask(0))
HMeeMax = HMeeMax.max(HMee)
return ones.subtract(ones.subtract(HMeeMax).multiply(ones.subtract(img)))
def rename_l8(img: ee.Image) -> ee.Image:
'''
Args
- img: ee.Image, Landsat 8 image
Returns
- img: ee.Image, with bands renamed
See: https://developers.google.com/earth-engine/datasets/catalog/LANDSAT_LC08_C01_T1_SR
Name Scale Factor Description
B1 0.0001 Band 1 (Ultra Blue) surface reflectance, 0.435-0.451 um
B2 0.0001 Band 2 (Blue) surface reflectance, 0.452-0.512 um
B3 0.0001 Band 3 (Green) surface reflectance, 0.533-0.590 um
B4 0.0001 Band 4 (Red) surface reflectance, 0.636-0.673 um
B5 0.0001 Band 5 (Near Infrared) surface reflectance, 0.851-0.879 um
B6 0.0001 Band 6 (Shortwave Infrared 1) surface reflectance, 1.566-1.651 um
B7 0.0001 Band 7 (Shortwave Infrared 2) surface reflectance, 2.107-2.294 um
B10 0.1 Band 10 brightness temperature (Kelvin), 10.60-11.19 um
B11 0.1 Band 11 brightness temperature (Kelvin), 11.50-12.51 um
sr_aerosol Aerosol attributes, see Aerosol QA table
pixel_qa Pixel quality attributes, see Pixel QA table
radsat_qa Radiometric saturation QA, see Radsat QA table
'''
newnames = [
'AEROS', 'BLUE', 'GREEN', 'RED', 'NIR', 'SWIR1', 'SWIR2', 'TEMP1',
'TEMP2', 'sr_aerosol', 'pixel_qa', 'radsat_qa'
]
return img.rename(newnames)
def rename_l57(img: ee.Image) -> ee.Image:
'''
Args
- img: ee.Image, Landsat 5/7 image
Returns
- img: ee.Image, with bands renamed
See: https://developers.google.com/earth-engine/datasets/catalog/LANDSAT_LT05_C01_T1_SR
https://developers.google.com/earth-engine/datasets/catalog/LANDSAT_LE07_C01_T1_SR
Name Scale Factor Description
B1 0.0001 Band 1 (blue) surface reflectance, 0.45-0.52 um
B2 0.0001 Band 2 (green) surface reflectance, 0.52-0.60 um
B3 0.0001 Band 3 (red) surface reflectance, 0.63-0.69 um
B4 0.0001 Band 4 (near infrared) surface reflectance, 0.77-0.90 um
B5 0.0001 Band 5 (shortwave infrared 1) surface reflectance, 1.55-1.75 um
B6 0.1 Band 6 brightness temperature (Kelvin), 10.40-12.50 um
B7 0.0001 Band 7 (shortwave infrared 2) surface reflectance, 2.08-2.35 um
sr_atmos_opacity 0.001 Atmospheric opacity; < 0.1 = clear; 0.1 - 0.3 = average; > 0.3 = hazy
sr_cloud_qa Cloud quality attributes, see SR Cloud QA table. Note:
pixel_qa is likely to present more accurate results
than sr_cloud_qa for cloud masking. See page 14 in
the LEDAPS product guide.
pixel_qa Pixel quality attributes generated from the CFMASK algorithm,
see Pixel QA table
radsat_qa Radiometric saturation QA, see Radiometric Saturation QA table
'''
newnames = [
'BLUE', 'GREEN', 'RED', 'NIR', 'SWIR1', 'TEMP1', 'SWIR2',
'sr_atmos_opacity', 'sr_cloud_qa', 'pixel_qa', 'radsat_qa'
]
return img.rename(newnames)
def export(
image: ee.Image,
region: ee.Geometry,
filename: str,
drive_folder: str,
monitor: bool = False,
) -> ee.batch.Export:
task = ee.batch.Export.image(
image.clip(region),
filename,
{
"scale": 10,
"region": region,
"maxPixels": 1e13,
"driveFolder": drive_folder
},
)
try:
task.start()
except ee.ee_exception.EEException as e:
print(f"Task not started! Got exception {e}")
return task
if monitor:
monitor_task(task)
return task
def _joinFilteredETI(self, et, i):
time_filter = EEFilter.equals(leftField="system:time_start",
rightField="system:time_start")
join = ee.Join.inner()
joinCollETI = EEImageCollection(join.apply(et, i, time_filter))
return joinCollETI.map(lambda element: EEImage.cat(
element.get('primary'), element.get('secondary'))).sort(
'system:time_start')
def maxvalue(x: ee.Image, scale: Optional[float] = None) -> float:
"""Get a maximum value.
Returns the maximum value of an ee.Image. The return value
will be an approximation if the polygon (x.geometry())
contains too many pixels at the native scale.
Args:
x: An ee.Image.
scale: A nominal scale in meters of the projection
to work in. Defaults image x$geometry()$projection()$nominalScale().
Returns:
An float number describing the x (ee.Image) maximum value.
Examples:
>>> import ee
>>> import ee_extra
>>> ee.Initialize()
>>> img = ee.Image.random()
>>> maxvalue(img)
"""
if scale is None:
scale = x.geometry().projection().nominalScale()
# Create a clean geometry i.e. geodesic = FALSE
img_geom_local = x.geometry().getInfo()["coordinates"]
ee_geom = ee.Geometry.Polygon(
coords=img_geom_local,
proj="EPSG:4326",
evenOdd=True,
maxError=1.0,
geodesic=False,
)
# get max values
maxval = ee.Image.reduceRegion(
image=x,
reducer=ee.Reducer.max(),
scale=scale,
geometry=ee_geom,
bestEffort=True,
).getInfo()
return maxval
def extract_deforestation_events(LT_segments: ee.Image, start_year: int,
end_year: int,
dsnr_threshold: float) -> ee.Image:
"""
Given an image with information about loss segments, assumed to be the
output of get_segment_data, bounds on start and end year, and a threshold
on the disturbance signal-to-noise ratio (DSNR), filter to only events
within the years of interest which pass the DSNR threshold.
Note: the decision to extract the most recent event, rather than the
selecting the largest or using a different selection mechanism, is made
because this was written with the goal of identifying deforestation events
followed by a recovery period.
Parameters
----------
LT_segments: ee.Image
Landtrendr segment data, assumed to be the output of get_segment_data
start_year: int
The first year to consider for deforestation events.
end_year: int
The last year to consider for deforestation events.
dsnr_threshold: float
The threshold on the disturbance signal-to-noise ratio (DSNR) for the
deforestation segments.
Returns
-------
ee.Image
The input image filtered to the most recent event in each pixel which
passes the given threshold.
"""
start_years = LT_segments.arraySlice(0, 0, 1)
end_years = LT_segments.arraySlice(0, 1, 2)
dsnr = LT_segments.arraySlice(0, 7, 8)
mask = (start_years.gte(ee.Image(start_year)).And(
end_years.lte(ee.Image(end_year))).And(
dsnr.gte(ee.Image(dsnr_threshold))))
masked_segments = LT_segments.arrayMask(mask)
# Extract the most recent segments for each pixel
# factor of -1 to flip delta, since arraySort is ascending
sort_by = masked_segments.arraySlice(0, 0, 1).toArray(0).multiply(-1)
segments_sorted = masked_segments.arraySort(sort_by)
return segments_sorted.arraySlice(1, 0, 1)
def addLayer(image: ee.Image,
visParams=None,
name=None,
shown=True,
opacity=1.0):
"""
Adds a given EE object to the map as a layer.
https://developers.google.com/earth-engine/api_docs#map.addlayer
Uses:
>>> from ee_plugin import Map
>>> Map.addLayer(.....)
"""
if not isinstance(image, ee.Image) and not isinstance(
image, ee.FeatureCollection) and not isinstance(
image, ee.Feature) and not isinstance(image, ee.Geometry):
err_str = "\n\nThe image argument in 'addLayer' function must be an instace of one of ee.Image, ee.Geometry, ee.Feature or ee.FeatureCollection."
raise AttributeError(err_str)
if isinstance(image, ee.Geometry) or isinstance(
image, ee.Feature) or isinstance(image, ee.FeatureCollection):
features = ee.FeatureCollection(image)
color = '000000'
if visParams and 'color' in visParams:
color = visParams['color']
image = features.style(**{'color': color})
else:
if isinstance(image, ee.Image) and visParams:
image = image.visualize(**visParams)
if name is None:
# extract name from id
try:
name = json.loads(
image.id().serialize())["scope"][0][1]["arguments"]["id"]
except:
name = "untitled"
ee_plugin.utils.add_or_update_ee_image_layer(image, name, shown, opacity)
def extract_latitude_longitude_pixel(image: ee.Image, geometry: ee.Geometry, bands: list, scale: int = 30, tile_scale: int = 16):
# extract pixels lat and lons
coordinates = image.addBands(image.pixelLonLat()).select(['longitude', 'latitude']+bands)
coordinates = coordinates.reduceRegion(reducer=ee.Reducer.toList(), geometry=geometry, scale=scale, bestEffort=False, tileScale=tile_scale)
# add bands
band_values = []
for band in bands:
band_values.append(np.array(ee.List(coordinates.get(band)).getInfo(), dtype=np.float64))
# build results
band_values = np.array(band_values)
result = np.zeros(shape=(2+band_values.shape[0], band_values.shape[1]))
result[0] = np.array(ee.List(coordinates.get('longitude')).getInfo(), dtype=np.float64)
result[1] = np.array(ee.List(coordinates.get('latitude')).getInfo(), dtype=np.float64)
for i, band_value in enumerate(band_values):
result[i+2] = band_value
# result
return np.stack(result, axis=1)
def rescale_l8(img: ee.Image) -> ee.Image:
'''
Args
- img: ee.Image, Landsat 8 image, with bands already renamed
by rename_l8()
Returns
- img: ee.Image, with bands rescaled
'''
opt = img.select(
['AEROS', 'BLUE', 'GREEN', 'RED', 'NIR', 'SWIR1', 'SWIR2'])
therm = img.select(['TEMP1', 'TEMP2'])
masks = img.select(['sr_aerosol', 'pixel_qa', 'radsat_qa'])
opt = opt.multiply(0.0001)
therm = therm.multiply(0.1)
scaled = ee.Image.cat([opt, therm, masks]).copyProperties(img)
# system properties are not copied
scaled = scaled.set('system:time_start', img.get('system:time_start'))
return scaled
def rescale_l57(img: ee.Image) -> ee.Image:
'''
Args
- img: ee.Image, Landsat 5/7 image, with bands already renamed
by rename_157()
Returns
- img: ee.Image, with bands rescaled
'''
opt = img.select(['BLUE', 'GREEN', 'RED', 'NIR', 'SWIR1', 'SWIR2'])
atmos = img.select(['sr_atmos_opacity'])
therm = img.select(['TEMP1'])
masks = img.select(['sr_cloud_qa', 'pixel_qa', 'radsat_qa'])
opt = opt.multiply(0.0001)
atmos = atmos.multiply(0.001)
therm = therm.multiply(0.1)
scaled = ee.Image.cat([opt, therm, masks, atmos]).copyProperties(img)
# system properties are not copied
scaled = scaled.set('system:time_start', img.get('system:time_start'))
return scaled
def mask_qaclear(img: ee.Image) -> ee.Image:
'''
Args
- img: ee.Image
Returns
- img: ee.Image, input image with cloud-shadow, snow, cloud, and unclear
pixels masked out
'''
qam = decode_qamask(img)
cloudshadow_mask = qam.select('pxqa_cloudshadow')
snow_mask = qam.select('pxqa_snow')
cloud_mask = qam.select('pxqa_cloud')
return img.updateMask(cloudshadow_mask).updateMask(snow_mask).updateMask(
cloud_mask)
def addLayer(image: ee.Image, visParams=None, name='untitled', shown=True, opacity=1.0):
"""
Mimique addLayer GEE function
Uses:
>>> from ee_plugin import Map
>>> Map.addLayer(.....)
"""
if not isinstance(image, ee.Image):
err_str = "\n\nThe image argument in 'addLayer' function must be a 'ee.Image' instance."
raise AttributeError(err_str)
if visParams:
image = image.visualize(**visParams)
ee_plugin.utils.add_or_update_ee_image_layer(image, name, shown, opacity)
def get_array_patches(img: ee.Image,
scale: Numeric,
ksize: Numeric,
points: ee.FeatureCollection,
export: str,
prefix: str,
fname: str,
selectors: Optional[ee.List] = None,
dropselectors: Optional[ee.List] = None,
bucket: Optional[str] = None) -> ee.batch.Task:
'''Creates and starts a task to export square image patches in TFRecord
format to Google Drive or Google Cloud Storage (GCS). The image patches are
sampled from the given ee.Image at specific coordinates.
Args
- img: ee.Image, image covering the entire region of interest
- scale: int or float, scale in meters of the projection to sample in
- ksize: int or float, radius of square image patch
- points: ee.FeatureCollection, coordinates from which to sample patches
- export: str, 'drive' for Google Drive, 'gcs' for GCS
- prefix: str, folder name in Drive or GCS to export to, no trailing '/'
- fname: str, filename for export
- selectors: None or ee.List, names of properties to include in output,
set to None to include all properties
- dropselectors: None or ee.List, names of properties to exclude
- bucket: None or str, name of GCS bucket, only used if export=='gcs'
Returns: ee.batch.Task
'''
kern = ee.Kernel.square(radius=ksize, units='pixels')
patches_array = img.neighborhoodToArray(kern)
# ee.Image.sampleRegions() does not cut it for larger collections,
# using mapped sample instead
samples = points.map(lambda pt: sample_patch(pt, patches_array, scale))
# export to a TFRecord file which can be loaded directly in TensorFlow
return tfexporter(collection=samples,
export=export,
prefix=prefix,
fname=fname,
selectors=selectors,
dropselectors=dropselectors,
bucket=bucket)
def sample_patch(point: ee.Feature, patches_array: ee.Image,
scale: Numeric) -> ee.Feature:
'''Extracts an image patch at a specific point.
Args
- point: ee.Feature
- patches_array: ee.Image, Array Image
- scale: int or float, scale in meters of the projection to sample in
Returns: ee.Feature, 1 property per band from the input image
'''
arrays_samples = patches_array.sample(region=point.geometry(),
scale=scale,
projection='EPSG:3857',
factor=None,
numPixels=None,
dropNulls=False,
tileScale=12)
return arrays_samples.first().copyProperties(point)
def decode_qamask(img: ee.Image) -> ee.Image:
'''
Args
- img: ee.Image, Landsat 5/7/8 image containing 'pixel_qa' band
Returns
- masks: ee.Image, contains 5 bands of masks
Pixel QA Bit Flags (universal across Landsat 5/7/8)
Bit Attribute
0 Fill
1 Clear
2 Water
3 Cloud Shadow
4 Snow
5 Cloud
'''
qa = img.select('pixel_qa')
clear = qa.bitwiseAnd(2).neq(0) # 0 = not clear, 1 = clear
clear = clear.updateMask(clear).rename(['pxqa_clear'])
water = qa.bitwiseAnd(4).neq(0) # 0 = not water, 1 = water
water = water.updateMask(water).rename(['pxqa_water'])
cloud_shadow = qa.bitwiseAnd(8).eq(0) # 0 = shadow, 1 = not shadow
cloud_shadow = cloud_shadow.updateMask(cloud_shadow).rename(
['pxqa_cloudshadow'])
snow = qa.bitwiseAnd(16).eq(0) # 0 = snow, 1 = not snow
snow = snow.updateMask(snow).rename(['pxqa_snow'])
cloud = qa.bitwiseAnd(32).eq(0) # 0 = cloud, 1 = not cloud
cloud = cloud.updateMask(cloud).rename(['pxqa_cloud'])
masks = ee.Image.cat([clear, water, cloud_shadow, snow, cloud])
return masks
def addStats(img: ee.Image) -> ee.Image:
patch = ee.Geometry(img.get('patch'))
patch_area = patch.area(0.001)
img_footprint = ee.Algorithms.GeometryConstructors.Polygon(
ee.Geometry(img.get('system:footprint')).coordinates())
intersection = patch.intersection(img_footprint, ee.ErrorMargin(0.001))
intersection_area = intersection.area(0.001)
coverage = ee.Number(intersection_area).divide(patch_area)
cloud_area_img = img.select('clouds').multiply(ee.Image.pixelArea())
stats = cloud_area_img.reduceRegion(reducer=ee.Reducer.sum(),
geometry=patch,
scale=10,
maxPixels=1e12)
cloud_area = stats.get('clouds')
cloud_coverage = ee.Number(cloud_area).divide(patch_area)
img = img.set('patchCoverage', coverage)
img = img.set('patchCloudCoverage', cloud_coverage)
score = coverage.multiply(cloud_coverage)
img = img.set('patchScore', score)
return img
def normalize_mapper(img: ee.Image):
return img.subtract(min_value).divide(max_value - min_value).clamp(
0, 1).copyProperties(img)