def test_generate_mask(self):
band = 'B1'
mask = cloud_mask.sentinel2(cloud_S2)
cld_score = self.score.generate_score(mask, band)
vis = cld_score.visualize(
min=0, max=1, palette=['b9936c', 'dac292', 'e6e2d3', 'c4b7a6'])
region = cloud_S2.geometry().centroid().buffer(5000)
url = vis.getThumbUrl({'region': tools.getRegion(region)})
print('\nCloud Distance Score: {}\n'.format(url))
self.assertEqual(cld_score.getInfo()['type'], 'Image')
def apply_mask_cloud(image_coll, collection_name, cloudy_pix_flag):
"""
Different input_collections need different steps to be taken to handle
cloudy image. The first step is to reject images that more than X%
cloudy pixels (here X=5). The next step is to mask cloudy pixels. This
will hopefully mean that when we take the median of the ImageCollection,
we ignore cloudy pixels.
Parameters
----------
image_coll : ee.ImageCollection
The ImageCollection of images from which we want to remove cloud.
collection_name : str
Name of the collection so that we can apply collection specific
masking.
cloud_pix_flag : str
Name of the flag which details the fraction of cloudy pixels in each
image.
Returns
----------
image_coll
Image collection with very cloudy images removed, and masked images
containing a tolerable amount of cloud.
"""
# construct cloud mask if availible
if collection_name == 'COPERNICUS/S2':
mask_func = cloud_mask.sentinel2()
elif collection_name == 'LANDSAT/LC08/C01/T1_SR':
mask_func = cloud_mask.landsat8SRPixelQA()
elif (collection_name == 'LANDSAT/LE07/C01/T1_SR'
or collection_name == 'LANDSAT/LT05/C01/T1_SR'
or collection_name == 'LANDSAT/LT04/C01/T1_SR'):
mask_func = cloud_mask.landsat457SRPixelQA()
else:
print("No cloud mask logic defined for input collection {}"\
.format(collection_name))
return image_coll
# images with more than this percent of cloud pixels are removed
cloud_pix_frac = 50
# remove images that have more than `cloud_pix_frac`% cloudy pixels
if cloudy_pix_flag != 'None':
image_coll = image_coll.filter(
ee.Filter.lt(cloudy_pix_flag, cloud_pix_frac))
# apply per pixel cloud mask
image_coll = image_coll.map(mask_func)
return image_coll
def test_with_zeros(self):
col = Collection.Sentinel2()
score = MaskPercent()
image = cloud_mask.sentinel2()(cloud_S2)
bounds = image.geometry().buffer(-50000)
computed = score.map(col, bounds)(image)
maskprop = computed.get(score.name).getInfo()
maskband = tools.get_value(computed,
image.geometry().centroid(), 10,
'client')[score.name]
print(maskband, maskprop)
self.assertEqual(maskband, maskprop)
def test(self):
iid = 'COPERNICUS/S2'
collection = ee.ImageCollection(iid)
date = '2017-03-07'
p_cloud = ee.Geometry.Point([-65.84304, -24.82382])
p_clear = ee.Geometry.Point([-65.88415, -24.82608])
image = getimage(collection, date, area)
masked = cloud_mask.sentinel2()(image)
vals = get_value(masked, p_cloud, 30, 'client')
show(iid, image, masked, bandsS2, 0, 5000)
self.assertEqual(vals["B1"], None)
def mask_clouds(ee_img):
mask_all = cloud_mask.sentinel2()
masked_ee_img = mask_all(ee_img)
return masked_ee_img
def test_cirrus():
masked = cloud_mask.sentinel2(['cirrus'])(image)
vals = getValue(masked, p_cirrus, 30)
assert vals.get("B1").getInfo() == None
def test_cloud_mask(self):
from geetools import cloud_mask
# LANDSAT 4 TOA
masked_l4toa = cloud_mask.fmask("fmask")(self.l4toa)
p_l4toa = ee.Geometry.Point([-72.2736, -44.6062])
vals_l4toa = tools.get_value(masked_l4toa, p_l4toa, 30)
self.assertEqual(vals_l4toa["B1"], None)
# LANDSAT 5 USGS
p_l5usgs = ee.Geometry.Point([-65.4991, -24.534])
masked_l5usgs = cloud_mask.fmask("cfmask")(self.l5SR)
vals_l5usgs = tools.get_value(masked_l5usgs, p_l5usgs, 30)
self.assertEqual(vals_l5usgs["B1"], None)
# LANDSAT 5 LEDAPS
p_l5led = ee.Geometry.Point([-70.3455, -43.8306])
masked_l5led = cloud_mask.ledaps(self.l5led)
vals_l5led = tools.get_value(masked_l5led, p_l5led, 30)
self.assertEqual(vals_l5led["B1"], None)
# LANDSAT 5 TOA
masked_l5toa = cloud_mask.fmask("fmask")(self.l5toa)
p_l5toa = ee.Geometry.Point([-70.9003, -39.7421])
vals_l5toa = tools.get_value(masked_l5toa, p_l5toa, 30)
self.assertEqual(vals_l5toa["B1"], None)
# LANDSAT 7 USGS
p_l7usgs = ee.Geometry.Point([-64.8533, -26.1052])
masked_l7usgs = cloud_mask.fmask("cfmask")(self.l7SR)
vals_l7usgs = tools.get_value(masked_l7usgs, p_l7usgs, 30)
self.assertEqual(vals_l7usgs["B1"], None)
# LANDSAT 7 LEDAPS
p_l7led = ee.Geometry.Point([-64.9141, -25.2264])
masked_l7led = cloud_mask.ledaps(self.l7led)
vals_l7led = tools.get_value(masked_l7led, p_l7led, 30)
self.assertEqual(vals_l7led["B1"], None)
# LANDSAT 7 TOA
masked_l7toa = cloud_mask.fmask("fmask")(self.l7toa)
p_l7toa = ee.Geometry.Point([-64.8495, -25.2354])
vals_l7toa = tools.get_value(masked_l7toa, p_l7toa, 30)
self.assertEqual(vals_l7toa["B1"], None)
# LANDSAT 8 USGS
p_l8usgs = ee.Geometry.Point([-65.5568, -24.3327])
masked_l8usgs = cloud_mask.fmask("cfmask")(self.l8SR)
vals_l8usgs = tools.get_value(masked_l8usgs, p_l8usgs, 30)
self.assertEqual(vals_l8usgs["B1"], None)
# LANDSAT 8 TOA
masked_l8toa = cloud_mask.fmask("fmask")(self.l8toa)
p_l8toa = ee.Geometry.Point([-71.7833, -43.6634])
vals_l8toa = tools.get_value(masked_l8toa, p_l8toa, 30)
self.assertEqual(vals_l8toa["B1"], None)
# LANDSAT SENTINEL 2
masked_s2 = cloud_mask.sentinel2(self.sentinel2)
p_s2 = ee.Geometry.Point([-72.2104, -42.7592])
vals_s2 = tools.get_value(masked_s2, p_s2, 10)
self.assertEqual(vals_s2["B1"], None)
# LANDSAT MODIS AQ
masked_ma = cloud_mask.modis(self.modis_aqua)
p_ma = ee.Geometry.Point([-69.071, -43.755])
vals_ma = tools.get_value(masked_ma, p_ma, 500)
self.assertEqual(vals_ma["sur_refl_b01"], None)
# LANDSAT MODIS TE
masked_te = cloud_mask.modis(self.modis_terra)
p_te = ee.Geometry.Point([-71.993, -43.692])
vals_te = tools.get_value(masked_te, p_te, 500)
self.assertEqual(vals_te["sur_refl_b01"], None)
import ee
import numpy as np
import geetools
from geetools import ui, cloud_mask
import os, datetime
import pandas as pd
import itertools
import tensorflow as tf
cloud_mask_landsatSR = cloud_mask.landsatSR()
cloud_mask_sentinel2 = cloud_mask.sentinel2()
# tfrecord helper functions ----------------------------------------------------
# https://stackoverflow.com/questions/52324515/passing-multiple-inputs-to-keras-model-from-tf-dataset-api
# https://www.tensorflow.org/tutorials/load_data/tfrecord
def _bytes_feature(value):
"""Returns a bytes_list from a string / byte."""
# If the value is an eager tensor BytesList won't unpack a string from an EagerTensor.
if isinstance(value, type(tf.constant(0))):
value = value.numpy()
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def _float_feature(value):
"""Returns a float_list from a float / double."""
return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
def _int64_feature(value):
def Sentinel2(cls):
escalables = ["B2", "B3", "B4", "B8", "B11", "B12"]
bandscale = dict(B1=60,
B2=10,
B3=10,
B4=10,
B5=20,
B6=20,
B7=20,
B8=10,
B8a=20,
B9=60,
B10=60,
B11=20,
B12=20)
obj = cls(
BLUE="B2",
GREEN="B3",
RED="B4",
NIR="B8",
SWIR="B11",
SWIR2="B12",
to_scale=escalables,
process="TOA",
clouds_fld="CLOUD_COVERAGE_ASSESSMENT",
max=10000,
fclouds=cld.sentinel2(),
scale=10,
bandscale=bandscale,
bandmask="B2",
family="Sentinel",
ini=2015,
short="S2",
#col_id=13
)
obj.threshold = {
'NIR': {
'min': 700,
'max': 4500
},
'RED': {
'min': 50,
'max': 2000
},
'SWIR': {
'min': 400,
'max': 3500
},
'SWIR2': {
'min': 150,
'max': 2800
},
'BLUE': {
'min': 0,
'max': 1000
},
'GREEN': {
'min': 100,
'max': 1500
}
}
# obj.ID = "COPERNICUS/S2"
obj.ID = IDS[obj.short]
return obj
