def main():
#os.system("earthengine authenticate")
ee.Initialize()
now = time.time()
# We will export the information for each kml (Trigo, Girasol, Barbecho, etc.)
for kml in kmls:
table = ee.FeatureCollection(kml[1])
if buffer_value < 0:
table = getBuffer(table, buffer_value)
# Get radar
for orbit in orbits:
for sentinel in sentinels:
nameOutputFile = kml[0] + "_" + sentinel + "_" + orbit
# Check if the file already exists
if not os.path.exists(
os.path.join("dataEE", nameOutputFile + ".csv")):
# Load the imagery with the current feature collection
s1 = loadSentinel1(table, start_date, end_date, sentinel,
orbit)
# Export the table (sentinel-1)
if natural_data == 1:
print("Descargando datos en valores naturales")
elif natural_data == 0:
print("Descargando datos en decibelios")
else:
print(
"Error en el valor de la variable 'natural_data'. Abortando..."
)
exit()
tableSeries = getTimeSeriesTask(s1[natural_data], table)
#s1[0] = db; s1[1] = natural
datasetTask = exportTableSeries(tableSeries,
nameOutputFile,
indexes_sentinel1)
# Start the task.
datasetTask.start()
while datasetTask.active():
print('Descargando datos de %s...' % (nameOutputFile))
time.sleep(30)
print('Terminado.')
else:
print("Fichero %s ya descargado." % (nameOutputFile))
# Check if we have to download sentinel 2 indexes
if indexes_sentinel2:
nameOutputFile = kml[0] + "_s2"
# Check if the file already exists
if not os.path.exists(
os.path.join("dataEE", nameOutputFile + ".csv")):
# Get Sentinel-2
# Load the imagery with the current feature collection
s2 = loadSentinel2(table, start_date, end_date)
# Export the table (sentinel-2)
tableSeries_s2 = getTimeSeriesTask(s2, table)
datasetTask_s2 = exportTableSeries(tableSeries_s2,
kml[0] + "_s2",
indexes_sentinel2)
# Start the task.
datasetTask_s2.start()
while datasetTask_s2.active():
print('Descargando datos de %s...' % (kml[0] + "_s2"))
time.sleep(30)
print('Terminado.')
else:
print("Fichero %s ya descargado." % (nameOutputFile))
end = time.time()
elapsed = end - now
time_convert(elapsed)
import ee
import sys
from oauth2client.service_account import ServiceAccountCredentials
from aoi import Aoi
from image_spec import ImageSpec
credentials = ServiceAccountCredentials.from_p12_keyfile(
service_account_email=sys.argv[1],
filename=sys.argv[2],
private_key_password='******',
scopes=ee.oauth.SCOPE + ' https://www.googleapis.com/auth/drive ')
ee.Initialize(credentials)
'min':7000, 'max':10000, 'size': '800', 'region': maliau_region Good luck! Cheers, Andrew Cotham [email protected] ''' import ee,oauth2client.client, config, datetime, sys, json, math from IPython.core.display import Image ee.Initialize(ee.ServiceAccountCredentials(config.MY_SERVICE_ACCOUNT, config.MY_PRIVATE_KEY_FILE)) maliau_image = ee.Image("LC81170572013170LGN00") maliau_region = [[116.676364283,4.95945957299],[116.676364283,4.64623224023],[117.100028398,4.64623224023],[117.100028398,4.95945957299]] uncorrected_image = ee.Image.cat([maliau_image.expression("b('B4')+770"),maliau_image.select('B3'),maliau_image.expression("b('B2')-1230")]); uncorrected_thumbnail = uncorrected_image.getThumbUrl({ 'bands': 'B4,B3,B2', 'min':7000, 'max':10000, 'size': '800', 'region': maliau_region }) Image(url=uncorrected_thumbnail) # <rawcell> # Get the Digital Terrain Model and the image metadata to help produce the Illumination Condition
def check_images_available(inputs):
"""
Create the structure of subfolders for each satellite mission
KV WRL 2018
Arguments:
-----------
inputs: dict
inputs dictionnary
Returns:
-----------
im_dict_T1: list of dict
list of images in Tier 1 and Level-1C
im_dict_T2: list of dict
list of images in Tier 2 (Landsat only)
"""
# check if EE was initialised or not
try:
ee.ImageCollection('LANDSAT/LT05/C01/T1_TOA')
except:
ee.Initialize()
print('Images available between %s and %s:' %
(inputs['dates'][0], inputs['dates'][1]),
end='\n')
# check how many images are available in Tier 1 and Sentinel Level-1C
col_names_T1 = {
'L5': 'LANDSAT/LT05/C01/T1_TOA',
'L7': 'LANDSAT/LE07/C01/T1_TOA',
'L8': 'LANDSAT/LC08/C01/T1_TOA',
'S2': 'COPERNICUS/S2'
}
print('- In Landsat Tier 1 & Sentinel-2 Level-1C:')
im_dict_T1 = dict([])
sum_img = 0
for satname in inputs['sat_list']:
# get list of images in EE collection
while True:
try:
ee_col = ee.ImageCollection(col_names_T1[satname])
col = ee_col.filterBounds(ee.Geometry.Polygon(inputs['polygon']))\
.filterDate(inputs['dates'][0],inputs['dates'][1])
im_list = col.getInfo().get('features')
break
except:
continue
# remove very cloudy images (>95% cloud cover)
im_list_upt = remove_cloudy_images(im_list, satname)
sum_img = sum_img + len(im_list_upt)
print(' %s: %d images' % (satname, len(im_list_upt)))
im_dict_T1[satname] = im_list_upt
print(' Total: %d images' % sum_img)
# in only S2 is in sat_list, stop here
if len(inputs['sat_list']) == 1 and inputs['sat_list'][0] == 'S2':
return im_dict_T1, []
# otherwise check how many images are available in Landsat Tier 2
col_names_T2 = {
'L5': 'LANDSAT/LT05/C01/T2_TOA',
'L7': 'LANDSAT/LE07/C01/T2_TOA',
'L8': 'LANDSAT/LC08/C01/T2_TOA'
}
print('- In Landsat Tier 2:', end='\n')
im_dict_T2 = dict([])
sum_img = 0
for satname in inputs['sat_list']:
if satname == 'S2': continue
# get list of images in EE collection
while True:
try:
ee_col = ee.ImageCollection(col_names_T2[satname])
col = ee_col.filterBounds(ee.Geometry.Polygon(inputs['polygon']))\
.filterDate(inputs['dates'][0],inputs['dates'][1])
im_list = col.getInfo().get('features')
break
except:
continue
# remove very cloudy images (>95% cloud cover)
im_list_upt = remove_cloudy_images(im_list, satname)
sum_img = sum_img + len(im_list_upt)
print(' %s: %d images' % (satname, len(im_list_upt)))
im_dict_T2[satname] = im_list_upt
print(' Total: %d images' % sum_img)
return im_dict_T1, im_dict_T2
def download_data(R:RegionST, times, products, bands, path_save, scale=10, max_cloud_fraction=None,
use_least_cloudy=None):
ee.Initialize()
path_save.mkdir(exist_ok=True, parents=True)
if not ((path_save/f'download.{bands[0]}.tif').is_file() and
(path_save/f'download.{bands[1]}.tif').is_file() and
(path_save/f'download.{bands[2]}.tif').is_file()):
sR = [R] if R.shape[0] <= 32 else split_region(R, size=32, cls=RegionST)
fsaves = []
#for j, R in tqdm(enumerate(sR), total=len(sR)):
for j, R in enumerate(sR):
region = (f"[[{R.bbox.left}, {R.bbox.bottom}], [{R.bbox.right}, {R.bbox.bottom}], " +
f"[{R.bbox.right}, {R.bbox.top}], [{R.bbox.left}, {R.bbox.top}]]")
if not ((path_save/f'download.{bands[0]}_{j}.tif').is_file() and
(path_save/f'download.{bands[1]}_{j}.tif').is_file() and
(path_save/f'download.{bands[2]}_{j}.tif').is_file()):
# Merge products to single image collection
imCol = ee.ImageCollection(products[0])
for i in range(1, len(products)):
imCol = imCol.merge(ee.ImageCollection(products[i]))
imCol = filter_region(imCol, R, times=times, bands=bands)
if max_cloud_fraction is not None:
imCol = filter_cloudy(imCol, max_cloud_fraction=max_cloud_fraction)
if use_least_cloudy is not None:
imCol = n_least_cloudy(imCol, n=use_least_cloudy)
im = imCol.median()
imCol = ee.ImageCollection([im])
colList = imCol.toList(imCol.size())
# info = colList.getInfo()
# data_times = [pd.to_datetime(o['properties']['system:time_start'], unit='ms') for o in info]
# data_cloudy = [o['properties']['CLOUDY_PIXEL_PERCENTAGE'] for o in info]
# Download each image
for i in range(colList.size().getInfo()):
image = ee.Image(colList.get(i))
fname = 'download'
#fname = image.get('system:id').getInfo().split('/')[-1]
fnames_full = [f'{fname}.{b}.tif' for b in bands]
fnames_partial0 = [f'{fname}.{b}_{j}.tif' for b in bands]
fnames_full = all([(path_save/f).is_file() for f in fnames_full])
fnames_partial = all([(path_save/f).is_file() for f in fnames_partial0])
if not fnames_full:
fsaves.append([path_save/f for f in fnames_partial0])
if not fnames_partial:
zip_error = True
for i in range(10): # Try 10 times
if zip_error:
try:
url = image.getDownloadURL(
{'scale': scale, 'crs': 'EPSG:4326',
'region': f'{region}'})
r = requests.get(url)
with open(str(path_save/'data.zip'), 'wb') as f:
f.write(r.content)
with zipfile.ZipFile(str(path_save/'data.zip'), 'r') as f:
files = f.namelist()
f.extractall(str(path_save))
os.remove(str(path_save/'data.zip'))
zip_error = False
except:
zip_error = True
os.remove(str(path_save/'data.zip'))
time.sleep(10)
if zip_error: raise Exception(f'Failed to process {url}')
for f in files:
f = path_save/f
os.rename(str(f), str(path_save/f'{f.stem}_{j}{f.suffix}'))
# Merge files
suffix = '.tif'
files = path_save.ls(include=[suffix])
#files = np.unique(fsaves)
files = [o.stem for o in files]
ref = np.unique(['_'.join(o.split('_')[:-1])
for o in files if len(o.split('_')[-1]) < 6])
ids = np.unique([int(o.split('_')[-1])
for o in files if len(o.split('_')[-1]) < 6])
#file_groups = [[path_save/f'{r}_{i}{suffix}' for i in ids] for r in ref]
file_groups = [[path_save/f'{r}_{i}{suffix}' for i in ids
if f'{r}_{i}' in files] for r in ref]
for fs in file_groups:
if len(fs) < 500:
fsave = '_'.join(fs[0].stem.split('_')[:-1]) + suffix
merge_tifs(fs, fsave, delete=True)
else:
fs_break = np.array(fs)[:(len(fs)//500)*500].reshape(len(fs)//500,-1).tolist()
if len(fs[(len(fs)//500)*500:]) > 0:
fs_break.append(fs[(len(fs)//500)*500:])
for fsi, fs2 in enumerate(fs_break):
fsave = '_'.join(fs2[0].stem.split('_')[:-1]) + f'_break{fsi}' + suffix
merge_tifs(fs2, fsave, delete=True)
files = path_save.ls(include=[suffix, '_break'])
files = [o.stem for o in files]
ref = np.unique(['_'.join(o.split('_')[:-1])
for o in files if len(o.split('_')[-1]) < 11])
ids = np.unique([o.split('_')[-1]
for o in files if len(o.split('_')[-1]) < 11])
#file_groups = [[path_save/f'{r}_{i}{suffix}' for i in ids] for r in ref]
file_groups = [[path_save/f'{r}_{i}{suffix}' for i in ids
if f'{r}_{i}' in files] for r in ref]
for fs in file_groups:
fsave = '_'.join(fs[0].stem.split('_')[:-1]) + suffix
merge_tifs(fs, fsave, delete=True)
def testDynamicClasses(self):
"""Verifies dynamic class initialization."""
# Use a custom set of known functions.
def MockSend(path, unused_params, unused_method=None, unused_raw=None):
if path == '/algorithms':
return {
'Array': {
'type':
'Algorithm',
'args': [{
'name': 'values',
'type': 'Serializable',
'description': ''
}],
'description':
'',
'returns':
'Array'
},
'Array.cos': {
'type':
'Algorithm',
'args': [{
'type': 'Array',
'description': '',
'name': 'input'
}],
'description':
'',
'returns':
'Array'
},
'Kernel.circle': {
'returns':
'Kernel',
'args': [{
'type': 'float',
'description': '',
'name': 'radius',
}, {
'default': 1.0,
'type': 'float',
'optional': True,
'description': '',
'name': 'scale'
}, {
'default': True,
'type': 'boolean',
'optional': True,
'description': '',
'name': 'normalize'
}],
'type':
'Algorithm',
'description':
''
},
'Reducer.mean': {
'returns': 'Reducer',
'args': []
},
'fakeFunction': {
'returns':
'Array',
'args': [{
'type': 'Reducer',
'description': '',
'name': 'kernel',
}]
}
}
ee.data.send_ = MockSend
ee.Initialize(None)
# Verify that the expected classes got generated.
self.assertTrue(hasattr(ee, 'Array'))
self.assertTrue(hasattr(ee, 'Kernel'))
self.assertTrue(hasattr(ee.Array, 'cos'))
self.assertTrue(hasattr(ee.Kernel, 'circle'))
# Try out the constructors.
kernel = ee.ApiFunction('Kernel.circle').call(1, 2)
self.assertEquals(kernel, ee.Kernel.circle(1, 2))
array = ee.ApiFunction('Array').call([1, 2])
self.assertEquals(array, ee.Array([1, 2]))
self.assertEquals(array, ee.Array(ee.Array([1, 2])))
# Try out the member function.
self.assertEquals(
ee.ApiFunction('Array.cos').call(array),
ee.Array([1, 2]).cos())
# Test argument promotion.
f1 = ee.ApiFunction('Array.cos').call([1, 2])
f2 = ee.ApiFunction('Array.cos').call(ee.Array([1, 2]))
self.assertEquals(f1, f2)
self.assertTrue(isinstance(f1, ee.Array))
f3 = ee.call('fakeFunction', 'mean')
f4 = ee.call('fakeFunction', ee.Reducer.mean())
self.assertEquals(f3, f4)
try:
ee.call('fakeFunction', 'moo')
self.fail()
except ee.EEException as e:
self.assertTrue('Unknown algorithm: Reducer.moo' in str(e))
def testCallAndApply(self):
"""Verifies library initialization."""
# Use a custom set of known functions.
def MockSend(path, params, unused_method=None, unused_raw=None):
if path == '/algorithms':
return {
'fakeFunction': {
'type':
'Algorithm',
'args': [{
'name': 'image1',
'type': 'Image'
}, {
'name': 'image2',
'type': 'Image'
}],
'returns':
'Image'
},
'Image.constant':
apitestcase.BUILTIN_FUNCTIONS['Image.constant']
}
else:
raise Exception('Unexpected API call to %s with %s' %
(path, params))
ee.data.send_ = MockSend
ee.Initialize(None)
image1 = ee.Image(1)
image2 = ee.Image(2)
expected = ee.Image(
ee.ComputedObject(ee.ApiFunction.lookup('fakeFunction'), {
'image1': image1,
'image2': image2
}))
applied_with_images = ee.apply('fakeFunction', {
'image1': image1,
'image2': image2
})
self.assertEquals(expected, applied_with_images)
applied_with_numbers = ee.apply('fakeFunction', {
'image1': 1,
'image2': 2
})
self.assertEquals(expected, applied_with_numbers)
called_with_numbers = ee.call('fakeFunction', 1, 2)
self.assertEquals(expected, called_with_numbers)
# Test call and apply() with a custom function.
sig = {'returns': 'Image', 'args': [{'name': 'foo', 'type': 'Image'}]}
func = ee.CustomFunction(sig,
lambda foo: ee.call('fakeFunction', 42, foo))
expected_custom_function_call = ee.Image(
ee.ComputedObject(func, {'foo': ee.Image(13)}))
self.assertEquals(expected_custom_function_call, ee.call(func, 13))
self.assertEquals(expected_custom_function_call,
ee.apply(func, {'foo': 13}))
# Test None promotion.
called_with_null = ee.call('fakeFunction', None, 1)
self.assertEquals(None, called_with_null.args['image1'])
import ee
import itertools
from gee_functions.classification import create_features, create_training_areas, classify_irrigated_areas
from gee_functions.constants import GEE_USER_PATH
if __name__ == '__main__':
ee.Initialize() # Initialize the Google Earth Engine
# Load the feature collection containing the area of interest
# aoi = ee.FeatureCollection(f'{GEE_USER_PATH}/vector/outline/outline_region_de_murcia')
aoi = ee.FeatureCollection(f'{GEE_USER_PATH}/vector/outline/outline_cdc_3857')
aoi_coordinates = aoi.geometry().bounds().getInfo()['coordinates']
# Dictionary containing the date ranges for each year, will be used to select sat. imagery
years = {
'88': ('1987-01-01', '1989-01-01'),
'97': ('1996-01-01', '1998-01-01'),
'00': ('1999-01-01', '2001-01-01'),
'05': ('2004-01-01', '2006-01-01'),
'09': ('2008-01-01', '2010-01-01'),
}
stats = ['median', 'min', 'max']
stats_combos = list(itertools.combinations(stats, 3))
# stats_combos = list(itertools.combinations(stats, 3)) + \
# list(itertools.combinations(stats, 2)) + \
# list(itertools.combinations(stats, 3))
def __init__(self):
self.initialize = ee.Initialize()
self.credentials = ee.Credentials()
self.service = discovery.build(serviceName='drive', version='v3', cache_discovery=False, credentials=self.credentials)
def extr_L8_ts_GEE(lon, lat, bufferSize=20):
ee.Reset()
ee.Initialize()
def createAvg(image):
gee_roi = ee.Geometry.Point(lon, lat).buffer(bufferSize)
# aerosols
image = image.updateMask(image.select('sr_aerosol').eq(2).bitwiseOr(
image.select('sr_aerosol').eq(32).bitwiseOr(
image.select('sr_aerosol').eq(96).bitwiseOr(
image.select('sr_aerosol').eq(160).bitwiseOr(
image.select('sr_aerosol').eq(66).bitwiseOr(
image.select('sr_aerosol').eq(130)
))))))
# clouds
def getQABits(image, start, end, newName):
pattern = 0
for i in range(start, end + 1):
pattern = pattern + int(math.pow(2, i))
return image.select([0], [newName]).bitwiseAnd(pattern).rightShift(start)
def cloud_shadows(image):
QA = image.select('pixel_qa')
return getQABits(QA, 3, 3, 'Cloud_shadows').eq(0)
def clouds(image):
QA = image.select('pixel_qa')
return getQABits(QA, 5, 5, 'Cloud').eq(0)
image = image.updateMask(cloud_shadows(image))
image = image.updateMask(clouds(image))
# # radiometric saturation
# image = image.updateMask(image.select('radsat_qa').eq(2))
reduced_img_data = image.reduceRegion(ee.Reducer.mean(), gee_roi, 30)
return ee.Feature(None, {'result': reduced_img_data})
def setresample(image):
image = image.resample()
return (image)
def mask_tree_cover(image):
tree_cover_image = ee.ImageCollection("GLCF/GLS_TCC").filterBounds(gee_roi).filter(
ee.Filter.eq('year', 2010)).mosaic()
treemask = tree_cover_image.select('tree_canopy_cover').clip(gee_roi).lte(20)
# load lc
glbcvr = ee.Image("ESA/GLOBCOVER_L4_200901_200912_V2_3").select('landcover')
# mask water
watermask = glbcvr.neq(210)
return image.updateMask(treemask.And(watermask))
# load collection
gee_l8_collection = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR') # .map(setresample)
# filter collection
gee_roi = ee.Geometry.Point(lon, lat).buffer(bufferSize)
gee_l8_fltd = gee_l8_collection.filterBounds(gee_roi)
# extract time series
#gee_l8_fltd = gee_l8_fltd.map(mask_tree_cover)
gee_l8_mpd = gee_l8_fltd.map(createAvg)
tmp = gee_l8_mpd.getInfo()
b1 = np.array([x['properties']['result']['B1'] for x in tmp['features']], dtype=np.float)
b2 = np.array([x['properties']['result']['B2'] for x in tmp['features']], dtype=np.float)
b3 = np.array([x['properties']['result']['B3'] for x in tmp['features']], dtype=np.float)
b4 = np.array([x['properties']['result']['B4'] for x in tmp['features']], dtype=np.float)
b5 = np.array([x['properties']['result']['B5'] for x in tmp['features']], dtype=np.float)
b6 = np.array([x['properties']['result']['B6'] for x in tmp['features']], dtype=np.float)
b7 = np.array([x['properties']['result']['B7'] for x in tmp['features']], dtype=np.float)
b10 = np.array([x['properties']['result']['B7'] for x in tmp['features']], dtype=np.float)
b11 = np.array([x['properties']['result']['B7'] for x in tmp['features']], dtype=np.float)
ge_dates = np.array([datetime.strptime(x['id'][12::], '%Y%m%d') for x in tmp['features']])
valid = np.where(np.isfinite(b2))
# cut out invalid values
b1 = b1[valid]
b2 = b2[valid]
b3 = b3[valid]
b4 = b4[valid]
b5 = b5[valid]
b6 = b6[valid]
b7 = b7[valid]
b10 = b10[valid]
b11 = b11[valid]
ge_dates = ge_dates[valid]
if b1.size == 0:
return None
else:
return pd.DataFrame({'B1': b1,
'B2': b2,
'B3': b3,
'B4': b4,
'B5': b5,
'B6': b6,
'B7': b7,
'B10': b10,
'B11': b11}, index=ge_dates)
def extr_SIG0_LIA_ts_GEE(lon, lat,
bufferSize=20,
maskwinter=False,
lcmask=False,
globcover_mask=False,
trackflt=None,
masksnow=False,
varmask=False,
ssmcor=None,
dual_pol=True,
desc=False,
tempfilter=False,
returnLIA=False,
datesonly=False,
datefilter=None,
S1B=False,
treemask=False,
radcor=False):
ee.Initialize()
def mask_lc(image):
tmp = ee.Image(image)
# load land cover info
corine = ee.Image('users/felixgreifeneder/corine')
# create lc mask
valLClist = [10, 11, 12, 13, 18, 19, 20, 21, 26, 27, 28, 29]
lcmask = corine.eq(valLClist[0]).bitwiseOr(corine.eq(valLClist[1])) \
.bitwiseOr(corine.eq(valLClist[2])) \
.bitwiseOr(corine.eq(valLClist[3])) \
.bitwiseOr(corine.eq(valLClist[4])) \
.bitwiseOr(corine.eq(valLClist[5])) \
.bitwiseOr(corine.eq(valLClist[6])) \
.bitwiseOr(corine.eq(valLClist[7])) \
.bitwiseOr(corine.eq(valLClist[8])) \
.bitwiseOr(corine.eq(valLClist[9])) \
.bitwiseOr(corine.eq(valLClist[10])) \
.bitwiseOr(corine.eq(valLClist[11]))
tmp = tmp.updateMask(lcmask)
return tmp
def mask_lc_globcover(image):
tmp = ee.Image(image)
# load lc
glbcvr = ee.Image("ESA/GLOBCOVER_L4_200901_200912_V2_3").select('landcover')
valLClist = [11, 14, 20, 30, 120, 140, 150]
lcmask = glbcvr.eq(valLClist[0]) \
.bitwiseOr(glbcvr.eq(valLClist[1])) \
.bitwiseOr(glbcvr.eq(valLClist[2])) \
.bitwiseOr(glbcvr.eq(valLClist[3])) \
.bitwiseOr(glbcvr.eq(valLClist[4])) \
.bitwiseOr(glbcvr.eq(valLClist[5])) \
.bitwiseOr(glbcvr.eq(valLClist[6]))
tmp = tmp.updateMask(lcmask)
return tmp
def mask_tree_cover(image):
copernicus_collection = ee.ImageCollection('COPERNICUS/Landcover/100m/Proba-V/Global')
copernicus_image = ee.Image(copernicus_collection.toList(1000).get(0))
treemask = copernicus_image.select('tree-coverfraction').clip(gee_roi).lte(20)
# load lc
glbcvr = ee.Image("ESA/GLOBCOVER_L4_200901_200912_V2_3").select('landcover')
# mask water
watermask = copernicus_image.select('discrete_classification').neq(80)
return image.updateMask(treemask.And(watermask))
def addRefSM(image):
tmp = ee.Image(image)
img_date = ee.Date(tmp.get('system:time_start'))
RefSMtmp = RefSMcollection.filterDate(img_date.format('Y-M-d'))
current_ssm = ee.ImageCollection(RefSMtmp).toList(10).get(0)
out_image = tmp.addBands(ee.Image(current_ssm))
return (out_image)
def s1_simplyfy_date(image):
return (image.set('system:time_start', ee.Date(ee.Date(image.get('system:time_start')).format('Y-M-d'))))
def applyCorrelationMask(image):
mask = ssm_vv_cor.select('correlation').gt(0.1)
return (image.updateMask(mask))
import timeit
tic = timeit.default_timer()
# load S1 data
gee_s1_collection = ee.ImageCollection('COPERNICUS/S1_GRD') # .map(setresample)
# filter collection
gee_roi = ee.Geometry.Point(lon, lat).buffer(bufferSize)
gee_s1_filtered = gee_s1_collection.filter(ee.Filter.eq('instrumentMode', 'IW')) \
.filterBounds(gee_roi) \
.filter(ee.Filter.listContains('transmitterReceiverPolarisation',
'VV'))
if S1B == False:
gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.eq('platform_number', 'A'))
if datefilter is not None:
gee_s1_filtered = gee_s1_filtered.filterDate(datefilter[0], datefilter[1])
if desc == False:
# Select only acquisition from ascending tracks
gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.eq('orbitProperties_pass', 'ASCENDING'))
# else:
# gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.eq('orbitProperties_pass', 'DESCENDING'))
if dual_pol == True:
# select only acquisitions with VV AND VH
gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.listContains('transmitterReceiverPolarisation', 'VH'))
if maskwinter == True:
# Mask winter based on the DOY
gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.dayOfYear(121, 304))
if trackflt is not None:
# Select only data from a specific S1 track
if isinstance(trackflt, list):
gee_s1_filtered = gee_s1_filtered.filter(
ee.Filter.inList(ee.List(trackflt), 'relativeOrbitNumber_start'))
else:
gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.eq('relativeOrbitNumber_start', trackflt))
if lcmask == True:
# Mask pixels based on Corine land-cover
gee_s1_filtered = gee_s1_filtered.map(mask_lc)
if globcover_mask == True:
# Mask pixels based on the Globcover land-cover classification
gee_s1_filtered = gee_s1_filtered.map(mask_lc_globcover)
if treemask == True:
gee_s1_filtered = gee_s1_filtered.map(mask_tree_cover)
if ssmcor is not None:
# Mask pixels with a low correlation toe coarse resolution soil moisture
# Mostly relevant if aggregating over a larger area
RefSMlist = list()
ssmcor = ssmcor.resample('D').mean().dropna()
ssmcor = ssmcor.astype(np.float)
for i in range(len(ssmcor)):
ssm_img = ee.Image(ssmcor[i]).clip(gee_roi).float()
ssm_img = ssm_img.set('system:time_start', ssmcor.index[i])
RefSMlist.append(ssm_img)
RefSMcollection = ee.ImageCollection(RefSMlist)
# prepare the join
s1_joined = gee_s1_filtered.map(s1_simplyfy_date)
join_filter = ee.Filter.equals(leftField='system:time_start', rightField='system:time_start')
simple_join = ee.Join.simple()
s1_joined = simple_join.apply(s1_joined, RefSMcollection, join_filter)
# create ssm reference SM, image collection
s1_plus_RefSM = ee.ImageCollection(s1_joined.map(addRefSM, True))
ssm_vv_cor = s1_plus_RefSM.select(['VV', 'constant']).reduce(ee.Reducer.pearsonsCorrelation())
gee_s1_filtered = gee_s1_filtered.map(applyCorrelationMask)
# get the track numbers
tmp = gee_s1_filtered.getInfo()
track_series = np.array([x['properties']['relativeOrbitNumber_start'] for x in tmp['features']])
dir_series = np.array([x['properties']['orbitProperties_pass'] for x in tmp['features']])
available_tracks, uidx = np.unique(track_series, return_index=True)
available_directions = dir_series[uidx]
# create dict with orbit directions
available_directions = pd.Series(np.where(available_directions == 'ASCENDING', 1, 0), index=available_tracks)
print('Extracting data from ' + str(len(available_tracks)) + ' Sentinel-1 tracks...')
print(available_tracks)
out_dict = {}
lgths = list()
for track_nr in available_tracks:
if datesonly == False:
out_dict[str(int(track_nr))] = _s1_track_ts(lon, lat, bufferSize,
gee_s1_filtered,
track_nr,
dual_pol,
varmask,
returnLIA,
masksnow,
tempfilter,
datesonly,
radcor)
else:
tmp_dates = _s1_track_ts(lon, lat, bufferSize,
gee_s1_filtered,
track_nr,
dual_pol,
varmask,
returnLIA,
masksnow,
tempfilter,
datesonly,
radcor)
lgths.append(tmp_dates)
toc = timeit.default_timer()
if datesonly == True:
return np.array(lgths)
print('Time-series extraction finished in ' + "{:10.2f}".format(toc - tic) + 'seconds')
return out_dict, available_directions
begin = self.request.get('begin')
end = self.request.get('end')
content = GetWetnessTimeSeries(aoi, begin, end)
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(content))
self.response.headers['Content-Type'] = 'application/json'
self.response.headers['Content-Disposition'] = 'attachment; filename="data.json"'
class RefreshAccessToken(webapp2.RequestHandler):
def post(self):
self.redirect('/')
app = webapp2.WSGIApplication([
('/get_aoi_image_info_time_series', GetImageInfoHandler),
('/get_ndwi_time_series', GetWetnessTimeSeriesHandler),
# html pages in current directory
('/(.*\.html)', MainPageHandler),
('/', MainPageHandler)
], debug=True)
# Initialize the EE API.
# Use our App Engine service account's credentials.
EE_CREDENTIALS = ee.ServiceAccountCredentials(config.EE_ACCOUNT, config.EE_PRIVATE_KEY_FILE)
ee.Initialize(EE_CREDENTIALS)
def main():
# Reading data
data = pd.read_csv(input_file, sep='\t', usecols=[column_latitude, column_longitude])
data = data.values
for satellite_name in satellites:
for day in dates:
for dam in range(data.shape[0]):
ee.Initialize()
if satellite_name == "sentinel":
# Use these bands for prediction.
bands = ['B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'B8', 'B8A', 'B9', 'B10', 'B11', 'B12']
# Use Sentinel 2 surface reflectance data.
satellite = ee.ImageCollection("COPERNICUS/S2")
flag_clouds = 'CLOUDY_PIXEL_PERCENTAGE'
def maskS2clouds(image):
cloudShadowBitMask = ee.Number(2).pow(3).int()
cloudsBitMask = ee.Number(2).pow(5).int()
qa = image.select('QA60')
mask = qa.bitwiseAnd(cloudShadowBitMask).eq(0).And(
qa.bitwiseAnd(cloudsBitMask).eq(0))
return image.updateMask(mask).select(bands).divide(10000)
mask = maskS2clouds
elif satellite_name == "landsat8":
# Use these bands for prediction.
bands = ['B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'B8', 'B9', 'B10', 'B11']
# Use Sentinel 2 surface reflectance data.
satellite = ee.ImageCollection("LANDSAT/LC08/C01/T1_TOA")
flag_clouds = 'CLOUD_COVER'
# Cloud masking function.
def maskL8sr(image):
cloudsBitMask = ee.Number(2).pow(4).int()
qa = image.select('BQA')
mask = qa.bitwiseAnd(cloudsBitMask).eq(0)
return image.updateMask(mask).select(bands).divide(10000)
mask = maskL8sr
else:
raise NotImplementedError
# Logitude, Latitude
x, y = degree_conv(data[dam][1]), degree_conv(data[dam][0])
# x, y = float(data[dam][1].replace(",", ".")), float(data[dam][0].replace(",", "."))
# if not_dam:
# # North
# try:
# send_not_dam_task(x, y, bands, satellite, flag_clouds, mask, data, "N", satellite_name)
# except:
# _print("Error in {0}{1:0=3d}_{2}".format("N", dam + 1, day[0][:4]))
# try:
# _print("Trying image {0}{1:0=3d}_{2} again".format("N", dam + 1, day[0][:4]))
# send_not_dam_task(x, y, bands, satellite, flag_clouds, mask, data, "N", satellite_name)
# except:
# _print("Image {0}{1:0=3d}_{2} could not be downloaded".format("N", dam + 1, day[0][:4]))
#
# # South
# try:
# send_not_dam_task(x, y, bands, satellite, flag_clouds, mask, data, "S", satellite_name)
# except:
# _print("Error in {0}{1:0=3d}_{2}".format("S", dam + 1, day[0][:4]))
# try:
# _print("Trying image {0}{1:0=3d}_{2} again".format("S", dam + 1, day[0][:4]))
# send_not_dam_task(x, y, bands, satellite, flag_clouds, mask, data, "S", satellite_name)
# except:
# _print("Image {0}{1:0=3d}_{2} could not be downloaded".format("S", dam + 1, day[0][:4]))
#
# else:
# Dam
try:
send_task(satellite_name, satellite, bands, x, y, day, flag_clouds, mask, dam)
except:
_print("Error in {0:0=3d}_{1}".format(dam + 1, day[0][:4]))
try:
_print("Trying image {0:0=3d}_{1} again".format(dam + 1, day[0][:4]))
send_task(satellite_name, satellite, bands, x, y, day, flag_clouds, mask, dam)
except:
_print("Image {0:0=3d}_{1} could not be downloaded".format(dam + 1, day[0][:4]))
def main(ini_path=None,
overwrite_flag=False,
delay=0,
key=None,
cron_flag=False,
reverse_flag=False):
"""Compute daily Tcorr images
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files if the export dates are the same and
generate new images (but with different export dates) even if the tile
lists are the same. The default is False.
delay : float, optional
Delay time between each export task (the default is 0).
key : str, optional
File path to an Earth Engine json key file (the default is None).
cron_flag : bool, optional
If True, only compute Tcorr daily image if existing image does not have
all available image (using the 'wrs2_tiles' property) and limit the
date range to the last 64 days (~2 months).
reverse_flag : bool, optional
If True, process dates in reverse order.
"""
logging.info('\nCompute daily Tcorr images')
ini = utils.read_ini(ini_path)
model_name = 'SSEBOP'
# model_name = ini['INPUTS']['et_model'].upper()
if (ini[model_name]['tmax_source'].upper() == 'CIMIS'
and ini['INPUTS']['end_date'] < '2003-10-01'):
logging.error(
'\nCIMIS is not currently available before 2003-10-01, exiting\n')
sys.exit()
elif (ini[model_name]['tmax_source'].upper() == 'DAYMET'
and ini['INPUTS']['end_date'] > '2017-12-31'):
logging.warning('\nDAYMET is not currently available past 2017-12-31, '
'using median Tmax values\n')
# sys.exit()
# elif (ini[model_name]['tmax_source'].upper() == 'TOPOWX' and
# ini['INPUTS']['end_date'] > '2017-12-31'):
# logging.warning(
# '\nDAYMET is not currently available past 2017-12-31, '
# 'using median Tmax values\n')
# # sys.exit()
logging.info('\nInitializing Earth Engine')
if key:
logging.info(' Using service account key file: {}'.format(key))
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('deadbeef', key_file=key))
else:
ee.Initialize()
# Output Tcorr daily image collection
tcorr_daily_coll_id = '{}/{}_daily'.format(
ini['EXPORT']['export_coll'], ini[model_name]['tmax_source'].lower())
# Get a Tmax image to set the Tcorr values to
logging.debug('\nTmax properties')
tmax_name = ini[model_name]['tmax_source']
tmax_source = tmax_name.split('_', 1)[0]
tmax_version = tmax_name.split('_', 1)[1]
tmax_coll_id = 'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower())
tmax_coll = ee.ImageCollection(tmax_coll_id)
tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0)
logging.debug(' Collection: {}'.format(tmax_coll_id))
logging.debug(' Source: {}'.format(tmax_source))
logging.debug(' Version: {}'.format(tmax_version))
logging.debug('\nExport properties')
export_geo = ee.Image(tmax_mask).projection().getInfo()['transform']
export_crs = ee.Image(tmax_mask).projection().getInfo()['crs']
export_shape = ee.Image(tmax_mask).getInfo()['bands'][0]['dimensions']
export_extent = [
export_geo[2], export_geo[5] + export_shape[1] * export_geo[4],
export_geo[2] + export_shape[0] * export_geo[0], export_geo[5]
]
logging.debug(' CRS: {}'.format(export_crs))
logging.debug(' Extent: {}'.format(export_extent))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
# # Limit export to a user defined study area or geometry?
# export_geom = ee.Geometry.Rectangle(
# [-125, 24, -65, 50], proj='EPSG:4326', geodesic=False) # CONUS
# export_geom = ee.Geometry.Rectangle(
# [-124, 35, -119, 42], proj='EPSG:4326', geodesic=False) # California
# If cell_size parameter is set in the INI,
# adjust the output cellsize and recompute the transform and shape
try:
export_cs = float(ini['EXPORT']['cell_size'])
export_shape = [
int(math.ceil(abs((export_shape[0] * export_geo[0]) / export_cs))),
int(math.ceil(abs((export_shape[1] * export_geo[4]) / export_cs)))
]
export_geo = [
export_cs, 0.0, export_geo[2], 0.0, -export_cs, export_geo[5]
]
logging.debug(' Custom export cell size: {}'.format(export_cs))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
except KeyError:
pass
# Get current asset list
if ini['EXPORT']['export_dest'].upper() == 'ASSET':
logging.debug('\nGetting asset list')
# DEADBEEF - daily is hardcoded in the asset_id for now
asset_list = utils.get_ee_assets(tcorr_daily_coll_id)
else:
raise ValueError('invalid export destination: {}'.format(
ini['EXPORT']['export_dest']))
# Get current running tasks
tasks = utils.get_ee_tasks()
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
logging.debug(' Tasks: {}\n'.format(len(tasks)))
input('ENTER')
collections = [x.strip() for x in ini['INPUTS']['collections'].split(',')]
# Limit by year and month
try:
month_list = sorted(list(utils.parse_int_set(ini['TCORR']['months'])))
except:
logging.info('\nTCORR "months" parameter not set in the INI,'
'\n Defaulting to all months (1-12)\n')
month_list = list(range(1, 13))
try:
year_list = sorted(list(utils.parse_int_set(ini['TCORR']['years'])))
except:
logging.info('\nTCORR "years" parameter not set in the INI,'
'\n Defaulting to all available years\n')
year_list = []
# Key is cycle day, value is a reference date on that cycle
# Data from: https://landsat.usgs.gov/landsat_acq
# I only need to use 8 cycle days because of 5/7 and 7/8 are offset
cycle_dates = {
7: '1970-01-01',
8: '1970-01-02',
1: '1970-01-03',
2: '1970-01-04',
3: '1970-01-05',
4: '1970-01-06',
5: '1970-01-07',
6: '1970-01-08',
}
# cycle_dates = {
# 1: '2000-01-06',
# 2: '2000-01-07',
# 3: '2000-01-08',
# 4: '2000-01-09',
# 5: '2000-01-10',
# 6: '2000-01-11',
# 7: '2000-01-12',
# 8: '2000-01-13',
# # 9: '2000-01-14',
# # 10: '2000-01-15',
# # 11: '2000-01-16',
# # 12: '2000-01-01',
# # 13: '2000-01-02',
# # 14: '2000-01-03',
# # 15: '2000-01-04',
# # 16: '2000-01-05',
# }
cycle_base_dt = datetime.datetime.strptime(cycle_dates[1], '%Y-%m-%d')
if cron_flag:
# CGM - This seems like a silly way of getting the date as a datetime
# Why am I doing this and not using the commented out line?
iter_end_dt = datetime.date.today().strftime('%Y-%m-%d')
iter_end_dt = datetime.datetime.strptime(iter_end_dt, '%Y-%m-%d')
iter_end_dt = iter_end_dt + datetime.timedelta(days=-4)
# iter_end_dt = datetime.datetime.today() + datetime.timedelta(days=-1)
iter_start_dt = iter_end_dt + datetime.timedelta(days=-64)
else:
iter_start_dt = datetime.datetime.strptime(ini['INPUTS']['start_date'],
'%Y-%m-%d')
iter_end_dt = datetime.datetime.strptime(ini['INPUTS']['end_date'],
'%Y-%m-%d')
logging.debug('Start Date: {}'.format(iter_start_dt.strftime('%Y-%m-%d')))
logging.debug('End Date: {}\n'.format(iter_end_dt.strftime('%Y-%m-%d')))
for export_dt in sorted(utils.date_range(iter_start_dt, iter_end_dt),
reverse=reverse_flag):
export_date = export_dt.strftime('%Y-%m-%d')
next_date = (export_dt +
datetime.timedelta(days=1)).strftime('%Y-%m-%d')
# if ((month_list and export_dt.month not in month_list) or
# (year_list and export_dt.year not in year_list)):
if month_list and export_dt.month not in month_list:
logging.debug(f'Date: {export_date} - month not in INI - skipping')
continue
elif export_date >= datetime.datetime.today().strftime('%Y-%m-%d'):
logging.debug(f'Date: {export_date} - unsupported date - skipping')
continue
elif export_date < '1984-03-23':
logging.debug(f'Date: {export_date} - no Landsat 5+ images before '
'1984-03-16 - skipping')
continue
logging.info(f'Date: {export_date}')
export_id = ini['EXPORT']['export_id_fmt'] \
.format(
product=tmax_name.lower(),
date=export_dt.strftime('%Y%m%d'),
export=datetime.datetime.today().strftime('%Y%m%d'),
dest=ini['EXPORT']['export_dest'].lower())
logging.debug(' Export ID: {}'.format(export_id))
if ini['EXPORT']['export_dest'] == 'ASSET':
asset_id = '{}/{}_{}'.format(
tcorr_daily_coll_id, export_dt.strftime('%Y%m%d'),
datetime.datetime.today().strftime('%Y%m%d'))
logging.debug(' Asset ID: {}'.format(asset_id))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if (ini['EXPORT']['export_dest'].upper() == 'ASSET'
and asset_id in asset_list):
logging.debug(' Asset already exists, removing')
ee.data.deleteAsset(asset_id)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, exiting')
continue
elif (ini['EXPORT']['export_dest'].upper() == 'ASSET'
and asset_id in asset_list):
logging.debug(' Asset already exists, skipping')
continue
# Build and merge the Landsat collections
model_obj = ssebop.Collection(
collections=collections,
start_date=export_dt.strftime('%Y-%m-%d'),
end_date=(export_dt +
datetime.timedelta(days=1)).strftime('%Y-%m-%d'),
cloud_cover_max=float(ini['INPUTS']['cloud_cover']),
geometry=tmax_mask.geometry(),
# model_args=model_args,
# filter_args=filter_args,
)
landsat_coll = model_obj.overpass(variables=['ndvi'])
# wrs2_tiles_all = model_obj.get_image_ids()
# pprint.pprint(landsat_coll.aggregate_array('system:id').getInfo())
# input('ENTER')
logging.debug(' Getting available WRS2 tile list')
landsat_id_list = landsat_coll.aggregate_array('system:id').getInfo()
wrs2_tiles_all = set([id.split('_')[-2] for id in landsat_id_list])
if not wrs2_tiles_all:
logging.info(' No available images - skipping')
continue
# If overwriting, start a new export no matter what
# The default is to no overwrite, so this mode will not be used often
if not overwrite_flag:
# Check if there are any previous images for this date
# If so, only build a new Tcorr image if there are new wrs2_tiles
# that were not used in the previous image.
# Should this code only be run in cron mode or is this the expected
# operation when (re)running for any date range?
# Should we only test the last image
# or all previous images for the date?
logging.debug(
' Checking for previous exports/versions of daily image')
tcorr_daily_coll = ee.ImageCollection(tcorr_daily_coll_id)\
.filterDate(export_date, next_date)\
.limit(1, 'date_ingested', False)
tcorr_daily_info = tcorr_daily_coll.getInfo()
if tcorr_daily_info['features']:
# Assume we won't be building a new image and only set flag
# to True if the WRS2 tile lists are different
export_flag = False
# The ".limit(1, ..." on the tcorr_daily_coll above makes this
# for loop and break statement unnecessary, but leaving for now
for tcorr_img in tcorr_daily_info['features']:
# If the full WRS2 list is not present, rebuild the image
# This should only happen for much older Tcorr images
if 'wrs2_available' not in tcorr_img['properties'].keys():
logging.debug(
' "wrs2_available" property not present in '
'previous export')
export_flag = True
break
wrs2_tiles_old = set(
tcorr_img['properties']['wrs2_available'].split(','))
if wrs2_tiles_all != wrs2_tiles_old:
logging.debug(' Tile Lists')
logging.debug(' Previous: {}'.format(', '.join(
sorted(wrs2_tiles_old))))
logging.debug(' Available: {}'.format(', '.join(
sorted(wrs2_tiles_all))))
logging.debug(' New: {}'.format(', '.join(
sorted(
wrs2_tiles_all.difference(wrs2_tiles_old)))))
logging.debug(' Dropped: {}'.format(', '.join(
sorted(
wrs2_tiles_old.difference(wrs2_tiles_all)))))
export_flag = True
break
if not export_flag:
logging.debug(' No new WRS2 tiles/images - skipping')
continue
# else:
# logging.debug(' Building new version')
else:
logging.debug(' No previous exports')
def tcorr_img_func(image):
t_stats = ssebop.Image.from_landsat_c1_toa(
ee.Image(image),
tdiff_threshold=float(ini[model_name]['tdiff_threshold'])) \
.tcorr_stats
t_stats = ee.Dictionary(t_stats) \
.combine({'tcorr_p5': 0, 'tcorr_count': 0},
overwrite=False)
tcorr = ee.Number(t_stats.get('tcorr_p5'))
count = ee.Number(t_stats.get('tcorr_count'))
# Remove the merged collection indices from the system:index
scene_id = ee.List(
ee.String(image.get('system:index')).split('_')).slice(-3)
scene_id = ee.String(scene_id.get(0)).cat('_') \
.cat(ee.String(scene_id.get(1))).cat('_') \
.cat(ee.String(scene_id.get(2)))
return tmax_mask.add(tcorr) \
.rename(['tcorr']) \
.clip(image.geometry()) \
.set({
'system:time_start': image.get('system:time_start'),
'scene_id': scene_id,
'wrs2_tile': scene_id.slice(5, 11),
'spacecraft_id': image.get('SPACECRAFT_ID'),
'tcorr': tcorr,
'count': count,
})
# Test for one image
# pprint.pprint(tcorr_img_func(ee.Image(landsat_coll \
# .filterMetadata('WRS_PATH', 'equals', 36) \
# .filterMetadata('WRS_ROW', 'equals', 33).first())).getInfo())
# input('ENTER')
# (Re)build the Landsat collection from the image IDs
landsat_coll = ee.ImageCollection(landsat_id_list)
tcorr_img_coll = ee.ImageCollection(landsat_coll.map(tcorr_img_func)) \
.filterMetadata('count', 'not_less_than',
float(ini['TCORR']['min_pixel_count']))
# If there are no Tcorr values, return an empty image
tcorr_img = ee.Algorithms.If(tcorr_img_coll.size().gt(0),
tcorr_img_coll.median(),
tmax_mask.updateMask(0))
def unique_properties(coll, property):
return ee.String(
ee.List(
ee.Dictionary(
coll.aggregate_histogram(property)).keys()).join(','))
wrs2_tile_list = ee.String('').cat(
unique_properties(tcorr_img_coll, 'wrs2_tile'))
landsat_list = ee.String('').cat(
unique_properties(tcorr_img_coll, 'spacecraft_id'))
# Cast to float and set properties
tcorr_img = ee.Image(tcorr_img).rename(['tcorr']).double() \
.set({
'system:time_start': utils.millis(export_dt),
'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'),
'date': export_dt.strftime('%Y-%m-%d'),
'year': int(export_dt.year),
'month': int(export_dt.month),
'day': int(export_dt.day),
'doy': int(export_dt.strftime('%j')),
'cycle_day': ((export_dt - cycle_base_dt).days % 8) + 1,
'landsat': landsat_list,
'model_name': model_name,
'model_version': ssebop.__version__,
'tmax_source': tmax_source.upper(),
'tmax_version': tmax_version.upper(),
'wrs2_tiles': wrs2_tile_list,
'wrs2_available': ','.join(sorted(wrs2_tiles_all)),
})
# Build export tasks
if ini['EXPORT']['export_dest'] == 'ASSET':
logging.debug(' Building export task')
task = ee.batch.Export.image.toAsset(
image=ee.Image(tcorr_img),
description=export_id,
assetId=asset_id,
crs=export_crs,
crsTransform='[' + ','.join(list(map(str, export_geo))) + ']',
dimensions='{0}x{1}'.format(*export_shape),
)
logging.info(' Starting export task')
utils.ee_task_start(task)
# Pause before starting next task
utils.delay_task(delay)
logging.debug('')
def main(country, year, resolution=30):
ee.Initialize()
def last_day_of_month(any_day):
next_month = any_day.replace(day=28) + datetime.timedelta(days=4)
return next_month - datetime.timedelta(days=next_month.day)
# Reference: https://www.satimagingcorp.com/satellite-sensors/other-satellite-sensors/sentinel-2a/
band_blue = 'B2' #10m
band_green = 'B3' #10m
band_red = "B4" #10m
band_nir = 'B8' #10m
def calc_NDVI(img):
ndvi = ee.Image(img.normalizedDifference([band_nir, band_red])).rename(
["ndvi"]).copyProperties(img, img.propertyNames())
composite = img.addBands(ndvi)
return composite
# SAVI = ((NIR – Red) / (NIR + Red + L)) x (1 + L)
def calc_SAVI(img):
"""A function to compute Soil Adjusted Vegetation Index."""
savi = ee.Image(
img.expression('(1 + L) * float(nir - red)/ (nir + red + L)', {
'nir': img.select(band_nir),
'red': img.select(band_red),
'L': 0.5
})).rename(["savi"]).copyProperties(img, img.propertyNames())
composite = img.addBands(savi)
return composite
# EVI = 2.5 * ((NIR – Red) / ((NIR) + (C1 * Red) – (C2 * Blue) + L))
# C1=6, C2=7.5, and L=1
def calc_EVI(img):
"""A function to compute Soil Adjusted Vegetation Index."""
evi = ee.Image(
img.expression(
'(2.5) * float(nir - red)/ ((nir) + (C1*red) - (C2*blue) + L)',
{
'nir': img.select(band_nir),
'red': img.select(band_red),
'blue': img.select(band_blue),
'L': 0.2,
'C1': 6,
'C2': 7.5
})).rename(["evi"]).copyProperties(img, img.propertyNames())
composite = img.addBands(evi)
return composite
def add_landcover(img):
landcover = ee.Image("USGS/GFSAD1000_V1")
composite = img.addBands(landcover)
return composite
def calc_YYYYMM(img):
return img.set('YYYYMM', img.date().format("YYYYMM"))
########################################################################################
world_df = pd.read_csv(
'../data/Global_Samples_Balanced_n100_m250_s210.csv', index_col=[0])
world_df.head()
world_df['geometry'] = world_df['geometry'].apply(wkt.loads)
world_df['samples'] = world_df['samples'].apply(literal_eval)
world_df['n_samples'] = world_df.apply(lambda row: len(row['samples']),
axis=1)
world_gdf = geopandas.GeoDataFrame(world_df, geometry='geometry')
country_gdf = world_gdf[(world_gdf['country_code'] == country)
& (world_gdf['n_samples'] > 0)]
country_gdf = country_gdf.reset_index()
country_polygon_dict = dict()
for index, row in country_gdf.iterrows():
country_polygon_dict[country + '_' + str(index)] = (row['geometry'],
row['samples'])
for k, v in country_polygon_dict.items():
country_segment = k
print('==================== {} ===================='.format(
country_segment))
area_of_interest_shapely, random_pixels = v
area_of_interest_ee = ee.Geometry.Polygon(
list(area_of_interest_shapely.exterior.coords))
for month in range(1, 13):
month_start = datetime.date(year, month, 1)
month_end = last_day_of_month(datetime.date(year, month, 1))
# Create image collection that contains the area of interest
img_collect = (
ee.ImageCollection('COPERNICUS/S2').filterDate(
str(month_start),
str(month_end)).filterBounds(area_of_interest_ee)
# Remove image that's too small (likely to be partial image)
# Size of a full image: 1,276,131,371; size of a partial image: 276,598,191
# .filter(ee.Filter.gt('system:asset_size', 800000000))
.filterMetadata("CLOUDY_PIXEL_PERCENTAGE", "less_than", 50))
if img_collect.size().getInfo() == 0:
warnings.warn('No valid image.')
continue
print("Total number of images in the collection: ",
img_collect.size().getInfo())
# Extract tile information from each image
# Note: tiles can overlap a little bit
unique_tiles = set([
item['properties']['MGRS_TILE']
for item in img_collect.getInfo()['features']
])
if len(unique_tiles) > 1:
print('Number of tiles selected: ', len(unique_tiles))
# if img_collect_no_partial.size().getInfo() < img_collect.size().getInfo():
# warnings.warn('There are partial images in the collection. Proceed with caution.')
# print('Number of partial images: ', img_collect.size().getInfo()-img_collect_no_partial.size().getInfo())
img_collect_calc = img_collect.map(calc_YYYYMM).map(calc_NDVI).map(
calc_SAVI).map(calc_EVI).map(add_landcover)
unique_month = list(
set([
item['properties']['YYYYMM']
for item in img_collect_calc.getInfo()['features']
]))
unique_month.sort()
print(unique_month)
# 1. min_lat, min_lon, max_lat, max_lon for the country
# min_lon, min_lat, max_lon, max_lat = area_of_interest_shapely.bounds
# 2. from left to right, top to bottom, define 0.1 degree by 0.1 degree overlapping rectangles
# 3. only keep the rectangles that's within the country's boundary (possbily using spatial join)
# 4. for rectangle in a list of recntagles:
# .reduceRegion(geometry=rectangle)
# save to csv
img_calc_month_dict = dict()
data_dict = dict()
for month in unique_month:
img_calc_month_dict[month] = img_collect_calc.filter(
ee.Filter.eq('YYYYMM', month)).median()
img_calc_month2 = img_calc_month_dict[month].addBands(
ee.Image.pixelLonLat())
# EEException: Output of image computation is too large (20 bands for 851968 pixels = 126.8 MiB > 80.0 MiB).
# If this is a reduction, try specifying a larger 'tileScale' parameter.
# EEException: ReduceRegion.AggregationContainer: Valid tileScales are 1 to 16.
data_dict[month] = pd.DataFrame(columns=[
"lat", "lon", 'landcover', month + '_NDVI', month +
'_SAVI', month + '_EVI'
])
for p in range(len(random_pixels)):
# `reduceRegion` doesn't like ee.Geometry.MultiPoint as geometry
data_month_lst = img_calc_month2.reduceRegion(reducer=ee.Reducer.toList(), \
geometry=ee.Geometry.Point(random_pixels[p]), maxPixels=1e13, scale=resolution)
# For some reason, ee.Array(data_month_lst.get("...")).getInfo()[0] runs really slow
pixel_dict = dict()
pixel_dict['lat'] = ee.Array(
data_month_lst.get("latitude")).getInfo()[0]
pixel_dict['lon'] = ee.Array(
data_month_lst.get("longitude")).getInfo()[0]
try:
pixel_dict['landcover'] = ee.Array(
data_month_lst.get("landcover")).getInfo()[0]
pixel_dict[month + '_NDVI'] = ee.Array(
data_month_lst.get("ndvi")).getInfo()[0]
pixel_dict[month + '_SAVI'] = ee.Array(
data_month_lst.get("savi")).getInfo()[0]
pixel_dict[month + '_EVI'] = ee.Array(
data_month_lst.get("evi")).getInfo()[0]
except:
warnings.warn('Missing satellite data.')
pixel_dict['landcover'] = None
pixel_dict[month + '_NDVI'] = None
pixel_dict[month + '_SAVI'] = None
pixel_dict[month + '_EVI'] = None
data_dict[month] = data_dict[month].append(
pixel_dict, ignore_index=True)
data_dict[month].to_csv(
'../data/GEE/Sentinel2_samples/{}_{}.csv'.format(
country_segment, month))
def initialize():
# get GBDX catalog
catalog = gb.catalog.Catalog()
# initilaize EE, used to get water occurrence data
ee.Initialize()
def arrayToPairs(array, startDate, EndDate, delta=1, CLOUDY_PIXEL_PERCENTAGE=10):
ee.Initialize()
area = ee.Geometry.Polygon(array)
# query
collection_NDVI = ee.ImageCollection("COPERNICUS/S2").filterBounds(area) \
.filterDate(startDate, EndDate) \
.filterMetadata("CLOUDY_PIXEL_PERCENTAGE", "less_than", CLOUDY_PIXEL_PERCENTAGE) \
.select(['B8', 'B4', 'QA60']) \
.sort('date')
NDVI_size = collection_NDVI.size().getInfo()
l_NDVI = collection_NDVI.toList(NDVI_size)
l_NDVI_dates = getDates(l_NDVI, NDVI_size, process=False)
# use NDVI dates to make date-filter for SAR data, taking into account
valid_dates = None
for i_NDVI in range(NDVI_size):
NDVI_date = l_NDVI_dates[i_NDVI]
if not valid_dates:
valid_dates = ee.Filter.date(NDVI_date.advance(-delta, 'day'), NDVI_date.advance(delta, 'day'))
else:
valid_dates = ee.Filter.Or(valid_dates, ee.Filter.date(NDVI_date.advance(-delta, 'day'),
NDVI_date.advance(delta, 'day')))
if (not valid_dates):
raise ValueError("no valid dates for the image")
collection_SAR = ee.ImageCollection('COPERNICUS/S1_GRD').filterBounds(area) \
.filter(valid_dates) \
.select(['VH', 'VV'])
SAR_size = collection_SAR.size().getInfo()
l_SAR = collection_SAR.toList(SAR_size)
l_SAR_dates = getDates(l_SAR, SAR_size)
l_NDVI_dates = getDates(l_NDVI, NDVI_size)
# map collections to their respective special data formats
collection_NDVI = collection_NDVI.map(getNDVI.getNDVI)
l_NDVI = collection_NDVI.toList(NDVI_size)
# NDVI-[SAR] list
pairs_i = {}
for i_SAR in range(SAR_size):
SAR_date = l_SAR_dates[i_SAR]
for i_NDVI in range(NDVI_size):
NDVI_date = l_NDVI_dates[i_NDVI]
if abs(SAR_date - NDVI_date) <= timedelta(days=delta):
if i_NDVI not in pairs_i:
pairs_i[i_NDVI] = [i_SAR]
else:
pairs_i[i_NDVI].append(i_SAR)
sorted_pairs = sorted(pairs_i)
print(pairs_i)
arr = []
precomputed_SAR = dict()
LatLonImgsNDVI = ee.List(
[common.fastLatLonImg(ee.Image(l_NDVI.get(NDVI)), area) for NDVI in sorted_pairs])
LatLonImgsSAR = ee.List([LatLonImgVHVV(ee.Image(l_SAR.get(SAR)), area) for SAR in range(SAR_size)])
both_lists = ee.List([LatLonImgsNDVI, LatLonImgsSAR]).getInfo()
LatLonImgsNDVI = both_lists[0]
LatLonImgsSAR = both_lists[1]
for NDVI in range(len(LatLonImgsNDVI)):
i = sorted_pairs[NDVI]
for SAR in pairs_i[i]:
ndvi_temp = (LatLonImgsNDVI[NDVI][0], LatLonImgsNDVI[NDVI][1], LatLonImgsNDVI[NDVI][2])
if SAR not in precomputed_SAR:
lats = LatLonImgsSAR[SAR][0]
lons = LatLonImgsSAR[SAR][1]
vh = LatLonImgsSAR[SAR][2]
vv = LatLonImgsSAR[SAR][3]
precomputed_SAR[SAR] = []
precomputed_SAR[SAR].append((lats, lons, vh) + (f'SAR (VH) {l_SAR_dates[SAR]:%B %d, %Y}',))
precomputed_SAR[SAR].append((lats, lons, vv) + (f'SAR (VV) {l_SAR_dates[SAR]:%B %d, %Y}',))
arr.append(ndvi_temp + (f'NDVI {l_NDVI_dates[NDVI]:%B %d, %Y}',))
arr.extend(precomputed_SAR[SAR])
return rasteriser.rasteriseImages(arr)
def __init__(self):
"""Initialize the environment."""
# Initialize the Earth Engine object, using the authentication credentials.
ee.Initialize()
self.dem = ee.Image("JAXA/ALOS/AW3D30_V1_1").select(["AVE"])
self.epsg = "EPSG:32717"
##########################################
# variable for the landsat data request #
##########################################
self.metadataCloudCoverMax = 80
##########################################
# Export variables #
##########################################
self.assetId = "projects/Sacha/PreprocessedData/L8_Biweekly_V6/"
self.name = "LS_BW_"
self.exportScale = 20
##########################################
# variable for the shadowMask algorithm #
##########################################
# zScoreThresh: Threshold for cloud shadow masking- lower number masks out
# less. Between -0.8 and -1.2 generally works well
self.zScoreThresh = -0.9
# shadowSumThresh: Sum of IR bands to include as shadows within TDOM and the
# shadow shift method (lower number masks out less)
self.shadowSumThresh = 0.4
# contractPixels: The radius of the number of pixels to contract (negative buffer) clouds and cloud shadows by. Intended to eliminate smaller cloud
# patches that are likely errors (1.5 results in a -1 pixel buffer)(0.5 results in a -0 pixel buffer)
# (1.5 or 2.5 generally is sufficient)
self.contractPixels = 1.5
# dilatePixels: The radius of the number of pixels to dilate (buffer) clouds
# and cloud shadows by. Intended to include edges of clouds/cloud shadows
# that are often missed (1.5 results in a 1 pixel buffer)(0.5 results in a 0 pixel buffer)
# (2.5 or 3.5 generally is sufficient)
self.dilatePixels = 3.25
##########################################
# variable for cloudScore algorithm #
##########################################
# 9. Cloud and cloud shadow masking parameters.
# If cloudScoreTDOM is chosen
# cloudScoreThresh: If using the cloudScoreTDOMShift method-Threshold for cloud
# masking (lower number masks more clouds. Between 10 and 30 generally works best)
self.cloudScoreThresh = 1
# Percentile of cloud score to pull from time series to represent a minimum for
# the cloud score over time for a given pixel. Reduces commission errors over
# cool bright surfaces. Generally between 5 and 10 works well. 0 generally is a bit noisy
self.cloudScorePctl = 8
self.hazeThresh = 195
##########################################
# variable for terrain algorithm #
##########################################
self.terrainScale = 600
##########################################
# variable band selection #
##########################################
self.percentiles = [25, 75]
self.medianPercentileBands = ee.List([
'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'date',
'pixel_qa', 'cloudScore'
])
self.divideBands = ee.List(
['blue', 'green', 'red', 'nir', 'swir1', 'swir2'])
self.medoidBands = ee.List(
['blue', 'green', 'red', 'nir', 'swir1', 'swir2'])
self.medoidIncludeBands = ee.List(
['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa'])
self.noScaleBands = ee.List([
'date', 'year', 'cloudMask', 'count', 'TDOMMask', 'pixel_qa',
'cloudScore'
])
self.bandNamesLandsat = ee.List([
'blue', 'green', 'red', 'nir', 'swir1', 'thermal', 'swir2',
'sr_atmos_opacity', 'pixel_qa', 'radsat_qa'
])
self.sensorBandDictLandsatSR = ee.Dictionary({'L8' : ee.List([1,2,3,4,5,7,6,9,10,11]),\
'L7' : ee.List([0,1,2,3,4,5,6,7,9,10]),\
'L5' : ee.List([0,1,2,3,4,5,6,7,9,10]),\
'L4' : ee.List([0,1,2,3,4,5,6,7,9,10])})
##########################################
# enable / disable modules #
##########################################
self.maskSR = True
self.cloudMask = False
self.hazeMask = False
self.shadowMask = False
self.brdfCorrect = True
self.terrainCorrection = True
self.includePercentiles = True
self.compositingMethod = 'Medoid'
def testDynamicConstructor(self):
# Test the behavior of the dynamic class constructor.
# Use a custom set of known functions for classes Foo and Bar.
# Foo Foo(arg1, [arg2])
# Bar Foo.makeBar()
# Bar Foo.takeBar(Bar bar)
# Baz Foo.baz()
def MockSend(path, unused_params, unused_method=None, unused_raw=None):
if path == '/algorithms':
return {
'Foo': {
'returns':
'Foo',
'args': [{
'name': 'arg1',
'type': 'Object'
}, {
'name': 'arg2',
'type': 'Object',
'optional': True
}]
},
'Foo.makeBar': {
'returns': 'Bar',
'args': [{
'name': 'foo',
'type': 'Foo'
}]
},
'Foo.takeBar': {
'returns':
'Bar',
'args': [{
'name': 'foo',
'type': 'Foo'
}, {
'name': 'bar',
'type': 'Bar'
}]
},
'Bar.baz': {
'returns': 'Baz',
'args': [{
'name': 'bar',
'type': 'Bar'
}]
}
}
ee.data.send_ = MockSend
ee.Initialize(None)
# Try to cast something that's already of the right class.
x = ee.Foo('argument')
self.assertEquals(ee.Foo(x), x)
# Tests for dynamic classes, where there is a constructor.
#
# If there's more than 1 arg, call the constructor.
x = ee.Foo('a')
y = ee.Foo(x, 'b')
ctor = ee.ApiFunction.lookup('Foo')
self.assertEquals(y.func, ctor)
self.assertEquals(y.args, {'arg1': x, 'arg2': 'b'})
# Can't cast a primitive; call the constructor.
self.assertEquals(ctor, ee.Foo(1).func)
# A computed object, but not this class; call the constructor.
self.assertEquals(ctor, ee.Foo(ee.List([1, 2, 3])).func)
# Tests for dynamic classes, where there isn't a constructor.
#
# Foo.makeBar and Foo.takeBar should have caused Bar to be generated.
self.assertTrue(hasattr(ee, 'Bar'))
# Make sure we can create a Bar.
bar = ee.Foo(1).makeBar()
self.assertTrue(isinstance(bar, ee.Bar))
# Now cast something else to a Bar and verify it was just a cast.
cast = ee.Bar(ee.Foo(1))
self.assertTrue(isinstance(cast, ee.Bar))
self.assertEquals(ctor, cast.func)
# We shouldn't be able to cast with more than 1 arg.
try:
ee.Bar(x, 'foo')
self.fail('Expected an exception.')
except ee.EEException as e:
self.assertTrue('Too many arguments for ee.Bar' in str(e))
# We shouldn't be able to cast a primitive.
try:
ee.Bar(1)
self.fail('Expected an exception.')
except ee.EEException as e:
self.assertTrue('Must be a ComputedObject' in str(e))
def main(args=None):
setup_logging()
parser = argparse.ArgumentParser(
description='Google Earth Engine Batch Asset Manager')
parser.add_argument('-s',
'--service-account',
help='Google Earth Engine service account.',
required=False)
parser.add_argument('-k',
'--private-key',
help='Google Earth Engine private key file.',
required=False)
subparsers = parser.add_subparsers()
parser_delete = subparsers.add_parser(
'delete',
help=
'Deletes collection and all items inside. Supports Unix-like wildcards.'
)
parser_delete.add_argument(
'id',
help=
'Full path to asset for deletion. Recursively removes all folders, collections and images.'
)
parser_delete.set_defaults(func=delete_collection_from_parser)
parser_upload = subparsers.add_parser('upload',
help='Batch Asset Uploader.')
required_named = parser_upload.add_argument_group(
'Required named arguments.')
required_named.add_argument(
'--source',
help='Path to the directory with images for upload.',
required=True)
required_named.add_argument(
'--dest',
help=
'Destination. Full path for upload to Google Earth Engine, e.g. users/pinkiepie/myponycollection',
required=True)
optional_named = parser_upload.add_argument_group(
'Optional named arguments')
optional_named.add_argument('-m',
'--metadata',
help='Path to CSV with metadata.')
optional_named.add_argument(
'--large',
action='store_true',
help='(Advanced) Use multipart upload. Might help if upload of large '
'files is failing on some systems. Might cause other issues.')
optional_named.add_argument(
'--nodata',
type=int,
help='The value to burn into the raster as NoData (missing data)')
optional_named.add_argument(
'--bands',
type=_comma_separated_strings,
help='Comma-separated list of names to use for the image bands. Spaces'
'or other special characters are not allowed.')
required_named.add_argument('-u',
'--user',
help='Google account name (gmail address).')
optional_named.add_argument('-b',
'--bucket',
help='Google Cloud Storage bucket name.')
optional_named.add_argument(
'-e',
'--upload-catch-error',
action='store_true',
help='Return exit code 1 when upload catches an error')
optional_named.add_argument('-a',
'--tolerate-assets-already-exist',
action='store_true',
help='Return exit 0 when assets already exist')
optional_named.add_argument(
'--headless',
help='Run the browser in headless mode (i.e. no user interface).',
action='store_true')
parser_upload.set_defaults(func=upload_from_parser)
parser_cancel = subparsers.add_parser('cancel',
help='Cancel all running tasks')
parser_cancel.set_defaults(func=cancel_all_running_tasks_from_parser)
parser_info = subparsers.add_parser('report',
help='Produce summary of all assets.')
parser_info.set_defaults(func=produce_report)
parser_info.add_argument('--filename',
help='File name for the output CSV (optional)')
parser_copy = subparsers.add_parser(
'copy',
help=
'Batch copy of assets. Helps in migrating assets from Google Maps to GEE'
)
parser_copy.set_defaults(func=batch_copy)
parser_copy.add_argument(
'--source',
help='File with the following structure: [asset name],[asset id in GME]'
)
parser_copy.add_argument(
'--dest', help='Full path to the directory or collection in EE')
args = parser.parse_args()
if args.service_account:
credentials = ee.ServiceAccountCredentials(args.service_account,
args.private_key)
ee.Initialize(credentials)
else:
ee.Initialize()
if args.private_key is not None:
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.private_key
if 'func' in args:
args.func(args)
else:
parser.print_help()
def retrieve_images(inputs):
"""
Downloads all images from Landsat 5, Landsat 7, Landsat 8 and Sentinel-2
covering the area of interest and acquired between the specified dates.
The downloaded images are in .TIF format and organised in subfolders, divided
by satellite mission. The bands are also subdivided by pixel resolution.
KV WRL 2018
Arguments:
-----------
inputs: dict with the following keys
'sitename': str
name of the site
'polygon': list
polygon containing the lon/lat coordinates to be extracted,
longitudes in the first column and latitudes in the second column,
there are 5 pairs of lat/lon with the fifth point equal to the first point:
```
polygon = [[[151.3, -33.7],[151.4, -33.7],[151.4, -33.8],[151.3, -33.8],
[151.3, -33.7]]]
```
'dates': list of str
list that contains 2 strings with the initial and final dates in
format 'yyyy-mm-dd':
```
dates = ['1987-01-01', '2018-01-01']
```
'sat_list': list of str
list that contains the names of the satellite missions to include:
```
sat_list = ['L5', 'L7', 'L8', 'S2']
```
'filepath_data': str
filepath to the directory where the images are downloaded
Returns:
-----------
metadata: dict
contains the information about the satellite images that were downloaded:
date, filename, georeferencing accuracy and image coordinate reference system
"""
# initialise connection with GEE server
ee.Initialize()
# check image availabiliy and retrieve list of images
im_dict_T1, im_dict_T2 = check_images_available(inputs)
# if user also wants to download T2 images, merge both lists
if 'include_T2' in inputs.keys():
for key in inputs['sat_list']:
if key == 'S2': continue
else: im_dict_T1[key] += im_dict_T2[key]
# remove UTM duplicates in S2 collections (they provide several projections for same images)
if 'S2' in inputs['sat_list'] and len(im_dict_T1['S2']) > 0:
im_dict_T1['S2'] = filter_S2_collection(im_dict_T1['S2'])
# create a new directory for this site with the name of the site
im_folder = os.path.join(inputs['filepath'], inputs['sitename'])
if not os.path.exists(im_folder): os.makedirs(im_folder)
print('\nDownloading images:')
suffix = '.tif'
for satname in im_dict_T1.keys():
print('%s: %d images' % (satname, len(im_dict_T1[satname])))
# create subfolder structure to store the different bands
filepaths = create_folder_structure(im_folder, satname)
# initialise variables and loop through images
georef_accs = []
filenames = []
all_names = []
im_epsg = []
for i in range(len(im_dict_T1[satname])):
im_meta = im_dict_T1[satname][i]
# get time of acquisition (UNIX time) and convert to datetime
t = im_meta['properties']['system:time_start']
im_timestamp = datetime.fromtimestamp(t / 1000, tz=pytz.utc)
im_date = im_timestamp.strftime('%Y-%m-%d-%H-%M-%S')
# get epsg code
im_epsg.append(int(im_meta['bands'][0]['crs'][5:]))
# get geometric accuracy
if satname in ['L5', 'L7', 'L8']:
if 'GEOMETRIC_RMSE_MODEL' in im_meta['properties'].keys():
acc_georef = im_meta['properties']['GEOMETRIC_RMSE_MODEL']
else:
acc_georef = 12 # default value of accuracy (RMSE = 12m)
elif satname in ['S2']:
# Sentinel-2 products don't provide a georeferencing accuracy (RMSE as in Landsat)
# but they have a flag indicating if the geometric quality control was passed or failed
# if passed a value of 1 is stored if failed a value of -1 is stored in the metadata
skip_geo_check = False
if 'GEOMETRIC_QUALITY_FLAG' in im_meta['properties'].keys():
key = 'GEOMETRIC_QUALITY_FLAG'
elif 'quality_check' in im_meta['properties'].keys():
key = 'quality_check'
else:
acc_georef = -1
skip_geo_check = True
if not skip_geo_check:
if im_meta['properties'][key] == 'PASSED': acc_georef = 1
else: acc_georef = -1
georef_accs.append(acc_georef)
bands = dict([])
im_fn = dict([])
# first delete dimensions key from dictionnary
# otherwise the entire image is extracted (don't know why)
im_bands = im_meta['bands']
for j in range(len(im_bands)):
del im_bands[j]['dimensions']
# Landsat 5 download
if satname == 'L5':
bands[''] = [
im_bands[0], im_bands[1], im_bands[2], im_bands[3],
im_bands[4], im_bands[7]
]
im_fn[''] = im_date + '_' + satname + '_' + inputs[
'sitename'] + suffix
# if two images taken at the same date add 'dup' to the name (duplicate)
if any(im_fn[''] in _ for _ in all_names):
im_fn[''] = im_date + '_' + satname + '_' + inputs[
'sitename'] + '_dup' + suffix
all_names.append(im_fn[''])
filenames.append(im_fn[''])
# download .tif from EE
while True:
try:
im_ee = ee.Image(im_meta['id'])
local_data = download_tif(im_ee, inputs['polygon'],
bands[''], filepaths[1])
break
except:
continue
# rename the file as the image is downloaded as 'data.tif'
try:
os.rename(local_data, os.path.join(filepaths[1],
im_fn['']))
except: # overwrite if already exists
os.remove(os.path.join(filepaths[1], im_fn['']))
os.rename(local_data, os.path.join(filepaths[1],
im_fn['']))
# metadata for .txt file
filename_txt = im_fn[''].replace('.tif', '')
metadict = {
'filename': im_fn[''],
'acc_georef': georef_accs[i],
'epsg': im_epsg[i]
}
# Landsat 7 and 8 download
elif satname in ['L7', 'L8']:
if satname == 'L7':
bands['pan'] = [im_bands[8]] # panchromatic band
bands['ms'] = [
im_bands[0], im_bands[1], im_bands[2], im_bands[3],
im_bands[4], im_bands[9]
] # multispectral bands
else:
bands['pan'] = [im_bands[7]] # panchromatic band
bands['ms'] = [
im_bands[1], im_bands[2], im_bands[3], im_bands[4],
im_bands[5], im_bands[11]
] # multispectral bands
for key in bands.keys():
im_fn[key] = im_date + '_' + satname + '_' + inputs[
'sitename'] + '_' + key + suffix
# if two images taken at the same date add 'dup' to the name (duplicate)
if any(im_fn['pan'] in _ for _ in all_names):
for key in bands.keys():
im_fn[key] = im_date + '_' + satname + '_' + inputs[
'sitename'] + '_' + key + '_dup' + suffix
all_names.append(im_fn['pan'])
filenames.append(im_fn['pan'])
# download .tif from EE (panchromatic band and multispectral bands)
while True:
try:
im_ee = ee.Image(im_meta['id'])
local_data_pan = download_tif(im_ee, inputs['polygon'],
bands['pan'],
filepaths[1])
local_data_ms = download_tif(im_ee, inputs['polygon'],
bands['ms'], filepaths[2])
break
except:
continue
# rename the files as the image is downloaded as 'data.tif'
try: # panchromatic
os.rename(local_data_pan,
os.path.join(filepaths[1], im_fn['pan']))
except: # overwrite if already exists
os.remove(os.path.join(filepaths[1], im_fn['pan']))
os.rename(local_data_pan,
os.path.join(filepaths[1], im_fn['pan']))
try: # multispectral
os.rename(local_data_ms,
os.path.join(filepaths[2], im_fn['ms']))
except: # overwrite if already exists
os.remove(os.path.join(filepaths[2], im_fn['ms']))
os.rename(local_data_ms,
os.path.join(filepaths[2], im_fn['ms']))
# metadata for .txt file
filename_txt = im_fn['pan'].replace('_pan',
'').replace('.tif', '')
metadict = {
'filename': im_fn['pan'],
'acc_georef': georef_accs[i],
'epsg': im_epsg[i]
}
# Sentinel-2 download
elif satname in ['S2']:
bands['10m'] = [
im_bands[1], im_bands[2], im_bands[3], im_bands[7]
] # multispectral bands
bands['20m'] = [im_bands[11]] # SWIR band
bands['60m'] = [im_bands[15]] # QA band
for key in bands.keys():
im_fn[key] = im_date + '_' + satname + '_' + inputs[
'sitename'] + '_' + key + suffix
# if two images taken at the same date add 'dup' to the name (duplicate)
if any(im_fn['10m'] in _ for _ in all_names):
for key in bands.keys():
im_fn[key] = im_date + '_' + satname + '_' + inputs[
'sitename'] + '_' + key + '_dup' + suffix
# also check for triplicates (only on S2 imagery) and add 'tri' to the name
if im_fn['10m'] in all_names:
for key in bands.keys():
im_fn[
key] = im_date + '_' + satname + '_' + inputs[
'sitename'] + '_' + key + '_tri' + suffix
all_names.append(im_fn['10m'])
filenames.append(im_fn['10m'])
# download .tif from EE (multispectral bands at 3 different resolutions)
while True:
try:
im_ee = ee.Image(im_meta['id'])
local_data_10m = download_tif(im_ee, inputs['polygon'],
bands['10m'],
filepaths[1])
local_data_20m = download_tif(im_ee, inputs['polygon'],
bands['20m'],
filepaths[2])
local_data_60m = download_tif(im_ee, inputs['polygon'],
bands['60m'],
filepaths[3])
break
except:
continue
# rename the files as the image is downloaded as 'data.tif'
try: # 10m
os.rename(local_data_10m,
os.path.join(filepaths[1], im_fn['10m']))
except: # overwrite if already exists
os.remove(os.path.join(filepaths[1], im_fn['10m']))
os.rename(local_data_10m,
os.path.join(filepaths[1], im_fn['10m']))
try: # 20m
os.rename(local_data_20m,
os.path.join(filepaths[2], im_fn['20m']))
except: # overwrite if already exists
os.remove(os.path.join(filepaths[2], im_fn['20m']))
os.rename(local_data_20m,
os.path.join(filepaths[2], im_fn['20m']))
try: # 60m
os.rename(local_data_60m,
os.path.join(filepaths[3], im_fn['60m']))
except: # overwrite if already exists
os.remove(os.path.join(filepaths[3], im_fn['60m']))
os.rename(local_data_60m,
os.path.join(filepaths[3], im_fn['60m']))
# metadata for .txt file
filename_txt = im_fn['10m'].replace('_10m',
'').replace('.tif', '')
metadict = {
'filename': im_fn['10m'],
'acc_georef': georef_accs[i],
'epsg': im_epsg[i]
}
# write metadata
with open(os.path.join(filepaths[0], filename_txt + '.txt'),
'w') as f:
for key in metadict.keys():
f.write('%s\t%s\n' % (key, metadict[key]))
# print percentage completion for user
print('\r%d%%' % int((i + 1) / len(im_dict_T1[satname]) * 100),
end='')
print('')
# once all images have been downloaded, load metadata from .txt files
metadata = get_metadata(inputs)
# merge overlapping images (necessary only if the polygon is at the boundary of an image)
if 'S2' in metadata.keys():
try:
metadata = merge_overlapping_images(metadata, inputs)
except:
print(
'WARNING: there was an error while merging overlapping S2 images,'
+
' please open an issue on Github at https://github.com/kvos/CoastSat/issues'
+ ' and include your script so we can find out what happened.')
# save metadata dict
with open(
os.path.join(im_folder, inputs['sitename'] + '_metadata' + '.pkl'),
'wb') as f:
pickle.dump(metadata, f)
return metadata
def main():
"""Using the Drive v3 API to download products from GEE for upload to S3."""
yearly, year, bucket = parseCmdLine()
if yearly:
productName = 'YearlyChange' + year
bucketName = bucket
csvString = "\'SWIR-Custom-Change-Between-*\'"
else:
productName = 'LatestChange'
bucketName = 'current-year-to-date/'
csvString = "\'SWIR-Latest-Change-Between-*\'"
successfulDownloads = 0
creds = None
#pdb.set_trace()
# The file token.pickle stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
if os.path.exists(drive_key_file):
with open(drive_key_file, 'rb') as token:
creds = pickle.load(token)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
credentials_file, SCOPES)
creds = flow.run_local_server(port=0)
# Save the credentials for the next run
with open(drive_key_file, 'wb') as token:
pickle.dump(creds, token)
service = build('drive', 'v3', credentials=creds)
ee.Initialize()
text_file = open(ids_file, "r")
ids = text_file.read().split(',')
ids = list(filter(None, ids))
tasks = ee.data.getTaskList()
myTasks = list(filter(lambda x: x['id'] in ids, tasks))
print('Begin download at {0}'.format(
datetime.datetime.now().strftime("%a, %d %B %Y %I:%M:%S")))
while True:
results = service.files().list(
pageSize=100,
q="mimeType = 'image/tiff'",
fields="nextPageToken, files(id, name)").execute()
# get the specifics of the items on drive for download
items = results.get('files', [])
# get any completed tasks and download them
successfulDownloads = len(downloadMultiple(items, service))
# if there's nothing let to do, quit
if not (pendingTasks(ids) or completedTasksRemaining(ids)
or successfulDownloads):
break
else:
print("Waiting for exports to complete.")
time.sleep(5 * 60) # Delay for 5 minutes.
# find IDs for the scenesBegin and scenesEnd CSVs
queryStr = "mimeType != 'image/tiff' and name contains " + csvString
results = service.files().list(
pageSize=100, q=queryStr,
fields="nextPageToken, files(id, name)").execute()
# get the fileIDs of the items download
items = results.get('files', [])
# get CSVs and download them
downloadedFiles = downloadMultiple(items, service)
successfulDownloads = len(downloadedFiles)
#pdb.set_trace()
p = re.compile(
'(NDVI.?|SWIR.?|NDMI.?)(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}\w*(L8|S2)'
)
satelliteName = re.search(p, downloadedFiles[0]).group(3)
if successfulDownloads == 2:
upload_file(downloadDir + downloadedFiles[0], "data.southfact.com",
bucketName + downloadedFiles[0])
upload_file(downloadDir + downloadedFiles[1], "data.southfact.com",
bucketName + downloadedFiles[1])
# Mainland Southern states and PR VI mosaics
print('Begin mosaic to geotiff at {0}'.format(
datetime.datetime.now().strftime("%a, %d %B %Y %I:%M:%S")))
os.chdir(downloadDir)
# no metadata for now
#mosaicDownloadedToGeotiff(re.compile('SWIR(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}datesBegin(L8|S2)CONUS'), 'swirdatesBegin' + productName + satelliteName + 'CONUS.tif')
#mosaicDownloadedToGeotiff(re.compile('SWIR(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}datesEnd(L8|S2)CONUS'), 'swirdatesEnd' + productName + satelliteName + 'CONUS.tif')
#mosaicDownloadedToGeotiff(re.compile('SWIR(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}datesBegin(L8|S2)PRVI'), 'swirdatesBegin' + productName + satelliteName + 'PRVI.tif')
#mosaicDownloadedToGeotiff(re.compile('SWIR(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}datesEnd(L8|S2)PRVI'), 'swirdatesEnd' + productName + satelliteName + 'PRVI.tif')
if yearly:
# for yearly statewide products produce a shapefile of change polys and a regional GeoTIFF
#pdb.set_trace()
downloadedToShape(
re.compile(
'SWIR.?(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}(LA|AR|MS|KY|TN|OK|VA|SC|NC|GA|AL|TX|FL|PR|VI)(L8|S2)',
'swir' + productName + satelliteName))
else:
mosaicDownloadedToGeotiff(
re.compile(
'SWIR.?(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}(L8|S2)CONUS'
), 'swir' + productName + satelliteName + 'CONUS.tif')
mosaicDownloadedToGeotiff(
re.compile(
'SWIR.?(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}(L8|S2)PRVI'
), 'swir' + productName + satelliteName + 'PRVI.tif')
mosaicDownloadedToGeotiff(
re.compile(
'NDMI.?(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}(L8|S2)CONUS'
), 'ndmi' + productName + satelliteName + 'CONUS.tif')
mosaicDownloadedToGeotiff(
re.compile(
'NDMI.?(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}(L8|S2)PRVI'
), 'ndmi' + productName + satelliteName + 'PRVI.tif')
mosaicDownloadedToGeotiff(
re.compile(
'NDVI.?(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}(L8|S2)CONUS'
), 'ndvi' + productName + satelliteName + 'CONUS.tif')
mosaicDownloadedToGeotiff(
re.compile(
'NDVI.?(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}(L8|S2)PRVI'
), 'ndvi' + productName + satelliteName + 'PRVI.tif')
onlyfiles = [
f for f in os.listdir(outputDir) if isfile(join(outputDir, f))
]
for file in onlyfiles:
upload_file(outputDir + file, "data.southfact.com", bucketName + file)
#clean up my mess
shutil.rmtree('/mnt/efs/fs1/GeoTIFF')
shutil.rmtree('/mnt/efs/fs1/output')
print('Finished at {0}'.format(
datetime.datetime.now().strftime("%a, %d %B %Y %I:%M:%S")))
#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" Collections module. Add some needed parameters to collections and create
collection groups to generate a Best Available Pixel Composite """
import ee
import ee.data
if not ee.data._initialized: ee.Initialize()
import indices
from geetools import cloud_mask as cld
from copy import deepcopy
import functions
from geetools import tools
from datetime import date
initialized = True
ACTUAL_YEAR = date.today().year
IDS = {
'L1': 'LANDSAT/LM1_L1T',
'L2': 'LANDSAT/LM2_L1T',
'L3': 'LANDSAT/LM3_L1T',
'L4TOA': 'LANDSAT/LT04/C01/T1_TOA',
'L4USGS': 'LANDSAT/LT04/C01/T1_SR',
'L5TOA': 'LANDSAT/LT05/C01/T1_TOA',
'L5USGS': 'LANDSAT/LT05/C01/T1_SR',
'L5LED': 'LEDAPS/LT5_L1T_SR',
'L7TOA': 'LANDSAT/LE07/C01/T1_TOA',
'L7USGS': 'LANDSAT/LE07/C01/T1_SR',
def main(ini_path=None, overwrite_flag=False, delay=0, key=None):
"""Compute daily Tcorr images
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
delay : float, optional
Delay time between each export task (the default is 0).
key : str, optional
File path to an Earth Engine json key file (the default is None).
"""
logging.info('\nCompute daily Tcorr images')
ini = utils.read_ini(ini_path)
model_name = 'SSEBOP'
# model_name = ini['INPUTS']['et_model'].upper()
if (ini[model_name]['tmax_source'].upper() == 'CIMIS' and
ini['INPUTS']['end_date'] < '2003-10-01'):
logging.error(
'\nCIMIS is not currently available before 2003-10-01, exiting\n')
sys.exit()
elif (ini[model_name]['tmax_source'].upper() == 'DAYMET' and
ini['INPUTS']['end_date'] > '2017-12-31'):
logging.warning(
'\nDAYMET is not currently available past 2017-12-31, '
'using median Tmax values\n')
# sys.exit()
# elif (ini[model_name]['tmax_source'].upper() == 'TOPOWX' and
# ini['INPUTS']['end_date'] > '2017-12-31'):
# logging.warning(
# '\nDAYMET is not currently available past 2017-12-31, '
# 'using median Tmax values\n')
# # sys.exit()
logging.info('\nInitializing Earth Engine')
if key:
logging.info(' Using service account key file: {}'.format(key))
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('deadbeef', key_file=key))
else:
ee.Initialize()
# Output Tcorr daily image collection
tcorr_daily_coll_id = '{}/{}_daily'.format(
ini['EXPORT']['export_coll'], tmax_name.lower())
# Get a Tmax image to set the Tcorr values to
logging.debug('\nTmax properties')
tmax_name = ini[model_name]['tmax_source']
tmax_source = tmax_name.split('_', 1)[0]
tmax_version = tmax_name.split('_', 1)[1]
tmax_coll_id = 'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower())
tmax_coll = ee.ImageCollection(tmax_coll_id)
tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0)
logging.debug(' Collection: {}'.format(tmax_coll_id))
logging.debug(' Source: {}'.format(tmax_source))
logging.debug(' Version: {}'.format(tmax_version))
logging.debug('\nExport properties')
export_geo = ee.Image(tmax_mask).projection().getInfo()['transform']
export_crs = ee.Image(tmax_mask).projection().getInfo()['crs']
export_shape = ee.Image(tmax_mask).getInfo()['bands'][0]['dimensions']
export_extent = [
export_geo[2], export_geo[5] + export_shape[1] * export_geo[4],
export_geo[2] + export_shape[0] * export_geo[0], export_geo[5]]
logging.debug(' CRS: {}'.format(export_crs))
logging.debug(' Extent: {}'.format(export_extent))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
# # Limit export to a user defined study area or geometry?
# export_geom = ee.Geometry.Rectangle(
# [-125, 24, -65, 50], proj='EPSG:4326', geodesic=False) # CONUS
# export_geom = ee.Geometry.Rectangle(
# [-124, 35, -119, 42], proj='EPSG:4326', geodesic=False) # California
# If cell_size parameter is set in the INI,
# adjust the output cellsize and recompute the transform and shape
try:
export_cs = float(ini['EXPORT']['cell_size'])
export_shape = [
int(math.ceil(abs((export_shape[0] * export_geo[0]) / export_cs))),
int(math.ceil(abs((export_shape[1] * export_geo[4]) / export_cs)))]
export_geo = [export_cs, 0.0, export_geo[2], 0.0, -export_cs, export_geo[5]]
logging.debug(' Custom export cell size: {}'.format(export_cs))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
except KeyError:
pass
# Get current asset list
if ini['EXPORT']['export_dest'].upper() == 'ASSET':
logging.debug('\nGetting asset list')
# DEADBEEF - daily is hardcoded in the asset_id for now
asset_list = utils.get_ee_assets(tcorr_daily_coll_id)
else:
raise ValueError('invalid export destination: {}'.format(
ini['EXPORT']['export_dest']))
# Get current running tasks
tasks = utils.get_ee_tasks()
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
logging.debug(' Tasks: {}\n'.format(len(tasks)))
input('ENTER')
# Limit by year and month
try:
month_list = sorted(list(utils.parse_int_set(ini['TCORR']['months'])))
except:
logging.info('\nTCORR "months" parameter not set in the INI,'
'\n Defaulting to all months (1-12)\n')
month_list = list(range(1, 13))
try:
year_list = sorted(list(utils.parse_int_set(ini['TCORR']['years'])))
except:
logging.info('\nTCORR "years" parameter not set in the INI,'
'\n Defaulting to all available years\n')
year_list = []
# Key is cycle day, value is a reference date on that cycle
# Data from: https://landsat.usgs.gov/landsat_acq
# I only need to use 8 cycle days because of 5/7 and 7/8 are offset
cycle_dates = {
7: '1970-01-01',
8: '1970-01-02',
1: '1970-01-03',
2: '1970-01-04',
3: '1970-01-05',
4: '1970-01-06',
5: '1970-01-07',
6: '1970-01-08',
}
# cycle_dates = {
# 1: '2000-01-06',
# 2: '2000-01-07',
# 3: '2000-01-08',
# 4: '2000-01-09',
# 5: '2000-01-10',
# 6: '2000-01-11',
# 7: '2000-01-12',
# 8: '2000-01-13',
# # 9: '2000-01-14',
# # 10: '2000-01-15',
# # 11: '2000-01-16',
# # 12: '2000-01-01',
# # 13: '2000-01-02',
# # 14: '2000-01-03',
# # 15: '2000-01-04',
# # 16: '2000-01-05',
# }
cycle_base_dt = datetime.datetime.strptime(cycle_dates[1], '%Y-%m-%d')
iter_start_dt = datetime.datetime.strptime(
ini['INPUTS']['start_date'], '%Y-%m-%d')
iter_end_dt = datetime.datetime.strptime(
ini['INPUTS']['end_date'], '%Y-%m-%d')
# Iterate over date ranges
for export_dt in utils.date_range(iter_start_dt, iter_end_dt):
export_date = export_dt.strftime('%Y-%m-%d')
if ((month_list and export_dt.month not in month_list) or
( year_list and export_dt.year not in year_list)):
logging.debug('Date: {} - skipping'.format(export_date))
continue
logging.info('Date: {}'.format(export_date))
if export_date >= datetime.datetime.today().strftime('%Y-%m-%d'):
logging.info(' Unsupported date, skipping')
continue
elif export_date < '1984-03-23':
logging.info(' No Landsat 5+ images before 1984-03-16, skipping')
continue
export_id = ini['EXPORT']['export_id_fmt'] \
.format(
product=tmax_name.lower(),
date=export_dt.strftime('%Y%m%d'),
export=ini['EXPORT']['export_dest'].lower())
logging.debug(' Export ID: {}'.format(export_id))
if ini['EXPORT']['export_dest'] == 'ASSET':
# DEADBEEF - daily is hardcoded in the asset_id for now
asset_id = '{}/{}'.format(
tcorr_daily_coll_id, export_dt.strftime('%Y%m%d'))
logging.debug(' Asset ID: {}'.format(asset_id))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if (ini['EXPORT']['export_dest'].upper() == 'ASSET' and
asset_id in asset_list):
logging.debug(' Asset already exists, removing')
ee.data.deleteAsset(asset_id)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, exiting')
continue
elif (ini['EXPORT']['export_dest'].upper() == 'ASSET' and
asset_id in asset_list):
logging.debug(' Asset already exists, skipping')
continue
# Build and merge the Landsat collections
# Time filters are to remove bad (L5) and pre-op (L8) images
# .filterBounds(export_geom) \
l8_coll = ee.ImageCollection('LANDSAT/LC08/C01/T1_RT_TOA') \
.filterDate(export_dt, export_dt + datetime.timedelta(days=1)) \
.filterBounds(tmax_mask.geometry()) \
.filterMetadata('CLOUD_COVER_LAND', 'less_than',
float(ini['INPUTS']['cloud_cover'])) \
.filterMetadata('DATA_TYPE', 'equals', 'L1TP') \
.filter(ee.Filter.gt('system:time_start',
ee.Date('2013-03-24').millis()))
l7_coll = ee.ImageCollection('LANDSAT/LE07/C01/T1_RT_TOA') \
.filterDate(export_dt, export_dt + datetime.timedelta(days=1)) \
.filterBounds(tmax_mask.geometry()) \
.filterMetadata('CLOUD_COVER_LAND', 'less_than',
float(ini['INPUTS']['cloud_cover'])) \
.filterMetadata('DATA_TYPE', 'equals', 'L1TP')
l5_coll = ee.ImageCollection('LANDSAT/LT05/C01/T1_TOA') \
.filterDate(export_dt, export_dt + datetime.timedelta(days=1)) \
.filterBounds(tmax_mask.geometry()) \
.filterMetadata('CLOUD_COVER_LAND', 'less_than',
float(ini['INPUTS']['cloud_cover'])) \
.filterMetadata('DATA_TYPE', 'equals', 'L1TP') \
.filter(ee.Filter.lt('system:time_start',
ee.Date('2011-12-31').millis()))
# l4_coll = ee.ImageCollection('LANDSAT/LT04/C01/T1_TOA') \
# .filterDate(export_dt, export_dt + datetime.timedelta(days=1)) \
# .filterBounds(tmax_img.geometry()) \
# .filterMetadata('CLOUD_COVER_LAND', 'less_than',
# float(ini['INPUTS']['cloud_cover'])) \
# .filterMetadata('DATA_TYPE', 'equals', 'L1TP')
# if export_date <= '1993-12-31':
# landsat_coll = ee.ImageCollection(l5_coll.merge(l4_coll))
if export_date < '1999-01-01':
landsat_coll = l5_coll
elif export_date <= '2011-12-31':
landsat_coll = ee.ImageCollection(l7_coll.merge(l5_coll))
elif export_date <= '2013-03-24':
landsat_coll = l7_coll
else:
landsat_coll = ee.ImageCollection(l8_coll.merge(l7_coll))
# pprint.pprint(landsat_coll.aggregate_histogram('system:index').getInfo())
# pprint.pprint(ee.Image(landsat_coll.first()).getInfo())
# input('ENTER')
def tcorr_img_func(image):
t_stats = ssebop.Image.from_landsat_c1_toa(
ee.Image(image),
tdiff_threshold=float(ini[model_name]['tdiff_threshold'])) \
.tcorr_stats
t_stats = ee.Dictionary(t_stats) \
.combine({'tcorr_p5': 0, 'tcorr_count': 0},
overwrite=False)
# tcorr = ee.Algorithms.If(
# t_stats.get('tcorr_p5'), ee.Number(t_stats.get('tcorr_p5')), 0)
tcorr = ee.Number(t_stats.get('tcorr_p5'))
count = ee.Number(t_stats.get('tcorr_count'))
# Remove the merged collection indices from the system:index
scene_id = ee.List(
ee.String(image.get('system:index')).split('_')).slice(-3)
scene_id = ee.String(scene_id.get(0)).cat('_') \
.cat(ee.String(scene_id.get(1))).cat('_') \
.cat(ee.String(scene_id.get(2)))
# return ee.Image([
# tmax_img.select([0], ['tcorr']).multiply(0) \
# .add(ee.Image.constant(tcorr)).float(),
# tmax_img.select([0], ['count']).multiply(0)
# .add(ee.Image.constant(count)).int()]) \
return tmax_mask.add(tcorr) \
.rename(['tcorr']) \
.clip(image.geometry()) \
.set({
'system:time_start': image.get('system:time_start'),
'scene_id': scene_id,
'wrs2_tile': scene_id.slice(5, 11),
'spacecraft_id': image.get('SPACECRAFT_ID'),
'tcorr': tcorr,
'count': count,
})
# # Test for one image
# pprint.pprint(tcorr_img_func(ee.Image(landsat_coll \
# .filterMetadata('WRS_PATH', 'equals', 36) \
# .filterMetadata('WRS_ROW', 'equals', 33).first())).getInfo())
# input('ENTER')
tcorr_img_coll = ee.ImageCollection(landsat_coll.map(tcorr_img_func)) \
.filterMetadata('count', 'not_less_than',
float(ini['TCORR']['min_pixel_count']))
# pprint.pprint(tcorr_img_coll.aggregate_histogram('system:index').getInfo())
# pprint.pprint(ee.Image(tcorr_img_coll.first()).getInfo())
# input('ENTER')
# If there are no Tcorr values, return an empty image
tcorr_img = ee.Algorithms.If(
tcorr_img_coll.size().gt(0),
tcorr_img_coll.median(),
tmax_mask.updateMask(0))
# pprint.pprint(tcorr_img.getInfo())
# pprint.pprint(tcorr_img_coll.size().getInfo())
# input('ENTER')
# # This doesn't work (median is returning no bands for some dates)
# tcorr_img = tmax_mask.add(tcorr_img_coll.median()) \
# .updateMask(0)
def unique_properties(coll, property):
return ee.String(ee.List(ee.Dictionary(
coll.aggregate_histogram(property)).keys()).join(','))
wrs2_tile_list = ee.String('').cat(unique_properties(
tcorr_img_coll, 'wrs2_tile'))
landsat_list = ee.String('').cat(unique_properties(
tcorr_img_coll, 'spacecraft_id'))
# # Is there a better way of building these strings?
# wrs2_tile_list = ee.Algorithms.If(
# tcorr_img_coll.size().gt(0),
# ee.String(ee.List(ee.Dictionary(tcorr_img_coll \
# .aggregate_histogram('WRS2_TILE')).keys()).join(',')),
# ee.String(''))
# landsat_list = ee.Algorithms.If(
# tcorr_img_coll.size().gt(0),
# ee.String(ee.List(ee.Dictionary(tcorr_img_coll\
# .aggregate_histogram('SPACECRAFT_ID')).keys()).join(',')),
# ee.String(''))
# Cast to float and set properties
tcorr_img = ee.Image(tcorr_img).rename(['tcorr']).double() \
.set({
'system:time_start': utils.millis(export_dt),
'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'),
'date': export_dt.strftime('%Y-%m-%d'),
'year': int(export_dt.year),
'month': int(export_dt.month),
'day': int(export_dt.day),
'doy': int(export_dt.strftime('%j')),
'cycle_day': ((export_dt - cycle_base_dt).days % 8) + 1,
'landsat': landsat_list,
'model_name': model_name,
'model_version': ssebop.__version__,
'tmax_source': tmax_source.upper(),
'tmax_version': tmax_version.upper(),
'wrs2_tiles': wrs2_tile_list,
})
# pprint.pprint(tcorr_img.getInfo())
# input('ENTER')
# Build export tasks
if ini['EXPORT']['export_dest'] == 'ASSET':
logging.debug(' Building export task')
task = ee.batch.Export.image.toAsset(
image=ee.Image(tcorr_img),
description=export_id,
assetId=asset_id,
crs=export_crs,
crsTransform='[' + ','.join(list(map(str, export_geo))) + ']',
dimensions='{0}x{1}'.format(*export_shape),
)
logging.debug(' Starting export task')
utils.ee_task_start(task)
# Pause before starting next task
utils.delay_task(delay)
logging.debug('')
def download_data_ts(R:RegionST, products, bands, path_save, scale=10):
ee.Initialize()
times = (R.times[0], R.times[-1])
path_save.mkdir(exist_ok=True, parents=True)
sR = [R] if R.shape[0] <= 32 else split_region(R, size=32, cls=RegionST)
for j, R in enumerate(sR):
region = (f"[[{R.bbox.left}, {R.bbox.bottom}], [{R.bbox.right}, {R.bbox.bottom}], " +
f"[{R.bbox.right}, {R.bbox.top}], [{R.bbox.left}, {R.bbox.top}]]")
# Merge products to single image collection
imCol = ee.ImageCollection(products[0])
for i in range(1, len(products)):
imCol = imCol.merge(ee.ImageCollection(products[i]))
imCol = filter_region(imCol, R, times=times, bands=bands)
imCol = ee.ImageCollection(imCol)
colList = imCol.toList(imCol.size())
# Download each image
for i in range(colList.size().getInfo()):
image = ee.Image(colList.get(i))
zip_error = True
for i in range(10): # Try 10 times
if zip_error:
try:
url = image.getDownloadURL(
{'scale': scale, 'crs': 'EPSG:4326',
'region': f'{region}'})
r = requests.get(url)
with open(str(path_save/'data.zip'), 'wb') as f:
f.write(r.content)
with zipfile.ZipFile(str(path_save/'data.zip'), 'r') as f:
files = f.namelist()
f.extractall(str(path_save))
os.remove(str(path_save/'data.zip'))
zip_error = False
except:
zip_error = True
os.remove(str(path_save/'data.zip'))
time.sleep(10)
if zip_error: raise Exception(f'Failed to process {url}')
for f in files:
f = path_save/f
os.rename(str(f), str(path_save/f'{f.stem}_{j}{f.suffix}'))
# Merge files
suffix = '.tif'
files = path_save.ls(include=[suffix])
files = [o.stem for o in files]
ref = np.unique(['_'.join(o.split('_')[:-1])
for o in files if len(o.split('_')[-1]) < 6])
ids = np.unique([int(o.split('_')[-1])
for o in files if len(o.split('_')[-1]) < 6])
file_groups = [[path_save/f'{r}_{i}{suffix}' for i in ids
if f'{r}_{i}' in files] for r in ref]
for fs in file_groups:
if len(fs) < 500:
fsave = '_'.join(fs[0].stem.split('_')[:-1]) + suffix
merge_tifs(fs, fsave, delete=True)
else:
fs_break = np.array(fs)[:(len(fs)//500)*500].reshape(len(fs)//500,-1).tolist()
if len(fs[(len(fs)//500)*500:]) > 0:
fs_break.append(fs[(len(fs)//500)*500:])
for fsi, fs2 in enumerate(fs_break):
fsave = '_'.join(fs2[0].stem.split('_')[:-1]) + f'_break{fsi}' + suffix
merge_tifs(fs2, fsave, delete=True)
files = path_save.ls(include=[suffix, '_break'])
files = [o.stem for o in files]
ref = np.unique(['_'.join(o.split('_')[:-1])
for o in files if len(o.split('_')[-1]) < 11])
ids = np.unique([o.split('_')[-1]
for o in files if len(o.split('_')[-1]) < 11])
file_groups = [[path_save/f'{r}_{i}{suffix}' for i in ids
if f'{r}_{i}' in files] for r in ref]
for fs in file_groups:
fsave = '_'.join(fs[0].stem.split('_')[:-1]) + suffix
merge_tifs(fs, fsave, delete=True)
def main(ini_path=None, overwrite_flag=False, delay_time=0, gee_key_file=None,
max_ready=-1, reverse_flag=False):
"""Compute annual Tcorr images from scene images
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
delay_time : float, optional
Delay time in seconds between starting export tasks (or checking the
number of queued tasks, see "max_ready" parameter). The default is 0.
gee_key_file : str, None, optional
Earth Engine service account JSON key file (the default is None).
max_ready: int, optional
Maximum number of queued "READY" tasks. The default is -1 which is
implies no limit to the number of tasks that will be submitted.
reverse_flag : bool, optional
If True, process WRS2 tiles in reverse order.
"""
logging.info('\nCompute annual Tcorr images from scene images')
ini = utils.read_ini(ini_path)
model_name = 'SSEBOP'
# model_name = ini['INPUTS']['et_model'].upper()
tmax_name = ini[model_name]['tmax_source']
export_id_fmt = 'tcorr_scene_{product}_{wrs2}_annual_from_scene'
asset_id_fmt = '{coll_id}/{wrs2}'
tcorr_annual_coll_id = '{}/{}_annual_from_scene'.format(
ini['EXPORT']['export_coll'], tmax_name.lower())
wrs2_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/wrs2_descending_custom'
wrs2_tile_field = 'WRS2_TILE'
# wrs2_path_field = 'ROW'
# wrs2_row_field = 'PATH'
try:
wrs2_tiles = str(ini['INPUTS']['wrs2_tiles'])
wrs2_tiles = [x.strip() for x in wrs2_tiles.split(',')]
wrs2_tiles = sorted([x.lower() for x in wrs2_tiles if x])
except KeyError:
wrs2_tiles = []
logging.debug(' wrs2_tiles: not set in INI, defaulting to []')
except Exception as e:
raise e
try:
study_area_extent = str(ini['INPUTS']['study_area_extent']) \
.replace('[', '').replace(']', '').split(',')
study_area_extent = [float(x.strip()) for x in study_area_extent]
except KeyError:
study_area_extent = None
logging.debug(' study_area_extent: not set in INI, defaulting to None')
except Exception as e:
raise e
# TODO: Add try/except blocks and default values?
# TODO: Filter Tcorr scene collection based on collections parameter
# collections = [x.strip() for x in ini['INPUTS']['collections'].split(',')]
cloud_cover = float(ini['INPUTS']['cloud_cover'])
min_pixel_count = float(ini['TCORR']['min_pixel_count'])
min_scene_count = float(ini['TCORR']['min_scene_count'])
if (tmax_name.upper() == 'CIMIS' and
ini['INPUTS']['end_date'] < '2003-10-01'):
logging.error(
'\nCIMIS is not currently available before 2003-10-01, exiting\n')
sys.exit()
elif (tmax_name.upper() == 'DAYMET' and
ini['INPUTS']['end_date'] > '2018-12-31'):
logging.warning(
'\nDAYMET is not currently available past 2018-12-31, '
'using median Tmax values\n')
# sys.exit()
# elif (tmax_name.upper() == 'TOPOWX' and
# ini['INPUTS']['end_date'] > '2017-12-31'):
# logging.warning(
# '\nDAYMET is not currently available past 2017-12-31, '
# 'using median Tmax values\n')
# # sys.exit()
logging.info('\nInitializing Earth Engine')
if gee_key_file:
logging.info(' Using service account key file: {}'.format(gee_key_file))
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('x', key_file=gee_key_file),
use_cloud_api=True)
else:
ee.Initialize(use_cloud_api=True)
logging.debug('\nTmax properties')
tmax_source = tmax_name.split('_', 1)[0]
tmax_version = tmax_name.split('_', 1)[1]
tmax_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower())
tmax_coll = ee.ImageCollection(tmax_coll_id)
tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0)
logging.debug(' Collection: {}'.format(tmax_coll_id))
logging.debug(' Source: {}'.format(tmax_source))
logging.debug(' Version: {}'.format(tmax_version))
# Get the Tcorr scene image collection properties
logging.debug('\nTcorr scene collection')
tcorr_scene_coll_id = '{}/{}_scene'.format(
ini['EXPORT']['export_coll'], tmax_name.lower())
logging.debug('\nExport properties')
export_info = utils.get_info(ee.Image(tmax_mask))
if 'daymet' in tmax_name.lower():
# Custom smaller extent for DAYMET focused on CONUS
export_extent = [-1999750, -1890500, 2500250, 1109500]
export_shape = [4500, 3000]
export_geo = [1000, 0, -1999750, 0, -1000, 1109500]
# Custom medium extent for DAYMET of CONUS, Mexico, and southern Canada
# export_extent = [-2099750, -3090500, 2900250, 1909500]
# export_shape = [5000, 5000]
# export_geo = [1000, 0, -2099750, 0, -1000, 1909500]
export_crs = export_info['bands'][0]['crs']
else:
export_crs = export_info['bands'][0]['crs']
export_geo = export_info['bands'][0]['crs_transform']
export_shape = export_info['bands'][0]['dimensions']
# export_geo = ee.Image(tmax_mask).projection().getInfo()['transform']
# export_crs = ee.Image(tmax_mask).projection().getInfo()['crs']
# export_shape = ee.Image(tmax_mask).getInfo()['bands'][0]['dimensions']
export_extent = [
export_geo[2], export_geo[5] + export_shape[1] * export_geo[4],
export_geo[2] + export_shape[0] * export_geo[0], export_geo[5]]
export_geom = ee.Geometry.Rectangle(
export_extent, proj=export_crs, geodesic=False)
logging.debug(' CRS: {}'.format(export_crs))
logging.debug(' Extent: {}'.format(export_extent))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
if study_area_extent is None:
if 'daymet' in tmax_name.lower():
# CGM - For now force DAYMET to a slightly smaller "CONUS" extent
study_area_extent = [-125, 25, -65, 49]
# study_area_extent = [-125, 25, -65, 52]
elif 'cimis' in tmax_name.lower():
study_area_extent = [-124, 35, -119, 42]
else:
# TODO: Make sure output from bounds is in WGS84
study_area_extent = tmax_mask.geometry().bounds().getInfo()
logging.debug(f'\nStudy area extent not set in INI, '
f'default to {study_area_extent}')
study_area_geom = ee.Geometry.Rectangle(
study_area_extent, proj='EPSG:4326', geodesic=False)
if not ee.data.getInfo(tcorr_annual_coll_id):
logging.info('\nExport collection does not exist and will be built'
'\n {}'.format(tcorr_annual_coll_id))
input('Press ENTER to continue')
ee.data.createAsset({'type': 'IMAGE_COLLECTION'}, tcorr_annual_coll_id)
# Get current asset list
logging.debug('\nGetting GEE asset list')
asset_list = utils.get_ee_assets(tcorr_annual_coll_id)
# if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
# pprint.pprint(asset_list[:10])
# Get current running tasks
tasks = utils.get_ee_tasks()
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
logging.debug(' Tasks: {}\n'.format(len(tasks)))
input('ENTER')
# Limit by year
month_list = list(range(1, 13))
# try:
# month_list = sorted(list(utils.parse_int_set(ini['TCORR']['months'])))
# except:
# logging.info('\nTCORR "months" parameter not set in the INI,'
# '\n Defaulting to all months (1-12)\n')
# month_list = list(range(1, 13))
try:
year_list = sorted(list(utils.parse_int_set(ini['TCORR']['years'])))
except:
logging.info('\nTCORR "years" parameter not set in the INI,'
'\n Defaulting to all available years\n')
year_list = []
# Get the list of WRS2 tiles that intersect the data area and study area
wrs2_coll = ee.FeatureCollection(wrs2_coll_id) \
.filterBounds(export_geom) \
.filterBounds(study_area_geom)
if wrs2_tiles:
wrs2_coll = wrs2_coll.filter(ee.Filter.inList(wrs2_tile_field, wrs2_tiles))
wrs2_info = wrs2_coll.getInfo()['features']
# Iterate over date ranges
for wrs2_ftr in sorted(wrs2_info,
key=lambda k: k['properties']['WRS2_TILE'],
reverse=reverse_flag):
wrs2_tile = wrs2_ftr['properties'][wrs2_tile_field]
logging.info('{}'.format(wrs2_tile))
wrs2_path = int(wrs2_tile[1:4])
wrs2_row = int(wrs2_tile[5:8])
# wrs2_path = wrs2_ftr['properties'][wrs2_path_field]
# wrs2_row = wrs2_ftr['properties'][wrs2_row_field]
export_id = export_id_fmt.format(
product=tmax_name.lower(), wrs2=wrs2_tile)
logging.debug(' Export ID: {}'.format(export_id))
asset_id = asset_id_fmt.format(
coll_id=tcorr_annual_coll_id, wrs2=wrs2_tile)
logging.debug(' Asset ID: {}'.format(asset_id))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id]['id'])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if asset_id in asset_list:
logging.debug(' Asset already exists, removing')
ee.data.deleteAsset(asset_id)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, exiting')
continue
elif asset_id in asset_list:
logging.debug(' Asset already exists, skipping')
continue
tcorr_coll = ee.ImageCollection(tcorr_scene_coll_id) \
.filterMetadata('wrs2_tile', 'equals', wrs2_tile) \
.filterMetadata('tcorr_pixel_count', 'not_less_than', min_pixel_count) \
.filter(ee.Filter.inList('year', year_list))
# TODO: Should CLOUD_COVER_LAND filter should be re-applied here?
# .filterMetadata('CLOUD_COVER_LAND', 'less_than', cloud_cover)
# .filterDate(start_date, end_date)
# .filterBounds(ee.Geometry(wrs2_ftr['geometry']))
# Use a common reducer for the images and property stats
reducer = ee.Reducer.median() \
.combine(ee.Reducer.count(), sharedInputs=True)
# Compute stats from the collection images
# This might be used when Tcorr is spatial
# tcorr_img = tcorr_coll.reduce(reducer).rename(['tcorr', 'count'])
# Compute stats from the image properties
tcorr_stats = ee.List(tcorr_coll.aggregate_array('tcorr_value')) \
.reduce(reducer)
tcorr_stats = ee.Dictionary(tcorr_stats) \
.combine({'median': 0, 'count': 0}, overwrite=False)
tcorr = ee.Number(tcorr_stats.get('median'))
count = ee.Number(tcorr_stats.get('count'))
index = count.lt(min_scene_count).multiply(7).add(2)
# index = ee.Algorithms.If(count.gte(min_scene_count), 2, 9)
# Clip the mask image to the Landsat footprint
# Change mask values to 1 if count >= threshold
# Mask values of 0 will be set to nodata
mask_img = tmax_mask.add(count.gte(min_scene_count)) \
.clip(ee.Geometry(wrs2_ftr['geometry']))
output_img = ee.Image(
[mask_img.multiply(tcorr), mask_img.multiply(count)]) \
.rename(['tcorr', 'count']) \
.updateMask(mask_img.unmask(0))
# # Write an empty image if the pixel count is too low
# # CGM: Check/test if this can be combined into a single If()
# tcorr_img = ee.Algorithms.If(
# count.gte(min_scene_count),
# tmax_mask.add(tcorr), tmax_mask.updateMask(0))
# count_img = ee.Algorithms.If(
# count.gte(min_scene_count),
# tmax_mask.add(count), tmax_mask.updateMask(0))
#
# # Clip to the Landsat image footprint
# output_img = ee.Image([tcorr_img, count_img]) \
# .rename(['tcorr', 'count']) \
# .clip(ee.Geometry(wrs2_ftr['geometry']))
# # Clear the transparency mask
# output_img = output_img.updateMask(output_img.unmask(0))
output_img = output_img.set({
'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'),
'model_name': model_name,
'model_version': ssebop.__version__,
# 'system:time_start': utils.millis(start_dt),
'tcorr_value': tcorr,
'tcorr_index': index,
'tcorr_scene_count': count,
'tmax_source': tmax_source.upper(),
'tmax_version': tmax_version.upper(),
'wrs2_path': wrs2_path,
'wrs2_row': wrs2_row,
'wrs2_tile': wrs2_tile,
'years': ','.join(map(str, year_list)),
# 'year_start': year_list[0],
# 'year_end': year_list[-1],
})
# pprint.pprint(output_img.getInfo())
# input('ENTER')
logging.debug(' Building export task')
task = ee.batch.Export.image.toAsset(
image=output_img,
description=export_id,
assetId=asset_id,
crs=export_crs,
crsTransform='[' + ','.join(list(map(str, export_geo))) + ']',
dimensions='{0}x{1}'.format(*export_shape),
)
logging.info(' Starting export task')
utils.ee_task_start(task)
# Pause before starting the next export task
utils.delay_task(delay_time, max_ready)
logging.debug('')
def main(ini_path=None, overwrite_flag=False, delay_time=0, gee_key_file=None,
max_ready=-1, reverse_flag=False):
"""Compute default Tcorr images by WRS2 tile
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
delay_time : float, optional
Delay time in seconds between starting export tasks (or checking the
number of queued tasks, see "max_ready" parameter). The default is 0.
gee_key_file : str, None, optional
Earth Engine service account JSON key file (the default is None).
max_ready: int, optional
Maximum number of queued "READY" tasks. The default is -1 which is
implies no limit to the number of tasks that will be submitted.
reverse_flag : bool, optional
If True, process WRS2 tiles in reverse order.
"""
logging.info('\nCompute default Tcorr images by WRS2 tile')
ini = utils.read_ini(ini_path)
model_name = 'SSEBOP'
# model_name = ini['INPUTS']['et_model'].upper()
tmax_name = ini[model_name]['tmax_source']
export_id_fmt = 'tcorr_scene_{product}_{wrs2}_default'
asset_id_fmt = '{coll_id}/{wrs2}'
tcorr_default_coll_id = '{}/{}_default'.format(
ini['EXPORT']['export_coll'], tmax_name.lower())
wrs2_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/wrs2_descending_custom'
wrs2_tile_field = 'WRS2_TILE'
# wrs2_path_field = 'ROW'
# wrs2_row_field = 'PATH'
try:
wrs2_tiles = str(ini['INPUTS']['wrs2_tiles'])
wrs2_tiles = [x.strip() for x in wrs2_tiles.split(',')]
wrs2_tiles = sorted([x.lower() for x in wrs2_tiles if x])
except KeyError:
wrs2_tiles = []
logging.debug(' wrs2_tiles: not set in INI, defaulting to []')
except Exception as e:
raise e
try:
study_area_extent = str(ini['INPUTS']['study_area_extent']) \
.replace('[', '').replace(']', '').split(',')
study_area_extent = [float(x.strip()) for x in study_area_extent]
except KeyError:
study_area_extent = None
logging.debug(' study_area_extent: not set in INI')
except Exception as e:
raise e
try:
tcorr_default = ini[model_name]['tcorr_default']
except:
tcorr_default = 0.978
logging.info('\nInitializing Earth Engine')
if gee_key_file:
logging.info(' Using service account key file: {}'.format(gee_key_file))
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('x', key_file=gee_key_file),
use_cloud_api=True)
else:
ee.Initialize(use_cloud_api=True)
logging.debug('\nTmax properties')
tmax_source = tmax_name.split('_', 1)[0]
tmax_version = tmax_name.split('_', 1)[1]
tmax_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower())
tmax_coll = ee.ImageCollection(tmax_coll_id)
tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0)
logging.debug(' Collection: {}'.format(tmax_coll_id))
logging.debug(' Source: {}'.format(tmax_source))
logging.debug(' Version: {}'.format(tmax_version))
# # Get the Tcorr scene image collection properties
# logging.debug('\nTcorr scene collection')
# tcorr_scene_coll_id = '{}/{}_scene'.format(
# ini['EXPORT']['export_coll'], tmax_name.lower())
logging.debug('\nExport properties')
export_info = utils.get_info(ee.Image(tmax_mask))
if 'daymet' in tmax_name.lower():
# Custom smaller extent for DAYMET focused on CONUS
export_extent = [-1999750, -1890500, 2500250, 1109500]
export_shape = [4500, 3000]
export_geo = [1000, 0, -1999750, 0, -1000, 1109500]
# Custom medium extent for DAYMET of CONUS, Mexico, and southern Canada
# export_extent = [-2099750, -3090500, 2900250, 1909500]
# export_shape = [5000, 5000]
# export_geo = [1000, 0, -2099750, 0, -1000, 1909500]
export_crs = export_info['bands'][0]['crs']
else:
export_crs = export_info['bands'][0]['crs']
export_geo = export_info['bands'][0]['crs_transform']
export_shape = export_info['bands'][0]['dimensions']
# export_geo = ee.Image(tmax_mask).projection().getInfo()['transform']
# export_crs = ee.Image(tmax_mask).projection().getInfo()['crs']
# export_shape = ee.Image(tmax_mask).getInfo()['bands'][0]['dimensions']
export_extent = [
export_geo[2], export_geo[5] + export_shape[1] * export_geo[4],
export_geo[2] + export_shape[0] * export_geo[0], export_geo[5]]
export_geom = ee.Geometry.Rectangle(
export_extent, proj=export_crs, geodesic=False)
logging.debug(' CRS: {}'.format(export_crs))
logging.debug(' Extent: {}'.format(export_extent))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
if study_area_extent is None:
if 'daymet' in tmax_name.lower():
# CGM - For now force DAYMET to a slightly smaller "CONUS" extent
study_area_extent = [-125, 25, -65, 50]
# study_area_extent = [-125, 25, -65, 49]
# study_area_extent = [-125, 25, -65, 52]
elif 'cimis' in tmax_name.lower():
study_area_extent = [-124, 35, -119, 42]
else:
# TODO: Make sure output from bounds is in WGS84
study_area_extent = tmax_mask.geometry().bounds().getInfo()
logging.debug(f'\nStudy area extent not set in INI, '
f'default to {study_area_extent}')
study_area_geom = ee.Geometry.Rectangle(
study_area_extent, proj='EPSG:4326', geodesic=False)
if not ee.data.getInfo(tcorr_default_coll_id):
logging.info('\nExport collection does not exist and will be built'
'\n {}'.format(tcorr_default_coll_id))
input('Press ENTER to continue')
ee.data.createAsset({'type': 'IMAGE_COLLECTION'}, tcorr_default_coll_id)
# Get current asset list
logging.debug('\nGetting GEE asset list')
asset_list = utils.get_ee_assets(tcorr_default_coll_id)
# if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
# pprint.pprint(asset_list[:10])
# Get current running tasks
tasks = utils.get_ee_tasks()
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
logging.debug(' Tasks: {}\n'.format(len(tasks)))
input('ENTER')
# Get the list of WRS2 tiles that intersect the data area and study area
wrs2_coll = ee.FeatureCollection(wrs2_coll_id)\
.filterBounds(export_geom)\
.filterBounds(study_area_geom)
if wrs2_tiles:
wrs2_coll = wrs2_coll.filter(ee.Filter.inList(wrs2_tile_field, wrs2_tiles))
wrs2_info = wrs2_coll.getInfo()['features']
for wrs2_ftr in sorted(wrs2_info,
key=lambda k: k['properties']['WRS2_TILE'],
reverse=reverse_flag):
wrs2_tile = wrs2_ftr['properties'][wrs2_tile_field]
logging.info('{}'.format(wrs2_tile))
wrs2_path = int(wrs2_tile[1:4])
wrs2_row = int(wrs2_tile[5:8])
# wrs2_path = wrs2_ftr['properties'][wrs2_path_field]
# wrs2_row = wrs2_ftr['properties'][wrs2_row_field]
export_id = export_id_fmt.format(
product=tmax_name.lower(), wrs2=wrs2_tile)
logging.debug(' Export ID: {}'.format(export_id))
asset_id = asset_id_fmt.format(
coll_id=tcorr_default_coll_id, wrs2=wrs2_tile)
logging.debug(' Asset ID: {}'.format(asset_id))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id]['id'])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if asset_id in asset_list:
logging.debug(' Asset already exists, removing')
ee.data.deleteAsset(asset_id)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, exiting')
continue
elif asset_id in asset_list:
logging.debug(' Asset already exists, skipping')
continue
# Clip the mask image to the Landsat footprint
mask_img = tmax_mask.add(1).clip(ee.Geometry(wrs2_ftr['geometry']))
# Apply the default Tcorr value and then clear the transparency mask
output_img = mask_img.multiply(0.978).rename(['tcorr'])\
.updateMask(mask_img.unmask(0))
# # Clip to the Landsat image footprint
# output_img = tmax_mask.add(tcorr_default) \
# .rename(['tcorr']) \
# .clip(ee.Geometry(wrs2_ftr['geometry']))
# # Clear the transparency mask
# output_img = output_img.updateMask(output_img.unmask(0))
output_img = output_img.set({
'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'),
'model_name': model_name,
'model_version': ssebop.__version__,
# 'system:time_start': utils.millis(start_dt),
'tcorr_value': tcorr_default,
'tcorr_index': 3,
'tmax_source': tmax_source.upper(),
'tmax_version': tmax_version.upper(),
'wrs2_path': wrs2_path,
'wrs2_row': wrs2_row,
'wrs2_tile': wrs2_tile,
})
# pprint.pprint(output_img.getInfo())
# input('ENTER')
logging.debug(' Building export task')
task = ee.batch.Export.image.toAsset(
image=output_img,
description=export_id,
assetId=asset_id,
crs=export_crs,
crsTransform='[' + ','.join(list(map(str, export_geo))) + ']',
dimensions='{0}x{1}'.format(*export_shape),
)
logging.info(' Starting export task')
utils.ee_task_start(task)
# Pause before starting the next export task
utils.delay_task(delay_time, max_ready)
logging.debug('')
target = aligned_data_file
mask = aligned_mask_file
processed_data_file = os.path.join(data_dir,dataset_name+'.tif')
util_files.mask_geotiff(target, mask, processed_data_file, nodata=-128)
'''
Upload processed data to Google Earth Engine
'''
# set up Google Cloud Storage project and bucket objects
gcs_client = storage.Client(os.environ.get("CLOUDSDK_CORE_PROJECT"))
gcs_bucket = gcs_client.bucket(os.environ.get("GEE_STAGING_BUCKET"))
# initialize ee (Google Earth Engine Python API) for uploading to GEE
auth = ee.ServiceAccountCredentials(os.getenv('GEE_SERVICE_ACCOUNT'), os.getenv('GOOGLE_APPLICATION_CREDENTIALS'))
ee.Initialize(auth)
logger.info('Uploading processed data to Google Cloud Storage.')
gcs_uris = util_cloud.gcs_upload(processed_data_file, dataset_name, gcs_bucket=gcs_bucket)
logger.info('Uploading processed data to Google Earth Engine.')
# generate bands component of GEE upload manifest
mf_bands = util_cloud.gee_manifest_bands(data_dict, dataset_name)
# upload processed data file to GEE
asset_name = f'projects/resource-watch-gee/{dataset_name}'
manifest = util_cloud.gee_manifest_complete(asset_name, gcs_uris[0], mf_bands)
logger.debug(manifest)
task_id = util_cloud.gee_ingest(manifest, public=True)
util_cloud.gcs_remove(gcs_uris, gcs_bucket=gcs_bucket)
import ee
ee.Initialize()
ROI = ee.Geometry.Rectangle([8.501, 10.470, 10.701, 8.287])
test = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR').filterBounds(ROI)
# Scale Class
class Scale(object):
def __init__(self, scale=0.0001):
self.scale = scale
def f_scale(self, img):
imgs_to_scale = img.select(['B', 'G', 'R', 'NIR', 'SWIR1',
'SWIR2']).multiply(self.scale).float()
imgs_to_append = img.select(['pixel_qa'])
imgs = imgs_to_scale.addBands(imgs_to_append) \
.copyProperties(source=img).set('system:time_start', img.get('system:time_start'))
return imgs
def map(self, imgcol):
imgcol = imgcol.map(self.f_scale)
return imgcol
# PBC class
class PBC(object):
def __init__(self, imgcol=None, masks=None, scale=None, reducer=None):
self._imgcol = imgcol
self._id = imgcol.getInfo()['id'].split("/")[1:4:2]
self._masks = masks
def ee_beamer_et(ini_path=None, overwrite_flag=False):
"""Earth Engine Beamer ET Image Download
Args:
ini_path (str):
overwrite_flag (bool): if True, overwrite existing files
Returns:
None
"""
logging.info('\nEarth Engine Beamer ETg Image Download')
# Read config file
ini = inputs.read(ini_path)
inputs.parse_section(ini, section='INPUTS')
inputs.parse_section(ini, section='SPATIAL')
inputs.parse_section(ini, section='IMAGES')
inputs.parse_section(ini, section='BEAMER')
ini['IMAGES']['download_bands'] = [
'etg_mean', 'etg_lci', 'etg_uci', 'etg_lpi', 'etg_upi'
]
stat_list = ['median', 'mean']
nodata_value = -9999
zips_folder = 'zips'
images_folder = 'images'
annuals_folder = 'annuals'
# Regular expression is only used to extract year from SCENE_ID
landsat_re = re.compile('L[ETC]0[4578]_\d{3}XXX_(?P<YEAR>\d{4})\d{2}\d{2}')
# if end_doy and end_doy > 273:
# logging.error(
# '\nERROR: End DOY has to be in the same water year as start DOY')
# sys.exit()
# Get ee features from shapefile
zone_geom_list = gdc.shapefile_2_geom_list_func(
ini['INPUTS']['zone_shp_path'],
zone_field=ini['INPUTS']['zone_field'],
reverse_flag=False)
# zone_count = len(zone_geom_list)
# output_fmt = '_{0:0%sd}.csv' % str(int(math.log10(zone_count)) + 1)
# Check if the zone_names are unique
# Eventually support merging common zone_names
if len(set([z[1] for z in zone_geom_list])) != len(zone_geom_list):
logging.error(
'\nERROR: There appear to be duplicate zone ID/name values.'
'\n Currently, the values in "{}" must be unique.'
'\n Exiting.'.format(ini['INPUTS']['zone_field']))
return False
# Filter features by FID
if ini['INPUTS']['fid_keep_list']:
zone_geom_list = [
zone_obj for zone_obj in zone_geom_list
if zone_obj[0] in ini['INPUTS']['fid_keep_list']
]
if ini['INPUTS']['fid_skip_list']:
zone_geom_list = [
zone_obj for zone_obj in zone_geom_list
if zone_obj[0] not in ini['INPUTS']['fid_skip_list']
]
# Merge geometries
if ini['INPUTS']['merge_geom_flag']:
merge_geom = ogr.Geometry(ogr.wkbMultiPolygon)
for zone in zone_geom_list:
zone_multipolygon = ogr.ForceToMultiPolygon(
ogr.CreateGeometryFromJson(json.dumps(zone[2])))
for zone_polygon in zone_multipolygon:
merge_geom.AddGeometry(zone_polygon)
# merge_json = json.loads(merge_mp.ExportToJson())
zone_geom_list = [[
0, ini['INPUTS']['zone_filename'],
json.loads(merge_geom.ExportToJson())
]]
ini['INPUTS']['zone_field'] = ''
# Set all zone specific parameters into a dictionary
zone = {}
# Need zone_shp_path projection to build EE geometries
zone['osr'] = gdc.feature_path_osr(ini['INPUTS']['zone_shp_path'])
zone['proj'] = gdc.osr_wkt(zone['osr'])
# zone['proj'] = ee.Projection(zone['proj']).wkt().getInfo()
# zone['proj'] = zone['proj'].replace('\n', '').replace(' ', '')
# logging.debug(' Zone Projection: {}'.format(zone['proj']))
# Check that shapefile has matching spatial reference
if not gdc.matching_spatref(zone['osr'], ini['SPATIAL']['osr']):
logging.warning(' Zone OSR:\n{}\n'.format(zone['osr']))
logging.warning(' Output OSR:\n{}\n'.format(
ini['SPATIAL']['osr'].ExportToWkt()))
logging.warning(' Zone Proj4: {}'.format(
zone['osr'].ExportToProj4()))
logging.warning(' Output Proj4: {}'.format(
ini['SPATIAL']['osr'].ExportToProj4()))
logging.warning(
'\nWARNING: \n'
'The output and zone spatial references do not appear to match\n'
'This will likely cause problems!')
input('Press ENTER to continue')
else:
logging.debug(' Zone Projection:\n{}\n'.format(
zone['osr'].ExportToWkt()))
logging.debug(' Output Projection:\n{}\n'.format(
ini['SPATIAL']['osr'].ExportToWkt()))
logging.debug(' Output Cellsize: {}'.format(
ini['SPATIAL']['cellsize']))
# Initialize Earth Engine API key
logging.info('\nInitializing Earth Engine')
ee.Initialize()
utils.ee_request(ee.Number(1).getInfo())
# Get list of path/row strings to centroid coordinates
if ini['INPUTS']['tile_keep_list']:
ini['INPUTS']['tile_geom'] = [
wrs2.tile_centroids[tile]
for tile in ini['INPUTS']['tile_keep_list']
if tile in wrs2.tile_centroids.keys()
]
ini['INPUTS']['tile_geom'] = ee.Geometry.MultiPoint(
ini['INPUTS']['tile_geom'], 'EPSG:4326')
else:
ini['INPUTS']['tile_geom'] = None
# Read in ETo and PPT data from file
if (ini['BEAMER']['eto_source'] == 'file'
or ini['BEAMER']['ppt_source'] == 'file'):
data_array = np.genfromtxt(ini['BEAMER']['data_path'],
delimiter=',',
names=True,
dtype=None)
data_fields = data_array.dtype.names
logging.debug(' CSV fields: {}'.format(', '.join(data_fields)))
# DEADBEEF - Compare fields names assuming all upper case
data_fields = [f.upper() for f in data_fields]
eto_dict = defaultdict(dict)
ppt_dict = defaultdict(dict)
for row in data_array:
z = str(row[data_fields.index(ini['BEAMER']['data_zone_field'])])
y = int(row[data_fields.index(ini['BEAMER']['data_year_field'])])
if ini['BEAMER']['eto_source'] == 'file':
# DEADBEEF - Compare fields names assuming all upper case
eto_dict[z][y] = row[data_fields.index(
ini['BEAMER']['data_eto_field'].upper())]
if ini['BEAMER']['ppt_source'] == 'file':
# DEADBEEF - Compare fields names assuming all upper case
ppt_dict[z][y] = row[data_fields.index(
ini['BEAMER']['data_ppt_field'].upper())]
# Get filtered/merged/prepped Landsat collection
landsat_args = {
k: v
for section in ['INPUTS'] for k, v in ini[section].items() if k in [
'landsat4_flag', 'landsat5_flag', 'landsat7_flag', 'landsat8_flag',
'fmask_flag', 'acca_flag', 'start_year', 'end_year', 'start_month',
'end_month', 'start_doy', 'end_doy', 'scene_id_keep_list',
'scene_id_skip_list', 'path_keep_list', 'row_keep_list',
'tile_geom', 'adjust_method', 'mosaic_method', 'refl_sur_method'
]
}
landsat = ee_common.Landsat(landsat_args)
# Download images for each feature separately
for zone_fid, zone_name, zone_json in zone_geom_list:
zone['fid'] = zone_fid
zone['name'] = zone_name.replace(' ', '_')
zone['json'] = zone_json
logging.info('ZONE: {} (FID: {})'.format(zone['name'], zone['fid']))
# Build EE geometry object for zonal stats
zone['geom'] = ee.Geometry(geo_json=zone['json'],
opt_proj=zone['proj'],
opt_geodesic=False)
# logging.debug(' Centroid: {}'.format(
# zone['geom'].centroid(100).getInfo()['coordinates']))
# Use feature geometry to build extent, transform, and shape
zone['extent'] = gdc.Extent(
ogr.CreateGeometryFromJson(json.dumps(zone['json'])).GetEnvelope())
# zone['extent'] = gdc.Extent(zone['geom'].GetEnvelope())
zone['extent'] = zone['extent'].ogrenv_swap()
zone['extent'] = zone['extent'].adjust_to_snap(
'EXPAND', ini['SPATIAL']['snap_x'], ini['SPATIAL']['snap_y'],
ini['SPATIAL']['cellsize'])
zone['geo'] = zone['extent'].geo(ini['SPATIAL']['cellsize'])
zone['transform'] = gdc.geo_2_ee_transform(zone['geo'])
# zone['transform'] = '[' + ','.join(map(str, zone['transform'])) + ']'
zone['shape'] = zone['extent'].shape(ini['SPATIAL']['cellsize'])
logging.debug(' Zone Shape: {}'.format(zone['shape']))
logging.debug(' Zone Transform: {}'.format(zone['transform']))
logging.debug(' Zone Extent: {}'.format(zone['extent']))
# logging.debug(' Zone Geom: {}'.format(zone['geom'].getInfo()))
# Assume all pixels in all 14+2 images could be reduced
zone['max_pixels'] = zone['shape'][0] * zone['shape'][1]
logging.debug(' Max Pixels: {}'.format(zone['max_pixels']))
# Set output spatial reference
# Eventually allow user to manually set these
# output_crs = zone['proj']
logging.debug(' Image Projection: {}'.format(ini['SPATIAL']['crs']))
# output_transform = zone['transform'][:]
output_transform = '[' + ','.join(map(str, zone['transform'])) + ']'
output_shape = '{1}x{0}'.format(*zone['shape'])
logging.debug(' Image Transform: {}'.format(output_transform))
logging.debug(' Image Shape: {}'.format(output_shape))
zone_output_ws = os.path.join(ini['IMAGES']['output_ws'], zone_name)
zone_zips_ws = os.path.join(zone_output_ws, zips_folder)
zone_images_ws = os.path.join(zone_output_ws, images_folder)
zone_annuals_ws = os.path.join(zone_output_ws, annuals_folder)
if not os.path.isdir(zone_zips_ws):
os.makedirs(zone_zips_ws)
if not os.path.isdir(zone_images_ws):
os.makedirs(zone_images_ws)
if not os.path.isdir(zone_annuals_ws):
os.makedirs(zone_annuals_ws)
# Initialize the Landsat object
# Limit Landsat products for getting SCENE IDs
landsat.products = []
landsat.zone_geom = zone['geom']
landsat_coll = landsat.get_collection()
# if ee.Image(landsat_coll.first()).getInfo() is None:
# logging.info(' No images, skipping')
# continue
# Get the full list of scene IDs
logging.debug(' Getting SCENE_ID list')
scene_id_list = sorted(
utils.ee_getinfo(landsat_coll.aggregate_histogram('SCENE_ID')))
logging.debug(' {} scenes'.format(len(scene_id_list)))
# Switch Landsat products for computing ETg
landsat.products = ['evi_sur']
# Process each image in the collection by date
for image_id in scene_id_list:
logging.info('{}'.format(image_id))
zip_path = os.path.join(zone_zips_ws, '{}.zip'.format(image_id))
logging.debug(' Zip: {}'.format(zip_path))
if os.path.isfile(zip_path) and overwrite_flag:
logging.debug(' Output already exists, removing zip')
os.remove(zip_path)
elif os.path.isfile(zip_path) and not overwrite_flag:
# Check that existing ZIP files can be opened
try:
with zipfile.ZipFile(zip_path, 'r') as z:
pass
logging.debug(' Output already exists, skipping')
continue
except Exception as e:
logging.warning(' Zip file error, removing')
os.remove(zip_path)
# Getting the date directly from the SCENE_ID
image_start_dt = datetime.datetime.strptime(
image_id[12:], '%Y%m%d')
image_end_dt = image_start_dt + datetime.timedelta(days=1)
logging.debug(' {} {}'.format(image_start_dt.date(),
image_end_dt.date()))
year = image_start_dt.year
# Filter the GRIDMET collection
wy_start_date = '{}-10-01'.format(year - 1)
wy_end_date = '{}-10-01'.format(year)
logging.debug(' WY: {} {}'.format(wy_start_date, wy_end_date))
gridmet_coll = ee.ImageCollection('IDAHO_EPSCOR/GRIDMET') \
.filterDate(wy_start_date, wy_end_date)
# # PRISM collection was uploaded as an asset
# if ini['BEAMER']['ppt_source'] == 'prism':
# def prism_time_start(input_image):
# """Set time_start property on PRISM water year PPT collection"""
# # Assume year is the 4th item separated by "_"
# water_year = ee.String(input_image.get('system:index')).split('_').get(3)
# date_start = ee.Date(ee.String(water_year).cat('-10-01'))
# return input_image.select([0], ['ppt']).set({
# 'system:time_start': date_start.millis()
# })
# prism_coll = ee.ImageCollection('users/cgmorton/prism_800m_ppt_wy')
# prism_coll = prism_coll.map(prism_time_start) \
# .filterDate(wy_start_date, wy_end_date)
# Get water year PPT from file
# Convert all input data to mm to match GRIDMET data
if ini['BEAMER']['ppt_source'] == 'file':
wy_ppt_input = ppt_dict[zone_name][year]
if ini['BEAMER']['data_ppt_units'] == 'mm':
pass
elif ini['BEAMER']['data_ppt_units'] == 'inches':
wy_ppt_input *= 25.4
elif ini['BEAMER']['data_ppt_units'] == 'feet':
wy_ppt_input *= (25.4 * 12)
elif ini['BEAMER']['ppt_source'] == 'gridmet':
# GET GRIDMET value at centroid of geometry
wy_ppt_input = float(
utils.ee_getinfo(
ee.ImageCollection(
gridmet_coll.select(['pr'],
['ppt']).sum()).getRegion(
zone['geom'].centroid(1),
500))[1][4])
# Calculate GRIDMET zonal mean of geometry
# wy_ppt_input = float(ee.ImageCollection(
# gridmet_coll.select(['pr'], ['ppt'])).reduceRegion(
# reducer=ee.Reducer.sum(),
# geometry=zone['geom'],
# crs=ini['SPATIAL']['crs'],
# crsTransform=zone['transform'],
# bestEffort=False,
# tileScale=1).getInfo()['ppt']
# elif ini['BEAMER']['ppt_source'] == 'prism':
# # Calculate PRISM zonal mean of geometry
# wy_ppt_input = float(ee.ImageCollection(
# prism_coll.map(ee_common.prism_ppt_func)).sum().reduceRegion(
# reducer=ee.Reducer.mean(),
# geometry=zone['geom'],
# crs=ini['SPATIAL']['crs'],
# crsTransform=zone['transform'],
# bestEffort=False,
# tileScale=1).getInfo()['ppt'])
# Get water year ETo read from file
# Convert all input data to mm for Beamer Method
if ini['BEAMER']['eto_source'] == 'file':
wy_eto_input = eto_dict[zone_name][year]
if ini['BEAMER']['data_eto_units'] == 'mm':
pass
elif ini['BEAMER']['data_eto_units'] == 'inches':
wy_eto_input *= 25.4
elif ini['BEAMER']['data_eto_units'] == 'feet':
wy_eto_input *= (25.4 * 12)
# This assumes GRIMET data is in millimeters
elif ini['BEAMER']['eto_source'] == 'gridmet':
wy_eto_input = float(
utils.ee_getinfo(
ee.ImageCollection(gridmet_coll.select(
['eto']).sum()).getRegion(zone['geom'].centroid(1),
500))[1][4])
# wy_eto_input = float(utils.ee_getinfo(ee.ImageCollection(
# gridmet_coll.select(['eto'])).reduceRegion(
# reducer=ee.Reducer.sum(),
# geometry=zone['geom'],
# crs=zone_proj,
# crsTransform=zone_transform,
# bestEffort=False,
# tileScale=1).getInfo()))
logging.debug(' Input ETO: {} mm PPT: {} mm'.format(
wy_eto_input, wy_ppt_input))
# Scale ETo & PPT
wy_eto_input *= ini['BEAMER']['eto_factor']
wy_ppt_input *= ini['BEAMER']['ppt_factor']
# Convert output units from mm
wy_ppt_output = wy_ppt_input
wy_eto_output = wy_eto_input
if ini['IMAGES']['ppt_units'] == 'mm':
pass
elif ini['IMAGES']['ppt_units'] == 'in':
wy_ppt_output /= 25.4
elif ini['IMAGES']['ppt_units'] == 'ft':
wy_ppt_output /= (25.4 * 12)
if ini['IMAGES']['eto_units'] == 'mm':
pass
elif ini['IMAGES']['eto_units'] == 'in':
wy_eto_output /= 25.4
elif ini['IMAGES']['eto_units'] == 'ft':
wy_eto_output /= (25.4 * 12)
logging.debug(' Output ETO: {} {} PPT: {} {}'.format(
wy_eto_output, ini['IMAGES']['eto_units'], wy_ppt_output,
ini['IMAGES']['ppt_units']))
# Add water year ETo and PPT values to each image
def eto_ppt_func(img):
""""""
return ee.Image(img).setMulti({
'wy_eto': wy_eto_output,
'wy_ppt': wy_ppt_output
})
# Compute EVI_SUR, add ETo and PPT, then Compute ETg
# Set the masked values to a nodata value
# so that the TIF can have a nodata value other than 0 set
landsat_image = eto_ppt_func(
landsat.get_image(landsat,
image_start_dt.year,
image_start_dt.strftime('%j'),
path=image_id[5:8],
row=None))
etg_image = ee.Image(ee_common.beamer_func(landsat_image)) \
.clip(zone['geom']) \
.unmask(nodata_value, False)
# Get the download URL
logging.debug(' Requesting URL')
zip_url = utils.ee_request(
etg_image.getDownloadURL({
'name': image_id,
'crs': ini['SPATIAL']['crs'],
'crs_transform': output_transform,
'dimensions': output_shape
}))
del etg_image
# Remove the scene from scen list if it's not going to work
if not zip_url:
scene_id_list.remove(image_id)
continue
# Try downloading a few times
logging.info(' Downloading')
for i in range(1, 10):
try:
response = urlrequest.urlopen(zip_url)
with open(zip_path, 'wb') as output_f:
shutil.copyfileobj(response, output_f)
break
except Exception as e:
logging.info(' Resending query')
logging.debug(' {}'.format(e))
sleep(i**2)
os.remove(zip_path)
# Remove the scene from scen list if it's not going to work
if not os.path.isfile(zip_path):
scene_id_list.remove(image_id)
logging.info('\nExtracting images')
for image_id in scene_id_list:
logging.info('{}'.format(image_id))
zip_path = os.path.join(zone_zips_ws, '{}.zip'.format(image_id))
logging.debug(' Zip: {}'.format(zip_path))
if not os.path.isfile(zip_path):
logging.debug(' zip file does not exist, skipping')
# Skip if all output images are present
image_band_list = [
os.path.join(zone_images_ws,
'{}.{}.tif'.format(image_id, band))
for band in ini['IMAGES']['download_bands']
]
if (not overwrite_flag
and all(os.path.isfile(x) for x in image_band_list)):
logging.debug(' all images present, skipping')
continue
# Otherwise, remove existing images
for image_path in image_band_list:
for file_path in glob.glob(image_path.replace('.tif', '.*')):
os.remove(file_path)
# Try extracting the files
try:
logging.debug(' Extracting')
with zipfile.ZipFile(zip_path, 'r') as z:
z.extractall(zone_images_ws)
except Exception as e:
logging.warning(' Error: could not extract'.format(i))
logging.debug(' {}'.format(e))
try:
os.remove(zip_path)
except Exception as e:
pass
# Set nodata value
for item in os.listdir(zone_images_ws):
if item.startswith(image_id) and item.endswith('.tif'):
gdc.raster_path_set_nodata(
os.path.join(zone_images_ws, item), nodata_value)
raster_statistics(os.path.join(zone_images_ws, item))
logging.info('\nComputing annual means')
for band in ini['IMAGES']['download_bands']:
logging.info(' {}'.format(band))
for year in range(ini['INPUTS']['start_year'],
ini['INPUTS']['end_year'] + 1):
logging.info(' {}'.format(year))
mean_path = os.path.join(
# zone_annuals_ws, 'etg_{}_{}.{}.tif'.format(
zone_annuals_ws,
'{}_{}.{}.tif'.format(zone_name.lower().replace(' ', '_'),
year, band))
logging.debug(' {}'.format(mean_path))
# if os.path.isfile(mean_path) and not overwrite_flag:
# logging.debug(' file already exists, skipping')
# continue
image_band_list = [
os.path.join(zone_images_ws, item)
for item in os.listdir(zone_images_ws)
if (item.endswith('.{}.tif'.format(band)) and (
int(landsat_re.match(item).group('YEAR')) == year))
]
# for image_path in image_band_list:
# raster_path_set_nodata(image_path, nodata_value)
if not image_band_list:
continue
# Use GDAL to compute the composite
cell_statistics(image_band_list, mean_path, 'mean')
raster_statistics(mean_path)
logging.info('\nComputing composite rasters from annual means')
for stat in stat_list:
logging.info(' Stat: {}'.format(stat))
for band in ini['IMAGES']['download_bands']:
logging.info(' {}'.format(band))
image_band_list = [
os.path.join(zone_annuals_ws, item)
for item in os.listdir(zone_annuals_ws)
if item.endswith('.{}.tif'.format(band))
]
# for image_path in image_band_list:
# raster_path_set_nodata(image_path, nodata_value)
output_path = os.path.join(
zone_output_ws,
'{}_{}.{}.tif'.format(zone_name.lower().replace(' ', '_'),
stat.lower(), band.lower()))
logging.debug(' {}'.format(output_path))
# Use GDAL to compute the composite raster
cell_statistics(image_band_list, output_path, 'mean')
raster_statistics(output_path)
