def InitializeApi(self):
"""Initializes the library with standard API methods.
This is normally invoked during setUp(), but subclasses may invoke
it manually instead if they prefer.
"""
self.last_download_call = None
self.last_thumb_call = None
self.last_table_call = None
def MockSend(path, params, unused_method=None, unused_raw=None):
if path == '/algorithms':
return BUILTIN_FUNCTIONS
elif path == '/value':
return {'value': 'fakeValue'}
elif path == '/mapid':
return {'mapid': 'fakeMapId'}
elif path == '/download':
# Hang on to the call arguments.
self.last_download_call = {'url': path, 'data': params}
return {'docid': '1', 'token': '2'}
elif path == '/thumb':
# Hang on to the call arguments.
self.last_thumb_call = {'url': path, 'data': params}
return {'thumbid': '3', 'token': '4'}
elif path == '/table':
# Hang on to the call arguments.
self.last_table_call = {'url': path, 'data': params}
return {'docid': '5', 'token': '6'}
else:
raise Exception('Unexpected API call to %s with %s' % (path, params))
ee.data.send_ = MockSend
ee.Reset()
ee.Initialize(None, '')
def testInitialization(self):
"""Verifies library initialization."""
def MockAlgorithms():
return {}
ee.data.getAlgorithms = MockAlgorithms
# Verify that the base state is uninitialized.
self.assertFalse(ee.data._initialized)
self.assertEqual(ee.data._api_base_url, None)
self.assertEqual(ee.ApiFunction._api, None)
self.assertFalse(ee.Image._initialized)
# Verify that ee.Initialize() sets the URL and initializes classes.
ee.Initialize(None, 'foo')
self.assertTrue(ee.data._initialized)
self.assertEqual(ee.data._api_base_url, 'foo/api')
self.assertEqual(ee.ApiFunction._api, {})
self.assertTrue(ee.Image._initialized)
# Verify that ee.Initialize(None) does not override custom URLs.
ee.Initialize(None)
self.assertTrue(ee.data._initialized)
self.assertEqual(ee.data._api_base_url, 'foo/api')
# Verify that ee.Reset() reverts everything to the base state.
ee.Reset()
self.assertFalse(ee.data._initialized)
self.assertEqual(ee.data._api_base_url, None)
self.assertEqual(ee.ApiFunction._api, None)
self.assertFalse(ee.Image._initialized)
def testInitialization(self):
"""Verifies library initialization."""
def MockSend(path, params, unused_method=None, unused_raw=None):
if path == '/algorithms':
return {}
else:
raise Exception('Unexpected API call to %s with %s' %
(path, params))
ee.data.send_ = MockSend
# Verify that the base state is uninitialized.
self.assertFalse(ee.data._initialized)
self.assertEqual(ee.data._api_base_url, None)
self.assertEqual(ee.ApiFunction._api, None)
self.assertFalse(ee.Image._initialized)
# Verify that ee.Initialize() sets the URL and initializes classes.
ee.Initialize(None, 'foo', use_cloud_api=False)
self.assertTrue(ee.data._initialized)
self.assertEqual(ee.data._api_base_url, 'foo/api')
self.assertEqual(ee.ApiFunction._api, {})
self.assertTrue(ee.Image._initialized)
# Verify that ee.Initialize(None) does not override custom URLs.
ee.Initialize(None, use_cloud_api=False)
self.assertTrue(ee.data._initialized)
self.assertEqual(ee.data._api_base_url, 'foo/api')
# Verify that ee.Reset() reverts everything to the base state.
ee.Reset()
self.assertFalse(ee.data._initialized)
self.assertEqual(ee.data._api_base_url, None)
self.assertEqual(ee.ApiFunction._api, None)
self.assertFalse(ee.Image._initialized)
def extr_GLDAS_ts_GEE(lon, lat, varname='SoilMoi0_10cm_inst', bufferSize=20, yearlist=None):
ee.Reset()
ee.Initialize()
def createAvg(image):
gee_roi = ee.Geometry.Point(lon, lat).buffer(bufferSize)
reduced_img_data = image.reduceRegion(ee.Reducer.median(), gee_roi, 30)
return ee.Feature(None, {'result': reduced_img_data})
if yearlist == None:
# yearlist = range(1987,2018)
yearlist = range(2011, 2018)
SM_list = list()
for iyear in yearlist:
# ee.Reset()
# ee.Initialize()
print(iyear)
# load collection
if iyear < 2000:
GLDAS_collection = ee.ImageCollection('NASA/GLDAS/V20/NOAH/G025/T3H').select(varname)
else:
GLDAS_collection = ee.ImageCollection('NASA/GLDAS/V021/NOAH/G025/T3H').select(varname)
GLDAS_collection = GLDAS_collection.filterDate(str(iyear) + '-01-01', str(iyear) + '-12-31')
GLDAS_collection = GLDAS_collection.filter(ee.Filter.calendarRange(16, 18, 'hour'))
# clip
roi = ee.Geometry.Point(lon, lat).buffer(bufferSize)
GLDAS_collection = GLDAS_collection.map(lambda image: image.clip(roi))
# extract time series
time_series = GLDAS_collection.map(createAvg)
tmp = time_series.getInfo()
SM = np.array([x['properties']['result'][varname] for x in tmp['features']], dtype=np.float)
ge_dates = np.array([datetime.strptime(x['id'], 'A%Y%m%d_%H%M') for x in tmp['features']])
valid = np.where(np.isfinite(SM))
# cut out invalid values
SM = SM[valid]
ge_dates = ge_dates[valid]
SM_series = pd.Series(SM, index=ge_dates, copy=True, name='GLDAS')
SM_list.append(SM_series)
return (pd.concat(SM_list))
def InitializeApi(self):
"""Initializes the library with standard API methods.
This is normally invoked during setUp(), but subclasses may invoke
it manually instead if they prefer.
"""
self.last_download_call = None
self.last_thumb_call = None
self.last_table_call = None
ee.data.send_ = self.MockSend
ee.Reset()
ee.Initialize(None, '')
def init_ee(account_name):
import json
import oauth2client
global gee_account_name
cred_dict = json.load(open(gee_credentials_file))
refresh_token = cred_dict.get(account_name)
if refresh_token is None:
raise Exception(
'Credentials not defined for account name {}!'.format(account_name))
credentials = oauth2client.client.OAuth2Credentials(
None, ee.oauth.CLIENT_ID, ee.oauth.CLIENT_SECRET, refresh_token, None, 'https://accounts.google.com/o/oauth2/token', None)
ee.Reset()
ee.Initialize(credentials=credentials)
gee_account_name = account_name
def getWrsCodes(regionsNames):
ee.Initialize(EE_CREDENTIALS)
regions = ee.FeatureCollection("users/lapig/countries")
wrs = ee.FeatureCollection("users/lapig/WRS2")
query = ee.Filter.inList('ISO', regionsNames)
selectedCountries = ee.Feature(regions.filter(query))
selectedRegion = ee.FeatureCollection(selectedCountries).flatten()
wrs_filtered = wrs.filterBounds(selectedRegion.geometry())
wrs_list = wrs_filtered.toList(wrs_filtered.size())
listWrs = list(wrs_list.map(getWRS).getInfo())
listWrs.sort()
ee.Reset()
return listWrs
def extr_MODIS_MOD13Q1_ts_GEE(lon, lat, bufferSize=100, datefilter=None):
ee.Reset()
ee.Initialize()
def createAvg(image):
# mask image
immask = image.select('SummaryQA').eq(ee.Image(0))
image = image.updateMask(immask)
reduced_img_data = image.reduceRegion(ee.Reducer.median(),
ee.Geometry.Point(lon, lat).buffer(bufferSize),
50)
return ee.Feature(None, {'result': reduced_img_data})
# load collection
gee_l8_collection = ee.ImageCollection('MODIS/006/MOD13Q1')
# filter collection
gee_roi = ee.Geometry.Point(lon, lat).buffer(bufferSize)
gee_l8_fltd = gee_l8_collection.filterBounds(gee_roi)
if datefilter is not None:
gee_l8_fltd = gee_l8_fltd.filterDate(datefilter[0], datefilter[1])
# extract time series
gee_l8_mpd = gee_l8_fltd.map(createAvg)
tmp = gee_l8_mpd.getInfo()
EVI = np.array([x['properties']['result']['EVI'] for x in tmp['features']], dtype=np.float)
ge_dates = np.array([datetime.strptime(x['id'], '%Y_%m_%d') for x in tmp['features']])
valid = np.where(np.isfinite(EVI))
if len(valid[0]) == 0:
success = 0
return (None, success)
# cut out invalid values
EVI = EVI[valid]
ge_dates = ge_dates[valid]
success = 1
return (pd.Series(EVI, index=ge_dates, name='EVI'), success)
def setUp(self): ee.Reset() ee.data._install_cloud_api_resource = lambda: None
def setUp(self):
ee.Reset()
print(mosaicId + ' exists and is valid.')
client = MongoClient(MONGO_HOST, MONGO_PORT)
db = client.tvi
gee_multi_credentials(CREDENTIALS_DIR)
if len(sys.argv) > 4:
TILES_BR = getWrsCodes(sys.argv[4])
for year in range(1985, 2021 + 1):
ee.Initialize()
LANDSAT_5 = ee.ImageCollection("LANDSAT/LT05/C01/T1_TOA")
LANDSAT_7 = ee.ImageCollection("LANDSAT/LE07/C01/T1_TOA")
LANDSAT_8 = ee.ImageCollection("LANDSAT/LC08/C01/T1_TOA")
for satellite in SATELLITES:
if (satellite == 'L8' and year < 2013) or (satellite == 'L5'
and year > 2011):
continue
if (satellite == 'L7' and year < 2000):
continue
processPeriod(TILES_BR, PERIODS_BR)
ee.Reset()
def extr_GEE_array_reGE(minlon, minlat, maxlon, maxlat, year, month, day, tempfilter=True, applylcmask=True,
sampling=20, dualpol=True, trackflt=None, maskwinter=True, masksnow=True):
def maskterrain(image):
# srtm dem
gee_srtm = ee.Image("USGS/SRTMGL1_003")
gee_srtm_slope = ee.Terrain.slope(gee_srtm)
tmp = ee.Image(image)
mask = gee_srtm_slope.lt(20)
mask2 = tmp.lt(0).bitwiseAnd(tmp.gt(-25))
# mask2 = tmp.gte(-25)
mask = mask.bitwiseAnd(mask2)
tmp = tmp.updateMask(mask)
return (tmp)
def masklc(image):
# load land cover info
corine = ee.Image('users/felixgreifeneder/corine')
# create lc mask
valLClist = [10, 11, 12, 13, 18, 19, 20, 21, 26, 27, 28, 29]
lcmask = corine.eq(valLClist[0]).bitwiseOr(corine.eq(valLClist[1])) \
.bitwiseOr(corine.eq(valLClist[2])) \
.bitwiseOr(corine.eq(valLClist[3])) \
.bitwiseOr(corine.eq(valLClist[4])) \
.bitwiseOr(corine.eq(valLClist[5])) \
.bitwiseOr(corine.eq(valLClist[6])) \
.bitwiseOr(corine.eq(valLClist[7])) \
.bitwiseOr(corine.eq(valLClist[8])) \
.bitwiseOr(corine.eq(valLClist[9])) \
.bitwiseOr(corine.eq(valLClist[10])) \
.bitwiseOr(corine.eq(valLClist[11]))
tmp = ee.Image(image)
tmp = tmp.updateMask(lcmask)
return (tmp)
def setresample(image):
image = image.resample()
return (image)
def toln(image):
tmp = ee.Image(image)
# Convert to linear
vv = ee.Image(10).pow(tmp.select('VV').divide(10))
if dualpol == True:
vh = ee.Image(10).pow(tmp.select('VH').divide(10))
# Convert to ln
out = vv.log()
if dualpol == True:
out = out.addBands(vh.log())
out = out.select(['constant', 'constant_1'], ['VV', 'VH'])
else:
out = out.select(['constant'], ['VV'])
return out.set('system:time_start', tmp.get('system:time_start'))
def tolin(image):
tmp = ee.Image(image)
# Covert to linear
vv = ee.Image(10).pow(tmp.select('VV').divide(10))
if dualpol == True:
vh = ee.Image(10).pow(tmp.select('VH').divide(10))
# Convert to
if dualpol == True:
out = vv.addBands(vh)
out = out.select(['constant', 'constant_1'], ['VV', 'VH'])
else:
out = vv.select(['constant'], ['VV'])
return out.set('system:time_start', tmp.get('system:time_start'))
def todb(image):
tmp = ee.Image(image)
return ee.Image(10).multiply(tmp.log10()).set('system:time_start', tmp.get('system:time_start'))
def applysnowmask(image):
tmp = ee.Image(image)
sdiff = tmp.select('VH').subtract(snowref)
wetsnowmap = sdiff.lte(-2.6).focal_mode(100, 'square', 'meters', 3)
return (tmp.updateMask(wetsnowmap.eq(0)))
ee.Reset()
ee.Initialize()
# load S1 data
gee_s1_collection = ee.ImageCollection('COPERNICUS/S1_GRD')
# for lc_id in range(1, len(valLClist)):
# tmpmask = corine.eq(valLClist[lc_id])
# lcmask = lcmask.bitwiseAnd(tmpmask)
# construct roi
roi = ee.Geometry.Polygon([[minlon, minlat], [minlon, maxlat],
[maxlon, maxlat], [maxlon, minlat],
[minlon, minlat]])
# roi = get_south_tyrol_roi()
# ASCENDING acquisitions
gee_s1_filtered = gee_s1_collection.filter(ee.Filter.eq('instrumentMode', 'IW')) \
.filterBounds(roi) \
.filter(ee.Filter.eq('orbitProperties_pass', 'ASCENDING')) \
.filter(ee.Filter.eq('platform_number', 'A')) \
.filter(ee.Filter.listContains('transmitterReceiverPolarisation', 'VV'))
if dualpol == True:
gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.listContains('transmitterReceiverPolarisation', 'VH'))
if trackflt is not None:
gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.eq('relativeOrbitNumber_start', trackflt))
if maskwinter == True:
gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.dayOfYear(121, 304))
if applylcmask == True:
gee_s1_filtered = gee_s1_filtered.map(masklc)
gee_s1_filtered = gee_s1_filtered.map(setresample)
gee_s1_filtered = gee_s1_filtered.map(maskterrain)
# apply wet snow mask
if masksnow == True:
gee_s1_linear_vh = gee_s1_filtered.map(tolin).select('VH')
snowref = ee.Image(10).multiply(gee_s1_linear_vh.reduce(ee.Reducer.intervalMean(5, 100)).log10())
# snowdiff = s1_sig0_vh.subtract(snowref)
# wetsnowmap = snowdiff.lte(-2.6).focal_mode(100, 'square', 'meters', 3)
# s1_sig0_vv = s1_sig0_vv.updateMask(wetsnowmap.eq(0))
# if dualpol == True:
# s1_sig0_vh = s1_sig0_vh.updateMask(wetsnowmap.eq(0))
gee_s1_filtered = gee_s1_filtered.map(applysnowmask)
# create a list of availalbel dates
tmp = gee_s1_filtered.getInfo()
tmp_ids = [x['properties']['system:index'] for x in tmp['features']]
dates = np.array([dt.date(year=int(x[17:21]), month=int(x[21:23]), day=int(x[23:25])) for x in tmp_ids])
# find the closest acquisitions
doi = dt.date(year=year, month=month, day=day)
doi_index = np.argmin(np.abs(dates - doi))
date_selected = dates[doi_index]
# filter imagecollection for respective date
gee_s1_list = gee_s1_filtered.toList(2000)
doi_indices = np.where(dates == date_selected)[0]
gee_s1_drange = ee.ImageCollection(gee_s1_list.slice(np.int(doi_indices[0]), np.int(doi_indices[-1] + 1)))
s1_sig0 = gee_s1_drange.mosaic()
s1_sig0 = ee.Image(s1_sig0.copyProperties(gee_s1_drange.first()))
# fetch image from image collection
s1_lia = s1_sig0.select('angle').clip(roi)
# get the track number
s1_sig0_info = s1_sig0.getInfo()
track_nr = s1_sig0_info['properties']['relativeOrbitNumber_start']
# despeckle
if tempfilter == True:
radius = 7
units = 'pixels'
gee_s1_linear = gee_s1_filtered.map(tolin)
gee_s1_dspckld_vv = multitemporalDespeckle(gee_s1_linear.select('VV'), radius, units,
{'before': -12, 'after': 12, 'units': 'month'})
gee_s1_dspckld_vv = gee_s1_dspckld_vv.map(todb)
gee_s1_list_vv = gee_s1_dspckld_vv.toList(2000)
gee_s1_fltrd_vv = ee.ImageCollection(gee_s1_list_vv.slice(np.int(doi_indices[0]), np.int(doi_indices[-1] + 1)))
s1_sig0_vv = gee_s1_fltrd_vv.mosaic()
# s1_sig0_vv = ee.Image(gee_s1_list_vv.get(doi_index))
if dualpol == True:
gee_s1_dspckld_vh = multitemporalDespeckle(gee_s1_linear.select('VH'), radius, units,
{'before': -12, 'after': 12, 'units': 'month'})
gee_s1_dspckld_vh = gee_s1_dspckld_vh.map(todb)
gee_s1_list_vh = gee_s1_dspckld_vh.toList(2000)
gee_s1_fltrd_vh = ee.ImageCollection(gee_s1_list_vh.slice(np.int(doi_indices[0]), np.int(doi_indices[-1] + 1)))
s1_sig0_vh = gee_s1_fltrd_vh.mosaic()
if dualpol == True:
s1_sig0 = s1_sig0_vv.addBands(s1_sig0_vh).select(['constant', 'constant_1'], ['VV', 'VH'])
else:
s1_sig0 = s1_sig0_vv.select(['constant'], ['VV'])
# s1_sig0_dsc = s1_sig0_vv_dsc.select(['constant'], ['VV'])
# extract information
s1_sig0_vv = s1_sig0.select('VV')
s1_sig0_vv = s1_sig0_vv.clip(roi)
if dualpol == True:
s1_sig0_vh = s1_sig0.select('VH')
s1_sig0_vh = s1_sig0_vh.clip(roi)
# compute temporal statistics
gee_s1_filtered = gee_s1_filtered.filterMetadata('relativeOrbitNumber_start', 'equals', track_nr)
gee_s1_ln = gee_s1_filtered.map(toln)
# gee_s1_ln = gee_s1_ln.clip(roi)
k1vv = ee.Image(gee_s1_ln.select('VV').mean()).clip(roi)
k2vv = ee.Image(gee_s1_ln.select('VV').reduce(ee.Reducer.stdDev())).clip(roi)
if dualpol == True:
k1vh = ee.Image(gee_s1_ln.select('VH').mean()).clip(roi)
k2vh = ee.Image(gee_s1_ln.select('VH').reduce(ee.Reducer.stdDev())).clip(roi)
# mask insensitive pixels
# if dualpol == True:
# smask = k2vv.gt(0.4).And(k2vh.gt(0.4))
# else:
# smask = k2vv.gt(0.4)
#
# s1_sig0_vv = s1_sig0_vv.updateMask(smask)
# s1_lia = s1_lia.updateMask(smask)
# k1vv = k1vv.updateMask(smask)
# k2vv = k2vv.updateMask(smask)
#
# if dualpol == True:
# s1_sig0_vh = s1_sig0_vh.updateMask(smask)
# k1vh = k1vh.updateMask(smask)
# k2vh = k2vh.updateMask(smask)
# export
if dualpol == False:
# s1_sig0_vv = s1_sig0_vv.reproject(s1_lia.projection())
return (s1_sig0_vv, s1_lia, k1vv, k2vv, roi, date_selected)
else:
return (s1_lia.focal_median(sampling, 'square', 'meters'),
s1_sig0_vv.focal_median(sampling, 'square', 'meters'),
s1_sig0_vh.focal_median(sampling, 'square', 'meters'),
k1vv.focal_median(sampling, 'square', 'meters'),
k1vh.focal_median(sampling, 'square', 'meters'),
k2vv.focal_median(sampling, 'square', 'meters'),
k2vh.focal_median(sampling, 'square', 'meters'),
roi,
date_selected)
def extr_L8_ts_GEE(lon, lat, bufferSize=20):
ee.Reset()
ee.Initialize()
def createAvg(image):
gee_roi = ee.Geometry.Point(lon, lat).buffer(bufferSize)
# aerosols
image = image.updateMask(image.select('sr_aerosol').eq(2).bitwiseOr(
image.select('sr_aerosol').eq(32).bitwiseOr(
image.select('sr_aerosol').eq(96).bitwiseOr(
image.select('sr_aerosol').eq(160).bitwiseOr(
image.select('sr_aerosol').eq(66).bitwiseOr(
image.select('sr_aerosol').eq(130)
))))))
# clouds
def getQABits(image, start, end, newName):
pattern = 0
for i in range(start, end + 1):
pattern = pattern + int(math.pow(2, i))
return image.select([0], [newName]).bitwiseAnd(pattern).rightShift(start)
def cloud_shadows(image):
QA = image.select('pixel_qa')
return getQABits(QA, 3, 3, 'Cloud_shadows').eq(0)
def clouds(image):
QA = image.select('pixel_qa')
return getQABits(QA, 5, 5, 'Cloud').eq(0)
image = image.updateMask(cloud_shadows(image))
image = image.updateMask(clouds(image))
# # radiometric saturation
# image = image.updateMask(image.select('radsat_qa').eq(2))
reduced_img_data = image.reduceRegion(ee.Reducer.mean(), gee_roi, 30)
return ee.Feature(None, {'result': reduced_img_data})
def setresample(image):
image = image.resample()
return (image)
def mask_tree_cover(image):
tree_cover_image = ee.ImageCollection("GLCF/GLS_TCC").filterBounds(gee_roi).filter(
ee.Filter.eq('year', 2010)).mosaic()
treemask = tree_cover_image.select('tree_canopy_cover').clip(gee_roi).lte(20)
# load lc
glbcvr = ee.Image("ESA/GLOBCOVER_L4_200901_200912_V2_3").select('landcover')
# mask water
watermask = glbcvr.neq(210)
return image.updateMask(treemask.And(watermask))
# load collection
gee_l8_collection = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR') # .map(setresample)
# filter collection
gee_roi = ee.Geometry.Point(lon, lat).buffer(bufferSize)
gee_l8_fltd = gee_l8_collection.filterBounds(gee_roi)
# extract time series
#gee_l8_fltd = gee_l8_fltd.map(mask_tree_cover)
gee_l8_mpd = gee_l8_fltd.map(createAvg)
tmp = gee_l8_mpd.getInfo()
b1 = np.array([x['properties']['result']['B1'] for x in tmp['features']], dtype=np.float)
b2 = np.array([x['properties']['result']['B2'] for x in tmp['features']], dtype=np.float)
b3 = np.array([x['properties']['result']['B3'] for x in tmp['features']], dtype=np.float)
b4 = np.array([x['properties']['result']['B4'] for x in tmp['features']], dtype=np.float)
b5 = np.array([x['properties']['result']['B5'] for x in tmp['features']], dtype=np.float)
b6 = np.array([x['properties']['result']['B6'] for x in tmp['features']], dtype=np.float)
b7 = np.array([x['properties']['result']['B7'] for x in tmp['features']], dtype=np.float)
b10 = np.array([x['properties']['result']['B7'] for x in tmp['features']], dtype=np.float)
b11 = np.array([x['properties']['result']['B7'] for x in tmp['features']], dtype=np.float)
ge_dates = np.array([datetime.strptime(x['id'][12::], '%Y%m%d') for x in tmp['features']])
valid = np.where(np.isfinite(b2))
# cut out invalid values
b1 = b1[valid]
b2 = b2[valid]
b3 = b3[valid]
b4 = b4[valid]
b5 = b5[valid]
b6 = b6[valid]
b7 = b7[valid]
b10 = b10[valid]
b11 = b11[valid]
ge_dates = ge_dates[valid]
if b1.size == 0:
return None
else:
return pd.DataFrame({'B1': b1,
'B2': b2,
'B3': b3,
'B4': b4,
'B5': b5,
'B6': b6,
'B7': b7,
'B10': b10,
'B11': b11}, index=ge_dates)
