def InitializeApi(self): """Initializes the library with standard API methods. This is normally invoked during setUp(), but subclasses may invoke it manually instead if they prefer. """ self.last_download_call = None self.last_thumb_call = None self.last_table_call = None def MockSend(path, params, unused_method=None, unused_raw=None): if path == '/algorithms': return BUILTIN_FUNCTIONS elif path == '/value': return {'value': 'fakeValue'} elif path == '/mapid': return {'mapid': 'fakeMapId'} elif path == '/download': # Hang on to the call arguments. self.last_download_call = {'url': path, 'data': params} return {'docid': '1', 'token': '2'} elif path == '/thumb': # Hang on to the call arguments. self.last_thumb_call = {'url': path, 'data': params} return {'thumbid': '3', 'token': '4'} elif path == '/table': # Hang on to the call arguments. self.last_table_call = {'url': path, 'data': params} return {'docid': '5', 'token': '6'} else: raise Exception('Unexpected API call to %s with %s' % (path, params)) ee.data.send_ = MockSend ee.Reset() ee.Initialize(None, '')
def testInitialization(self): """Verifies library initialization.""" def MockAlgorithms(): return {} ee.data.getAlgorithms = MockAlgorithms # Verify that the base state is uninitialized. self.assertFalse(ee.data._initialized) self.assertEqual(ee.data._api_base_url, None) self.assertEqual(ee.ApiFunction._api, None) self.assertFalse(ee.Image._initialized) # Verify that ee.Initialize() sets the URL and initializes classes. ee.Initialize(None, 'foo') self.assertTrue(ee.data._initialized) self.assertEqual(ee.data._api_base_url, 'foo/api') self.assertEqual(ee.ApiFunction._api, {}) self.assertTrue(ee.Image._initialized) # Verify that ee.Initialize(None) does not override custom URLs. ee.Initialize(None) self.assertTrue(ee.data._initialized) self.assertEqual(ee.data._api_base_url, 'foo/api') # Verify that ee.Reset() reverts everything to the base state. ee.Reset() self.assertFalse(ee.data._initialized) self.assertEqual(ee.data._api_base_url, None) self.assertEqual(ee.ApiFunction._api, None) self.assertFalse(ee.Image._initialized)
def testInitialization(self): """Verifies library initialization.""" def MockSend(path, params, unused_method=None, unused_raw=None): if path == '/algorithms': return {} else: raise Exception('Unexpected API call to %s with %s' % (path, params)) ee.data.send_ = MockSend # Verify that the base state is uninitialized. self.assertFalse(ee.data._initialized) self.assertEqual(ee.data._api_base_url, None) self.assertEqual(ee.ApiFunction._api, None) self.assertFalse(ee.Image._initialized) # Verify that ee.Initialize() sets the URL and initializes classes. ee.Initialize(None, 'foo', use_cloud_api=False) self.assertTrue(ee.data._initialized) self.assertEqual(ee.data._api_base_url, 'foo/api') self.assertEqual(ee.ApiFunction._api, {}) self.assertTrue(ee.Image._initialized) # Verify that ee.Initialize(None) does not override custom URLs. ee.Initialize(None, use_cloud_api=False) self.assertTrue(ee.data._initialized) self.assertEqual(ee.data._api_base_url, 'foo/api') # Verify that ee.Reset() reverts everything to the base state. ee.Reset() self.assertFalse(ee.data._initialized) self.assertEqual(ee.data._api_base_url, None) self.assertEqual(ee.ApiFunction._api, None) self.assertFalse(ee.Image._initialized)
def extr_GLDAS_ts_GEE(lon, lat, varname='SoilMoi0_10cm_inst', bufferSize=20, yearlist=None): ee.Reset() ee.Initialize() def createAvg(image): gee_roi = ee.Geometry.Point(lon, lat).buffer(bufferSize) reduced_img_data = image.reduceRegion(ee.Reducer.median(), gee_roi, 30) return ee.Feature(None, {'result': reduced_img_data}) if yearlist == None: # yearlist = range(1987,2018) yearlist = range(2011, 2018) SM_list = list() for iyear in yearlist: # ee.Reset() # ee.Initialize() print(iyear) # load collection if iyear < 2000: GLDAS_collection = ee.ImageCollection('NASA/GLDAS/V20/NOAH/G025/T3H').select(varname) else: GLDAS_collection = ee.ImageCollection('NASA/GLDAS/V021/NOAH/G025/T3H').select(varname) GLDAS_collection = GLDAS_collection.filterDate(str(iyear) + '-01-01', str(iyear) + '-12-31') GLDAS_collection = GLDAS_collection.filter(ee.Filter.calendarRange(16, 18, 'hour')) # clip roi = ee.Geometry.Point(lon, lat).buffer(bufferSize) GLDAS_collection = GLDAS_collection.map(lambda image: image.clip(roi)) # extract time series time_series = GLDAS_collection.map(createAvg) tmp = time_series.getInfo() SM = np.array([x['properties']['result'][varname] for x in tmp['features']], dtype=np.float) ge_dates = np.array([datetime.strptime(x['id'], 'A%Y%m%d_%H%M') for x in tmp['features']]) valid = np.where(np.isfinite(SM)) # cut out invalid values SM = SM[valid] ge_dates = ge_dates[valid] SM_series = pd.Series(SM, index=ge_dates, copy=True, name='GLDAS') SM_list.append(SM_series) return (pd.concat(SM_list))
def InitializeApi(self): """Initializes the library with standard API methods. This is normally invoked during setUp(), but subclasses may invoke it manually instead if they prefer. """ self.last_download_call = None self.last_thumb_call = None self.last_table_call = None ee.data.send_ = self.MockSend ee.Reset() ee.Initialize(None, '')
def init_ee(account_name): import json import oauth2client global gee_account_name cred_dict = json.load(open(gee_credentials_file)) refresh_token = cred_dict.get(account_name) if refresh_token is None: raise Exception( 'Credentials not defined for account name {}!'.format(account_name)) credentials = oauth2client.client.OAuth2Credentials( None, ee.oauth.CLIENT_ID, ee.oauth.CLIENT_SECRET, refresh_token, None, 'https://accounts.google.com/o/oauth2/token', None) ee.Reset() ee.Initialize(credentials=credentials) gee_account_name = account_name
def getWrsCodes(regionsNames): ee.Initialize(EE_CREDENTIALS) regions = ee.FeatureCollection("users/lapig/countries") wrs = ee.FeatureCollection("users/lapig/WRS2") query = ee.Filter.inList('ISO', regionsNames) selectedCountries = ee.Feature(regions.filter(query)) selectedRegion = ee.FeatureCollection(selectedCountries).flatten() wrs_filtered = wrs.filterBounds(selectedRegion.geometry()) wrs_list = wrs_filtered.toList(wrs_filtered.size()) listWrs = list(wrs_list.map(getWRS).getInfo()) listWrs.sort() ee.Reset() return listWrs
def extr_MODIS_MOD13Q1_ts_GEE(lon, lat, bufferSize=100, datefilter=None): ee.Reset() ee.Initialize() def createAvg(image): # mask image immask = image.select('SummaryQA').eq(ee.Image(0)) image = image.updateMask(immask) reduced_img_data = image.reduceRegion(ee.Reducer.median(), ee.Geometry.Point(lon, lat).buffer(bufferSize), 50) return ee.Feature(None, {'result': reduced_img_data}) # load collection gee_l8_collection = ee.ImageCollection('MODIS/006/MOD13Q1') # filter collection gee_roi = ee.Geometry.Point(lon, lat).buffer(bufferSize) gee_l8_fltd = gee_l8_collection.filterBounds(gee_roi) if datefilter is not None: gee_l8_fltd = gee_l8_fltd.filterDate(datefilter[0], datefilter[1]) # extract time series gee_l8_mpd = gee_l8_fltd.map(createAvg) tmp = gee_l8_mpd.getInfo() EVI = np.array([x['properties']['result']['EVI'] for x in tmp['features']], dtype=np.float) ge_dates = np.array([datetime.strptime(x['id'], '%Y_%m_%d') for x in tmp['features']]) valid = np.where(np.isfinite(EVI)) if len(valid[0]) == 0: success = 0 return (None, success) # cut out invalid values EVI = EVI[valid] ge_dates = ge_dates[valid] success = 1 return (pd.Series(EVI, index=ge_dates, name='EVI'), success)
def setUp(self): ee.Reset() ee.data._install_cloud_api_resource = lambda: None
def setUp(self): ee.Reset()
print(mosaicId + ' exists and is valid.') client = MongoClient(MONGO_HOST, MONGO_PORT) db = client.tvi gee_multi_credentials(CREDENTIALS_DIR) if len(sys.argv) > 4: TILES_BR = getWrsCodes(sys.argv[4]) for year in range(1985, 2021 + 1): ee.Initialize() LANDSAT_5 = ee.ImageCollection("LANDSAT/LT05/C01/T1_TOA") LANDSAT_7 = ee.ImageCollection("LANDSAT/LE07/C01/T1_TOA") LANDSAT_8 = ee.ImageCollection("LANDSAT/LC08/C01/T1_TOA") for satellite in SATELLITES: if (satellite == 'L8' and year < 2013) or (satellite == 'L5' and year > 2011): continue if (satellite == 'L7' and year < 2000): continue processPeriod(TILES_BR, PERIODS_BR) ee.Reset()
def extr_GEE_array_reGE(minlon, minlat, maxlon, maxlat, year, month, day, tempfilter=True, applylcmask=True, sampling=20, dualpol=True, trackflt=None, maskwinter=True, masksnow=True): def maskterrain(image): # srtm dem gee_srtm = ee.Image("USGS/SRTMGL1_003") gee_srtm_slope = ee.Terrain.slope(gee_srtm) tmp = ee.Image(image) mask = gee_srtm_slope.lt(20) mask2 = tmp.lt(0).bitwiseAnd(tmp.gt(-25)) # mask2 = tmp.gte(-25) mask = mask.bitwiseAnd(mask2) tmp = tmp.updateMask(mask) return (tmp) def masklc(image): # load land cover info corine = ee.Image('users/felixgreifeneder/corine') # create lc mask valLClist = [10, 11, 12, 13, 18, 19, 20, 21, 26, 27, 28, 29] lcmask = corine.eq(valLClist[0]).bitwiseOr(corine.eq(valLClist[1])) \ .bitwiseOr(corine.eq(valLClist[2])) \ .bitwiseOr(corine.eq(valLClist[3])) \ .bitwiseOr(corine.eq(valLClist[4])) \ .bitwiseOr(corine.eq(valLClist[5])) \ .bitwiseOr(corine.eq(valLClist[6])) \ .bitwiseOr(corine.eq(valLClist[7])) \ .bitwiseOr(corine.eq(valLClist[8])) \ .bitwiseOr(corine.eq(valLClist[9])) \ .bitwiseOr(corine.eq(valLClist[10])) \ .bitwiseOr(corine.eq(valLClist[11])) tmp = ee.Image(image) tmp = tmp.updateMask(lcmask) return (tmp) def setresample(image): image = image.resample() return (image) def toln(image): tmp = ee.Image(image) # Convert to linear vv = ee.Image(10).pow(tmp.select('VV').divide(10)) if dualpol == True: vh = ee.Image(10).pow(tmp.select('VH').divide(10)) # Convert to ln out = vv.log() if dualpol == True: out = out.addBands(vh.log()) out = out.select(['constant', 'constant_1'], ['VV', 'VH']) else: out = out.select(['constant'], ['VV']) return out.set('system:time_start', tmp.get('system:time_start')) def tolin(image): tmp = ee.Image(image) # Covert to linear vv = ee.Image(10).pow(tmp.select('VV').divide(10)) if dualpol == True: vh = ee.Image(10).pow(tmp.select('VH').divide(10)) # Convert to if dualpol == True: out = vv.addBands(vh) out = out.select(['constant', 'constant_1'], ['VV', 'VH']) else: out = vv.select(['constant'], ['VV']) return out.set('system:time_start', tmp.get('system:time_start')) def todb(image): tmp = ee.Image(image) return ee.Image(10).multiply(tmp.log10()).set('system:time_start', tmp.get('system:time_start')) def applysnowmask(image): tmp = ee.Image(image) sdiff = tmp.select('VH').subtract(snowref) wetsnowmap = sdiff.lte(-2.6).focal_mode(100, 'square', 'meters', 3) return (tmp.updateMask(wetsnowmap.eq(0))) ee.Reset() ee.Initialize() # load S1 data gee_s1_collection = ee.ImageCollection('COPERNICUS/S1_GRD') # for lc_id in range(1, len(valLClist)): # tmpmask = corine.eq(valLClist[lc_id]) # lcmask = lcmask.bitwiseAnd(tmpmask) # construct roi roi = ee.Geometry.Polygon([[minlon, minlat], [minlon, maxlat], [maxlon, maxlat], [maxlon, minlat], [minlon, minlat]]) # roi = get_south_tyrol_roi() # ASCENDING acquisitions gee_s1_filtered = gee_s1_collection.filter(ee.Filter.eq('instrumentMode', 'IW')) \ .filterBounds(roi) \ .filter(ee.Filter.eq('orbitProperties_pass', 'ASCENDING')) \ .filter(ee.Filter.eq('platform_number', 'A')) \ .filter(ee.Filter.listContains('transmitterReceiverPolarisation', 'VV')) if dualpol == True: gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.listContains('transmitterReceiverPolarisation', 'VH')) if trackflt is not None: gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.eq('relativeOrbitNumber_start', trackflt)) if maskwinter == True: gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.dayOfYear(121, 304)) if applylcmask == True: gee_s1_filtered = gee_s1_filtered.map(masklc) gee_s1_filtered = gee_s1_filtered.map(setresample) gee_s1_filtered = gee_s1_filtered.map(maskterrain) # apply wet snow mask if masksnow == True: gee_s1_linear_vh = gee_s1_filtered.map(tolin).select('VH') snowref = ee.Image(10).multiply(gee_s1_linear_vh.reduce(ee.Reducer.intervalMean(5, 100)).log10()) # snowdiff = s1_sig0_vh.subtract(snowref) # wetsnowmap = snowdiff.lte(-2.6).focal_mode(100, 'square', 'meters', 3) # s1_sig0_vv = s1_sig0_vv.updateMask(wetsnowmap.eq(0)) # if dualpol == True: # s1_sig0_vh = s1_sig0_vh.updateMask(wetsnowmap.eq(0)) gee_s1_filtered = gee_s1_filtered.map(applysnowmask) # create a list of availalbel dates tmp = gee_s1_filtered.getInfo() tmp_ids = [x['properties']['system:index'] for x in tmp['features']] dates = np.array([dt.date(year=int(x[17:21]), month=int(x[21:23]), day=int(x[23:25])) for x in tmp_ids]) # find the closest acquisitions doi = dt.date(year=year, month=month, day=day) doi_index = np.argmin(np.abs(dates - doi)) date_selected = dates[doi_index] # filter imagecollection for respective date gee_s1_list = gee_s1_filtered.toList(2000) doi_indices = np.where(dates == date_selected)[0] gee_s1_drange = ee.ImageCollection(gee_s1_list.slice(np.int(doi_indices[0]), np.int(doi_indices[-1] + 1))) s1_sig0 = gee_s1_drange.mosaic() s1_sig0 = ee.Image(s1_sig0.copyProperties(gee_s1_drange.first())) # fetch image from image collection s1_lia = s1_sig0.select('angle').clip(roi) # get the track number s1_sig0_info = s1_sig0.getInfo() track_nr = s1_sig0_info['properties']['relativeOrbitNumber_start'] # despeckle if tempfilter == True: radius = 7 units = 'pixels' gee_s1_linear = gee_s1_filtered.map(tolin) gee_s1_dspckld_vv = multitemporalDespeckle(gee_s1_linear.select('VV'), radius, units, {'before': -12, 'after': 12, 'units': 'month'}) gee_s1_dspckld_vv = gee_s1_dspckld_vv.map(todb) gee_s1_list_vv = gee_s1_dspckld_vv.toList(2000) gee_s1_fltrd_vv = ee.ImageCollection(gee_s1_list_vv.slice(np.int(doi_indices[0]), np.int(doi_indices[-1] + 1))) s1_sig0_vv = gee_s1_fltrd_vv.mosaic() # s1_sig0_vv = ee.Image(gee_s1_list_vv.get(doi_index)) if dualpol == True: gee_s1_dspckld_vh = multitemporalDespeckle(gee_s1_linear.select('VH'), radius, units, {'before': -12, 'after': 12, 'units': 'month'}) gee_s1_dspckld_vh = gee_s1_dspckld_vh.map(todb) gee_s1_list_vh = gee_s1_dspckld_vh.toList(2000) gee_s1_fltrd_vh = ee.ImageCollection(gee_s1_list_vh.slice(np.int(doi_indices[0]), np.int(doi_indices[-1] + 1))) s1_sig0_vh = gee_s1_fltrd_vh.mosaic() if dualpol == True: s1_sig0 = s1_sig0_vv.addBands(s1_sig0_vh).select(['constant', 'constant_1'], ['VV', 'VH']) else: s1_sig0 = s1_sig0_vv.select(['constant'], ['VV']) # s1_sig0_dsc = s1_sig0_vv_dsc.select(['constant'], ['VV']) # extract information s1_sig0_vv = s1_sig0.select('VV') s1_sig0_vv = s1_sig0_vv.clip(roi) if dualpol == True: s1_sig0_vh = s1_sig0.select('VH') s1_sig0_vh = s1_sig0_vh.clip(roi) # compute temporal statistics gee_s1_filtered = gee_s1_filtered.filterMetadata('relativeOrbitNumber_start', 'equals', track_nr) gee_s1_ln = gee_s1_filtered.map(toln) # gee_s1_ln = gee_s1_ln.clip(roi) k1vv = ee.Image(gee_s1_ln.select('VV').mean()).clip(roi) k2vv = ee.Image(gee_s1_ln.select('VV').reduce(ee.Reducer.stdDev())).clip(roi) if dualpol == True: k1vh = ee.Image(gee_s1_ln.select('VH').mean()).clip(roi) k2vh = ee.Image(gee_s1_ln.select('VH').reduce(ee.Reducer.stdDev())).clip(roi) # mask insensitive pixels # if dualpol == True: # smask = k2vv.gt(0.4).And(k2vh.gt(0.4)) # else: # smask = k2vv.gt(0.4) # # s1_sig0_vv = s1_sig0_vv.updateMask(smask) # s1_lia = s1_lia.updateMask(smask) # k1vv = k1vv.updateMask(smask) # k2vv = k2vv.updateMask(smask) # # if dualpol == True: # s1_sig0_vh = s1_sig0_vh.updateMask(smask) # k1vh = k1vh.updateMask(smask) # k2vh = k2vh.updateMask(smask) # export if dualpol == False: # s1_sig0_vv = s1_sig0_vv.reproject(s1_lia.projection()) return (s1_sig0_vv, s1_lia, k1vv, k2vv, roi, date_selected) else: return (s1_lia.focal_median(sampling, 'square', 'meters'), s1_sig0_vv.focal_median(sampling, 'square', 'meters'), s1_sig0_vh.focal_median(sampling, 'square', 'meters'), k1vv.focal_median(sampling, 'square', 'meters'), k1vh.focal_median(sampling, 'square', 'meters'), k2vv.focal_median(sampling, 'square', 'meters'), k2vh.focal_median(sampling, 'square', 'meters'), roi, date_selected)
def extr_L8_ts_GEE(lon, lat, bufferSize=20): ee.Reset() ee.Initialize() def createAvg(image): gee_roi = ee.Geometry.Point(lon, lat).buffer(bufferSize) # aerosols image = image.updateMask(image.select('sr_aerosol').eq(2).bitwiseOr( image.select('sr_aerosol').eq(32).bitwiseOr( image.select('sr_aerosol').eq(96).bitwiseOr( image.select('sr_aerosol').eq(160).bitwiseOr( image.select('sr_aerosol').eq(66).bitwiseOr( image.select('sr_aerosol').eq(130) )))))) # clouds def getQABits(image, start, end, newName): pattern = 0 for i in range(start, end + 1): pattern = pattern + int(math.pow(2, i)) return image.select([0], [newName]).bitwiseAnd(pattern).rightShift(start) def cloud_shadows(image): QA = image.select('pixel_qa') return getQABits(QA, 3, 3, 'Cloud_shadows').eq(0) def clouds(image): QA = image.select('pixel_qa') return getQABits(QA, 5, 5, 'Cloud').eq(0) image = image.updateMask(cloud_shadows(image)) image = image.updateMask(clouds(image)) # # radiometric saturation # image = image.updateMask(image.select('radsat_qa').eq(2)) reduced_img_data = image.reduceRegion(ee.Reducer.mean(), gee_roi, 30) return ee.Feature(None, {'result': reduced_img_data}) def setresample(image): image = image.resample() return (image) def mask_tree_cover(image): tree_cover_image = ee.ImageCollection("GLCF/GLS_TCC").filterBounds(gee_roi).filter( ee.Filter.eq('year', 2010)).mosaic() treemask = tree_cover_image.select('tree_canopy_cover').clip(gee_roi).lte(20) # load lc glbcvr = ee.Image("ESA/GLOBCOVER_L4_200901_200912_V2_3").select('landcover') # mask water watermask = glbcvr.neq(210) return image.updateMask(treemask.And(watermask)) # load collection gee_l8_collection = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR') # .map(setresample) # filter collection gee_roi = ee.Geometry.Point(lon, lat).buffer(bufferSize) gee_l8_fltd = gee_l8_collection.filterBounds(gee_roi) # extract time series #gee_l8_fltd = gee_l8_fltd.map(mask_tree_cover) gee_l8_mpd = gee_l8_fltd.map(createAvg) tmp = gee_l8_mpd.getInfo() b1 = np.array([x['properties']['result']['B1'] for x in tmp['features']], dtype=np.float) b2 = np.array([x['properties']['result']['B2'] for x in tmp['features']], dtype=np.float) b3 = np.array([x['properties']['result']['B3'] for x in tmp['features']], dtype=np.float) b4 = np.array([x['properties']['result']['B4'] for x in tmp['features']], dtype=np.float) b5 = np.array([x['properties']['result']['B5'] for x in tmp['features']], dtype=np.float) b6 = np.array([x['properties']['result']['B6'] for x in tmp['features']], dtype=np.float) b7 = np.array([x['properties']['result']['B7'] for x in tmp['features']], dtype=np.float) b10 = np.array([x['properties']['result']['B7'] for x in tmp['features']], dtype=np.float) b11 = np.array([x['properties']['result']['B7'] for x in tmp['features']], dtype=np.float) ge_dates = np.array([datetime.strptime(x['id'][12::], '%Y%m%d') for x in tmp['features']]) valid = np.where(np.isfinite(b2)) # cut out invalid values b1 = b1[valid] b2 = b2[valid] b3 = b3[valid] b4 = b4[valid] b5 = b5[valid] b6 = b6[valid] b7 = b7[valid] b10 = b10[valid] b11 = b11[valid] ge_dates = ge_dates[valid] if b1.size == 0: return None else: return pd.DataFrame({'B1': b1, 'B2': b2, 'B3': b3, 'B4': b4, 'B5': b5, 'B6': b6, 'B7': b7, 'B10': b10, 'B11': b11}, index=ge_dates)