def main(): #os.system("earthengine authenticate") ee.Initialize() now = time.time() # We will export the information for each kml (Trigo, Girasol, Barbecho, etc.) for kml in kmls: table = ee.FeatureCollection(kml[1]) if buffer_value < 0: table = getBuffer(table, buffer_value) # Get radar for orbit in orbits: for sentinel in sentinels: nameOutputFile = kml[0] + "_" + sentinel + "_" + orbit # Check if the file already exists if not os.path.exists( os.path.join("dataEE", nameOutputFile + ".csv")): # Load the imagery with the current feature collection s1 = loadSentinel1(table, start_date, end_date, sentinel, orbit) # Export the table (sentinel-1) if natural_data == 1: print("Descargando datos en valores naturales") elif natural_data == 0: print("Descargando datos en decibelios") else: print( "Error en el valor de la variable 'natural_data'. Abortando..." ) exit() tableSeries = getTimeSeriesTask(s1[natural_data], table) #s1[0] = db; s1[1] = natural datasetTask = exportTableSeries(tableSeries, nameOutputFile, indexes_sentinel1) # Start the task. datasetTask.start() while datasetTask.active(): print('Descargando datos de %s...' % (nameOutputFile)) time.sleep(30) print('Terminado.') else: print("Fichero %s ya descargado." % (nameOutputFile)) # Check if we have to download sentinel 2 indexes if indexes_sentinel2: nameOutputFile = kml[0] + "_s2" # Check if the file already exists if not os.path.exists( os.path.join("dataEE", nameOutputFile + ".csv")): # Get Sentinel-2 # Load the imagery with the current feature collection s2 = loadSentinel2(table, start_date, end_date) # Export the table (sentinel-2) tableSeries_s2 = getTimeSeriesTask(s2, table) datasetTask_s2 = exportTableSeries(tableSeries_s2, kml[0] + "_s2", indexes_sentinel2) # Start the task. datasetTask_s2.start() while datasetTask_s2.active(): print('Descargando datos de %s...' % (kml[0] + "_s2")) time.sleep(30) print('Terminado.') else: print("Fichero %s ya descargado." % (nameOutputFile)) end = time.time() elapsed = end - now time_convert(elapsed)
import ee import sys from oauth2client.service_account import ServiceAccountCredentials from aoi import Aoi from image_spec import ImageSpec credentials = ServiceAccountCredentials.from_p12_keyfile( service_account_email=sys.argv[1], filename=sys.argv[2], private_key_password='******', scopes=ee.oauth.SCOPE + ' https://www.googleapis.com/auth/drive ') ee.Initialize(credentials)
'min':7000, 'max':10000, 'size': '800', 'region': maliau_region Good luck! Cheers, Andrew Cotham [email protected] ''' import ee,oauth2client.client, config, datetime, sys, json, math from IPython.core.display import Image ee.Initialize(ee.ServiceAccountCredentials(config.MY_SERVICE_ACCOUNT, config.MY_PRIVATE_KEY_FILE)) maliau_image = ee.Image("LC81170572013170LGN00") maliau_region = [[116.676364283,4.95945957299],[116.676364283,4.64623224023],[117.100028398,4.64623224023],[117.100028398,4.95945957299]] uncorrected_image = ee.Image.cat([maliau_image.expression("b('B4')+770"),maliau_image.select('B3'),maliau_image.expression("b('B2')-1230")]); uncorrected_thumbnail = uncorrected_image.getThumbUrl({ 'bands': 'B4,B3,B2', 'min':7000, 'max':10000, 'size': '800', 'region': maliau_region }) Image(url=uncorrected_thumbnail) # <rawcell> # Get the Digital Terrain Model and the image metadata to help produce the Illumination Condition
def check_images_available(inputs): """ Create the structure of subfolders for each satellite mission KV WRL 2018 Arguments: ----------- inputs: dict inputs dictionnary Returns: ----------- im_dict_T1: list of dict list of images in Tier 1 and Level-1C im_dict_T2: list of dict list of images in Tier 2 (Landsat only) """ # check if EE was initialised or not try: ee.ImageCollection('LANDSAT/LT05/C01/T1_TOA') except: ee.Initialize() print('Images available between %s and %s:' % (inputs['dates'][0], inputs['dates'][1]), end='\n') # check how many images are available in Tier 1 and Sentinel Level-1C col_names_T1 = { 'L5': 'LANDSAT/LT05/C01/T1_TOA', 'L7': 'LANDSAT/LE07/C01/T1_TOA', 'L8': 'LANDSAT/LC08/C01/T1_TOA', 'S2': 'COPERNICUS/S2' } print('- In Landsat Tier 1 & Sentinel-2 Level-1C:') im_dict_T1 = dict([]) sum_img = 0 for satname in inputs['sat_list']: # get list of images in EE collection while True: try: ee_col = ee.ImageCollection(col_names_T1[satname]) col = ee_col.filterBounds(ee.Geometry.Polygon(inputs['polygon']))\ .filterDate(inputs['dates'][0],inputs['dates'][1]) im_list = col.getInfo().get('features') break except: continue # remove very cloudy images (>95% cloud cover) im_list_upt = remove_cloudy_images(im_list, satname) sum_img = sum_img + len(im_list_upt) print(' %s: %d images' % (satname, len(im_list_upt))) im_dict_T1[satname] = im_list_upt print(' Total: %d images' % sum_img) # in only S2 is in sat_list, stop here if len(inputs['sat_list']) == 1 and inputs['sat_list'][0] == 'S2': return im_dict_T1, [] # otherwise check how many images are available in Landsat Tier 2 col_names_T2 = { 'L5': 'LANDSAT/LT05/C01/T2_TOA', 'L7': 'LANDSAT/LE07/C01/T2_TOA', 'L8': 'LANDSAT/LC08/C01/T2_TOA' } print('- In Landsat Tier 2:', end='\n') im_dict_T2 = dict([]) sum_img = 0 for satname in inputs['sat_list']: if satname == 'S2': continue # get list of images in EE collection while True: try: ee_col = ee.ImageCollection(col_names_T2[satname]) col = ee_col.filterBounds(ee.Geometry.Polygon(inputs['polygon']))\ .filterDate(inputs['dates'][0],inputs['dates'][1]) im_list = col.getInfo().get('features') break except: continue # remove very cloudy images (>95% cloud cover) im_list_upt = remove_cloudy_images(im_list, satname) sum_img = sum_img + len(im_list_upt) print(' %s: %d images' % (satname, len(im_list_upt))) im_dict_T2[satname] = im_list_upt print(' Total: %d images' % sum_img) return im_dict_T1, im_dict_T2
def download_data(R:RegionST, times, products, bands, path_save, scale=10, max_cloud_fraction=None, use_least_cloudy=None): ee.Initialize() path_save.mkdir(exist_ok=True, parents=True) if not ((path_save/f'download.{bands[0]}.tif').is_file() and (path_save/f'download.{bands[1]}.tif').is_file() and (path_save/f'download.{bands[2]}.tif').is_file()): sR = [R] if R.shape[0] <= 32 else split_region(R, size=32, cls=RegionST) fsaves = [] #for j, R in tqdm(enumerate(sR), total=len(sR)): for j, R in enumerate(sR): region = (f"[[{R.bbox.left}, {R.bbox.bottom}], [{R.bbox.right}, {R.bbox.bottom}], " + f"[{R.bbox.right}, {R.bbox.top}], [{R.bbox.left}, {R.bbox.top}]]") if not ((path_save/f'download.{bands[0]}_{j}.tif').is_file() and (path_save/f'download.{bands[1]}_{j}.tif').is_file() and (path_save/f'download.{bands[2]}_{j}.tif').is_file()): # Merge products to single image collection imCol = ee.ImageCollection(products[0]) for i in range(1, len(products)): imCol = imCol.merge(ee.ImageCollection(products[i])) imCol = filter_region(imCol, R, times=times, bands=bands) if max_cloud_fraction is not None: imCol = filter_cloudy(imCol, max_cloud_fraction=max_cloud_fraction) if use_least_cloudy is not None: imCol = n_least_cloudy(imCol, n=use_least_cloudy) im = imCol.median() imCol = ee.ImageCollection([im]) colList = imCol.toList(imCol.size()) # info = colList.getInfo() # data_times = [pd.to_datetime(o['properties']['system:time_start'], unit='ms') for o in info] # data_cloudy = [o['properties']['CLOUDY_PIXEL_PERCENTAGE'] for o in info] # Download each image for i in range(colList.size().getInfo()): image = ee.Image(colList.get(i)) fname = 'download' #fname = image.get('system:id').getInfo().split('/')[-1] fnames_full = [f'{fname}.{b}.tif' for b in bands] fnames_partial0 = [f'{fname}.{b}_{j}.tif' for b in bands] fnames_full = all([(path_save/f).is_file() for f in fnames_full]) fnames_partial = all([(path_save/f).is_file() for f in fnames_partial0]) if not fnames_full: fsaves.append([path_save/f for f in fnames_partial0]) if not fnames_partial: zip_error = True for i in range(10): # Try 10 times if zip_error: try: url = image.getDownloadURL( {'scale': scale, 'crs': 'EPSG:4326', 'region': f'{region}'}) r = requests.get(url) with open(str(path_save/'data.zip'), 'wb') as f: f.write(r.content) with zipfile.ZipFile(str(path_save/'data.zip'), 'r') as f: files = f.namelist() f.extractall(str(path_save)) os.remove(str(path_save/'data.zip')) zip_error = False except: zip_error = True os.remove(str(path_save/'data.zip')) time.sleep(10) if zip_error: raise Exception(f'Failed to process {url}') for f in files: f = path_save/f os.rename(str(f), str(path_save/f'{f.stem}_{j}{f.suffix}')) # Merge files suffix = '.tif' files = path_save.ls(include=[suffix]) #files = np.unique(fsaves) files = [o.stem for o in files] ref = np.unique(['_'.join(o.split('_')[:-1]) for o in files if len(o.split('_')[-1]) < 6]) ids = np.unique([int(o.split('_')[-1]) for o in files if len(o.split('_')[-1]) < 6]) #file_groups = [[path_save/f'{r}_{i}{suffix}' for i in ids] for r in ref] file_groups = [[path_save/f'{r}_{i}{suffix}' for i in ids if f'{r}_{i}' in files] for r in ref] for fs in file_groups: if len(fs) < 500: fsave = '_'.join(fs[0].stem.split('_')[:-1]) + suffix merge_tifs(fs, fsave, delete=True) else: fs_break = np.array(fs)[:(len(fs)//500)*500].reshape(len(fs)//500,-1).tolist() if len(fs[(len(fs)//500)*500:]) > 0: fs_break.append(fs[(len(fs)//500)*500:]) for fsi, fs2 in enumerate(fs_break): fsave = '_'.join(fs2[0].stem.split('_')[:-1]) + f'_break{fsi}' + suffix merge_tifs(fs2, fsave, delete=True) files = path_save.ls(include=[suffix, '_break']) files = [o.stem for o in files] ref = np.unique(['_'.join(o.split('_')[:-1]) for o in files if len(o.split('_')[-1]) < 11]) ids = np.unique([o.split('_')[-1] for o in files if len(o.split('_')[-1]) < 11]) #file_groups = [[path_save/f'{r}_{i}{suffix}' for i in ids] for r in ref] file_groups = [[path_save/f'{r}_{i}{suffix}' for i in ids if f'{r}_{i}' in files] for r in ref] for fs in file_groups: fsave = '_'.join(fs[0].stem.split('_')[:-1]) + suffix merge_tifs(fs, fsave, delete=True)
def testDynamicClasses(self): """Verifies dynamic class initialization.""" # Use a custom set of known functions. def MockSend(path, unused_params, unused_method=None, unused_raw=None): if path == '/algorithms': return { 'Array': { 'type': 'Algorithm', 'args': [{ 'name': 'values', 'type': 'Serializable', 'description': '' }], 'description': '', 'returns': 'Array' }, 'Array.cos': { 'type': 'Algorithm', 'args': [{ 'type': 'Array', 'description': '', 'name': 'input' }], 'description': '', 'returns': 'Array' }, 'Kernel.circle': { 'returns': 'Kernel', 'args': [{ 'type': 'float', 'description': '', 'name': 'radius', }, { 'default': 1.0, 'type': 'float', 'optional': True, 'description': '', 'name': 'scale' }, { 'default': True, 'type': 'boolean', 'optional': True, 'description': '', 'name': 'normalize' }], 'type': 'Algorithm', 'description': '' }, 'Reducer.mean': { 'returns': 'Reducer', 'args': [] }, 'fakeFunction': { 'returns': 'Array', 'args': [{ 'type': 'Reducer', 'description': '', 'name': 'kernel', }] } } ee.data.send_ = MockSend ee.Initialize(None) # Verify that the expected classes got generated. self.assertTrue(hasattr(ee, 'Array')) self.assertTrue(hasattr(ee, 'Kernel')) self.assertTrue(hasattr(ee.Array, 'cos')) self.assertTrue(hasattr(ee.Kernel, 'circle')) # Try out the constructors. kernel = ee.ApiFunction('Kernel.circle').call(1, 2) self.assertEquals(kernel, ee.Kernel.circle(1, 2)) array = ee.ApiFunction('Array').call([1, 2]) self.assertEquals(array, ee.Array([1, 2])) self.assertEquals(array, ee.Array(ee.Array([1, 2]))) # Try out the member function. self.assertEquals( ee.ApiFunction('Array.cos').call(array), ee.Array([1, 2]).cos()) # Test argument promotion. f1 = ee.ApiFunction('Array.cos').call([1, 2]) f2 = ee.ApiFunction('Array.cos').call(ee.Array([1, 2])) self.assertEquals(f1, f2) self.assertTrue(isinstance(f1, ee.Array)) f3 = ee.call('fakeFunction', 'mean') f4 = ee.call('fakeFunction', ee.Reducer.mean()) self.assertEquals(f3, f4) try: ee.call('fakeFunction', 'moo') self.fail() except ee.EEException as e: self.assertTrue('Unknown algorithm: Reducer.moo' in str(e))
def testCallAndApply(self): """Verifies library initialization.""" # Use a custom set of known functions. def MockSend(path, params, unused_method=None, unused_raw=None): if path == '/algorithms': return { 'fakeFunction': { 'type': 'Algorithm', 'args': [{ 'name': 'image1', 'type': 'Image' }, { 'name': 'image2', 'type': 'Image' }], 'returns': 'Image' }, 'Image.constant': apitestcase.BUILTIN_FUNCTIONS['Image.constant'] } else: raise Exception('Unexpected API call to %s with %s' % (path, params)) ee.data.send_ = MockSend ee.Initialize(None) image1 = ee.Image(1) image2 = ee.Image(2) expected = ee.Image( ee.ComputedObject(ee.ApiFunction.lookup('fakeFunction'), { 'image1': image1, 'image2': image2 })) applied_with_images = ee.apply('fakeFunction', { 'image1': image1, 'image2': image2 }) self.assertEquals(expected, applied_with_images) applied_with_numbers = ee.apply('fakeFunction', { 'image1': 1, 'image2': 2 }) self.assertEquals(expected, applied_with_numbers) called_with_numbers = ee.call('fakeFunction', 1, 2) self.assertEquals(expected, called_with_numbers) # Test call and apply() with a custom function. sig = {'returns': 'Image', 'args': [{'name': 'foo', 'type': 'Image'}]} func = ee.CustomFunction(sig, lambda foo: ee.call('fakeFunction', 42, foo)) expected_custom_function_call = ee.Image( ee.ComputedObject(func, {'foo': ee.Image(13)})) self.assertEquals(expected_custom_function_call, ee.call(func, 13)) self.assertEquals(expected_custom_function_call, ee.apply(func, {'foo': 13})) # Test None promotion. called_with_null = ee.call('fakeFunction', None, 1) self.assertEquals(None, called_with_null.args['image1'])
import ee import itertools from gee_functions.classification import create_features, create_training_areas, classify_irrigated_areas from gee_functions.constants import GEE_USER_PATH if __name__ == '__main__': ee.Initialize() # Initialize the Google Earth Engine # Load the feature collection containing the area of interest # aoi = ee.FeatureCollection(f'{GEE_USER_PATH}/vector/outline/outline_region_de_murcia') aoi = ee.FeatureCollection(f'{GEE_USER_PATH}/vector/outline/outline_cdc_3857') aoi_coordinates = aoi.geometry().bounds().getInfo()['coordinates'] # Dictionary containing the date ranges for each year, will be used to select sat. imagery years = { '88': ('1987-01-01', '1989-01-01'), '97': ('1996-01-01', '1998-01-01'), '00': ('1999-01-01', '2001-01-01'), '05': ('2004-01-01', '2006-01-01'), '09': ('2008-01-01', '2010-01-01'), } stats = ['median', 'min', 'max'] stats_combos = list(itertools.combinations(stats, 3)) # stats_combos = list(itertools.combinations(stats, 3)) + \ # list(itertools.combinations(stats, 2)) + \ # list(itertools.combinations(stats, 3))
def __init__(self): self.initialize = ee.Initialize() self.credentials = ee.Credentials() self.service = discovery.build(serviceName='drive', version='v3', cache_discovery=False, credentials=self.credentials)
def extr_L8_ts_GEE(lon, lat, bufferSize=20): ee.Reset() ee.Initialize() def createAvg(image): gee_roi = ee.Geometry.Point(lon, lat).buffer(bufferSize) # aerosols image = image.updateMask(image.select('sr_aerosol').eq(2).bitwiseOr( image.select('sr_aerosol').eq(32).bitwiseOr( image.select('sr_aerosol').eq(96).bitwiseOr( image.select('sr_aerosol').eq(160).bitwiseOr( image.select('sr_aerosol').eq(66).bitwiseOr( image.select('sr_aerosol').eq(130) )))))) # clouds def getQABits(image, start, end, newName): pattern = 0 for i in range(start, end + 1): pattern = pattern + int(math.pow(2, i)) return image.select([0], [newName]).bitwiseAnd(pattern).rightShift(start) def cloud_shadows(image): QA = image.select('pixel_qa') return getQABits(QA, 3, 3, 'Cloud_shadows').eq(0) def clouds(image): QA = image.select('pixel_qa') return getQABits(QA, 5, 5, 'Cloud').eq(0) image = image.updateMask(cloud_shadows(image)) image = image.updateMask(clouds(image)) # # radiometric saturation # image = image.updateMask(image.select('radsat_qa').eq(2)) reduced_img_data = image.reduceRegion(ee.Reducer.mean(), gee_roi, 30) return ee.Feature(None, {'result': reduced_img_data}) def setresample(image): image = image.resample() return (image) def mask_tree_cover(image): tree_cover_image = ee.ImageCollection("GLCF/GLS_TCC").filterBounds(gee_roi).filter( ee.Filter.eq('year', 2010)).mosaic() treemask = tree_cover_image.select('tree_canopy_cover').clip(gee_roi).lte(20) # load lc glbcvr = ee.Image("ESA/GLOBCOVER_L4_200901_200912_V2_3").select('landcover') # mask water watermask = glbcvr.neq(210) return image.updateMask(treemask.And(watermask)) # load collection gee_l8_collection = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR') # .map(setresample) # filter collection gee_roi = ee.Geometry.Point(lon, lat).buffer(bufferSize) gee_l8_fltd = gee_l8_collection.filterBounds(gee_roi) # extract time series #gee_l8_fltd = gee_l8_fltd.map(mask_tree_cover) gee_l8_mpd = gee_l8_fltd.map(createAvg) tmp = gee_l8_mpd.getInfo() b1 = np.array([x['properties']['result']['B1'] for x in tmp['features']], dtype=np.float) b2 = np.array([x['properties']['result']['B2'] for x in tmp['features']], dtype=np.float) b3 = np.array([x['properties']['result']['B3'] for x in tmp['features']], dtype=np.float) b4 = np.array([x['properties']['result']['B4'] for x in tmp['features']], dtype=np.float) b5 = np.array([x['properties']['result']['B5'] for x in tmp['features']], dtype=np.float) b6 = np.array([x['properties']['result']['B6'] for x in tmp['features']], dtype=np.float) b7 = np.array([x['properties']['result']['B7'] for x in tmp['features']], dtype=np.float) b10 = np.array([x['properties']['result']['B7'] for x in tmp['features']], dtype=np.float) b11 = np.array([x['properties']['result']['B7'] for x in tmp['features']], dtype=np.float) ge_dates = np.array([datetime.strptime(x['id'][12::], '%Y%m%d') for x in tmp['features']]) valid = np.where(np.isfinite(b2)) # cut out invalid values b1 = b1[valid] b2 = b2[valid] b3 = b3[valid] b4 = b4[valid] b5 = b5[valid] b6 = b6[valid] b7 = b7[valid] b10 = b10[valid] b11 = b11[valid] ge_dates = ge_dates[valid] if b1.size == 0: return None else: return pd.DataFrame({'B1': b1, 'B2': b2, 'B3': b3, 'B4': b4, 'B5': b5, 'B6': b6, 'B7': b7, 'B10': b10, 'B11': b11}, index=ge_dates)
def extr_SIG0_LIA_ts_GEE(lon, lat, bufferSize=20, maskwinter=False, lcmask=False, globcover_mask=False, trackflt=None, masksnow=False, varmask=False, ssmcor=None, dual_pol=True, desc=False, tempfilter=False, returnLIA=False, datesonly=False, datefilter=None, S1B=False, treemask=False, radcor=False): ee.Initialize() def mask_lc(image): tmp = ee.Image(image) # load land cover info corine = ee.Image('users/felixgreifeneder/corine') # create lc mask valLClist = [10, 11, 12, 13, 18, 19, 20, 21, 26, 27, 28, 29] lcmask = corine.eq(valLClist[0]).bitwiseOr(corine.eq(valLClist[1])) \ .bitwiseOr(corine.eq(valLClist[2])) \ .bitwiseOr(corine.eq(valLClist[3])) \ .bitwiseOr(corine.eq(valLClist[4])) \ .bitwiseOr(corine.eq(valLClist[5])) \ .bitwiseOr(corine.eq(valLClist[6])) \ .bitwiseOr(corine.eq(valLClist[7])) \ .bitwiseOr(corine.eq(valLClist[8])) \ .bitwiseOr(corine.eq(valLClist[9])) \ .bitwiseOr(corine.eq(valLClist[10])) \ .bitwiseOr(corine.eq(valLClist[11])) tmp = tmp.updateMask(lcmask) return tmp def mask_lc_globcover(image): tmp = ee.Image(image) # load lc glbcvr = ee.Image("ESA/GLOBCOVER_L4_200901_200912_V2_3").select('landcover') valLClist = [11, 14, 20, 30, 120, 140, 150] lcmask = glbcvr.eq(valLClist[0]) \ .bitwiseOr(glbcvr.eq(valLClist[1])) \ .bitwiseOr(glbcvr.eq(valLClist[2])) \ .bitwiseOr(glbcvr.eq(valLClist[3])) \ .bitwiseOr(glbcvr.eq(valLClist[4])) \ .bitwiseOr(glbcvr.eq(valLClist[5])) \ .bitwiseOr(glbcvr.eq(valLClist[6])) tmp = tmp.updateMask(lcmask) return tmp def mask_tree_cover(image): copernicus_collection = ee.ImageCollection('COPERNICUS/Landcover/100m/Proba-V/Global') copernicus_image = ee.Image(copernicus_collection.toList(1000).get(0)) treemask = copernicus_image.select('tree-coverfraction').clip(gee_roi).lte(20) # load lc glbcvr = ee.Image("ESA/GLOBCOVER_L4_200901_200912_V2_3").select('landcover') # mask water watermask = copernicus_image.select('discrete_classification').neq(80) return image.updateMask(treemask.And(watermask)) def addRefSM(image): tmp = ee.Image(image) img_date = ee.Date(tmp.get('system:time_start')) RefSMtmp = RefSMcollection.filterDate(img_date.format('Y-M-d')) current_ssm = ee.ImageCollection(RefSMtmp).toList(10).get(0) out_image = tmp.addBands(ee.Image(current_ssm)) return (out_image) def s1_simplyfy_date(image): return (image.set('system:time_start', ee.Date(ee.Date(image.get('system:time_start')).format('Y-M-d')))) def applyCorrelationMask(image): mask = ssm_vv_cor.select('correlation').gt(0.1) return (image.updateMask(mask)) import timeit tic = timeit.default_timer() # load S1 data gee_s1_collection = ee.ImageCollection('COPERNICUS/S1_GRD') # .map(setresample) # filter collection gee_roi = ee.Geometry.Point(lon, lat).buffer(bufferSize) gee_s1_filtered = gee_s1_collection.filter(ee.Filter.eq('instrumentMode', 'IW')) \ .filterBounds(gee_roi) \ .filter(ee.Filter.listContains('transmitterReceiverPolarisation', 'VV')) if S1B == False: gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.eq('platform_number', 'A')) if datefilter is not None: gee_s1_filtered = gee_s1_filtered.filterDate(datefilter[0], datefilter[1]) if desc == False: # Select only acquisition from ascending tracks gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.eq('orbitProperties_pass', 'ASCENDING')) # else: # gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.eq('orbitProperties_pass', 'DESCENDING')) if dual_pol == True: # select only acquisitions with VV AND VH gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.listContains('transmitterReceiverPolarisation', 'VH')) if maskwinter == True: # Mask winter based on the DOY gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.dayOfYear(121, 304)) if trackflt is not None: # Select only data from a specific S1 track if isinstance(trackflt, list): gee_s1_filtered = gee_s1_filtered.filter( ee.Filter.inList(ee.List(trackflt), 'relativeOrbitNumber_start')) else: gee_s1_filtered = gee_s1_filtered.filter(ee.Filter.eq('relativeOrbitNumber_start', trackflt)) if lcmask == True: # Mask pixels based on Corine land-cover gee_s1_filtered = gee_s1_filtered.map(mask_lc) if globcover_mask == True: # Mask pixels based on the Globcover land-cover classification gee_s1_filtered = gee_s1_filtered.map(mask_lc_globcover) if treemask == True: gee_s1_filtered = gee_s1_filtered.map(mask_tree_cover) if ssmcor is not None: # Mask pixels with a low correlation toe coarse resolution soil moisture # Mostly relevant if aggregating over a larger area RefSMlist = list() ssmcor = ssmcor.resample('D').mean().dropna() ssmcor = ssmcor.astype(np.float) for i in range(len(ssmcor)): ssm_img = ee.Image(ssmcor[i]).clip(gee_roi).float() ssm_img = ssm_img.set('system:time_start', ssmcor.index[i]) RefSMlist.append(ssm_img) RefSMcollection = ee.ImageCollection(RefSMlist) # prepare the join s1_joined = gee_s1_filtered.map(s1_simplyfy_date) join_filter = ee.Filter.equals(leftField='system:time_start', rightField='system:time_start') simple_join = ee.Join.simple() s1_joined = simple_join.apply(s1_joined, RefSMcollection, join_filter) # create ssm reference SM, image collection s1_plus_RefSM = ee.ImageCollection(s1_joined.map(addRefSM, True)) ssm_vv_cor = s1_plus_RefSM.select(['VV', 'constant']).reduce(ee.Reducer.pearsonsCorrelation()) gee_s1_filtered = gee_s1_filtered.map(applyCorrelationMask) # get the track numbers tmp = gee_s1_filtered.getInfo() track_series = np.array([x['properties']['relativeOrbitNumber_start'] for x in tmp['features']]) dir_series = np.array([x['properties']['orbitProperties_pass'] for x in tmp['features']]) available_tracks, uidx = np.unique(track_series, return_index=True) available_directions = dir_series[uidx] # create dict with orbit directions available_directions = pd.Series(np.where(available_directions == 'ASCENDING', 1, 0), index=available_tracks) print('Extracting data from ' + str(len(available_tracks)) + ' Sentinel-1 tracks...') print(available_tracks) out_dict = {} lgths = list() for track_nr in available_tracks: if datesonly == False: out_dict[str(int(track_nr))] = _s1_track_ts(lon, lat, bufferSize, gee_s1_filtered, track_nr, dual_pol, varmask, returnLIA, masksnow, tempfilter, datesonly, radcor) else: tmp_dates = _s1_track_ts(lon, lat, bufferSize, gee_s1_filtered, track_nr, dual_pol, varmask, returnLIA, masksnow, tempfilter, datesonly, radcor) lgths.append(tmp_dates) toc = timeit.default_timer() if datesonly == True: return np.array(lgths) print('Time-series extraction finished in ' + "{:10.2f}".format(toc - tic) + 'seconds') return out_dict, available_directions
begin = self.request.get('begin') end = self.request.get('end') content = GetWetnessTimeSeries(aoi, begin, end) self.response.headers['Content-Type'] = 'application/json' self.response.out.write(json.dumps(content)) self.response.headers['Content-Type'] = 'application/json' self.response.headers['Content-Disposition'] = 'attachment; filename="data.json"' class RefreshAccessToken(webapp2.RequestHandler): def post(self): self.redirect('/') app = webapp2.WSGIApplication([ ('/get_aoi_image_info_time_series', GetImageInfoHandler), ('/get_ndwi_time_series', GetWetnessTimeSeriesHandler), # html pages in current directory ('/(.*\.html)', MainPageHandler), ('/', MainPageHandler) ], debug=True) # Initialize the EE API. # Use our App Engine service account's credentials. EE_CREDENTIALS = ee.ServiceAccountCredentials(config.EE_ACCOUNT, config.EE_PRIVATE_KEY_FILE) ee.Initialize(EE_CREDENTIALS)
def main(): # Reading data data = pd.read_csv(input_file, sep='\t', usecols=[column_latitude, column_longitude]) data = data.values for satellite_name in satellites: for day in dates: for dam in range(data.shape[0]): ee.Initialize() if satellite_name == "sentinel": # Use these bands for prediction. bands = ['B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'B8', 'B8A', 'B9', 'B10', 'B11', 'B12'] # Use Sentinel 2 surface reflectance data. satellite = ee.ImageCollection("COPERNICUS/S2") flag_clouds = 'CLOUDY_PIXEL_PERCENTAGE' def maskS2clouds(image): cloudShadowBitMask = ee.Number(2).pow(3).int() cloudsBitMask = ee.Number(2).pow(5).int() qa = image.select('QA60') mask = qa.bitwiseAnd(cloudShadowBitMask).eq(0).And( qa.bitwiseAnd(cloudsBitMask).eq(0)) return image.updateMask(mask).select(bands).divide(10000) mask = maskS2clouds elif satellite_name == "landsat8": # Use these bands for prediction. bands = ['B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'B8', 'B9', 'B10', 'B11'] # Use Sentinel 2 surface reflectance data. satellite = ee.ImageCollection("LANDSAT/LC08/C01/T1_TOA") flag_clouds = 'CLOUD_COVER' # Cloud masking function. def maskL8sr(image): cloudsBitMask = ee.Number(2).pow(4).int() qa = image.select('BQA') mask = qa.bitwiseAnd(cloudsBitMask).eq(0) return image.updateMask(mask).select(bands).divide(10000) mask = maskL8sr else: raise NotImplementedError # Logitude, Latitude x, y = degree_conv(data[dam][1]), degree_conv(data[dam][0]) # x, y = float(data[dam][1].replace(",", ".")), float(data[dam][0].replace(",", ".")) # if not_dam: # # North # try: # send_not_dam_task(x, y, bands, satellite, flag_clouds, mask, data, "N", satellite_name) # except: # _print("Error in {0}{1:0=3d}_{2}".format("N", dam + 1, day[0][:4])) # try: # _print("Trying image {0}{1:0=3d}_{2} again".format("N", dam + 1, day[0][:4])) # send_not_dam_task(x, y, bands, satellite, flag_clouds, mask, data, "N", satellite_name) # except: # _print("Image {0}{1:0=3d}_{2} could not be downloaded".format("N", dam + 1, day[0][:4])) # # # South # try: # send_not_dam_task(x, y, bands, satellite, flag_clouds, mask, data, "S", satellite_name) # except: # _print("Error in {0}{1:0=3d}_{2}".format("S", dam + 1, day[0][:4])) # try: # _print("Trying image {0}{1:0=3d}_{2} again".format("S", dam + 1, day[0][:4])) # send_not_dam_task(x, y, bands, satellite, flag_clouds, mask, data, "S", satellite_name) # except: # _print("Image {0}{1:0=3d}_{2} could not be downloaded".format("S", dam + 1, day[0][:4])) # # else: # Dam try: send_task(satellite_name, satellite, bands, x, y, day, flag_clouds, mask, dam) except: _print("Error in {0:0=3d}_{1}".format(dam + 1, day[0][:4])) try: _print("Trying image {0:0=3d}_{1} again".format(dam + 1, day[0][:4])) send_task(satellite_name, satellite, bands, x, y, day, flag_clouds, mask, dam) except: _print("Image {0:0=3d}_{1} could not be downloaded".format(dam + 1, day[0][:4]))
def main(ini_path=None, overwrite_flag=False, delay=0, key=None, cron_flag=False, reverse_flag=False): """Compute daily Tcorr images Parameters ---------- ini_path : str Input file path. overwrite_flag : bool, optional If True, overwrite existing files if the export dates are the same and generate new images (but with different export dates) even if the tile lists are the same. The default is False. delay : float, optional Delay time between each export task (the default is 0). key : str, optional File path to an Earth Engine json key file (the default is None). cron_flag : bool, optional If True, only compute Tcorr daily image if existing image does not have all available image (using the 'wrs2_tiles' property) and limit the date range to the last 64 days (~2 months). reverse_flag : bool, optional If True, process dates in reverse order. """ logging.info('\nCompute daily Tcorr images') ini = utils.read_ini(ini_path) model_name = 'SSEBOP' # model_name = ini['INPUTS']['et_model'].upper() if (ini[model_name]['tmax_source'].upper() == 'CIMIS' and ini['INPUTS']['end_date'] < '2003-10-01'): logging.error( '\nCIMIS is not currently available before 2003-10-01, exiting\n') sys.exit() elif (ini[model_name]['tmax_source'].upper() == 'DAYMET' and ini['INPUTS']['end_date'] > '2017-12-31'): logging.warning('\nDAYMET is not currently available past 2017-12-31, ' 'using median Tmax values\n') # sys.exit() # elif (ini[model_name]['tmax_source'].upper() == 'TOPOWX' and # ini['INPUTS']['end_date'] > '2017-12-31'): # logging.warning( # '\nDAYMET is not currently available past 2017-12-31, ' # 'using median Tmax values\n') # # sys.exit() logging.info('\nInitializing Earth Engine') if key: logging.info(' Using service account key file: {}'.format(key)) # The "EE_ACCOUNT" parameter is not used if the key file is valid ee.Initialize(ee.ServiceAccountCredentials('deadbeef', key_file=key)) else: ee.Initialize() # Output Tcorr daily image collection tcorr_daily_coll_id = '{}/{}_daily'.format( ini['EXPORT']['export_coll'], ini[model_name]['tmax_source'].lower()) # Get a Tmax image to set the Tcorr values to logging.debug('\nTmax properties') tmax_name = ini[model_name]['tmax_source'] tmax_source = tmax_name.split('_', 1)[0] tmax_version = tmax_name.split('_', 1)[1] tmax_coll_id = 'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower()) tmax_coll = ee.ImageCollection(tmax_coll_id) tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0) logging.debug(' Collection: {}'.format(tmax_coll_id)) logging.debug(' Source: {}'.format(tmax_source)) logging.debug(' Version: {}'.format(tmax_version)) logging.debug('\nExport properties') export_geo = ee.Image(tmax_mask).projection().getInfo()['transform'] export_crs = ee.Image(tmax_mask).projection().getInfo()['crs'] export_shape = ee.Image(tmax_mask).getInfo()['bands'][0]['dimensions'] export_extent = [ export_geo[2], export_geo[5] + export_shape[1] * export_geo[4], export_geo[2] + export_shape[0] * export_geo[0], export_geo[5] ] logging.debug(' CRS: {}'.format(export_crs)) logging.debug(' Extent: {}'.format(export_extent)) logging.debug(' Geo: {}'.format(export_geo)) logging.debug(' Shape: {}'.format(export_shape)) # # Limit export to a user defined study area or geometry? # export_geom = ee.Geometry.Rectangle( # [-125, 24, -65, 50], proj='EPSG:4326', geodesic=False) # CONUS # export_geom = ee.Geometry.Rectangle( # [-124, 35, -119, 42], proj='EPSG:4326', geodesic=False) # California # If cell_size parameter is set in the INI, # adjust the output cellsize and recompute the transform and shape try: export_cs = float(ini['EXPORT']['cell_size']) export_shape = [ int(math.ceil(abs((export_shape[0] * export_geo[0]) / export_cs))), int(math.ceil(abs((export_shape[1] * export_geo[4]) / export_cs))) ] export_geo = [ export_cs, 0.0, export_geo[2], 0.0, -export_cs, export_geo[5] ] logging.debug(' Custom export cell size: {}'.format(export_cs)) logging.debug(' Geo: {}'.format(export_geo)) logging.debug(' Shape: {}'.format(export_shape)) except KeyError: pass # Get current asset list if ini['EXPORT']['export_dest'].upper() == 'ASSET': logging.debug('\nGetting asset list') # DEADBEEF - daily is hardcoded in the asset_id for now asset_list = utils.get_ee_assets(tcorr_daily_coll_id) else: raise ValueError('invalid export destination: {}'.format( ini['EXPORT']['export_dest'])) # Get current running tasks tasks = utils.get_ee_tasks() if logging.getLogger().getEffectiveLevel() == logging.DEBUG: logging.debug(' Tasks: {}\n'.format(len(tasks))) input('ENTER') collections = [x.strip() for x in ini['INPUTS']['collections'].split(',')] # Limit by year and month try: month_list = sorted(list(utils.parse_int_set(ini['TCORR']['months']))) except: logging.info('\nTCORR "months" parameter not set in the INI,' '\n Defaulting to all months (1-12)\n') month_list = list(range(1, 13)) try: year_list = sorted(list(utils.parse_int_set(ini['TCORR']['years']))) except: logging.info('\nTCORR "years" parameter not set in the INI,' '\n Defaulting to all available years\n') year_list = [] # Key is cycle day, value is a reference date on that cycle # Data from: https://landsat.usgs.gov/landsat_acq # I only need to use 8 cycle days because of 5/7 and 7/8 are offset cycle_dates = { 7: '1970-01-01', 8: '1970-01-02', 1: '1970-01-03', 2: '1970-01-04', 3: '1970-01-05', 4: '1970-01-06', 5: '1970-01-07', 6: '1970-01-08', } # cycle_dates = { # 1: '2000-01-06', # 2: '2000-01-07', # 3: '2000-01-08', # 4: '2000-01-09', # 5: '2000-01-10', # 6: '2000-01-11', # 7: '2000-01-12', # 8: '2000-01-13', # # 9: '2000-01-14', # # 10: '2000-01-15', # # 11: '2000-01-16', # # 12: '2000-01-01', # # 13: '2000-01-02', # # 14: '2000-01-03', # # 15: '2000-01-04', # # 16: '2000-01-05', # } cycle_base_dt = datetime.datetime.strptime(cycle_dates[1], '%Y-%m-%d') if cron_flag: # CGM - This seems like a silly way of getting the date as a datetime # Why am I doing this and not using the commented out line? iter_end_dt = datetime.date.today().strftime('%Y-%m-%d') iter_end_dt = datetime.datetime.strptime(iter_end_dt, '%Y-%m-%d') iter_end_dt = iter_end_dt + datetime.timedelta(days=-4) # iter_end_dt = datetime.datetime.today() + datetime.timedelta(days=-1) iter_start_dt = iter_end_dt + datetime.timedelta(days=-64) else: iter_start_dt = datetime.datetime.strptime(ini['INPUTS']['start_date'], '%Y-%m-%d') iter_end_dt = datetime.datetime.strptime(ini['INPUTS']['end_date'], '%Y-%m-%d') logging.debug('Start Date: {}'.format(iter_start_dt.strftime('%Y-%m-%d'))) logging.debug('End Date: {}\n'.format(iter_end_dt.strftime('%Y-%m-%d'))) for export_dt in sorted(utils.date_range(iter_start_dt, iter_end_dt), reverse=reverse_flag): export_date = export_dt.strftime('%Y-%m-%d') next_date = (export_dt + datetime.timedelta(days=1)).strftime('%Y-%m-%d') # if ((month_list and export_dt.month not in month_list) or # (year_list and export_dt.year not in year_list)): if month_list and export_dt.month not in month_list: logging.debug(f'Date: {export_date} - month not in INI - skipping') continue elif export_date >= datetime.datetime.today().strftime('%Y-%m-%d'): logging.debug(f'Date: {export_date} - unsupported date - skipping') continue elif export_date < '1984-03-23': logging.debug(f'Date: {export_date} - no Landsat 5+ images before ' '1984-03-16 - skipping') continue logging.info(f'Date: {export_date}') export_id = ini['EXPORT']['export_id_fmt'] \ .format( product=tmax_name.lower(), date=export_dt.strftime('%Y%m%d'), export=datetime.datetime.today().strftime('%Y%m%d'), dest=ini['EXPORT']['export_dest'].lower()) logging.debug(' Export ID: {}'.format(export_id)) if ini['EXPORT']['export_dest'] == 'ASSET': asset_id = '{}/{}_{}'.format( tcorr_daily_coll_id, export_dt.strftime('%Y%m%d'), datetime.datetime.today().strftime('%Y%m%d')) logging.debug(' Asset ID: {}'.format(asset_id)) if overwrite_flag: if export_id in tasks.keys(): logging.debug(' Task already submitted, cancelling') ee.data.cancelTask(tasks[export_id]) # This is intentionally not an "elif" so that a task can be # cancelled and an existing image/file/asset can be removed if (ini['EXPORT']['export_dest'].upper() == 'ASSET' and asset_id in asset_list): logging.debug(' Asset already exists, removing') ee.data.deleteAsset(asset_id) else: if export_id in tasks.keys(): logging.debug(' Task already submitted, exiting') continue elif (ini['EXPORT']['export_dest'].upper() == 'ASSET' and asset_id in asset_list): logging.debug(' Asset already exists, skipping') continue # Build and merge the Landsat collections model_obj = ssebop.Collection( collections=collections, start_date=export_dt.strftime('%Y-%m-%d'), end_date=(export_dt + datetime.timedelta(days=1)).strftime('%Y-%m-%d'), cloud_cover_max=float(ini['INPUTS']['cloud_cover']), geometry=tmax_mask.geometry(), # model_args=model_args, # filter_args=filter_args, ) landsat_coll = model_obj.overpass(variables=['ndvi']) # wrs2_tiles_all = model_obj.get_image_ids() # pprint.pprint(landsat_coll.aggregate_array('system:id').getInfo()) # input('ENTER') logging.debug(' Getting available WRS2 tile list') landsat_id_list = landsat_coll.aggregate_array('system:id').getInfo() wrs2_tiles_all = set([id.split('_')[-2] for id in landsat_id_list]) if not wrs2_tiles_all: logging.info(' No available images - skipping') continue # If overwriting, start a new export no matter what # The default is to no overwrite, so this mode will not be used often if not overwrite_flag: # Check if there are any previous images for this date # If so, only build a new Tcorr image if there are new wrs2_tiles # that were not used in the previous image. # Should this code only be run in cron mode or is this the expected # operation when (re)running for any date range? # Should we only test the last image # or all previous images for the date? logging.debug( ' Checking for previous exports/versions of daily image') tcorr_daily_coll = ee.ImageCollection(tcorr_daily_coll_id)\ .filterDate(export_date, next_date)\ .limit(1, 'date_ingested', False) tcorr_daily_info = tcorr_daily_coll.getInfo() if tcorr_daily_info['features']: # Assume we won't be building a new image and only set flag # to True if the WRS2 tile lists are different export_flag = False # The ".limit(1, ..." on the tcorr_daily_coll above makes this # for loop and break statement unnecessary, but leaving for now for tcorr_img in tcorr_daily_info['features']: # If the full WRS2 list is not present, rebuild the image # This should only happen for much older Tcorr images if 'wrs2_available' not in tcorr_img['properties'].keys(): logging.debug( ' "wrs2_available" property not present in ' 'previous export') export_flag = True break wrs2_tiles_old = set( tcorr_img['properties']['wrs2_available'].split(',')) if wrs2_tiles_all != wrs2_tiles_old: logging.debug(' Tile Lists') logging.debug(' Previous: {}'.format(', '.join( sorted(wrs2_tiles_old)))) logging.debug(' Available: {}'.format(', '.join( sorted(wrs2_tiles_all)))) logging.debug(' New: {}'.format(', '.join( sorted( wrs2_tiles_all.difference(wrs2_tiles_old))))) logging.debug(' Dropped: {}'.format(', '.join( sorted( wrs2_tiles_old.difference(wrs2_tiles_all))))) export_flag = True break if not export_flag: logging.debug(' No new WRS2 tiles/images - skipping') continue # else: # logging.debug(' Building new version') else: logging.debug(' No previous exports') def tcorr_img_func(image): t_stats = ssebop.Image.from_landsat_c1_toa( ee.Image(image), tdiff_threshold=float(ini[model_name]['tdiff_threshold'])) \ .tcorr_stats t_stats = ee.Dictionary(t_stats) \ .combine({'tcorr_p5': 0, 'tcorr_count': 0}, overwrite=False) tcorr = ee.Number(t_stats.get('tcorr_p5')) count = ee.Number(t_stats.get('tcorr_count')) # Remove the merged collection indices from the system:index scene_id = ee.List( ee.String(image.get('system:index')).split('_')).slice(-3) scene_id = ee.String(scene_id.get(0)).cat('_') \ .cat(ee.String(scene_id.get(1))).cat('_') \ .cat(ee.String(scene_id.get(2))) return tmax_mask.add(tcorr) \ .rename(['tcorr']) \ .clip(image.geometry()) \ .set({ 'system:time_start': image.get('system:time_start'), 'scene_id': scene_id, 'wrs2_tile': scene_id.slice(5, 11), 'spacecraft_id': image.get('SPACECRAFT_ID'), 'tcorr': tcorr, 'count': count, }) # Test for one image # pprint.pprint(tcorr_img_func(ee.Image(landsat_coll \ # .filterMetadata('WRS_PATH', 'equals', 36) \ # .filterMetadata('WRS_ROW', 'equals', 33).first())).getInfo()) # input('ENTER') # (Re)build the Landsat collection from the image IDs landsat_coll = ee.ImageCollection(landsat_id_list) tcorr_img_coll = ee.ImageCollection(landsat_coll.map(tcorr_img_func)) \ .filterMetadata('count', 'not_less_than', float(ini['TCORR']['min_pixel_count'])) # If there are no Tcorr values, return an empty image tcorr_img = ee.Algorithms.If(tcorr_img_coll.size().gt(0), tcorr_img_coll.median(), tmax_mask.updateMask(0)) def unique_properties(coll, property): return ee.String( ee.List( ee.Dictionary( coll.aggregate_histogram(property)).keys()).join(',')) wrs2_tile_list = ee.String('').cat( unique_properties(tcorr_img_coll, 'wrs2_tile')) landsat_list = ee.String('').cat( unique_properties(tcorr_img_coll, 'spacecraft_id')) # Cast to float and set properties tcorr_img = ee.Image(tcorr_img).rename(['tcorr']).double() \ .set({ 'system:time_start': utils.millis(export_dt), 'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'), 'date': export_dt.strftime('%Y-%m-%d'), 'year': int(export_dt.year), 'month': int(export_dt.month), 'day': int(export_dt.day), 'doy': int(export_dt.strftime('%j')), 'cycle_day': ((export_dt - cycle_base_dt).days % 8) + 1, 'landsat': landsat_list, 'model_name': model_name, 'model_version': ssebop.__version__, 'tmax_source': tmax_source.upper(), 'tmax_version': tmax_version.upper(), 'wrs2_tiles': wrs2_tile_list, 'wrs2_available': ','.join(sorted(wrs2_tiles_all)), }) # Build export tasks if ini['EXPORT']['export_dest'] == 'ASSET': logging.debug(' Building export task') task = ee.batch.Export.image.toAsset( image=ee.Image(tcorr_img), description=export_id, assetId=asset_id, crs=export_crs, crsTransform='[' + ','.join(list(map(str, export_geo))) + ']', dimensions='{0}x{1}'.format(*export_shape), ) logging.info(' Starting export task') utils.ee_task_start(task) # Pause before starting next task utils.delay_task(delay) logging.debug('')
def main(country, year, resolution=30): ee.Initialize() def last_day_of_month(any_day): next_month = any_day.replace(day=28) + datetime.timedelta(days=4) return next_month - datetime.timedelta(days=next_month.day) # Reference: https://www.satimagingcorp.com/satellite-sensors/other-satellite-sensors/sentinel-2a/ band_blue = 'B2' #10m band_green = 'B3' #10m band_red = "B4" #10m band_nir = 'B8' #10m def calc_NDVI(img): ndvi = ee.Image(img.normalizedDifference([band_nir, band_red])).rename( ["ndvi"]).copyProperties(img, img.propertyNames()) composite = img.addBands(ndvi) return composite # SAVI = ((NIR – Red) / (NIR + Red + L)) x (1 + L) def calc_SAVI(img): """A function to compute Soil Adjusted Vegetation Index.""" savi = ee.Image( img.expression('(1 + L) * float(nir - red)/ (nir + red + L)', { 'nir': img.select(band_nir), 'red': img.select(band_red), 'L': 0.5 })).rename(["savi"]).copyProperties(img, img.propertyNames()) composite = img.addBands(savi) return composite # EVI = 2.5 * ((NIR – Red) / ((NIR) + (C1 * Red) – (C2 * Blue) + L)) # C1=6, C2=7.5, and L=1 def calc_EVI(img): """A function to compute Soil Adjusted Vegetation Index.""" evi = ee.Image( img.expression( '(2.5) * float(nir - red)/ ((nir) + (C1*red) - (C2*blue) + L)', { 'nir': img.select(band_nir), 'red': img.select(band_red), 'blue': img.select(band_blue), 'L': 0.2, 'C1': 6, 'C2': 7.5 })).rename(["evi"]).copyProperties(img, img.propertyNames()) composite = img.addBands(evi) return composite def add_landcover(img): landcover = ee.Image("USGS/GFSAD1000_V1") composite = img.addBands(landcover) return composite def calc_YYYYMM(img): return img.set('YYYYMM', img.date().format("YYYYMM")) ######################################################################################## world_df = pd.read_csv( '../data/Global_Samples_Balanced_n100_m250_s210.csv', index_col=[0]) world_df.head() world_df['geometry'] = world_df['geometry'].apply(wkt.loads) world_df['samples'] = world_df['samples'].apply(literal_eval) world_df['n_samples'] = world_df.apply(lambda row: len(row['samples']), axis=1) world_gdf = geopandas.GeoDataFrame(world_df, geometry='geometry') country_gdf = world_gdf[(world_gdf['country_code'] == country) & (world_gdf['n_samples'] > 0)] country_gdf = country_gdf.reset_index() country_polygon_dict = dict() for index, row in country_gdf.iterrows(): country_polygon_dict[country + '_' + str(index)] = (row['geometry'], row['samples']) for k, v in country_polygon_dict.items(): country_segment = k print('==================== {} ===================='.format( country_segment)) area_of_interest_shapely, random_pixels = v area_of_interest_ee = ee.Geometry.Polygon( list(area_of_interest_shapely.exterior.coords)) for month in range(1, 13): month_start = datetime.date(year, month, 1) month_end = last_day_of_month(datetime.date(year, month, 1)) # Create image collection that contains the area of interest img_collect = ( ee.ImageCollection('COPERNICUS/S2').filterDate( str(month_start), str(month_end)).filterBounds(area_of_interest_ee) # Remove image that's too small (likely to be partial image) # Size of a full image: 1,276,131,371; size of a partial image: 276,598,191 # .filter(ee.Filter.gt('system:asset_size', 800000000)) .filterMetadata("CLOUDY_PIXEL_PERCENTAGE", "less_than", 50)) if img_collect.size().getInfo() == 0: warnings.warn('No valid image.') continue print("Total number of images in the collection: ", img_collect.size().getInfo()) # Extract tile information from each image # Note: tiles can overlap a little bit unique_tiles = set([ item['properties']['MGRS_TILE'] for item in img_collect.getInfo()['features'] ]) if len(unique_tiles) > 1: print('Number of tiles selected: ', len(unique_tiles)) # if img_collect_no_partial.size().getInfo() < img_collect.size().getInfo(): # warnings.warn('There are partial images in the collection. Proceed with caution.') # print('Number of partial images: ', img_collect.size().getInfo()-img_collect_no_partial.size().getInfo()) img_collect_calc = img_collect.map(calc_YYYYMM).map(calc_NDVI).map( calc_SAVI).map(calc_EVI).map(add_landcover) unique_month = list( set([ item['properties']['YYYYMM'] for item in img_collect_calc.getInfo()['features'] ])) unique_month.sort() print(unique_month) # 1. min_lat, min_lon, max_lat, max_lon for the country # min_lon, min_lat, max_lon, max_lat = area_of_interest_shapely.bounds # 2. from left to right, top to bottom, define 0.1 degree by 0.1 degree overlapping rectangles # 3. only keep the rectangles that's within the country's boundary (possbily using spatial join) # 4. for rectangle in a list of recntagles: # .reduceRegion(geometry=rectangle) # save to csv img_calc_month_dict = dict() data_dict = dict() for month in unique_month: img_calc_month_dict[month] = img_collect_calc.filter( ee.Filter.eq('YYYYMM', month)).median() img_calc_month2 = img_calc_month_dict[month].addBands( ee.Image.pixelLonLat()) # EEException: Output of image computation is too large (20 bands for 851968 pixels = 126.8 MiB > 80.0 MiB). # If this is a reduction, try specifying a larger 'tileScale' parameter. # EEException: ReduceRegion.AggregationContainer: Valid tileScales are 1 to 16. data_dict[month] = pd.DataFrame(columns=[ "lat", "lon", 'landcover', month + '_NDVI', month + '_SAVI', month + '_EVI' ]) for p in range(len(random_pixels)): # `reduceRegion` doesn't like ee.Geometry.MultiPoint as geometry data_month_lst = img_calc_month2.reduceRegion(reducer=ee.Reducer.toList(), \ geometry=ee.Geometry.Point(random_pixels[p]), maxPixels=1e13, scale=resolution) # For some reason, ee.Array(data_month_lst.get("...")).getInfo()[0] runs really slow pixel_dict = dict() pixel_dict['lat'] = ee.Array( data_month_lst.get("latitude")).getInfo()[0] pixel_dict['lon'] = ee.Array( data_month_lst.get("longitude")).getInfo()[0] try: pixel_dict['landcover'] = ee.Array( data_month_lst.get("landcover")).getInfo()[0] pixel_dict[month + '_NDVI'] = ee.Array( data_month_lst.get("ndvi")).getInfo()[0] pixel_dict[month + '_SAVI'] = ee.Array( data_month_lst.get("savi")).getInfo()[0] pixel_dict[month + '_EVI'] = ee.Array( data_month_lst.get("evi")).getInfo()[0] except: warnings.warn('Missing satellite data.') pixel_dict['landcover'] = None pixel_dict[month + '_NDVI'] = None pixel_dict[month + '_SAVI'] = None pixel_dict[month + '_EVI'] = None data_dict[month] = data_dict[month].append( pixel_dict, ignore_index=True) data_dict[month].to_csv( '../data/GEE/Sentinel2_samples/{}_{}.csv'.format( country_segment, month))
def initialize(): # get GBDX catalog catalog = gb.catalog.Catalog() # initilaize EE, used to get water occurrence data ee.Initialize()
def arrayToPairs(array, startDate, EndDate, delta=1, CLOUDY_PIXEL_PERCENTAGE=10): ee.Initialize() area = ee.Geometry.Polygon(array) # query collection_NDVI = ee.ImageCollection("COPERNICUS/S2").filterBounds(area) \ .filterDate(startDate, EndDate) \ .filterMetadata("CLOUDY_PIXEL_PERCENTAGE", "less_than", CLOUDY_PIXEL_PERCENTAGE) \ .select(['B8', 'B4', 'QA60']) \ .sort('date') NDVI_size = collection_NDVI.size().getInfo() l_NDVI = collection_NDVI.toList(NDVI_size) l_NDVI_dates = getDates(l_NDVI, NDVI_size, process=False) # use NDVI dates to make date-filter for SAR data, taking into account valid_dates = None for i_NDVI in range(NDVI_size): NDVI_date = l_NDVI_dates[i_NDVI] if not valid_dates: valid_dates = ee.Filter.date(NDVI_date.advance(-delta, 'day'), NDVI_date.advance(delta, 'day')) else: valid_dates = ee.Filter.Or(valid_dates, ee.Filter.date(NDVI_date.advance(-delta, 'day'), NDVI_date.advance(delta, 'day'))) if (not valid_dates): raise ValueError("no valid dates for the image") collection_SAR = ee.ImageCollection('COPERNICUS/S1_GRD').filterBounds(area) \ .filter(valid_dates) \ .select(['VH', 'VV']) SAR_size = collection_SAR.size().getInfo() l_SAR = collection_SAR.toList(SAR_size) l_SAR_dates = getDates(l_SAR, SAR_size) l_NDVI_dates = getDates(l_NDVI, NDVI_size) # map collections to their respective special data formats collection_NDVI = collection_NDVI.map(getNDVI.getNDVI) l_NDVI = collection_NDVI.toList(NDVI_size) # NDVI-[SAR] list pairs_i = {} for i_SAR in range(SAR_size): SAR_date = l_SAR_dates[i_SAR] for i_NDVI in range(NDVI_size): NDVI_date = l_NDVI_dates[i_NDVI] if abs(SAR_date - NDVI_date) <= timedelta(days=delta): if i_NDVI not in pairs_i: pairs_i[i_NDVI] = [i_SAR] else: pairs_i[i_NDVI].append(i_SAR) sorted_pairs = sorted(pairs_i) print(pairs_i) arr = [] precomputed_SAR = dict() LatLonImgsNDVI = ee.List( [common.fastLatLonImg(ee.Image(l_NDVI.get(NDVI)), area) for NDVI in sorted_pairs]) LatLonImgsSAR = ee.List([LatLonImgVHVV(ee.Image(l_SAR.get(SAR)), area) for SAR in range(SAR_size)]) both_lists = ee.List([LatLonImgsNDVI, LatLonImgsSAR]).getInfo() LatLonImgsNDVI = both_lists[0] LatLonImgsSAR = both_lists[1] for NDVI in range(len(LatLonImgsNDVI)): i = sorted_pairs[NDVI] for SAR in pairs_i[i]: ndvi_temp = (LatLonImgsNDVI[NDVI][0], LatLonImgsNDVI[NDVI][1], LatLonImgsNDVI[NDVI][2]) if SAR not in precomputed_SAR: lats = LatLonImgsSAR[SAR][0] lons = LatLonImgsSAR[SAR][1] vh = LatLonImgsSAR[SAR][2] vv = LatLonImgsSAR[SAR][3] precomputed_SAR[SAR] = [] precomputed_SAR[SAR].append((lats, lons, vh) + (f'SAR (VH) {l_SAR_dates[SAR]:%B %d, %Y}',)) precomputed_SAR[SAR].append((lats, lons, vv) + (f'SAR (VV) {l_SAR_dates[SAR]:%B %d, %Y}',)) arr.append(ndvi_temp + (f'NDVI {l_NDVI_dates[NDVI]:%B %d, %Y}',)) arr.extend(precomputed_SAR[SAR]) return rasteriser.rasteriseImages(arr)
def __init__(self): """Initialize the environment.""" # Initialize the Earth Engine object, using the authentication credentials. ee.Initialize() self.dem = ee.Image("JAXA/ALOS/AW3D30_V1_1").select(["AVE"]) self.epsg = "EPSG:32717" ########################################## # variable for the landsat data request # ########################################## self.metadataCloudCoverMax = 80 ########################################## # Export variables # ########################################## self.assetId = "projects/Sacha/PreprocessedData/L8_Biweekly_V6/" self.name = "LS_BW_" self.exportScale = 20 ########################################## # variable for the shadowMask algorithm # ########################################## # zScoreThresh: Threshold for cloud shadow masking- lower number masks out # less. Between -0.8 and -1.2 generally works well self.zScoreThresh = -0.9 # shadowSumThresh: Sum of IR bands to include as shadows within TDOM and the # shadow shift method (lower number masks out less) self.shadowSumThresh = 0.4 # contractPixels: The radius of the number of pixels to contract (negative buffer) clouds and cloud shadows by. Intended to eliminate smaller cloud # patches that are likely errors (1.5 results in a -1 pixel buffer)(0.5 results in a -0 pixel buffer) # (1.5 or 2.5 generally is sufficient) self.contractPixels = 1.5 # dilatePixels: The radius of the number of pixels to dilate (buffer) clouds # and cloud shadows by. Intended to include edges of clouds/cloud shadows # that are often missed (1.5 results in a 1 pixel buffer)(0.5 results in a 0 pixel buffer) # (2.5 or 3.5 generally is sufficient) self.dilatePixels = 3.25 ########################################## # variable for cloudScore algorithm # ########################################## # 9. Cloud and cloud shadow masking parameters. # If cloudScoreTDOM is chosen # cloudScoreThresh: If using the cloudScoreTDOMShift method-Threshold for cloud # masking (lower number masks more clouds. Between 10 and 30 generally works best) self.cloudScoreThresh = 1 # Percentile of cloud score to pull from time series to represent a minimum for # the cloud score over time for a given pixel. Reduces commission errors over # cool bright surfaces. Generally between 5 and 10 works well. 0 generally is a bit noisy self.cloudScorePctl = 8 self.hazeThresh = 195 ########################################## # variable for terrain algorithm # ########################################## self.terrainScale = 600 ########################################## # variable band selection # ########################################## self.percentiles = [25, 75] self.medianPercentileBands = ee.List([ 'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'date', 'pixel_qa', 'cloudScore' ]) self.divideBands = ee.List( ['blue', 'green', 'red', 'nir', 'swir1', 'swir2']) self.medoidBands = ee.List( ['blue', 'green', 'red', 'nir', 'swir1', 'swir2']) self.medoidIncludeBands = ee.List( ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa']) self.noScaleBands = ee.List([ 'date', 'year', 'cloudMask', 'count', 'TDOMMask', 'pixel_qa', 'cloudScore' ]) self.bandNamesLandsat = ee.List([ 'blue', 'green', 'red', 'nir', 'swir1', 'thermal', 'swir2', 'sr_atmos_opacity', 'pixel_qa', 'radsat_qa' ]) self.sensorBandDictLandsatSR = ee.Dictionary({'L8' : ee.List([1,2,3,4,5,7,6,9,10,11]),\ 'L7' : ee.List([0,1,2,3,4,5,6,7,9,10]),\ 'L5' : ee.List([0,1,2,3,4,5,6,7,9,10]),\ 'L4' : ee.List([0,1,2,3,4,5,6,7,9,10])}) ########################################## # enable / disable modules # ########################################## self.maskSR = True self.cloudMask = False self.hazeMask = False self.shadowMask = False self.brdfCorrect = True self.terrainCorrection = True self.includePercentiles = True self.compositingMethod = 'Medoid'
def testDynamicConstructor(self): # Test the behavior of the dynamic class constructor. # Use a custom set of known functions for classes Foo and Bar. # Foo Foo(arg1, [arg2]) # Bar Foo.makeBar() # Bar Foo.takeBar(Bar bar) # Baz Foo.baz() def MockSend(path, unused_params, unused_method=None, unused_raw=None): if path == '/algorithms': return { 'Foo': { 'returns': 'Foo', 'args': [{ 'name': 'arg1', 'type': 'Object' }, { 'name': 'arg2', 'type': 'Object', 'optional': True }] }, 'Foo.makeBar': { 'returns': 'Bar', 'args': [{ 'name': 'foo', 'type': 'Foo' }] }, 'Foo.takeBar': { 'returns': 'Bar', 'args': [{ 'name': 'foo', 'type': 'Foo' }, { 'name': 'bar', 'type': 'Bar' }] }, 'Bar.baz': { 'returns': 'Baz', 'args': [{ 'name': 'bar', 'type': 'Bar' }] } } ee.data.send_ = MockSend ee.Initialize(None) # Try to cast something that's already of the right class. x = ee.Foo('argument') self.assertEquals(ee.Foo(x), x) # Tests for dynamic classes, where there is a constructor. # # If there's more than 1 arg, call the constructor. x = ee.Foo('a') y = ee.Foo(x, 'b') ctor = ee.ApiFunction.lookup('Foo') self.assertEquals(y.func, ctor) self.assertEquals(y.args, {'arg1': x, 'arg2': 'b'}) # Can't cast a primitive; call the constructor. self.assertEquals(ctor, ee.Foo(1).func) # A computed object, but not this class; call the constructor. self.assertEquals(ctor, ee.Foo(ee.List([1, 2, 3])).func) # Tests for dynamic classes, where there isn't a constructor. # # Foo.makeBar and Foo.takeBar should have caused Bar to be generated. self.assertTrue(hasattr(ee, 'Bar')) # Make sure we can create a Bar. bar = ee.Foo(1).makeBar() self.assertTrue(isinstance(bar, ee.Bar)) # Now cast something else to a Bar and verify it was just a cast. cast = ee.Bar(ee.Foo(1)) self.assertTrue(isinstance(cast, ee.Bar)) self.assertEquals(ctor, cast.func) # We shouldn't be able to cast with more than 1 arg. try: ee.Bar(x, 'foo') self.fail('Expected an exception.') except ee.EEException as e: self.assertTrue('Too many arguments for ee.Bar' in str(e)) # We shouldn't be able to cast a primitive. try: ee.Bar(1) self.fail('Expected an exception.') except ee.EEException as e: self.assertTrue('Must be a ComputedObject' in str(e))
def main(args=None): setup_logging() parser = argparse.ArgumentParser( description='Google Earth Engine Batch Asset Manager') parser.add_argument('-s', '--service-account', help='Google Earth Engine service account.', required=False) parser.add_argument('-k', '--private-key', help='Google Earth Engine private key file.', required=False) subparsers = parser.add_subparsers() parser_delete = subparsers.add_parser( 'delete', help= 'Deletes collection and all items inside. Supports Unix-like wildcards.' ) parser_delete.add_argument( 'id', help= 'Full path to asset for deletion. Recursively removes all folders, collections and images.' ) parser_delete.set_defaults(func=delete_collection_from_parser) parser_upload = subparsers.add_parser('upload', help='Batch Asset Uploader.') required_named = parser_upload.add_argument_group( 'Required named arguments.') required_named.add_argument( '--source', help='Path to the directory with images for upload.', required=True) required_named.add_argument( '--dest', help= 'Destination. Full path for upload to Google Earth Engine, e.g. users/pinkiepie/myponycollection', required=True) optional_named = parser_upload.add_argument_group( 'Optional named arguments') optional_named.add_argument('-m', '--metadata', help='Path to CSV with metadata.') optional_named.add_argument( '--large', action='store_true', help='(Advanced) Use multipart upload. Might help if upload of large ' 'files is failing on some systems. Might cause other issues.') optional_named.add_argument( '--nodata', type=int, help='The value to burn into the raster as NoData (missing data)') optional_named.add_argument( '--bands', type=_comma_separated_strings, help='Comma-separated list of names to use for the image bands. Spaces' 'or other special characters are not allowed.') required_named.add_argument('-u', '--user', help='Google account name (gmail address).') optional_named.add_argument('-b', '--bucket', help='Google Cloud Storage bucket name.') optional_named.add_argument( '-e', '--upload-catch-error', action='store_true', help='Return exit code 1 when upload catches an error') optional_named.add_argument('-a', '--tolerate-assets-already-exist', action='store_true', help='Return exit 0 when assets already exist') optional_named.add_argument( '--headless', help='Run the browser in headless mode (i.e. no user interface).', action='store_true') parser_upload.set_defaults(func=upload_from_parser) parser_cancel = subparsers.add_parser('cancel', help='Cancel all running tasks') parser_cancel.set_defaults(func=cancel_all_running_tasks_from_parser) parser_info = subparsers.add_parser('report', help='Produce summary of all assets.') parser_info.set_defaults(func=produce_report) parser_info.add_argument('--filename', help='File name for the output CSV (optional)') parser_copy = subparsers.add_parser( 'copy', help= 'Batch copy of assets. Helps in migrating assets from Google Maps to GEE' ) parser_copy.set_defaults(func=batch_copy) parser_copy.add_argument( '--source', help='File with the following structure: [asset name],[asset id in GME]' ) parser_copy.add_argument( '--dest', help='Full path to the directory or collection in EE') args = parser.parse_args() if args.service_account: credentials = ee.ServiceAccountCredentials(args.service_account, args.private_key) ee.Initialize(credentials) else: ee.Initialize() if args.private_key is not None: os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.private_key if 'func' in args: args.func(args) else: parser.print_help()
def retrieve_images(inputs): """ Downloads all images from Landsat 5, Landsat 7, Landsat 8 and Sentinel-2 covering the area of interest and acquired between the specified dates. The downloaded images are in .TIF format and organised in subfolders, divided by satellite mission. The bands are also subdivided by pixel resolution. KV WRL 2018 Arguments: ----------- inputs: dict with the following keys 'sitename': str name of the site 'polygon': list polygon containing the lon/lat coordinates to be extracted, longitudes in the first column and latitudes in the second column, there are 5 pairs of lat/lon with the fifth point equal to the first point: ``` polygon = [[[151.3, -33.7],[151.4, -33.7],[151.4, -33.8],[151.3, -33.8], [151.3, -33.7]]] ``` 'dates': list of str list that contains 2 strings with the initial and final dates in format 'yyyy-mm-dd': ``` dates = ['1987-01-01', '2018-01-01'] ``` 'sat_list': list of str list that contains the names of the satellite missions to include: ``` sat_list = ['L5', 'L7', 'L8', 'S2'] ``` 'filepath_data': str filepath to the directory where the images are downloaded Returns: ----------- metadata: dict contains the information about the satellite images that were downloaded: date, filename, georeferencing accuracy and image coordinate reference system """ # initialise connection with GEE server ee.Initialize() # check image availabiliy and retrieve list of images im_dict_T1, im_dict_T2 = check_images_available(inputs) # if user also wants to download T2 images, merge both lists if 'include_T2' in inputs.keys(): for key in inputs['sat_list']: if key == 'S2': continue else: im_dict_T1[key] += im_dict_T2[key] # remove UTM duplicates in S2 collections (they provide several projections for same images) if 'S2' in inputs['sat_list'] and len(im_dict_T1['S2']) > 0: im_dict_T1['S2'] = filter_S2_collection(im_dict_T1['S2']) # create a new directory for this site with the name of the site im_folder = os.path.join(inputs['filepath'], inputs['sitename']) if not os.path.exists(im_folder): os.makedirs(im_folder) print('\nDownloading images:') suffix = '.tif' for satname in im_dict_T1.keys(): print('%s: %d images' % (satname, len(im_dict_T1[satname]))) # create subfolder structure to store the different bands filepaths = create_folder_structure(im_folder, satname) # initialise variables and loop through images georef_accs = [] filenames = [] all_names = [] im_epsg = [] for i in range(len(im_dict_T1[satname])): im_meta = im_dict_T1[satname][i] # get time of acquisition (UNIX time) and convert to datetime t = im_meta['properties']['system:time_start'] im_timestamp = datetime.fromtimestamp(t / 1000, tz=pytz.utc) im_date = im_timestamp.strftime('%Y-%m-%d-%H-%M-%S') # get epsg code im_epsg.append(int(im_meta['bands'][0]['crs'][5:])) # get geometric accuracy if satname in ['L5', 'L7', 'L8']: if 'GEOMETRIC_RMSE_MODEL' in im_meta['properties'].keys(): acc_georef = im_meta['properties']['GEOMETRIC_RMSE_MODEL'] else: acc_georef = 12 # default value of accuracy (RMSE = 12m) elif satname in ['S2']: # Sentinel-2 products don't provide a georeferencing accuracy (RMSE as in Landsat) # but they have a flag indicating if the geometric quality control was passed or failed # if passed a value of 1 is stored if failed a value of -1 is stored in the metadata skip_geo_check = False if 'GEOMETRIC_QUALITY_FLAG' in im_meta['properties'].keys(): key = 'GEOMETRIC_QUALITY_FLAG' elif 'quality_check' in im_meta['properties'].keys(): key = 'quality_check' else: acc_georef = -1 skip_geo_check = True if not skip_geo_check: if im_meta['properties'][key] == 'PASSED': acc_georef = 1 else: acc_georef = -1 georef_accs.append(acc_georef) bands = dict([]) im_fn = dict([]) # first delete dimensions key from dictionnary # otherwise the entire image is extracted (don't know why) im_bands = im_meta['bands'] for j in range(len(im_bands)): del im_bands[j]['dimensions'] # Landsat 5 download if satname == 'L5': bands[''] = [ im_bands[0], im_bands[1], im_bands[2], im_bands[3], im_bands[4], im_bands[7] ] im_fn[''] = im_date + '_' + satname + '_' + inputs[ 'sitename'] + suffix # if two images taken at the same date add 'dup' to the name (duplicate) if any(im_fn[''] in _ for _ in all_names): im_fn[''] = im_date + '_' + satname + '_' + inputs[ 'sitename'] + '_dup' + suffix all_names.append(im_fn['']) filenames.append(im_fn['']) # download .tif from EE while True: try: im_ee = ee.Image(im_meta['id']) local_data = download_tif(im_ee, inputs['polygon'], bands[''], filepaths[1]) break except: continue # rename the file as the image is downloaded as 'data.tif' try: os.rename(local_data, os.path.join(filepaths[1], im_fn[''])) except: # overwrite if already exists os.remove(os.path.join(filepaths[1], im_fn[''])) os.rename(local_data, os.path.join(filepaths[1], im_fn[''])) # metadata for .txt file filename_txt = im_fn[''].replace('.tif', '') metadict = { 'filename': im_fn[''], 'acc_georef': georef_accs[i], 'epsg': im_epsg[i] } # Landsat 7 and 8 download elif satname in ['L7', 'L8']: if satname == 'L7': bands['pan'] = [im_bands[8]] # panchromatic band bands['ms'] = [ im_bands[0], im_bands[1], im_bands[2], im_bands[3], im_bands[4], im_bands[9] ] # multispectral bands else: bands['pan'] = [im_bands[7]] # panchromatic band bands['ms'] = [ im_bands[1], im_bands[2], im_bands[3], im_bands[4], im_bands[5], im_bands[11] ] # multispectral bands for key in bands.keys(): im_fn[key] = im_date + '_' + satname + '_' + inputs[ 'sitename'] + '_' + key + suffix # if two images taken at the same date add 'dup' to the name (duplicate) if any(im_fn['pan'] in _ for _ in all_names): for key in bands.keys(): im_fn[key] = im_date + '_' + satname + '_' + inputs[ 'sitename'] + '_' + key + '_dup' + suffix all_names.append(im_fn['pan']) filenames.append(im_fn['pan']) # download .tif from EE (panchromatic band and multispectral bands) while True: try: im_ee = ee.Image(im_meta['id']) local_data_pan = download_tif(im_ee, inputs['polygon'], bands['pan'], filepaths[1]) local_data_ms = download_tif(im_ee, inputs['polygon'], bands['ms'], filepaths[2]) break except: continue # rename the files as the image is downloaded as 'data.tif' try: # panchromatic os.rename(local_data_pan, os.path.join(filepaths[1], im_fn['pan'])) except: # overwrite if already exists os.remove(os.path.join(filepaths[1], im_fn['pan'])) os.rename(local_data_pan, os.path.join(filepaths[1], im_fn['pan'])) try: # multispectral os.rename(local_data_ms, os.path.join(filepaths[2], im_fn['ms'])) except: # overwrite if already exists os.remove(os.path.join(filepaths[2], im_fn['ms'])) os.rename(local_data_ms, os.path.join(filepaths[2], im_fn['ms'])) # metadata for .txt file filename_txt = im_fn['pan'].replace('_pan', '').replace('.tif', '') metadict = { 'filename': im_fn['pan'], 'acc_georef': georef_accs[i], 'epsg': im_epsg[i] } # Sentinel-2 download elif satname in ['S2']: bands['10m'] = [ im_bands[1], im_bands[2], im_bands[3], im_bands[7] ] # multispectral bands bands['20m'] = [im_bands[11]] # SWIR band bands['60m'] = [im_bands[15]] # QA band for key in bands.keys(): im_fn[key] = im_date + '_' + satname + '_' + inputs[ 'sitename'] + '_' + key + suffix # if two images taken at the same date add 'dup' to the name (duplicate) if any(im_fn['10m'] in _ for _ in all_names): for key in bands.keys(): im_fn[key] = im_date + '_' + satname + '_' + inputs[ 'sitename'] + '_' + key + '_dup' + suffix # also check for triplicates (only on S2 imagery) and add 'tri' to the name if im_fn['10m'] in all_names: for key in bands.keys(): im_fn[ key] = im_date + '_' + satname + '_' + inputs[ 'sitename'] + '_' + key + '_tri' + suffix all_names.append(im_fn['10m']) filenames.append(im_fn['10m']) # download .tif from EE (multispectral bands at 3 different resolutions) while True: try: im_ee = ee.Image(im_meta['id']) local_data_10m = download_tif(im_ee, inputs['polygon'], bands['10m'], filepaths[1]) local_data_20m = download_tif(im_ee, inputs['polygon'], bands['20m'], filepaths[2]) local_data_60m = download_tif(im_ee, inputs['polygon'], bands['60m'], filepaths[3]) break except: continue # rename the files as the image is downloaded as 'data.tif' try: # 10m os.rename(local_data_10m, os.path.join(filepaths[1], im_fn['10m'])) except: # overwrite if already exists os.remove(os.path.join(filepaths[1], im_fn['10m'])) os.rename(local_data_10m, os.path.join(filepaths[1], im_fn['10m'])) try: # 20m os.rename(local_data_20m, os.path.join(filepaths[2], im_fn['20m'])) except: # overwrite if already exists os.remove(os.path.join(filepaths[2], im_fn['20m'])) os.rename(local_data_20m, os.path.join(filepaths[2], im_fn['20m'])) try: # 60m os.rename(local_data_60m, os.path.join(filepaths[3], im_fn['60m'])) except: # overwrite if already exists os.remove(os.path.join(filepaths[3], im_fn['60m'])) os.rename(local_data_60m, os.path.join(filepaths[3], im_fn['60m'])) # metadata for .txt file filename_txt = im_fn['10m'].replace('_10m', '').replace('.tif', '') metadict = { 'filename': im_fn['10m'], 'acc_georef': georef_accs[i], 'epsg': im_epsg[i] } # write metadata with open(os.path.join(filepaths[0], filename_txt + '.txt'), 'w') as f: for key in metadict.keys(): f.write('%s\t%s\n' % (key, metadict[key])) # print percentage completion for user print('\r%d%%' % int((i + 1) / len(im_dict_T1[satname]) * 100), end='') print('') # once all images have been downloaded, load metadata from .txt files metadata = get_metadata(inputs) # merge overlapping images (necessary only if the polygon is at the boundary of an image) if 'S2' in metadata.keys(): try: metadata = merge_overlapping_images(metadata, inputs) except: print( 'WARNING: there was an error while merging overlapping S2 images,' + ' please open an issue on Github at https://github.com/kvos/CoastSat/issues' + ' and include your script so we can find out what happened.') # save metadata dict with open( os.path.join(im_folder, inputs['sitename'] + '_metadata' + '.pkl'), 'wb') as f: pickle.dump(metadata, f) return metadata
def main(): """Using the Drive v3 API to download products from GEE for upload to S3.""" yearly, year, bucket = parseCmdLine() if yearly: productName = 'YearlyChange' + year bucketName = bucket csvString = "\'SWIR-Custom-Change-Between-*\'" else: productName = 'LatestChange' bucketName = 'current-year-to-date/' csvString = "\'SWIR-Latest-Change-Between-*\'" successfulDownloads = 0 creds = None #pdb.set_trace() # The file token.pickle stores the user's access and refresh tokens, and is # created automatically when the authorization flow completes for the first # time. if os.path.exists(drive_key_file): with open(drive_key_file, 'rb') as token: creds = pickle.load(token) # If there are no (valid) credentials available, let the user log in. if not creds or not creds.valid: if creds and creds.expired and creds.refresh_token: creds.refresh(Request()) else: flow = InstalledAppFlow.from_client_secrets_file( credentials_file, SCOPES) creds = flow.run_local_server(port=0) # Save the credentials for the next run with open(drive_key_file, 'wb') as token: pickle.dump(creds, token) service = build('drive', 'v3', credentials=creds) ee.Initialize() text_file = open(ids_file, "r") ids = text_file.read().split(',') ids = list(filter(None, ids)) tasks = ee.data.getTaskList() myTasks = list(filter(lambda x: x['id'] in ids, tasks)) print('Begin download at {0}'.format( datetime.datetime.now().strftime("%a, %d %B %Y %I:%M:%S"))) while True: results = service.files().list( pageSize=100, q="mimeType = 'image/tiff'", fields="nextPageToken, files(id, name)").execute() # get the specifics of the items on drive for download items = results.get('files', []) # get any completed tasks and download them successfulDownloads = len(downloadMultiple(items, service)) # if there's nothing let to do, quit if not (pendingTasks(ids) or completedTasksRemaining(ids) or successfulDownloads): break else: print("Waiting for exports to complete.") time.sleep(5 * 60) # Delay for 5 minutes. # find IDs for the scenesBegin and scenesEnd CSVs queryStr = "mimeType != 'image/tiff' and name contains " + csvString results = service.files().list( pageSize=100, q=queryStr, fields="nextPageToken, files(id, name)").execute() # get the fileIDs of the items download items = results.get('files', []) # get CSVs and download them downloadedFiles = downloadMultiple(items, service) successfulDownloads = len(downloadedFiles) #pdb.set_trace() p = re.compile( '(NDVI.?|SWIR.?|NDMI.?)(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}\w*(L8|S2)' ) satelliteName = re.search(p, downloadedFiles[0]).group(3) if successfulDownloads == 2: upload_file(downloadDir + downloadedFiles[0], "data.southfact.com", bucketName + downloadedFiles[0]) upload_file(downloadDir + downloadedFiles[1], "data.southfact.com", bucketName + downloadedFiles[1]) # Mainland Southern states and PR VI mosaics print('Begin mosaic to geotiff at {0}'.format( datetime.datetime.now().strftime("%a, %d %B %Y %I:%M:%S"))) os.chdir(downloadDir) # no metadata for now #mosaicDownloadedToGeotiff(re.compile('SWIR(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}datesBegin(L8|S2)CONUS'), 'swirdatesBegin' + productName + satelliteName + 'CONUS.tif') #mosaicDownloadedToGeotiff(re.compile('SWIR(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}datesEnd(L8|S2)CONUS'), 'swirdatesEnd' + productName + satelliteName + 'CONUS.tif') #mosaicDownloadedToGeotiff(re.compile('SWIR(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}datesBegin(L8|S2)PRVI'), 'swirdatesBegin' + productName + satelliteName + 'PRVI.tif') #mosaicDownloadedToGeotiff(re.compile('SWIR(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}datesEnd(L8|S2)PRVI'), 'swirdatesEnd' + productName + satelliteName + 'PRVI.tif') if yearly: # for yearly statewide products produce a shapefile of change polys and a regional GeoTIFF #pdb.set_trace() downloadedToShape( re.compile( 'SWIR.?(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}(LA|AR|MS|KY|TN|OK|VA|SC|NC|GA|AL|TX|FL|PR|VI)(L8|S2)', 'swir' + productName + satelliteName)) else: mosaicDownloadedToGeotiff( re.compile( 'SWIR.?(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}(L8|S2)CONUS' ), 'swir' + productName + satelliteName + 'CONUS.tif') mosaicDownloadedToGeotiff( re.compile( 'SWIR.?(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}(L8|S2)PRVI' ), 'swir' + productName + satelliteName + 'PRVI.tif') mosaicDownloadedToGeotiff( re.compile( 'NDMI.?(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}(L8|S2)CONUS' ), 'ndmi' + productName + satelliteName + 'CONUS.tif') mosaicDownloadedToGeotiff( re.compile( 'NDMI.?(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}(L8|S2)PRVI' ), 'ndmi' + productName + satelliteName + 'PRVI.tif') mosaicDownloadedToGeotiff( re.compile( 'NDVI.?(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}(L8|S2)CONUS' ), 'ndvi' + productName + satelliteName + 'CONUS.tif') mosaicDownloadedToGeotiff( re.compile( 'NDVI.?(-Latest|-Custom)-Change-Between-[0-9]{4}-and-[0-9]{4}(L8|S2)PRVI' ), 'ndvi' + productName + satelliteName + 'PRVI.tif') onlyfiles = [ f for f in os.listdir(outputDir) if isfile(join(outputDir, f)) ] for file in onlyfiles: upload_file(outputDir + file, "data.southfact.com", bucketName + file) #clean up my mess shutil.rmtree('/mnt/efs/fs1/GeoTIFF') shutil.rmtree('/mnt/efs/fs1/output') print('Finished at {0}'.format( datetime.datetime.now().strftime("%a, %d %B %Y %I:%M:%S")))
#!/usr/bin/env python # -*- coding: utf-8 -*- """ Collections module. Add some needed parameters to collections and create collection groups to generate a Best Available Pixel Composite """ import ee import ee.data if not ee.data._initialized: ee.Initialize() import indices from geetools import cloud_mask as cld from copy import deepcopy import functions from geetools import tools from datetime import date initialized = True ACTUAL_YEAR = date.today().year IDS = { 'L1': 'LANDSAT/LM1_L1T', 'L2': 'LANDSAT/LM2_L1T', 'L3': 'LANDSAT/LM3_L1T', 'L4TOA': 'LANDSAT/LT04/C01/T1_TOA', 'L4USGS': 'LANDSAT/LT04/C01/T1_SR', 'L5TOA': 'LANDSAT/LT05/C01/T1_TOA', 'L5USGS': 'LANDSAT/LT05/C01/T1_SR', 'L5LED': 'LEDAPS/LT5_L1T_SR', 'L7TOA': 'LANDSAT/LE07/C01/T1_TOA', 'L7USGS': 'LANDSAT/LE07/C01/T1_SR',
def main(ini_path=None, overwrite_flag=False, delay=0, key=None): """Compute daily Tcorr images Parameters ---------- ini_path : str Input file path. overwrite_flag : bool, optional If True, overwrite existing files (the default is False). delay : float, optional Delay time between each export task (the default is 0). key : str, optional File path to an Earth Engine json key file (the default is None). """ logging.info('\nCompute daily Tcorr images') ini = utils.read_ini(ini_path) model_name = 'SSEBOP' # model_name = ini['INPUTS']['et_model'].upper() if (ini[model_name]['tmax_source'].upper() == 'CIMIS' and ini['INPUTS']['end_date'] < '2003-10-01'): logging.error( '\nCIMIS is not currently available before 2003-10-01, exiting\n') sys.exit() elif (ini[model_name]['tmax_source'].upper() == 'DAYMET' and ini['INPUTS']['end_date'] > '2017-12-31'): logging.warning( '\nDAYMET is not currently available past 2017-12-31, ' 'using median Tmax values\n') # sys.exit() # elif (ini[model_name]['tmax_source'].upper() == 'TOPOWX' and # ini['INPUTS']['end_date'] > '2017-12-31'): # logging.warning( # '\nDAYMET is not currently available past 2017-12-31, ' # 'using median Tmax values\n') # # sys.exit() logging.info('\nInitializing Earth Engine') if key: logging.info(' Using service account key file: {}'.format(key)) # The "EE_ACCOUNT" parameter is not used if the key file is valid ee.Initialize(ee.ServiceAccountCredentials('deadbeef', key_file=key)) else: ee.Initialize() # Output Tcorr daily image collection tcorr_daily_coll_id = '{}/{}_daily'.format( ini['EXPORT']['export_coll'], tmax_name.lower()) # Get a Tmax image to set the Tcorr values to logging.debug('\nTmax properties') tmax_name = ini[model_name]['tmax_source'] tmax_source = tmax_name.split('_', 1)[0] tmax_version = tmax_name.split('_', 1)[1] tmax_coll_id = 'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower()) tmax_coll = ee.ImageCollection(tmax_coll_id) tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0) logging.debug(' Collection: {}'.format(tmax_coll_id)) logging.debug(' Source: {}'.format(tmax_source)) logging.debug(' Version: {}'.format(tmax_version)) logging.debug('\nExport properties') export_geo = ee.Image(tmax_mask).projection().getInfo()['transform'] export_crs = ee.Image(tmax_mask).projection().getInfo()['crs'] export_shape = ee.Image(tmax_mask).getInfo()['bands'][0]['dimensions'] export_extent = [ export_geo[2], export_geo[5] + export_shape[1] * export_geo[4], export_geo[2] + export_shape[0] * export_geo[0], export_geo[5]] logging.debug(' CRS: {}'.format(export_crs)) logging.debug(' Extent: {}'.format(export_extent)) logging.debug(' Geo: {}'.format(export_geo)) logging.debug(' Shape: {}'.format(export_shape)) # # Limit export to a user defined study area or geometry? # export_geom = ee.Geometry.Rectangle( # [-125, 24, -65, 50], proj='EPSG:4326', geodesic=False) # CONUS # export_geom = ee.Geometry.Rectangle( # [-124, 35, -119, 42], proj='EPSG:4326', geodesic=False) # California # If cell_size parameter is set in the INI, # adjust the output cellsize and recompute the transform and shape try: export_cs = float(ini['EXPORT']['cell_size']) export_shape = [ int(math.ceil(abs((export_shape[0] * export_geo[0]) / export_cs))), int(math.ceil(abs((export_shape[1] * export_geo[4]) / export_cs)))] export_geo = [export_cs, 0.0, export_geo[2], 0.0, -export_cs, export_geo[5]] logging.debug(' Custom export cell size: {}'.format(export_cs)) logging.debug(' Geo: {}'.format(export_geo)) logging.debug(' Shape: {}'.format(export_shape)) except KeyError: pass # Get current asset list if ini['EXPORT']['export_dest'].upper() == 'ASSET': logging.debug('\nGetting asset list') # DEADBEEF - daily is hardcoded in the asset_id for now asset_list = utils.get_ee_assets(tcorr_daily_coll_id) else: raise ValueError('invalid export destination: {}'.format( ini['EXPORT']['export_dest'])) # Get current running tasks tasks = utils.get_ee_tasks() if logging.getLogger().getEffectiveLevel() == logging.DEBUG: logging.debug(' Tasks: {}\n'.format(len(tasks))) input('ENTER') # Limit by year and month try: month_list = sorted(list(utils.parse_int_set(ini['TCORR']['months']))) except: logging.info('\nTCORR "months" parameter not set in the INI,' '\n Defaulting to all months (1-12)\n') month_list = list(range(1, 13)) try: year_list = sorted(list(utils.parse_int_set(ini['TCORR']['years']))) except: logging.info('\nTCORR "years" parameter not set in the INI,' '\n Defaulting to all available years\n') year_list = [] # Key is cycle day, value is a reference date on that cycle # Data from: https://landsat.usgs.gov/landsat_acq # I only need to use 8 cycle days because of 5/7 and 7/8 are offset cycle_dates = { 7: '1970-01-01', 8: '1970-01-02', 1: '1970-01-03', 2: '1970-01-04', 3: '1970-01-05', 4: '1970-01-06', 5: '1970-01-07', 6: '1970-01-08', } # cycle_dates = { # 1: '2000-01-06', # 2: '2000-01-07', # 3: '2000-01-08', # 4: '2000-01-09', # 5: '2000-01-10', # 6: '2000-01-11', # 7: '2000-01-12', # 8: '2000-01-13', # # 9: '2000-01-14', # # 10: '2000-01-15', # # 11: '2000-01-16', # # 12: '2000-01-01', # # 13: '2000-01-02', # # 14: '2000-01-03', # # 15: '2000-01-04', # # 16: '2000-01-05', # } cycle_base_dt = datetime.datetime.strptime(cycle_dates[1], '%Y-%m-%d') iter_start_dt = datetime.datetime.strptime( ini['INPUTS']['start_date'], '%Y-%m-%d') iter_end_dt = datetime.datetime.strptime( ini['INPUTS']['end_date'], '%Y-%m-%d') # Iterate over date ranges for export_dt in utils.date_range(iter_start_dt, iter_end_dt): export_date = export_dt.strftime('%Y-%m-%d') if ((month_list and export_dt.month not in month_list) or ( year_list and export_dt.year not in year_list)): logging.debug('Date: {} - skipping'.format(export_date)) continue logging.info('Date: {}'.format(export_date)) if export_date >= datetime.datetime.today().strftime('%Y-%m-%d'): logging.info(' Unsupported date, skipping') continue elif export_date < '1984-03-23': logging.info(' No Landsat 5+ images before 1984-03-16, skipping') continue export_id = ini['EXPORT']['export_id_fmt'] \ .format( product=tmax_name.lower(), date=export_dt.strftime('%Y%m%d'), export=ini['EXPORT']['export_dest'].lower()) logging.debug(' Export ID: {}'.format(export_id)) if ini['EXPORT']['export_dest'] == 'ASSET': # DEADBEEF - daily is hardcoded in the asset_id for now asset_id = '{}/{}'.format( tcorr_daily_coll_id, export_dt.strftime('%Y%m%d')) logging.debug(' Asset ID: {}'.format(asset_id)) if overwrite_flag: if export_id in tasks.keys(): logging.debug(' Task already submitted, cancelling') ee.data.cancelTask(tasks[export_id]) # This is intentionally not an "elif" so that a task can be # cancelled and an existing image/file/asset can be removed if (ini['EXPORT']['export_dest'].upper() == 'ASSET' and asset_id in asset_list): logging.debug(' Asset already exists, removing') ee.data.deleteAsset(asset_id) else: if export_id in tasks.keys(): logging.debug(' Task already submitted, exiting') continue elif (ini['EXPORT']['export_dest'].upper() == 'ASSET' and asset_id in asset_list): logging.debug(' Asset already exists, skipping') continue # Build and merge the Landsat collections # Time filters are to remove bad (L5) and pre-op (L8) images # .filterBounds(export_geom) \ l8_coll = ee.ImageCollection('LANDSAT/LC08/C01/T1_RT_TOA') \ .filterDate(export_dt, export_dt + datetime.timedelta(days=1)) \ .filterBounds(tmax_mask.geometry()) \ .filterMetadata('CLOUD_COVER_LAND', 'less_than', float(ini['INPUTS']['cloud_cover'])) \ .filterMetadata('DATA_TYPE', 'equals', 'L1TP') \ .filter(ee.Filter.gt('system:time_start', ee.Date('2013-03-24').millis())) l7_coll = ee.ImageCollection('LANDSAT/LE07/C01/T1_RT_TOA') \ .filterDate(export_dt, export_dt + datetime.timedelta(days=1)) \ .filterBounds(tmax_mask.geometry()) \ .filterMetadata('CLOUD_COVER_LAND', 'less_than', float(ini['INPUTS']['cloud_cover'])) \ .filterMetadata('DATA_TYPE', 'equals', 'L1TP') l5_coll = ee.ImageCollection('LANDSAT/LT05/C01/T1_TOA') \ .filterDate(export_dt, export_dt + datetime.timedelta(days=1)) \ .filterBounds(tmax_mask.geometry()) \ .filterMetadata('CLOUD_COVER_LAND', 'less_than', float(ini['INPUTS']['cloud_cover'])) \ .filterMetadata('DATA_TYPE', 'equals', 'L1TP') \ .filter(ee.Filter.lt('system:time_start', ee.Date('2011-12-31').millis())) # l4_coll = ee.ImageCollection('LANDSAT/LT04/C01/T1_TOA') \ # .filterDate(export_dt, export_dt + datetime.timedelta(days=1)) \ # .filterBounds(tmax_img.geometry()) \ # .filterMetadata('CLOUD_COVER_LAND', 'less_than', # float(ini['INPUTS']['cloud_cover'])) \ # .filterMetadata('DATA_TYPE', 'equals', 'L1TP') # if export_date <= '1993-12-31': # landsat_coll = ee.ImageCollection(l5_coll.merge(l4_coll)) if export_date < '1999-01-01': landsat_coll = l5_coll elif export_date <= '2011-12-31': landsat_coll = ee.ImageCollection(l7_coll.merge(l5_coll)) elif export_date <= '2013-03-24': landsat_coll = l7_coll else: landsat_coll = ee.ImageCollection(l8_coll.merge(l7_coll)) # pprint.pprint(landsat_coll.aggregate_histogram('system:index').getInfo()) # pprint.pprint(ee.Image(landsat_coll.first()).getInfo()) # input('ENTER') def tcorr_img_func(image): t_stats = ssebop.Image.from_landsat_c1_toa( ee.Image(image), tdiff_threshold=float(ini[model_name]['tdiff_threshold'])) \ .tcorr_stats t_stats = ee.Dictionary(t_stats) \ .combine({'tcorr_p5': 0, 'tcorr_count': 0}, overwrite=False) # tcorr = ee.Algorithms.If( # t_stats.get('tcorr_p5'), ee.Number(t_stats.get('tcorr_p5')), 0) tcorr = ee.Number(t_stats.get('tcorr_p5')) count = ee.Number(t_stats.get('tcorr_count')) # Remove the merged collection indices from the system:index scene_id = ee.List( ee.String(image.get('system:index')).split('_')).slice(-3) scene_id = ee.String(scene_id.get(0)).cat('_') \ .cat(ee.String(scene_id.get(1))).cat('_') \ .cat(ee.String(scene_id.get(2))) # return ee.Image([ # tmax_img.select([0], ['tcorr']).multiply(0) \ # .add(ee.Image.constant(tcorr)).float(), # tmax_img.select([0], ['count']).multiply(0) # .add(ee.Image.constant(count)).int()]) \ return tmax_mask.add(tcorr) \ .rename(['tcorr']) \ .clip(image.geometry()) \ .set({ 'system:time_start': image.get('system:time_start'), 'scene_id': scene_id, 'wrs2_tile': scene_id.slice(5, 11), 'spacecraft_id': image.get('SPACECRAFT_ID'), 'tcorr': tcorr, 'count': count, }) # # Test for one image # pprint.pprint(tcorr_img_func(ee.Image(landsat_coll \ # .filterMetadata('WRS_PATH', 'equals', 36) \ # .filterMetadata('WRS_ROW', 'equals', 33).first())).getInfo()) # input('ENTER') tcorr_img_coll = ee.ImageCollection(landsat_coll.map(tcorr_img_func)) \ .filterMetadata('count', 'not_less_than', float(ini['TCORR']['min_pixel_count'])) # pprint.pprint(tcorr_img_coll.aggregate_histogram('system:index').getInfo()) # pprint.pprint(ee.Image(tcorr_img_coll.first()).getInfo()) # input('ENTER') # If there are no Tcorr values, return an empty image tcorr_img = ee.Algorithms.If( tcorr_img_coll.size().gt(0), tcorr_img_coll.median(), tmax_mask.updateMask(0)) # pprint.pprint(tcorr_img.getInfo()) # pprint.pprint(tcorr_img_coll.size().getInfo()) # input('ENTER') # # This doesn't work (median is returning no bands for some dates) # tcorr_img = tmax_mask.add(tcorr_img_coll.median()) \ # .updateMask(0) def unique_properties(coll, property): return ee.String(ee.List(ee.Dictionary( coll.aggregate_histogram(property)).keys()).join(',')) wrs2_tile_list = ee.String('').cat(unique_properties( tcorr_img_coll, 'wrs2_tile')) landsat_list = ee.String('').cat(unique_properties( tcorr_img_coll, 'spacecraft_id')) # # Is there a better way of building these strings? # wrs2_tile_list = ee.Algorithms.If( # tcorr_img_coll.size().gt(0), # ee.String(ee.List(ee.Dictionary(tcorr_img_coll \ # .aggregate_histogram('WRS2_TILE')).keys()).join(',')), # ee.String('')) # landsat_list = ee.Algorithms.If( # tcorr_img_coll.size().gt(0), # ee.String(ee.List(ee.Dictionary(tcorr_img_coll\ # .aggregate_histogram('SPACECRAFT_ID')).keys()).join(',')), # ee.String('')) # Cast to float and set properties tcorr_img = ee.Image(tcorr_img).rename(['tcorr']).double() \ .set({ 'system:time_start': utils.millis(export_dt), 'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'), 'date': export_dt.strftime('%Y-%m-%d'), 'year': int(export_dt.year), 'month': int(export_dt.month), 'day': int(export_dt.day), 'doy': int(export_dt.strftime('%j')), 'cycle_day': ((export_dt - cycle_base_dt).days % 8) + 1, 'landsat': landsat_list, 'model_name': model_name, 'model_version': ssebop.__version__, 'tmax_source': tmax_source.upper(), 'tmax_version': tmax_version.upper(), 'wrs2_tiles': wrs2_tile_list, }) # pprint.pprint(tcorr_img.getInfo()) # input('ENTER') # Build export tasks if ini['EXPORT']['export_dest'] == 'ASSET': logging.debug(' Building export task') task = ee.batch.Export.image.toAsset( image=ee.Image(tcorr_img), description=export_id, assetId=asset_id, crs=export_crs, crsTransform='[' + ','.join(list(map(str, export_geo))) + ']', dimensions='{0}x{1}'.format(*export_shape), ) logging.debug(' Starting export task') utils.ee_task_start(task) # Pause before starting next task utils.delay_task(delay) logging.debug('')
def download_data_ts(R:RegionST, products, bands, path_save, scale=10): ee.Initialize() times = (R.times[0], R.times[-1]) path_save.mkdir(exist_ok=True, parents=True) sR = [R] if R.shape[0] <= 32 else split_region(R, size=32, cls=RegionST) for j, R in enumerate(sR): region = (f"[[{R.bbox.left}, {R.bbox.bottom}], [{R.bbox.right}, {R.bbox.bottom}], " + f"[{R.bbox.right}, {R.bbox.top}], [{R.bbox.left}, {R.bbox.top}]]") # Merge products to single image collection imCol = ee.ImageCollection(products[0]) for i in range(1, len(products)): imCol = imCol.merge(ee.ImageCollection(products[i])) imCol = filter_region(imCol, R, times=times, bands=bands) imCol = ee.ImageCollection(imCol) colList = imCol.toList(imCol.size()) # Download each image for i in range(colList.size().getInfo()): image = ee.Image(colList.get(i)) zip_error = True for i in range(10): # Try 10 times if zip_error: try: url = image.getDownloadURL( {'scale': scale, 'crs': 'EPSG:4326', 'region': f'{region}'}) r = requests.get(url) with open(str(path_save/'data.zip'), 'wb') as f: f.write(r.content) with zipfile.ZipFile(str(path_save/'data.zip'), 'r') as f: files = f.namelist() f.extractall(str(path_save)) os.remove(str(path_save/'data.zip')) zip_error = False except: zip_error = True os.remove(str(path_save/'data.zip')) time.sleep(10) if zip_error: raise Exception(f'Failed to process {url}') for f in files: f = path_save/f os.rename(str(f), str(path_save/f'{f.stem}_{j}{f.suffix}')) # Merge files suffix = '.tif' files = path_save.ls(include=[suffix]) files = [o.stem for o in files] ref = np.unique(['_'.join(o.split('_')[:-1]) for o in files if len(o.split('_')[-1]) < 6]) ids = np.unique([int(o.split('_')[-1]) for o in files if len(o.split('_')[-1]) < 6]) file_groups = [[path_save/f'{r}_{i}{suffix}' for i in ids if f'{r}_{i}' in files] for r in ref] for fs in file_groups: if len(fs) < 500: fsave = '_'.join(fs[0].stem.split('_')[:-1]) + suffix merge_tifs(fs, fsave, delete=True) else: fs_break = np.array(fs)[:(len(fs)//500)*500].reshape(len(fs)//500,-1).tolist() if len(fs[(len(fs)//500)*500:]) > 0: fs_break.append(fs[(len(fs)//500)*500:]) for fsi, fs2 in enumerate(fs_break): fsave = '_'.join(fs2[0].stem.split('_')[:-1]) + f'_break{fsi}' + suffix merge_tifs(fs2, fsave, delete=True) files = path_save.ls(include=[suffix, '_break']) files = [o.stem for o in files] ref = np.unique(['_'.join(o.split('_')[:-1]) for o in files if len(o.split('_')[-1]) < 11]) ids = np.unique([o.split('_')[-1] for o in files if len(o.split('_')[-1]) < 11]) file_groups = [[path_save/f'{r}_{i}{suffix}' for i in ids if f'{r}_{i}' in files] for r in ref] for fs in file_groups: fsave = '_'.join(fs[0].stem.split('_')[:-1]) + suffix merge_tifs(fs, fsave, delete=True)
def main(ini_path=None, overwrite_flag=False, delay_time=0, gee_key_file=None, max_ready=-1, reverse_flag=False): """Compute annual Tcorr images from scene images Parameters ---------- ini_path : str Input file path. overwrite_flag : bool, optional If True, overwrite existing files (the default is False). delay_time : float, optional Delay time in seconds between starting export tasks (or checking the number of queued tasks, see "max_ready" parameter). The default is 0. gee_key_file : str, None, optional Earth Engine service account JSON key file (the default is None). max_ready: int, optional Maximum number of queued "READY" tasks. The default is -1 which is implies no limit to the number of tasks that will be submitted. reverse_flag : bool, optional If True, process WRS2 tiles in reverse order. """ logging.info('\nCompute annual Tcorr images from scene images') ini = utils.read_ini(ini_path) model_name = 'SSEBOP' # model_name = ini['INPUTS']['et_model'].upper() tmax_name = ini[model_name]['tmax_source'] export_id_fmt = 'tcorr_scene_{product}_{wrs2}_annual_from_scene' asset_id_fmt = '{coll_id}/{wrs2}' tcorr_annual_coll_id = '{}/{}_annual_from_scene'.format( ini['EXPORT']['export_coll'], tmax_name.lower()) wrs2_coll_id = 'projects/earthengine-legacy/assets/' \ 'projects/usgs-ssebop/wrs2_descending_custom' wrs2_tile_field = 'WRS2_TILE' # wrs2_path_field = 'ROW' # wrs2_row_field = 'PATH' try: wrs2_tiles = str(ini['INPUTS']['wrs2_tiles']) wrs2_tiles = [x.strip() for x in wrs2_tiles.split(',')] wrs2_tiles = sorted([x.lower() for x in wrs2_tiles if x]) except KeyError: wrs2_tiles = [] logging.debug(' wrs2_tiles: not set in INI, defaulting to []') except Exception as e: raise e try: study_area_extent = str(ini['INPUTS']['study_area_extent']) \ .replace('[', '').replace(']', '').split(',') study_area_extent = [float(x.strip()) for x in study_area_extent] except KeyError: study_area_extent = None logging.debug(' study_area_extent: not set in INI, defaulting to None') except Exception as e: raise e # TODO: Add try/except blocks and default values? # TODO: Filter Tcorr scene collection based on collections parameter # collections = [x.strip() for x in ini['INPUTS']['collections'].split(',')] cloud_cover = float(ini['INPUTS']['cloud_cover']) min_pixel_count = float(ini['TCORR']['min_pixel_count']) min_scene_count = float(ini['TCORR']['min_scene_count']) if (tmax_name.upper() == 'CIMIS' and ini['INPUTS']['end_date'] < '2003-10-01'): logging.error( '\nCIMIS is not currently available before 2003-10-01, exiting\n') sys.exit() elif (tmax_name.upper() == 'DAYMET' and ini['INPUTS']['end_date'] > '2018-12-31'): logging.warning( '\nDAYMET is not currently available past 2018-12-31, ' 'using median Tmax values\n') # sys.exit() # elif (tmax_name.upper() == 'TOPOWX' and # ini['INPUTS']['end_date'] > '2017-12-31'): # logging.warning( # '\nDAYMET is not currently available past 2017-12-31, ' # 'using median Tmax values\n') # # sys.exit() logging.info('\nInitializing Earth Engine') if gee_key_file: logging.info(' Using service account key file: {}'.format(gee_key_file)) # The "EE_ACCOUNT" parameter is not used if the key file is valid ee.Initialize(ee.ServiceAccountCredentials('x', key_file=gee_key_file), use_cloud_api=True) else: ee.Initialize(use_cloud_api=True) logging.debug('\nTmax properties') tmax_source = tmax_name.split('_', 1)[0] tmax_version = tmax_name.split('_', 1)[1] tmax_coll_id = 'projects/earthengine-legacy/assets/' \ 'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower()) tmax_coll = ee.ImageCollection(tmax_coll_id) tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0) logging.debug(' Collection: {}'.format(tmax_coll_id)) logging.debug(' Source: {}'.format(tmax_source)) logging.debug(' Version: {}'.format(tmax_version)) # Get the Tcorr scene image collection properties logging.debug('\nTcorr scene collection') tcorr_scene_coll_id = '{}/{}_scene'.format( ini['EXPORT']['export_coll'], tmax_name.lower()) logging.debug('\nExport properties') export_info = utils.get_info(ee.Image(tmax_mask)) if 'daymet' in tmax_name.lower(): # Custom smaller extent for DAYMET focused on CONUS export_extent = [-1999750, -1890500, 2500250, 1109500] export_shape = [4500, 3000] export_geo = [1000, 0, -1999750, 0, -1000, 1109500] # Custom medium extent for DAYMET of CONUS, Mexico, and southern Canada # export_extent = [-2099750, -3090500, 2900250, 1909500] # export_shape = [5000, 5000] # export_geo = [1000, 0, -2099750, 0, -1000, 1909500] export_crs = export_info['bands'][0]['crs'] else: export_crs = export_info['bands'][0]['crs'] export_geo = export_info['bands'][0]['crs_transform'] export_shape = export_info['bands'][0]['dimensions'] # export_geo = ee.Image(tmax_mask).projection().getInfo()['transform'] # export_crs = ee.Image(tmax_mask).projection().getInfo()['crs'] # export_shape = ee.Image(tmax_mask).getInfo()['bands'][0]['dimensions'] export_extent = [ export_geo[2], export_geo[5] + export_shape[1] * export_geo[4], export_geo[2] + export_shape[0] * export_geo[0], export_geo[5]] export_geom = ee.Geometry.Rectangle( export_extent, proj=export_crs, geodesic=False) logging.debug(' CRS: {}'.format(export_crs)) logging.debug(' Extent: {}'.format(export_extent)) logging.debug(' Geo: {}'.format(export_geo)) logging.debug(' Shape: {}'.format(export_shape)) if study_area_extent is None: if 'daymet' in tmax_name.lower(): # CGM - For now force DAYMET to a slightly smaller "CONUS" extent study_area_extent = [-125, 25, -65, 49] # study_area_extent = [-125, 25, -65, 52] elif 'cimis' in tmax_name.lower(): study_area_extent = [-124, 35, -119, 42] else: # TODO: Make sure output from bounds is in WGS84 study_area_extent = tmax_mask.geometry().bounds().getInfo() logging.debug(f'\nStudy area extent not set in INI, ' f'default to {study_area_extent}') study_area_geom = ee.Geometry.Rectangle( study_area_extent, proj='EPSG:4326', geodesic=False) if not ee.data.getInfo(tcorr_annual_coll_id): logging.info('\nExport collection does not exist and will be built' '\n {}'.format(tcorr_annual_coll_id)) input('Press ENTER to continue') ee.data.createAsset({'type': 'IMAGE_COLLECTION'}, tcorr_annual_coll_id) # Get current asset list logging.debug('\nGetting GEE asset list') asset_list = utils.get_ee_assets(tcorr_annual_coll_id) # if logging.getLogger().getEffectiveLevel() == logging.DEBUG: # pprint.pprint(asset_list[:10]) # Get current running tasks tasks = utils.get_ee_tasks() if logging.getLogger().getEffectiveLevel() == logging.DEBUG: logging.debug(' Tasks: {}\n'.format(len(tasks))) input('ENTER') # Limit by year month_list = list(range(1, 13)) # try: # month_list = sorted(list(utils.parse_int_set(ini['TCORR']['months']))) # except: # logging.info('\nTCORR "months" parameter not set in the INI,' # '\n Defaulting to all months (1-12)\n') # month_list = list(range(1, 13)) try: year_list = sorted(list(utils.parse_int_set(ini['TCORR']['years']))) except: logging.info('\nTCORR "years" parameter not set in the INI,' '\n Defaulting to all available years\n') year_list = [] # Get the list of WRS2 tiles that intersect the data area and study area wrs2_coll = ee.FeatureCollection(wrs2_coll_id) \ .filterBounds(export_geom) \ .filterBounds(study_area_geom) if wrs2_tiles: wrs2_coll = wrs2_coll.filter(ee.Filter.inList(wrs2_tile_field, wrs2_tiles)) wrs2_info = wrs2_coll.getInfo()['features'] # Iterate over date ranges for wrs2_ftr in sorted(wrs2_info, key=lambda k: k['properties']['WRS2_TILE'], reverse=reverse_flag): wrs2_tile = wrs2_ftr['properties'][wrs2_tile_field] logging.info('{}'.format(wrs2_tile)) wrs2_path = int(wrs2_tile[1:4]) wrs2_row = int(wrs2_tile[5:8]) # wrs2_path = wrs2_ftr['properties'][wrs2_path_field] # wrs2_row = wrs2_ftr['properties'][wrs2_row_field] export_id = export_id_fmt.format( product=tmax_name.lower(), wrs2=wrs2_tile) logging.debug(' Export ID: {}'.format(export_id)) asset_id = asset_id_fmt.format( coll_id=tcorr_annual_coll_id, wrs2=wrs2_tile) logging.debug(' Asset ID: {}'.format(asset_id)) if overwrite_flag: if export_id in tasks.keys(): logging.debug(' Task already submitted, cancelling') ee.data.cancelTask(tasks[export_id]['id']) # This is intentionally not an "elif" so that a task can be # cancelled and an existing image/file/asset can be removed if asset_id in asset_list: logging.debug(' Asset already exists, removing') ee.data.deleteAsset(asset_id) else: if export_id in tasks.keys(): logging.debug(' Task already submitted, exiting') continue elif asset_id in asset_list: logging.debug(' Asset already exists, skipping') continue tcorr_coll = ee.ImageCollection(tcorr_scene_coll_id) \ .filterMetadata('wrs2_tile', 'equals', wrs2_tile) \ .filterMetadata('tcorr_pixel_count', 'not_less_than', min_pixel_count) \ .filter(ee.Filter.inList('year', year_list)) # TODO: Should CLOUD_COVER_LAND filter should be re-applied here? # .filterMetadata('CLOUD_COVER_LAND', 'less_than', cloud_cover) # .filterDate(start_date, end_date) # .filterBounds(ee.Geometry(wrs2_ftr['geometry'])) # Use a common reducer for the images and property stats reducer = ee.Reducer.median() \ .combine(ee.Reducer.count(), sharedInputs=True) # Compute stats from the collection images # This might be used when Tcorr is spatial # tcorr_img = tcorr_coll.reduce(reducer).rename(['tcorr', 'count']) # Compute stats from the image properties tcorr_stats = ee.List(tcorr_coll.aggregate_array('tcorr_value')) \ .reduce(reducer) tcorr_stats = ee.Dictionary(tcorr_stats) \ .combine({'median': 0, 'count': 0}, overwrite=False) tcorr = ee.Number(tcorr_stats.get('median')) count = ee.Number(tcorr_stats.get('count')) index = count.lt(min_scene_count).multiply(7).add(2) # index = ee.Algorithms.If(count.gte(min_scene_count), 2, 9) # Clip the mask image to the Landsat footprint # Change mask values to 1 if count >= threshold # Mask values of 0 will be set to nodata mask_img = tmax_mask.add(count.gte(min_scene_count)) \ .clip(ee.Geometry(wrs2_ftr['geometry'])) output_img = ee.Image( [mask_img.multiply(tcorr), mask_img.multiply(count)]) \ .rename(['tcorr', 'count']) \ .updateMask(mask_img.unmask(0)) # # Write an empty image if the pixel count is too low # # CGM: Check/test if this can be combined into a single If() # tcorr_img = ee.Algorithms.If( # count.gte(min_scene_count), # tmax_mask.add(tcorr), tmax_mask.updateMask(0)) # count_img = ee.Algorithms.If( # count.gte(min_scene_count), # tmax_mask.add(count), tmax_mask.updateMask(0)) # # # Clip to the Landsat image footprint # output_img = ee.Image([tcorr_img, count_img]) \ # .rename(['tcorr', 'count']) \ # .clip(ee.Geometry(wrs2_ftr['geometry'])) # # Clear the transparency mask # output_img = output_img.updateMask(output_img.unmask(0)) output_img = output_img.set({ 'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'), 'model_name': model_name, 'model_version': ssebop.__version__, # 'system:time_start': utils.millis(start_dt), 'tcorr_value': tcorr, 'tcorr_index': index, 'tcorr_scene_count': count, 'tmax_source': tmax_source.upper(), 'tmax_version': tmax_version.upper(), 'wrs2_path': wrs2_path, 'wrs2_row': wrs2_row, 'wrs2_tile': wrs2_tile, 'years': ','.join(map(str, year_list)), # 'year_start': year_list[0], # 'year_end': year_list[-1], }) # pprint.pprint(output_img.getInfo()) # input('ENTER') logging.debug(' Building export task') task = ee.batch.Export.image.toAsset( image=output_img, description=export_id, assetId=asset_id, crs=export_crs, crsTransform='[' + ','.join(list(map(str, export_geo))) + ']', dimensions='{0}x{1}'.format(*export_shape), ) logging.info(' Starting export task') utils.ee_task_start(task) # Pause before starting the next export task utils.delay_task(delay_time, max_ready) logging.debug('')
def main(ini_path=None, overwrite_flag=False, delay_time=0, gee_key_file=None, max_ready=-1, reverse_flag=False): """Compute default Tcorr images by WRS2 tile Parameters ---------- ini_path : str Input file path. overwrite_flag : bool, optional If True, overwrite existing files (the default is False). delay_time : float, optional Delay time in seconds between starting export tasks (or checking the number of queued tasks, see "max_ready" parameter). The default is 0. gee_key_file : str, None, optional Earth Engine service account JSON key file (the default is None). max_ready: int, optional Maximum number of queued "READY" tasks. The default is -1 which is implies no limit to the number of tasks that will be submitted. reverse_flag : bool, optional If True, process WRS2 tiles in reverse order. """ logging.info('\nCompute default Tcorr images by WRS2 tile') ini = utils.read_ini(ini_path) model_name = 'SSEBOP' # model_name = ini['INPUTS']['et_model'].upper() tmax_name = ini[model_name]['tmax_source'] export_id_fmt = 'tcorr_scene_{product}_{wrs2}_default' asset_id_fmt = '{coll_id}/{wrs2}' tcorr_default_coll_id = '{}/{}_default'.format( ini['EXPORT']['export_coll'], tmax_name.lower()) wrs2_coll_id = 'projects/earthengine-legacy/assets/' \ 'projects/usgs-ssebop/wrs2_descending_custom' wrs2_tile_field = 'WRS2_TILE' # wrs2_path_field = 'ROW' # wrs2_row_field = 'PATH' try: wrs2_tiles = str(ini['INPUTS']['wrs2_tiles']) wrs2_tiles = [x.strip() for x in wrs2_tiles.split(',')] wrs2_tiles = sorted([x.lower() for x in wrs2_tiles if x]) except KeyError: wrs2_tiles = [] logging.debug(' wrs2_tiles: not set in INI, defaulting to []') except Exception as e: raise e try: study_area_extent = str(ini['INPUTS']['study_area_extent']) \ .replace('[', '').replace(']', '').split(',') study_area_extent = [float(x.strip()) for x in study_area_extent] except KeyError: study_area_extent = None logging.debug(' study_area_extent: not set in INI') except Exception as e: raise e try: tcorr_default = ini[model_name]['tcorr_default'] except: tcorr_default = 0.978 logging.info('\nInitializing Earth Engine') if gee_key_file: logging.info(' Using service account key file: {}'.format(gee_key_file)) # The "EE_ACCOUNT" parameter is not used if the key file is valid ee.Initialize(ee.ServiceAccountCredentials('x', key_file=gee_key_file), use_cloud_api=True) else: ee.Initialize(use_cloud_api=True) logging.debug('\nTmax properties') tmax_source = tmax_name.split('_', 1)[0] tmax_version = tmax_name.split('_', 1)[1] tmax_coll_id = 'projects/earthengine-legacy/assets/' \ 'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower()) tmax_coll = ee.ImageCollection(tmax_coll_id) tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0) logging.debug(' Collection: {}'.format(tmax_coll_id)) logging.debug(' Source: {}'.format(tmax_source)) logging.debug(' Version: {}'.format(tmax_version)) # # Get the Tcorr scene image collection properties # logging.debug('\nTcorr scene collection') # tcorr_scene_coll_id = '{}/{}_scene'.format( # ini['EXPORT']['export_coll'], tmax_name.lower()) logging.debug('\nExport properties') export_info = utils.get_info(ee.Image(tmax_mask)) if 'daymet' in tmax_name.lower(): # Custom smaller extent for DAYMET focused on CONUS export_extent = [-1999750, -1890500, 2500250, 1109500] export_shape = [4500, 3000] export_geo = [1000, 0, -1999750, 0, -1000, 1109500] # Custom medium extent for DAYMET of CONUS, Mexico, and southern Canada # export_extent = [-2099750, -3090500, 2900250, 1909500] # export_shape = [5000, 5000] # export_geo = [1000, 0, -2099750, 0, -1000, 1909500] export_crs = export_info['bands'][0]['crs'] else: export_crs = export_info['bands'][0]['crs'] export_geo = export_info['bands'][0]['crs_transform'] export_shape = export_info['bands'][0]['dimensions'] # export_geo = ee.Image(tmax_mask).projection().getInfo()['transform'] # export_crs = ee.Image(tmax_mask).projection().getInfo()['crs'] # export_shape = ee.Image(tmax_mask).getInfo()['bands'][0]['dimensions'] export_extent = [ export_geo[2], export_geo[5] + export_shape[1] * export_geo[4], export_geo[2] + export_shape[0] * export_geo[0], export_geo[5]] export_geom = ee.Geometry.Rectangle( export_extent, proj=export_crs, geodesic=False) logging.debug(' CRS: {}'.format(export_crs)) logging.debug(' Extent: {}'.format(export_extent)) logging.debug(' Geo: {}'.format(export_geo)) logging.debug(' Shape: {}'.format(export_shape)) if study_area_extent is None: if 'daymet' in tmax_name.lower(): # CGM - For now force DAYMET to a slightly smaller "CONUS" extent study_area_extent = [-125, 25, -65, 50] # study_area_extent = [-125, 25, -65, 49] # study_area_extent = [-125, 25, -65, 52] elif 'cimis' in tmax_name.lower(): study_area_extent = [-124, 35, -119, 42] else: # TODO: Make sure output from bounds is in WGS84 study_area_extent = tmax_mask.geometry().bounds().getInfo() logging.debug(f'\nStudy area extent not set in INI, ' f'default to {study_area_extent}') study_area_geom = ee.Geometry.Rectangle( study_area_extent, proj='EPSG:4326', geodesic=False) if not ee.data.getInfo(tcorr_default_coll_id): logging.info('\nExport collection does not exist and will be built' '\n {}'.format(tcorr_default_coll_id)) input('Press ENTER to continue') ee.data.createAsset({'type': 'IMAGE_COLLECTION'}, tcorr_default_coll_id) # Get current asset list logging.debug('\nGetting GEE asset list') asset_list = utils.get_ee_assets(tcorr_default_coll_id) # if logging.getLogger().getEffectiveLevel() == logging.DEBUG: # pprint.pprint(asset_list[:10]) # Get current running tasks tasks = utils.get_ee_tasks() if logging.getLogger().getEffectiveLevel() == logging.DEBUG: logging.debug(' Tasks: {}\n'.format(len(tasks))) input('ENTER') # Get the list of WRS2 tiles that intersect the data area and study area wrs2_coll = ee.FeatureCollection(wrs2_coll_id)\ .filterBounds(export_geom)\ .filterBounds(study_area_geom) if wrs2_tiles: wrs2_coll = wrs2_coll.filter(ee.Filter.inList(wrs2_tile_field, wrs2_tiles)) wrs2_info = wrs2_coll.getInfo()['features'] for wrs2_ftr in sorted(wrs2_info, key=lambda k: k['properties']['WRS2_TILE'], reverse=reverse_flag): wrs2_tile = wrs2_ftr['properties'][wrs2_tile_field] logging.info('{}'.format(wrs2_tile)) wrs2_path = int(wrs2_tile[1:4]) wrs2_row = int(wrs2_tile[5:8]) # wrs2_path = wrs2_ftr['properties'][wrs2_path_field] # wrs2_row = wrs2_ftr['properties'][wrs2_row_field] export_id = export_id_fmt.format( product=tmax_name.lower(), wrs2=wrs2_tile) logging.debug(' Export ID: {}'.format(export_id)) asset_id = asset_id_fmt.format( coll_id=tcorr_default_coll_id, wrs2=wrs2_tile) logging.debug(' Asset ID: {}'.format(asset_id)) if overwrite_flag: if export_id in tasks.keys(): logging.debug(' Task already submitted, cancelling') ee.data.cancelTask(tasks[export_id]['id']) # This is intentionally not an "elif" so that a task can be # cancelled and an existing image/file/asset can be removed if asset_id in asset_list: logging.debug(' Asset already exists, removing') ee.data.deleteAsset(asset_id) else: if export_id in tasks.keys(): logging.debug(' Task already submitted, exiting') continue elif asset_id in asset_list: logging.debug(' Asset already exists, skipping') continue # Clip the mask image to the Landsat footprint mask_img = tmax_mask.add(1).clip(ee.Geometry(wrs2_ftr['geometry'])) # Apply the default Tcorr value and then clear the transparency mask output_img = mask_img.multiply(0.978).rename(['tcorr'])\ .updateMask(mask_img.unmask(0)) # # Clip to the Landsat image footprint # output_img = tmax_mask.add(tcorr_default) \ # .rename(['tcorr']) \ # .clip(ee.Geometry(wrs2_ftr['geometry'])) # # Clear the transparency mask # output_img = output_img.updateMask(output_img.unmask(0)) output_img = output_img.set({ 'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'), 'model_name': model_name, 'model_version': ssebop.__version__, # 'system:time_start': utils.millis(start_dt), 'tcorr_value': tcorr_default, 'tcorr_index': 3, 'tmax_source': tmax_source.upper(), 'tmax_version': tmax_version.upper(), 'wrs2_path': wrs2_path, 'wrs2_row': wrs2_row, 'wrs2_tile': wrs2_tile, }) # pprint.pprint(output_img.getInfo()) # input('ENTER') logging.debug(' Building export task') task = ee.batch.Export.image.toAsset( image=output_img, description=export_id, assetId=asset_id, crs=export_crs, crsTransform='[' + ','.join(list(map(str, export_geo))) + ']', dimensions='{0}x{1}'.format(*export_shape), ) logging.info(' Starting export task') utils.ee_task_start(task) # Pause before starting the next export task utils.delay_task(delay_time, max_ready) logging.debug('')
target = aligned_data_file mask = aligned_mask_file processed_data_file = os.path.join(data_dir,dataset_name+'.tif') util_files.mask_geotiff(target, mask, processed_data_file, nodata=-128) ''' Upload processed data to Google Earth Engine ''' # set up Google Cloud Storage project and bucket objects gcs_client = storage.Client(os.environ.get("CLOUDSDK_CORE_PROJECT")) gcs_bucket = gcs_client.bucket(os.environ.get("GEE_STAGING_BUCKET")) # initialize ee (Google Earth Engine Python API) for uploading to GEE auth = ee.ServiceAccountCredentials(os.getenv('GEE_SERVICE_ACCOUNT'), os.getenv('GOOGLE_APPLICATION_CREDENTIALS')) ee.Initialize(auth) logger.info('Uploading processed data to Google Cloud Storage.') gcs_uris = util_cloud.gcs_upload(processed_data_file, dataset_name, gcs_bucket=gcs_bucket) logger.info('Uploading processed data to Google Earth Engine.') # generate bands component of GEE upload manifest mf_bands = util_cloud.gee_manifest_bands(data_dict, dataset_name) # upload processed data file to GEE asset_name = f'projects/resource-watch-gee/{dataset_name}' manifest = util_cloud.gee_manifest_complete(asset_name, gcs_uris[0], mf_bands) logger.debug(manifest) task_id = util_cloud.gee_ingest(manifest, public=True) util_cloud.gcs_remove(gcs_uris, gcs_bucket=gcs_bucket)
import ee ee.Initialize() ROI = ee.Geometry.Rectangle([8.501, 10.470, 10.701, 8.287]) test = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR').filterBounds(ROI) # Scale Class class Scale(object): def __init__(self, scale=0.0001): self.scale = scale def f_scale(self, img): imgs_to_scale = img.select(['B', 'G', 'R', 'NIR', 'SWIR1', 'SWIR2']).multiply(self.scale).float() imgs_to_append = img.select(['pixel_qa']) imgs = imgs_to_scale.addBands(imgs_to_append) \ .copyProperties(source=img).set('system:time_start', img.get('system:time_start')) return imgs def map(self, imgcol): imgcol = imgcol.map(self.f_scale) return imgcol # PBC class class PBC(object): def __init__(self, imgcol=None, masks=None, scale=None, reducer=None): self._imgcol = imgcol self._id = imgcol.getInfo()['id'].split("/")[1:4:2] self._masks = masks
def ee_beamer_et(ini_path=None, overwrite_flag=False): """Earth Engine Beamer ET Image Download Args: ini_path (str): overwrite_flag (bool): if True, overwrite existing files Returns: None """ logging.info('\nEarth Engine Beamer ETg Image Download') # Read config file ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='SPATIAL') inputs.parse_section(ini, section='IMAGES') inputs.parse_section(ini, section='BEAMER') ini['IMAGES']['download_bands'] = [ 'etg_mean', 'etg_lci', 'etg_uci', 'etg_lpi', 'etg_upi' ] stat_list = ['median', 'mean'] nodata_value = -9999 zips_folder = 'zips' images_folder = 'images' annuals_folder = 'annuals' # Regular expression is only used to extract year from SCENE_ID landsat_re = re.compile('L[ETC]0[4578]_\d{3}XXX_(?P<YEAR>\d{4})\d{2}\d{2}') # if end_doy and end_doy > 273: # logging.error( # '\nERROR: End DOY has to be in the same water year as start DOY') # sys.exit() # Get ee features from shapefile zone_geom_list = gdc.shapefile_2_geom_list_func( ini['INPUTS']['zone_shp_path'], zone_field=ini['INPUTS']['zone_field'], reverse_flag=False) # zone_count = len(zone_geom_list) # output_fmt = '_{0:0%sd}.csv' % str(int(math.log10(zone_count)) + 1) # Check if the zone_names are unique # Eventually support merging common zone_names if len(set([z[1] for z in zone_geom_list])) != len(zone_geom_list): logging.error( '\nERROR: There appear to be duplicate zone ID/name values.' '\n Currently, the values in "{}" must be unique.' '\n Exiting.'.format(ini['INPUTS']['zone_field'])) return False # Filter features by FID if ini['INPUTS']['fid_keep_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] in ini['INPUTS']['fid_keep_list'] ] if ini['INPUTS']['fid_skip_list']: zone_geom_list = [ zone_obj for zone_obj in zone_geom_list if zone_obj[0] not in ini['INPUTS']['fid_skip_list'] ] # Merge geometries if ini['INPUTS']['merge_geom_flag']: merge_geom = ogr.Geometry(ogr.wkbMultiPolygon) for zone in zone_geom_list: zone_multipolygon = ogr.ForceToMultiPolygon( ogr.CreateGeometryFromJson(json.dumps(zone[2]))) for zone_polygon in zone_multipolygon: merge_geom.AddGeometry(zone_polygon) # merge_json = json.loads(merge_mp.ExportToJson()) zone_geom_list = [[ 0, ini['INPUTS']['zone_filename'], json.loads(merge_geom.ExportToJson()) ]] ini['INPUTS']['zone_field'] = '' # Set all zone specific parameters into a dictionary zone = {} # Need zone_shp_path projection to build EE geometries zone['osr'] = gdc.feature_path_osr(ini['INPUTS']['zone_shp_path']) zone['proj'] = gdc.osr_wkt(zone['osr']) # zone['proj'] = ee.Projection(zone['proj']).wkt().getInfo() # zone['proj'] = zone['proj'].replace('\n', '').replace(' ', '') # logging.debug(' Zone Projection: {}'.format(zone['proj'])) # Check that shapefile has matching spatial reference if not gdc.matching_spatref(zone['osr'], ini['SPATIAL']['osr']): logging.warning(' Zone OSR:\n{}\n'.format(zone['osr'])) logging.warning(' Output OSR:\n{}\n'.format( ini['SPATIAL']['osr'].ExportToWkt())) logging.warning(' Zone Proj4: {}'.format( zone['osr'].ExportToProj4())) logging.warning(' Output Proj4: {}'.format( ini['SPATIAL']['osr'].ExportToProj4())) logging.warning( '\nWARNING: \n' 'The output and zone spatial references do not appear to match\n' 'This will likely cause problems!') input('Press ENTER to continue') else: logging.debug(' Zone Projection:\n{}\n'.format( zone['osr'].ExportToWkt())) logging.debug(' Output Projection:\n{}\n'.format( ini['SPATIAL']['osr'].ExportToWkt())) logging.debug(' Output Cellsize: {}'.format( ini['SPATIAL']['cellsize'])) # Initialize Earth Engine API key logging.info('\nInitializing Earth Engine') ee.Initialize() utils.ee_request(ee.Number(1).getInfo()) # Get list of path/row strings to centroid coordinates if ini['INPUTS']['tile_keep_list']: ini['INPUTS']['tile_geom'] = [ wrs2.tile_centroids[tile] for tile in ini['INPUTS']['tile_keep_list'] if tile in wrs2.tile_centroids.keys() ] ini['INPUTS']['tile_geom'] = ee.Geometry.MultiPoint( ini['INPUTS']['tile_geom'], 'EPSG:4326') else: ini['INPUTS']['tile_geom'] = None # Read in ETo and PPT data from file if (ini['BEAMER']['eto_source'] == 'file' or ini['BEAMER']['ppt_source'] == 'file'): data_array = np.genfromtxt(ini['BEAMER']['data_path'], delimiter=',', names=True, dtype=None) data_fields = data_array.dtype.names logging.debug(' CSV fields: {}'.format(', '.join(data_fields))) # DEADBEEF - Compare fields names assuming all upper case data_fields = [f.upper() for f in data_fields] eto_dict = defaultdict(dict) ppt_dict = defaultdict(dict) for row in data_array: z = str(row[data_fields.index(ini['BEAMER']['data_zone_field'])]) y = int(row[data_fields.index(ini['BEAMER']['data_year_field'])]) if ini['BEAMER']['eto_source'] == 'file': # DEADBEEF - Compare fields names assuming all upper case eto_dict[z][y] = row[data_fields.index( ini['BEAMER']['data_eto_field'].upper())] if ini['BEAMER']['ppt_source'] == 'file': # DEADBEEF - Compare fields names assuming all upper case ppt_dict[z][y] = row[data_fields.index( ini['BEAMER']['data_ppt_field'].upper())] # Get filtered/merged/prepped Landsat collection landsat_args = { k: v for section in ['INPUTS'] for k, v in ini[section].items() if k in [ 'landsat4_flag', 'landsat5_flag', 'landsat7_flag', 'landsat8_flag', 'fmask_flag', 'acca_flag', 'start_year', 'end_year', 'start_month', 'end_month', 'start_doy', 'end_doy', 'scene_id_keep_list', 'scene_id_skip_list', 'path_keep_list', 'row_keep_list', 'tile_geom', 'adjust_method', 'mosaic_method', 'refl_sur_method' ] } landsat = ee_common.Landsat(landsat_args) # Download images for each feature separately for zone_fid, zone_name, zone_json in zone_geom_list: zone['fid'] = zone_fid zone['name'] = zone_name.replace(' ', '_') zone['json'] = zone_json logging.info('ZONE: {} (FID: {})'.format(zone['name'], zone['fid'])) # Build EE geometry object for zonal stats zone['geom'] = ee.Geometry(geo_json=zone['json'], opt_proj=zone['proj'], opt_geodesic=False) # logging.debug(' Centroid: {}'.format( # zone['geom'].centroid(100).getInfo()['coordinates'])) # Use feature geometry to build extent, transform, and shape zone['extent'] = gdc.Extent( ogr.CreateGeometryFromJson(json.dumps(zone['json'])).GetEnvelope()) # zone['extent'] = gdc.Extent(zone['geom'].GetEnvelope()) zone['extent'] = zone['extent'].ogrenv_swap() zone['extent'] = zone['extent'].adjust_to_snap( 'EXPAND', ini['SPATIAL']['snap_x'], ini['SPATIAL']['snap_y'], ini['SPATIAL']['cellsize']) zone['geo'] = zone['extent'].geo(ini['SPATIAL']['cellsize']) zone['transform'] = gdc.geo_2_ee_transform(zone['geo']) # zone['transform'] = '[' + ','.join(map(str, zone['transform'])) + ']' zone['shape'] = zone['extent'].shape(ini['SPATIAL']['cellsize']) logging.debug(' Zone Shape: {}'.format(zone['shape'])) logging.debug(' Zone Transform: {}'.format(zone['transform'])) logging.debug(' Zone Extent: {}'.format(zone['extent'])) # logging.debug(' Zone Geom: {}'.format(zone['geom'].getInfo())) # Assume all pixels in all 14+2 images could be reduced zone['max_pixels'] = zone['shape'][0] * zone['shape'][1] logging.debug(' Max Pixels: {}'.format(zone['max_pixels'])) # Set output spatial reference # Eventually allow user to manually set these # output_crs = zone['proj'] logging.debug(' Image Projection: {}'.format(ini['SPATIAL']['crs'])) # output_transform = zone['transform'][:] output_transform = '[' + ','.join(map(str, zone['transform'])) + ']' output_shape = '{1}x{0}'.format(*zone['shape']) logging.debug(' Image Transform: {}'.format(output_transform)) logging.debug(' Image Shape: {}'.format(output_shape)) zone_output_ws = os.path.join(ini['IMAGES']['output_ws'], zone_name) zone_zips_ws = os.path.join(zone_output_ws, zips_folder) zone_images_ws = os.path.join(zone_output_ws, images_folder) zone_annuals_ws = os.path.join(zone_output_ws, annuals_folder) if not os.path.isdir(zone_zips_ws): os.makedirs(zone_zips_ws) if not os.path.isdir(zone_images_ws): os.makedirs(zone_images_ws) if not os.path.isdir(zone_annuals_ws): os.makedirs(zone_annuals_ws) # Initialize the Landsat object # Limit Landsat products for getting SCENE IDs landsat.products = [] landsat.zone_geom = zone['geom'] landsat_coll = landsat.get_collection() # if ee.Image(landsat_coll.first()).getInfo() is None: # logging.info(' No images, skipping') # continue # Get the full list of scene IDs logging.debug(' Getting SCENE_ID list') scene_id_list = sorted( utils.ee_getinfo(landsat_coll.aggregate_histogram('SCENE_ID'))) logging.debug(' {} scenes'.format(len(scene_id_list))) # Switch Landsat products for computing ETg landsat.products = ['evi_sur'] # Process each image in the collection by date for image_id in scene_id_list: logging.info('{}'.format(image_id)) zip_path = os.path.join(zone_zips_ws, '{}.zip'.format(image_id)) logging.debug(' Zip: {}'.format(zip_path)) if os.path.isfile(zip_path) and overwrite_flag: logging.debug(' Output already exists, removing zip') os.remove(zip_path) elif os.path.isfile(zip_path) and not overwrite_flag: # Check that existing ZIP files can be opened try: with zipfile.ZipFile(zip_path, 'r') as z: pass logging.debug(' Output already exists, skipping') continue except Exception as e: logging.warning(' Zip file error, removing') os.remove(zip_path) # Getting the date directly from the SCENE_ID image_start_dt = datetime.datetime.strptime( image_id[12:], '%Y%m%d') image_end_dt = image_start_dt + datetime.timedelta(days=1) logging.debug(' {} {}'.format(image_start_dt.date(), image_end_dt.date())) year = image_start_dt.year # Filter the GRIDMET collection wy_start_date = '{}-10-01'.format(year - 1) wy_end_date = '{}-10-01'.format(year) logging.debug(' WY: {} {}'.format(wy_start_date, wy_end_date)) gridmet_coll = ee.ImageCollection('IDAHO_EPSCOR/GRIDMET') \ .filterDate(wy_start_date, wy_end_date) # # PRISM collection was uploaded as an asset # if ini['BEAMER']['ppt_source'] == 'prism': # def prism_time_start(input_image): # """Set time_start property on PRISM water year PPT collection""" # # Assume year is the 4th item separated by "_" # water_year = ee.String(input_image.get('system:index')).split('_').get(3) # date_start = ee.Date(ee.String(water_year).cat('-10-01')) # return input_image.select([0], ['ppt']).set({ # 'system:time_start': date_start.millis() # }) # prism_coll = ee.ImageCollection('users/cgmorton/prism_800m_ppt_wy') # prism_coll = prism_coll.map(prism_time_start) \ # .filterDate(wy_start_date, wy_end_date) # Get water year PPT from file # Convert all input data to mm to match GRIDMET data if ini['BEAMER']['ppt_source'] == 'file': wy_ppt_input = ppt_dict[zone_name][year] if ini['BEAMER']['data_ppt_units'] == 'mm': pass elif ini['BEAMER']['data_ppt_units'] == 'inches': wy_ppt_input *= 25.4 elif ini['BEAMER']['data_ppt_units'] == 'feet': wy_ppt_input *= (25.4 * 12) elif ini['BEAMER']['ppt_source'] == 'gridmet': # GET GRIDMET value at centroid of geometry wy_ppt_input = float( utils.ee_getinfo( ee.ImageCollection( gridmet_coll.select(['pr'], ['ppt']).sum()).getRegion( zone['geom'].centroid(1), 500))[1][4]) # Calculate GRIDMET zonal mean of geometry # wy_ppt_input = float(ee.ImageCollection( # gridmet_coll.select(['pr'], ['ppt'])).reduceRegion( # reducer=ee.Reducer.sum(), # geometry=zone['geom'], # crs=ini['SPATIAL']['crs'], # crsTransform=zone['transform'], # bestEffort=False, # tileScale=1).getInfo()['ppt'] # elif ini['BEAMER']['ppt_source'] == 'prism': # # Calculate PRISM zonal mean of geometry # wy_ppt_input = float(ee.ImageCollection( # prism_coll.map(ee_common.prism_ppt_func)).sum().reduceRegion( # reducer=ee.Reducer.mean(), # geometry=zone['geom'], # crs=ini['SPATIAL']['crs'], # crsTransform=zone['transform'], # bestEffort=False, # tileScale=1).getInfo()['ppt']) # Get water year ETo read from file # Convert all input data to mm for Beamer Method if ini['BEAMER']['eto_source'] == 'file': wy_eto_input = eto_dict[zone_name][year] if ini['BEAMER']['data_eto_units'] == 'mm': pass elif ini['BEAMER']['data_eto_units'] == 'inches': wy_eto_input *= 25.4 elif ini['BEAMER']['data_eto_units'] == 'feet': wy_eto_input *= (25.4 * 12) # This assumes GRIMET data is in millimeters elif ini['BEAMER']['eto_source'] == 'gridmet': wy_eto_input = float( utils.ee_getinfo( ee.ImageCollection(gridmet_coll.select( ['eto']).sum()).getRegion(zone['geom'].centroid(1), 500))[1][4]) # wy_eto_input = float(utils.ee_getinfo(ee.ImageCollection( # gridmet_coll.select(['eto'])).reduceRegion( # reducer=ee.Reducer.sum(), # geometry=zone['geom'], # crs=zone_proj, # crsTransform=zone_transform, # bestEffort=False, # tileScale=1).getInfo())) logging.debug(' Input ETO: {} mm PPT: {} mm'.format( wy_eto_input, wy_ppt_input)) # Scale ETo & PPT wy_eto_input *= ini['BEAMER']['eto_factor'] wy_ppt_input *= ini['BEAMER']['ppt_factor'] # Convert output units from mm wy_ppt_output = wy_ppt_input wy_eto_output = wy_eto_input if ini['IMAGES']['ppt_units'] == 'mm': pass elif ini['IMAGES']['ppt_units'] == 'in': wy_ppt_output /= 25.4 elif ini['IMAGES']['ppt_units'] == 'ft': wy_ppt_output /= (25.4 * 12) if ini['IMAGES']['eto_units'] == 'mm': pass elif ini['IMAGES']['eto_units'] == 'in': wy_eto_output /= 25.4 elif ini['IMAGES']['eto_units'] == 'ft': wy_eto_output /= (25.4 * 12) logging.debug(' Output ETO: {} {} PPT: {} {}'.format( wy_eto_output, ini['IMAGES']['eto_units'], wy_ppt_output, ini['IMAGES']['ppt_units'])) # Add water year ETo and PPT values to each image def eto_ppt_func(img): """""" return ee.Image(img).setMulti({ 'wy_eto': wy_eto_output, 'wy_ppt': wy_ppt_output }) # Compute EVI_SUR, add ETo and PPT, then Compute ETg # Set the masked values to a nodata value # so that the TIF can have a nodata value other than 0 set landsat_image = eto_ppt_func( landsat.get_image(landsat, image_start_dt.year, image_start_dt.strftime('%j'), path=image_id[5:8], row=None)) etg_image = ee.Image(ee_common.beamer_func(landsat_image)) \ .clip(zone['geom']) \ .unmask(nodata_value, False) # Get the download URL logging.debug(' Requesting URL') zip_url = utils.ee_request( etg_image.getDownloadURL({ 'name': image_id, 'crs': ini['SPATIAL']['crs'], 'crs_transform': output_transform, 'dimensions': output_shape })) del etg_image # Remove the scene from scen list if it's not going to work if not zip_url: scene_id_list.remove(image_id) continue # Try downloading a few times logging.info(' Downloading') for i in range(1, 10): try: response = urlrequest.urlopen(zip_url) with open(zip_path, 'wb') as output_f: shutil.copyfileobj(response, output_f) break except Exception as e: logging.info(' Resending query') logging.debug(' {}'.format(e)) sleep(i**2) os.remove(zip_path) # Remove the scene from scen list if it's not going to work if not os.path.isfile(zip_path): scene_id_list.remove(image_id) logging.info('\nExtracting images') for image_id in scene_id_list: logging.info('{}'.format(image_id)) zip_path = os.path.join(zone_zips_ws, '{}.zip'.format(image_id)) logging.debug(' Zip: {}'.format(zip_path)) if not os.path.isfile(zip_path): logging.debug(' zip file does not exist, skipping') # Skip if all output images are present image_band_list = [ os.path.join(zone_images_ws, '{}.{}.tif'.format(image_id, band)) for band in ini['IMAGES']['download_bands'] ] if (not overwrite_flag and all(os.path.isfile(x) for x in image_band_list)): logging.debug(' all images present, skipping') continue # Otherwise, remove existing images for image_path in image_band_list: for file_path in glob.glob(image_path.replace('.tif', '.*')): os.remove(file_path) # Try extracting the files try: logging.debug(' Extracting') with zipfile.ZipFile(zip_path, 'r') as z: z.extractall(zone_images_ws) except Exception as e: logging.warning(' Error: could not extract'.format(i)) logging.debug(' {}'.format(e)) try: os.remove(zip_path) except Exception as e: pass # Set nodata value for item in os.listdir(zone_images_ws): if item.startswith(image_id) and item.endswith('.tif'): gdc.raster_path_set_nodata( os.path.join(zone_images_ws, item), nodata_value) raster_statistics(os.path.join(zone_images_ws, item)) logging.info('\nComputing annual means') for band in ini['IMAGES']['download_bands']: logging.info(' {}'.format(band)) for year in range(ini['INPUTS']['start_year'], ini['INPUTS']['end_year'] + 1): logging.info(' {}'.format(year)) mean_path = os.path.join( # zone_annuals_ws, 'etg_{}_{}.{}.tif'.format( zone_annuals_ws, '{}_{}.{}.tif'.format(zone_name.lower().replace(' ', '_'), year, band)) logging.debug(' {}'.format(mean_path)) # if os.path.isfile(mean_path) and not overwrite_flag: # logging.debug(' file already exists, skipping') # continue image_band_list = [ os.path.join(zone_images_ws, item) for item in os.listdir(zone_images_ws) if (item.endswith('.{}.tif'.format(band)) and ( int(landsat_re.match(item).group('YEAR')) == year)) ] # for image_path in image_band_list: # raster_path_set_nodata(image_path, nodata_value) if not image_band_list: continue # Use GDAL to compute the composite cell_statistics(image_band_list, mean_path, 'mean') raster_statistics(mean_path) logging.info('\nComputing composite rasters from annual means') for stat in stat_list: logging.info(' Stat: {}'.format(stat)) for band in ini['IMAGES']['download_bands']: logging.info(' {}'.format(band)) image_band_list = [ os.path.join(zone_annuals_ws, item) for item in os.listdir(zone_annuals_ws) if item.endswith('.{}.tif'.format(band)) ] # for image_path in image_band_list: # raster_path_set_nodata(image_path, nodata_value) output_path = os.path.join( zone_output_ws, '{}_{}.{}.tif'.format(zone_name.lower().replace(' ', '_'), stat.lower(), band.lower())) logging.debug(' {}'.format(output_path)) # Use GDAL to compute the composite raster cell_statistics(image_band_list, output_path, 'mean') raster_statistics(output_path)