from pydrive.drive import GoogleDrive from pydrive.auth import GoogleAuth from ee import batch BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) latitude_val = float(sys.argv[1]) longitude_val = float(sys.argv[2]) manipulated_lat = latitude_val + 0.2 manipulated_lng = longitude_val + 0.2 ee.Initialize() # Define the image collection we will be using collection = ee.ImageCollection('LANDSAT/LC08/C01/T1_TOA') # Creates a polygon which will be used as the bounds of the image # Flagstaff Long: -111.6512, Lat: 35.1982 # region_polygon = ee.Geometry.Polygon( # [[[-111.791491098128,35.150445194872816], # [-111.49829346629207,35.150445194872816], # [-111.49829346629207,35.385344417919924], # [-111.791491098128,35.385344417919924], # [-111.791491098128,35.150445194872816]]]) region_polygon = ee.Geometry.Polygon([[[longitude_val, latitude_val], [manipulated_lng, latitude_val], [manipulated_lng, manipulated_lat], [longitude_val, manipulated_lat], [longitude_val, latitude_val]]])
# GitHub URL: https://github.com/giswqs/qgis-earthengine-examples/tree/master/ImageCollection/sort_by_cloud_and_date.py import ee from ee_plugin import Map # This function masks the input with a threshold on the simple cloud score. def cloudMask(img): cloudscore = ee.Algorithms.Landsat.simpleCloudScore(img).select('cloud') return img.updateMask(cloudscore.lt(50)) # Load a Landsat 5 image collection. collection = ee.ImageCollection('LANDSAT/LT5_L1T_TOA') \ .filterDate('2008-04-01', '2010-04-01') \ .filterBounds(ee.Geometry.Point(-122.2627, 37.8735)) \ .map(cloudMask) \ .select(['B4', 'B3']) \ .sort('system:time_start', True) #Sort the collection in chronological order. print(collection.size().getInfo()) first = collection.first().get('system:id') print(first.getInfo())
Map = emap.Map(center=[40,-100], zoom=4) Map.add_basemap('ROADMAP') # Add Google Map Map # %% """ ## Add Earth Engine Python script """ # %% # Add Earth Engine dataset roi = ee.Geometry.Point([-99.2182, 46.7824]) collection = ee.ImageCollection('USDA/NAIP/DOQQ') \ .filterBounds(roi) \ .filter(ee.Filter.listContains("system:band_names", "N")) print(collection.size().getInfo()) first = collection.first() Map.centerObject(first, 13) Map.addLayer(first, {'bands': ['N', 'R', 'G']}, 'NAIP') # %% """ ## Display Earth Engine data layers """ # %% Map.addLayerControl() # This line is not needed for ipyleaflet-based Map. Map
months_min = month - MONTHS_OFFSET if months_min < 1: months_min = 1 months_max = month + MONTHS_OFFSET if months_max > 12: months_max = 12 # -------------------------------------------------- # IMPORT ImageCollections # -------------------------------------------------- imgCol_L5_SR = ee.ImageCollection('LANDSAT/LT05/C01/T1_SR') \ .filterBounds(ROI) \ .filter(ee.Filter.calendarRange(year_min, year_max, 'year')) \ .filter(ee.Filter.calendarRange(months_min, months_max, 'month')) \ .filter(ee.Filter.lt('CLOUD_COVER_LAND', CLOUD_COVER)) \ .map(prepro.rename_bands_l5) \ .map(prepro.mask_landsat_sr(bits)) \ .map(prepro.scale_img(0.0001, ['B', 'G', 'R', 'NIR', 'SWIR1', 'SWIR2'], ['TIR'])) \ .map(prepro.scale_img(0.1, ['TIR'], ['B', 'G', 'R', 'NIR', 'SWIR1', 'SWIR2'])) imgCol_L7_SR = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR') \ .filterBounds(ROI) \ .filter(ee.Filter.calendarRange(year_min, year_max, 'year')) \ .filter(ee.Filter.calendarRange(months_min, months_max, 'month')) \ .filter(ee.Filter.lt('CLOUD_COVER_LAND', CLOUD_COVER)) \ .map(prepro.rename_bands_l7) \ .map(prepro.mask_landsat_sr(bits)) \ .map(prepro.scale_img(0.0001, ['B', 'G', 'R', 'NIR', 'SWIR1', 'SWIR2'], ['TIR'])) \ .map(prepro.scale_img(0.1, ['TIR'], ['B', 'G', 'R', 'NIR', 'SWIR1', 'SWIR2']))
def whittakerSmoothing(imageCollection, isCompositional=False, lamb=5): # quick configs to set defaults def toFl(image): return image.toFloat() # procedure start ic = imageCollection.map(toFl) dimension = ic.size() identity_mat = ee.Array.identity(dimension) difference_mat = getDifferenceMatrix(identity_mat, 3) difference_mat_transpose = difference_mat.transpose() lamda_difference_mat = difference_mat_transpose.multiply(lamb) res_mat = lamda_difference_mat.matrixMultiply(difference_mat) hat_matrix = res_mat.add(identity_mat) # backing up original data original = ic def getProperties(image): return ee.Image(image).toDictionary() # get original image properties properties = ic.toList(10000).map(getProperties) # if data is compositional # calculate the logratio of an image between 0 and 100. First # clamps between delta and 100-delta, where delta is a small positive value. if (isCompositional): def clampImage(image): delta = 0.001 bands = image.bandNames() image = image.clamp(delta, 100 - delta) image = (ee.Image.constant(100).subtract(image) ).divide(image).log().rename(bands) return image ic = ic.map(clampImage) arrayImage = original.toArray() coeffimage = ee.Image(hat_matrix).updateMask(arrayImage.mask()) smoothImage = coeffimage.matrixSolve( arrayImage.unmask(hat_matrix.multiply(0))) def getImageId(image): return ee.Image(image).id() idlist = ic.toList(10000).map(getImageId) bandlist = ee.Image(ic.first()).bandNames() flatImage = smoothImage.arrayFlatten([idlist, bandlist]) smoothCollection = ee.ImageCollection(unpack(flatImage, idlist, bandlist)) if (isCompositional): smoothCollection = smoothCollection.map(inverseLogRatio) def addSuffix(band): return ee.String(band).cat('_fitted') # get new band names by adding suffix fitted newBandNames = bandlist.map(addSuffix) # rename the bands in smoothened images smoothCollection = smoothCollection.select(bandlist, newBandNames) # a really dumb way to loose the google earth engine generated ID so that the two # images can be combined for the chart dumbimg = arrayImage.arrayFlatten([idlist, bandlist]) dumbcoll = ee.ImageCollection(unpack(dumbimg, idlist, bandlist)) outCollection = dumbcoll.combine(smoothCollection) outCollList = outCollection.toList(10000) def addPropBack(image): return ee.Image(image).set(properties.get(outCollList.indexOf(image))) outCollectionProp = outCollList.map(addPropBack) residue_sq = smoothImage.subtract(arrayImage).pow( ee.Image(2)).divide(dimension) rmse_array = residue_sq.arrayReduce(ee.Reducer.sum(), [0]).pow(ee.Image(1 / 2)) rmseImage = rmse_array.arrayFlatten([["rmse"], bandlist]) return (ee.ImageCollection(outCollectionProp), rmseImage)
def vegET_model(daily_imageColl, bbox): """ Calculate Daily Soil Water Index (SWI) :param start_date: ee.Date First date for analysis. Used to calculate initial SWI and then removed from collection. :param daily_imageColl: ee.ImageCollection Collection of daily images with bands for ndvi, precip, pet, canopy intercept :param whc_grid_img: ee.Image Static water holding capacity image :return: ee.ImageCollection imageCollection of daily Soil Water Index """ # TODO: Verify if these should be user inputs # Define constant variables VARA = ee.Number(1.25) VARB = ee.Number(0.2) # Define whc_grid whc_grid_img = ee.Image(daily_imageColl.first().select('whc')) # Drainage Coefficient (ee.Image to add as band for .expression()) dc_coeff = ee.Image(0.65) # rf coefficient (ee.Image to add as band for .expression()) rf_coeff = ee.Image(1.0).subtract(dc_coeff) # Calculate initial values where necessary init_effppt = eff_intercept_precip(daily_imageColl.first()) initial_images = init_image_create(daily_imageColl, whc_grid_img, init_effppt.select('effppt')) # Create list for dynamic variables to be used in .iterate() outputs_list = ee.List([initial_images]) # Create constant image for calculations in daily_vegET_calc() const_img = ee.Image(1.0) # TODO: Update docstring def daily_vegET_calc(daily_img, outputs_list): """ Function to run imageCollection.iterate(). Takes latest value from outputs_list as previous time-step SWI, current day whc_image and daily_image :param swi_list: :param whc_image: :param daily_image: :return: """ # Outputs from previous day as inputs to current day. # NOTE: needs to be cast to list then image. see: https://developers.google.com/earth-engine/ic_iterating prev_outputs = ee.Image(ee.List(outputs_list).get(-1)) # Calculate rain_frac rain_frac = rain_frac_calc(daily_img, bbox) # Calculate effective precipitation and intercepted precip effective_precip = eff_intercept_precip(daily_img) # Calculate amount of effective precipitation as rain rain = rain_frac.multiply( effective_precip.select('effppt')).rename('rain') # TODO: Double check this is ok. Essentially skips first run as is in the esri version # TODO: This will be combined with snowpack and swf at end of run to make new outputs_list append swe = ee.Image(const_img.subtract(rain_frac)).multiply( effective_precip.select('effppt')).rename('swe') def melt_rate_calc(image): """ Calculate melt_rate :param image: ee.Image Image with max and min temp bands :return: ee.Image """ melt_rate = image.expression( '0.06 * ((tmax * tmax) - (tmax * tmin))', { 'tmax': image.select('tmaxC'), 'tmin': image.select('tminC') }) melt_rate = melt_rate.set( 'system:time_start', image.get('system:time_start')).rename('melt_rate') return melt_rate melt_rate = melt_rate_calc(daily_img) # TODO: Update docstring def snow_melt_calc(melt_rate_img, swe_current, prev_sw_image, geometry): """ Calculate snow melt :param melt_rate_img: ee.Image Melt rate image :param swe_current: ee.Image Current time-step calculation of swe :param prev_sw_image: ee.Image Snowpack band from previous timestep list :return: ee.Image """ # Combine images to allow for band selection in expression snow_melt_img = melt_rate_img.addBands([ swe_current, prev_sw_image.select('snowpack') ]).rename(['melt_rate', 'swe', 'snowpack']) snow_melt = snow_melt_img.expression( "(b('melt_rate') <= (b('swe') + b('snowpack'))) ? (b('melt_rate'))" + ": (b('swe') + b('snowpack'))").clip(geometry).rename( ['snowmelt']) return snow_melt snow_melt = snow_melt_calc(melt_rate, swe, prev_outputs.select('snowpack'), bbox) def snowpack_calc(prev_sw_image, swe_current, snow_melt): """ Calculate snowpack :param prev_sw_image: ee.Image image with band for snowpack at previous timestep :param swe_current: ee.Image current swe :param snow_melt: ee.Image current snowmelt :return: ee.Image """ snwpk1 = prev_sw_image.add(swe_current).subtract(snow_melt) snowpack = snwpk1.where(snwpk1.lt(0.0), 0.0) return snowpack snowpack = snowpack_calc(prev_outputs.select('snowpack'), swe, snow_melt) swi_current = ee.Image( prev_outputs.select('swf').add(rain).add(snow_melt)).rename('swi') sat_fc = daily_img.select('soil_sat').subtract( daily_img.select('fcap')).rename('sat_fc') rf1 = swi_current.subtract(whc_grid_img).rename('rf1') rf = rf1.where(rf1.lt(0.0), 0.0).rename('rf') def srf_calc(rf_img, sat_fc_img, rf_coeff, geometry): """ Calculate srf :param rf_img: ee.Image image for rf :param sat_fc_img: ee.Image sat_fc_image :param rf_coeff: ee.Image rf coefficient :return: ee.Image """ # add bands to make single image for .expression srf_input_img = rf_img.addBands([sat_fc_img, rf_coeff]).rename( ['rf', 'sat_fc', 'rf_coeff']) srf = srf_input_img.expression( "(b('rf') <= b('sat_fc')) ? (b('rf') * b('rf_coeff'))" + ": (b('rf') - b('sat_fc')) + b('rf_coeff') * b('sat_fc')" ).clip(geometry).rename('srf') return srf srf = srf_calc(rf, sat_fc, rf_coeff, bbox) # Deep drainage ddrain = rf.subtract(srf).double().rename('ddrain') # TODO: Verify if this is still needed # def rfi_calc(image1, image2): # """ # Calculate runoff as swi - whc # :param image1: ee.Image # Soil Water Index image # :param image2: ee.Image # Water holding capacity image # :return: ee.Image # Runoff as only positive valued image # """ # # rf = image1.subtract(image2) # # # Set value to 0 if rf < 0 # rfi = rf.where(rf.lt(0), 0) # # rfi = rfi.set('system:time_start', image1.get('system:time_start')) # # return ee.Image(rfi) # # rfi = rfi_calc(swi_current, whc_grid_img) etasw1A = ee.Image( daily_img.select('ndvi').multiply(VARA).add(VARB)).multiply( daily_img.select('eto')).rename('etasw1A') etasw1B = ee.Image( daily_img.select('ndvi').multiply(VARA).multiply( daily_img.select('eto'))).rename('etasw1B') # TODO: consider ee.Algorithms.If() for conditional statements # If ndvi is > 0.4, return etasw1A pixel value, otherwise, return etasw1B etasw1 = etasw1B.where(daily_img.select('ndvi').gt(0.4), etasw1A).rename('etasw1') etasw2 = etasw1.multiply(swi_current.divide( whc_grid_img.multiply(0.5))).rename('etasw2') # use etasw1 pixel when swi_current > whc*0.5, use etasw2 pixel otherwise etasw3 = etasw2.where(swi_current.gt(whc_grid_img.multiply(0.5)), etasw1).rename('etasw3') # use swi_current pixel where etasw3 > swi_current, use etasw3 pixel otherwise etasw4 = etasw3.where(etasw3.gt(swi_current), swi_current).rename('etasw4') # use whc pixel where etasw4 is > whc, use etasw4 otherwise etasw = etasw4.where(etasw4.gt(whc_grid_img), whc_grid_img).rename('etasw') swf1 = swi_current.subtract(etasw).rename('swf1') bigswi = ee.Image(whc_grid_img.subtract(etasw)).rename('bigswi') swf1_thresh = swf1.where(swf1.lt(0.0), 0.0).rename('swf_thresh') swf = swf1_thresh.where(swi_current.gt(whc_grid_img), bigswi).rename('swf') # # TODO: This has been replaced by geetools addMultiBands # def output_image_create(ref_img, swf_img, swe_img, snowpack_img): # """ # Combine images to create single output image with multiple bands # :param daily_img: ee.Image # reference image for timestamp # :param swf_img: ee.Image # :param swe_img: ee.Image # :param snowpack_img: ee.Image # :return: ee.Image # """ # # output_img = swf_img.addBands([swe_img, snowpack_img]).rename(['swi', 'swe', 'snowpack']) \ # .set({ # 'system:index': ref_img.get('system:index'), # 'system:time_start': ref_img.get('system:time_start') # }) # # return ee.Image(output_img) # # # Create output image # output_image = output_image_create(daily_img, swf, swe, snowpack) results = ee.Image( utils.addMultiBands(daily_img, [ rain_frac, effective_precip, rain, swe, melt_rate, snow_melt, snowpack, swi_current, sat_fc, rf1, rf, srf, ddrain, etasw1A, etasw1B, etasw1, etasw2, etasw3, etasw4, etasw, swf1, bigswi, swf1_thresh, swf ])) return ee.List(outputs_list).add(results) return ee.ImageCollection( ee.List(daily_imageColl.iterate(daily_vegET_calc, outputs_list)))
import ee import ee.mapclient ee.Initialize() collection = ee.ImageCollection('USDA/NAIP/DOQQ') polys = ee.Geometry.Polygon([[[-99.29615020751953, 46.725459351792374], [-99.2116928100586, 46.72404725733022], [-99.21443939208984, 46.772037733479884], [-99.30267333984375, 46.77321343419932]]]) centroid = polys.centroid() lng, lat = centroid.getInfo()['coordinates'] print("lng = {}, lat = {}".format(lng, lat)) lng_lat = ee.Geometry.Point(lng, lat) naip = collection.filterBounds(polys) naip_2015 = naip.filterDate('2015-01-01', '2015-12-31') ppr = naip_2015.mosaic() count = naip_2015.size().getInfo() print("Count: ", count) # print(naip_2015.size().getInfo()) # vis = {'bands': ['N', 'R', 'G']} # ee.mapclient.centerMap(lng, lat, 12) # ee.mapclient.addToMap(ppr,vis) # ee.mapclient.addToMap(polys) downConfig = {
def getS2collection(): s2 = ee.ImageCollection('COPERNICUS/S2') \ .filterBounds(poly) \ .filterDate(ee.Date(w_startdate.value),ee.Date(w_enddate.value)) \ .sort('CLOUDY_PIXEL_PERCENTAGE',True) return s2.filter(ee.Filter.contains(rightValue=poly, leftField='.geo'))
def on_collect_button_clicked(b): global result,collection,count,imList,poly,timestamplist1,timestamps2, \ s2_image,rons,mean_incidence,collectionmosaic,collectionfirst,archive_crs,coords,wc with w_out: try: if (w_collection.value == 'COPERNICUS/S1_GRD') or (w_collection.value == ''): w_out.clear_output() print( 'running on GEE archive COPERNICUS/S1_GRD (please wait for raster overlay) ...' ) # coords = ee.List(poly.bounds().coordinates().get(0)) collection = getS1collection() if w_relativeorbitnumber.value > 0: collection = collection.filter( ee.Filter.eq('relativeOrbitNumber_start', int(w_relativeorbitnumber.value))) if w_platform.value != 'Both': collection = collection.filter( ee.Filter.eq('platform_number', w_platform.value)) collection = collection.sort('system:time_start') acquisition_times = ee.List( collection.aggregate_array('system:time_start')).getInfo() count = len(acquisition_times) if count < 2: raise ValueError('Less than 2 images found') timestamplist = [] for timestamp in acquisition_times: tmp = time.gmtime(int(timestamp) / 1000) timestamplist.append(time.strftime('%x', tmp)) # make timestamps in YYYYMMDD format timestamplist = [x.replace('/', '') for x in timestamplist] timestamplist = ['T20' + x[4:] + x[0:4] for x in timestamplist] timestamplist = timestamplist[::int(w_stride.value)] # in case of duplicates add running integer timestamplist1 = [ timestamplist[i] + '_' + str(i + 1) for i in range(len(timestamplist)) ] count = len(timestamplist) if count < 2: raise ValueError( 'Less than 2 images found, decrease stride') relativeorbitnumbers = map( int, ee.List( collection.aggregate_array( 'relativeOrbitNumber_start')).getInfo()) rons = list(set(relativeorbitnumbers)) print('Images found: %i, platform: %s' % (count, w_platform.value)) print('Number of 10m pixels contained: %i' % math.floor(poly.area().getInfo() / 100.0)) print('Acquisition dates: %s to %s' % (str(timestamplist[0]), str(timestamplist[-1]))) print('Relative orbit numbers: ' + str(rons)) if len(rons) == 1: mean_incidence = get_incidence_angle(collection.first()) print('Mean incidence angle: %f' % mean_incidence) else: mean_incidence = 'undefined' print('Mean incidence angle: (select one rel. orbit)') pcollection = collection.map(get_vvvh) collectionfirst = ee.Image(pcollection.first()) w_exportscale.value = collectionfirst.projection( ).nominalScale().getInfo() pList = pcollection.toList(500) first = ee.Dictionary({ 'imlist': ee.List([]), 'poly': poly, 'enl': ee.Number(w_enl.value), 'ctr': ee.Number(0), 'stride': ee.Number(int(w_stride.value)) }) imList = ee.List( ee.Dictionary(pList.iterate(clipList, first)).get('imlist')) # get a vorschau as collection mean collectionmosaic = collection.mosaic().select(0, 1).rename( 'b0', 'b1') percentiles = collectionmosaic.reduceRegion( ee.Reducer.percentile([2, 98]), geometry=poly, scale=w_exportscale.value, maxPixels=10e9) mn = ee.Number(percentiles.get('b0_p2')) mx = ee.Number(percentiles.get('b0_p98')) vorschau = collectionmosaic.select(0).visualize( min=mn, max=mx, opacity=w_opacity.value) else: w_out.clear_output() collection = ee.ImageCollection(w_collection.value) print( 'running on local collection %s \n ignoring start and end dates (please wait for raster overlay) ...' % w_collection.value) count = collection.size().getInfo() print('Images found: %i' % count) collectionfirst = ee.Image(collection.first()) poly = collectionfirst.geometry() # coords = ee.List(poly.bounds().coordinates().get(0)) center = poly.centroid().coordinates().getInfo() center.reverse() m.center = center w_exportscale.value = collectionfirst.projection( ).nominalScale().getInfo() if collectionfirst.get( 'system:time_start').getInfo() is not None: acquisition_times = ee.List( collection.aggregate_array( 'system:time_start')).getInfo() timestamplist1 = [] for timestamp in acquisition_times: tmp = time.gmtime(int(timestamp) / 1000) timestamplist1.append(time.strftime('%x', tmp)) timestamplist1 = [ x.replace('/', '') for x in timestamplist1 ] timestamplist1 = [ 'T20' + x[4:] + x[0:4] for x in timestamplist1 ] print('Acquisition dates: %s' % str(timestamplist1)) else: timestamplist1 = ['T%i' % (i + 1) for i in range(count)] print('No time property available: acquisitions: %s' % str(timestamplist1)) # get a vorschau from collection mean collectionmosaic = collection.mosaic().clip(poly) percentiles = collectionmosaic.select(0).rename( 'b0').reduceRegion(ee.Reducer.percentile([2, 98]), scale=w_exportscale.value, maxPixels=10e9) mn = ee.Number(percentiles.get('b0_p2')) mx = ee.Number(percentiles.get('b0_p98')) vorschau = collectionmosaic.select(0).visualize( min=mn, max=mx, opacity=w_opacity.value) imList = collection.toList(100) # get GEE S1 archive crs for eventual image series export # archive_crs = ee.Image(getS1collection(coords).first()).select(0).projection().crs().getInfo() archive_crs = ee.Image(getS1collection().first()).select( 0).projection().crs().getInfo() # run the algorithm result = omnibus(imList, w_significance.value, w_enl.value, w_median.value) w_preview.disabled = False w_ENL.disabled = False w_export_atsf.disabled = True s2_image = None # display collection or S2 if len(m.layers) > 3: m.remove_layer(m.layers[3]) if w_S2.value: # display sentinel-2 if available collection2 = getS2collection() count1 = collection2.size().getInfo() if count1 > 0: s2_image = ee.Image(collection2.first()).select( ['B2', 'B3', 'B4']) percentiles = s2_image.reduceRegion( ee.Reducer.percentile([2, 98]), scale=w_exportscale.value, maxPixels=10e9) mn = percentiles.values(['B2_p2', 'B3_p2', 'B4_p2']) mx = percentiles.values(['B2_p98', 'B3_p98', 'B4_p98']) vorschau = s2_image.visualize(min=mn, max=mx, opacity=w_opacity.value) timestamp = s2_image.get('system:time_start').getInfo() timestamp = time.gmtime(int(timestamp) / 1000) timestamp = time.strftime('%x', timestamp).replace('/', '') timestamps2 = '20' + timestamp[4:] + timestamp[0:4] print('Sentinel-2 from %s' % timestamps2) w_export_s2.disabled = False m.add_layer(TileLayer(url=GetTileLayerUrl(vorschau))) except Exception as e: print('Error: %s' % e)
score = score.min(rescale(img, 'img.red + img.green + img.blue', [0.2, 0.8])) # Clouds are reasonably bright in all infrared bands. score = score.min( rescale(img, 'img.nir + img.swir1 + img.swir2', [0.3, 0.8])) # Clouds are reasonably cool in temperature. score = score.min(rescale(img, 'img.temp', [300, 290])) # However, clouds are not snow. ndsi = img.normalizedDifference(['green', 'swir1']) return score.min(rescale(ndsi, 'img', [0.8, 0.6])) # Filter the TOA collection to a time-range and add the cloudscore band. collection = ee.ImageCollection('LANDSAT/LC08/C01/T1_TOA') \ .filterDate('2017-05-01', '2017-07-01') def func_wwa(img): # Invert the cloudscore so 1 is least cloudy, and rename the band. score = cloudScore(img.select(LC8_BANDS, STD_NAMES)) score = ee.Image(1).subtract(score).select([0], ['cloudscore']) return img.addBands(score) \ .map(func_wwa) # Define visualization parameters for a True color image.
Map = emap.Map(center=[40, -100], zoom=4) Map.add_basemap('ROADMAP') # Add Google Map Map # %% """ ## Add Earth Engine Python script """ # %% # Add Earth Engine dataset roi = ee.FeatureCollection('TIGER/2018/States') \ .filter(ee.Filter.eq('STUSPS', 'ND')) dataset = ee.ImageCollection('USDA/NAIP/DOQQ') \ .filter(ee.Filter.date('2016-01-01', '2017-12-31'))\ .filterBounds(roi) TrueColor = dataset.select(['N', 'R', 'G']).mosaic() TrueColorVis = { 'min': 0.0, 'max': 255.0, } # Map.setCenter(-73.9958, 40.7278, 15) Map.centerObject(roi, 8) Map.addLayer(TrueColor, TrueColorVis, 'True Color') # %% """ ## Display Earth Engine data layers """
# -*- coding: utf-8 -*- """ Created on Wed Jun 3 16:03:52 2020 @author: MEvans """ import ee service_account = '*****@*****.**' key = 'C:/Users/mevans/Downloads/acd-app-04a10fe38611.json' credentials = ee.ServiceAccountCredentials(service_account, key) ee.Initialize(credentials) geometry = ee.Geometry.Point([-111.54765625, 35.4227332452855]) imageCollection = ee.ImageCollection("COPERNICUS/S2") filtered = imageCollection.filterBounds(geometry) test = ee.Image(filtered.first()) print(test.bandNames().getInfo())
[-149.4140625, 58.99531118795094], [-145.810546875, 59.977005492196], [-140.7568359375, 59.153403092050375], [-140.90746584324359, 65.67611691802753], [-140.888671875, 70.1851027549897], [-146.162109375, 70.55417853776078], [-154.248046875, 71.49703690095419], [-160.3125, 71.24435551310674], [-167.255859375, 68.97416358340674]]]) # geometries end ------------------------------------------------------------------------------------------------- all_samp = ee.FeatureCollection( "users/masseyr44/shapefiles/all_samp_postbin_v8") ls5 = ee.ImageCollection("LANDSAT/LT05/C01/T1_SR") ls7 = ee.ImageCollection("LANDSAT/LE07/C01/T1_SR") ls8 = ee.ImageCollection("LANDSAT/LC08/C01/T1_SR") elevation = ee.Image('USGS/GMTED2010') # collections end ---------------------------------------------------------------------------------------- elev_scale_factor = 10000 pctl = 50 index = 'NDVI' unmask_val = -9999 startJulian1 = 90 endJulian1 = 165 startJulian2 = 180
This step creates an interactive map using [folium](https://github.com/python-visualization/folium). The default basemap is the OpenStreetMap. Additional basemaps can be added using the `Map.setOptions()` function. The optional basemaps can be `ROADMAP`, `SATELLITE`, `HYBRID`, `TERRAIN`, or `ESRI`. ''' # %% Map = folium.Map(location=[40, -100], zoom_start=4) Map.setOptions('HYBRID') # %% ''' ## Add Earth Engine Python script ''' # %% dataset = ee.ImageCollection('NRCan/CDEM') elevation = dataset.select('elevation').mosaic() elevationVis = { 'min': -50.0, 'max': 1500.0, 'palette': ['0905ff', 'ffefc4', 'ffffff'], } Map.setCenter(-139.3643, 63.3213, 9) Map.addLayer(elevation, elevationVis, 'Elevation') # %% ''' ## Display Earth Engine data layers '''
def p_restrend(year_start, year_end, ndvi_1yr, climate_1yr, logger): logger.debug("Entering p_restrend function.") def f_img_coll(ndvi_stack): img_coll = ee.List([]) for k in range(year_start, year_end + 1): ndvi_img = ndvi_stack.select('y{}'.format(k))\ .addBands(climate_1yr.select('y{}'.format(k)))\ .rename(['ndvi', 'clim']).set({'year': k}) img_coll = img_coll.add(ndvi_img) return ee.ImageCollection(img_coll) ## Function to predict NDVI from climate first = ee.List([]) def f_ndvi_clim_p(image, list): ndvi = lf_clim_ndvi.select('offset').add( (lf_clim_ndvi.select('scale').multiply(image))).set( {'year': image.get('year')}) return ee.List(list).add(ndvi) ## Function to compute residuals (ndvi obs - ndvi pred) def f_ndvi_clim_r_img(year): ndvi_o = ndvi_1yr_coll.filter(ee.Filter.eq( 'year', year)).select('ndvi').median() ndvi_p = ndvi_1yr_p.filter(ee.Filter.eq('year', year)).median() ndvi_r = ee.Image(year).float().addBands(ndvi_o.subtract(ndvi_p)) return ndvi_r.rename(['year', 'ndvi_res']) # Function to compute differences between observed and predicted NDVI and compilation in an image collection def stack(year_start, year_end): img_coll = ee.List([]) for k in range(year_start, year_end + 1): ndvi = ndvi_1yr_o.filter(ee.Filter.eq('year', k)).select('ndvi').median() clim = clim_1yr_o.filter(ee.Filter.eq('year', k)).select('ndvi').median() img = ndvi.addBands(clim.addBands(ee.Image(k).float())).rename( ['ndvi', 'clim', 'year']).set({'year': k}) img_coll = img_coll.add(img) return ee.ImageCollection(img_coll) ## Function create image collection of residuals def f_ndvi_clim_r_coll(year_start, year_end): res_list = ee.List([]) #for(i = year_start i <= year_end i += 1): for i in range(year_start, year_end + 1): res_image = f_ndvi_clim_r_img(i) res_list = res_list.add(res_image) return ee.ImageCollection(res_list) ## Apply function to create image collection of ndvi and climate ndvi_1yr_coll = f_img_coll(ndvi_1yr) ## Compute linear trend function to predict ndvi based on climate (independent are followed by dependent var lf_clim_ndvi = ndvi_1yr_coll.select(['clim', 'ndvi' ]).reduce(ee.Reducer.linearFit()) ## Apply function to predict NDVI based on climate ndvi_1yr_p = ee.ImageCollection( ee.List(ndvi_1yr_coll.select('clim').iterate(f_ndvi_clim_p, first))) ## Apply function to compute NDVI annual residuals ndvi_1yr_r = f_ndvi_clim_r_coll(year_start, year_end) ## Fit a linear regression to the NDVI residuals lf_trend = ndvi_1yr_r.select(['year', 'ndvi_res']).reduce(ee.Reducer.linearFit()) ## Compute Kendall statistics mk_trend = stats.mann_kendall(ndvi_1yr_r.select('ndvi_res')) return (lf_trend, mk_trend)
def get_data(begin, end, aoi, stat, season='yearly'): """ Function to gather all the spectral imagery to be analysed. :param begin: Begin date of the period for which the analysis is to be done :param end: End date of the period for which the analysis is to be done :param aoi: Area of interest :param season: Season to be analysed, currently supports winter, summer or the whole year. :return: Image containing all the spectral indexes added as bands. """ # Calculate the slope for the area of interest elevation = ee.Image('JAXA/ALOS/AW3D30/V2_2').select('AVE_DSM') slope = ee.Terrain.slope(elevation).clip(aoi).rename('slope') # Select Landsat imagery for the period of interest and join the collections ls_5 = landsat.get_ls5_image_collection(begin, end, aoi) ls_7 = landsat.get_ls7_image_collection(begin, end, aoi) ls = ls_5.merge(ls_7) # Convert the RGB bands in the landsat collection to Hue, Saturation and Value ls_hsv = ls.map(landsat.rgb_to_hsv) # Calculate spectral indices to be analysed. ls_ndvi = ls.map(landsat.add_ndvi_ls457).filter( ee.Filter.listContains('system:band_names', 'NDVI')) ls_gcvi = ls.map(landsat.add_gcvi_ls457).filter( ee.Filter.listContains('system:band_names', 'GCVI')) ls_ndwi = ls.map(landsat.add_ndwi_ls457).filter( ee.Filter.listContains('system:band_names', 'NDWI')) ls_ndbi = ls_ndvi.map(landsat.add_ndbi_ls457).filter( ee.Filter.listContains('system:band_names', 'NDBI')) ls_bu = ls_ndbi.map(landsat.add_bu_ls457).filter( ee.Filter.listContains('system:band_names', 'BU')) ls_pr = landsat.join_precipitation(ls_ndvi, begin, end, aoi).filter( ee.Filter.listContains('system:band_names', 'pr')) ls_wgi = ls_gcvi.map(landsat.add_ndwi_ls457).filter(ee.Filter.listContains('system:band_names', 'NDWI'))\ .map(landsat.add_wgi_ls457).filter(ee.Filter.listContains('system:band_names', 'WGI')) ls_ndwi_greenhouses = ls.map(landsat.add_ndwi_mcfeeters_ls457)\ .filter(ee.Filter.listContains('system:band_names', 'NDWIGH')) # # Calculate the monthly values per spectral index ls_monthly_hue = landsat.create_monthly_index_images( image_collection=ls_hsv, band='hue', start_date=begin, end_date=end, aoi=aoi, stats=['mean', 'median']) ls_monthly_saturation = landsat.create_monthly_index_images( image_collection=ls_hsv, band='saturation', start_date=begin, end_date=end, aoi=aoi, stats=['mean', 'median']) ls_monthly_value = landsat.create_monthly_index_images( image_collection=ls_hsv, band='value', start_date=begin, end_date=end, aoi=aoi, stats=['mean', 'median']) ls_monthly_blue = landsat.create_monthly_index_images( image_collection=ls, band='B1', start_date=begin, end_date=end, aoi=aoi, stats=['mean', 'median']) ls_monthly_green = landsat.create_monthly_index_images( image_collection=ls, band='B2', start_date=begin, end_date=end, aoi=aoi, stats=['mean', 'median']) ls_monthly_red = landsat.create_monthly_index_images( image_collection=ls, band='B3', start_date=begin, end_date=end, aoi=aoi, stats=['mean', 'median']) ls_monthly_ndvi = landsat.create_monthly_index_images( image_collection=ls_ndvi, band='NDVI', start_date=begin, end_date=end, aoi=aoi, stats=['mean', 'median']) ls_monthly_gcvi = landsat.create_monthly_index_images( image_collection=ls_gcvi, band='GCVI', start_date=begin, end_date=end, aoi=aoi, stats=['mean', 'median']) ls_monthly_ndwi = landsat.create_monthly_index_images( image_collection=ls_ndwi, band='NDWI', start_date=begin, end_date=end, aoi=aoi, stats=['mean', 'median']) ls_monthly_wgi = landsat.create_monthly_index_images( image_collection=ls_wgi, band='WGI', start_date=begin, end_date=end, aoi=aoi, stats=['mean', 'median']) ls_monthly_ndbi = landsat.create_monthly_index_images( image_collection=ls_ndbi, band='NDBI', start_date=begin, end_date=end, aoi=aoi, stats=['mean', 'median']) ls_monthly_bu = landsat.create_monthly_index_images( image_collection=ls_bu, band='BU', start_date=begin, end_date=end, aoi=aoi, stats=['mean', 'median']) ls_monthly_ndvi_pr = ls_ndvi_pr = ls_pr.map(landsat.ndvi_precipitation_correction)\ .filter(ee.Filter.listContains('system:band_names', 'NDVI_pr')).select('NDVI_pr') ls_monthly_ndwi_greenhouses = landsat.create_monthly_index_images( image_collection=ls_ndwi_greenhouses, band='NDWIGH', start_date=begin, end_date=end, aoi=aoi, stats=['mean', 'median']) # Finally cobine all the data layers into a single image for the desired season if season == 'yearly': # Create images containing the mean monthly median and max value for the period on interest ls_mean_monthly_mean_hue = ls_monthly_hue.select('mean').mean().rename( 'hue mean') ls_mean_monthly_max_hue = ls_monthly_hue.select('mean').max().rename( 'hue max') ls_mean_monthly_min_hue = ls_monthly_hue.select('mean').min().rename( 'hue min') ls_mean_monthly_mean_saturation = ls_monthly_saturation.select( 'mean').mean().rename('saturation mean') ls_mean_monthly_max_saturation = ls_monthly_saturation.select( 'mean').max().rename('saturation max') ls_mean_monthly_min_saturation = ls_monthly_saturation.select( 'mean').min().rename('saturation min') ls_mean_monthly_mean_value = ls_monthly_value.select( 'mean').mean().rename('value mean') ls_mean_monthly_max_value = ls_monthly_value.select( 'mean').max().rename('value max') ls_mean_monthly_min_value = ls_monthly_value.select( 'mean').min().rename('value min') ls_mean_monthly_mean_blue = ls_monthly_blue.select( 'mean').mean().rename('blue mean') ls_mean_monthly_max_blue = ls_monthly_blue.select('mean').max().rename( 'blue max') ls_mean_monthly_min_blue = ls_monthly_blue.select('mean').min().rename( 'blue min') ls_mean_monthly_mean_green = ls_monthly_green.select( 'mean').mean().rename('green mean') ls_mean_monthly_max_green = ls_monthly_green.select( 'mean').max().rename('green max') ls_mean_monthly_min_green = ls_monthly_green.select( 'mean').min().rename('green min') ls_mean_monthly_mean_red = ls_monthly_red.select('mean').mean().rename( 'red mean') ls_mean_monthly_max_red = ls_monthly_red.select('mean').max().rename( 'red max') ls_mean_monthly_min_red = ls_monthly_red.select('mean').min().rename( 'red min') ls_mean_monthly_mean_ndvi = ls_monthly_ndvi.select( 'mean').mean().rename('NDVI mean') ls_mean_monthly_max_ndvi = ls_monthly_ndvi.select('mean').max().rename( 'NDVI max') ls_mean_monthly_min_ndvi = ls_monthly_ndvi.select('mean').min().rename( 'NDVI min') ls_mean_monthly_mean_gcvi = ls_monthly_gcvi.select( 'mean').mean().rename('GCVI mean') ls_mean_monthly_max_gcvi = ls_monthly_gcvi.select('mean').max().rename( 'GCVI max') ls_mean_monthly_min_gcvi = ls_monthly_gcvi.select('mean').min().rename( 'GCVI min') ls_mean_monthly_mean_ndwi = ls_monthly_ndwi.select( 'mean').mean().rename('NDWI mean') ls_mean_monthly_max_ndwi = ls_monthly_ndwi.select('mean').max().rename( 'NDWI max') ls_mean_monthly_min_ndwi = ls_monthly_ndwi.select('mean').min().rename( 'NDWI min') ls_mean_monthly_mean_wgi = ls_monthly_wgi.select('mean').mean().rename( 'WGI mean') ls_mean_monthly_max_wgi = ls_monthly_wgi.select('mean').max().rename( 'WGI max') ls_mean_monthly_min_wgi = ls_monthly_wgi.select('mean').min().rename( 'WGI min') ls_mean_monthly_mean_ndwi_greenhouses = ls_monthly_ndwi_greenhouses.select( 'mean').mean().rename('NDWIGH mean') ls_mean_monthly_max_ndwi_greenhouses = ls_monthly_ndwi_greenhouses.select( 'mean').max().rename('NDWIGH max') ls_mean_monthly_min_ndwi_greenhouses = ls_monthly_ndwi_greenhouses.select( 'mean').min().rename('NDWIGH min') # Next create standard deviation maps for both the seasonal and yearly NDWI and NDVI values. yearly_std_ndvi = landsat.get_yearly_band_std(ls_monthly_ndvi, [stat], begin, end, aoi) yearly_std_ndwi = landsat.get_yearly_band_std(ls_monthly_ndwi, [stat], begin, end, aoi) yearly_std_gcvi = landsat.get_yearly_band_std(ls_monthly_gcvi, [stat], begin, end, aoi) yearly_std_wgi = landsat.get_yearly_band_std(ls_monthly_wgi, [stat], begin, end, aoi) # Take the mean for the periods that will be used during thresholding. yearly_ndvi_std_mean = yearly_std_ndvi.select(f'{stat}_std').mean() yearly_ndwi_std_mean = yearly_std_ndwi.select(f'{stat}_std').mean() yearly_gcvi_std_mean = yearly_std_gcvi.select(f'{stat}_std').mean() yearly_wgi_std_mean = yearly_std_wgi.select(f'{stat}_std').mean() crop_data = ee.ImageCollection([ ls_mean_monthly_mean_hue, ls_mean_monthly_max_hue, ls_mean_monthly_min_hue, ls_mean_monthly_mean_saturation, ls_mean_monthly_max_saturation, ls_mean_monthly_min_saturation, ls_mean_monthly_mean_value, ls_mean_monthly_max_value, ls_mean_monthly_min_value, ls_mean_monthly_mean_blue, ls_mean_monthly_max_blue, ls_mean_monthly_min_blue, ls_mean_monthly_mean_green, ls_mean_monthly_max_green, ls_mean_monthly_min_green, ls_mean_monthly_mean_red, ls_mean_monthly_max_red, ls_mean_monthly_min_red, ls_mean_monthly_mean_ndvi, ls_mean_monthly_max_ndvi, ls_mean_monthly_min_ndvi, ls_mean_monthly_mean_gcvi, ls_mean_monthly_max_gcvi, ls_mean_monthly_min_gcvi, ls_mean_monthly_mean_ndwi, ls_mean_monthly_max_ndwi, ls_mean_monthly_min_ndwi, ls_mean_monthly_mean_ndwi_greenhouses, ls_mean_monthly_max_ndwi_greenhouses, ls_mean_monthly_min_ndwi_greenhouses, ls_mean_monthly_mean_wgi, ls_mean_monthly_max_wgi, ls_mean_monthly_min_wgi, yearly_ndvi_std_mean.rename('NDVI_std'), yearly_ndwi_std_mean.rename('NDWI_std'), yearly_gcvi_std_mean.rename('GCVI_std'), yearly_wgi_std_mean.rename('WGI_std'), slope, ]) return crop_data.toBands().regexpRename('([0-9]*_)', '') elif season == 'summer': # # Create separate image collections containing only data for the summer months ls_monthly_hue_summer = ls_monthly_hue.filter( ee.Filter.rangeContains('month', 4, 9)) ls_monthly_saturation_summer = ls_monthly_saturation.filter( ee.Filter.rangeContains('month', 4, 9)) ls_monthly_value_summer = ls_monthly_value.filter( ee.Filter.rangeContains('month', 4, 9)) ls_monthly_blue_summer = ls_monthly_blue.filter( ee.Filter.rangeContains('month', 4, 9)) ls_monthly_green_summer = ls_monthly_green.filter( ee.Filter.rangeContains('month', 4, 9)) ls_monthly_red_summer = ls_monthly_red.filter( ee.Filter.rangeContains('month', 4, 9)) ls_monthly_ndvi_summer = ls_monthly_ndvi.filter( ee.Filter.rangeContains('month', 4, 9)) ls_monthly_gcvi_summer = ls_monthly_gcvi.filter( ee.Filter.rangeContains('month', 4, 9)) ls_monthly_ndwi_summer = ls_monthly_ndwi.filter( ee.Filter.rangeContains('month', 4, 9)) ls_monthly_wgi_summer = ls_monthly_wgi.filter( ee.Filter.rangeContains('month', 4, 9)) ls_monthly_ndbi_summer = ls_monthly_ndbi.filter( ee.Filter.rangeContains('month', 4, 9)) ls_monthly_bu_summer = ls_monthly_bu.filter( ee.Filter.rangeContains('month', 4, 9)) ls_monthly_ndvi_pr_summer = ls_monthly_ndvi_pr.filter( ee.Filter.rangeContains('month', 4, 9)) ls_monthly_ndwi_greenhouses_summer = ls_monthly_ndwi_greenhouses.filter( ee.Filter.rangeContains('month', 4, 9)) # # Same principle only now for the summer months ls_mean_monthly_mean_hue_summer = ls_monthly_hue_summer.select( 'mean').mean().rename('hue mean') ls_mean_monthly_max_hue_summer = ls_monthly_hue_summer.select( 'mean').max().rename('hue max') ls_mean_monthly_min_hue_summer = ls_monthly_hue_summer.select( 'mean').min().rename('hue min') ls_mean_monthly_mean_saturation_summer = ls_monthly_saturation_summer.select( 'mean').mean().rename('saturation mean') ls_mean_monthly_max_saturation_summer = ls_monthly_saturation_summer.select( 'mean').max().rename('saturation max') ls_mean_monthly_min_saturation_summer = ls_monthly_saturation_summer.select( 'mean').min().rename('saturation min') ls_mean_monthly_mean_value_summer = ls_monthly_value_summer.select( 'mean').mean().rename('value mean') ls_mean_monthly_max_value_summer = ls_monthly_value_summer.select( 'mean').max().rename('value max') ls_mean_monthly_min_value_summer = ls_monthly_value_summer.select( 'mean').min().rename('value min') ls_mean_monthly_mean_blue_summer = ls_monthly_blue_summer.select( 'mean').mean().rename('blue mean') ls_mean_monthly_max_blue_summer = ls_monthly_blue_summer.select( 'mean').max().rename('blue max') ls_mean_monthly_min_blue_summer = ls_monthly_blue_summer.select( 'mean').min().rename('blue min') ls_mean_monthly_mean_green_summer = ls_monthly_green_summer.select( 'mean').mean().rename('green mean') ls_mean_monthly_max_green_summer = ls_monthly_green_summer.select( 'mean').max().rename('green max') ls_mean_monthly_min_green_summer = ls_monthly_green_summer.select( 'mean').min().rename('green min') ls_mean_monthly_mean_red_summer = ls_monthly_red_summer.select( 'mean').mean().rename('red mean') ls_mean_monthly_max_red_summer = ls_monthly_red_summer.select( 'mean').max().rename('red max') ls_mean_monthly_min_red_summer = ls_monthly_red_summer.select( 'mean').min().rename('red min') ls_mean_monthly_mean_ndvi_summer = ls_monthly_ndvi_summer.select( 'mean').mean().rename('NDVI mean') ls_mean_monthly_max_ndvi_summer = ls_monthly_ndvi_summer.select( 'mean').max().rename('NDVI max') ls_mean_monthly_min_ndvi_summer = ls_monthly_ndvi_summer.select( 'mean').min().rename('NDVI min') ls_mean_monthly_mean_gcvi_summer = ls_monthly_gcvi_summer.select( 'mean').mean().rename('GCVI mean') ls_mean_monthly_max_gcvi_summer = ls_monthly_gcvi_summer.select( 'mean').max().rename('GCVI max') ls_mean_monthly_min_gcvi_summer = ls_monthly_gcvi_summer.select( 'mean').min().rename('GCVI min') ls_mean_monthly_mean_ndwi_summer = ls_monthly_ndwi_summer.select( 'mean').mean().rename('NDWI mean') ls_mean_monthly_max_ndwi_summer = ls_monthly_ndwi_summer.select( 'mean').max().rename('NDWI max') ls_mean_monthly_min_ndwi_summer = ls_monthly_ndwi_summer.select( 'mean').min().rename('NDWI min') ls_mean_monthly_mean_wgi_summer = ls_monthly_wgi_summer.select( 'mean').mean().rename('WGI mean') ls_mean_monthly_max_wgi_summer = ls_monthly_wgi_summer.select( 'mean').max().rename('WGI max') ls_mean_monthly_min_wgi_summer = ls_monthly_wgi_summer.select( 'mean').min().rename('WGI min') ls_mean_monthly_mean_ndbi_summer = ls_monthly_ndbi_summer.select( 'mean').mean().rename('NDBI mean') ls_mean_monthly_max_ndbi_summer = ls_monthly_ndbi_summer.select( 'mean').max().rename('NDBI max') ls_mean_monthly_min_ndbi_summer = ls_monthly_ndbi_summer.select( 'mean').min().rename('NDBI min') ls_mean_monthly_mean_bu_summer = ls_monthly_bu_summer.select( 'mean').mean().rename('BU mean') ls_mean_monthly_max_bu_summer = ls_monthly_bu_summer.select( 'mean').max().rename('BU max') ls_mean_monthly_min_bu_summer = ls_monthly_bu_summer.select( 'mean').min().rename('BU min') ls_mean_monthly_mean_ndvi_pr_summer = ls_monthly_ndvi_pr_summer.select( 'NDVI_pr').mean().rename('NDVI_pr mean') ls_mean_monthly_max_ndvi_pr_summer = ls_monthly_ndvi_pr_summer.select( 'NDVI_pr').max().rename('NDVI_pr max') ls_mean_monthly_min_ndvi_pr_summer = ls_monthly_ndvi_pr_summer.select( 'NDVI_pr').min().rename('NDVI_pr min') ls_mean_monthly_mean_ndwi_greenhouses_summer = ls_monthly_ndwi_greenhouses_summer.select( 'mean').mean().rename('NDWIGH mean') ls_mean_monthly_max_ndwi_greenhouses_summer = ls_monthly_ndwi_greenhouses_summer.select( 'mean').max().rename('NDWIGH max') ls_mean_monthly_min_ndwi_greenhouses_summer = ls_monthly_ndwi_greenhouses_summer.select( 'mean').min().rename('NDWIGH min') # Next create standard deviation maps for both the seasonal and yearly NDWI and NDVI values. summer_std_ndvi = landsat.get_yearly_band_std(ls_monthly_ndvi, [stat], begin, end, aoi, season='summer') summer_std_ndwi = landsat.get_yearly_band_std(ls_monthly_ndwi, [stat], begin, end, aoi, season='summer') summer_std_gcvi = landsat.get_yearly_band_std(ls_monthly_gcvi, [stat], begin, end, aoi, season='summer') summer_std_wgi = landsat.get_yearly_band_std(ls_monthly_wgi, [stat], begin, end, aoi, season='summer') summer_ndvi_std_mean = summer_std_ndvi.select(f'{stat}_std').mean() summer_ndwi_std_mean = summer_std_ndwi.select(f'{stat}_std').mean() summer_gcvi_std_mean = summer_std_gcvi.select(f'{stat}_std').mean() summer_wgi_std_mean = summer_std_wgi.select(f'{stat}_std').mean() crop_data = ee.ImageCollection([ ls_mean_monthly_mean_hue_summer, ls_mean_monthly_max_hue_summer, ls_mean_monthly_min_hue_summer, ls_mean_monthly_mean_saturation_summer, ls_mean_monthly_max_saturation_summer, ls_mean_monthly_min_saturation_summer, ls_mean_monthly_mean_value_summer, ls_mean_monthly_max_value_summer, ls_mean_monthly_min_value_summer, ls_mean_monthly_mean_blue_summer, ls_mean_monthly_max_blue_summer, ls_mean_monthly_min_blue_summer, ls_mean_monthly_mean_green_summer, ls_mean_monthly_max_green_summer, ls_mean_monthly_min_green_summer, ls_mean_monthly_mean_red_summer, ls_mean_monthly_max_red_summer, ls_mean_monthly_min_red_summer, ls_mean_monthly_mean_ndvi_summer, ls_mean_monthly_max_ndvi_summer, ls_mean_monthly_min_ndvi_summer, ls_mean_monthly_mean_gcvi_summer, ls_mean_monthly_max_gcvi_summer, ls_mean_monthly_min_gcvi_summer, ls_mean_monthly_mean_ndwi_summer, ls_mean_monthly_max_ndwi_summer, ls_mean_monthly_min_ndwi_summer, ls_mean_monthly_mean_wgi_summer, ls_mean_monthly_max_wgi_summer, ls_mean_monthly_min_wgi_summer, ls_mean_monthly_mean_ndbi_summer, ls_mean_monthly_max_ndbi_summer, ls_mean_monthly_min_ndbi_summer, ls_mean_monthly_mean_bu_summer, ls_mean_monthly_max_bu_summer, ls_mean_monthly_min_bu_summer, ls_mean_monthly_mean_ndvi_pr_summer, ls_mean_monthly_max_ndvi_pr_summer, ls_mean_monthly_min_ndvi_pr_summer, ls_mean_monthly_mean_ndwi_greenhouses_summer, ls_mean_monthly_max_ndwi_greenhouses_summer, ls_mean_monthly_min_ndwi_greenhouses_summer, summer_ndvi_std_mean.rename('NDVI_std'), summer_ndwi_std_mean.rename('NDWI_std'), summer_gcvi_std_mean.rename('GCVI_std'), summer_wgi_std_mean.rename('WGI_std'), slope, ]) return crop_data.toBands().regexpRename('([0-9]*_)', '') elif season == 'winter': early_filter = ee.Filter.rangeContains('month', 1, 3) late_filter = ee.Filter.rangeContains('month', 10, 12) ls_monthly_hue_winter = ls_monthly_hue.filter( ee.Filter.Or(early_filter, late_filter)) ls_monthly_saturation_winter = ls_monthly_saturation.filter( ee.Filter.Or(early_filter, late_filter)) ls_monthly_value_winter = ls_monthly_value.filter( ee.Filter.Or(early_filter, late_filter)) ls_monthly_blue_winter = ls_monthly_blue.filter( ee.Filter.Or(early_filter, late_filter)) ls_monthly_green_winter = ls_monthly_green.filter( ee.Filter.Or(early_filter, late_filter)) ls_monthly_red_winter = ls_monthly_red.filter( ee.Filter.Or(early_filter, late_filter)) ls_monthly_ndvi_winter = ls_monthly_ndvi.filter( ee.Filter.Or(early_filter, late_filter)) ls_monthly_gcvi_winter = ls_monthly_gcvi.filter( ee.Filter.Or(early_filter, late_filter)) ls_monthly_ndwi_winter = ls_monthly_ndwi.filter( ee.Filter.Or(early_filter, late_filter)) ls_monthly_wgi_winter = ls_monthly_wgi.filter( ee.Filter.Or(early_filter, late_filter)) ls_monthly_ndbi_winter = ls_monthly_ndbi.filter( ee.Filter.Or(early_filter, late_filter)) ls_monthly_bu_winter = ls_monthly_bu.filter( ee.Filter.Or(early_filter, late_filter)) ls_monthly_ndvi_pr_winter = ls_monthly_ndvi_pr.filter( ee.Filter.Or(early_filter, late_filter)) ls_monthly_ndwi_greenhouses_winter = ls_monthly_ndwi_greenhouses.filter( ee.Filter.Or(early_filter, late_filter)) # And Winter ls_mean_monthly_mean_hue_winter = ls_monthly_hue_winter.select( 'mean').mean().rename('hue mean') ls_mean_monthly_max_hue_winter = ls_monthly_hue_winter.select( 'mean').max().rename('hue max') ls_mean_monthly_min_hue_winter = ls_monthly_hue_winter.select( 'mean').min().rename('hue min') ls_mean_monthly_mean_saturation_winter = ls_monthly_saturation_winter.select( 'mean').mean().rename('saturation mean') ls_mean_monthly_max_saturation_winter = ls_monthly_saturation_winter.select( 'mean').max().rename('saturation max') ls_mean_monthly_min_saturation_winter = ls_monthly_saturation_winter.select( 'mean').min().rename('saturation min') ls_mean_monthly_mean_value_winter = ls_monthly_value_winter.select( 'mean').mean().rename('value mean') ls_mean_monthly_max_value_winter = ls_monthly_value_winter.select( 'mean').max().rename('value max') ls_mean_monthly_min_value_winter = ls_monthly_value_winter.select( 'mean').min().rename('value min') ls_mean_monthly_mean_blue_winter = ls_monthly_blue_winter.select( 'mean').mean().rename('blue mean') ls_mean_monthly_max_blue_winter = ls_monthly_blue_winter.select( 'mean').max().rename('blue max') ls_mean_monthly_min_blue_winter = ls_monthly_blue_winter.select( 'mean').min().rename('blue min') ls_mean_monthly_mean_green_winter = ls_monthly_green_winter.select( 'mean').mean().rename('green mean') ls_mean_monthly_max_green_winter = ls_monthly_green_winter.select( 'mean').max().rename('green max') ls_mean_monthly_min_green_winter = ls_monthly_green_winter.select( 'mean').min().rename('green min') ls_mean_monthly_mean_red_winter = ls_monthly_red_winter.select( 'mean').mean().rename('red mean') ls_mean_monthly_max_red_winter = ls_monthly_red_winter.select( 'mean').max().rename('red max') ls_mean_monthly_min_red_winter = ls_monthly_red_winter.select( 'mean').min().rename('red min') ls_mean_monthly_mean_ndvi_winter = ls_monthly_ndvi_winter.select( 'mean').mean().rename('NDVI mean') ls_mean_monthly_max_ndvi_winter = ls_monthly_ndvi_winter.select( 'mean').max().rename('NDVI max') ls_mean_monthly_min_ndvi_winter = ls_monthly_ndvi_winter.select( 'mean').min().rename('NDVI min') ls_mean_monthly_mean_gcvi_winter = ls_monthly_gcvi_winter.select( 'mean').mean().rename('GCVI mean') ls_mean_monthly_max_gcvi_winter = ls_monthly_gcvi_winter.select( 'mean').max().rename('GCVI max') ls_mean_monthly_min_gcvi_winter = ls_monthly_gcvi_winter.select( 'mean').min().rename('GCVI min') ls_mean_monthly_mean_ndwi_winter = ls_monthly_ndwi_winter.select( 'mean').mean().rename('NDWI mean') ls_mean_monthly_max_ndwi_winter = ls_monthly_ndwi_winter.select( 'mean').max().rename('NDWI max') ls_mean_monthly_min_ndwi_winter = ls_monthly_ndwi_winter.select( 'mean').min().rename('NDWI min') ls_mean_monthly_mean_wgi_winter = ls_monthly_wgi_winter.select( 'mean').mean().rename('WGI mean') ls_mean_monthly_max_wgi_winter = ls_monthly_wgi_winter.select( 'mean').max().rename('WGI max') ls_mean_monthly_min_wgi_winter = ls_monthly_wgi_winter.select( 'mean').min().rename('WGI min') ls_mean_monthly_mean_ndbi_winter = ls_monthly_ndbi_winter.select( 'mean').mean().rename('NDBI mean') ls_mean_monthly_max_ndbi_winter = ls_monthly_ndbi_winter.select( 'mean').max().rename('NDBI max') ls_mean_monthly_min_ndbi_winter = ls_monthly_ndbi_winter.select( 'mean').min().rename('NDBI min') ls_mean_monthly_mean_bu_winter = ls_monthly_bu_winter.select( 'mean').mean().rename('BU mean') ls_mean_monthly_max_bu_winter = ls_monthly_bu_winter.select( 'mean').max().rename('BU max') ls_mean_monthly_min_bu_winter = ls_monthly_bu_winter.select( 'mean').min().rename('BU min') ls_mean_monthly_mean_ndvi_pr_winter = ls_monthly_ndvi_pr_winter.select( 'NDVI_pr').mean().rename('NDVI_pr mean') ls_mean_monthly_max_ndvi_pr_winter = ls_monthly_ndvi_pr_winter.select( 'NDVI_pr').max().rename('NDVI_pr max') ls_mean_monthly_min_ndvi_pr_winter = ls_monthly_ndvi_pr_winter.select( 'NDVI_pr').min().rename('NDVI_pr min') ls_mean_monthly_mean_ndwi_greenhouses_winter = ls_monthly_ndwi_greenhouses_winter.select( 'mean').mean().rename('NDWIGH mean') ls_mean_monthly_max_ndwi_greenhouses_winter = ls_monthly_ndwi_greenhouses_winter.select( 'mean').max().rename('NDWIGH max') ls_mean_monthly_min_ndwi_greenhouses_winter = ls_monthly_ndwi_greenhouses_winter.select( 'mean').min().rename('NDWIGH min') # Next create standard deviation maps for both the seasonal and yearly NDWI and NDVI values. winter_std_ndvi = landsat.get_yearly_band_std(ls_monthly_ndvi, [f'{stat}'], begin, end, aoi, season='winter') winter_std_ndwi = landsat.get_yearly_band_std(ls_monthly_ndwi, [f'{stat}'], begin, end, aoi, season='winter') winter_std_gcvi = landsat.get_yearly_band_std(ls_monthly_gcvi, [f'{stat}'], begin, end, aoi, season='winter') winter_std_wgi = landsat.get_yearly_band_std(ls_monthly_wgi, [f'{stat}'], begin, end, aoi, season='winter') winter_ndvi_std_mean = winter_std_ndvi.select(f'{stat}_std').mean() winter_ndwi_std_mean = winter_std_ndwi.select(f'{stat}_std').mean() winter_gcvi_std_mean = winter_std_gcvi.select(f'{stat}_std').mean() winter_wgi_std_mean = winter_std_wgi.select(f'{stat}_std').mean() crop_data = ee.ImageCollection([ ls_mean_monthly_mean_hue_winter, ls_mean_monthly_max_hue_winter, ls_mean_monthly_min_hue_winter, ls_mean_monthly_mean_saturation_winter, ls_mean_monthly_max_saturation_winter, ls_mean_monthly_min_saturation_winter, ls_mean_monthly_mean_value_winter, ls_mean_monthly_max_value_winter, ls_mean_monthly_min_value_winter, ls_mean_monthly_mean_blue_winter, ls_mean_monthly_max_blue_winter, ls_mean_monthly_min_blue_winter, ls_mean_monthly_mean_green_winter, ls_mean_monthly_max_green_winter, ls_mean_monthly_min_green_winter, ls_mean_monthly_mean_red_winter, ls_mean_monthly_max_red_winter, ls_mean_monthly_min_red_winter, ls_mean_monthly_mean_ndvi_winter, ls_mean_monthly_max_ndvi_winter, ls_mean_monthly_min_ndvi_winter, ls_mean_monthly_mean_gcvi_winter, ls_mean_monthly_max_gcvi_winter, ls_mean_monthly_min_gcvi_winter, ls_mean_monthly_mean_ndwi_winter, ls_mean_monthly_max_ndwi_winter, ls_mean_monthly_min_ndwi_winter, ls_mean_monthly_mean_wgi_winter, ls_mean_monthly_max_wgi_winter, ls_mean_monthly_min_wgi_winter, ls_mean_monthly_mean_ndbi_winter, ls_mean_monthly_max_ndbi_winter, ls_mean_monthly_min_ndbi_winter, ls_mean_monthly_mean_bu_winter, ls_mean_monthly_max_bu_winter, ls_mean_monthly_min_bu_winter, ls_mean_monthly_mean_ndvi_pr_winter, ls_mean_monthly_max_ndvi_pr_winter, ls_mean_monthly_min_ndvi_pr_winter, ls_mean_monthly_mean_ndwi_greenhouses_winter, ls_mean_monthly_max_ndwi_greenhouses_winter, ls_mean_monthly_min_ndwi_greenhouses_winter, winter_ndvi_std_mean.rename('NDVI_std'), winter_ndwi_std_mean.rename('NDWI_std'), winter_gcvi_std_mean.rename('GCVI_std'), winter_wgi_std_mean.rename('WGI_std'), slope, ]) return crop_data.toBands().regexpRename('([0-9]*_)', '')
import ee from ee_plugin import Map image = ee.ImageCollection('COPERNICUS/S2') \ .filterDate('2017-01-01', '2017-01-02').median() \ .divide(10000).visualize(**{'bands': ['B12', 'B8', 'B4'], 'min': 0.05, 'max': 0.5}) Map.setCenter(35.2, 31, 13) Map.addLayer(image, {}, 'Sentinel-2 images January, 2018')
def exp(collection, folderpath, start, end, geojson, bandnames, operator, typ, ff): # typ = ee.data.getInfo(collection)['type'] bandnames = ast.literal_eval(bandnames) try: if geojson.endswith(".geojson"): with open(geojson) as aoi: aoi_resp = json.load(aoi) aoi_geom = ee.Geometry.Polygon( aoi_resp["features"][0]["geometry"]["coordinates"] ) boundbox = aoi_geom.bounds() elif geojson.endswith(".json"): with open(geojson) as aoi: aoi_resp = json.load(aoi) aoi_geom = ee.Geometry.Polygon( aoi_resp["config"][0]["config"]["coordinates"] ) boundbox = aoi_geom.bounds() elif geojson.endswith(".kml"): getcoord = kml2coord(geojson) aoi_geom = ee.Geometry.Polygon(getcoord) boundbox = aoi_geom.bounds() except Exception as e: print("Could not parse geometry") print(e) if typ == "image" and operator == "bb": userCollection = ee.ImageCollection(collection).select(bandnames) clipname = os.path.basename(geojson).split(".")[0] fileName = str(collection).split("/")[-1] + "_" + str(clipname) + "_bb" firstband = ee.Image(collection).bandNames().getInfo()[0] scale = int( ee.Image(collection).select(firstband).projection().nominalScale().getInfo() ) task = ee.batch.Export.image.toDrive( image=ee.Image(collection).clip(boundbox), description=fileName, folder=folderpath, maxPixels=1e13, region=boundbox.getInfo()["coordinates"][0], scale=scale, ) task.start() print("Finished creating export tasks") print("") elif typ == "image" and operator == None: userCollection = ee.ImageCollection(collection).select(bandnames) clipname = os.path.basename(geojson).split(".")[0] fileName = str(collection).split("/")[-1] + "_" + str(clipname) firstband = ee.Image(collection).bandNames().getInfo()[0] scale = int( ee.Image(collection).select(firstband).projection().nominalScale().getInfo() ) task = ee.batch.Export.image.toDrive( image=ee.Image(collection).clip(aoi_geom), description=fileName, folder=folderpath, maxPixels=1e13, region=aoi_geom.getInfo()["coordinates"][0], scale=scale, ) task.start() print("Finished creating export tasks") print("") elif typ == "collection" and operator == "bb": userCollection = ( ee.ImageCollection(collection) .filterBounds(aoi_geom) .filterDate(start, end) .select(bandnames) ) clipname = os.path.basename(geojson).split(".")[0] imageList = ee.List(userCollection.toList(userCollection.size().add(1))) length = userCollection.size().getInfo() if int(length) == 0: print("No images found exiting export function") sys.exit() else: print("Total images in filtered collection: " + str(length)) def exportImage(img): fileName = ee.String(img.get("system:index")).getInfo() firstband = img.bandNames().getInfo()[0] scale = int(img.select(firstband).projection().nominalScale().getInfo()) # get geometry of image task = ee.batch.Export.image.toDrive( image=img.clip(boundbox), description=fileName + "_" + str(clipname) + "_bb", folder=folderpath, maxPixels=1e13, region=boundbox.getInfo()["coordinates"][0], scale=scale, ) task.start() index = 0 while index < int(length): print("Export #: " + str(index + 1) + " of " + str(length)) img2export = ee.Image(imageList.get(index)) exportImage(img2export) index = index + 1 # time.sleep(10) print("Finished creating export tasks") print("") elif typ == "collection" and operator == None: userCollection = ( ee.ImageCollection(collection) .filterBounds(aoi_geom) .filterDate(start, end) .select(bandnames) ) clipname = os.path.basename(geojson).split(".")[0] imageList = ee.List(userCollection.toList(userCollection.size().add(1))) length = userCollection.size().getInfo() if int(length) == 0: print("No images found exiting export function") sys.exit() else: print("Total images in filtered collection: " + str(length)) def exportImage(img): fileName = ee.String(img.get("system:index")).getInfo() firstband = img.bandNames().getInfo()[0] scale = int(img.select(firstband).projection().nominalScale().getInfo()) # get geometry of image task = ee.batch.Export.image.toDrive( image=img.clip(aoi_geom), description=fileName + "_" + str(clipname), folder=folderpath, maxPixels=1e13, region=aoi_geom.getInfo()["coordinates"][0], scale=scale, ) task.start() index = 0 while index < int(length): print("Export #: " + str(index + 1) + " of " + str(length)) img2export = ee.Image(imageList.get(index)) exportImage(img2export) index = index + 1 # time.sleep(10) print("Finished creating export tasks") print("") elif typ == "table" and operator == "bb": if ff is None: ff = "SHP" userCollection = ee.FeatureCollection(collection) clipname = os.path.basename(geojson).split(".")[0] fileName = str(collection).split("/")[-1] + "_" + str(clipname) + "_bb" task = ee.batch.Export.table.toDrive( collection=ee.FeatureCollection(collection).filterBounds(boundbox), description=fileName, folder=folderpath, fileFormat=ff, ) task.start() print("Finished creating export tasks") print("") elif typ == "table" and operator == None: if ff is None: ff = "SHP" userCollection = ee.FeatureCollection(collection) clipname = os.path.basename(geojson).split(".")[0] fileName = str(collection).split("/")[-1] + "_" + str(clipname) task = ee.batch.Export.table.toDrive( collection=ee.FeatureCollection(collection), description=fileName, folder=folderpath, fileFormat=ff, ) task.start() print("Finished creating export tasks") print("")
[-102.05, 41.0], # Colorado [-109.05, 41.0], [-111.05, 41.0], [-111.05, 42.0], # Utah [-114.05, 42.0], [-114.05, 37.0], [-109.05, 37.0] ]], 'geodesic': False }) # Create a Landsat 7 composite for Spring of 2000, and filter by # the bounds of the FeatureCollection. collection = ee.ImageCollection('LANDSAT/LE07/C01/T1') \ .filterDate('2000-04-01', '2000-07-01') \ .filterBounds(polygon) # Compute the median in each band, in each pixel. median = collection.median() # Select the red, green and blue bands. result = median.select('B3', 'B2', 'B1') Map.addLayer(result, {'gain': [1.4, 1.4, 1.1]}) Map.setCenter(-110, 40, 5) # %% """ ## Display Earth Engine data layers """
def prep_cnn_np(survey_df, satellite, kernel_size, year): ''' Creates numpy arrays for CNN Input: df - pandas dataframe lat_name - name of latitude variable in df lon_name - name of longitude variable in df Output: geopandas dataframe ''' # Setup -------------------------------------------------------------------- # Survey to FeatureCollection survey_fc = survey_to_fc(survey_df) # Define kernel for neighborhood array list = ee.List.repeat(1, kernel_size) lists = ee.List.repeat(list, kernel_size) kernel = ee.Kernel.fixed(kernel_size, kernel_size, lists) # Define scale if satellite in ['l7', 'l8']: SCALE = 30 elif satellite in ['s2']: SCALE = 10 # Prep NTL ----------------------------------------------------------------- # Year # VIIRS starts in 2012. At minimum, use 2013 to have year before and after if False: if year <= 2013: year_use = 2013 else: year_use = year year_plus = year_use + 1 year_minus = year_use - 1 year_minus_str = str(year_minus) + '-01-01' year_plus_str = str(year_plus) + '-12-31' # Reduce image collection ntl_image = ee.ImageCollection('NOAA/VIIRS/DNB/MONTHLY_V1/VCMCFG')\ .filterDate(year_minus_str, year_plus_str)\ .median() # Select Bands ntl_image = ntl_image.select(['avg_rad']) # Image to neighborhood array ntl_arrays = ntl_image.neighborhoodToArray(kernel) # Extract values from GEE ntl_values_ee = ntl_arrays.sample( region=survey_fc, scale=SCALE, tileScale=10 #8 ) ntl_dict_ee = ntl_values_ee.getInfo() # Convert values to numpy array #n_rows = survey_df.shape[0] ntl_f = ntl_dict_ee['features'] # l7 ---------------------------------------------------------------- if satellite == "l7": # Bands b_b = 'B1' g_b = 'B2' r_b = 'B3' nir_b = 'B4' swir_b = 'B5' #other_bs = ['B5', 'B6', 'B7'] #BANDS = single_bs.copy() BANDS = [b_b].copy() BANDS.append(g_b) BANDS.append(r_b) BANDS.append(nir_b) BANDS.append(swir_b) # Year year_use = year year_plus = year_use + 1 year_minus = year_use - 1 year_minus_str = str(year_minus) + '-01-01' year_plus_str = str(year_plus) + '-12-31' image = ee.ImageCollection('LANDSAT/LC07/C01/T1_SR')\ .filterDate(year_minus_str, year_plus_str)\ .map(cloud_mask_landsatSR)\ .median() #\ #.multiply(0.0001) # l8 ---------------------------------------------------------------- if satellite == "l8": # Bands # FOR COLLECTION 2 #b_b = 'SR_B2' #g_b = 'SR_B3' #r_b = 'SR_B4' #nir_b = 'SR_B5' #other_bs = ['SR_B6', 'SR_B7', 'ST_B10'] # FOR COLLECTION 1 b_b = 'B2' g_b = 'B3' r_b = 'B4' nir_b = 'B5' swir_b = 'B6' #other_bs = ['B6', 'B7', 'B10'] #BANDS = single_bs.copy() BANDS = [b_b].copy() BANDS.append(g_b) BANDS.append(r_b) BANDS.append(nir_b) BANDS.append(swir_b) # Year # landsat 8 starts in April 2013; if year is less than # 2014, use 2014 as year (to ensure have year before and after) if year < 2014: year_use = 2014 else: year_use = year year_plus = year_use + 1 year_minus = year_use - 1 year_minus_str = str(year_minus) + '-01-01' year_plus_str = str(year_plus) + '-12-31' #image = ee.ImageCollection('LANDSAT/LC08/C02/T1_L2')\ # .filterDate(year_minus_str, year_plus_str)\ # #.map(cloud_mask_landsatSR)\ #TODO cloud_mask_landsatSR doesn't work with landsat collection 2 # .median() #\ # #.multiply(0.0001) image = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR')\ .filterDate(year_minus_str, year_plus_str)\ .map(cloud_mask_landsatSR)\ .median() #image = ee.ImageCollection('LANDSAT/LC08/C02/T1_L2')\ # .filterDate(year_minus_str, year_plus_str)\ # .median() # s2 ---------------------------------------------------------------- if satellite == "s2": # Bands b_b = 'B2' g_b = 'B3' r_b = 'B4' nir_b = 'B8' swir_b = 'B11' #other_bs = ['B5', 'B6', 'B7', 'B8A', 'B11', 'B12', 'AOT'] #BANDS = single_bs.copy() BANDS = [b_b].copy() BANDS.append(g_b) BANDS.append(r_b) BANDS.append(nir_b) BANDS.append(swir_b) # Year # sentinel starts in March 2017; juse use 2018 year_use = 2019 year_plus = year_use + 1 year_minus = year_use - 1 year_minus_str = str(year_minus) + '-01-01' year_plus_str = str(year_plus) + '-12-31' # Number of bands changes in sentinel, so need to select here before aggregate # https://gis.stackexchange.com/questions/374010/gee-tile-error-expected-a-homogeneous-image-collection-but-an-image-with-incom image = ee.ImageCollection('COPERNICUS/S2_SR')\ .filterDate(year_minus_str, year_plus_str)\ .map(cloud_mask_sentinel2)\ .select(BANDS)\ .median() # \ #.multiply(0.0001) # Select Bands image = image.select(BANDS) # Create Indices # https://www.linkedin.com/pulse/ndvi-ndbi-ndwi-calculation-using-landsat-7-8-tek-bahadur-kshetri ndvi = image.normalizedDifference([nir_b, r_b]).rename('NDVI') ndbi = image.normalizedDifference([swir_b, nir_b]).rename('NDBI') image = image.addBands(ndvi) image = image.addBands(ndbi) bu = image.select('NDBI').subtract(image.select('NDVI')).rename('BU') image = image.addBands(bu) # Subset bands; don't need those used to create NDVI and NDBI image = image.select([b_b, g_b, r_b, 'NDVI', 'BU']) # Image to neighborhood array arrays = image.neighborhoodToArray(kernel) # New --------- #neighborhoodImage = myImageToBeSampled.neighborhoodToArray(kernel) #samples = arrays.sampleRegions(collection=survey_fc) # ee.batch.Export.table.toCloudStorage # ee.batch.Export.table.toDrive # Export.table.toDrive #mytask = ee.batch.Export.table.toDrive( # collection = samples, # fileFormat = 'TFRecord', # description = 'test123', # folder = 'gee_extracts', # selectors = [b_b, g_b, r_b, 'NDVI', 'BU'] + ['uid', 'ntl_group']) #return mytask # OLD --------- # Extract values from GEE values_ee = arrays.sample( region=survey_fc, scale=SCALE, tileScale=12 # 8 ) dict_ee = values_ee.getInfo() # Convert values to numpy array n_rows = survey_df.shape[0] daytime_f = dict_ee['features'] # Extract data out_ex_proto_list = ee_to_np_daytime(daytime_f, survey_df, n_rows, b_b, g_b, r_b) return out_ex_proto_list # https://csaybar.github.io/blog/2019/05/30/eetf/ # https://stackoverflow.com/questions/63000565/extract-10000-images-from-google-earth-engine # https://colab.research.google.com/github/google/earthengine-api/blob/master/python/examples/ipynb/UNET_regression_demo.ipynb#scrollTo=-IlgXu-vcUEY
# To make sure you're logged in, run: earthengine authenticate # Edit the DATA to be one of the values listed above # Edit NUMBER_OF_POINTS to be the number of data points you want # Edit FILE_NAME to be the name of the output file # run: python3 data_extract.py # The result will be a new CSV file in your Google Drive DATA = RADIATION NUMBER_OF_POINTS = 100000 FILE_NAME = 'Radiation-100k' # ------------------------------------ if DATA in [ELEVATION, SOIL_TYPE, PRECIPITATION]: image = ee.Image(DATA) # Use this if the data is already an Image type else: image_collection = ee.ImageCollection(DATA) # Use this if the data is an ImageCollection if DATA in [HUMAN_MODIF]: image = image_collection.median() # Use this for non-time based data else: # The image input data is a 2018 cloud-masked median composite. # image = image_collection.filterDate('2018-01-01', '2018-12-31').median() # Use this for time-based data image = image_collection.filterDate('2017-01-01', '2017-12-31').median() # Use this for time-based data # Change the following two lines to use your own training data. region = regions.california() labels = ee.FeatureCollection.randomPoints(region, NUMBER_OF_POINTS) # Sample the image at the points and add a random column. sample = image.sampleRegions(collection=labels, scale=30) #.randomColumn()
Map = folium.Map(location=[40, -100], zoom_start=4) Map.setOptions('HYBRID') # %% ''' ## Add Earth Engine Python script ''' # %% Map.setCenter(-110, 40, 5) fc = ee.FeatureCollection('TIGER/2018/States').filter( ee.Filter.eq('STUSPS', 'MN')) # Create a Landsat 7, median-pixel composite for Spring of 2000. collection = ee.ImageCollection('LE7_L1T').filterDate("2000-05-01", "2000-10-31") \ .filterBounds(fc) image1 = collection.median() # Map.addLayer(image1) # # Clip to the output image to the California state boundary. # # fc = (ee.FeatureCollection('ft:1fRY18cjsHzDgGiJiS2nnpUU3v9JPDc2HNaR7Xk8') # # .filter(ee.Filter().eq('Name', 'Minnesota'))) image2 = image1.clipToCollection(fc) # Select the red, green and blue bands. image = image2.select('B4', 'B3', 'B2') Map.addLayer(image, {'gain': [1.4, 1.4, 1.1]}, 'Landsat 7') # %% '''
""" import ee from ee import batch from pandas.tseries.offsets import DateOffset import pandas as pd import folium ## Initialize (a ee python thing) ee.Initialize() #################################################### roi = ee.FeatureCollection("users/kkraoj/west_usa") fireCol = ee.ImageCollection("MODIS/006/MCD64A1") prism = ee.ImageCollection("OREGONSTATE/PRISM/AN81m") # prismMean = ee.ImageCollection("OREGONSTATE/PRISM/Norm81m").filter(ee.Filter.calendarRange(3, 9, 'month')).mean() # prism = prism.filterDate("2001-01-01","2019-12-31").filter(ee.Filter.calendarRange(3, 9, 'month')) fireCol = fireCol.filterDate("2001-01-01", "2021-03-31") def addMonthYear(image): month = ee.Date(image.get("system:time_start")).get("month") year = ee.Date(image.get("system:time_start")).get("year") image = image.set("month", month) image = image.set("year", year) return image
def main(ini_path=None, overwrite_flag=False, delay_time=0, gee_key_file=None, ready_task_max=-1, reverse_flag=False, tiles=None, update_flag=False, log_tasks=True, recent_days=0, start_dt=None, end_dt=None): """Compute gridded Tcorr images by WRS2 tile Parameters ---------- ini_path : str Input file path. overwrite_flag : bool, optional If True, overwrite existing files if the export dates are the same and generate new images (but with different export dates) even if the tile lists are the same. The default is False. delay_time : float, optional Delay time in seconds between starting export tasks (or checking the number of queued tasks, see "max_ready" parameter). The default is 0. gee_key_file : str, None, optional Earth Engine service account JSON key file (the default is None). ready_task_max: int, optional Maximum number of queued "READY" tasks. reverse_flag : bool, optional If True, process WRS2 tiles in reverse order (the default is False). tiles : str, None, optional List of MGRS tiles to process (the default is None). update_flag : bool, optional If True, only overwrite scenes with an older model version. recent_days : int, optional Limit start/end date range to this many days before the current date (the default is 0 which is equivalent to not setting the parameter and will use the INI start/end date directly). start_dt : datetime, optional Override the start date in the INI file (the default is None which will use the INI start date). end_dt : datetime, optional Override the (inclusive) end date in the INI file (the default is None which will use the INI end date). """ logging.info('\nCompute gridded Tcorr images by WRS2 tile') # CGM - Which format should we use for the WRS2 tile? wrs2_tile_fmt = 'p{:03d}r{:03d}' # wrs2_tile_fmt = '{:03d}{:03d}' wrs2_tile_re = re.compile('p?(\d{1,3})r?(\d{1,3})') # List of path/rows to skip wrs2_skip_list = [ 'p049r026', # Vancouver Island, Canada # 'p047r031', # North California coast 'p042r037', # San Nicholas Island, California # 'p041r037', # South California coast 'p040r038', 'p039r038', 'p038r038', # Mexico (by California) 'p037r039', 'p036r039', 'p035r039', # Mexico (by Arizona) 'p034r039', 'p033r039', # Mexico (by New Mexico) 'p032r040', # Mexico (West Texas) 'p029r041', 'p028r042', 'p027r043', 'p026r043', # Mexico (South Texas) 'p019r040', 'p018r040', # West Florida coast 'p016r043', 'p015r043', # South Florida coast 'p014r041', 'p014r042', 'p014r043', # East Florida coast 'p013r035', 'p013r036', # North Carolina Outer Banks 'p013r026', 'p012r026', # Canada (by Maine) 'p011r032', # Rhode Island coast ] wrs2_path_skip_list = [9, 49] wrs2_row_skip_list = [25, 24, 43] mgrs_skip_list = [] export_id_fmt = 'tcorr_gridded_{product}_{scene_id}' asset_id_fmt = '{coll_id}/{scene_id}' # TODO: Move to INI or function input parameter clip_ocean_flag = True # Read config file ini = configparser.ConfigParser(interpolation=None) ini.read_file(open(ini_path, 'r')) # ini = utils.read_ini(ini_path) model_name = 'SSEBOP' try: study_area_coll_id = str(ini['INPUTS']['study_area_coll']) except KeyError: raise ValueError('"study_area_coll" parameter was not set in INI') except Exception as e: raise e try: start_date = str(ini['INPUTS']['start_date']) except KeyError: raise ValueError('"start_date" parameter was not set in INI') except Exception as e: raise e try: end_date = str(ini['INPUTS']['end_date']) except KeyError: raise ValueError('"end_date" parameter was not set in INI') except Exception as e: raise e try: collections = str(ini['INPUTS']['collections']) collections = sorted([x.strip() for x in collections.split(',')]) except KeyError: raise ValueError('"collections" parameter was not set in INI') except Exception as e: raise e try: mgrs_ftr_coll_id = str(ini['EXPORT']['mgrs_ftr_coll']) except KeyError: raise ValueError('"mgrs_ftr_coll" parameter was not set in INI') except Exception as e: raise e # Optional parameters try: study_area_property = str(ini['INPUTS']['study_area_property']) except KeyError: study_area_property = None logging.debug(' study_area_property: not set in INI, defaulting to None') except Exception as e: raise e try: study_area_features = str(ini['INPUTS']['study_area_features']) study_area_features = sorted([ x.strip() for x in study_area_features.split(',')]) except KeyError: study_area_features = [] logging.debug(' study_area_features: not set in INI, defaulting to []') except Exception as e: raise e try: wrs2_tiles = str(ini['INPUTS']['wrs2_tiles'])\ .replace('"', '').replace("'", '') wrs2_tiles = sorted([x.strip() for x in wrs2_tiles.split(',')]) except KeyError: wrs2_tiles = [] logging.debug(' wrs2_tiles: not set in INI, defaulting to []') except Exception as e: raise e try: mgrs_tiles = str(ini['EXPORT']['mgrs_tiles']) mgrs_tiles = sorted([x.strip() for x in mgrs_tiles.split(',')]) # CGM - Remove empty strings caused by trailing or extra commas mgrs_tiles = [x.upper() for x in mgrs_tiles if x] logging.debug(f' mgrs_tiles: {mgrs_tiles}') except KeyError: mgrs_tiles = [] logging.debug(' mgrs_tiles: not set in INI, defaulting to []') except Exception as e: raise e try: utm_zones = str(ini['EXPORT']['utm_zones']) utm_zones = sorted([int(x.strip()) for x in utm_zones.split(',')]) logging.debug(f' utm_zones: {utm_zones}') except KeyError: utm_zones = [] logging.debug(' utm_zones: not set in INI, defaulting to []') except Exception as e: raise e try: fill_with_climo_flag = str(ini['EXPORT']['fill_with_climo']) if fill_with_climo_flag.lower() in ['t', 'true']: fill_with_climo_flag = True else: fill_with_climo_flag = False except KeyError: fill_with_climo_flag = False logging.debug(' fill_with_climo: not set in INI, defaulting to False') except Exception as e: raise e # TODO: Add try/except blocks and default values? cloud_cover = float(ini['INPUTS']['cloud_cover']) # Model specific parameters # Set the property name to lower case and try to cast values to numbers model_args = { k.lower(): float(v) if utils.is_number(v) else v for k, v in dict(ini[model_name]).items()} filter_args = {} # TODO: Add try/except blocks tmax_source = ini[model_name]['tmax_source'] tcorr_source = ini[model_name]['tcorr_source'] tcorr_scene_coll_id = '{}'.format(ini['EXPORT']['export_coll']) # tcorr_scene_coll_id = '{}/{}_scene'.format( # ini['EXPORT']['export_coll'], tmax_source.lower()) tcorr_month_coll_id = f'{tcorr_scene_coll_id}_monthly' if tcorr_source.upper() not in ['GRIDDED_COLD', 'GRIDDED_COLD_1KM', 'GRIDDED']: raise ValueError('unsupported tcorr_source for these tools') # For now only support reading specific Tmax sources if (tmax_source.upper() not in ['DAYMET_MEDIAN_V2'] and not re.match('^projects/.+/tmax/.+_(mean|median)_\d{4}_\d{4}(_\w+)?', tmax_source)): raise ValueError(f'unsupported tmax_source: {tmax_source}') # if (tmax_source.upper() == 'CIMIS' and # ini['INPUTS']['end_date'] < '2003-10-01'): # raise ValueError('CIMIS is not currently available before 2003-10-01') # elif (tmax_source.upper() == 'DAYMET' and # ini['INPUTS']['end_date'] > '2018-12-31'): # logging.warning('\nDAYMET is not currently available past 2018-12-31, ' # 'using median Tmax values\n') # If the user set the tiles argument, use these instead of the INI values if tiles: logging.info('\nOverriding INI mgrs_tiles and utm_zones parameters') logging.info(f' user tiles: {tiles}') mgrs_tiles = sorted([y.strip() for x in tiles for y in x.split(',')]) mgrs_tiles = [x.upper() for x in mgrs_tiles if x] logging.info(f' mgrs_tiles: {", ".join(mgrs_tiles)}') utm_zones = sorted(list(set([int(x[:2]) for x in mgrs_tiles]))) logging.info(f' utm_zones: {", ".join(map(str, utm_zones))}') today_dt = datetime.datetime.now() today_dt = today_dt.replace(hour=0, minute=0, second=0, microsecond=0) if recent_days: logging.info('\nOverriding INI "start_date" and "end_date" parameters') logging.info(f' Recent days: {recent_days}') end_dt = today_dt - datetime.timedelta(days=1) start_dt = today_dt - datetime.timedelta(days=recent_days) start_date = start_dt.strftime('%Y-%m-%d') end_date = end_dt.strftime('%Y-%m-%d') elif start_dt and end_dt: # Attempt to use the function start/end dates logging.info('\nOverriding INI "start_date" and "end_date" parameters') logging.info(' Custom date range') start_date = start_dt.strftime('%Y-%m-%d') end_date = end_dt.strftime('%Y-%m-%d') else: # Parse the INI start/end dates logging.info('\nINI date range') try: start_dt = datetime.datetime.strptime(start_date, '%Y-%m-%d') end_dt = datetime.datetime.strptime(end_date, '%Y-%m-%d') except Exception as e: raise e logging.info(f' Start: {start_date}') logging.info(f' End: {end_date}') # TODO: Add a few more checks on the dates if end_dt < start_dt: raise ValueError('end date can not be before start date') # logging.debug('\nInterpolation date range') # iter_start_dt = start_dt # iter_end_dt = end_dt + datetime.timedelta(days=1) # iter_start_dt = start_dt - datetime.timedelta(days=interp_days) # iter_end_dt = end_dt + datetime.timedelta(days=interp_days+1) # logging.debug(' Start: {}'.format(iter_start_dt.strftime('%Y-%m-%d'))) # logging.debug(' End: {}'.format(iter_end_dt.strftime('%Y-%m-%d'))) logging.info('\nInitializing Earth Engine') if gee_key_file: logging.info(f' Using service account key file: {gee_key_file}') # The "EE_ACCOUNT" parameter is not used if the key file is valid ee.Initialize(ee.ServiceAccountCredentials('x', key_file=gee_key_file)) else: ee.Initialize() logging.debug('\nTmax properties') logging.debug(f' Source: {tmax_source}') # # DEADBEEF - Not needed with gridded Tcorr # # Get a Tmax image to set the Tcorr values to # if 'MEDIAN' in tmax_name.upper(): # tmax_coll_id = 'projects/earthengine-legacy/assets/' \ # 'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower()) # tmax_coll = ee.ImageCollection(tmax_coll_id) # tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0) # # else: # # raise ValueError(f'unsupported tmax_source: {tmax_name}') # logging.debug(f' Collection: {tmax_coll_id}') # Build output collection and folder if necessary logging.debug(f'\nExport Collection: {tcorr_scene_coll_id}') if not ee.data.getInfo(tcorr_scene_coll_id.rsplit('/', 1)[0]): logging.info('\nExport folder does not exist and will be built' '\n {}'.format(tcorr_scene_coll_id.rsplit('/', 1)[0])) input('Press ENTER to continue') ee.data.createAsset({'type': 'FOLDER'}, tcorr_scene_coll_id.rsplit('/', 1)[0]) if not ee.data.getInfo(tcorr_scene_coll_id): logging.info('\nExport collection does not exist and will be built' '\n {}'.format(tcorr_scene_coll_id)) input('Press ENTER to continue') ee.data.createAsset({'type': 'IMAGE_COLLECTION'}, tcorr_scene_coll_id) # Get current running tasks if ready_task_max == -9999: # CGM - Getting the task list can take awhile so set ready tasks to # -9999 to skip requesting it. Only run this if you are going to # manually avoid running existing tasks. # TODO: Check if this should disable delay_task() or set the # ready_task_max to a large value tasks = {} ready_task_count = 0 else: logging.info('\nRequesting Task List') tasks = utils.get_ee_tasks() if logging.getLogger().getEffectiveLevel() == logging.DEBUG: utils.print_ee_tasks(tasks) input('ENTER') ready_task_count = sum(1 for t in tasks.values() if t['state'] == 'READY') # ready_task_count = len(tasks.keys()) logging.info(f' Tasks: {ready_task_count}') # CGM - I'm still not sure if it makes sense to hold here or after the # first task is started. ready_task_count = delay_task( delay_time=0, task_max=ready_task_max, task_count=ready_task_count) # # Get current running tasks # tasks = utils.get_ee_tasks() # ready_task_count = sum(1 for t in tasks.values() if t['state'] == 'READY') # # ready_task_count = delay_task(ready_task_count, delay_time, max_ready) # if logging.getLogger().getEffectiveLevel() == logging.DEBUG: # utils.print_ee_tasks(tasks) # input('ENTER') # DEADBEEF - The asset list will be retrieved before each WRS2 tile is processed # Get current asset list # logging.debug('\nGetting GEE asset list') # asset_list = utils.get_ee_assets(tcorr_scene_coll_id) # if logging.getLogger().getEffectiveLevel() == logging.DEBUG: # pprint.pprint(asset_list[:10]) # Get list of MGRS tiles that intersect the study area logging.debug('\nMGRS Tiles/Zones') export_list = mgrs_export_tiles( study_area_coll_id=study_area_coll_id, mgrs_coll_id=mgrs_ftr_coll_id, study_area_property=study_area_property, study_area_features=study_area_features, mgrs_tiles=mgrs_tiles, mgrs_skip_list=mgrs_skip_list, utm_zones=utm_zones, wrs2_tiles=wrs2_tiles, ) if not export_list: logging.error('\nEmpty export list, exiting') return False # pprint.pprint(export_list) # input('ENTER') # Build the complete/filtered WRS2 list wrs2_tile_list = list(set( wrs2 for tile_info in export_list for wrs2 in tile_info['wrs2_tiles'])) if wrs2_skip_list: wrs2_tile_list = [wrs2 for wrs2 in wrs2_tile_list if wrs2 not in wrs2_skip_list] if wrs2_path_skip_list: wrs2_tile_list = [wrs2 for wrs2 in wrs2_tile_list if int(wrs2[1:4]) not in wrs2_path_skip_list] if wrs2_row_skip_list: wrs2_tile_list = [wrs2 for wrs2 in wrs2_tile_list if int(wrs2[5:8]) not in wrs2_row_skip_list] wrs2_tile_list = sorted(wrs2_tile_list, reverse=not(reverse_flag)) wrs2_tile_count = len(wrs2_tile_list) # Process each WRS2 tile separately logging.info('\nImage Exports') for wrs2_i, wrs2_tile in enumerate(wrs2_tile_list): wrs2_path, wrs2_row = map(int, wrs2_tile_re.findall(wrs2_tile)[0]) logging.info('{} ({}/{})'.format(wrs2_tile, wrs2_i + 1, wrs2_tile_count)) for coll_id in collections: filter_args[coll_id] = [ {'type': 'equals', 'leftField': 'WRS_PATH', 'rightValue': wrs2_path}, {'type': 'equals', 'leftField': 'WRS_ROW', 'rightValue': wrs2_row}] # logging.debug(f' Filter Args: {filter_args}') # Build and merge the Landsat collections model_obj = ssebop.Collection( collections=collections, start_date=start_dt.strftime('%Y-%m-%d'), end_date=(end_dt + datetime.timedelta(days=1)).strftime('%Y-%m-%d'), cloud_cover_max=cloud_cover, geometry=ee.Geometry.Point(openet.core.wrs2.centroids[wrs2_tile]), model_args=model_args, filter_args=filter_args, ) landsat_coll = model_obj.overpass(variables=['ndvi']) # pprint.pprint(landsat_coll.aggregate_array('system:id').getInfo()) # input('ENTER') try: image_id_list = landsat_coll.aggregate_array('system:id').getInfo() except Exception as e: logging.warning(' Error getting image ID list, skipping tile') logging.debug(f' {e}') continue # Get list of existing images for the target tile logging.debug(' Getting GEE asset list') asset_coll = ee.ImageCollection(tcorr_scene_coll_id) \ .filterDate(start_dt.strftime('%Y-%m-%d'), (end_dt + datetime.timedelta(days=1)).strftime('%Y-%m-%d')) \ .filterMetadata('wrs2_tile', 'equals', wrs2_tile_fmt.format(wrs2_path, wrs2_row)) asset_props = {f'{tcorr_scene_coll_id}/{x["properties"]["system:index"]}': x['properties'] for x in utils.get_info(asset_coll)['features']} # asset_props = {x['id']: x['properties'] for x in assets_info['features']} # Sort image ID list by date image_id_list = sorted( image_id_list, key=lambda k: k.split('/')[-1].split('_')[-1], reverse=reverse_flag) # Sort by date for image_id in image_id_list: coll_id, scene_id = image_id.rsplit('/', 1) logging.info(f'{scene_id}') export_dt = datetime.datetime.strptime(scene_id.split('_')[-1], '%Y%m%d') export_date = export_dt.strftime('%Y-%m-%d') logging.debug(f' Date: {export_date}') export_id = export_id_fmt.format( product=tmax_source.split('/')[-1].lower(), scene_id=scene_id) logging.debug(f' Export ID: {export_id}') asset_id = asset_id_fmt.format( coll_id=tcorr_scene_coll_id, scene_id=scene_id) logging.debug(f' Collection: {os.path.dirname(asset_id)}') logging.debug(f' Image ID: {os.path.basename(asset_id)}') if update_flag: def version_number(version_str): return list(map(int, version_str.split('.'))) if export_id in tasks.keys(): logging.info(' Task already submitted, skipping') continue # In update mode only overwrite if the version is old if asset_props and asset_id in asset_props.keys(): model_ver = version_number(ssebop.__version__) asset_ver = version_number( asset_props[asset_id]['model_version']) if asset_ver < model_ver: logging.info(' Existing asset model version is old, ' 'removing') logging.debug(f' asset: {asset_ver}\n' f' model: {model_ver}') try: ee.data.deleteAsset(asset_id) except: logging.info(' Error removing asset, skipping') continue elif (('T1_RT_TOA' in asset_props[asset_id]['coll_id']) and ('T1_RT_TOA' not in image_id)): logging.info(' Existing asset is from realtime ' 'Landsat collection, removing') try: ee.data.deleteAsset(asset_id) except: logging.info(' Error removing asset, skipping') continue else: logging.debug(' Asset is up to date, skipping') continue elif overwrite_flag: if export_id in tasks.keys(): logging.info(' Task already submitted, cancelling') ee.data.cancelTask(tasks[export_id]['id']) # This is intentionally not an "elif" so that a task can be # cancelled and an existing image/file/asset can be removed if asset_props and asset_id in asset_props.keys(): logging.info(' Asset already exists, removing') ee.data.deleteAsset(asset_id) else: if export_id in tasks.keys(): logging.info(' Task already submitted, skipping') continue elif asset_props and asset_id in asset_props.keys(): logging.info(' Asset already exists, skipping') continue # Get the input image grid and spatial reference image_info = ee.Image(image_id).select([3]).getInfo() image_geo = image_info['bands'][0]['crs_transform'] image_crs = image_info['bands'][0]['crs'] image_shape = image_info['bands'][0]['dimensions'] # Transform format: [30, 0, 591285, 0, -30, 4256115] image_extent = [ image_geo[2], image_geo[5] + image_shape[1] * image_geo[4], image_geo[2] + image_shape[0] * image_geo[0], image_geo[5]] logging.debug(f' Image CRS: {image_crs}') logging.debug(f' Image Geo: {image_geo}') logging.debug(f' Image Extent: {image_extent}') logging.debug(f' Image Shape: {image_shape}') # Adjust the image extent to the coarse resolution grid # EXPORT_GEO = [5000, 0, 15, 0, -5000, 15] export_cs = EXPORT_GEO[0] export_extent = [ round(math.floor((image_extent[0] - EXPORT_GEO[2]) / export_cs) * export_cs + EXPORT_GEO[2], 8), round(math.floor((image_extent[1] - EXPORT_GEO[5]) / export_cs) * export_cs + EXPORT_GEO[5], 8), round(math.ceil((image_extent[2] - EXPORT_GEO[2]) / export_cs) * export_cs + EXPORT_GEO[2], 8), round(math.ceil((image_extent[3] - EXPORT_GEO[5]) / export_cs) * export_cs + EXPORT_GEO[5], 8), ] export_geo = [export_cs, 0, export_extent[0], 0, -export_cs, export_extent[3]] export_shape = [ int(abs(export_extent[2] - export_extent[0]) / EXPORT_GEO[0]), int(abs(export_extent[3] - export_extent[1]) / EXPORT_GEO[0])] logging.debug(f' Export CRS: {image_crs}') logging.debug(f' Export Geo: {export_geo}') logging.debug(f' Export Extent: {export_extent}') logging.debug(f' Export Shape: {export_shape}') # CGM - Why are we not using the from_image_id() method? # t_obj = ssebop.Image.from_image_id(ee.Image(image_id), **model_args) if coll_id.endswith('_L2'): t_obj = ssebop.Image.from_landsat_c2_sr( sr_image=ee.Image(image_id), cloudmask_args={'cirrus_flag': True, 'dilate_flag': True, 'shadow_flag': True, 'snow_flag': True}, **model_args) elif coll_id.endswith('_SR'): t_obj = ssebop.Image.from_landsat_c1_sr( ee.Image(image_id), **model_args) elif coll_id.endswith('_TOA'): t_obj = ssebop.Image.from_landsat_c1_toa( ee.Image(image_id), **model_args) else: raise ValueError('Could not determine Landsat type') # CGM - Intentionally not calling the tcorr method directly since # there may be compositing with climos or the scene average if tcorr_source == 'GRIDDED': tcorr_img = t_obj.tcorr_gridded elif tcorr_source == 'GRIDDED_COLD': tcorr_img = t_obj.tcorr_gridded_cold elif tcorr_source == 'GRIDDED_COLD_1KM': tcorr_img = t_obj.tcorr_gridded_cold_1km # tcorr_img = t_obj.tcorr # Replace masked tcorr images with climos # Note, If the month climo doesn't exist this will keep the # existing masked Tcorr image (we may want to change that) # Does the tcorr_coarse count need to be set to a value like 0? if fill_with_climo_flag and tcorr_source == 'GRIDDED_COLD': logging.debug(' Checking if monthly climo should be applied') tcorr_month_coll = ee.ImageCollection(tcorr_month_coll_id)\ .filterMetadata('wrs2_tile', 'equals', wrs2_tile)\ .filterMetadata('month', 'equals', export_dt.month)\ .select(['tcorr']) # Setting the quality to 0 here causes it to get masked below # We might want to have it actually be 0 instead tcorr_month_img = tcorr_month_coll.first()\ .addBands([tcorr_month_coll.first().multiply(0).rename(['quality'])])\ .set({'tcorr_coarse_count': None}) tcorr_img = ee.Algorithms.If( ee.Number(tcorr_img.get('tcorr_index')).eq(9) .And(tcorr_month_coll.size().gt(0)), tcorr_month_img, tcorr_img) elif fill_with_climo_flag and tcorr_source == 'GRIDDED_COLD_1KM': logging.debug(' Checking if monthly climo should be applied') tcorr_month_coll = ee.ImageCollection(tcorr_month_coll_id) \ .filterMetadata('wrs2_tile', 'equals', wrs2_tile) \ .filterMetadata('month', 'equals', export_dt.month) \ .select(['tcorr']) # Setting the quality to 0 here causes it to get masked below # We might want to have it actually be 0 instead tcorr_month_img = tcorr_month_coll.first() \ .addBands([tcorr_month_coll.first().multiply(0).rename(['quality'])]) \ .set({'tcorr_coarse_count': None}) tcorr_img = ee.Algorithms.If( ee.Number(tcorr_img.get('tcorr_index')).eq(9) .And(tcorr_month_coll.size().gt(0)), tcorr_month_img, tcorr_img) # Clip to the Landsat image footprint tcorr_img = ee.Image(tcorr_img).clip(ee.Image(image_id).geometry()) # Clear the transparency mask (from clipping) tcorr_img = tcorr_img.updateMask(tcorr_img.unmask(0)) if clip_ocean_flag: tcorr_img = tcorr_img\ .updateMask(ee.Image('projects/openet/ocean_mask')) # # CGM - The NLCD mask will only work for CONUS # output_img = output_img.updateMask( # ee.Image('USGS/NLCD/NLCD2016').select(['landcover']).mask()) tcorr_img = tcorr_img\ .set({ 'CLOUD_COVER': image_info['properties']['CLOUD_COVER'], 'CLOUD_COVER_LAND': image_info['properties']['CLOUD_COVER_LAND'], # 'SPACECRAFT_ID': image.get('SPACECRAFT_ID'), 'coll_id': coll_id, 'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'), 'date': export_dt.strftime('%Y-%m-%d'), 'doy': int(export_dt.strftime('%j')), 'image_id': image_id, 'model_name': model_name, 'model_version': ssebop.__version__, 'month': int(export_dt.month), 'realtime': 'true' if '/T1_RT' in coll_id else 'false', 'scene_id': scene_id, 'system:time_start': image_info['properties']['system:time_start'], # 'tcorr_index': TCORR_INDICES[tcorr_source.upper()], 'tcorr_source': tcorr_source, 'tmax_source': tmax_source, # 'tmax_source': tmax_source.replace( # 'projects/earthengine-legacy/assets/', ''), 'tool_name': TOOL_NAME, 'tool_version': TOOL_VERSION, 'wrs2_path': wrs2_path, 'wrs2_row': wrs2_row, 'wrs2_tile': wrs2_tile_fmt.format(wrs2_path, wrs2_row), 'year': int(export_dt.year), }) # pprint.pprint(tcorr_img.getInfo()['properties']) # input('ENTER') logging.debug(' Building export task') task = ee.batch.Export.image.toAsset( image=tcorr_img, description=export_id, assetId=asset_id, crs=image_crs, crsTransform='[' + ','.join(list(map(str, export_geo))) + ']', dimensions='{0}x{1}'.format(*export_shape), ) logging.info(' Starting export task') utils.ee_task_start(task) ready_task_count += 1 # logging.debug(f' Ready tasks: {ready_task_count}') # Pause before starting the next date (not export task) ready_task_count = delay_task( delay_time=delay_time, task_max=ready_task_max, task_count=ready_task_count)
import ee from ee_plugin import Map # Define a region of interest as a point. Change the coordinates # to get a classification of any place where there is imagery. roi = ee.Geometry.Point(-122.3942, 37.7295) # Load Landsat 5 input imagery. landsat = ee.Image(ee.ImageCollection('LANDSAT/LT05/C01/T1_TOA') # Filter to get only one year of images. \ .filterDate('2011-01-01', '2011-12-31') # Filter to get only images under the region of interest. \ .filterBounds(roi) # Sort by scene cloudiness, ascending. \ .sort('CLOUD_COVER') # Get the first (least cloudy) scene. \ .first()) # Compute cloud score. cloudScore = ee.Algorithms.Landsat.simpleCloudScore(landsat).select('cloud') # Mask the input for clouds. Compute the min of the input mask to mask # pixels where any band is masked. Combine that with the cloud mask. input = landsat.updateMask(landsat.mask().reduce('min').And(cloudScore.lte(50))) # Use MODIS land cover, IGBP classification, for training. modis = ee.Image('MODIS/051/MCD12Q1/2011_01_01') \ .select('Land_Cover_Type_1') # Sample the input imagery to get a FeatureCollection of training data. training = input.addBands(modis).sample(**{
def addNDVI(image): return image.addBands(image.normalizedDifference(['B5', 'B4'])) # This function masks cloudy pixels. def cloudMask(image): clouds = ee.Algorithms.Landsat.simpleCloudScore(image).select(['cloud']) return image.updateMask(clouds.lt(10)) # Load a Landsat collection, map the NDVI and cloud masking functions over it. collection = ee.ImageCollection('LANDSAT/LC08/C01/T1_TOA') \ .filterBounds(ee.Geometry.Point([-122.262, 37.8719])) \ .filterDate('2014-03-01', '2014-05-31') \ .map(addNDVI) \ .map(cloudMask) # Reduce the collection to the mean of each pixel and display. meanImage = collection.reduce(ee.Reducer.mean()) vizParams = {'bands': ['B5_mean', 'B4_mean', 'B3_mean'], 'min': 0, 'max': 0.5} Map.setCenter(-122.262, 37.8719, 10) Map.addLayer(meanImage, vizParams, 'mean') # Load a region in which to compute the mean and display it. counties = ee.FeatureCollection('TIGER/2016/Counties') santaClara = ee.Feature( counties.filter(ee.Filter.eq('NAME', 'Santa Clara')).first()) Map.addLayer(ee.Image().paint(santaClara, 0, 2), {'palette': 'yellow'}, 'Santa Clara')
pdf = pdf.iloc[:48] X = pdf.values return X X = [] #create empty arrays Y = [] ee.Initialize() # %% get data 18/19 for l in range(df_1819.shape[0]): punto = df_1819.iloc[l] p = ee.Geometry.Point(punto.Longitud, punto.Latitud).buffer(100) S2_collection = ee.ImageCollection("COPERNICUS/S2") \ .filterBounds(p) \ .filterDate('2018-09-01', '2019-04-30') \ .sort('system:id', opt_ascending=True) \ p_get = p featCol = S2_collection.map(get_data) pdf = get_pd(featCol) X.append(get_X(pdf)) Y.append(punto.Cultivo) # %% get data 19/20 for l in range(df_1920.shape[0]): punto = df_1920.iloc[l] p = ee.Geometry.Point(punto.Longitud, punto.Latitud).buffer(100) S2_collection = ee.ImageCollection("COPERNICUS/S2") \ .filterBounds(p) \ .filterDate('2019-09-01', '2020-04-30') \
# Add a band containing image date as years since 1990. def createTimeBand(img): year = img.date().difference(ee.Date('1990-01-01'), 'year') return ee.Image(year).float().addBands(img) # function createTimeBand(img) { # year = img.date().difference(ee.Date('1990-01-01'), 'year') # return ee.Image(year).float().addBands(img) # } # Fit a linear trend to the nighttime lights collection. collection = ee.ImageCollection('NOAA/DMSP-OLS/CALIBRATED_LIGHTS_V4') \ .select('avg_vis') \ .map(createTimeBand) fit = collection.reduce(ee.Reducer.linearFit()) # Display a single image Map.addLayer(ee.Image(collection.select('avg_vis').first()), { 'min': 0, 'max': 63 }, 'stable lights first asset') # Display trend in red/blue, brightness in green. Map.setCenter(30, 45, 4) Map.addLayer(fit, { 'min': 0, 'max': [0.18, 20, -0.18], 'bands': ['scale', 'offset', 'scale']
def ts_extract(sensor, start, tiers=['T1', 'T2'], lon=None, lat=None, end=datetime.today(), radius=None, feature=None, bands=None, stats='mean'): """Perform a spatio temporal query to extract Landsat surface reflectance data from gee Args: lon (float): Center longitude in decimal degree lat (float): Center latitude in decimal degree sensor (str): Landsat sensor to query data from. Must be one of 'LT4', 'LT5', 'LE7', 'LC8' tiers (list): List of tiers to order. ``'T1'`` corresponds to tiers 1. Default is ``['T1', 'T2']`` start (datetime.datetime): Start date end (datetime.datetime): Optional end date; automatically set as today if unset radius (float): Optional radius around center point in meters. If unset, time-series of a single pixel are queried. Otherwise a reducer is used to spatially aggregate the pixels intersecting the circular feature built. feature (dict): Optional dictionary representation of a polygon feature in longlat CRS. If unset, time-series of a single pixel are queried. Otherwise a reducer is used to spatially aggregate the pixels intersecting the given feature. bands (list): List of Landsat band names. Optional, defaults to ['B1', 'B2', 'B3', 'B4', 'B5', 'B7'] in the case of LC8 sensor and to ['B1', 'B2', 'B3', 'B4', 'B5', 'B7'] otherwise. stats (str): Spatial aggregation function to use. Only relevant if a radius value is set. Returns: dict: A dictionary representation of the json data returned by the gee platform. Example: >>> import geextract >>> from pprint import pprint >>> from datetime import datetime >>> lon = -89.8107197 >>> lat = 20.4159611 >>> out = geextract.ts_extract(lon=lon, lat=lat, sensor='LE7', start=datetime(1980, 1, 1, 0, 0), >>> radius=500) >>> pprint(out) """ # Define some internal functions to be mapped over imageCollections def _mask_clouds(image): """Cloud masking function""" # collection 1 cloud masking example # https://code.earthengine.google.com/52e39cc00de3471c905509e374c52284 # Pre collecction masking example # https://code.earthengine.google.com/37ffd688d1b2d2c977fa5c536a023356 # collection must be a variable of the parent environment clear = image.select('pixel_qa').bitwiseAnd(0x2).neq(0) valid_range_mask = image.gte(0).And(image.lte(10000)) return image.updateMask(clear).updateMask(valid_range_mask) # Check inputs if sensor not in ['LT4', 'LT5', 'LC8', 'LE7']: raise ValueError('Unknown sensor (Must be one of LT4, LT5, LE7, LC8)') if bands is None: if sensor in ['LT4', 'LT5', 'LE7']: bands = ['B1', 'B2', 'B3', 'B4', 'B5', 'B7'] else: bands = ['B2', 'B3', 'B4', 'B5', 'B6', 'B7'] sensor = re.sub(r'(LC|LT|LE)(\d{1})', r'\g<1>0\g<2>', sensor) collection_name_template = 'LANDSAT/%s/C01/%%s_SR' % sensor # Iterate over tiers to load and merge all corresponding image collections landsat_ic = ee.ImageCollection(collection_name_template % tiers[0]) for tier in tiers[1:]: tier_ic = ee.ImageCollection(collection_name_template % tier) landsat_ic = ee.ImageCollection(landsat_ic.merge(tier_ic)) # Prepare image collection landsat = landsat_ic.\ filterDate(start=start, opt_end=end)\ .map(_mask_clouds)\ .select(bands) if radius is not None or feature is not None: # Define spatial aggregation function if stats == 'mean': fun = ee.Reducer.mean() elif stats == 'median': fun = ee.Reducer.median() elif stats == 'max': fun = ee.Reducer.max() elif stats == 'min': fun = ee.Reducer.min() else: raise ValueError( 'Unknown spatial aggregation function. Must be one of mean, median, max, or min' ) if feature is not None: geometry = ee.Geometry.Polygon(feature['geometry']['coordinates']) else: # Geometry defined by point and radius geometry = ee.Geometry.Point(lon, lat).buffer(radius) # Define function to map over imageCollection to perform spatial aggregation def _reduce_region(image): """Spatial aggregation function for a single image and a polygon feature""" stat_dict = image.reduceRegion(fun, geometry, 30) # FEature needs to be rebuilt because the backend doesn't accept to map # functions that return dictionaries return ee.Feature(None, stat_dict) fc = landsat.filterBounds(geometry).map(_reduce_region).getInfo() out = simplify(fc) else: # Extraction with a point, no spatial aggregation, etc geometry = ee.Geometry.Point(lon, lat) l = landsat.filterBounds(geometry).getRegion(geometry, 30).getInfo() out = dictify(l) # pop longitude and lattitude keys from dict collection so that band aliases can # be replaced by their color names [d.pop('longitude', None) for d in out] [d.pop('latitude', None) for d in out] [d.pop('time', None) for d in out] return out
def Get_Original_and_Fitting_df(self): # if the get_original_val is False, skip Fourier transformation if self.get_original_val: # Making the Fourier map self.Stp_1_Create_hamonic_names() self.Stp_2_Add_harmonics() self.Stp_3_Harmonic_fit() print('1/5 ==> Fourier Transformation Finished!') else: print('1/5 ==> Fourier Transformation skiped!') #__________________________Get the original [NDVI,NDBI,EVI] from harmonicLandsat_____________________ # if the get_original_val is False, skip Original Image Value acquiration if self.get_original_val: # get the necessary data original_value = self.harmonicLandsat.select( self.Normalized_Index).getRegion(self.point, 30).getInfo() # convert original value to dataframe original_df = pd.DataFrame(original_value[1:], columns=original_value[0]) original_df['time'] = pd.to_datetime(original_df['time'], unit='ms') # formating the df original_df.drop(['id', 'longitude', 'latitude'], 1, inplace=True) original_df.sort_values('time', inplace=True) original_df.reset_index(inplace=True, drop=True) # add attribute to self self.original_df = original_df print('2/5 ==> Original Image Value df Acquired!') else: print('2/5 ==> Original Image Value acquiration skiped!') #____________________________________Create indpendents_____________________________________________ independents = {} # set a list to store t date_list = ee.List.sequence(0, 365.25 * self.advance_year_num, 1) date_list = date_list.map( lambda x: ee.Date(self.start_date).advance(x, 'day')) # put the data to independents time_stamp = date_list.map(lambda x: ee.Date(x).format()).getInfo() independents.update({'time': [i[:10] for i in time_stamp]}) # Compute time in fractional years since the epoch. years = date_list.map(lambda x: ee.Date(x).difference( ee.Date('1970-01-01'), 'year').multiply(2 * 3.1415926)).getInfo() # add t to independents independents['t'] = np.array(years) # get other derivative time independents for i in range(1, self.harmonics + 1): # Get the cosine terms. cos = {f'cos_{i}': np.cos(np.array(years) * np.array(i))} # Get the sin terms. sin = {f'sin_{i}': np.sin(np.array(years) * np.array(i))} # add cos and sin bands to image independents.update(cos) independents.update(sin) # add constant to independents independents.update({'constant': 1}) independents = pd.DataFrame(independents) # add independents to self for check purpose self.independents = independents print('3/5 ==> Independents Values Computation Completed!') #___________________________Get the coefficients from Fourier img value________________________ # if the get_original_val is False, skip Fourier transformation if self.get_fitted_val: # if the year_range is not in the research periods, and the harmonic is not 3, # then do the calculation all again year_span = [f'{i}_{i+2}' for i in range(1990, 2018, 3)] if self.year_range in year_span[:5] and self.harmonics == 3: # import Fourier img print( '________________Using GEE assest to reduce time of Fourier transformation!__________' ) Fourier_img = ee.Image( f"users/Jinzhu_Deakin/North_China_Plain/Fourier_img_harmonic_3/Fourier_img_{self.year_range}_harmonic_3" ) elif self.year_range in year_span[5:] and self.harmonics == 3: # import Fourier img print( '________________Using GEE assest to reduce time of Fourier transformation!__________' ) Fourier_img = ee.Image( f"users/wang8052664/North_China_Plain/Fourier_imgs/Fourier_img_{self.year_range}_harmonic_3" ) else: # get the Fourier_img Fourier_img = self.harmonicTrendCoefficients # get the value of Foutire img at Fig_pt, #recall that the img was multipied by 1000 and converted into an integer Fig_pt_img_value = ee.ImageCollection(Fourier_img).getRegion( self.point, 30).getInfo() # get the coefficients and drop uesless columns coefficients = pd.DataFrame(Fig_pt_img_value[1:], columns=Fig_pt_img_value[0]) coefficients.drop(['id', 'longitude', 'latitude', 'time'], 1, inplace=True) coefficients = coefficients.T # divede by 1000, because the Fourier has been multiplied by 1000 at the Fourier fitting process coefficients = coefficients / 1000 # add coefficients to self for check purpose self.coefficients = coefficients print('4/5 ==> Coefficients Acquiring Finished!') #________________________Restore the fitted value using matrix multiply___________________________ fitted_df_list = [] # loop through indexes and use matrix multiply to get the fitted values for idx in self.Normalized_Index: X = independents[self.independents_idx].values y = coefficients.loc[[ s for s in coefficients.index if idx in s ]].values fitted_df_list.append( pd.DataFrame({idx: (X @ y).reshape(1, -1)[0]})) #_____________________________get the fitted value and add time to it_________________________________ fitted_df = pd.concat(fitted_df_list, 1) fitted_df['time'] = pd.to_datetime(independents['time']) self.fitted_df = fitted_df print('5/5 ==> Fitted Value Computation Completed!') else: print('5/5 ==> Fitted Value Computation Skipped!')