from pydrive.drive import GoogleDrive
from pydrive.auth import GoogleAuth
from ee import batch
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
latitude_val = float(sys.argv[1])
longitude_val = float(sys.argv[2])
manipulated_lat = latitude_val + 0.2
manipulated_lng = longitude_val + 0.2
ee.Initialize()
# Define the image collection we will be using
collection = ee.ImageCollection('LANDSAT/LC08/C01/T1_TOA')
# Creates a polygon which will be used as the bounds of the image
# Flagstaff Long: -111.6512, Lat: 35.1982
# region_polygon = ee.Geometry.Polygon(
# [[[-111.791491098128,35.150445194872816],
# [-111.49829346629207,35.150445194872816],
# [-111.49829346629207,35.385344417919924],
# [-111.791491098128,35.385344417919924],
# [-111.791491098128,35.150445194872816]]])
region_polygon = ee.Geometry.Polygon([[[longitude_val, latitude_val],
[manipulated_lng, latitude_val],
[manipulated_lng, manipulated_lat],
[longitude_val, manipulated_lat],
[longitude_val, latitude_val]]])
# GitHub URL: https://github.com/giswqs/qgis-earthengine-examples/tree/master/ImageCollection/sort_by_cloud_and_date.py
import ee
from ee_plugin import Map
# This function masks the input with a threshold on the simple cloud score.
def cloudMask(img):
cloudscore = ee.Algorithms.Landsat.simpleCloudScore(img).select('cloud')
return img.updateMask(cloudscore.lt(50))
# Load a Landsat 5 image collection.
collection = ee.ImageCollection('LANDSAT/LT5_L1T_TOA') \
.filterDate('2008-04-01', '2010-04-01') \
.filterBounds(ee.Geometry.Point(-122.2627, 37.8735)) \
.map(cloudMask) \
.select(['B4', 'B3']) \
.sort('system:time_start', True) #Sort the collection in chronological order.
print(collection.size().getInfo())
first = collection.first().get('system:id')
print(first.getInfo())
Map = emap.Map(center=[40,-100], zoom=4)
Map.add_basemap('ROADMAP') # Add Google Map
Map
# %%
"""
## Add Earth Engine Python script
"""
# %%
# Add Earth Engine dataset
roi = ee.Geometry.Point([-99.2182, 46.7824])
collection = ee.ImageCollection('USDA/NAIP/DOQQ') \
.filterBounds(roi) \
.filter(ee.Filter.listContains("system:band_names", "N"))
print(collection.size().getInfo())
first = collection.first()
Map.centerObject(first, 13)
Map.addLayer(first, {'bands': ['N', 'R', 'G']}, 'NAIP')
# %%
"""
## Display Earth Engine data layers
"""
# %%
Map.addLayerControl() # This line is not needed for ipyleaflet-based Map.
Map
months_min = month - MONTHS_OFFSET
if months_min < 1:
months_min = 1
months_max = month + MONTHS_OFFSET
if months_max > 12:
months_max = 12
# --------------------------------------------------
# IMPORT ImageCollections
# --------------------------------------------------
imgCol_L5_SR = ee.ImageCollection('LANDSAT/LT05/C01/T1_SR') \
.filterBounds(ROI) \
.filter(ee.Filter.calendarRange(year_min, year_max, 'year')) \
.filter(ee.Filter.calendarRange(months_min, months_max, 'month')) \
.filter(ee.Filter.lt('CLOUD_COVER_LAND', CLOUD_COVER)) \
.map(prepro.rename_bands_l5) \
.map(prepro.mask_landsat_sr(bits)) \
.map(prepro.scale_img(0.0001, ['B', 'G', 'R', 'NIR', 'SWIR1', 'SWIR2'], ['TIR'])) \
.map(prepro.scale_img(0.1, ['TIR'], ['B', 'G', 'R', 'NIR', 'SWIR1', 'SWIR2']))
imgCol_L7_SR = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR') \
.filterBounds(ROI) \
.filter(ee.Filter.calendarRange(year_min, year_max, 'year')) \
.filter(ee.Filter.calendarRange(months_min, months_max, 'month')) \
.filter(ee.Filter.lt('CLOUD_COVER_LAND', CLOUD_COVER)) \
.map(prepro.rename_bands_l7) \
.map(prepro.mask_landsat_sr(bits)) \
.map(prepro.scale_img(0.0001, ['B', 'G', 'R', 'NIR', 'SWIR1', 'SWIR2'], ['TIR'])) \
.map(prepro.scale_img(0.1, ['TIR'], ['B', 'G', 'R', 'NIR', 'SWIR1', 'SWIR2']))
def whittakerSmoothing(imageCollection, isCompositional=False, lamb=5):
# quick configs to set defaults
def toFl(image):
return image.toFloat()
# procedure start
ic = imageCollection.map(toFl)
dimension = ic.size()
identity_mat = ee.Array.identity(dimension)
difference_mat = getDifferenceMatrix(identity_mat, 3)
difference_mat_transpose = difference_mat.transpose()
lamda_difference_mat = difference_mat_transpose.multiply(lamb)
res_mat = lamda_difference_mat.matrixMultiply(difference_mat)
hat_matrix = res_mat.add(identity_mat)
# backing up original data
original = ic
def getProperties(image):
return ee.Image(image).toDictionary()
# get original image properties
properties = ic.toList(10000).map(getProperties)
# if data is compositional
# calculate the logratio of an image between 0 and 100. First
# clamps between delta and 100-delta, where delta is a small positive value.
if (isCompositional):
def clampImage(image):
delta = 0.001
bands = image.bandNames()
image = image.clamp(delta, 100 - delta)
image = (ee.Image.constant(100).subtract(image)
).divide(image).log().rename(bands)
return image
ic = ic.map(clampImage)
arrayImage = original.toArray()
coeffimage = ee.Image(hat_matrix).updateMask(arrayImage.mask())
smoothImage = coeffimage.matrixSolve(
arrayImage.unmask(hat_matrix.multiply(0)))
def getImageId(image):
return ee.Image(image).id()
idlist = ic.toList(10000).map(getImageId)
bandlist = ee.Image(ic.first()).bandNames()
flatImage = smoothImage.arrayFlatten([idlist, bandlist])
smoothCollection = ee.ImageCollection(unpack(flatImage, idlist, bandlist))
if (isCompositional):
smoothCollection = smoothCollection.map(inverseLogRatio)
def addSuffix(band):
return ee.String(band).cat('_fitted')
# get new band names by adding suffix fitted
newBandNames = bandlist.map(addSuffix)
# rename the bands in smoothened images
smoothCollection = smoothCollection.select(bandlist, newBandNames)
# a really dumb way to loose the google earth engine generated ID so that the two
# images can be combined for the chart
dumbimg = arrayImage.arrayFlatten([idlist, bandlist])
dumbcoll = ee.ImageCollection(unpack(dumbimg, idlist, bandlist))
outCollection = dumbcoll.combine(smoothCollection)
outCollList = outCollection.toList(10000)
def addPropBack(image):
return ee.Image(image).set(properties.get(outCollList.indexOf(image)))
outCollectionProp = outCollList.map(addPropBack)
residue_sq = smoothImage.subtract(arrayImage).pow(
ee.Image(2)).divide(dimension)
rmse_array = residue_sq.arrayReduce(ee.Reducer.sum(),
[0]).pow(ee.Image(1 / 2))
rmseImage = rmse_array.arrayFlatten([["rmse"], bandlist])
return (ee.ImageCollection(outCollectionProp), rmseImage)
def vegET_model(daily_imageColl, bbox):
"""
Calculate Daily Soil Water Index (SWI)
:param start_date: ee.Date
First date for analysis. Used to calculate initial SWI and then removed from collection.
:param daily_imageColl: ee.ImageCollection
Collection of daily images with bands for ndvi, precip, pet, canopy intercept
:param whc_grid_img: ee.Image
Static water holding capacity image
:return: ee.ImageCollection
imageCollection of daily Soil Water Index
"""
# TODO: Verify if these should be user inputs
# Define constant variables
VARA = ee.Number(1.25)
VARB = ee.Number(0.2)
# Define whc_grid
whc_grid_img = ee.Image(daily_imageColl.first().select('whc'))
# Drainage Coefficient (ee.Image to add as band for .expression())
dc_coeff = ee.Image(0.65)
# rf coefficient (ee.Image to add as band for .expression())
rf_coeff = ee.Image(1.0).subtract(dc_coeff)
# Calculate initial values where necessary
init_effppt = eff_intercept_precip(daily_imageColl.first())
initial_images = init_image_create(daily_imageColl, whc_grid_img,
init_effppt.select('effppt'))
# Create list for dynamic variables to be used in .iterate()
outputs_list = ee.List([initial_images])
# Create constant image for calculations in daily_vegET_calc()
const_img = ee.Image(1.0)
# TODO: Update docstring
def daily_vegET_calc(daily_img, outputs_list):
"""
Function to run imageCollection.iterate(). Takes latest value from outputs_list as previous
time-step SWI, current day whc_image and daily_image
:param swi_list:
:param whc_image:
:param daily_image:
:return:
"""
# Outputs from previous day as inputs to current day.
# NOTE: needs to be cast to list then image. see: https://developers.google.com/earth-engine/ic_iterating
prev_outputs = ee.Image(ee.List(outputs_list).get(-1))
# Calculate rain_frac
rain_frac = rain_frac_calc(daily_img, bbox)
# Calculate effective precipitation and intercepted precip
effective_precip = eff_intercept_precip(daily_img)
# Calculate amount of effective precipitation as rain
rain = rain_frac.multiply(
effective_precip.select('effppt')).rename('rain')
# TODO: Double check this is ok. Essentially skips first run as is in the esri version
# TODO: This will be combined with snowpack and swf at end of run to make new outputs_list append
swe = ee.Image(const_img.subtract(rain_frac)).multiply(
effective_precip.select('effppt')).rename('swe')
def melt_rate_calc(image):
"""
Calculate melt_rate
:param image: ee.Image
Image with max and min temp bands
:return: ee.Image
"""
melt_rate = image.expression(
'0.06 * ((tmax * tmax) - (tmax * tmin))', {
'tmax': image.select('tmaxC'),
'tmin': image.select('tminC')
})
melt_rate = melt_rate.set(
'system:time_start',
image.get('system:time_start')).rename('melt_rate')
return melt_rate
melt_rate = melt_rate_calc(daily_img)
# TODO: Update docstring
def snow_melt_calc(melt_rate_img, swe_current, prev_sw_image,
geometry):
"""
Calculate snow melt
:param melt_rate_img: ee.Image
Melt rate image
:param swe_current: ee.Image
Current time-step calculation of swe
:param prev_sw_image: ee.Image
Snowpack band from previous timestep list
:return: ee.Image
"""
# Combine images to allow for band selection in expression
snow_melt_img = melt_rate_img.addBands([
swe_current, prev_sw_image.select('snowpack')
]).rename(['melt_rate', 'swe', 'snowpack'])
snow_melt = snow_melt_img.expression(
"(b('melt_rate') <= (b('swe') + b('snowpack'))) ? (b('melt_rate'))"
+ ": (b('swe') + b('snowpack'))").clip(geometry).rename(
['snowmelt'])
return snow_melt
snow_melt = snow_melt_calc(melt_rate, swe,
prev_outputs.select('snowpack'), bbox)
def snowpack_calc(prev_sw_image, swe_current, snow_melt):
"""
Calculate snowpack
:param prev_sw_image: ee.Image
image with band for snowpack at previous timestep
:param swe_current: ee.Image
current swe
:param snow_melt: ee.Image
current snowmelt
:return: ee.Image
"""
snwpk1 = prev_sw_image.add(swe_current).subtract(snow_melt)
snowpack = snwpk1.where(snwpk1.lt(0.0), 0.0)
return snowpack
snowpack = snowpack_calc(prev_outputs.select('snowpack'), swe,
snow_melt)
swi_current = ee.Image(
prev_outputs.select('swf').add(rain).add(snow_melt)).rename('swi')
sat_fc = daily_img.select('soil_sat').subtract(
daily_img.select('fcap')).rename('sat_fc')
rf1 = swi_current.subtract(whc_grid_img).rename('rf1')
rf = rf1.where(rf1.lt(0.0), 0.0).rename('rf')
def srf_calc(rf_img, sat_fc_img, rf_coeff, geometry):
"""
Calculate srf
:param rf_img: ee.Image
image for rf
:param sat_fc_img: ee.Image
sat_fc_image
:param rf_coeff: ee.Image
rf coefficient
:return: ee.Image
"""
# add bands to make single image for .expression
srf_input_img = rf_img.addBands([sat_fc_img, rf_coeff]).rename(
['rf', 'sat_fc', 'rf_coeff'])
srf = srf_input_img.expression(
"(b('rf') <= b('sat_fc')) ? (b('rf') * b('rf_coeff'))" +
": (b('rf') - b('sat_fc')) + b('rf_coeff') * b('sat_fc')"
).clip(geometry).rename('srf')
return srf
srf = srf_calc(rf, sat_fc, rf_coeff, bbox)
# Deep drainage
ddrain = rf.subtract(srf).double().rename('ddrain')
# TODO: Verify if this is still needed
# def rfi_calc(image1, image2):
# """
# Calculate runoff as swi - whc
# :param image1: ee.Image
# Soil Water Index image
# :param image2: ee.Image
# Water holding capacity image
# :return: ee.Image
# Runoff as only positive valued image
# """
#
# rf = image1.subtract(image2)
#
# # Set value to 0 if rf < 0
# rfi = rf.where(rf.lt(0), 0)
#
# rfi = rfi.set('system:time_start', image1.get('system:time_start'))
#
# return ee.Image(rfi)
#
# rfi = rfi_calc(swi_current, whc_grid_img)
etasw1A = ee.Image(
daily_img.select('ndvi').multiply(VARA).add(VARB)).multiply(
daily_img.select('eto')).rename('etasw1A')
etasw1B = ee.Image(
daily_img.select('ndvi').multiply(VARA).multiply(
daily_img.select('eto'))).rename('etasw1B')
# TODO: consider ee.Algorithms.If() for conditional statements
# If ndvi is > 0.4, return etasw1A pixel value, otherwise, return etasw1B
etasw1 = etasw1B.where(daily_img.select('ndvi').gt(0.4),
etasw1A).rename('etasw1')
etasw2 = etasw1.multiply(swi_current.divide(
whc_grid_img.multiply(0.5))).rename('etasw2')
# use etasw1 pixel when swi_current > whc*0.5, use etasw2 pixel otherwise
etasw3 = etasw2.where(swi_current.gt(whc_grid_img.multiply(0.5)),
etasw1).rename('etasw3')
# use swi_current pixel where etasw3 > swi_current, use etasw3 pixel otherwise
etasw4 = etasw3.where(etasw3.gt(swi_current),
swi_current).rename('etasw4')
# use whc pixel where etasw4 is > whc, use etasw4 otherwise
etasw = etasw4.where(etasw4.gt(whc_grid_img),
whc_grid_img).rename('etasw')
swf1 = swi_current.subtract(etasw).rename('swf1')
bigswi = ee.Image(whc_grid_img.subtract(etasw)).rename('bigswi')
swf1_thresh = swf1.where(swf1.lt(0.0), 0.0).rename('swf_thresh')
swf = swf1_thresh.where(swi_current.gt(whc_grid_img),
bigswi).rename('swf')
# # TODO: This has been replaced by geetools addMultiBands
# def output_image_create(ref_img, swf_img, swe_img, snowpack_img):
# """
# Combine images to create single output image with multiple bands
# :param daily_img: ee.Image
# reference image for timestamp
# :param swf_img: ee.Image
# :param swe_img: ee.Image
# :param snowpack_img: ee.Image
# :return: ee.Image
# """
#
# output_img = swf_img.addBands([swe_img, snowpack_img]).rename(['swi', 'swe', 'snowpack']) \
# .set({
# 'system:index': ref_img.get('system:index'),
# 'system:time_start': ref_img.get('system:time_start')
# })
#
# return ee.Image(output_img)
#
# # Create output image
# output_image = output_image_create(daily_img, swf, swe, snowpack)
results = ee.Image(
utils.addMultiBands(daily_img, [
rain_frac, effective_precip, rain, swe, melt_rate, snow_melt,
snowpack, swi_current, sat_fc, rf1, rf, srf, ddrain, etasw1A,
etasw1B, etasw1, etasw2, etasw3, etasw4, etasw, swf1, bigswi,
swf1_thresh, swf
]))
return ee.List(outputs_list).add(results)
return ee.ImageCollection(
ee.List(daily_imageColl.iterate(daily_vegET_calc, outputs_list)))
import ee
import ee.mapclient
ee.Initialize()
collection = ee.ImageCollection('USDA/NAIP/DOQQ')
polys = ee.Geometry.Polygon([[[-99.29615020751953, 46.725459351792374],
[-99.2116928100586, 46.72404725733022],
[-99.21443939208984, 46.772037733479884],
[-99.30267333984375, 46.77321343419932]]])
centroid = polys.centroid()
lng, lat = centroid.getInfo()['coordinates']
print("lng = {}, lat = {}".format(lng, lat))
lng_lat = ee.Geometry.Point(lng, lat)
naip = collection.filterBounds(polys)
naip_2015 = naip.filterDate('2015-01-01', '2015-12-31')
ppr = naip_2015.mosaic()
count = naip_2015.size().getInfo()
print("Count: ", count)
# print(naip_2015.size().getInfo())
# vis = {'bands': ['N', 'R', 'G']}
# ee.mapclient.centerMap(lng, lat, 12)
# ee.mapclient.addToMap(ppr,vis)
# ee.mapclient.addToMap(polys)
downConfig = {
def getS2collection():
s2 = ee.ImageCollection('COPERNICUS/S2') \
.filterBounds(poly) \
.filterDate(ee.Date(w_startdate.value),ee.Date(w_enddate.value)) \
.sort('CLOUDY_PIXEL_PERCENTAGE',True)
return s2.filter(ee.Filter.contains(rightValue=poly, leftField='.geo'))
def on_collect_button_clicked(b):
global result,collection,count,imList,poly,timestamplist1,timestamps2, \
s2_image,rons,mean_incidence,collectionmosaic,collectionfirst,archive_crs,coords,wc
with w_out:
try:
if (w_collection.value
== 'COPERNICUS/S1_GRD') or (w_collection.value == ''):
w_out.clear_output()
print(
'running on GEE archive COPERNICUS/S1_GRD (please wait for raster overlay) ...'
)
# coords = ee.List(poly.bounds().coordinates().get(0))
collection = getS1collection()
if w_relativeorbitnumber.value > 0:
collection = collection.filter(
ee.Filter.eq('relativeOrbitNumber_start',
int(w_relativeorbitnumber.value)))
if w_platform.value != 'Both':
collection = collection.filter(
ee.Filter.eq('platform_number', w_platform.value))
collection = collection.sort('system:time_start')
acquisition_times = ee.List(
collection.aggregate_array('system:time_start')).getInfo()
count = len(acquisition_times)
if count < 2:
raise ValueError('Less than 2 images found')
timestamplist = []
for timestamp in acquisition_times:
tmp = time.gmtime(int(timestamp) / 1000)
timestamplist.append(time.strftime('%x', tmp))
# make timestamps in YYYYMMDD format
timestamplist = [x.replace('/', '') for x in timestamplist]
timestamplist = ['T20' + x[4:] + x[0:4] for x in timestamplist]
timestamplist = timestamplist[::int(w_stride.value)]
# in case of duplicates add running integer
timestamplist1 = [
timestamplist[i] + '_' + str(i + 1)
for i in range(len(timestamplist))
]
count = len(timestamplist)
if count < 2:
raise ValueError(
'Less than 2 images found, decrease stride')
relativeorbitnumbers = map(
int,
ee.List(
collection.aggregate_array(
'relativeOrbitNumber_start')).getInfo())
rons = list(set(relativeorbitnumbers))
print('Images found: %i, platform: %s' %
(count, w_platform.value))
print('Number of 10m pixels contained: %i' %
math.floor(poly.area().getInfo() / 100.0))
print('Acquisition dates: %s to %s' %
(str(timestamplist[0]), str(timestamplist[-1])))
print('Relative orbit numbers: ' + str(rons))
if len(rons) == 1:
mean_incidence = get_incidence_angle(collection.first())
print('Mean incidence angle: %f' % mean_incidence)
else:
mean_incidence = 'undefined'
print('Mean incidence angle: (select one rel. orbit)')
pcollection = collection.map(get_vvvh)
collectionfirst = ee.Image(pcollection.first())
w_exportscale.value = collectionfirst.projection(
).nominalScale().getInfo()
pList = pcollection.toList(500)
first = ee.Dictionary({
'imlist': ee.List([]),
'poly': poly,
'enl': ee.Number(w_enl.value),
'ctr': ee.Number(0),
'stride': ee.Number(int(w_stride.value))
})
imList = ee.List(
ee.Dictionary(pList.iterate(clipList,
first)).get('imlist'))
# get a vorschau as collection mean
collectionmosaic = collection.mosaic().select(0, 1).rename(
'b0', 'b1')
percentiles = collectionmosaic.reduceRegion(
ee.Reducer.percentile([2, 98]),
geometry=poly,
scale=w_exportscale.value,
maxPixels=10e9)
mn = ee.Number(percentiles.get('b0_p2'))
mx = ee.Number(percentiles.get('b0_p98'))
vorschau = collectionmosaic.select(0).visualize(
min=mn, max=mx, opacity=w_opacity.value)
else:
w_out.clear_output()
collection = ee.ImageCollection(w_collection.value)
print(
'running on local collection %s \n ignoring start and end dates (please wait for raster overlay) ...'
% w_collection.value)
count = collection.size().getInfo()
print('Images found: %i' % count)
collectionfirst = ee.Image(collection.first())
poly = collectionfirst.geometry()
# coords = ee.List(poly.bounds().coordinates().get(0))
center = poly.centroid().coordinates().getInfo()
center.reverse()
m.center = center
w_exportscale.value = collectionfirst.projection(
).nominalScale().getInfo()
if collectionfirst.get(
'system:time_start').getInfo() is not None:
acquisition_times = ee.List(
collection.aggregate_array(
'system:time_start')).getInfo()
timestamplist1 = []
for timestamp in acquisition_times:
tmp = time.gmtime(int(timestamp) / 1000)
timestamplist1.append(time.strftime('%x', tmp))
timestamplist1 = [
x.replace('/', '') for x in timestamplist1
]
timestamplist1 = [
'T20' + x[4:] + x[0:4] for x in timestamplist1
]
print('Acquisition dates: %s' % str(timestamplist1))
else:
timestamplist1 = ['T%i' % (i + 1) for i in range(count)]
print('No time property available: acquisitions: %s' %
str(timestamplist1))
# get a vorschau from collection mean
collectionmosaic = collection.mosaic().clip(poly)
percentiles = collectionmosaic.select(0).rename(
'b0').reduceRegion(ee.Reducer.percentile([2, 98]),
scale=w_exportscale.value,
maxPixels=10e9)
mn = ee.Number(percentiles.get('b0_p2'))
mx = ee.Number(percentiles.get('b0_p98'))
vorschau = collectionmosaic.select(0).visualize(
min=mn, max=mx, opacity=w_opacity.value)
imList = collection.toList(100)
# get GEE S1 archive crs for eventual image series export
# archive_crs = ee.Image(getS1collection(coords).first()).select(0).projection().crs().getInfo()
archive_crs = ee.Image(getS1collection().first()).select(
0).projection().crs().getInfo()
# run the algorithm
result = omnibus(imList, w_significance.value, w_enl.value,
w_median.value)
w_preview.disabled = False
w_ENL.disabled = False
w_export_atsf.disabled = True
s2_image = None
# display collection or S2
if len(m.layers) > 3:
m.remove_layer(m.layers[3])
if w_S2.value:
# display sentinel-2 if available
collection2 = getS2collection()
count1 = collection2.size().getInfo()
if count1 > 0:
s2_image = ee.Image(collection2.first()).select(
['B2', 'B3', 'B4'])
percentiles = s2_image.reduceRegion(
ee.Reducer.percentile([2, 98]),
scale=w_exportscale.value,
maxPixels=10e9)
mn = percentiles.values(['B2_p2', 'B3_p2', 'B4_p2'])
mx = percentiles.values(['B2_p98', 'B3_p98', 'B4_p98'])
vorschau = s2_image.visualize(min=mn,
max=mx,
opacity=w_opacity.value)
timestamp = s2_image.get('system:time_start').getInfo()
timestamp = time.gmtime(int(timestamp) / 1000)
timestamp = time.strftime('%x', timestamp).replace('/', '')
timestamps2 = '20' + timestamp[4:] + timestamp[0:4]
print('Sentinel-2 from %s' % timestamps2)
w_export_s2.disabled = False
m.add_layer(TileLayer(url=GetTileLayerUrl(vorschau)))
except Exception as e:
print('Error: %s' % e)
score = score.min(rescale(img, 'img.red + img.green + img.blue', [0.2, 0.8]))
# Clouds are reasonably bright in all infrared bands.
score = score.min(
rescale(img, 'img.nir + img.swir1 + img.swir2', [0.3, 0.8]))
# Clouds are reasonably cool in temperature.
score = score.min(rescale(img, 'img.temp', [300, 290]))
# However, clouds are not snow.
ndsi = img.normalizedDifference(['green', 'swir1'])
return score.min(rescale(ndsi, 'img', [0.8, 0.6]))
# Filter the TOA collection to a time-range and add the cloudscore band.
collection = ee.ImageCollection('LANDSAT/LC08/C01/T1_TOA') \
.filterDate('2017-05-01', '2017-07-01')
def func_wwa(img):
# Invert the cloudscore so 1 is least cloudy, and rename the band.
score = cloudScore(img.select(LC8_BANDS, STD_NAMES))
score = ee.Image(1).subtract(score).select([0], ['cloudscore'])
return img.addBands(score) \
.map(func_wwa)
# Define visualization parameters for a True color image.
Map = emap.Map(center=[40, -100], zoom=4)
Map.add_basemap('ROADMAP') # Add Google Map
Map
# %%
"""
## Add Earth Engine Python script
"""
# %%
# Add Earth Engine dataset
roi = ee.FeatureCollection('TIGER/2018/States') \
.filter(ee.Filter.eq('STUSPS', 'ND'))
dataset = ee.ImageCollection('USDA/NAIP/DOQQ') \
.filter(ee.Filter.date('2016-01-01', '2017-12-31'))\
.filterBounds(roi)
TrueColor = dataset.select(['N', 'R', 'G']).mosaic()
TrueColorVis = {
'min': 0.0,
'max': 255.0,
}
# Map.setCenter(-73.9958, 40.7278, 15)
Map.centerObject(roi, 8)
Map.addLayer(TrueColor, TrueColorVis, 'True Color')
# %%
"""
## Display Earth Engine data layers
"""
# -*- coding: utf-8 -*-
"""
Created on Wed Jun 3 16:03:52 2020
@author: MEvans
"""
import ee
service_account = '*****@*****.**'
key = 'C:/Users/mevans/Downloads/acd-app-04a10fe38611.json'
credentials = ee.ServiceAccountCredentials(service_account, key)
ee.Initialize(credentials)
geometry = ee.Geometry.Point([-111.54765625, 35.4227332452855])
imageCollection = ee.ImageCollection("COPERNICUS/S2")
filtered = imageCollection.filterBounds(geometry)
test = ee.Image(filtered.first())
print(test.bandNames().getInfo())
[-149.4140625, 58.99531118795094],
[-145.810546875, 59.977005492196],
[-140.7568359375, 59.153403092050375],
[-140.90746584324359, 65.67611691802753],
[-140.888671875, 70.1851027549897],
[-146.162109375, 70.55417853776078],
[-154.248046875, 71.49703690095419],
[-160.3125, 71.24435551310674],
[-167.255859375, 68.97416358340674]]])
# geometries end -------------------------------------------------------------------------------------------------
all_samp = ee.FeatureCollection(
"users/masseyr44/shapefiles/all_samp_postbin_v8")
ls5 = ee.ImageCollection("LANDSAT/LT05/C01/T1_SR")
ls7 = ee.ImageCollection("LANDSAT/LE07/C01/T1_SR")
ls8 = ee.ImageCollection("LANDSAT/LC08/C01/T1_SR")
elevation = ee.Image('USGS/GMTED2010')
# collections end ----------------------------------------------------------------------------------------
elev_scale_factor = 10000
pctl = 50
index = 'NDVI'
unmask_val = -9999
startJulian1 = 90
endJulian1 = 165
startJulian2 = 180
This step creates an interactive map using [folium](https://github.com/python-visualization/folium). The default basemap is the OpenStreetMap. Additional basemaps can be added using the `Map.setOptions()` function.
The optional basemaps can be `ROADMAP`, `SATELLITE`, `HYBRID`, `TERRAIN`, or `ESRI`.
'''
# %%
Map = folium.Map(location=[40, -100], zoom_start=4)
Map.setOptions('HYBRID')
# %%
'''
## Add Earth Engine Python script
'''
# %%
dataset = ee.ImageCollection('NRCan/CDEM')
elevation = dataset.select('elevation').mosaic()
elevationVis = {
'min': -50.0,
'max': 1500.0,
'palette': ['0905ff', 'ffefc4', 'ffffff'],
}
Map.setCenter(-139.3643, 63.3213, 9)
Map.addLayer(elevation, elevationVis, 'Elevation')
# %%
'''
## Display Earth Engine data layers
'''
def p_restrend(year_start, year_end, ndvi_1yr, climate_1yr, logger):
logger.debug("Entering p_restrend function.")
def f_img_coll(ndvi_stack):
img_coll = ee.List([])
for k in range(year_start, year_end + 1):
ndvi_img = ndvi_stack.select('y{}'.format(k))\
.addBands(climate_1yr.select('y{}'.format(k)))\
.rename(['ndvi', 'clim']).set({'year': k})
img_coll = img_coll.add(ndvi_img)
return ee.ImageCollection(img_coll)
## Function to predict NDVI from climate
first = ee.List([])
def f_ndvi_clim_p(image, list):
ndvi = lf_clim_ndvi.select('offset').add(
(lf_clim_ndvi.select('scale').multiply(image))).set(
{'year': image.get('year')})
return ee.List(list).add(ndvi)
## Function to compute residuals (ndvi obs - ndvi pred)
def f_ndvi_clim_r_img(year):
ndvi_o = ndvi_1yr_coll.filter(ee.Filter.eq(
'year', year)).select('ndvi').median()
ndvi_p = ndvi_1yr_p.filter(ee.Filter.eq('year', year)).median()
ndvi_r = ee.Image(year).float().addBands(ndvi_o.subtract(ndvi_p))
return ndvi_r.rename(['year', 'ndvi_res'])
# Function to compute differences between observed and predicted NDVI and compilation in an image collection
def stack(year_start, year_end):
img_coll = ee.List([])
for k in range(year_start, year_end + 1):
ndvi = ndvi_1yr_o.filter(ee.Filter.eq('year',
k)).select('ndvi').median()
clim = clim_1yr_o.filter(ee.Filter.eq('year',
k)).select('ndvi').median()
img = ndvi.addBands(clim.addBands(ee.Image(k).float())).rename(
['ndvi', 'clim', 'year']).set({'year': k})
img_coll = img_coll.add(img)
return ee.ImageCollection(img_coll)
## Function create image collection of residuals
def f_ndvi_clim_r_coll(year_start, year_end):
res_list = ee.List([])
#for(i = year_start i <= year_end i += 1):
for i in range(year_start, year_end + 1):
res_image = f_ndvi_clim_r_img(i)
res_list = res_list.add(res_image)
return ee.ImageCollection(res_list)
## Apply function to create image collection of ndvi and climate
ndvi_1yr_coll = f_img_coll(ndvi_1yr)
## Compute linear trend function to predict ndvi based on climate (independent are followed by dependent var
lf_clim_ndvi = ndvi_1yr_coll.select(['clim', 'ndvi'
]).reduce(ee.Reducer.linearFit())
## Apply function to predict NDVI based on climate
ndvi_1yr_p = ee.ImageCollection(
ee.List(ndvi_1yr_coll.select('clim').iterate(f_ndvi_clim_p, first)))
## Apply function to compute NDVI annual residuals
ndvi_1yr_r = f_ndvi_clim_r_coll(year_start, year_end)
## Fit a linear regression to the NDVI residuals
lf_trend = ndvi_1yr_r.select(['year',
'ndvi_res']).reduce(ee.Reducer.linearFit())
## Compute Kendall statistics
mk_trend = stats.mann_kendall(ndvi_1yr_r.select('ndvi_res'))
return (lf_trend, mk_trend)
def get_data(begin, end, aoi, stat, season='yearly'):
"""
Function to gather all the spectral imagery to be analysed.
:param begin: Begin date of the period for which the analysis is to be done
:param end: End date of the period for which the analysis is to be done
:param aoi: Area of interest
:param season: Season to be analysed, currently supports winter, summer or the whole year.
:return: Image containing all the spectral indexes added as bands.
"""
# Calculate the slope for the area of interest
elevation = ee.Image('JAXA/ALOS/AW3D30/V2_2').select('AVE_DSM')
slope = ee.Terrain.slope(elevation).clip(aoi).rename('slope')
# Select Landsat imagery for the period of interest and join the collections
ls_5 = landsat.get_ls5_image_collection(begin, end, aoi)
ls_7 = landsat.get_ls7_image_collection(begin, end, aoi)
ls = ls_5.merge(ls_7)
# Convert the RGB bands in the landsat collection to Hue, Saturation and Value
ls_hsv = ls.map(landsat.rgb_to_hsv)
# Calculate spectral indices to be analysed.
ls_ndvi = ls.map(landsat.add_ndvi_ls457).filter(
ee.Filter.listContains('system:band_names', 'NDVI'))
ls_gcvi = ls.map(landsat.add_gcvi_ls457).filter(
ee.Filter.listContains('system:band_names', 'GCVI'))
ls_ndwi = ls.map(landsat.add_ndwi_ls457).filter(
ee.Filter.listContains('system:band_names', 'NDWI'))
ls_ndbi = ls_ndvi.map(landsat.add_ndbi_ls457).filter(
ee.Filter.listContains('system:band_names', 'NDBI'))
ls_bu = ls_ndbi.map(landsat.add_bu_ls457).filter(
ee.Filter.listContains('system:band_names', 'BU'))
ls_pr = landsat.join_precipitation(ls_ndvi, begin, end, aoi).filter(
ee.Filter.listContains('system:band_names', 'pr'))
ls_wgi = ls_gcvi.map(landsat.add_ndwi_ls457).filter(ee.Filter.listContains('system:band_names', 'NDWI'))\
.map(landsat.add_wgi_ls457).filter(ee.Filter.listContains('system:band_names', 'WGI'))
ls_ndwi_greenhouses = ls.map(landsat.add_ndwi_mcfeeters_ls457)\
.filter(ee.Filter.listContains('system:band_names', 'NDWIGH'))
# # Calculate the monthly values per spectral index
ls_monthly_hue = landsat.create_monthly_index_images(
image_collection=ls_hsv,
band='hue',
start_date=begin,
end_date=end,
aoi=aoi,
stats=['mean', 'median'])
ls_monthly_saturation = landsat.create_monthly_index_images(
image_collection=ls_hsv,
band='saturation',
start_date=begin,
end_date=end,
aoi=aoi,
stats=['mean', 'median'])
ls_monthly_value = landsat.create_monthly_index_images(
image_collection=ls_hsv,
band='value',
start_date=begin,
end_date=end,
aoi=aoi,
stats=['mean', 'median'])
ls_monthly_blue = landsat.create_monthly_index_images(
image_collection=ls,
band='B1',
start_date=begin,
end_date=end,
aoi=aoi,
stats=['mean', 'median'])
ls_monthly_green = landsat.create_monthly_index_images(
image_collection=ls,
band='B2',
start_date=begin,
end_date=end,
aoi=aoi,
stats=['mean', 'median'])
ls_monthly_red = landsat.create_monthly_index_images(
image_collection=ls,
band='B3',
start_date=begin,
end_date=end,
aoi=aoi,
stats=['mean', 'median'])
ls_monthly_ndvi = landsat.create_monthly_index_images(
image_collection=ls_ndvi,
band='NDVI',
start_date=begin,
end_date=end,
aoi=aoi,
stats=['mean', 'median'])
ls_monthly_gcvi = landsat.create_monthly_index_images(
image_collection=ls_gcvi,
band='GCVI',
start_date=begin,
end_date=end,
aoi=aoi,
stats=['mean', 'median'])
ls_monthly_ndwi = landsat.create_monthly_index_images(
image_collection=ls_ndwi,
band='NDWI',
start_date=begin,
end_date=end,
aoi=aoi,
stats=['mean', 'median'])
ls_monthly_wgi = landsat.create_monthly_index_images(
image_collection=ls_wgi,
band='WGI',
start_date=begin,
end_date=end,
aoi=aoi,
stats=['mean', 'median'])
ls_monthly_ndbi = landsat.create_monthly_index_images(
image_collection=ls_ndbi,
band='NDBI',
start_date=begin,
end_date=end,
aoi=aoi,
stats=['mean', 'median'])
ls_monthly_bu = landsat.create_monthly_index_images(
image_collection=ls_bu,
band='BU',
start_date=begin,
end_date=end,
aoi=aoi,
stats=['mean', 'median'])
ls_monthly_ndvi_pr = ls_ndvi_pr = ls_pr.map(landsat.ndvi_precipitation_correction)\
.filter(ee.Filter.listContains('system:band_names', 'NDVI_pr')).select('NDVI_pr')
ls_monthly_ndwi_greenhouses = landsat.create_monthly_index_images(
image_collection=ls_ndwi_greenhouses,
band='NDWIGH',
start_date=begin,
end_date=end,
aoi=aoi,
stats=['mean', 'median'])
# Finally cobine all the data layers into a single image for the desired season
if season == 'yearly':
# Create images containing the mean monthly median and max value for the period on interest
ls_mean_monthly_mean_hue = ls_monthly_hue.select('mean').mean().rename(
'hue mean')
ls_mean_monthly_max_hue = ls_monthly_hue.select('mean').max().rename(
'hue max')
ls_mean_monthly_min_hue = ls_monthly_hue.select('mean').min().rename(
'hue min')
ls_mean_monthly_mean_saturation = ls_monthly_saturation.select(
'mean').mean().rename('saturation mean')
ls_mean_monthly_max_saturation = ls_monthly_saturation.select(
'mean').max().rename('saturation max')
ls_mean_monthly_min_saturation = ls_monthly_saturation.select(
'mean').min().rename('saturation min')
ls_mean_monthly_mean_value = ls_monthly_value.select(
'mean').mean().rename('value mean')
ls_mean_monthly_max_value = ls_monthly_value.select(
'mean').max().rename('value max')
ls_mean_monthly_min_value = ls_monthly_value.select(
'mean').min().rename('value min')
ls_mean_monthly_mean_blue = ls_monthly_blue.select(
'mean').mean().rename('blue mean')
ls_mean_monthly_max_blue = ls_monthly_blue.select('mean').max().rename(
'blue max')
ls_mean_monthly_min_blue = ls_monthly_blue.select('mean').min().rename(
'blue min')
ls_mean_monthly_mean_green = ls_monthly_green.select(
'mean').mean().rename('green mean')
ls_mean_monthly_max_green = ls_monthly_green.select(
'mean').max().rename('green max')
ls_mean_monthly_min_green = ls_monthly_green.select(
'mean').min().rename('green min')
ls_mean_monthly_mean_red = ls_monthly_red.select('mean').mean().rename(
'red mean')
ls_mean_monthly_max_red = ls_monthly_red.select('mean').max().rename(
'red max')
ls_mean_monthly_min_red = ls_monthly_red.select('mean').min().rename(
'red min')
ls_mean_monthly_mean_ndvi = ls_monthly_ndvi.select(
'mean').mean().rename('NDVI mean')
ls_mean_monthly_max_ndvi = ls_monthly_ndvi.select('mean').max().rename(
'NDVI max')
ls_mean_monthly_min_ndvi = ls_monthly_ndvi.select('mean').min().rename(
'NDVI min')
ls_mean_monthly_mean_gcvi = ls_monthly_gcvi.select(
'mean').mean().rename('GCVI mean')
ls_mean_monthly_max_gcvi = ls_monthly_gcvi.select('mean').max().rename(
'GCVI max')
ls_mean_monthly_min_gcvi = ls_monthly_gcvi.select('mean').min().rename(
'GCVI min')
ls_mean_monthly_mean_ndwi = ls_monthly_ndwi.select(
'mean').mean().rename('NDWI mean')
ls_mean_monthly_max_ndwi = ls_monthly_ndwi.select('mean').max().rename(
'NDWI max')
ls_mean_monthly_min_ndwi = ls_monthly_ndwi.select('mean').min().rename(
'NDWI min')
ls_mean_monthly_mean_wgi = ls_monthly_wgi.select('mean').mean().rename(
'WGI mean')
ls_mean_monthly_max_wgi = ls_monthly_wgi.select('mean').max().rename(
'WGI max')
ls_mean_monthly_min_wgi = ls_monthly_wgi.select('mean').min().rename(
'WGI min')
ls_mean_monthly_mean_ndwi_greenhouses = ls_monthly_ndwi_greenhouses.select(
'mean').mean().rename('NDWIGH mean')
ls_mean_monthly_max_ndwi_greenhouses = ls_monthly_ndwi_greenhouses.select(
'mean').max().rename('NDWIGH max')
ls_mean_monthly_min_ndwi_greenhouses = ls_monthly_ndwi_greenhouses.select(
'mean').min().rename('NDWIGH min')
# Next create standard deviation maps for both the seasonal and yearly NDWI and NDVI values.
yearly_std_ndvi = landsat.get_yearly_band_std(ls_monthly_ndvi, [stat],
begin, end, aoi)
yearly_std_ndwi = landsat.get_yearly_band_std(ls_monthly_ndwi, [stat],
begin, end, aoi)
yearly_std_gcvi = landsat.get_yearly_band_std(ls_monthly_gcvi, [stat],
begin, end, aoi)
yearly_std_wgi = landsat.get_yearly_band_std(ls_monthly_wgi, [stat],
begin, end, aoi)
# Take the mean for the periods that will be used during thresholding.
yearly_ndvi_std_mean = yearly_std_ndvi.select(f'{stat}_std').mean()
yearly_ndwi_std_mean = yearly_std_ndwi.select(f'{stat}_std').mean()
yearly_gcvi_std_mean = yearly_std_gcvi.select(f'{stat}_std').mean()
yearly_wgi_std_mean = yearly_std_wgi.select(f'{stat}_std').mean()
crop_data = ee.ImageCollection([
ls_mean_monthly_mean_hue,
ls_mean_monthly_max_hue,
ls_mean_monthly_min_hue,
ls_mean_monthly_mean_saturation,
ls_mean_monthly_max_saturation,
ls_mean_monthly_min_saturation,
ls_mean_monthly_mean_value,
ls_mean_monthly_max_value,
ls_mean_monthly_min_value,
ls_mean_monthly_mean_blue,
ls_mean_monthly_max_blue,
ls_mean_monthly_min_blue,
ls_mean_monthly_mean_green,
ls_mean_monthly_max_green,
ls_mean_monthly_min_green,
ls_mean_monthly_mean_red,
ls_mean_monthly_max_red,
ls_mean_monthly_min_red,
ls_mean_monthly_mean_ndvi,
ls_mean_monthly_max_ndvi,
ls_mean_monthly_min_ndvi,
ls_mean_monthly_mean_gcvi,
ls_mean_monthly_max_gcvi,
ls_mean_monthly_min_gcvi,
ls_mean_monthly_mean_ndwi,
ls_mean_monthly_max_ndwi,
ls_mean_monthly_min_ndwi,
ls_mean_monthly_mean_ndwi_greenhouses,
ls_mean_monthly_max_ndwi_greenhouses,
ls_mean_monthly_min_ndwi_greenhouses,
ls_mean_monthly_mean_wgi,
ls_mean_monthly_max_wgi,
ls_mean_monthly_min_wgi,
yearly_ndvi_std_mean.rename('NDVI_std'),
yearly_ndwi_std_mean.rename('NDWI_std'),
yearly_gcvi_std_mean.rename('GCVI_std'),
yearly_wgi_std_mean.rename('WGI_std'),
slope,
])
return crop_data.toBands().regexpRename('([0-9]*_)', '')
elif season == 'summer':
# # Create separate image collections containing only data for the summer months
ls_monthly_hue_summer = ls_monthly_hue.filter(
ee.Filter.rangeContains('month', 4, 9))
ls_monthly_saturation_summer = ls_monthly_saturation.filter(
ee.Filter.rangeContains('month', 4, 9))
ls_monthly_value_summer = ls_monthly_value.filter(
ee.Filter.rangeContains('month', 4, 9))
ls_monthly_blue_summer = ls_monthly_blue.filter(
ee.Filter.rangeContains('month', 4, 9))
ls_monthly_green_summer = ls_monthly_green.filter(
ee.Filter.rangeContains('month', 4, 9))
ls_monthly_red_summer = ls_monthly_red.filter(
ee.Filter.rangeContains('month', 4, 9))
ls_monthly_ndvi_summer = ls_monthly_ndvi.filter(
ee.Filter.rangeContains('month', 4, 9))
ls_monthly_gcvi_summer = ls_monthly_gcvi.filter(
ee.Filter.rangeContains('month', 4, 9))
ls_monthly_ndwi_summer = ls_monthly_ndwi.filter(
ee.Filter.rangeContains('month', 4, 9))
ls_monthly_wgi_summer = ls_monthly_wgi.filter(
ee.Filter.rangeContains('month', 4, 9))
ls_monthly_ndbi_summer = ls_monthly_ndbi.filter(
ee.Filter.rangeContains('month', 4, 9))
ls_monthly_bu_summer = ls_monthly_bu.filter(
ee.Filter.rangeContains('month', 4, 9))
ls_monthly_ndvi_pr_summer = ls_monthly_ndvi_pr.filter(
ee.Filter.rangeContains('month', 4, 9))
ls_monthly_ndwi_greenhouses_summer = ls_monthly_ndwi_greenhouses.filter(
ee.Filter.rangeContains('month', 4, 9))
# # Same principle only now for the summer months
ls_mean_monthly_mean_hue_summer = ls_monthly_hue_summer.select(
'mean').mean().rename('hue mean')
ls_mean_monthly_max_hue_summer = ls_monthly_hue_summer.select(
'mean').max().rename('hue max')
ls_mean_monthly_min_hue_summer = ls_monthly_hue_summer.select(
'mean').min().rename('hue min')
ls_mean_monthly_mean_saturation_summer = ls_monthly_saturation_summer.select(
'mean').mean().rename('saturation mean')
ls_mean_monthly_max_saturation_summer = ls_monthly_saturation_summer.select(
'mean').max().rename('saturation max')
ls_mean_monthly_min_saturation_summer = ls_monthly_saturation_summer.select(
'mean').min().rename('saturation min')
ls_mean_monthly_mean_value_summer = ls_monthly_value_summer.select(
'mean').mean().rename('value mean')
ls_mean_monthly_max_value_summer = ls_monthly_value_summer.select(
'mean').max().rename('value max')
ls_mean_monthly_min_value_summer = ls_monthly_value_summer.select(
'mean').min().rename('value min')
ls_mean_monthly_mean_blue_summer = ls_monthly_blue_summer.select(
'mean').mean().rename('blue mean')
ls_mean_monthly_max_blue_summer = ls_monthly_blue_summer.select(
'mean').max().rename('blue max')
ls_mean_monthly_min_blue_summer = ls_monthly_blue_summer.select(
'mean').min().rename('blue min')
ls_mean_monthly_mean_green_summer = ls_monthly_green_summer.select(
'mean').mean().rename('green mean')
ls_mean_monthly_max_green_summer = ls_monthly_green_summer.select(
'mean').max().rename('green max')
ls_mean_monthly_min_green_summer = ls_monthly_green_summer.select(
'mean').min().rename('green min')
ls_mean_monthly_mean_red_summer = ls_monthly_red_summer.select(
'mean').mean().rename('red mean')
ls_mean_monthly_max_red_summer = ls_monthly_red_summer.select(
'mean').max().rename('red max')
ls_mean_monthly_min_red_summer = ls_monthly_red_summer.select(
'mean').min().rename('red min')
ls_mean_monthly_mean_ndvi_summer = ls_monthly_ndvi_summer.select(
'mean').mean().rename('NDVI mean')
ls_mean_monthly_max_ndvi_summer = ls_monthly_ndvi_summer.select(
'mean').max().rename('NDVI max')
ls_mean_monthly_min_ndvi_summer = ls_monthly_ndvi_summer.select(
'mean').min().rename('NDVI min')
ls_mean_monthly_mean_gcvi_summer = ls_monthly_gcvi_summer.select(
'mean').mean().rename('GCVI mean')
ls_mean_monthly_max_gcvi_summer = ls_monthly_gcvi_summer.select(
'mean').max().rename('GCVI max')
ls_mean_monthly_min_gcvi_summer = ls_monthly_gcvi_summer.select(
'mean').min().rename('GCVI min')
ls_mean_monthly_mean_ndwi_summer = ls_monthly_ndwi_summer.select(
'mean').mean().rename('NDWI mean')
ls_mean_monthly_max_ndwi_summer = ls_monthly_ndwi_summer.select(
'mean').max().rename('NDWI max')
ls_mean_monthly_min_ndwi_summer = ls_monthly_ndwi_summer.select(
'mean').min().rename('NDWI min')
ls_mean_monthly_mean_wgi_summer = ls_monthly_wgi_summer.select(
'mean').mean().rename('WGI mean')
ls_mean_monthly_max_wgi_summer = ls_monthly_wgi_summer.select(
'mean').max().rename('WGI max')
ls_mean_monthly_min_wgi_summer = ls_monthly_wgi_summer.select(
'mean').min().rename('WGI min')
ls_mean_monthly_mean_ndbi_summer = ls_monthly_ndbi_summer.select(
'mean').mean().rename('NDBI mean')
ls_mean_monthly_max_ndbi_summer = ls_monthly_ndbi_summer.select(
'mean').max().rename('NDBI max')
ls_mean_monthly_min_ndbi_summer = ls_monthly_ndbi_summer.select(
'mean').min().rename('NDBI min')
ls_mean_monthly_mean_bu_summer = ls_monthly_bu_summer.select(
'mean').mean().rename('BU mean')
ls_mean_monthly_max_bu_summer = ls_monthly_bu_summer.select(
'mean').max().rename('BU max')
ls_mean_monthly_min_bu_summer = ls_monthly_bu_summer.select(
'mean').min().rename('BU min')
ls_mean_monthly_mean_ndvi_pr_summer = ls_monthly_ndvi_pr_summer.select(
'NDVI_pr').mean().rename('NDVI_pr mean')
ls_mean_monthly_max_ndvi_pr_summer = ls_monthly_ndvi_pr_summer.select(
'NDVI_pr').max().rename('NDVI_pr max')
ls_mean_monthly_min_ndvi_pr_summer = ls_monthly_ndvi_pr_summer.select(
'NDVI_pr').min().rename('NDVI_pr min')
ls_mean_monthly_mean_ndwi_greenhouses_summer = ls_monthly_ndwi_greenhouses_summer.select(
'mean').mean().rename('NDWIGH mean')
ls_mean_monthly_max_ndwi_greenhouses_summer = ls_monthly_ndwi_greenhouses_summer.select(
'mean').max().rename('NDWIGH max')
ls_mean_monthly_min_ndwi_greenhouses_summer = ls_monthly_ndwi_greenhouses_summer.select(
'mean').min().rename('NDWIGH min')
# Next create standard deviation maps for both the seasonal and yearly NDWI and NDVI values.
summer_std_ndvi = landsat.get_yearly_band_std(ls_monthly_ndvi, [stat],
begin,
end,
aoi,
season='summer')
summer_std_ndwi = landsat.get_yearly_band_std(ls_monthly_ndwi, [stat],
begin,
end,
aoi,
season='summer')
summer_std_gcvi = landsat.get_yearly_band_std(ls_monthly_gcvi, [stat],
begin,
end,
aoi,
season='summer')
summer_std_wgi = landsat.get_yearly_band_std(ls_monthly_wgi, [stat],
begin,
end,
aoi,
season='summer')
summer_ndvi_std_mean = summer_std_ndvi.select(f'{stat}_std').mean()
summer_ndwi_std_mean = summer_std_ndwi.select(f'{stat}_std').mean()
summer_gcvi_std_mean = summer_std_gcvi.select(f'{stat}_std').mean()
summer_wgi_std_mean = summer_std_wgi.select(f'{stat}_std').mean()
crop_data = ee.ImageCollection([
ls_mean_monthly_mean_hue_summer,
ls_mean_monthly_max_hue_summer,
ls_mean_monthly_min_hue_summer,
ls_mean_monthly_mean_saturation_summer,
ls_mean_monthly_max_saturation_summer,
ls_mean_monthly_min_saturation_summer,
ls_mean_monthly_mean_value_summer,
ls_mean_monthly_max_value_summer,
ls_mean_monthly_min_value_summer,
ls_mean_monthly_mean_blue_summer,
ls_mean_monthly_max_blue_summer,
ls_mean_monthly_min_blue_summer,
ls_mean_monthly_mean_green_summer,
ls_mean_monthly_max_green_summer,
ls_mean_monthly_min_green_summer,
ls_mean_monthly_mean_red_summer,
ls_mean_monthly_max_red_summer,
ls_mean_monthly_min_red_summer,
ls_mean_monthly_mean_ndvi_summer,
ls_mean_monthly_max_ndvi_summer,
ls_mean_monthly_min_ndvi_summer,
ls_mean_monthly_mean_gcvi_summer,
ls_mean_monthly_max_gcvi_summer,
ls_mean_monthly_min_gcvi_summer,
ls_mean_monthly_mean_ndwi_summer,
ls_mean_monthly_max_ndwi_summer,
ls_mean_monthly_min_ndwi_summer,
ls_mean_monthly_mean_wgi_summer,
ls_mean_monthly_max_wgi_summer,
ls_mean_monthly_min_wgi_summer,
ls_mean_monthly_mean_ndbi_summer,
ls_mean_monthly_max_ndbi_summer,
ls_mean_monthly_min_ndbi_summer,
ls_mean_monthly_mean_bu_summer,
ls_mean_monthly_max_bu_summer,
ls_mean_monthly_min_bu_summer,
ls_mean_monthly_mean_ndvi_pr_summer,
ls_mean_monthly_max_ndvi_pr_summer,
ls_mean_monthly_min_ndvi_pr_summer,
ls_mean_monthly_mean_ndwi_greenhouses_summer,
ls_mean_monthly_max_ndwi_greenhouses_summer,
ls_mean_monthly_min_ndwi_greenhouses_summer,
summer_ndvi_std_mean.rename('NDVI_std'),
summer_ndwi_std_mean.rename('NDWI_std'),
summer_gcvi_std_mean.rename('GCVI_std'),
summer_wgi_std_mean.rename('WGI_std'),
slope,
])
return crop_data.toBands().regexpRename('([0-9]*_)', '')
elif season == 'winter':
early_filter = ee.Filter.rangeContains('month', 1, 3)
late_filter = ee.Filter.rangeContains('month', 10, 12)
ls_monthly_hue_winter = ls_monthly_hue.filter(
ee.Filter.Or(early_filter, late_filter))
ls_monthly_saturation_winter = ls_monthly_saturation.filter(
ee.Filter.Or(early_filter, late_filter))
ls_monthly_value_winter = ls_monthly_value.filter(
ee.Filter.Or(early_filter, late_filter))
ls_monthly_blue_winter = ls_monthly_blue.filter(
ee.Filter.Or(early_filter, late_filter))
ls_monthly_green_winter = ls_monthly_green.filter(
ee.Filter.Or(early_filter, late_filter))
ls_monthly_red_winter = ls_monthly_red.filter(
ee.Filter.Or(early_filter, late_filter))
ls_monthly_ndvi_winter = ls_monthly_ndvi.filter(
ee.Filter.Or(early_filter, late_filter))
ls_monthly_gcvi_winter = ls_monthly_gcvi.filter(
ee.Filter.Or(early_filter, late_filter))
ls_monthly_ndwi_winter = ls_monthly_ndwi.filter(
ee.Filter.Or(early_filter, late_filter))
ls_monthly_wgi_winter = ls_monthly_wgi.filter(
ee.Filter.Or(early_filter, late_filter))
ls_monthly_ndbi_winter = ls_monthly_ndbi.filter(
ee.Filter.Or(early_filter, late_filter))
ls_monthly_bu_winter = ls_monthly_bu.filter(
ee.Filter.Or(early_filter, late_filter))
ls_monthly_ndvi_pr_winter = ls_monthly_ndvi_pr.filter(
ee.Filter.Or(early_filter, late_filter))
ls_monthly_ndwi_greenhouses_winter = ls_monthly_ndwi_greenhouses.filter(
ee.Filter.Or(early_filter, late_filter))
# And Winter
ls_mean_monthly_mean_hue_winter = ls_monthly_hue_winter.select(
'mean').mean().rename('hue mean')
ls_mean_monthly_max_hue_winter = ls_monthly_hue_winter.select(
'mean').max().rename('hue max')
ls_mean_monthly_min_hue_winter = ls_monthly_hue_winter.select(
'mean').min().rename('hue min')
ls_mean_monthly_mean_saturation_winter = ls_monthly_saturation_winter.select(
'mean').mean().rename('saturation mean')
ls_mean_monthly_max_saturation_winter = ls_monthly_saturation_winter.select(
'mean').max().rename('saturation max')
ls_mean_monthly_min_saturation_winter = ls_monthly_saturation_winter.select(
'mean').min().rename('saturation min')
ls_mean_monthly_mean_value_winter = ls_monthly_value_winter.select(
'mean').mean().rename('value mean')
ls_mean_monthly_max_value_winter = ls_monthly_value_winter.select(
'mean').max().rename('value max')
ls_mean_monthly_min_value_winter = ls_monthly_value_winter.select(
'mean').min().rename('value min')
ls_mean_monthly_mean_blue_winter = ls_monthly_blue_winter.select(
'mean').mean().rename('blue mean')
ls_mean_monthly_max_blue_winter = ls_monthly_blue_winter.select(
'mean').max().rename('blue max')
ls_mean_monthly_min_blue_winter = ls_monthly_blue_winter.select(
'mean').min().rename('blue min')
ls_mean_monthly_mean_green_winter = ls_monthly_green_winter.select(
'mean').mean().rename('green mean')
ls_mean_monthly_max_green_winter = ls_monthly_green_winter.select(
'mean').max().rename('green max')
ls_mean_monthly_min_green_winter = ls_monthly_green_winter.select(
'mean').min().rename('green min')
ls_mean_monthly_mean_red_winter = ls_monthly_red_winter.select(
'mean').mean().rename('red mean')
ls_mean_monthly_max_red_winter = ls_monthly_red_winter.select(
'mean').max().rename('red max')
ls_mean_monthly_min_red_winter = ls_monthly_red_winter.select(
'mean').min().rename('red min')
ls_mean_monthly_mean_ndvi_winter = ls_monthly_ndvi_winter.select(
'mean').mean().rename('NDVI mean')
ls_mean_monthly_max_ndvi_winter = ls_monthly_ndvi_winter.select(
'mean').max().rename('NDVI max')
ls_mean_monthly_min_ndvi_winter = ls_monthly_ndvi_winter.select(
'mean').min().rename('NDVI min')
ls_mean_monthly_mean_gcvi_winter = ls_monthly_gcvi_winter.select(
'mean').mean().rename('GCVI mean')
ls_mean_monthly_max_gcvi_winter = ls_monthly_gcvi_winter.select(
'mean').max().rename('GCVI max')
ls_mean_monthly_min_gcvi_winter = ls_monthly_gcvi_winter.select(
'mean').min().rename('GCVI min')
ls_mean_monthly_mean_ndwi_winter = ls_monthly_ndwi_winter.select(
'mean').mean().rename('NDWI mean')
ls_mean_monthly_max_ndwi_winter = ls_monthly_ndwi_winter.select(
'mean').max().rename('NDWI max')
ls_mean_monthly_min_ndwi_winter = ls_monthly_ndwi_winter.select(
'mean').min().rename('NDWI min')
ls_mean_monthly_mean_wgi_winter = ls_monthly_wgi_winter.select(
'mean').mean().rename('WGI mean')
ls_mean_monthly_max_wgi_winter = ls_monthly_wgi_winter.select(
'mean').max().rename('WGI max')
ls_mean_monthly_min_wgi_winter = ls_monthly_wgi_winter.select(
'mean').min().rename('WGI min')
ls_mean_monthly_mean_ndbi_winter = ls_monthly_ndbi_winter.select(
'mean').mean().rename('NDBI mean')
ls_mean_monthly_max_ndbi_winter = ls_monthly_ndbi_winter.select(
'mean').max().rename('NDBI max')
ls_mean_monthly_min_ndbi_winter = ls_monthly_ndbi_winter.select(
'mean').min().rename('NDBI min')
ls_mean_monthly_mean_bu_winter = ls_monthly_bu_winter.select(
'mean').mean().rename('BU mean')
ls_mean_monthly_max_bu_winter = ls_monthly_bu_winter.select(
'mean').max().rename('BU max')
ls_mean_monthly_min_bu_winter = ls_monthly_bu_winter.select(
'mean').min().rename('BU min')
ls_mean_monthly_mean_ndvi_pr_winter = ls_monthly_ndvi_pr_winter.select(
'NDVI_pr').mean().rename('NDVI_pr mean')
ls_mean_monthly_max_ndvi_pr_winter = ls_monthly_ndvi_pr_winter.select(
'NDVI_pr').max().rename('NDVI_pr max')
ls_mean_monthly_min_ndvi_pr_winter = ls_monthly_ndvi_pr_winter.select(
'NDVI_pr').min().rename('NDVI_pr min')
ls_mean_monthly_mean_ndwi_greenhouses_winter = ls_monthly_ndwi_greenhouses_winter.select(
'mean').mean().rename('NDWIGH mean')
ls_mean_monthly_max_ndwi_greenhouses_winter = ls_monthly_ndwi_greenhouses_winter.select(
'mean').max().rename('NDWIGH max')
ls_mean_monthly_min_ndwi_greenhouses_winter = ls_monthly_ndwi_greenhouses_winter.select(
'mean').min().rename('NDWIGH min')
# Next create standard deviation maps for both the seasonal and yearly NDWI and NDVI values.
winter_std_ndvi = landsat.get_yearly_band_std(ls_monthly_ndvi,
[f'{stat}'],
begin,
end,
aoi,
season='winter')
winter_std_ndwi = landsat.get_yearly_band_std(ls_monthly_ndwi,
[f'{stat}'],
begin,
end,
aoi,
season='winter')
winter_std_gcvi = landsat.get_yearly_band_std(ls_monthly_gcvi,
[f'{stat}'],
begin,
end,
aoi,
season='winter')
winter_std_wgi = landsat.get_yearly_band_std(ls_monthly_wgi,
[f'{stat}'],
begin,
end,
aoi,
season='winter')
winter_ndvi_std_mean = winter_std_ndvi.select(f'{stat}_std').mean()
winter_ndwi_std_mean = winter_std_ndwi.select(f'{stat}_std').mean()
winter_gcvi_std_mean = winter_std_gcvi.select(f'{stat}_std').mean()
winter_wgi_std_mean = winter_std_wgi.select(f'{stat}_std').mean()
crop_data = ee.ImageCollection([
ls_mean_monthly_mean_hue_winter,
ls_mean_monthly_max_hue_winter,
ls_mean_monthly_min_hue_winter,
ls_mean_monthly_mean_saturation_winter,
ls_mean_monthly_max_saturation_winter,
ls_mean_monthly_min_saturation_winter,
ls_mean_monthly_mean_value_winter,
ls_mean_monthly_max_value_winter,
ls_mean_monthly_min_value_winter,
ls_mean_monthly_mean_blue_winter,
ls_mean_monthly_max_blue_winter,
ls_mean_monthly_min_blue_winter,
ls_mean_monthly_mean_green_winter,
ls_mean_monthly_max_green_winter,
ls_mean_monthly_min_green_winter,
ls_mean_monthly_mean_red_winter,
ls_mean_monthly_max_red_winter,
ls_mean_monthly_min_red_winter,
ls_mean_monthly_mean_ndvi_winter,
ls_mean_monthly_max_ndvi_winter,
ls_mean_monthly_min_ndvi_winter,
ls_mean_monthly_mean_gcvi_winter,
ls_mean_monthly_max_gcvi_winter,
ls_mean_monthly_min_gcvi_winter,
ls_mean_monthly_mean_ndwi_winter,
ls_mean_monthly_max_ndwi_winter,
ls_mean_monthly_min_ndwi_winter,
ls_mean_monthly_mean_wgi_winter,
ls_mean_monthly_max_wgi_winter,
ls_mean_monthly_min_wgi_winter,
ls_mean_monthly_mean_ndbi_winter,
ls_mean_monthly_max_ndbi_winter,
ls_mean_monthly_min_ndbi_winter,
ls_mean_monthly_mean_bu_winter,
ls_mean_monthly_max_bu_winter,
ls_mean_monthly_min_bu_winter,
ls_mean_monthly_mean_ndvi_pr_winter,
ls_mean_monthly_max_ndvi_pr_winter,
ls_mean_monthly_min_ndvi_pr_winter,
ls_mean_monthly_mean_ndwi_greenhouses_winter,
ls_mean_monthly_max_ndwi_greenhouses_winter,
ls_mean_monthly_min_ndwi_greenhouses_winter,
winter_ndvi_std_mean.rename('NDVI_std'),
winter_ndwi_std_mean.rename('NDWI_std'),
winter_gcvi_std_mean.rename('GCVI_std'),
winter_wgi_std_mean.rename('WGI_std'),
slope,
])
return crop_data.toBands().regexpRename('([0-9]*_)', '')
import ee
from ee_plugin import Map
image = ee.ImageCollection('COPERNICUS/S2') \
.filterDate('2017-01-01', '2017-01-02').median() \
.divide(10000).visualize(**{'bands': ['B12', 'B8', 'B4'], 'min': 0.05, 'max': 0.5})
Map.setCenter(35.2, 31, 13)
Map.addLayer(image, {}, 'Sentinel-2 images January, 2018')
def exp(collection, folderpath, start, end, geojson, bandnames, operator, typ, ff):
# typ = ee.data.getInfo(collection)['type']
bandnames = ast.literal_eval(bandnames)
try:
if geojson.endswith(".geojson"):
with open(geojson) as aoi:
aoi_resp = json.load(aoi)
aoi_geom = ee.Geometry.Polygon(
aoi_resp["features"][0]["geometry"]["coordinates"]
)
boundbox = aoi_geom.bounds()
elif geojson.endswith(".json"):
with open(geojson) as aoi:
aoi_resp = json.load(aoi)
aoi_geom = ee.Geometry.Polygon(
aoi_resp["config"][0]["config"]["coordinates"]
)
boundbox = aoi_geom.bounds()
elif geojson.endswith(".kml"):
getcoord = kml2coord(geojson)
aoi_geom = ee.Geometry.Polygon(getcoord)
boundbox = aoi_geom.bounds()
except Exception as e:
print("Could not parse geometry")
print(e)
if typ == "image" and operator == "bb":
userCollection = ee.ImageCollection(collection).select(bandnames)
clipname = os.path.basename(geojson).split(".")[0]
fileName = str(collection).split("/")[-1] + "_" + str(clipname) + "_bb"
firstband = ee.Image(collection).bandNames().getInfo()[0]
scale = int(
ee.Image(collection).select(firstband).projection().nominalScale().getInfo()
)
task = ee.batch.Export.image.toDrive(
image=ee.Image(collection).clip(boundbox),
description=fileName,
folder=folderpath,
maxPixels=1e13,
region=boundbox.getInfo()["coordinates"][0],
scale=scale,
)
task.start()
print("Finished creating export tasks")
print("")
elif typ == "image" and operator == None:
userCollection = ee.ImageCollection(collection).select(bandnames)
clipname = os.path.basename(geojson).split(".")[0]
fileName = str(collection).split("/")[-1] + "_" + str(clipname)
firstband = ee.Image(collection).bandNames().getInfo()[0]
scale = int(
ee.Image(collection).select(firstband).projection().nominalScale().getInfo()
)
task = ee.batch.Export.image.toDrive(
image=ee.Image(collection).clip(aoi_geom),
description=fileName,
folder=folderpath,
maxPixels=1e13,
region=aoi_geom.getInfo()["coordinates"][0],
scale=scale,
)
task.start()
print("Finished creating export tasks")
print("")
elif typ == "collection" and operator == "bb":
userCollection = (
ee.ImageCollection(collection)
.filterBounds(aoi_geom)
.filterDate(start, end)
.select(bandnames)
)
clipname = os.path.basename(geojson).split(".")[0]
imageList = ee.List(userCollection.toList(userCollection.size().add(1)))
length = userCollection.size().getInfo()
if int(length) == 0:
print("No images found exiting export function")
sys.exit()
else:
print("Total images in filtered collection: " + str(length))
def exportImage(img):
fileName = ee.String(img.get("system:index")).getInfo()
firstband = img.bandNames().getInfo()[0]
scale = int(img.select(firstband).projection().nominalScale().getInfo())
# get geometry of image
task = ee.batch.Export.image.toDrive(
image=img.clip(boundbox),
description=fileName + "_" + str(clipname) + "_bb",
folder=folderpath,
maxPixels=1e13,
region=boundbox.getInfo()["coordinates"][0],
scale=scale,
)
task.start()
index = 0
while index < int(length):
print("Export #: " + str(index + 1) + " of " + str(length))
img2export = ee.Image(imageList.get(index))
exportImage(img2export)
index = index + 1
# time.sleep(10)
print("Finished creating export tasks")
print("")
elif typ == "collection" and operator == None:
userCollection = (
ee.ImageCollection(collection)
.filterBounds(aoi_geom)
.filterDate(start, end)
.select(bandnames)
)
clipname = os.path.basename(geojson).split(".")[0]
imageList = ee.List(userCollection.toList(userCollection.size().add(1)))
length = userCollection.size().getInfo()
if int(length) == 0:
print("No images found exiting export function")
sys.exit()
else:
print("Total images in filtered collection: " + str(length))
def exportImage(img):
fileName = ee.String(img.get("system:index")).getInfo()
firstband = img.bandNames().getInfo()[0]
scale = int(img.select(firstband).projection().nominalScale().getInfo())
# get geometry of image
task = ee.batch.Export.image.toDrive(
image=img.clip(aoi_geom),
description=fileName + "_" + str(clipname),
folder=folderpath,
maxPixels=1e13,
region=aoi_geom.getInfo()["coordinates"][0],
scale=scale,
)
task.start()
index = 0
while index < int(length):
print("Export #: " + str(index + 1) + " of " + str(length))
img2export = ee.Image(imageList.get(index))
exportImage(img2export)
index = index + 1
# time.sleep(10)
print("Finished creating export tasks")
print("")
elif typ == "table" and operator == "bb":
if ff is None:
ff = "SHP"
userCollection = ee.FeatureCollection(collection)
clipname = os.path.basename(geojson).split(".")[0]
fileName = str(collection).split("/")[-1] + "_" + str(clipname) + "_bb"
task = ee.batch.Export.table.toDrive(
collection=ee.FeatureCollection(collection).filterBounds(boundbox),
description=fileName,
folder=folderpath,
fileFormat=ff,
)
task.start()
print("Finished creating export tasks")
print("")
elif typ == "table" and operator == None:
if ff is None:
ff = "SHP"
userCollection = ee.FeatureCollection(collection)
clipname = os.path.basename(geojson).split(".")[0]
fileName = str(collection).split("/")[-1] + "_" + str(clipname)
task = ee.batch.Export.table.toDrive(
collection=ee.FeatureCollection(collection),
description=fileName,
folder=folderpath,
fileFormat=ff,
)
task.start()
print("Finished creating export tasks")
print("")
[-102.05, 41.0], # Colorado
[-109.05, 41.0],
[-111.05, 41.0],
[-111.05, 42.0], # Utah
[-114.05, 42.0],
[-114.05, 37.0],
[-109.05, 37.0]
]],
'geodesic':
False
})
# Create a Landsat 7 composite for Spring of 2000, and filter by
# the bounds of the FeatureCollection.
collection = ee.ImageCollection('LANDSAT/LE07/C01/T1') \
.filterDate('2000-04-01', '2000-07-01') \
.filterBounds(polygon)
# Compute the median in each band, in each pixel.
median = collection.median()
# Select the red, green and blue bands.
result = median.select('B3', 'B2', 'B1')
Map.addLayer(result, {'gain': [1.4, 1.4, 1.1]})
Map.setCenter(-110, 40, 5)
# %%
"""
## Display Earth Engine data layers
"""
def prep_cnn_np(survey_df, satellite, kernel_size, year):
'''
Creates numpy arrays for CNN
Input: df - pandas dataframe
lat_name - name of latitude variable in df
lon_name - name of longitude variable in df
Output: geopandas dataframe
'''
# Setup --------------------------------------------------------------------
# Survey to FeatureCollection
survey_fc = survey_to_fc(survey_df)
# Define kernel for neighborhood array
list = ee.List.repeat(1, kernel_size)
lists = ee.List.repeat(list, kernel_size)
kernel = ee.Kernel.fixed(kernel_size, kernel_size, lists)
# Define scale
if satellite in ['l7', 'l8']:
SCALE = 30
elif satellite in ['s2']:
SCALE = 10
# Prep NTL -----------------------------------------------------------------
# Year
# VIIRS starts in 2012. At minimum, use 2013 to have year before and after
if False:
if year <= 2013:
year_use = 2013
else:
year_use = year
year_plus = year_use + 1
year_minus = year_use - 1
year_minus_str = str(year_minus) + '-01-01'
year_plus_str = str(year_plus) + '-12-31'
# Reduce image collection
ntl_image = ee.ImageCollection('NOAA/VIIRS/DNB/MONTHLY_V1/VCMCFG')\
.filterDate(year_minus_str, year_plus_str)\
.median()
# Select Bands
ntl_image = ntl_image.select(['avg_rad'])
# Image to neighborhood array
ntl_arrays = ntl_image.neighborhoodToArray(kernel)
# Extract values from GEE
ntl_values_ee = ntl_arrays.sample(
region=survey_fc,
scale=SCALE,
tileScale=10 #8
)
ntl_dict_ee = ntl_values_ee.getInfo()
# Convert values to numpy array
#n_rows = survey_df.shape[0]
ntl_f = ntl_dict_ee['features']
# l7 ----------------------------------------------------------------
if satellite == "l7":
# Bands
b_b = 'B1'
g_b = 'B2'
r_b = 'B3'
nir_b = 'B4'
swir_b = 'B5'
#other_bs = ['B5', 'B6', 'B7']
#BANDS = single_bs.copy()
BANDS = [b_b].copy()
BANDS.append(g_b)
BANDS.append(r_b)
BANDS.append(nir_b)
BANDS.append(swir_b)
# Year
year_use = year
year_plus = year_use + 1
year_minus = year_use - 1
year_minus_str = str(year_minus) + '-01-01'
year_plus_str = str(year_plus) + '-12-31'
image = ee.ImageCollection('LANDSAT/LC07/C01/T1_SR')\
.filterDate(year_minus_str, year_plus_str)\
.map(cloud_mask_landsatSR)\
.median() #\
#.multiply(0.0001)
# l8 ----------------------------------------------------------------
if satellite == "l8":
# Bands
# FOR COLLECTION 2
#b_b = 'SR_B2'
#g_b = 'SR_B3'
#r_b = 'SR_B4'
#nir_b = 'SR_B5'
#other_bs = ['SR_B6', 'SR_B7', 'ST_B10']
# FOR COLLECTION 1
b_b = 'B2'
g_b = 'B3'
r_b = 'B4'
nir_b = 'B5'
swir_b = 'B6'
#other_bs = ['B6', 'B7', 'B10']
#BANDS = single_bs.copy()
BANDS = [b_b].copy()
BANDS.append(g_b)
BANDS.append(r_b)
BANDS.append(nir_b)
BANDS.append(swir_b)
# Year
# landsat 8 starts in April 2013; if year is less than
# 2014, use 2014 as year (to ensure have year before and after)
if year < 2014:
year_use = 2014
else:
year_use = year
year_plus = year_use + 1
year_minus = year_use - 1
year_minus_str = str(year_minus) + '-01-01'
year_plus_str = str(year_plus) + '-12-31'
#image = ee.ImageCollection('LANDSAT/LC08/C02/T1_L2')\
# .filterDate(year_minus_str, year_plus_str)\
# #.map(cloud_mask_landsatSR)\ #TODO cloud_mask_landsatSR doesn't work with landsat collection 2
# .median() #\
# #.multiply(0.0001)
image = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR')\
.filterDate(year_minus_str, year_plus_str)\
.map(cloud_mask_landsatSR)\
.median()
#image = ee.ImageCollection('LANDSAT/LC08/C02/T1_L2')\
# .filterDate(year_minus_str, year_plus_str)\
# .median()
# s2 ----------------------------------------------------------------
if satellite == "s2":
# Bands
b_b = 'B2'
g_b = 'B3'
r_b = 'B4'
nir_b = 'B8'
swir_b = 'B11'
#other_bs = ['B5', 'B6', 'B7', 'B8A', 'B11', 'B12', 'AOT']
#BANDS = single_bs.copy()
BANDS = [b_b].copy()
BANDS.append(g_b)
BANDS.append(r_b)
BANDS.append(nir_b)
BANDS.append(swir_b)
# Year
# sentinel starts in March 2017; juse use 2018
year_use = 2019
year_plus = year_use + 1
year_minus = year_use - 1
year_minus_str = str(year_minus) + '-01-01'
year_plus_str = str(year_plus) + '-12-31'
# Number of bands changes in sentinel, so need to select here before aggregate
# https://gis.stackexchange.com/questions/374010/gee-tile-error-expected-a-homogeneous-image-collection-but-an-image-with-incom
image = ee.ImageCollection('COPERNICUS/S2_SR')\
.filterDate(year_minus_str, year_plus_str)\
.map(cloud_mask_sentinel2)\
.select(BANDS)\
.median() # \
#.multiply(0.0001)
# Select Bands
image = image.select(BANDS)
# Create Indices
# https://www.linkedin.com/pulse/ndvi-ndbi-ndwi-calculation-using-landsat-7-8-tek-bahadur-kshetri
ndvi = image.normalizedDifference([nir_b, r_b]).rename('NDVI')
ndbi = image.normalizedDifference([swir_b, nir_b]).rename('NDBI')
image = image.addBands(ndvi)
image = image.addBands(ndbi)
bu = image.select('NDBI').subtract(image.select('NDVI')).rename('BU')
image = image.addBands(bu)
# Subset bands; don't need those used to create NDVI and NDBI
image = image.select([b_b, g_b, r_b, 'NDVI', 'BU'])
# Image to neighborhood array
arrays = image.neighborhoodToArray(kernel)
# New ---------
#neighborhoodImage = myImageToBeSampled.neighborhoodToArray(kernel)
#samples = arrays.sampleRegions(collection=survey_fc)
# ee.batch.Export.table.toCloudStorage
# ee.batch.Export.table.toDrive
# Export.table.toDrive
#mytask = ee.batch.Export.table.toDrive(
# collection = samples,
# fileFormat = 'TFRecord',
# description = 'test123',
# folder = 'gee_extracts',
# selectors = [b_b, g_b, r_b, 'NDVI', 'BU'] + ['uid', 'ntl_group'])
#return mytask
# OLD ---------
# Extract values from GEE
values_ee = arrays.sample(
region=survey_fc,
scale=SCALE,
tileScale=12 # 8
)
dict_ee = values_ee.getInfo()
# Convert values to numpy array
n_rows = survey_df.shape[0]
daytime_f = dict_ee['features']
# Extract data
out_ex_proto_list = ee_to_np_daytime(daytime_f, survey_df, n_rows, b_b,
g_b, r_b)
return out_ex_proto_list
# https://csaybar.github.io/blog/2019/05/30/eetf/
# https://stackoverflow.com/questions/63000565/extract-10000-images-from-google-earth-engine
# https://colab.research.google.com/github/google/earthengine-api/blob/master/python/examples/ipynb/UNET_regression_demo.ipynb#scrollTo=-IlgXu-vcUEY
# To make sure you're logged in, run: earthengine authenticate
# Edit the DATA to be one of the values listed above
# Edit NUMBER_OF_POINTS to be the number of data points you want
# Edit FILE_NAME to be the name of the output file
# run: python3 data_extract.py
# The result will be a new CSV file in your Google Drive
DATA = RADIATION
NUMBER_OF_POINTS = 100000
FILE_NAME = 'Radiation-100k'
# ------------------------------------
if DATA in [ELEVATION, SOIL_TYPE, PRECIPITATION]:
image = ee.Image(DATA) # Use this if the data is already an Image type
else:
image_collection = ee.ImageCollection(DATA) # Use this if the data is an ImageCollection
if DATA in [HUMAN_MODIF]:
image = image_collection.median() # Use this for non-time based data
else:
# The image input data is a 2018 cloud-masked median composite.
# image = image_collection.filterDate('2018-01-01', '2018-12-31').median() # Use this for time-based data
image = image_collection.filterDate('2017-01-01', '2017-12-31').median() # Use this for time-based data
# Change the following two lines to use your own training data.
region = regions.california()
labels = ee.FeatureCollection.randomPoints(region, NUMBER_OF_POINTS)
# Sample the image at the points and add a random column.
sample = image.sampleRegions(collection=labels, scale=30) #.randomColumn()
Map = folium.Map(location=[40, -100], zoom_start=4)
Map.setOptions('HYBRID')
# %%
'''
## Add Earth Engine Python script
'''
# %%
Map.setCenter(-110, 40, 5)
fc = ee.FeatureCollection('TIGER/2018/States').filter(
ee.Filter.eq('STUSPS', 'MN'))
# Create a Landsat 7, median-pixel composite for Spring of 2000.
collection = ee.ImageCollection('LE7_L1T').filterDate("2000-05-01", "2000-10-31") \
.filterBounds(fc)
image1 = collection.median()
# Map.addLayer(image1)
# # Clip to the output image to the California state boundary.
# # fc = (ee.FeatureCollection('ft:1fRY18cjsHzDgGiJiS2nnpUU3v9JPDc2HNaR7Xk8')
# # .filter(ee.Filter().eq('Name', 'Minnesota')))
image2 = image1.clipToCollection(fc)
# Select the red, green and blue bands.
image = image2.select('B4', 'B3', 'B2')
Map.addLayer(image, {'gain': [1.4, 1.4, 1.1]}, 'Landsat 7')
# %%
'''
"""
import ee
from ee import batch
from pandas.tseries.offsets import DateOffset
import pandas as pd
import folium
## Initialize (a ee python thing)
ee.Initialize()
####################################################
roi = ee.FeatureCollection("users/kkraoj/west_usa")
fireCol = ee.ImageCollection("MODIS/006/MCD64A1")
prism = ee.ImageCollection("OREGONSTATE/PRISM/AN81m")
# prismMean = ee.ImageCollection("OREGONSTATE/PRISM/Norm81m").filter(ee.Filter.calendarRange(3, 9, 'month')).mean()
# prism = prism.filterDate("2001-01-01","2019-12-31").filter(ee.Filter.calendarRange(3, 9, 'month'))
fireCol = fireCol.filterDate("2001-01-01", "2021-03-31")
def addMonthYear(image):
month = ee.Date(image.get("system:time_start")).get("month")
year = ee.Date(image.get("system:time_start")).get("year")
image = image.set("month", month)
image = image.set("year", year)
return image
def main(ini_path=None, overwrite_flag=False, delay_time=0, gee_key_file=None,
ready_task_max=-1, reverse_flag=False, tiles=None, update_flag=False,
log_tasks=True, recent_days=0, start_dt=None, end_dt=None):
"""Compute gridded Tcorr images by WRS2 tile
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files if the export dates are the same and
generate new images (but with different export dates) even if the tile
lists are the same. The default is False.
delay_time : float, optional
Delay time in seconds between starting export tasks (or checking the
number of queued tasks, see "max_ready" parameter). The default is 0.
gee_key_file : str, None, optional
Earth Engine service account JSON key file (the default is None).
ready_task_max: int, optional
Maximum number of queued "READY" tasks.
reverse_flag : bool, optional
If True, process WRS2 tiles in reverse order (the default is False).
tiles : str, None, optional
List of MGRS tiles to process (the default is None).
update_flag : bool, optional
If True, only overwrite scenes with an older model version.
recent_days : int, optional
Limit start/end date range to this many days before the current date
(the default is 0 which is equivalent to not setting the parameter and
will use the INI start/end date directly).
start_dt : datetime, optional
Override the start date in the INI file
(the default is None which will use the INI start date).
end_dt : datetime, optional
Override the (inclusive) end date in the INI file
(the default is None which will use the INI end date).
"""
logging.info('\nCompute gridded Tcorr images by WRS2 tile')
# CGM - Which format should we use for the WRS2 tile?
wrs2_tile_fmt = 'p{:03d}r{:03d}'
# wrs2_tile_fmt = '{:03d}{:03d}'
wrs2_tile_re = re.compile('p?(\d{1,3})r?(\d{1,3})')
# List of path/rows to skip
wrs2_skip_list = [
'p049r026', # Vancouver Island, Canada
# 'p047r031', # North California coast
'p042r037', # San Nicholas Island, California
# 'p041r037', # South California coast
'p040r038', 'p039r038', 'p038r038', # Mexico (by California)
'p037r039', 'p036r039', 'p035r039', # Mexico (by Arizona)
'p034r039', 'p033r039', # Mexico (by New Mexico)
'p032r040', # Mexico (West Texas)
'p029r041', 'p028r042', 'p027r043', 'p026r043', # Mexico (South Texas)
'p019r040', 'p018r040', # West Florida coast
'p016r043', 'p015r043', # South Florida coast
'p014r041', 'p014r042', 'p014r043', # East Florida coast
'p013r035', 'p013r036', # North Carolina Outer Banks
'p013r026', 'p012r026', # Canada (by Maine)
'p011r032', # Rhode Island coast
]
wrs2_path_skip_list = [9, 49]
wrs2_row_skip_list = [25, 24, 43]
mgrs_skip_list = []
export_id_fmt = 'tcorr_gridded_{product}_{scene_id}'
asset_id_fmt = '{coll_id}/{scene_id}'
# TODO: Move to INI or function input parameter
clip_ocean_flag = True
# Read config file
ini = configparser.ConfigParser(interpolation=None)
ini.read_file(open(ini_path, 'r'))
# ini = utils.read_ini(ini_path)
model_name = 'SSEBOP'
try:
study_area_coll_id = str(ini['INPUTS']['study_area_coll'])
except KeyError:
raise ValueError('"study_area_coll" parameter was not set in INI')
except Exception as e:
raise e
try:
start_date = str(ini['INPUTS']['start_date'])
except KeyError:
raise ValueError('"start_date" parameter was not set in INI')
except Exception as e:
raise e
try:
end_date = str(ini['INPUTS']['end_date'])
except KeyError:
raise ValueError('"end_date" parameter was not set in INI')
except Exception as e:
raise e
try:
collections = str(ini['INPUTS']['collections'])
collections = sorted([x.strip() for x in collections.split(',')])
except KeyError:
raise ValueError('"collections" parameter was not set in INI')
except Exception as e:
raise e
try:
mgrs_ftr_coll_id = str(ini['EXPORT']['mgrs_ftr_coll'])
except KeyError:
raise ValueError('"mgrs_ftr_coll" parameter was not set in INI')
except Exception as e:
raise e
# Optional parameters
try:
study_area_property = str(ini['INPUTS']['study_area_property'])
except KeyError:
study_area_property = None
logging.debug(' study_area_property: not set in INI, defaulting to None')
except Exception as e:
raise e
try:
study_area_features = str(ini['INPUTS']['study_area_features'])
study_area_features = sorted([
x.strip() for x in study_area_features.split(',')])
except KeyError:
study_area_features = []
logging.debug(' study_area_features: not set in INI, defaulting to []')
except Exception as e:
raise e
try:
wrs2_tiles = str(ini['INPUTS']['wrs2_tiles'])\
.replace('"', '').replace("'", '')
wrs2_tiles = sorted([x.strip() for x in wrs2_tiles.split(',')])
except KeyError:
wrs2_tiles = []
logging.debug(' wrs2_tiles: not set in INI, defaulting to []')
except Exception as e:
raise e
try:
mgrs_tiles = str(ini['EXPORT']['mgrs_tiles'])
mgrs_tiles = sorted([x.strip() for x in mgrs_tiles.split(',')])
# CGM - Remove empty strings caused by trailing or extra commas
mgrs_tiles = [x.upper() for x in mgrs_tiles if x]
logging.debug(f' mgrs_tiles: {mgrs_tiles}')
except KeyError:
mgrs_tiles = []
logging.debug(' mgrs_tiles: not set in INI, defaulting to []')
except Exception as e:
raise e
try:
utm_zones = str(ini['EXPORT']['utm_zones'])
utm_zones = sorted([int(x.strip()) for x in utm_zones.split(',')])
logging.debug(f' utm_zones: {utm_zones}')
except KeyError:
utm_zones = []
logging.debug(' utm_zones: not set in INI, defaulting to []')
except Exception as e:
raise e
try:
fill_with_climo_flag = str(ini['EXPORT']['fill_with_climo'])
if fill_with_climo_flag.lower() in ['t', 'true']:
fill_with_climo_flag = True
else:
fill_with_climo_flag = False
except KeyError:
fill_with_climo_flag = False
logging.debug(' fill_with_climo: not set in INI, defaulting to False')
except Exception as e:
raise e
# TODO: Add try/except blocks and default values?
cloud_cover = float(ini['INPUTS']['cloud_cover'])
# Model specific parameters
# Set the property name to lower case and try to cast values to numbers
model_args = {
k.lower(): float(v) if utils.is_number(v) else v
for k, v in dict(ini[model_name]).items()}
filter_args = {}
# TODO: Add try/except blocks
tmax_source = ini[model_name]['tmax_source']
tcorr_source = ini[model_name]['tcorr_source']
tcorr_scene_coll_id = '{}'.format(ini['EXPORT']['export_coll'])
# tcorr_scene_coll_id = '{}/{}_scene'.format(
# ini['EXPORT']['export_coll'], tmax_source.lower())
tcorr_month_coll_id = f'{tcorr_scene_coll_id}_monthly'
if tcorr_source.upper() not in ['GRIDDED_COLD', 'GRIDDED_COLD_1KM', 'GRIDDED']:
raise ValueError('unsupported tcorr_source for these tools')
# For now only support reading specific Tmax sources
if (tmax_source.upper() not in ['DAYMET_MEDIAN_V2'] and
not re.match('^projects/.+/tmax/.+_(mean|median)_\d{4}_\d{4}(_\w+)?',
tmax_source)):
raise ValueError(f'unsupported tmax_source: {tmax_source}')
# if (tmax_source.upper() == 'CIMIS' and
# ini['INPUTS']['end_date'] < '2003-10-01'):
# raise ValueError('CIMIS is not currently available before 2003-10-01')
# elif (tmax_source.upper() == 'DAYMET' and
# ini['INPUTS']['end_date'] > '2018-12-31'):
# logging.warning('\nDAYMET is not currently available past 2018-12-31, '
# 'using median Tmax values\n')
# If the user set the tiles argument, use these instead of the INI values
if tiles:
logging.info('\nOverriding INI mgrs_tiles and utm_zones parameters')
logging.info(f' user tiles: {tiles}')
mgrs_tiles = sorted([y.strip() for x in tiles for y in x.split(',')])
mgrs_tiles = [x.upper() for x in mgrs_tiles if x]
logging.info(f' mgrs_tiles: {", ".join(mgrs_tiles)}')
utm_zones = sorted(list(set([int(x[:2]) for x in mgrs_tiles])))
logging.info(f' utm_zones: {", ".join(map(str, utm_zones))}')
today_dt = datetime.datetime.now()
today_dt = today_dt.replace(hour=0, minute=0, second=0, microsecond=0)
if recent_days:
logging.info('\nOverriding INI "start_date" and "end_date" parameters')
logging.info(f' Recent days: {recent_days}')
end_dt = today_dt - datetime.timedelta(days=1)
start_dt = today_dt - datetime.timedelta(days=recent_days)
start_date = start_dt.strftime('%Y-%m-%d')
end_date = end_dt.strftime('%Y-%m-%d')
elif start_dt and end_dt:
# Attempt to use the function start/end dates
logging.info('\nOverriding INI "start_date" and "end_date" parameters')
logging.info(' Custom date range')
start_date = start_dt.strftime('%Y-%m-%d')
end_date = end_dt.strftime('%Y-%m-%d')
else:
# Parse the INI start/end dates
logging.info('\nINI date range')
try:
start_dt = datetime.datetime.strptime(start_date, '%Y-%m-%d')
end_dt = datetime.datetime.strptime(end_date, '%Y-%m-%d')
except Exception as e:
raise e
logging.info(f' Start: {start_date}')
logging.info(f' End: {end_date}')
# TODO: Add a few more checks on the dates
if end_dt < start_dt:
raise ValueError('end date can not be before start date')
# logging.debug('\nInterpolation date range')
# iter_start_dt = start_dt
# iter_end_dt = end_dt + datetime.timedelta(days=1)
# iter_start_dt = start_dt - datetime.timedelta(days=interp_days)
# iter_end_dt = end_dt + datetime.timedelta(days=interp_days+1)
# logging.debug(' Start: {}'.format(iter_start_dt.strftime('%Y-%m-%d')))
# logging.debug(' End: {}'.format(iter_end_dt.strftime('%Y-%m-%d')))
logging.info('\nInitializing Earth Engine')
if gee_key_file:
logging.info(f' Using service account key file: {gee_key_file}')
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('x', key_file=gee_key_file))
else:
ee.Initialize()
logging.debug('\nTmax properties')
logging.debug(f' Source: {tmax_source}')
# # DEADBEEF - Not needed with gridded Tcorr
# # Get a Tmax image to set the Tcorr values to
# if 'MEDIAN' in tmax_name.upper():
# tmax_coll_id = 'projects/earthengine-legacy/assets/' \
# 'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower())
# tmax_coll = ee.ImageCollection(tmax_coll_id)
# tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0)
# # else:
# # raise ValueError(f'unsupported tmax_source: {tmax_name}')
# logging.debug(f' Collection: {tmax_coll_id}')
# Build output collection and folder if necessary
logging.debug(f'\nExport Collection: {tcorr_scene_coll_id}')
if not ee.data.getInfo(tcorr_scene_coll_id.rsplit('/', 1)[0]):
logging.info('\nExport folder does not exist and will be built'
'\n {}'.format(tcorr_scene_coll_id.rsplit('/', 1)[0]))
input('Press ENTER to continue')
ee.data.createAsset({'type': 'FOLDER'}, tcorr_scene_coll_id.rsplit('/', 1)[0])
if not ee.data.getInfo(tcorr_scene_coll_id):
logging.info('\nExport collection does not exist and will be built'
'\n {}'.format(tcorr_scene_coll_id))
input('Press ENTER to continue')
ee.data.createAsset({'type': 'IMAGE_COLLECTION'}, tcorr_scene_coll_id)
# Get current running tasks
if ready_task_max == -9999:
# CGM - Getting the task list can take awhile so set ready tasks to
# -9999 to skip requesting it. Only run this if you are going to
# manually avoid running existing tasks.
# TODO: Check if this should disable delay_task() or set the
# ready_task_max to a large value
tasks = {}
ready_task_count = 0
else:
logging.info('\nRequesting Task List')
tasks = utils.get_ee_tasks()
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
utils.print_ee_tasks(tasks)
input('ENTER')
ready_task_count = sum(1 for t in tasks.values() if t['state'] == 'READY')
# ready_task_count = len(tasks.keys())
logging.info(f' Tasks: {ready_task_count}')
# CGM - I'm still not sure if it makes sense to hold here or after the
# first task is started.
ready_task_count = delay_task(
delay_time=0, task_max=ready_task_max, task_count=ready_task_count)
# # Get current running tasks
# tasks = utils.get_ee_tasks()
# ready_task_count = sum(1 for t in tasks.values() if t['state'] == 'READY')
# # ready_task_count = delay_task(ready_task_count, delay_time, max_ready)
# if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
# utils.print_ee_tasks(tasks)
# input('ENTER')
# DEADBEEF - The asset list will be retrieved before each WRS2 tile is processed
# Get current asset list
# logging.debug('\nGetting GEE asset list')
# asset_list = utils.get_ee_assets(tcorr_scene_coll_id)
# if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
# pprint.pprint(asset_list[:10])
# Get list of MGRS tiles that intersect the study area
logging.debug('\nMGRS Tiles/Zones')
export_list = mgrs_export_tiles(
study_area_coll_id=study_area_coll_id,
mgrs_coll_id=mgrs_ftr_coll_id,
study_area_property=study_area_property,
study_area_features=study_area_features,
mgrs_tiles=mgrs_tiles,
mgrs_skip_list=mgrs_skip_list,
utm_zones=utm_zones,
wrs2_tiles=wrs2_tiles,
)
if not export_list:
logging.error('\nEmpty export list, exiting')
return False
# pprint.pprint(export_list)
# input('ENTER')
# Build the complete/filtered WRS2 list
wrs2_tile_list = list(set(
wrs2 for tile_info in export_list
for wrs2 in tile_info['wrs2_tiles']))
if wrs2_skip_list:
wrs2_tile_list = [wrs2 for wrs2 in wrs2_tile_list
if wrs2 not in wrs2_skip_list]
if wrs2_path_skip_list:
wrs2_tile_list = [wrs2 for wrs2 in wrs2_tile_list
if int(wrs2[1:4]) not in wrs2_path_skip_list]
if wrs2_row_skip_list:
wrs2_tile_list = [wrs2 for wrs2 in wrs2_tile_list
if int(wrs2[5:8]) not in wrs2_row_skip_list]
wrs2_tile_list = sorted(wrs2_tile_list, reverse=not(reverse_flag))
wrs2_tile_count = len(wrs2_tile_list)
# Process each WRS2 tile separately
logging.info('\nImage Exports')
for wrs2_i, wrs2_tile in enumerate(wrs2_tile_list):
wrs2_path, wrs2_row = map(int, wrs2_tile_re.findall(wrs2_tile)[0])
logging.info('{} ({}/{})'.format(wrs2_tile, wrs2_i + 1, wrs2_tile_count))
for coll_id in collections:
filter_args[coll_id] = [
{'type': 'equals', 'leftField': 'WRS_PATH', 'rightValue': wrs2_path},
{'type': 'equals', 'leftField': 'WRS_ROW', 'rightValue': wrs2_row}]
# logging.debug(f' Filter Args: {filter_args}')
# Build and merge the Landsat collections
model_obj = ssebop.Collection(
collections=collections,
start_date=start_dt.strftime('%Y-%m-%d'),
end_date=(end_dt + datetime.timedelta(days=1)).strftime('%Y-%m-%d'),
cloud_cover_max=cloud_cover,
geometry=ee.Geometry.Point(openet.core.wrs2.centroids[wrs2_tile]),
model_args=model_args,
filter_args=filter_args,
)
landsat_coll = model_obj.overpass(variables=['ndvi'])
# pprint.pprint(landsat_coll.aggregate_array('system:id').getInfo())
# input('ENTER')
try:
image_id_list = landsat_coll.aggregate_array('system:id').getInfo()
except Exception as e:
logging.warning(' Error getting image ID list, skipping tile')
logging.debug(f' {e}')
continue
# Get list of existing images for the target tile
logging.debug(' Getting GEE asset list')
asset_coll = ee.ImageCollection(tcorr_scene_coll_id) \
.filterDate(start_dt.strftime('%Y-%m-%d'),
(end_dt + datetime.timedelta(days=1)).strftime('%Y-%m-%d')) \
.filterMetadata('wrs2_tile', 'equals',
wrs2_tile_fmt.format(wrs2_path, wrs2_row))
asset_props = {f'{tcorr_scene_coll_id}/{x["properties"]["system:index"]}':
x['properties']
for x in utils.get_info(asset_coll)['features']}
# asset_props = {x['id']: x['properties'] for x in assets_info['features']}
# Sort image ID list by date
image_id_list = sorted(
image_id_list, key=lambda k: k.split('/')[-1].split('_')[-1],
reverse=reverse_flag)
# Sort by date
for image_id in image_id_list:
coll_id, scene_id = image_id.rsplit('/', 1)
logging.info(f'{scene_id}')
export_dt = datetime.datetime.strptime(scene_id.split('_')[-1], '%Y%m%d')
export_date = export_dt.strftime('%Y-%m-%d')
logging.debug(f' Date: {export_date}')
export_id = export_id_fmt.format(
product=tmax_source.split('/')[-1].lower(), scene_id=scene_id)
logging.debug(f' Export ID: {export_id}')
asset_id = asset_id_fmt.format(
coll_id=tcorr_scene_coll_id, scene_id=scene_id)
logging.debug(f' Collection: {os.path.dirname(asset_id)}')
logging.debug(f' Image ID: {os.path.basename(asset_id)}')
if update_flag:
def version_number(version_str):
return list(map(int, version_str.split('.')))
if export_id in tasks.keys():
logging.info(' Task already submitted, skipping')
continue
# In update mode only overwrite if the version is old
if asset_props and asset_id in asset_props.keys():
model_ver = version_number(ssebop.__version__)
asset_ver = version_number(
asset_props[asset_id]['model_version'])
if asset_ver < model_ver:
logging.info(' Existing asset model version is old, '
'removing')
logging.debug(f' asset: {asset_ver}\n'
f' model: {model_ver}')
try:
ee.data.deleteAsset(asset_id)
except:
logging.info(' Error removing asset, skipping')
continue
elif (('T1_RT_TOA' in asset_props[asset_id]['coll_id']) and
('T1_RT_TOA' not in image_id)):
logging.info(' Existing asset is from realtime '
'Landsat collection, removing')
try:
ee.data.deleteAsset(asset_id)
except:
logging.info(' Error removing asset, skipping')
continue
else:
logging.debug(' Asset is up to date, skipping')
continue
elif overwrite_flag:
if export_id in tasks.keys():
logging.info(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id]['id'])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if asset_props and asset_id in asset_props.keys():
logging.info(' Asset already exists, removing')
ee.data.deleteAsset(asset_id)
else:
if export_id in tasks.keys():
logging.info(' Task already submitted, skipping')
continue
elif asset_props and asset_id in asset_props.keys():
logging.info(' Asset already exists, skipping')
continue
# Get the input image grid and spatial reference
image_info = ee.Image(image_id).select([3]).getInfo()
image_geo = image_info['bands'][0]['crs_transform']
image_crs = image_info['bands'][0]['crs']
image_shape = image_info['bands'][0]['dimensions']
# Transform format: [30, 0, 591285, 0, -30, 4256115]
image_extent = [
image_geo[2], image_geo[5] + image_shape[1] * image_geo[4],
image_geo[2] + image_shape[0] * image_geo[0], image_geo[5]]
logging.debug(f' Image CRS: {image_crs}')
logging.debug(f' Image Geo: {image_geo}')
logging.debug(f' Image Extent: {image_extent}')
logging.debug(f' Image Shape: {image_shape}')
# Adjust the image extent to the coarse resolution grid
# EXPORT_GEO = [5000, 0, 15, 0, -5000, 15]
export_cs = EXPORT_GEO[0]
export_extent = [
round(math.floor((image_extent[0] - EXPORT_GEO[2]) / export_cs) *
export_cs + EXPORT_GEO[2], 8),
round(math.floor((image_extent[1] - EXPORT_GEO[5]) / export_cs) *
export_cs + EXPORT_GEO[5], 8),
round(math.ceil((image_extent[2] - EXPORT_GEO[2]) / export_cs) *
export_cs + EXPORT_GEO[2], 8),
round(math.ceil((image_extent[3] - EXPORT_GEO[5]) / export_cs) *
export_cs + EXPORT_GEO[5], 8),
]
export_geo = [export_cs, 0, export_extent[0], 0,
-export_cs, export_extent[3]]
export_shape = [
int(abs(export_extent[2] - export_extent[0]) / EXPORT_GEO[0]),
int(abs(export_extent[3] - export_extent[1]) / EXPORT_GEO[0])]
logging.debug(f' Export CRS: {image_crs}')
logging.debug(f' Export Geo: {export_geo}')
logging.debug(f' Export Extent: {export_extent}')
logging.debug(f' Export Shape: {export_shape}')
# CGM - Why are we not using the from_image_id() method?
# t_obj = ssebop.Image.from_image_id(ee.Image(image_id), **model_args)
if coll_id.endswith('_L2'):
t_obj = ssebop.Image.from_landsat_c2_sr(
sr_image=ee.Image(image_id),
cloudmask_args={'cirrus_flag': True, 'dilate_flag': True,
'shadow_flag': True, 'snow_flag': True},
**model_args)
elif coll_id.endswith('_SR'):
t_obj = ssebop.Image.from_landsat_c1_sr(
ee.Image(image_id), **model_args)
elif coll_id.endswith('_TOA'):
t_obj = ssebop.Image.from_landsat_c1_toa(
ee.Image(image_id), **model_args)
else:
raise ValueError('Could not determine Landsat type')
# CGM - Intentionally not calling the tcorr method directly since
# there may be compositing with climos or the scene average
if tcorr_source == 'GRIDDED':
tcorr_img = t_obj.tcorr_gridded
elif tcorr_source == 'GRIDDED_COLD':
tcorr_img = t_obj.tcorr_gridded_cold
elif tcorr_source == 'GRIDDED_COLD_1KM':
tcorr_img = t_obj.tcorr_gridded_cold_1km
# tcorr_img = t_obj.tcorr
# Replace masked tcorr images with climos
# Note, If the month climo doesn't exist this will keep the
# existing masked Tcorr image (we may want to change that)
# Does the tcorr_coarse count need to be set to a value like 0?
if fill_with_climo_flag and tcorr_source == 'GRIDDED_COLD':
logging.debug(' Checking if monthly climo should be applied')
tcorr_month_coll = ee.ImageCollection(tcorr_month_coll_id)\
.filterMetadata('wrs2_tile', 'equals', wrs2_tile)\
.filterMetadata('month', 'equals', export_dt.month)\
.select(['tcorr'])
# Setting the quality to 0 here causes it to get masked below
# We might want to have it actually be 0 instead
tcorr_month_img = tcorr_month_coll.first()\
.addBands([tcorr_month_coll.first().multiply(0).rename(['quality'])])\
.set({'tcorr_coarse_count': None})
tcorr_img = ee.Algorithms.If(
ee.Number(tcorr_img.get('tcorr_index')).eq(9)
.And(tcorr_month_coll.size().gt(0)),
tcorr_month_img,
tcorr_img)
elif fill_with_climo_flag and tcorr_source == 'GRIDDED_COLD_1KM':
logging.debug(' Checking if monthly climo should be applied')
tcorr_month_coll = ee.ImageCollection(tcorr_month_coll_id) \
.filterMetadata('wrs2_tile', 'equals', wrs2_tile) \
.filterMetadata('month', 'equals', export_dt.month) \
.select(['tcorr'])
# Setting the quality to 0 here causes it to get masked below
# We might want to have it actually be 0 instead
tcorr_month_img = tcorr_month_coll.first() \
.addBands([tcorr_month_coll.first().multiply(0).rename(['quality'])]) \
.set({'tcorr_coarse_count': None})
tcorr_img = ee.Algorithms.If(
ee.Number(tcorr_img.get('tcorr_index')).eq(9)
.And(tcorr_month_coll.size().gt(0)),
tcorr_month_img,
tcorr_img)
# Clip to the Landsat image footprint
tcorr_img = ee.Image(tcorr_img).clip(ee.Image(image_id).geometry())
# Clear the transparency mask (from clipping)
tcorr_img = tcorr_img.updateMask(tcorr_img.unmask(0))
if clip_ocean_flag:
tcorr_img = tcorr_img\
.updateMask(ee.Image('projects/openet/ocean_mask'))
# # CGM - The NLCD mask will only work for CONUS
# output_img = output_img.updateMask(
# ee.Image('USGS/NLCD/NLCD2016').select(['landcover']).mask())
tcorr_img = tcorr_img\
.set({
'CLOUD_COVER': image_info['properties']['CLOUD_COVER'],
'CLOUD_COVER_LAND': image_info['properties']['CLOUD_COVER_LAND'],
# 'SPACECRAFT_ID': image.get('SPACECRAFT_ID'),
'coll_id': coll_id,
'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'),
'date': export_dt.strftime('%Y-%m-%d'),
'doy': int(export_dt.strftime('%j')),
'image_id': image_id,
'model_name': model_name,
'model_version': ssebop.__version__,
'month': int(export_dt.month),
'realtime': 'true' if '/T1_RT' in coll_id else 'false',
'scene_id': scene_id,
'system:time_start': image_info['properties']['system:time_start'],
# 'tcorr_index': TCORR_INDICES[tcorr_source.upper()],
'tcorr_source': tcorr_source,
'tmax_source': tmax_source,
# 'tmax_source': tmax_source.replace(
# 'projects/earthengine-legacy/assets/', ''),
'tool_name': TOOL_NAME,
'tool_version': TOOL_VERSION,
'wrs2_path': wrs2_path,
'wrs2_row': wrs2_row,
'wrs2_tile': wrs2_tile_fmt.format(wrs2_path, wrs2_row),
'year': int(export_dt.year),
})
# pprint.pprint(tcorr_img.getInfo()['properties'])
# input('ENTER')
logging.debug(' Building export task')
task = ee.batch.Export.image.toAsset(
image=tcorr_img,
description=export_id,
assetId=asset_id,
crs=image_crs,
crsTransform='[' + ','.join(list(map(str, export_geo))) + ']',
dimensions='{0}x{1}'.format(*export_shape),
)
logging.info(' Starting export task')
utils.ee_task_start(task)
ready_task_count += 1
# logging.debug(f' Ready tasks: {ready_task_count}')
# Pause before starting the next date (not export task)
ready_task_count = delay_task(
delay_time=delay_time, task_max=ready_task_max,
task_count=ready_task_count)
import ee
from ee_plugin import Map
# Define a region of interest as a point. Change the coordinates
# to get a classification of any place where there is imagery.
roi = ee.Geometry.Point(-122.3942, 37.7295)
# Load Landsat 5 input imagery.
landsat = ee.Image(ee.ImageCollection('LANDSAT/LT05/C01/T1_TOA')
# Filter to get only one year of images. \
.filterDate('2011-01-01', '2011-12-31')
# Filter to get only images under the region of interest. \
.filterBounds(roi)
# Sort by scene cloudiness, ascending. \
.sort('CLOUD_COVER')
# Get the first (least cloudy) scene. \
.first())
# Compute cloud score.
cloudScore = ee.Algorithms.Landsat.simpleCloudScore(landsat).select('cloud')
# Mask the input for clouds. Compute the min of the input mask to mask
# pixels where any band is masked. Combine that with the cloud mask.
input = landsat.updateMask(landsat.mask().reduce('min').And(cloudScore.lte(50)))
# Use MODIS land cover, IGBP classification, for training.
modis = ee.Image('MODIS/051/MCD12Q1/2011_01_01') \
.select('Land_Cover_Type_1')
# Sample the input imagery to get a FeatureCollection of training data.
training = input.addBands(modis).sample(**{
def addNDVI(image):
return image.addBands(image.normalizedDifference(['B5', 'B4']))
# This function masks cloudy pixels.
def cloudMask(image):
clouds = ee.Algorithms.Landsat.simpleCloudScore(image).select(['cloud'])
return image.updateMask(clouds.lt(10))
# Load a Landsat collection, map the NDVI and cloud masking functions over it.
collection = ee.ImageCollection('LANDSAT/LC08/C01/T1_TOA') \
.filterBounds(ee.Geometry.Point([-122.262, 37.8719])) \
.filterDate('2014-03-01', '2014-05-31') \
.map(addNDVI) \
.map(cloudMask)
# Reduce the collection to the mean of each pixel and display.
meanImage = collection.reduce(ee.Reducer.mean())
vizParams = {'bands': ['B5_mean', 'B4_mean', 'B3_mean'], 'min': 0, 'max': 0.5}
Map.setCenter(-122.262, 37.8719, 10)
Map.addLayer(meanImage, vizParams, 'mean')
# Load a region in which to compute the mean and display it.
counties = ee.FeatureCollection('TIGER/2016/Counties')
santaClara = ee.Feature(
counties.filter(ee.Filter.eq('NAME', 'Santa Clara')).first())
Map.addLayer(ee.Image().paint(santaClara, 0, 2), {'palette': 'yellow'},
'Santa Clara')
pdf = pdf.iloc[:48]
X = pdf.values
return X
X = [] #create empty arrays
Y = []
ee.Initialize()
# %% get data 18/19
for l in range(df_1819.shape[0]):
punto = df_1819.iloc[l]
p = ee.Geometry.Point(punto.Longitud, punto.Latitud).buffer(100)
S2_collection = ee.ImageCollection("COPERNICUS/S2") \
.filterBounds(p) \
.filterDate('2018-09-01', '2019-04-30') \
.sort('system:id', opt_ascending=True) \
p_get = p
featCol = S2_collection.map(get_data)
pdf = get_pd(featCol)
X.append(get_X(pdf))
Y.append(punto.Cultivo)
# %% get data 19/20
for l in range(df_1920.shape[0]):
punto = df_1920.iloc[l]
p = ee.Geometry.Point(punto.Longitud, punto.Latitud).buffer(100)
S2_collection = ee.ImageCollection("COPERNICUS/S2") \
.filterBounds(p) \
.filterDate('2019-09-01', '2020-04-30') \
# Add a band containing image date as years since 1990.
def createTimeBand(img):
year = img.date().difference(ee.Date('1990-01-01'), 'year')
return ee.Image(year).float().addBands(img)
# function createTimeBand(img) {
# year = img.date().difference(ee.Date('1990-01-01'), 'year')
# return ee.Image(year).float().addBands(img)
# }
# Fit a linear trend to the nighttime lights collection.
collection = ee.ImageCollection('NOAA/DMSP-OLS/CALIBRATED_LIGHTS_V4') \
.select('avg_vis') \
.map(createTimeBand)
fit = collection.reduce(ee.Reducer.linearFit())
# Display a single image
Map.addLayer(ee.Image(collection.select('avg_vis').first()), {
'min': 0,
'max': 63
}, 'stable lights first asset')
# Display trend in red/blue, brightness in green.
Map.setCenter(30, 45, 4)
Map.addLayer(fit, {
'min': 0,
'max': [0.18, 20, -0.18],
'bands': ['scale', 'offset', 'scale']
def ts_extract(sensor,
start,
tiers=['T1', 'T2'],
lon=None,
lat=None,
end=datetime.today(),
radius=None,
feature=None,
bands=None,
stats='mean'):
"""Perform a spatio temporal query to extract Landsat surface reflectance data
from gee
Args:
lon (float): Center longitude in decimal degree
lat (float): Center latitude in decimal degree
sensor (str): Landsat sensor to query data from. Must be one of 'LT4',
'LT5', 'LE7', 'LC8'
tiers (list): List of tiers to order. ``'T1'`` corresponds to tiers 1.
Default is ``['T1', 'T2']``
start (datetime.datetime): Start date
end (datetime.datetime): Optional end date; automatically set as today if unset
radius (float): Optional radius around center point in meters. If unset,
time-series of a single pixel are queried. Otherwise a reducer is used
to spatially aggregate the pixels intersecting the circular feature
built.
feature (dict): Optional dictionary representation of a polygon feature
in longlat CRS. If unset, time-series of a single pixel are queried.
Otherwise a reducer is used to spatially aggregate the pixels intersecting
the given feature.
bands (list): List of Landsat band names. Optional, defaults to
['B1', 'B2', 'B3', 'B4', 'B5', 'B7'] in the case of LC8 sensor and to
['B1', 'B2', 'B3', 'B4', 'B5', 'B7'] otherwise.
stats (str): Spatial aggregation function to use. Only relevant
if a radius value is set.
Returns:
dict: A dictionary representation of the json data returned by the gee platform.
Example:
>>> import geextract
>>> from pprint import pprint
>>> from datetime import datetime
>>> lon = -89.8107197
>>> lat = 20.4159611
>>> out = geextract.ts_extract(lon=lon, lat=lat, sensor='LE7', start=datetime(1980, 1, 1, 0, 0),
>>> radius=500)
>>> pprint(out)
"""
# Define some internal functions to be mapped over imageCollections
def _mask_clouds(image):
"""Cloud masking function"""
# collection 1 cloud masking example
# https://code.earthengine.google.com/52e39cc00de3471c905509e374c52284
# Pre collecction masking example
# https://code.earthengine.google.com/37ffd688d1b2d2c977fa5c536a023356
# collection must be a variable of the parent environment
clear = image.select('pixel_qa').bitwiseAnd(0x2).neq(0)
valid_range_mask = image.gte(0).And(image.lte(10000))
return image.updateMask(clear).updateMask(valid_range_mask)
# Check inputs
if sensor not in ['LT4', 'LT5', 'LC8', 'LE7']:
raise ValueError('Unknown sensor (Must be one of LT4, LT5, LE7, LC8)')
if bands is None:
if sensor in ['LT4', 'LT5', 'LE7']:
bands = ['B1', 'B2', 'B3', 'B4', 'B5', 'B7']
else:
bands = ['B2', 'B3', 'B4', 'B5', 'B6', 'B7']
sensor = re.sub(r'(LC|LT|LE)(\d{1})', r'\g<1>0\g<2>', sensor)
collection_name_template = 'LANDSAT/%s/C01/%%s_SR' % sensor
# Iterate over tiers to load and merge all corresponding image collections
landsat_ic = ee.ImageCollection(collection_name_template % tiers[0])
for tier in tiers[1:]:
tier_ic = ee.ImageCollection(collection_name_template % tier)
landsat_ic = ee.ImageCollection(landsat_ic.merge(tier_ic))
# Prepare image collection
landsat = landsat_ic.\
filterDate(start=start, opt_end=end)\
.map(_mask_clouds)\
.select(bands)
if radius is not None or feature is not None:
# Define spatial aggregation function
if stats == 'mean':
fun = ee.Reducer.mean()
elif stats == 'median':
fun = ee.Reducer.median()
elif stats == 'max':
fun = ee.Reducer.max()
elif stats == 'min':
fun = ee.Reducer.min()
else:
raise ValueError(
'Unknown spatial aggregation function. Must be one of mean, median, max, or min'
)
if feature is not None:
geometry = ee.Geometry.Polygon(feature['geometry']['coordinates'])
else: # Geometry defined by point and radius
geometry = ee.Geometry.Point(lon, lat).buffer(radius)
# Define function to map over imageCollection to perform spatial aggregation
def _reduce_region(image):
"""Spatial aggregation function for a single image and a polygon feature"""
stat_dict = image.reduceRegion(fun, geometry, 30)
# FEature needs to be rebuilt because the backend doesn't accept to map
# functions that return dictionaries
return ee.Feature(None, stat_dict)
fc = landsat.filterBounds(geometry).map(_reduce_region).getInfo()
out = simplify(fc)
else:
# Extraction with a point, no spatial aggregation, etc
geometry = ee.Geometry.Point(lon, lat)
l = landsat.filterBounds(geometry).getRegion(geometry, 30).getInfo()
out = dictify(l)
# pop longitude and lattitude keys from dict collection so that band aliases can
# be replaced by their color names
[d.pop('longitude', None) for d in out]
[d.pop('latitude', None) for d in out]
[d.pop('time', None) for d in out]
return out
def Get_Original_and_Fitting_df(self):
# if the get_original_val is False, skip Fourier transformation
if self.get_original_val:
# Making the Fourier map
self.Stp_1_Create_hamonic_names()
self.Stp_2_Add_harmonics()
self.Stp_3_Harmonic_fit()
print('1/5 ==> Fourier Transformation Finished!')
else:
print('1/5 ==> Fourier Transformation skiped!')
#__________________________Get the original [NDVI,NDBI,EVI] from harmonicLandsat_____________________
# if the get_original_val is False, skip Original Image Value acquiration
if self.get_original_val:
# get the necessary data
original_value = self.harmonicLandsat.select(
self.Normalized_Index).getRegion(self.point, 30).getInfo()
# convert original value to dataframe
original_df = pd.DataFrame(original_value[1:],
columns=original_value[0])
original_df['time'] = pd.to_datetime(original_df['time'],
unit='ms')
# formating the df
original_df.drop(['id', 'longitude', 'latitude'], 1, inplace=True)
original_df.sort_values('time', inplace=True)
original_df.reset_index(inplace=True, drop=True)
# add attribute to self
self.original_df = original_df
print('2/5 ==> Original Image Value df Acquired!')
else:
print('2/5 ==> Original Image Value acquiration skiped!')
#____________________________________Create indpendents_____________________________________________
independents = {}
# set a list to store t
date_list = ee.List.sequence(0, 365.25 * self.advance_year_num, 1)
date_list = date_list.map(
lambda x: ee.Date(self.start_date).advance(x, 'day'))
# put the data to independents
time_stamp = date_list.map(lambda x: ee.Date(x).format()).getInfo()
independents.update({'time': [i[:10] for i in time_stamp]})
# Compute time in fractional years since the epoch.
years = date_list.map(lambda x: ee.Date(x).difference(
ee.Date('1970-01-01'), 'year').multiply(2 * 3.1415926)).getInfo()
# add t to independents
independents['t'] = np.array(years)
# get other derivative time independents
for i in range(1, self.harmonics + 1):
# Get the cosine terms.
cos = {f'cos_{i}': np.cos(np.array(years) * np.array(i))}
# Get the sin terms.
sin = {f'sin_{i}': np.sin(np.array(years) * np.array(i))}
# add cos and sin bands to image
independents.update(cos)
independents.update(sin)
# add constant to independents
independents.update({'constant': 1})
independents = pd.DataFrame(independents)
# add independents to self for check purpose
self.independents = independents
print('3/5 ==> Independents Values Computation Completed!')
#___________________________Get the coefficients from Fourier img value________________________
# if the get_original_val is False, skip Fourier transformation
if self.get_fitted_val:
# if the year_range is not in the research periods, and the harmonic is not 3,
# then do the calculation all again
year_span = [f'{i}_{i+2}' for i in range(1990, 2018, 3)]
if self.year_range in year_span[:5] and self.harmonics == 3:
# import Fourier img
print(
'________________Using GEE assest to reduce time of Fourier transformation!__________'
)
Fourier_img = ee.Image(
f"users/Jinzhu_Deakin/North_China_Plain/Fourier_img_harmonic_3/Fourier_img_{self.year_range}_harmonic_3"
)
elif self.year_range in year_span[5:] and self.harmonics == 3:
# import Fourier img
print(
'________________Using GEE assest to reduce time of Fourier transformation!__________'
)
Fourier_img = ee.Image(
f"users/wang8052664/North_China_Plain/Fourier_imgs/Fourier_img_{self.year_range}_harmonic_3"
)
else:
# get the Fourier_img
Fourier_img = self.harmonicTrendCoefficients
# get the value of Foutire img at Fig_pt,
#recall that the img was multipied by 1000 and converted into an integer
Fig_pt_img_value = ee.ImageCollection(Fourier_img).getRegion(
self.point, 30).getInfo()
# get the coefficients and drop uesless columns
coefficients = pd.DataFrame(Fig_pt_img_value[1:],
columns=Fig_pt_img_value[0])
coefficients.drop(['id', 'longitude', 'latitude', 'time'],
1,
inplace=True)
coefficients = coefficients.T
# divede by 1000, because the Fourier has been multiplied by 1000 at the Fourier fitting process
coefficients = coefficients / 1000
# add coefficients to self for check purpose
self.coefficients = coefficients
print('4/5 ==> Coefficients Acquiring Finished!')
#________________________Restore the fitted value using matrix multiply___________________________
fitted_df_list = []
# loop through indexes and use matrix multiply to get the fitted values
for idx in self.Normalized_Index:
X = independents[self.independents_idx].values
y = coefficients.loc[[
s for s in coefficients.index if idx in s
]].values
fitted_df_list.append(
pd.DataFrame({idx: (X @ y).reshape(1, -1)[0]}))
#_____________________________get the fitted value and add time to it_________________________________
fitted_df = pd.concat(fitted_df_list, 1)
fitted_df['time'] = pd.to_datetime(independents['time'])
self.fitted_df = fitted_df
print('5/5 ==> Fitted Value Computation Completed!')
else:
print('5/5 ==> Fitted Value Computation Skipped!')
