logging.basicConfig(filename='warnings.log',level=logging.DEBUG)
download_dest = "/home/leo/landsat8_1x1_grid/"
check_dest = "/home/leo/landsat_1x1_new/"
ee.Initialize()
# Select 1x1 degree tiles on North America
northamerica = ee.FeatureCollection('ft:1Ep6prwb941jMjmMxqD_BsISZj6JCiAE76mFcAiYl') \
.filterMetadata('CONTINENT', 'equals', 'North America')
tiles = ee.FeatureCollection('ft:15P7IFF53wKVGefe_c6Cb2RBnsUql-wgMAX5OBcmq')
spatialFilter = ee.Filter.intersects('.geo', None, '.geo', None, 10)
saveAllJoin = ee.Join.saveAll('scenes')
intersectJoined = saveAllJoin.apply(northamerica, tiles, spatialFilter)
northamerica_tiles = ee.List(intersectJoined.first().get('scenes'))
time.sleep(15)
numTiles = 4366
for i in range(35, numTiles - 1):
checkpath = os.path.join(check_dest, 'tile' + str(i) + '.zip')
savepath = os.path.join(download_dest, 'tile' + str(i) + '.zip')
#check if file exists
if(os.path.isfile(checkpath)):
logging.warning('tile:' + str(i) + ' exists, skipping...')
continue
if(os.path.isfile(savepath)):
def test_Container_Length(self):
"""Test the Container Emulation Methods"""
test = len(ee.List([1, 2, 3]))
self.assertIsInstance(test, int)
def test_binary1(self):
"""Test the eeList module for binary operators 1"""
binary1_tested = ee.List([1, 2, 3]) + ee.List([4, 5, 6]) * 5
self.assertIsInstance(binary1_tested, ee.ee_list.List)
def bootstrapOtsu(collection,target_date, reductionPolygons,
neg_buffer=-1500, # negative buffer for masking potential bad data
upper_threshold=-14, # upper limit for water threshold
canny_threshold=7, # threshold for canny edge detection
canny_sigma=1, # sigma value for gaussian filter
canny_lt=7, # lower threshold for canny detection
smoothing=100, # amount of smoothing in meters
connected_pixels=200, # maximum size of the neighborhood in pixels
edge_length=50, # minimum length of edges from canny detection
smooth_edges=100,
qualityBand=None,
reverse=False,
reductionScale=90):
tDate = ee.Date(target_date)
targetColl = collection.filterDate(tDate,tDate.advance(1,'day'))
nImgs = targetColl.size().getInfo()
if nImgs <= 0:
raise EEException('Selected date has no imagery, please try processing another date')
collGeom = targetColl.geometry()
polygons = reductionPolygons.filterBounds(collGeom)
nPolys = polygons.size().getInfo()
if nPolys > 0:
ids = ee.List(polygons.aggregate_array('id'))
random_ids = []
for i in range(3):
random_ids.append(random.randint(0, ids.size().subtract(1).getInfo()))
random_ids = ee.List(random_ids)
def getRandomIds(i):
return ids.get(i)
ids = random_ids.map(getRandomIds)
polygons = polygons.filter(ee.Filter.inList('id', ids))
if qualityBand == None:
target = targetColl.mosaic().set('system:footprint', collGeom.dissolve())
target = target.clip(target.geometry().buffer(neg_buffer))
smoothed = target.focal_median(smoothing, 'circle', 'meters')
histBand = ee.String(target.bandNames().get(0))
else:
target = targetColl.qualityMosaic(qualityBand).set('system:footprint', collGeom.dissolve())
target = target.clip(target.geometry().buffer(neg_buffer))
smoothed = target.focal_median(smoothing, 'circle', 'meters')
histBand = ee.String(qualityBand)
canny = ee.Algorithms.CannyEdgeDetector(smoothed,canny_threshold,canny_sigma)
connected = canny.mask(canny).lt(canny_lt).connectedPixelCount(connected_pixels, True)
edges = connected.gte(edge_length)
edgeBuffer = edges.focal_max(smooth_edges, 'square', 'meters')
histogram_image = smoothed.mask(edgeBuffer)
histogram = histogram_image.reduceRegion(ee.Reducer.histogram(255, 2),polygons.geometry(),reductionScale,bestEffort=True)
threshold = ee.Number(otsu_function(histogram.get(histBand))).min(upper_threshold)
else:
threshold = upper_threshold
water = smoothed.lt(threshold).clip(geeutils.LAND.geometry())
return water
def test_Container_Get_Item_By_Index(self):
"""Test the Container Emulation Methods"""
test = ee.List([1, 2, 3])[0]
self.assertIsInstance(test, ee.computedobject.ComputedObject)
def line(start, end):
return ee.Geometry.LineString(ee.List([start, end]))
def y(point):
return ee.Number(ee.List(point).get(1))
def used_ids_sample(image, aoi, ids):
samples = image.select('index').uint16().sample(region=aoi.geometry(),
scale=300,
factor=0.01)
usedIndexes = ee.List(samples.distinct('index').aggregate_array('index'))
return usedIndexes.map(lambda index: ids.get(index))
def arrayToPairs(array, startDate, EndDate, delta=1, CLOUDY_PIXEL_PERCENTAGE=10):
ee.Initialize()
area = ee.Geometry.Polygon(array)
# query
collection_NDVI = ee.ImageCollection("COPERNICUS/S2").filterBounds(area) \
.filterDate(startDate, EndDate) \
.filterMetadata("CLOUDY_PIXEL_PERCENTAGE", "less_than", CLOUDY_PIXEL_PERCENTAGE) \
.select(['B8', 'B4', 'QA60']) \
.sort('date')
NDVI_size = collection_NDVI.size().getInfo()
l_NDVI = collection_NDVI.toList(NDVI_size)
l_NDVI_dates = getDates(l_NDVI, NDVI_size, process=False)
# use NDVI dates to make date-filter for SAR data, taking into account
valid_dates = None
for i_NDVI in range(NDVI_size):
NDVI_date = l_NDVI_dates[i_NDVI]
if not valid_dates:
valid_dates = ee.Filter.date(NDVI_date.advance(-delta, 'day'), NDVI_date.advance(delta, 'day'))
else:
valid_dates = ee.Filter.Or(valid_dates, ee.Filter.date(NDVI_date.advance(-delta, 'day'),
NDVI_date.advance(delta, 'day')))
if (not valid_dates):
raise ValueError("no valid dates for the image")
collection_SAR = ee.ImageCollection('COPERNICUS/S1_GRD').filterBounds(area) \
.filter(valid_dates) \
.select(['VH', 'VV'])
SAR_size = collection_SAR.size().getInfo()
l_SAR = collection_SAR.toList(SAR_size)
l_SAR_dates = getDates(l_SAR, SAR_size)
l_NDVI_dates = getDates(l_NDVI, NDVI_size)
# map collections to their respective special data formats
collection_NDVI = collection_NDVI.map(getNDVI.getNDVI)
l_NDVI = collection_NDVI.toList(NDVI_size)
# NDVI-[SAR] list
pairs_i = {}
for i_SAR in range(SAR_size):
SAR_date = l_SAR_dates[i_SAR]
for i_NDVI in range(NDVI_size):
NDVI_date = l_NDVI_dates[i_NDVI]
if abs(SAR_date - NDVI_date) <= timedelta(days=delta):
if i_NDVI not in pairs_i:
pairs_i[i_NDVI] = [i_SAR]
else:
pairs_i[i_NDVI].append(i_SAR)
sorted_pairs = sorted(pairs_i)
print(pairs_i)
arr = []
precomputed_SAR = dict()
LatLonImgsNDVI = ee.List(
[common.fastLatLonImg(ee.Image(l_NDVI.get(NDVI)), area) for NDVI in sorted_pairs])
LatLonImgsSAR = ee.List([LatLonImgVHVV(ee.Image(l_SAR.get(SAR)), area) for SAR in range(SAR_size)])
both_lists = ee.List([LatLonImgsNDVI, LatLonImgsSAR]).getInfo()
LatLonImgsNDVI = both_lists[0]
LatLonImgsSAR = both_lists[1]
for NDVI in range(len(LatLonImgsNDVI)):
i = sorted_pairs[NDVI]
for SAR in pairs_i[i]:
ndvi_temp = (LatLonImgsNDVI[NDVI][0], LatLonImgsNDVI[NDVI][1], LatLonImgsNDVI[NDVI][2])
if SAR not in precomputed_SAR:
lats = LatLonImgsSAR[SAR][0]
lons = LatLonImgsSAR[SAR][1]
vh = LatLonImgsSAR[SAR][2]
vv = LatLonImgsSAR[SAR][3]
precomputed_SAR[SAR] = []
precomputed_SAR[SAR].append((lats, lons, vh) + (f'SAR (VH) {l_SAR_dates[SAR]:%B %d, %Y}',))
precomputed_SAR[SAR].append((lats, lons, vv) + (f'SAR (VV) {l_SAR_dates[SAR]:%B %d, %Y}',))
arr.append(ndvi_temp + (f'NDVI {l_NDVI_dates[NDVI]:%B %d, %Y}',))
arr.extend(precomputed_SAR[SAR])
return rasteriser.rasteriseImages(arr)
CRS = 'EPSG:4326'
SCALE = 30
NULL_VALUE = 0
MASK_VALUE = 0 # used for ADDON image
startJulian = 152 # start april
endJulian = 243 # early oct
# parts to divide into if 'too many values' error
n = 10
# wait time
wait = 10 # seconds
# values to copy from fusion table/feature collection
feat_properties = ['site']
KEYS = ee.List(feat_properties)
# properties to retrieve from scene
scene_properties = [
'CLOUD_COVER', 'GEOMETRIC_RMSE_MODEL', 'LANDSAT_ID', 'SOLAR_ZENITH_ANGLE'
]
PROPERTY_LIST = ee.List(scene_properties)
# Bands to retrieve
bands = ['B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'pixel_qa', 'radsat_qa']
BAND_LIST = ee.List(bands)
# addon asset and bands
ADDON = ee.Image('JRC/GSW1_0/GlobalSurfaceWater').float().unmask(MASK_VALUE)
ADDON_BANDLIST = ee.List(['max_extent'])
def _combine(this, prev):
return ee.List(prev).add(this)
# end and start date of period of interest
start_date = ee.Date.fromYMD(2001, 1, 1)
end_date = ee.Date.fromYMD(2019, 12, 31)
# add dn
collection = collection.map(add_dn_date_all(Year = False, days = 8))
# mask out crap pixels
MOD09masked = collection.filterDate(start_date, end_date).map(maskPixels)
# add NDVI as a new band
MOD09ndvi = MOD09masked.map(addNDVI).select('NDVI')
tmpseas = ["%02d" % x for x in list(range(1, 46+1))]
tmpseas1 = ee.List(tmpseas)
# get year from bash environment
year = int(os.environ["year"])
# climatology
yearp5 = ee.Number(year).add(5)
start_date = ee.Date.fromYMD(year,1,1)
end_date = ee.Date.fromYMD(yearp5,12,31)
MOD09ndviY = MOD09ndvi.filterDate(start_date, end_date)
MinNDVI = MOD09ndviY.min()
meanNDVI = MOD09ndviY.mean()
# climatology 5 years (monthly composites)
seasons = ee.ImageCollection.fromImages(tmpseas1.map(clim5y)) # problems start here!!!
def main(ini_path=None, overwrite_flag=False, delay_time=0, gee_key_file=None,
max_ready=-1, reverse_flag=False):
"""Compute annual Tcorr images from scene images
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
delay_time : float, optional
Delay time in seconds between starting export tasks (or checking the
number of queued tasks, see "max_ready" parameter). The default is 0.
gee_key_file : str, None, optional
Earth Engine service account JSON key file (the default is None).
max_ready: int, optional
Maximum number of queued "READY" tasks. The default is -1 which is
implies no limit to the number of tasks that will be submitted.
reverse_flag : bool, optional
If True, process WRS2 tiles in reverse order.
"""
logging.info('\nCompute annual Tcorr images from scene images')
ini = utils.read_ini(ini_path)
model_name = 'SSEBOP'
# model_name = ini['INPUTS']['et_model'].upper()
tmax_name = ini[model_name]['tmax_source']
export_id_fmt = 'tcorr_scene_{product}_{wrs2}_annual_from_scene'
asset_id_fmt = '{coll_id}/{wrs2}'
tcorr_annual_coll_id = '{}/{}_annual_from_scene'.format(
ini['EXPORT']['export_coll'], tmax_name.lower())
wrs2_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/wrs2_descending_custom'
wrs2_tile_field = 'WRS2_TILE'
# wrs2_path_field = 'ROW'
# wrs2_row_field = 'PATH'
try:
wrs2_tiles = str(ini['INPUTS']['wrs2_tiles'])
wrs2_tiles = [x.strip() for x in wrs2_tiles.split(',')]
wrs2_tiles = sorted([x.lower() for x in wrs2_tiles if x])
except KeyError:
wrs2_tiles = []
logging.debug(' wrs2_tiles: not set in INI, defaulting to []')
except Exception as e:
raise e
try:
study_area_extent = str(ini['INPUTS']['study_area_extent']) \
.replace('[', '').replace(']', '').split(',')
study_area_extent = [float(x.strip()) for x in study_area_extent]
except KeyError:
study_area_extent = None
logging.debug(' study_area_extent: not set in INI, defaulting to None')
except Exception as e:
raise e
# TODO: Add try/except blocks and default values?
# TODO: Filter Tcorr scene collection based on collections parameter
# collections = [x.strip() for x in ini['INPUTS']['collections'].split(',')]
cloud_cover = float(ini['INPUTS']['cloud_cover'])
min_pixel_count = float(ini['TCORR']['min_pixel_count'])
min_scene_count = float(ini['TCORR']['min_scene_count'])
if (tmax_name.upper() == 'CIMIS' and
ini['INPUTS']['end_date'] < '2003-10-01'):
logging.error(
'\nCIMIS is not currently available before 2003-10-01, exiting\n')
sys.exit()
elif (tmax_name.upper() == 'DAYMET' and
ini['INPUTS']['end_date'] > '2018-12-31'):
logging.warning(
'\nDAYMET is not currently available past 2018-12-31, '
'using median Tmax values\n')
# sys.exit()
# elif (tmax_name.upper() == 'TOPOWX' and
# ini['INPUTS']['end_date'] > '2017-12-31'):
# logging.warning(
# '\nDAYMET is not currently available past 2017-12-31, '
# 'using median Tmax values\n')
# # sys.exit()
logging.info('\nInitializing Earth Engine')
if gee_key_file:
logging.info(' Using service account key file: {}'.format(gee_key_file))
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('x', key_file=gee_key_file),
use_cloud_api=True)
else:
ee.Initialize(use_cloud_api=True)
logging.debug('\nTmax properties')
tmax_source = tmax_name.split('_', 1)[0]
tmax_version = tmax_name.split('_', 1)[1]
tmax_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower())
tmax_coll = ee.ImageCollection(tmax_coll_id)
tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0)
logging.debug(' Collection: {}'.format(tmax_coll_id))
logging.debug(' Source: {}'.format(tmax_source))
logging.debug(' Version: {}'.format(tmax_version))
# Get the Tcorr scene image collection properties
logging.debug('\nTcorr scene collection')
tcorr_scene_coll_id = '{}/{}_scene'.format(
ini['EXPORT']['export_coll'], tmax_name.lower())
logging.debug('\nExport properties')
export_info = utils.get_info(ee.Image(tmax_mask))
if 'daymet' in tmax_name.lower():
# Custom smaller extent for DAYMET focused on CONUS
export_extent = [-1999750, -1890500, 2500250, 1109500]
export_shape = [4500, 3000]
export_geo = [1000, 0, -1999750, 0, -1000, 1109500]
# Custom medium extent for DAYMET of CONUS, Mexico, and southern Canada
# export_extent = [-2099750, -3090500, 2900250, 1909500]
# export_shape = [5000, 5000]
# export_geo = [1000, 0, -2099750, 0, -1000, 1909500]
export_crs = export_info['bands'][0]['crs']
else:
export_crs = export_info['bands'][0]['crs']
export_geo = export_info['bands'][0]['crs_transform']
export_shape = export_info['bands'][0]['dimensions']
# export_geo = ee.Image(tmax_mask).projection().getInfo()['transform']
# export_crs = ee.Image(tmax_mask).projection().getInfo()['crs']
# export_shape = ee.Image(tmax_mask).getInfo()['bands'][0]['dimensions']
export_extent = [
export_geo[2], export_geo[5] + export_shape[1] * export_geo[4],
export_geo[2] + export_shape[0] * export_geo[0], export_geo[5]]
export_geom = ee.Geometry.Rectangle(
export_extent, proj=export_crs, geodesic=False)
logging.debug(' CRS: {}'.format(export_crs))
logging.debug(' Extent: {}'.format(export_extent))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
if study_area_extent is None:
if 'daymet' in tmax_name.lower():
# CGM - For now force DAYMET to a slightly smaller "CONUS" extent
study_area_extent = [-125, 25, -65, 49]
# study_area_extent = [-125, 25, -65, 52]
elif 'cimis' in tmax_name.lower():
study_area_extent = [-124, 35, -119, 42]
else:
# TODO: Make sure output from bounds is in WGS84
study_area_extent = tmax_mask.geometry().bounds().getInfo()
logging.debug(f'\nStudy area extent not set in INI, '
f'default to {study_area_extent}')
study_area_geom = ee.Geometry.Rectangle(
study_area_extent, proj='EPSG:4326', geodesic=False)
if not ee.data.getInfo(tcorr_annual_coll_id):
logging.info('\nExport collection does not exist and will be built'
'\n {}'.format(tcorr_annual_coll_id))
input('Press ENTER to continue')
ee.data.createAsset({'type': 'IMAGE_COLLECTION'}, tcorr_annual_coll_id)
# Get current asset list
logging.debug('\nGetting GEE asset list')
asset_list = utils.get_ee_assets(tcorr_annual_coll_id)
# if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
# pprint.pprint(asset_list[:10])
# Get current running tasks
tasks = utils.get_ee_tasks()
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
logging.debug(' Tasks: {}\n'.format(len(tasks)))
input('ENTER')
# Limit by year
month_list = list(range(1, 13))
# try:
# month_list = sorted(list(utils.parse_int_set(ini['TCORR']['months'])))
# except:
# logging.info('\nTCORR "months" parameter not set in the INI,'
# '\n Defaulting to all months (1-12)\n')
# month_list = list(range(1, 13))
try:
year_list = sorted(list(utils.parse_int_set(ini['TCORR']['years'])))
except:
logging.info('\nTCORR "years" parameter not set in the INI,'
'\n Defaulting to all available years\n')
year_list = []
# Get the list of WRS2 tiles that intersect the data area and study area
wrs2_coll = ee.FeatureCollection(wrs2_coll_id) \
.filterBounds(export_geom) \
.filterBounds(study_area_geom)
if wrs2_tiles:
wrs2_coll = wrs2_coll.filter(ee.Filter.inList(wrs2_tile_field, wrs2_tiles))
wrs2_info = wrs2_coll.getInfo()['features']
# Iterate over date ranges
for wrs2_ftr in sorted(wrs2_info,
key=lambda k: k['properties']['WRS2_TILE'],
reverse=reverse_flag):
wrs2_tile = wrs2_ftr['properties'][wrs2_tile_field]
logging.info('{}'.format(wrs2_tile))
wrs2_path = int(wrs2_tile[1:4])
wrs2_row = int(wrs2_tile[5:8])
# wrs2_path = wrs2_ftr['properties'][wrs2_path_field]
# wrs2_row = wrs2_ftr['properties'][wrs2_row_field]
export_id = export_id_fmt.format(
product=tmax_name.lower(), wrs2=wrs2_tile)
logging.debug(' Export ID: {}'.format(export_id))
asset_id = asset_id_fmt.format(
coll_id=tcorr_annual_coll_id, wrs2=wrs2_tile)
logging.debug(' Asset ID: {}'.format(asset_id))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id]['id'])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if asset_id in asset_list:
logging.debug(' Asset already exists, removing')
ee.data.deleteAsset(asset_id)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, exiting')
continue
elif asset_id in asset_list:
logging.debug(' Asset already exists, skipping')
continue
tcorr_coll = ee.ImageCollection(tcorr_scene_coll_id) \
.filterMetadata('wrs2_tile', 'equals', wrs2_tile) \
.filterMetadata('tcorr_pixel_count', 'not_less_than', min_pixel_count) \
.filter(ee.Filter.inList('year', year_list))
# TODO: Should CLOUD_COVER_LAND filter should be re-applied here?
# .filterMetadata('CLOUD_COVER_LAND', 'less_than', cloud_cover)
# .filterDate(start_date, end_date)
# .filterBounds(ee.Geometry(wrs2_ftr['geometry']))
# Use a common reducer for the images and property stats
reducer = ee.Reducer.median() \
.combine(ee.Reducer.count(), sharedInputs=True)
# Compute stats from the collection images
# This might be used when Tcorr is spatial
# tcorr_img = tcorr_coll.reduce(reducer).rename(['tcorr', 'count'])
# Compute stats from the image properties
tcorr_stats = ee.List(tcorr_coll.aggregate_array('tcorr_value')) \
.reduce(reducer)
tcorr_stats = ee.Dictionary(tcorr_stats) \
.combine({'median': 0, 'count': 0}, overwrite=False)
tcorr = ee.Number(tcorr_stats.get('median'))
count = ee.Number(tcorr_stats.get('count'))
index = count.lt(min_scene_count).multiply(7).add(2)
# index = ee.Algorithms.If(count.gte(min_scene_count), 2, 9)
# Clip the mask image to the Landsat footprint
# Change mask values to 1 if count >= threshold
# Mask values of 0 will be set to nodata
mask_img = tmax_mask.add(count.gte(min_scene_count)) \
.clip(ee.Geometry(wrs2_ftr['geometry']))
output_img = ee.Image(
[mask_img.multiply(tcorr), mask_img.multiply(count)]) \
.rename(['tcorr', 'count']) \
.updateMask(mask_img.unmask(0))
# # Write an empty image if the pixel count is too low
# # CGM: Check/test if this can be combined into a single If()
# tcorr_img = ee.Algorithms.If(
# count.gte(min_scene_count),
# tmax_mask.add(tcorr), tmax_mask.updateMask(0))
# count_img = ee.Algorithms.If(
# count.gte(min_scene_count),
# tmax_mask.add(count), tmax_mask.updateMask(0))
#
# # Clip to the Landsat image footprint
# output_img = ee.Image([tcorr_img, count_img]) \
# .rename(['tcorr', 'count']) \
# .clip(ee.Geometry(wrs2_ftr['geometry']))
# # Clear the transparency mask
# output_img = output_img.updateMask(output_img.unmask(0))
output_img = output_img.set({
'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'),
'model_name': model_name,
'model_version': ssebop.__version__,
# 'system:time_start': utils.millis(start_dt),
'tcorr_value': tcorr,
'tcorr_index': index,
'tcorr_scene_count': count,
'tmax_source': tmax_source.upper(),
'tmax_version': tmax_version.upper(),
'wrs2_path': wrs2_path,
'wrs2_row': wrs2_row,
'wrs2_tile': wrs2_tile,
'years': ','.join(map(str, year_list)),
# 'year_start': year_list[0],
# 'year_end': year_list[-1],
})
# pprint.pprint(output_img.getInfo())
# input('ENTER')
logging.debug(' Building export task')
task = ee.batch.Export.image.toAsset(
image=output_img,
description=export_id,
assetId=asset_id,
crs=export_crs,
crsTransform='[' + ','.join(list(map(str, export_geo))) + ']',
dimensions='{0}x{1}'.format(*export_shape),
)
logging.info(' Starting export task')
utils.ee_task_start(task)
# Pause before starting the next export task
utils.delay_task(delay_time, max_ready)
logging.debug('')
def unique_properties(coll, property):
return ee.String(ee.List(ee.Dictionary(
coll.aggregate_histogram(property)).keys()).join(','))
def get_data_from_image():
# static values
scale_value = 1000
nameOfArea = "polygon"
# Extract image or image collection name from request
image_name = request.json.get('imageName')
# Load the SRTM image. Handling error for image vs image collection
try:
image_data = ee.ImageCollection(image_name).filterBounds(polygon)
# load image
srtm = ee.Image(image_data.first()).clip(polygon)
except:
image_data = ee.Image(image_name).filterBounds(polygon)
# load first image from collection
srtm = ee.Image(image_data).clip(polygon)
# Polygon region
region = ee.Geometry.Rectangle(-122.2806, 37.1209, -122.0554, 37.2413)
# meanDict = srtm.reduceRegion(reducer=ee.Reducer.toList(), geometry=region, scale=30)
# Compute the mean elevation in the polygon.
meanDict = srtm.reduceRegion( reducer= ee.Reducer.toList(), geometry= polygon, scale= scale_value)
# print('meanDict:', meanDict)
# Get the mean from the dictionary and print it.
mean = meanDict.get('elevation');
# print('Mean elevation', mean);
# # TEST IMAGE
first = srtm.clip(polygon);
# get image projection
proj = first.select([0]).projection();
# get coordinates image
latlon = ee.Image.pixelLonLat().reproject(proj)
# Map.addLayer(first, {bands:[bands], min:0, max:500}, 'Image')
coords = latlon.select(['longitude', 'latitude']).reduceRegion(reducer=ee.Reducer.toList(),
geometry=region,
scale=30)
# get lat & lon
lat = ee.List(coords.get('latitude'))
lon = ee.List(coords.get('longitude'))
# zip them. Example: zip([1, 3],[2, 4]) --> [[1, 2], [3,4]]
point_list = lon.zip(lat)
print('point list', point_list)
csv_data = []
# TODO: format data to return JSON
return str(point_list)
def regionCover(image,
region,
bands=None,
scale=None,
operator='OR',
property_name='REGION_COVER',
crs=None,
crsTransform=None,
bestEffort=False,
maxPixels=1e13,
tileScale=1):
""" Compute the percentage of values greater than 1 in a region. If more
than one band is specified, it applies the specified operator """
operators = ['OR', 'AND']
if operator not in operators:
raise ValueError('operator must be one of {}'.format(operators))
if not bands:
bands = ee.List([image.bandNames().get(0)])
if operator == 'AND':
reducer = ee.Reducer.bitwiseAnd()
else:
reducer = ee.Reducer.bitwiseOr()
bandname = 'regionCover'
mask = image.select(bands).reduce(reducer).rename(bandname)
# get projection
projection = mask.projection()
if not scale:
scale = projection.nominalScale()
# Make an image with all ones
ones_i = ee.Image.constant(1).reproject(projection).rename(bandname)
# manage geometry types
if isinstance(region, (ee.Feature, ee.FeatureCollection)):
region = region.geometry()
unbounded = region.isUnbounded()
# Get total number of pixels
ones = ones_i.reduceRegion(
reducer=ee.Reducer.count(),
geometry=region,
scale=scale,
maxPixels=maxPixels,
crs=crs,
crsTransform=crsTransform,
bestEffort=bestEffort,
tileScale=tileScale,
).get(bandname)
ones = ee.Number(ones)
# select first band, unmask and get the inverse
image_to_compute = mask.selfMask()
# Get number of zeros in the given image
zeros_in_mask = image_to_compute.reduceRegion(
reducer=ee.Reducer.count(),
geometry=region,
scale=scale,
maxPixels=maxPixels,
crs=crs,
crsTransform=crsTransform,
bestEffort=bestEffort,
tileScale=tileScale).get(bandname)
zeros_in_mask = ee.Number(zeros_in_mask)
percentage = zeros_in_mask.divide(ones)
# Multiply by 100
cover = percentage.multiply(100)
# Return None if geometry is unbounded
final = ee.Number(ee.Algorithms.If(unbounded, 0, cover))
return image.set(property_name, final)
def line_from_coords(coordinates, fromIndex, toIndex):
return ee.Geometry.LineString(
ee.List([coordinates.get(fromIndex),
coordinates.get(toIndex)]))
def interpolation(band):
look = _lookup(source.getArray(band), target.getArray(band))
x = ee.List(look.get('x'))
y = ee.List(look.get('y'))
return sourceImg.select([band]).interpolate(x, y)
def x(point):
return ee.Number(ee.List(point).get(0))
def parametrize(image, range_from, range_to, bands=None, drop=False):
""" Parametrize from a original **known** range to a fixed new range
:param range_from: Original range. example: (0, 5000)
:type range_from: tuple
:param range_to: Fixed new range. example: (500, 1000)
:type range_to: tuple
:param bands: bands to parametrize. If *None* all bands will be
parametrized.
:type bands: list
:param drop: drop the bands that will not be parametrized
:type drop: bool
:return: the parsed image with the parsed bands parametrized
:rtype: ee.Image
"""
original_range = range_from if isinstance(range_from, ee.List) \
else ee.List(range_from)
final_range = range_to if isinstance(range_to, ee.List) \
else ee.List(range_to)
# original min and max
min0 = ee.Image.constant(original_range.get(0))
max0 = ee.Image.constant(original_range.get(1))
# range from min to max
rango0 = max0.subtract(min0)
# final min max images
min1 = ee.Image.constant(final_range.get(0))
max1 = ee.Image.constant(final_range.get(1))
# final range
rango1 = max1.subtract(min1)
# all bands
all = image.bandNames()
# bands to parametrize
if bands:
bands_ee = ee.List(bands)
else:
bands_ee = image.bandNames()
inter = ee_list.intersection(bands_ee, all)
diff = ee_list.difference(all, inter)
image_ = image.select(inter)
# Percentage corresponding to the actual value
percent = image_.subtract(min0).divide(rango0)
# Taking count of the percentage of the original value in the original
# range compute the final value corresponding to the final range.
# Percentage * final_range + final_min
final = percent.multiply(rango1).add(min1)
if not drop:
# Add the rest of the bands (no parametrized)
final = image.select(diff).addBands(final)
# return passProperty(image, final, 'system:time_start')
return ee.Image(final.copyProperties(source=image))
def testDynamicConstructor(self):
# Test the behavior of the dynamic class constructor.
# Use a custom set of known functions for classes Foo and Bar.
# Foo Foo(arg1, [arg2])
# Bar Foo.makeBar()
# Bar Foo.takeBar(Bar bar)
# Baz Foo.baz()
def MockSend(path, unused_params, unused_method=None, unused_raw=None):
if path == '/algorithms':
return {
'Foo': {
'returns':
'Foo',
'args': [{
'name': 'arg1',
'type': 'Object'
}, {
'name': 'arg2',
'type': 'Object',
'optional': True
}]
},
'Foo.makeBar': {
'returns': 'Bar',
'args': [{
'name': 'foo',
'type': 'Foo'
}]
},
'Foo.takeBar': {
'returns':
'Bar',
'args': [{
'name': 'foo',
'type': 'Foo'
}, {
'name': 'bar',
'type': 'Bar'
}]
},
'Bar.baz': {
'returns': 'Baz',
'args': [{
'name': 'bar',
'type': 'Bar'
}]
}
}
ee.data.send_ = MockSend
ee.Initialize(None)
# Try to cast something that's already of the right class.
x = ee.Foo('argument')
self.assertEquals(ee.Foo(x), x)
# Tests for dynamic classes, where there is a constructor.
#
# If there's more than 1 arg, call the constructor.
x = ee.Foo('a')
y = ee.Foo(x, 'b')
ctor = ee.ApiFunction.lookup('Foo')
self.assertEquals(y.func, ctor)
self.assertEquals(y.args, {'arg1': x, 'arg2': 'b'})
# Can't cast a primitive; call the constructor.
self.assertEquals(ctor, ee.Foo(1).func)
# A computed object, but not this class; call the constructor.
self.assertEquals(ctor, ee.Foo(ee.List([1, 2, 3])).func)
# Tests for dynamic classes, where there isn't a constructor.
#
# Foo.makeBar and Foo.takeBar should have caused Bar to be generated.
self.assertTrue(hasattr(ee, 'Bar'))
# Make sure we can create a Bar.
bar = ee.Foo(1).makeBar()
self.assertTrue(isinstance(bar, ee.Bar))
# Now cast something else to a Bar and verify it was just a cast.
cast = ee.Bar(ee.Foo(1))
self.assertTrue(isinstance(cast, ee.Bar))
self.assertEquals(ctor, cast.func)
# We shouldn't be able to cast with more than 1 arg.
try:
ee.Bar(x, 'foo')
self.fail('Expected an exception.')
except ee.EEException as e:
self.assertTrue('Too many arguments for ee.Bar' in str(e))
# We shouldn't be able to cast a primitive.
try:
ee.Bar(1)
self.fail('Expected an exception.')
except ee.EEException as e:
self.assertTrue('Must be a ComputedObject' in str(e))
def sum_bands(n, ini):
condition = ee.List(band_names).contains(n)
return ee.Algorithms.If(condition,
ee.Image(ini).add(image.select([n])),
ee.Image(ini))
def bmaxOtsu(collection,target_data,region,
smoothing=100,
qualityBand=None,
reductionScale=90,
initThresh=0,
reverse=False,
gridSize=0.1,
bmaxThresh=0.75,
maxBoxes=100,
seed=7):
def constuctGrid(i):
def contructXGrid(j):
j = ee.Number(j)
box = ee.Feature(ee.Geometry.Rectangle(j,i,j.add(gridSize),i.add(gridSize)))
out = ee.Algorithms.If(geom.contains(box.geometry()),box,null)
return ee.Feature(out)
i = ee.Number(i)
out = ee.List.sequence(west,east.subtract(gridSize),gridSize).map(constuctXGrid)
return out
def calcBmax(feature):
segment = target.clip(feature)
initial = segment.lt(initThresh)
p1 = ee.Number(initial.reduceRegion(
reducer= ee.Reducer.mean(),
geometry= feature.geometry(),
bestEffort= True,
scale= reductionScale,
).get(histBand))
p2 = ee.Number(1).subtract(p1)
m = segment.updateMask(initial).rename('m1').addBands(
segment.updateMask(initial.not()).rename('m2')
)
mReduced = m.reduceRegion(
reducer= ee.Reducer.mean(),
geometry= feature.geometry(),
bestEffort= True,
scale= reductionScale,
)
m1 = ee.Number(mReduced.get('m1'))
m2 = ee.Number(mReduced.get('m2'))
m1 = ee.Number(ee.Algorithms.If(m1,m1,globalLow))
m2 = ee.Number(ee.Algorithms.If(m2,m2,globalHigh))
sigmab = p1.multiply(p2.multiply(m1.subtract(m2).pow(2)))
sigmat = ee.Number(segment.reduceRegion(
reducer= ee.Reducer.variance(),
geometry= feature.geometry(),
bestEffort= True,
scale= reductionScale,
).get(histBand))
bmax = sigmab.divide(sigmat)
return feature.set({'bmax':bmax})
tDate = ee.Date(target_date)
targetColl = collection.filterDate(tDate,tDate.advance(1,'day'))
if qualityBand == None:
histBand = ee.String(target.bandNames().get(0))
target = targetColl.mosaic()\
.select(histBand)
else:
histBand = ee.String(qualityBand)
target = targetColl.qualityMosaic(qualityBand)\
.select(histBand)
searchRegion = ee.Feature(ee.List(targeColl.map(geeutilsgetGeom).toList(1)).get(0))
theoretical = target.reduceRegion(
reducer= ee.Reducer.percentile([10,90]),
geometry= searchRegion.geometry(),
bestEffort= True,
scale: 5000
)
globalLow = theoretical.get(histBand.cat('_p10'))
globalHigh = theoretical.get(histBand.cat('_p90'))
geom = searchRegion.geometry()
bounds = geom.bounds()
coords = ee.List(bounds.coordinates().get(0))
gridSize = ee.Number(gridSize)
west = ee.Number(ee.List(coords.get(0)).get(0))
south = ee.Number(ee.List(coords.get(0)).get(1))
east = ee.Number(ee.List(coords.get(2)).get(0))
north = ee.Number(ee.List(coords.get(2)).get(1))
west = west.subtract(west.mod(gridSize))
south = south.subtract(south.mod(gridSize))
east = east.add(gridSize.subtract(east.mod(gridSize)))
north = north.add(gridSize.subtract(north.mod(gridSize)))
grid = ee.FeatureCollection(
ee.List.sequence(south,north.subtract(gridSize),gridSize).map(constuctGrid).flatten()
)
bmaxes = grid.map(calcBmax).filter(ee.Filter.gt('bmax',bmaxThresh)).randomColumn('random',seed)
nBoxes = ee.Number(bmax.size())
randomThresh = maxBoxes.divide(nBoxes)
selection = bmaxes.filter(ee.Filter.lt('random',randomThresh))
histogram = histogram_image.reduceRegion(ee.Reducer.histogram(255, 2)\
.combine('mean', None, True)\
.combine('variance', None,True),selection,reductionScale,bestEffort=True,
tileScale=16)
threshold = geeutils.otsu_function(histogram.get(histBand.cat('_histogram')))
water = target.gt(threshold).clip(geeutils.LAND.geometry())
return water.rename('water')
def toGrid(image, size=1, band=None, geometry=None):
""" Create a grid from pixels in an image. Results may depend on the image
projection. Work fine in Landsat imagery.
IMPORTANT: This grid is not perfect, it can be misplaced and have some
holes due to projection.
:param image: the image
:type image: ee.Image
:param size: the size of each cell, according to:
- 1: 1 pixel
- 2: 9 pixels (3x3)
- 3: 25 pixels (5x5)
- and so on..
:type size: int
:param band: the band to get the projection (and so, the scale) from. If
None, the first one will be used
:type band: str
:param geometry: the geometry where the grid will be computed. If the image
is unbounded this parameter must be set in order to work. If None,
the image geometry will be used if not unbounded.
:type geometry: ee.Geometry or ee.Feature
"""
band = band if band else 0
iband = image.select(band)
if geometry:
if isinstance(geometry, ee.Feature):
geometry = geometry.geometry()
else:
geometry = image.geometry()
projection = iband.projection()
scale = projection.nominalScale()
scale = scale.multiply((int(size) * 2) - 1)
buffer = scale.divide(2)
# get coordinates image
latlon = ee.Image.pixelLonLat().reproject(projection)
# put each lon lat in a list
coords = latlon.select(['longitude', 'latitude'])
coords = coords.reduceRegion(reducer=ee.Reducer.toList(),
geometry=geometry.buffer(scale),
scale=scale)
# get lat & lon
lat = ee.List(coords.get('latitude'))
lon = ee.List(coords.get('longitude'))
# zip them. Example: zip([1, 3],[2, 4]) --> [[1, 2], [3,4]]
point_list = lon.zip(lat)
def over_list(p):
p = ee.List(p)
point = ee.Geometry.Point(p).buffer(buffer).bounds()
return ee.Feature(point)
# make grid
fc = ee.FeatureCollection(point_list.map(over_list))
return fc
def test_Container_Get_Item_By_Slice(self):
"""Test the Container Emulation Methods"""
test = ee.List([1, 2, 3])[0:2]
self.assertIsInstance(test, ee.ee_list.List)
def over_list(p):
p = ee.List(p)
point = ee.Geometry.Point(p).buffer(buffer).bounds()
return ee.Feature(point)
def test_Container_Contains(self):
"""Test the Container Emulation Methods"""
test = 2 in ee.List([1, 2, 3])
self.assertIsInstance(test, bool)
def raster_zonal_stats(i_zones,
i_values,
statistic_type,
geometry,
crs_transform,
crs="EPSG:4326"):
""" Zonal Statistics using raster zones and values.
-------------------------------------------------------------------------------
Output options include a ee.FeatureCollection, pd.DataFrame or ee.Image.
The count is always included in the output results.
crs transform can be obtained by the get_crs_transform function.
Note that if a zone does not contain data, the value is missing from the
dictionary in the list.
Args:
i_zones (ee.Image) : Integer image with zones.
i_values (ee.Image) : Image with values.
statistic_type (string) : Statistics type like 'mean', 'sum'.
geometry (ee.Geometry) : Geometry defining extent of calculation.
geometry can be server side.
nodata_value (integer) : nodata value. Defaults to -9999.
crs_transform (list) : crs transform.
crs (string) : crs, deafults to 'EPSG:4326'.
Returns:
result_list (ee.List) : list of dictionaries with keys 'zones','count'
and 'mean/max/sum etc.'.
"""
if statistic_type == "mean":
reducer = ee.Reducer.mean().combine(reducer2=ee.Reducer.count(),
sharedInputs=True).group(
groupField=1,
groupName="zones")
elif statistic_type == "max":
reducer = ee.Reducer.max().combine(reducer2=ee.Reducer.count(),
sharedInputs=True).group(
groupField=1, groupName="zones")
elif statistic_type == "sum":
reducer = ee.Reducer.sum().combine(reducer2=ee.Reducer.count(),
sharedInputs=True).group(
groupField=1, groupName="zones")
elif statistic_type == "count":
reducer = ee.Reducer.count().combine(reducer2=ee.Reducer.count(),
sharedInputs=True).group(
groupField=1,
groupName="zones")
elif statistic_type == "first":
reducer = ee.Reducer.first().combine(reducer2=ee.Reducer.count(),
sharedInputs=True).group(
groupField=1,
groupName="zones")
else:
raise UserWarning(
"Statistic_type not yet supported, please modify function")
total_image = ee.Image(i_values).addBands(ee.Image(i_zones))
result_list = total_image.reduceRegion(geometry=geometry,
reducer=reducer,
crsTransform=crs_transform,
crs=crs,
maxPixels=1e10).get("groups")
result_list = ee.List(result_list)
return result_list
def wrap(img):
escalables = functions.list_intersection(
img.bandNames(), ee.List(self.to_scale))
return tools.parametrize(rango_orig, final_range,
escalables)(img)
def f_ndvi_clim_p(image, list):
ndvi = lf_clim_ndvi.select('offset').add(
(lf_clim_ndvi.select('scale').multiply(image))).set(
{'year': image.get('year')})
return ee.List(list).add(ndvi)