def main(ini_path=None, overwrite_flag=False):
"""Export annual ET/ETrF/ETr/count WRS2 tile images
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False)
Returns
-------
None
"""
logging.info('\nExport annual ET/ETrF/ETr/count WRS2 tile images')
# Read config file
ini = inputs.read(ini_path)
inputs.parse_section(ini, section='INPUTS')
inputs.parse_section(ini, section='INTERPOLATE')
inputs.parse_section(ini, section='EXPORT')
inputs.parse_section(ini, section=ini['INPUTS']['et_model'])
if ini['EXPORT']['export_dest'] == 'ASSET':
logging.error('\nERROR: ASSET tile export is not supported')
return False
# The study area cannot be used to set the output spatial reference
# when exporting by WRS2 tile
# if ini['EXPORT']['output_osr'] is None:
# # Get output coordinate system from study area shapefile
# study_area_ds = ogr.Open(ini['INPUTS']['study_area_path'], 0)
# study_area_lyr = study_area_ds.GetLayer()
# ini['EXPORT']['output_osr'] = osr.SpatialReference()
# ini['EXPORT']['output_osr'] = study_area_lyr.GetSpatialRef()
# ini['EXPORT']['output_crs'] = str(
# ini['EXPORT']['output_osr'].ExportToWkt())
# study_area_ds = None
# del study_area_lyr, study_area_ds
# logging.debug('\n {:16s} {}'.format(
# 'Output crs:', ini['EXPORT']['output_crs']))
logging.debug('\nInitializing Earth Engine')
ee.Initialize()
# Get current running tasks
tasks = utils.get_ee_tasks()
# Get list of existing images/files
if ini['EXPORT']['export_dest'] == 'CLOUD':
logging.debug('\nGetting cloud storage file list')
cloud_list = utils.get_bucket_files(
ini['EXPORT']['project_name'], ini['EXPORT']['output_ws'])
# It may be necessary to remove image tile notation
elif ini['EXPORT']['export_dest'] == 'GDRIVE':
logging.debug('\nGetting Google drive file list')
gdrive_list = [
os.path.join(ini['EXPORT']['output_ws'], x)
for x in os.listdir(ini['EXPORT']['output_ws'])]
# It may be necessary to remove image tile notation
# Very large tiles may get split up automatically by EE
# Strip the EE tile notation data from the image list
# gdrive_list = list(set([
# re.sub('-\d{10}-\d{10}.tif', '.tif', x)
# for x in os.listdir(ini['EXPORT']['output_ws'])]))
# logging.debug(gdrive_list)
# Get list of WRS2 tiles that intersect the study area
logging.debug('\nBuilding export list')
export_list = list(wrs2_tile_export_generator(
ini['INPUTS']['study_area_path'],
wrs2_coll=ini['INPUTS']['wrs2_coll'],
cell_size=ini['EXPORT']['cell_size'],
output_crs=ini['EXPORT']['output_crs'],
output_osr=ini['EXPORT']['output_osr'],
wrs2_tile_list=ini['INPUTS']['wrs2_tiles'],
wrs2_tile_field=ini['INPUTS']['wrs2_tile_field'],
snap_x=ini['EXPORT']['snap_x'],
snap_y=ini['EXPORT']['snap_y'],
wrs2_buffer=ini['INPUTS']['wrs2_buffer']))
if not export_list:
logging.error('\nEmpty export list, exiting')
return False
# Save export list to json
with open('export_wrs2_tile.json', 'w') as json_f:
json.dump(export_list, json_f)
# Process each WRS2 tile separately
logging.info('\nImage Exports')
for export_n, export_info in enumerate(export_list):
# path, row = map(int, wrs2_tile_re.findall(export_info['index'])[0])
logging.info('WRS2 tile: {} ({}/{})'.format(
export_info['index'], export_n + 1, len(export_list)))
logging.debug(' Shape: {}'.format(export_info['shape']))
logging.debug(' Transform: {}'.format(export_info['geo']))
logging.debug(' Extent: {}'.format(export_info['extent']))
logging.debug(' MaxPixels: {}'.format(export_info['maxpixels']))
if ini['INPUTS']['et_model'] == 'EEFLUX':
# Get the Landsat collection
landsat_coll = landsat.get_landsat_coll(
wrs2_tile_list=export_info['wrs2_tiles'],
cloud_cover=ini['INPUTS']['cloud_cover'],
start_date=ini['INTERPOLATE']['start_date'],
end_date=ini['INTERPOLATE']['end_date'],
landsat5_flag=ini['INPUTS']['landsat5_flag'],
landsat7_flag=ini['INPUTS']['landsat7_flag'],
landsat8_flag=ini['INPUTS']['landsat8_flag'],
landsat_type='RAD')
# Compute ETf for each Landsat scene
# The 'BQA' band is also being returned by the etrf method
def apply_et_fraction(image):
etrf_obj = eeflux.EEFlux(ee.Image(image)).etrf
etrf_img = ee.Image(etrf_obj.select(['etrf'], ['etf'])) \
.clamp(-1, 2)
cloud_mask = landsat.landsat_bqa_cloud_mask_func(
ee.Image(etrf_obj. select(['BQA'])))
return etrf_img.updateMask(cloud_mask) \
.copyProperties(image, ['system:time_start'])
scene_et_fraction_coll = ee.ImageCollection(
landsat_coll.map(apply_et_fraction))
else:
logging.error('\nInvalid/unsupported ET Model: {}'.format(
ini['INPUTS']['et_model']))
return False
# Daily reference ET collection
# DEADBEEF - Hard coding to GRIDMET for now
# Should this be retrieved from the model?
daily_et_reference_coll = ee.ImageCollection('IDAHO_EPSCOR/GRIDMET') \
.filterDate(ini['INPUTS']['start_date'], ini['INPUTS']['end_date']) \
.select(['etr'], ['et_reference'])
# Compute composite/mosaic images for each image date
daily_et_fraction_coll = ee.ImageCollection(interpolate.aggregate_daily(
image_coll=scene_et_fraction_coll,
start_date=ini['INTERPOLATE']['start_date'],
end_date=ini['INTERPOLATE']['end_date']))
# Interpolate daily ETf, multiply by daily ETr, and sum to ET
daily_et_actual_coll = ee.ImageCollection(interpolate.interp_et_coll(
et_reference_coll=daily_et_reference_coll,
et_fraction_coll=daily_et_fraction_coll,
interp_days=ini['INTERPOLATE']['interp_days'],
interp_type=ini['INTERPOLATE']['interp_type']))
# Export products
for product in ini['EXPORT']['products']:
logging.debug('\n Product: {}'.format(product))
export_id = ini['EXPORT']['export_id_fmt'].format(
model=ini['INPUTS']['et_model'].lower(),
product=product.lower(),
study_area=ini['INPUTS']['study_area_name'],
index=export_info['index'],
start=ini['INPUTS']['start_date'],
end=ini['INPUTS']['end_date'],
export=ini['EXPORT']['export_dest'].lower())
export_id = export_id.replace('-', '')
logging.debug(' Export ID: {}'.format(export_id))
if product == 'scene_id':
# Export the scene list CSV to Google Drive
if ini['EXPORT']['export_dest'] == 'GDRIVE':
export_path = os.path.join(
ini['EXPORT']['output_ws'], export_id + '.csv')
elif ini['EXPORT']['export_dest'] == 'CLOUD':
export_path = '{}/{}/{}'.format(
ini['EXPORT']['output_ws'],
product, export_id + '.csv')
elif ini['EXPORT']['export_dest'] == 'CLOUD':
# Write each product to a separate folder
export_path = '{}/{}/{}'.format(
ini['EXPORT']['output_ws'], product, export_id + '.tif')
elif ini['EXPORT']['export_dest'] == 'GDRIVE':
export_path = os.path.join(
ini['EXPORT']['output_ws'], export_id + '.tif')
logging.debug(' Export folder: {}'.format(
os.path.dirname(export_path)))
logging.debug(' Export file: {}'.format(
os.path.basename(export_path)))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if (ini['EXPORT']['export_dest'] == 'CLOUD' and
export_path in cloud_list):
logging.debug(' Export image already exists')
# Files in cloud storage are easily overwritten
# so it is unneccesary to manually remove them
# # This would remove an existing file
# subprocess.call(['gsutil', 'rm', export_path])
elif (ini['EXPORT']['export_dest'] == 'GDRIVE' and
export_path in gdrive_list):
logging.debug(' Export image already exists, removing')
os.remove(export_path)
# Remove automatically generated image tiles
# for f in glob.glob(export_path.replace('.tif', '*.tif')):
# os.remove(f)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, skipping')
continue
elif (ini['EXPORT']['export_dest'] == 'CLOUD' and
export_path in cloud_list):
logging.debug(' Export file already exists, skipping')
continue
elif (ini['EXPORT']['export_dest'] == 'GDRIVE' and
os.path.isfile(export_path)):
logging.debug(' Export file already exists, skipping')
continue
# Compute target product
if product == 'scene_id':
def scene_id_extract(image):
return ee.Feature(None).setMulti({
'SCENE_ID': ee.String(image.get('SCENE_ID'))})
scene_id_coll = ee.FeatureCollection(
scene_et_fraction_coll.map(scene_id_extract)).sort('SCENE_ID')
elif product == 'et_actual':
# Sum daily ET to total ET
output_image = ee.Image(daily_et_actual_coll.sum())
elif product == 'et_reference':
# Sum daily reference ET to total reference ET
output_image = ee.Image(daily_et_reference_coll.sum())
elif product == 'et_fraction':
# Compute mean ETf (ET / ETr)
output_image = ee.Image(daily_et_actual_coll.sum()) \
.divide(ee.Image(daily_et_reference_coll.sum()))
elif product == 'count':
# Filter count date range to same period as reference ET
output_image = ee.Image(daily_et_fraction_coll.filterDate(
ini['INPUTS']['start_dt'],
ini['INPUTS']['end_dt'] + datetime.timedelta(days=1)).count())
# elif product == 'count_monthly':
# output_image = interpolate.aggregate_monthly(
# composite_etf_coll.filterDate(
# ini['INPUTS']['start_dt'],
# ini['INPUTS']['end_dt'] + datetime.timedelta(days=1)))
else:
logging.warning(' Unsupported product type, skipping')
continue
# Convert data types for export to Google Drive or Cloud Storage
if (product in ['et_actual', 'et_reference', 'et_fraction'] and
ini['EXPORT']['export_dest'] in ['CLOUD', 'GDRIVE']):
output_image = output_image.unmask(-9999, False).toFloat()
# elif (product in ['count', 'count_monthly'] and
elif (product in ['count'] and
ini['EXPORT']['export_dest'] in ['CLOUD', 'GDRIVE']):
output_image = output_image.unmask(255, False).toUint8()
elif ini['EXPORT']['export_dest'] in ['ASSET']:
pass
# Build export tasks
if product == 'scene_id':
if ini['EXPORT']['export_dest'] == 'CLOUD':
task = ee.batch.Export.table.toCloudStorage(
collection=scene_id_coll,
description=export_id,
bucket=ini['EXPORT']['bucket_name'],
fileNamePrefix='{}/{}'.format(product, export_id),
fileFormat='CSV')
elif ini['EXPORT']['export_dest'] == 'GDRIVE':
# Export the scene list CSV to Google Drive
task = ee.batch.Export.table.toDrive(
collection=scene_id_coll,
description=export_id,
folder=os.path.basename(ini['EXPORT']['output_ws']),
fileNamePrefix=export_id,
fileFormat='CSV')
elif ini['EXPORT']['export_dest'] == 'CLOUD':
# Export the image to cloud storage
task = ee.batch.Export.image.toCloudStorage(
image=output_image,
description=export_id,
bucket=ini['EXPORT']['bucket_name'],
fileNamePrefix='{}/{}'.format(product, export_id),
dimensions=export_info['shape'],
crs=export_info['crs'],
crsTransform=export_info['geo'],
# shardSize=,
# fileDimensions=,
maxPixels=export_info['maxpixels'])
elif ini['EXPORT']['export_dest'] == 'GDRIVE':
# Export the images to your Google Drive
task = ee.batch.Export.image.toDrive(
image=output_image,
description=export_id,
folder=os.path.basename(ini['EXPORT']['output_ws']),
fileNamePrefix=export_id,
dimensions=export_info['shape'],
crs=export_info['crs'],
crsTransform=export_info['geo'],
maxPixels=export_info['maxpixels'])
else:
logging.debug(' Export task not built, skipping')
continue
# Try to start the export task a few times
logging.debug(' Starting export task')
for i in range(1, 10):
try:
task.start()
break
except Exception as e:
logging.error(
' Error: {}\n Retrying ({}/10)'.format(e, i))
time.sleep(i ** 2)
i += 1
def main(ini_path=None, overwrite_flag=False,
tile_cols='', tile_rows='', delay=0):
"""Export annual ET/ETrF/ETr/count image ARG grid tiles
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
tile_cols : str
Comma separated list and/or range of ARD tile columns indices.
tile_rows : str
Comma separated list and/or range of ARD tile row indices.
delay : float, optional
Delay time between each export task (the default is 0).
Returns
-------
None
"""
logging.info('\nExport annual ET/ETrF/ETr/count image tiles')
# Read config file
ini = inputs.read(ini_path)
inputs.parse_section(ini, section='INPUTS')
inputs.parse_section(ini, section='INTERPOLATE')
inputs.parse_section(ini, section='EXPORT')
inputs.parse_section(ini, section=ini['INPUTS']['et_model'])
if os.name == 'posix':
shell_flag = False
else:
shell_flag = True
# Limit tile ranges from command line
# Eventually move to config file?
try:
tile_cols_list = list(utils.parse_int_set(tile_cols))
except:
tile_cols_list = []
try:
tile_rows_list = list(utils.parse_int_set(tile_rows))
except:
tile_rows_list = []
logging.debug('\nInitializing Earth Engine')
ee.Initialize()
# Get current running tasks
tasks = utils.get_ee_tasks()
# Get list of existing images/files
if ini['EXPORT']['export_dest'] == 'ASSET':
logging.debug('\nGetting GEE asset list')
asset_list = utils.get_ee_assets(
ini['EXPORT']['output_ws'], shell_flag=shell_flag)
logging.debug(asset_list)
# elif ini['EXPORT']['export_dest'] == 'CLOUD':
# logging.debug('\nGetting cloud storage file list')
# cloud_list = utils.get_bucket_files(
# ini['EXPORT']['project_name'], ini['EXPORT']['output_ws'],
# shell_flag=shell_flag)
# # It may be necessary to remove image tile notation
# elif ini['EXPORT']['export_dest'] == 'GDRIVE':
# logging.debug('\nGetting Google drive file list')
# gdrive_list = [
# os.path.join(ini['EXPORT']['output_ws'], x)
# for x in os.listdir(ini['EXPORT']['output_ws'])]
# # It may be necessary to remove image tile notation
# # Very large tiles may get split up automatically by EE
# # Strip the EE tile notation data from the image list
# # gdrive_list = list(set([
# # re.sub('-\d{10}-\d{10}.tif', '.tif', x)
# # for x in os.listdir(ini['EXPORT']['output_ws'])]))
# # logging.debug(gdrive_list)
# Get list of tiles that intersect the study area
logging.debug('\nBuilding export list')
export_list = list(ard_tile_export_generator(
ini['INPUTS']['study_area_path'],
wrs2_coll=ini['INPUTS']['wrs2_coll'],
cell_size=ini['EXPORT']['cell_size'],
wrs2_tile_list=ini['INPUTS']['wrs2_tiles'],
wrs2_tile_field=ini['INPUTS']['wrs2_tile_field'],
wrs2_buffer=ini['INPUTS']['wrs2_buffer']))
if not export_list:
logging.error('\nEmpty export list, exiting')
return False
# Save export list to json
with open('export_tiles.json', 'w') as json_f:
json.dump(export_list, json_f)
# Process each tile separately
logging.info('\nImage Exports')
for export_n, export_info in enumerate(export_list):
tile_col = int(export_info['index'][1:4])
tile_row = int(export_info['index'][5:8])
if tile_cols_list and int(tile_col) not in tile_cols_list:
logging.debug('ARD Tile: {} ({}/{}), skipping'.format(
export_info['index'], export_n + 1, len(export_list)))
continue
elif tile_rows_list and int(tile_row) not in tile_rows_list:
logging.debug('ARD Tile: {} ({}/{}), skipping'.format(
export_info['index'], export_n + 1, len(export_list)))
continue
else:
logging.info('ARD Tile: {} ({}/{})'.format(
export_info['index'], export_n + 1, len(export_list)))
logging.debug(' Shape: {}'.format(export_info['shape']))
logging.debug(' Transform: {}'.format(export_info['geo']))
logging.debug(' Extent: {}'.format(export_info['extent']))
logging.debug(' MaxPixels: {}'.format(export_info['maxpixels']))
logging.debug(' WRS2 tiles: {}'.format(
', '.join(export_info['wrs2_tiles'])))
if ini['INPUTS']['et_model'] == 'EEFLUX':
# Get the Landsat collection
landsat_coll = landsat.get_landsat_coll(
wrs2_tile_list=export_info['wrs2_tiles'],
cloud_cover=ini['INPUTS']['cloud_cover'],
start_date=ini['INTERPOLATE']['start_date'],
end_date=ini['INTERPOLATE']['end_date'],
landsat5_flag=ini['INPUTS']['landsat5_flag'],
landsat7_flag=ini['INPUTS']['landsat7_flag'],
landsat8_flag=ini['INPUTS']['landsat8_flag'],
landsat_type='RAD')
# Compute ETf for each Landsat scene
# The 'BQA' band is also being returned by the etrf method
def apply_et_fraction(image):
etrf_obj = eeflux.EEFlux(ee.Image(image)).etrf
etrf_img = ee.Image(etrf_obj.select(['etrf'], ['etf'])) \
.clamp(-1, 2)
cloud_mask = landsat.landsat_bqa_cloud_mask_func(
ee.Image(etrf_obj. select(['BQA'])))
return etrf_img.updateMask(cloud_mask) \
.copyProperties(image, ['system:time_start'])
scene_et_fraction_coll = ee.ImageCollection(
landsat_coll.map(apply_et_fraction))
else:
logging.error('\nInvalid/unsupported ET Model: {}'.format(
ini['INPUTS']['et_model']))
return False
# Daily reference ET collection
# Is the "refet_source" a function of the model, interpolation, or other?
# The "refet_type" parameter is currently being ignored
if ini[ini['INPUTS']['et_model']]['refet_source'] == 'GRIDMET':
daily_et_reference_coll = ee.ImageCollection('IDAHO_EPSCOR/GRIDMET') \
.filterDate(ini['INPUTS']['start_date'], ini['INPUTS']['end_date']) \
.select(['etr'], ['et_reference'])
elif ini[ini['INPUTS']['et_model']]['refet_source'] == 'CIMIS':
daily_et_reference_coll = ee.ImageCollection('projects/climate-engine/cimis/daily') \
.filterDate(ini['INPUTS']['start_date'],
ini['INPUTS']['end_date']) \
.select(['etr_asce'], ['et_reference'])
# Compute composite/mosaic images for each image date
daily_et_fraction_coll = ee.ImageCollection(interpolate.aggregate_daily(
image_coll=scene_et_fraction_coll,
start_date=ini['INTERPOLATE']['start_date'],
end_date=ini['INTERPOLATE']['end_date']))
# Interpolate daily ETf, multiply by daily ETr, and sum to ET
daily_et_actual_coll = ee.ImageCollection(interpolate.interp_et_coll(
et_reference_coll=daily_et_reference_coll,
et_fraction_coll=daily_et_fraction_coll,
interp_days=ini['INTERPOLATE']['interp_days'],
interp_type=ini['INTERPOLATE']['interp_type']))
# Export products
# for product in ini['EXPORT']['products']:
# logging.debug('\n Product: {}'.format(product))
export_id = ini['EXPORT']['export_id_fmt'].format(
model=ini['INPUTS']['et_model'].lower(),
# product=product.lower(),
study_area=ini['INPUTS']['study_area_name'],
index=export_info['index'],
start=ini['INPUTS']['start_date'],
end=ini['INPUTS']['end_date'],
export=ini['EXPORT']['export_dest'].lower())
export_id = export_id.replace('-', '')
logging.debug(' Export ID: {}'.format(export_id))
# if product == 'scene_id':
# # Export the scene list CSV to Google Drive
# if ini['EXPORT']['export_dest'] == 'GDRIVE':
# export_path = os.path.join(
# ini['EXPORT']['output_ws'], export_id + '.csv')
# elif ini['EXPORT']['export_dest'] == 'CLOUD':
# export_path = '{}/{}/{}'.format(
# ini['EXPORT']['output_ws'], product, export_id + '.csv')
# if ini['EXPORT']['export_dest'] == 'CLOUD':
# # Write each product to a separate folder
# export_path = '{}/{}/{}'.format(
# ini['EXPORT']['output_ws'], product, export_id + '.tif')
# elif ini['EXPORT']['export_dest'] == 'GDRIVE':
# export_path = os.path.join(
# ini['EXPORT']['output_ws'], export_id + '.tif')
if ini['EXPORT']['export_dest'] == 'ASSET':
# Write each product to a separate folder
export_path = '{}/{}'.format(
ini['EXPORT']['output_ws'], export_id)
else:
logging.warning(' Unsupported product type, skipping')
continue
logging.debug(' Export folder: {}'.format(
os.path.dirname(export_path)))
logging.debug(' Export file: {}'.format(
os.path.basename(export_path)))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if (ini['EXPORT']['export_dest'] == 'ASSET' and
export_path in asset_list):
logging.debug(' Asset already exists')
subprocess.check_output(
['earthengine', 'rm', export_path],
shell=shell_flag)
# Files in cloud storage are easily overwritten
# so it is unneccesary to manually remove them
# # This would remove an existing file
# subprocess.call(['gsutil', 'rm', export_path])
# if (ini['EXPORT']['export_dest'] == 'CLOUD' and
# export_path in cloud_list):
# logging.debug(' Export image already exists')
# # Files in cloud storage are easily overwritten
# # so it is unneccesary to manually remove them
# # # This would remove an existing file
# # subprocess.check_output(['gsutil', 'rm', export_path])
# elif (ini['EXPORT']['export_dest'] == 'GDRIVE' and
# export_path in gdrive_list):
# logging.debug(' Export image already exists, removing')
# os.remove(export_path)
# # Remove automatically generated image tiles
# # for f in glob.glob(export_path.replace('.tif', '*.tif')):
# # os.remove(f)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, skipping')
continue
if (ini['EXPORT']['export_dest'] == 'ASSET' and
export_path in asset_list):
logging.debug(' Asset already exists, skipping')
continue
# elif (ini['EXPORT']['export_dest'] == 'CLOUD' and
# export_path in cloud_list):
# logging.debug(' Export file already exists, skipping')
# continue
# elif (ini['EXPORT']['export_dest'] == 'GDRIVE' and
# os.path.isfile(export_path)):
# logging.debug(' Export file already exists, skipping')
# continue
# Compute target product
# if product == 'scene_id':
# def scene_id_extract(image):
# return ee.Feature(None).setMulti({
# 'SCENE_ID': ee.String(image.get('SCENE_ID'))})
# scene_id_coll = ee.FeatureCollection(
# scene_et_fraction_coll.map(scene_id_extract)).sort('SCENE_ID')
output_images = []
for product_i, product in enumerate(ini['EXPORT']['products']):
logging.debug(' Product: {}'.format(product))
if product == 'et_actual':
# Sum daily ET to total ET
output_images.append(
ee.Image(daily_et_actual_coll.sum()).toFloat())
elif product == 'et_reference':
# Sum daily reference ET to total reference ET
output_images.append(
ee.Image(daily_et_reference_coll.sum()).toFloat())
elif product == 'et_fraction':
# Compute mean ETf (ET / ETr)
output_images.append(
ee.Image(daily_et_actual_coll.sum()) \
.divide(ee.Image(daily_et_reference_coll.sum())).toFloat())
elif product == 'count':
# Filter count date range to same period as reference ET
output_images.append(ee.Image(
daily_et_fraction_coll.filterDate(
ini['INPUTS']['start_dt'],
ini['INPUTS']['end_dt'] + datetime.timedelta(days=1)).count())\
.toUint8())
# DEADEEF - Consider saving other input parameters
# CLOUD_COVER_LAND, number of interpolation days, ?
output_image = ee.Image(ee.Image(output_images) \
.rename(ini['EXPORT']['products']) \
.setMulti({
'system:time_start': ini['INPUTS']['start_date'],
'index': export_info['index']}))
# print(output_image.get('system:time_start').getInfo())
# input('ENTER')
# Build export tasks
# if product == 'scene_id':
# if ini['EXPORT']['export_dest'] == 'CLOUD':
# task = ee.batch.Export.table.toCloudStorage(
# scene_id_coll,
# description=export_id,
# bucket=ini['EXPORT']['bucket_name'],
# fileNamePrefix='{}/{}/{}'.format(
# ini['EXPORT']['bucket_folder'], product, export_id),
# fileFormat='CSV')
# elif ini['EXPORT']['export_dest'] == 'GDRIVE':
# # Export the scene list CSV to Google Drive
# task = ee.batch.Export.table.toDrive(
# scene_id_coll,
# description=export_id,
# folder=os.path.basename(ini['EXPORT']['output_ws']),
# fileNamePrefix=export_id,
# fileFormat='CSV')
# elif ini['EXPORT']['export_dest'] == 'CLOUD':
# # Export the image to cloud storage
# task = ee.batch.Export.image.toCloudStorage(
# output_image,
# description=export_id,
# bucket=ini['EXPORT']['bucket_name'],
# fileNamePrefix='{}/{}/{}'.format(
# ini['EXPORT']['bucket_folder'], product, export_id),
# dimensions=export_info['shape'],
# crs=export_info['crs'],
# crsTransform=export_info['geo'],
# # shardSize=,
# # fileDimensions=,
# maxPixels=export_info['maxpixels'])
# elif ini['EXPORT']['export_dest'] == 'GDRIVE':
# # Export the images to your Google Drive
# task = ee.batch.Export.image.toDrive(
# output_image,
# description=export_id,
# folder=os.path.basename(ini['EXPORT']['output_ws']),
# fileNamePrefix=export_id,
# dimensions=export_info['shape'],
# crs=export_info['crs'],
# crsTransform=export_info['geo'],
# maxPixels=export_info['maxpixels'])
if ini['EXPORT']['export_dest'] == 'ASSET':
# Export the image to cloud storage
task = ee.batch.Export.image.toAsset(
output_image,
description=export_id,
assetId='{}/{}'.format(ini['EXPORT']['output_ws'], export_id),
# pyramidingPolicy='mean',
dimensions=export_info['shape'],
crs=export_info['crs'],
crsTransform=export_info['geo'],
maxPixels=export_info['maxpixels'])
else:
logging.debug(' Export task not built, skipping')
# continue
# Try to start the export task a few times
logging.debug(' Starting export task')
for i in range(1, 10):
try:
task.start()
break
except Exception as e:
logging.error(
' Error: {}\n Retrying ({}/10)'.format(e, i))
time.sleep(i ** 2)
i += 1
# logging.debug(' Active: {}'.format(task.active()))
# logging.debug(' Status: {}'.format(task.status()))
if delay and delay > 0:
time.sleep(delay)
elif delay and delay == -1:
input('ENTER')
def main(ini_path=None, overwrite_flag=False, tile_i='', tile_j=''):
"""Export annual ET/ETrF/ETr/count image tiles
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
tile_i : str
Comma separated list and/or range of tile row indices.
tile_j : str
Comma separated list and/or range of tile columns indices.
Returns
-------
None
"""
logging.info('\nGenerate tile shapefile')
# Read config file
ini = inputs.read(ini_path)
inputs.parse_section(ini, section='INPUTS')
inputs.parse_section(ini, section='EXPORT')
output_path = ini['INPUTS']['study_area_path'].replace(
'.shp', '_tiles.shp')
if os.path.isfile(output_path) and not overwrite_flag:
logging.info(
'\nOutput shapefile already exists and overwrite_flag is False\n')
return False
# Limit tile ranges from command line
# Eventually move to config file?
try:
tile_i_list = list(utils.parse_int_set(tile_i))
except:
tile_i_list = []
try:
tile_j_list = list(utils.parse_int_set(tile_j))
except:
tile_j_list = []
# Use study area spatial reference if not set explicitly in INI
if ini['EXPORT']['output_osr'] is None:
# Get output coordinate system from study area shapefile
study_area_ds = ogr.Open(ini['INPUTS']['study_area_path'], 0)
study_area_lyr = study_area_ds.GetLayer()
ini['EXPORT']['output_osr'] = osr.SpatialReference()
ini['EXPORT']['output_osr'] = study_area_lyr.GetSpatialRef()
ini['EXPORT']['output_crs'] = str(
ini['EXPORT']['output_osr'].ExportToWkt())
study_area_ds = None
del study_area_lyr, study_area_ds
logging.debug('\n {:16s} {}'.format('Output crs:',
ini['EXPORT']['output_crs']))
# Get list of tiles that intersect the study area
logging.debug('\nBuilding export list')
export_list = list(
tile_export_generator(ini['INPUTS']['study_area_path'],
cell_size=ini['EXPORT']['cell_size'],
output_osr=ini['EXPORT']['output_osr'],
snap_x=ini['EXPORT']['snap_x'],
snap_y=ini['EXPORT']['snap_y'],
tile_cells=ini['TILE']['tile_cells']))
if not export_list:
logging.error('\nEmpty export list, exiting')
return False
# Build the output shapefile
# Write the scene Tcorr values to the shapefile
logging.info('\nWriting tiles to the shapefile')
logging.debug(' {}'.format(output_path))
shp_driver = ogr.GetDriverByName("ESRI Shapefile")
output_ds = shp_driver.CreateDataSource(output_path)
output_lyr = output_ds.CreateLayer(output_path,
ini['EXPORT']['output_osr'],
ogr.wkbPolygon)
field_name = ogr.FieldDefn('COL', ogr.OFTInteger)
field_name.SetWidth(3)
output_lyr.CreateField(field_name)
field_name = ogr.FieldDefn('ROW', ogr.OFTInteger)
field_name.SetWidth(3)
output_lyr.CreateField(field_name)
# Write each tile separately
for export_n, export_info in enumerate(export_list):
logging.info('Tile: {} ({}/{})'.format(export_info['index'],
export_n + 1,
len(export_list)))
logging.debug(' Extent: {}'.format(export_info['extent']))
tile_i, tile_j = map(int, export_info['index'].split('_'))
if tile_i_list and int(tile_i) not in tile_i_list:
logging.debug(' Skipping tile')
continue
elif tile_j_list and int(tile_j) not in tile_j_list:
logging.debug(' Skipping tile')
continue
feature = ogr.Feature(output_lyr.GetLayerDefn())
feature.SetField('COL', tile_j)
feature.SetField('ROW', tile_i)
polygon = ogr.CreateGeometryFromWkt(
"POLYGON(({0} {1}, {2} {1}, {2} {3}, {0} {3}, {0} {1}))".format(
*export_info['extent']))
feature.SetGeometry(polygon)
output_lyr.CreateFeature(feature)
feature = None
output_ds = None
def main(ini_path=None, overwrite_flag=False):
"""Export daily ET/ETrF/ETr/count images as EE assets or to Google Drive
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False)
Returns
-------
None
"""
logging.info('\nComputing daily ET/ETrF/ETr/count images')
# Read config file
ini = inputs.read(ini_path)
inputs.parse_section(ini, section='INPUTS')
inputs.parse_section(ini, section='EXPORT')
inputs.parse_section(ini, section=ini['INPUTS']['et_model'])
# # Use study area spatial reference if not set explicitly in INI
# if ini['EXPORT']['output_osr'] is None:
# # Get output coordinate system from study area shapefile
# study_area_ds = ogr.Open(ini['INPUTS']['study_area_path'], 0)
# study_area_lyr = study_area_ds.GetLayer()
# ini['EXPORT']['output_osr'] = osr.SpatialReference()
# ini['EXPORT']['output_osr'] = study_area_lyr.GetSpatialRef()
# ini['EXPORT']['output_crs'] = str(
# ini['EXPORT']['output_osr'].ExportToWkt())
# study_area_ds = None
# del study_area_lyr, study_area_ds
# logging.debug('\n {:16s} {}'.format(
# 'Output crs:', ini['EXPORT']['output_crs']))
logging.info('\nInitializing Earth Engine')
ee.Initialize()
# Get current running tasks
tasks = utils.get_ee_tasks()
# Get list of existing images/files
# if ini['EXPORT']['export_dest'] == 'ASSET':
# logging.debug('\nGetting EE asset list')
# try:
# asset_list = subprocess.check_output(
# ['earthengine', 'ls', ini['EXPORT']['output_ws']],
# universal_newlines=True)
# asset_list = [x.strip() for x in asset_list.split('\n') if x]
# # logging.debug(asset_list)
# except ValueError as e:
# logging.info(' Collection doesn\'t exist')
# logging.debug(' {}'.format(str(e)))
# asset_list = []
# except Exception as e:
# logging.error('\n Unknown error, returning False')
# logging.error(e)
# return False
if ini['EXPORT']['export_dest'] == 'CLOUD':
logging.debug('\nGetting cloud storage file list')
cloud_list = utils.get_bucket_files(ini['EXPORT']['project_name'],
ini['EXPORT']['output_ws'])
# It may be necessary to remove image tile notation
elif ini['EXPORT']['export_dest'] == 'GDRIVE':
logging.debug('\nGetting Google drive file list')
gdrive_list = [
os.path.join(ini['EXPORT']['output_ws'], x)
for x in os.listdir(ini['EXPORT']['output_ws'])
]
# Very large tiles may get split up automatically by EE
# Strip the EE tile notation data from the image list
gdrive_list = list(
set([re.sub('-\d{10}-\d{10}.tif', '.tif', x)
for x in gdrive_list]))
# logging.debug(gdrive_list)
# Remove scene_id product
if 'scene_id' in ini['EXPORT']['products']:
ini['EXPORT']['products'].remove('scene_id')
# Change "count" product to "count_mask" for single image exports
# This block would need to be removed if a separate mask product was added
ini['EXPORT']['products'] = [
p if p != 'count' else 'count_mask' for p in ini['EXPORT']['products']
]
# Get list of WRS2 tiles that intersect the study area
logging.debug('\nBuilding export list')
export_list = list(
wrs2_tile_export_generator(
ini['INPUTS']['study_area_path'],
wrs2_coll=ini['INPUTS']['wrs2_coll'],
cell_size=ini['EXPORT']['cell_size'],
output_crs=ini['EXPORT']['output_crs'],
output_osr=ini['EXPORT']['output_osr'],
wrs2_tile_list=ini['INPUTS']['wrs2_tiles'],
wrs2_tile_field=ini['INPUTS']['wrs2_tile_field'],
snap_x=ini['EXPORT']['snap_x'],
snap_y=ini['EXPORT']['snap_y'],
wrs2_buffer=ini['INPUTS']['wrs2_buffer']))
if not export_list:
logging.error('\nEmpty export list, exiting')
return False
# # Save export list to json
# with open('export_wrs2_tile.json', 'w') as json_f:
# json.dump(export_list, json_f)
# Process each WRS2 tile separately
logging.info('\nImage Exports')
for export_n, export_info in enumerate(export_list):
# path, row = map(int, path_row_re.findall(export_info['index'])[0])
logging.info('WRS2 tile: {} ({}/{})'.format(export_info['index'],
export_n + 1,
len(export_list)))
logging.debug(' Shape: {}'.format(export_info['shape']))
logging.debug(' Transform: {}'.format(export_info['geo']))
logging.debug(' Extent: {}'.format(export_info['extent']))
logging.debug(' MaxPixels: {}'.format(export_info['maxpixels']))
# Get the full Landsat collection
logging.debug(' Getting image IDs from EarthEngine')
landsat_coll = landsat.get_landsat_coll(
wrs2_tile_list=export_info['wrs2_tiles'],
cloud_cover=ini['INPUTS']['cloud_cover'],
start_date=ini['INPUTS']['start_date'],
end_date=ini['INPUTS']['end_date'],
landsat5_flag=ini['INPUTS']['landsat5_flag'],
landsat7_flag=ini['INPUTS']['landsat7_flag'],
landsat8_flag=ini['INPUTS']['landsat8_flag'],
)
scene_id_list = landsat_coll.aggregate_histogram('SCENE_ID')\
.getInfo().keys()
if not scene_id_list:
logging.info('\nNo Landsat images in date range, exiting')
sys.exit()
# Process each image in the collection by date
# image_id is the full Earth Engine ID to the asset
for scene_id in scene_id_list:
logging.info('{}'.format(scene_id))
l, p, r, year, month, day = landsat.parse_landsat_id(scene_id)
image_dt = datetime.datetime.strptime(
'{:04d}{:02d}{:02d}'.format(year, month, day), '%Y%m%d')
image_date = image_dt.date().isoformat()
# logging.debug(' Date: {0}'.format(image_date))
# logging.debug(' DOY: {0}'.format(doy))
# Export products
for product in ini['EXPORT']['products']:
export_id = ini['EXPORT']['export_id_fmt'] \
.replace('_{start}', '') \
.replace('_{end}', '') \
.format(
model=ini['INPUTS']['et_model'].lower(),
product=product.lower(),
study_area=ini['INPUTS']['study_area_name'],
index=scene_id,
export=ini['EXPORT']['export_dest'].lower())
export_id = export_id.replace('-', '')
logging.debug(' Export ID: {0}'.format(export_id))
if ini['EXPORT']['export_dest'] == 'CLOUD':
# Write each product to a separate folder
export_path = '{}/{}/{}'.format(ini['EXPORT']['output_ws'],
product,
export_id + '.tif')
elif ini['EXPORT']['export_dest'] == 'GDRIVE':
export_path = os.path.join(ini['EXPORT']['output_ws'],
export_id + '.tif')
logging.debug(' {}'.format(export_path))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if (ini['EXPORT']['export_dest'] == 'CLOUD'
and export_path in cloud_list):
logging.debug(' Export image already exists')
# Files in cloud storage are easily overwritten
# so it is unneccesary to manually remove them
# # This would remove an existing file
# subprocess.call(['gsutil', 'rm', export_path])
elif (ini['EXPORT']['export_dest'] == 'GDRIVE'
and export_path in gdrive_list):
logging.debug(
' Export image already exists, removing')
os.remove(export_path)
# Remove automatically generated image tiles
# for f in glob.glob(export_path.replace('.tif', '*.tif')):
# os.remove(f)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, skipping')
continue
elif (ini['EXPORT']['export_dest'] == 'CLOUD'
and export_path in cloud_list):
logging.debug(
' Export file already exists, skipping')
continue
elif (ini['EXPORT']['export_dest'] == 'GDRIVE'
and os.path.isfile(export_path)):
logging.debug(
' Export file already exists, skipping')
continue
if ini['INPUTS']['et_model'] == 'EEFLUX':
# Get a Landsat collection with only the target image
landsat_image = landsat.get_landsat_image(
wrs2_tile_list=export_info['wrs2_tiles'],
cloud_cover=ini['INPUTS']['cloud_cover'],
start_date=image_date,
landsat_type='RAD')
# The 'BQA' band is also being returned by the etrf method
etrf_obj = eeflux.EEFlux(ee.Image(landsat_image)).etrf
etrf_img = ee.Image(etrf_obj.select(['etrf'], ['etf'])) \
.clamp(-1, 2)
cloud_mask = landsat.landsat_bqa_cloud_mask_func(
ee.Image(etrf_obj.select(['BQA'])))
daily_et_fraction_image = etrf_img.updateMask(cloud_mask) \
.copyProperties(etrf_obj, ['system:time_start'])
# Image date reference ET
# DEADBEEF - Hard coding to GRIDMET for now
# Should this be retrieved from the model?
daily_et_reference_image = 'IDAHO_EPSCOR/GRIDMET/{}'.format(
image_dt.date().strftime('%Y%m%d'))
daily_et_reference_image = ee.Image(daily_et_reference_image) \
.select(['etr'], ['et_reference'])
else:
logging.error('\nInvalid/unsupported ET Model: {}'.format(
ini['INPUTS']['et_model']))
return False
# Compute target product
if product == 'et_actual':
output_image = daily_et_fraction_image \
.multiply(daily_et_reference_image)
elif product == 'et_reference':
output_image = daily_et_reference_image
elif product == 'et_fraction':
output_image = daily_et_fraction_image
elif product in ['count', 'count_mask']:
output_image = daily_et_fraction_image.mask()
# elif product == 'scene_id':
# continue
else:
logging.debug(
' Unsupported product {}, skipping'.format(product))
continue
# Convert data types for export to Google Drive or Cloud Storage
if (product in ['et_actual', 'et_reference', 'et_fraction'] and
ini['EXPORT']['export_dest'] in ['CLOUD', 'GDRIVE']):
output_image = output_image.unmask(-9999, False).toFloat()
elif (product in ['count']
and ini['EXPORT']['export_dest'] in ['CLOUD', 'GDRIVE']):
output_image = output_image.unmask(255, False).toUint8()
# output_image = output_image.toUint8()
elif ini['EXPORT']['export_dest'] in ['ASSET']:
pass
# Build export tasks
if ini['EXPORT']['export_dest'] == 'CLOUD':
# Export the image to cloud storage
task = ee.batch.Export.image.toCloudStorage(
output_image,
description=export_id,
bucket=ini['EXPORT']['bucket_name'],
fileNamePrefix='{}/{}'.format(product, export_id),
dimensions=export_info['shape'],
crs=export_info['crs'],
crsTransform=export_info['geo'],
# shardSize=,
# fileDimensions=,
maxPixels=export_info['maxpixels'])
elif ini['EXPORT']['export_dest'] == 'GDRIVE':
# Export the images to your Google Drive
task = ee.batch.Export.image.toDrive(
output_image,
description=export_id,
folder=os.path.basename(ini['EXPORT']['output_ws']),
fileNamePrefix=export_id,
dimensions=export_info['shape'],
crs=export_info['crs'],
crsTransform=export_info['geo'],
maxPixels=export_info['maxpixels'])
else:
logging.debug(' Export task not built, skipping')
continue
# Try to start the export task a few times
logging.debug(' Starting export task')
for i in range(1, 10):
try:
task.start()
break
except Exception as e:
logging.error(
' Error: {}\n Retrying ({}/10)'.format(e, i))
time.sleep(i**2)
i += 1