def main(ini_path=None,
overwrite_flag=False,
delay_time=0,
gee_key_file=None,
max_ready=-1):
"""Compute monthly Tcorr images
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
delay_time : float, optional
Delay time in seconds between starting export tasks (or checking the
number of queued tasks, see "max_ready" parameter). The default is 0.
gee_key_file : str, None, optional
Earth Engine service account JSON key file (the default is None).
max_ready: int, optional
Maximum number of queued "READY" tasks. The default is -1 which is
implies no limit to the number of tasks that will be submitted.
"""
logging.info('\nCompute monthly Tcorr images')
ini = utils.read_ini(ini_path)
model_name = 'SSEBOP'
# model_name = ini['INPUTS']['et_model'].upper()
tmax_name = ini[model_name]['tmax_source']
export_id_fmt = 'tcorr_image_{product}_month{month:02d}_cycle{cycle:02d}'
asset_id_fmt = '{coll_id}/month{month:02d}_cycle{cycle:02d}'
tcorr_monthly_coll_id = '{}/{}_monthly'.format(
ini['EXPORT']['export_coll'], tmax_name.lower())
wrs2_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/wrs2_descending_custom'
if (tmax_name.upper() == 'CIMIS'
and ini['INPUTS']['end_date'] < '2003-10-01'):
logging.error(
'\nCIMIS is not currently available before 2003-10-01, exiting\n')
sys.exit()
elif (tmax_name.upper() == 'DAYMET'
and ini['INPUTS']['end_date'] > '2018-12-31'):
logging.warning('\nDAYMET is not currently available past 2018-12-31, '
'using median Tmax values\n')
# sys.exit()
# elif (tmax_name.upper() == 'TOPOWX' and
# ini['INPUTS']['end_date'] > '2017-12-31'):
# logging.warning(
# '\nDAYMET is not currently available past 2017-12-31, '
# 'using median Tmax values\n')
# # sys.exit()
logging.info('\nInitializing Earth Engine')
if gee_key_file:
logging.info(
' Using service account key file: {}'.format(gee_key_file))
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('x', key_file=gee_key_file),
use_cloud_api=True)
else:
ee.Initialize(use_cloud_api=True)
logging.debug('\nTmax properties')
tmax_source = tmax_name.split('_', 1)[0]
tmax_version = tmax_name.split('_', 1)[1]
tmax_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower())
tmax_coll = ee.ImageCollection(tmax_coll_id)
tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0)
logging.debug(' Collection: {}'.format(tmax_coll_id))
logging.debug(' Source: {}'.format(tmax_source))
logging.debug(' Version: {}'.format(tmax_version))
# Get the Tcorr daily image collection properties
logging.debug('\nTcorr Image properties')
tcorr_daily_coll_id = '{}/{}_daily'.format(ini['EXPORT']['export_coll'],
tmax_name.lower())
tcorr_img = ee.Image(ee.ImageCollection(tcorr_daily_coll_id).first())
tcorr_info = utils.get_info(ee.Image(tcorr_img))
tcorr_geo = tcorr_info['bands'][0]['crs_transform']
tcorr_crs = tcorr_info['bands'][0]['crs']
tcorr_shape = tcorr_info['bands'][0]['dimensions']
# tcorr_geo = ee.Image(tcorr_img).projection().getInfo()['transform']
# tcorr_crs = ee.Image(tcorr_img).projection().getInfo()['crs']
# tcorr_shape = ee.Image(tcorr_img).getInfo()['bands'][0]['dimensions']
tcorr_extent = [
tcorr_geo[2], tcorr_geo[5] + tcorr_shape[1] * tcorr_geo[4],
tcorr_geo[2] + tcorr_shape[0] * tcorr_geo[0], tcorr_geo[5]
]
logging.debug(' Shape: {}'.format(tcorr_shape))
logging.debug(' Extent: {}'.format(tcorr_extent))
logging.debug(' Geo: {}'.format(tcorr_geo))
logging.debug(' CRS: {}'.format(tcorr_crs))
if not ee.data.getInfo(tcorr_monthly_coll_id):
logging.info('\nExport collection does not exist and will be built'
'\n {}'.format(tcorr_monthly_coll_id))
input('Press ENTER to continue')
ee.data.createAsset({'type': 'IMAGE_COLLECTION'},
tcorr_monthly_coll_id)
# Get current asset list
logging.debug('\nGetting GEE asset list')
asset_list = utils.get_ee_assets(tcorr_monthly_coll_id)
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
pprint.pprint(asset_list[:10])
# Get current running tasks
tasks = utils.get_ee_tasks()
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
logging.debug(' Tasks: {}\n'.format(len(tasks)))
input('ENTER')
# Limit by year and month
try:
month_list = sorted(list(utils.parse_int_set(ini['TCORR']['months'])))
except:
logging.info('\nTCORR "months" parameter not set in the INI,'
'\n Defaulting to all months (1-12)\n')
month_list = list(range(1, 13))
try:
year_list = sorted(list(utils.parse_int_set(ini['TCORR']['years'])))
except:
logging.info('\nTCORR "years" parameter not set in the INI,'
'\n Defaulting to all available years\n')
year_list = []
# Key is cycle day, value is a reference date on that cycle
# Data from: https://landsat.usgs.gov/landsat_acq
# I only need to use 8 cycle days because of 5/7 and 7/8 are offset
cycle_dates = {
7: '1970-01-01',
8: '1970-01-02',
1: '1970-01-03',
2: '1970-01-04',
3: '1970-01-05',
4: '1970-01-06',
5: '1970-01-07',
6: '1970-01-08',
}
# cycle_dates = {
# 1: '2000-01-06',
# 2: '2000-01-07',
# 3: '2000-01-08',
# 4: '2000-01-09',
# 5: '2000-01-10',
# 6: '2000-01-11',
# 7: '2000-01-12',
# 8: '2000-01-13',
# # 9: '2000-01-14',
# # 10: '2000-01-15',
# # 11: '2000-01-16',
# # 12: '2000-01-01',
# # 13: '2000-01-02',
# # 14: '2000-01-03',
# # 15: '2000-01-04',
# # 16: '2000-01-05',
# }
# Key is cycle day, values are list of paths
# First list is Landsat 8 paths, second list is Landsat 7 paths
cycle_paths = {
5:
[1, 17, 33, 49, 65, 81, 97, 106, 122, 138, 154, 170, 186, 202, 218] +
[9, 25, 41, 57, 73, 89, 98, 114, 130, 146, 162, 178, 194, 210, 226],
# 12: [ 2, 18, 34, 50, 66, 82, 107, 123, 139, 155, 171, 187, 203, 219] +
# [10, 26, 42, 58, 74, 99, 115, 131, 147, 163, 179, 195, 211, 227],
3: [3, 19, 35, 51, 67, 83, 108, 124, 140, 156, 172, 188, 204, 220] +
[11, 27, 43, 59, 75, 100, 116, 132, 148, 164, 180, 196, 212, 228],
# 10: [ 4, 20, 36, 52, 68, 84, 109, 125, 141, 157, 171, 189, 205, 221] +
# [12, 28, 44, 60, 76, 101, 117, 133, 149, 165, 181, 197, 213, 229],
1: [5, 21, 37, 53, 69, 85, 110, 126, 142, 158, 174, 190, 206, 222] +
[13, 29, 45, 61, 77, 102, 118, 134, 150, 166, 182, 198, 214, 230],
8: [6, 22, 38, 54, 70, 86, 111, 127, 143, 159, 175, 191, 207, 223] +
[14, 30, 46, 62, 78, 103, 119, 135, 151, 167, 183, 199, 215, 231],
# 15: [ 7, 23, 39, 55, 71, 87, 112, 128, 144, 160, 176, 192, 208, 224] +
# [15, 31, 47, 63, 79, 104, 120, 136, 152, 168, 184, 200, 216, 232],
6: [8, 24, 40, 56, 72, 88, 113, 129, 145, 161, 177, 193, 209, 225] +
[16, 32, 48, 64, 80, 105, 121, 137, 153, 169, 185, 201, 217, 233],
# 13: [ 9, 25, 41, 57, 73, 89, 98, 114, 130, 146, 162, 178, 194, 210, 226] +
# [ 1, 17, 33, 49, 65, 81, 90, 106, 122, 138, 154, 170, 186, 202, 218],
4:
[10, 26, 42, 58, 74, 90, 99, 115, 131, 147, 163, 179, 195, 211, 227] +
[2, 18, 34, 50, 66, 82, 91, 107, 123, 139, 155, 171, 187, 203, 219],
# 11: [11, 27, 43, 59, 75, 91, 100, 116, 132, 148, 164, 180, 196, 212, 228] +
# [ 3, 19, 35, 51, 67, 83, 92, 108, 124, 140, 156, 172, 188, 204, 220],
2:
[12, 28, 44, 60, 76, 92, 101, 117, 133, 149, 165, 181, 197, 213, 229] +
[4, 20, 36, 52, 68, 84, 93, 109, 125, 141, 157, 173, 189, 205, 221],
# 9: [13, 29, 45, 61, 77, 93, 102, 118, 134, 150, 166, 182, 198, 214, 230] +
# [ 5, 21, 37, 53, 69, 85, 94, 110, 126, 142, 158, 174, 190, 206, 222],
# 16: [14, 30, 46, 62, 78, 94, 103, 119, 135, 151, 167, 183, 199, 215, 231] +
# [ 6, 22, 38, 54, 70, 86, 95, 111, 127, 143, 159, 175, 191, 207, 223],
7:
[15, 31, 47, 63, 79, 95, 104, 120, 136, 152, 168, 184, 200, 216, 232] +
[7, 23, 39, 55, 71, 87, 96, 112, 128, 144, 160, 176, 192, 208, 224],
# 14: [16, 32, 48, 64, 80, 96, 105, 121, 137, 153, 169, 185, 201, 217, 233] +
# [ 8, 24, 40, 56, 72, 88, 97, 113, 129, 145, 161, 177, 193, 209, 225],
}
# Iterate over date ranges
for month in month_list:
logging.info('\nMonth: {}'.format(month))
for cycle_day, ref_date in sorted(cycle_dates.items()):
logging.info('Cycle Day: {}'.format(cycle_day))
# # DEADBEEF
# if cycle_day not in [2]:
# continue
ref_dt = datetime.datetime.strptime(ref_date, '%Y-%m-%d')
logging.debug(' Reference Date: {}'.format(ref_date))
date_list = sorted(
list(
utils.date_range(datetime.datetime(year_list[0], 1, 1),
datetime.datetime(year_list[-1], 12,
31))))
date_list = [
d.strftime('%Y-%m-%d') for d in date_list
if ((abs(d - ref_dt).days % 8 == 0) and (
int(d.month) == month) and (int(d.year) in year_list))
]
logging.debug(' Dates: {}'.format(', '.join(date_list)))
export_id = export_id_fmt.format(product=tmax_name.lower(),
month=month,
cycle=cycle_day)
logging.info(' Export ID: {}'.format(export_id))
asset_id = asset_id_fmt.format(coll_id=tcorr_monthly_coll_id,
month=month,
cycle=cycle_day)
logging.info(' Asset ID: {}'.format(asset_id))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id]['id'])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if asset_id in asset_list:
logging.debug(' Asset already exists, removing')
ee.data.deleteAsset(asset_id)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, exiting')
continue
elif asset_id in asset_list:
logging.debug(' Asset already exists, skipping')
continue
wrs2_coll = ee.FeatureCollection(wrs2_coll_id) \
.filterBounds(tmax_mask.geometry()) \
.filter(ee.Filter.inList('PATH', cycle_paths[cycle_day]))
tcorr_daily_coll = ee.ImageCollection(tcorr_daily_coll_id) \
.filter(ee.Filter.inList('date', date_list))
# .filterMetadata('cycle_day', 'equals', cycle_day)
def wrs2_tcorr(ftr):
# Build & merge the Landsat collections for the target path/row
# Time filters are to remove bad (L5) and pre-op (L8) images
path = ee.Number(ee.Feature(ftr).get('PATH'))
row = ee.Number(ee.Feature(ftr).get('ROW'))
def tcorr_img_func(image):
tcorr_daily_img = image \
.addBands(image.multiply(0).add(
int(ini['TCORR']['min_pixel_count']))) \
.rename(['tcorr', 'count'])
# Get Tcorr from the WRS2 centroid of the daily images
t_stats = tcorr_daily_img.reduceRegion(
reducer=ee.Reducer.first(),
scale=30,
geometry=ee.Feature(ftr).geometry().centroid(),
bestEffort=False,
tileScale=1)
# Add a 0 tcorr value for any image that can be computed
t_stats = ee.Dictionary(t_stats) \
.combine({'tcorr': 0}, overwrite=False)
tcorr = ee.Number(t_stats.get('tcorr'))
# Use a dummy pixel count (0 or Tcorr * 2000)
count = ee.Number(t_stats.get('tcorr')) \
.multiply(2 * int(ini['TCORR']['min_pixel_count']))
# count = ee.Number(t_stats.get('count'))
return tmax_mask.add(ee.Image.constant(tcorr)) \
.rename(['tcorr']) \
.set({
'system:time_start': image.get('system:time_start'),
'tcorr': tcorr,
'count': count,
})
# temp = tcorr_img_func(ee.Image(tcorr_daily_coll.first()))
# pprint.pprint(temp.getInfo())
# input('ENTER')
# Use a common reducer for the images and property stats
reducer = ee.Reducer.median() \
.combine(ee.Reducer.count(), sharedInputs=True)
# Compute median monthly value for all images in the WRS2 tile
wrs2_tcorr_coll = ee.ImageCollection(
tcorr_daily_coll.map(tcorr_img_func)) \
.filterMetadata('count', 'not_less_than',
int(ini['TCORR']['min_pixel_count']))
wrs2_tcorr_img = wrs2_tcorr_coll.reduce(reducer) \
.rename(['tcorr', 'count'])
# Compute stats from the properties also
wrs2_tcorr_stats = ee.Dictionary(
ee.List(wrs2_tcorr_coll.aggregate_array('tcorr')).reduce(
reducer))
wrs2_tcorr_stats = wrs2_tcorr_stats \
.combine({'median': 0, 'count': 0}, overwrite=False)
return wrs2_tcorr_img \
.clip(ee.Feature(ftr).geometry()) \
.set({
'wrs2_tile': path.format('%03d').cat(row.format('%03d')),
# 'wrs2_tile': ftr.get('WRS2_TILE'),
'tcorr': ee.Number(wrs2_tcorr_stats.get('median')),
'count': ee.Number(wrs2_tcorr_stats.get('count')),
'index': 1,
})
# # DEADBEEF
# for row in [35]:
# print('\nPATH: 36 ROW: {}'.format(row))
# wrs2_test_coll = wrs2_coll \
# .filterMetadata('PATH', 'equals', 36) \
# .filterMetadata('ROW', 'equals', row)
# pprint.pprint(wrs2_test_coll.getInfo())
# output_img = ee.Image(wrs2_tcorr(ee.Feature(wrs2_test_coll.first())))
# pprint.pprint(output_img.getInfo())
# input('ENTER')
# Combine WRS2 Tcorr monthly images to a single monthly image
output_img = ee.ImageCollection(wrs2_coll.map(wrs2_tcorr)) \
.filterMetadata('count', 'not_less_than',
float(ini['TCORR']['min_scene_count'])) \
.mean() \
.rename(['tcorr', 'count'])
# pprint.pprint(output_img.getInfo())
# input('ENTER')
# .updateMask(0) \
output_img = ee.Image([
tmax_mask.add(output_img.select(['tcorr'])).double(),
tmax_mask.add(output_img.select(['count'])).min(250).uint8()]) \
.rename(['tcorr', 'count']) \
.set({
# 'system:time_start': utils.millis(iter_start_dt),
'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'),
'month': int(month),
'years': ','.join(map(str, year_list)),
'cycle_day': int(cycle_day),
'model_name': model_name,
'model_version': ssebop.__version__,
'tmax_source': tmax_source.upper(),
'tmax_version': tmax_version.upper(),
})
# pprint.pprint(output_img.getInfo())
# input('ENTER')
logging.debug(' Building export task')
task = ee.batch.Export.image.toAsset(
image=ee.Image(output_img),
description=export_id,
assetId=asset_id,
crs=tcorr_crs,
crsTransform='[' + ','.join(list(map(str, tcorr_geo))) + ']',
dimensions='{0}x{1}'.format(*tcorr_shape),
)
logging.debug(' Starting export task')
utils.ee_task_start(task)
# Pause before starting the next export task
utils.delay_task(delay_time, max_ready)
logging.debug('')
def main(ini_path=None):
"""Remove earlier versions of daily tcorr images
Parameters
----------
ini_path : str
Input file path.
"""
logging.info('\nRemove earlier versions of daily tcorr images')
ini = utils.read_ini(ini_path)
model_name = 'SSEBOP'
# model_name = ini['INPUTS']['et_model'].upper()
start_dt = datetime.datetime.strptime(
ini['INPUTS']['start_date'], '%Y-%m-%d')
end_dt = datetime.datetime.strptime(
ini['INPUTS']['end_date'], '%Y-%m-%d')
logging.debug('Start Date: {}'.format(start_dt.strftime('%Y-%m-%d')))
logging.debug('End Date: {}\n'.format(end_dt.strftime('%Y-%m-%d')))
tcorr_source = 'IMAGE'
try:
tmax_source = str(ini[model_name]['tmax_source']).upper()
logging.debug('\ntmax_source:\n {}'.format(tmax_source))
except KeyError:
logging.error(' tmax_source: must be set in INI')
sys.exit()
# This check is limited to TOPOWX_MEDIAN_V0 because Tcorr images have only
# been built for that dataset
if tmax_source.upper() not in ['TOPOWX_MEDIAN_V0']:
raise ValueError('tmax_source must be TOPOWX')
if (tmax_source.upper() == 'CIMIS' and
ini['INPUTS']['end_date'] < '2003-10-01'):
logging.error(
'\nCIMIS is not currently available before 2003-10-01, exiting\n')
sys.exit()
elif (tmax_source.upper() == 'DAYMET' and
ini['INPUTS']['end_date'] > '2017-12-31'):
logging.warning(
'\nDAYMET is not currently available past 2017-12-31, '
'using median Tmax values\n')
# Output tcorr daily image collection
tcorr_daily_coll_id = '{}/{}_daily'.format(
ini['EXPORT']['export_coll'], tmax_source.lower())
logging.debug(' {}'.format(tcorr_daily_coll_id))
if os.name == 'posix':
shell_flag = False
else:
shell_flag = True
logging.info('\nInitializing Earth Engine')
ee.Initialize()
utils.get_info(ee.Number(1))
# Get list of existing images/files
logging.debug('\nGetting GEE asset list')
asset_list = utils.get_ee_assets(tcorr_daily_coll_id, shell_flag=shell_flag)
logging.debug('Displaying first 10 images in collection')
logging.debug(asset_list[:10])
# Filter asset list by INI start_date and end_date
logging.debug('\nFiltering by INI start_date and end_date')
asset_re = re.compile('(\d{8})_\d{8}')
asset_list = [
asset_id for asset_id in asset_list
if (asset_re.match(asset_id.split('/')[-1]) and
start_dt <= datetime.datetime.strptime(asset_re.findall(asset_id.split('/')[-1])[0], '%Y%m%d') and
datetime.datetime.strptime(asset_re.findall(asset_id.split('/')[-1])[0], '%Y%m%d') <= end_dt)]
if not asset_list:
logging.info('Empty asset ID list after filter by start/end date, '
'exiting')
return True
logging.debug('Displaying first 10 images in collection')
logging.debug(asset_list[:10])
# Group asset IDs by image date
asset_id_dict = defaultdict(list)
for asset_id in asset_list:
asset_dt = datetime.datetime.strptime(
asset_id.split('/')[-1].split('_')[0], '%Y%m%d')
asset_id_dict[asset_dt.strftime('%Y-%m-%d')].append(asset_id)
# pprint.pprint(asset_id_dict)
# Remove all but the last image when sorted by export date
logging.info('\nRemoving assets')
for key, asset_list in asset_id_dict.items():
# logging.debug('{}'.format(key))
if len(asset_list) >=2:
# logging.debug('\n Keep: {}'.format(sorted(asset_list)[-1]))
for asset_id in sorted(asset_list)[:-1]:
logging.info(' Delete: {}'.format(asset_id))
try:
ee.data.deleteAsset(asset_id)
except Exception as e:
logging.info(' Unhandled exception, skipping')
logging.debug(e)
continue
def main(ini_path=None,
overwrite_flag=False,
delay=0,
key=None,
cron_flag=False,
reverse_flag=False):
"""Compute daily dT images
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, generate new images (but with different export dates) even if
the dates already have images. If False, only generate images for
dates that are missing. The default is False.
delay : float, optional
Delay time between each export task (the default is 0).
key : str, optional
File path to an Earth Engine json key file (the default is None).
reverse_flag : bool, optional
If True, process dates in reverse order.
"""
logging.info('\nCompute daily dT images')
ini = utils.read_ini(ini_path)
model_name = 'SSEBOP'
# model_name = ini['INPUTS']['et_model'].upper()
if ini[model_name]['dt_source'].upper() == 'CIMIS':
daily_coll_id = 'projects/climate-engine/cimis/daily'
elif ini[model_name]['dt_source'].upper() == 'DAYMET':
daily_coll_id = 'NASA/ORNL/DAYMET_V3'
elif ini[model_name]['dt_source'].upper() == 'GRIDMET':
daily_coll_id = 'IDAHO_EPSCOR/GRIDMET'
else:
raise ValueError('dt_source must be CIMIS, DAYMET, or GRIDMET')
# Check dates
if (ini[model_name]['dt_source'].upper() == 'CIMIS'
and ini['INPUTS']['end_date'] < '2003-10-01'):
logging.error(
'\nCIMIS is not currently available before 2003-10-01, exiting\n')
sys.exit()
elif (ini[model_name]['dt_source'].upper() == 'DAYMET'
and ini['INPUTS']['end_date'] > '2017-12-31'):
logging.warning('\nDAYMET is not currently available past 2017-12-31, '
'using median Tmax values\n')
# sys.exit()
# elif (ini[model_name]['tmax_source'].upper() == 'TOPOWX' and
# ini['INPUTS']['end_date'] > '2017-12-31'):
# logging.warning(
# '\nDAYMET is not currently available past 2017-12-31, '
# 'using median Tmax values\n')
# # sys.exit()
logging.info('\nInitializing Earth Engine')
if key:
logging.info(' Using service account key file: {}'.format(key))
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('deadbeef', key_file=key))
else:
ee.Initialize()
# Output dT daily image collection
dt_daily_coll_id = '{}/{}_daily'.format(
ini['EXPORT']['export_coll'], ini[model_name]['dt_source'].lower())
# Get an input image to set the dT values to
logging.debug('\nInput properties')
dt_name = ini[model_name]['dt_source']
dt_source = dt_name.split('_', 1)[0]
# dt_version = dt_name.split('_', 1)[1]
daily_coll = ee.ImageCollection(daily_coll_id)
dt_img = ee.Image(daily_coll.first()).select([0])
dt_mask = dt_img.multiply(0)
logging.debug(' Collection: {}'.format(daily_coll_id))
logging.debug(' Source: {}'.format(dt_source))
# logging.debug(' Version: {}'.format(dt_version))
logging.debug('\nExport properties')
export_proj = dt_img.projection().getInfo()
export_geo = export_proj['transform']
if 'crs' in export_proj.keys():
export_crs = export_proj['crs']
elif 'wkt' in export_proj.keys():
export_crs = re.sub(',\s+', ',', export_proj['wkt'])
export_shape = dt_img.getInfo()['bands'][0]['dimensions']
export_extent = [
export_geo[2], export_geo[5] + export_shape[1] * export_geo[4],
export_geo[2] + export_shape[0] * export_geo[0], export_geo[5]
]
logging.debug(' CRS: {}'.format(export_crs))
logging.debug(' Extent: {}'.format(export_extent))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
# Get current asset list
if ini['EXPORT']['export_dest'].upper() == 'ASSET':
logging.debug('\nGetting asset list')
# DEADBEEF - daily is hardcoded in the asset_id for now
asset_list = utils.get_ee_assets(dt_daily_coll_id)
else:
raise ValueError('invalid export destination: {}'.format(
ini['EXPORT']['export_dest']))
# Get current running tasks
tasks = utils.get_ee_tasks()
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
logging.debug(' Tasks: {}\n'.format(len(tasks)))
input('ENTER')
# Limit by year and month
try:
month_list = sorted(list(utils.parse_int_set(ini['INPUTS']['months'])))
except:
logging.info('\nINPUTS "months" parameter not set in the INI,'
'\n Defaulting to all months (1-12)\n')
month_list = list(range(1, 13))
# try:
# year_list = sorted(list(utils.parse_int_set(ini['INPUTS']['years'])))
# except:
# logging.info('\nINPUTS "years" parameter not set in the INI,'
# '\n Defaulting to all available years\n')
# year_list = []
# Group asset IDs by image date
asset_id_dict = defaultdict(list)
for asset_id in asset_list:
asset_dt = datetime.datetime.strptime(
asset_id.split('/')[-1].split('_')[0], '%Y%m%d')
asset_id_dict[asset_dt.strftime('%Y-%m-%d')].append(asset_id)
# pprint.pprint(export_dt_dict)
iter_start_dt = datetime.datetime.strptime(ini['INPUTS']['start_date'],
'%Y-%m-%d')
iter_end_dt = datetime.datetime.strptime(ini['INPUTS']['end_date'],
'%Y-%m-%d')
logging.debug('Start Date: {}'.format(iter_start_dt.strftime('%Y-%m-%d')))
logging.debug('End Date: {}\n'.format(iter_end_dt.strftime('%Y-%m-%d')))
for export_dt in sorted(utils.date_range(iter_start_dt, iter_end_dt),
reverse=reverse_flag):
export_date = export_dt.strftime('%Y-%m-%d')
# if ((month_list and export_dt.month not in month_list) or
# (year_list and export_dt.year not in year_list)):
if month_list and export_dt.month not in month_list:
logging.debug(f'Date: {export_date} - month not in INI - skipping')
continue
elif export_date >= datetime.datetime.today().strftime('%Y-%m-%d'):
logging.debug(f'Date: {export_date} - unsupported date - skipping')
continue
logging.info(f'Date: {export_date}')
export_id = ini['EXPORT']['export_id_fmt'] \
.format(
product=dt_name.lower(),
date=export_dt.strftime('%Y%m%d'),
export=datetime.datetime.today().strftime('%Y%m%d'),
dest=ini['EXPORT']['export_dest'].lower())
logging.debug(' Export ID: {}'.format(export_id))
if ini['EXPORT']['export_dest'] == 'ASSET':
asset_id = '{}/{}_{}'.format(
dt_daily_coll_id, export_dt.strftime('%Y%m%d'),
datetime.datetime.today().strftime('%Y%m%d'))
logging.debug(' Asset ID: {}'.format(asset_id))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if (ini['EXPORT']['export_dest'].upper() == 'ASSET'
and asset_id in asset_list):
logging.debug(' Asset already exists, removing')
ee.data.deleteAsset(asset_id)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, exiting')
continue
elif (ini['EXPORT']['export_dest'].upper() == 'ASSET'
and asset_id in asset_list):
logging.debug(
' Asset with current export date already exists, '
'skipping')
continue
elif len(asset_id_dict[export_date]) > 0:
logging.debug(
' Asset with earlier export date already exists, '
'skipping')
continue
# Compute dT using a fake Landsat image
# The system:time_start property is the only needed value
model_obj = ssebop.Image(
ee.Image.constant([0, 0]).rename(['ndvi', 'lst']).set({
'system:time_start':
utils.millis(export_dt),
'system:index':
'LC08_043033_20170716',
'system:id':
'LC08_043033_20170716'
}),
dt_source=dt_source.upper(),
elev_source='SRTM',
dt_min=ini['SSEBOP']['dt_min'],
dt_max=ini['SSEBOP']['dt_max'],
)
# Cast to float and set properties
dt_img = model_obj.dt.float() \
.set({
'system:time_start': utils.millis(export_dt),
'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'),
'date': export_dt.strftime('%Y-%m-%d'),
'year': int(export_dt.year),
'month': int(export_dt.month),
'day': int(export_dt.day),
'doy': int(export_dt.strftime('%j')),
'model_name': model_name,
'model_version': ssebop.__version__,
'dt_source': dt_source.upper(),
# 'dt_version': dt_version.upper(),
})
# Build export tasks
if ini['EXPORT']['export_dest'] == 'ASSET':
logging.debug(' Building export task')
task = ee.batch.Export.image.toAsset(
image=ee.Image(dt_img),
description=export_id,
assetId=asset_id,
crs=export_crs,
crsTransform='[' + ','.join(list(map(str, export_geo))) + ']',
dimensions='{0}x{1}'.format(*export_shape),
)
logging.info(' Starting export task')
utils.ee_task_start(task)
# Pause before starting next task
utils.delay_task(delay)
logging.debug('')
def main(ini_path=None,
overwrite_flag=False,
delay_time=0,
gee_key_file=None,
max_ready=-1,
reverse_flag=False):
"""Compute monthly Tcorr images by WRS2 tile
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
delay_time : float, optional
Delay time in seconds between starting export tasks (or checking the
number of queued tasks, see "max_ready" parameter). The default is 0.
gee_key_file : str, None, optional
Earth Engine service account JSON key file (the default is None).
max_ready: int, optional
Maximum number of queued "READY" tasks. The default is -1 which is
implies no limit to the number of tasks that will be submitted.
reverse_flag : bool, optional
If True, process WRS2 tiles in reverse order.
"""
logging.info('\nCompute annual Tcorr images by WRS2 tile')
ini = utils.read_ini(ini_path)
model_name = 'SSEBOP'
# model_name = ini['INPUTS']['et_model'].upper()
tmax_name = ini[model_name]['tmax_source']
export_id_fmt = 'tcorr_scene_{product}_{wrs2}_annual'
asset_id_fmt = '{coll_id}/{wrs2}'
tcorr_annual_coll_id = '{}/{}_annual'.format(ini['EXPORT']['export_coll'],
tmax_name.lower())
wrs2_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/wrs2_descending_custom'
wrs2_tile_field = 'WRS2_TILE'
# wrs2_path_field = 'ROW'
# wrs2_row_field = 'PATH'
try:
wrs2_tiles = str(ini['INPUTS']['wrs2_tiles'])
wrs2_tiles = [x.strip() for x in wrs2_tiles.split(',')]
wrs2_tiles = sorted([x.lower() for x in wrs2_tiles if x])
except KeyError:
wrs2_tiles = []
logging.debug(' wrs2_tiles: not set in INI, defaulting to []')
except Exception as e:
raise e
try:
study_area_extent = str(ini['INPUTS']['study_area_extent']) \
.replace('[', '').replace(']', '').split(',')
study_area_extent = [float(x.strip()) for x in study_area_extent]
except KeyError:
study_area_extent = None
logging.debug(' study_area_extent: not set in INI')
except Exception as e:
raise e
# TODO: Add try/except blocks and default values?
collections = [x.strip() for x in ini['INPUTS']['collections'].split(',')]
cloud_cover = float(ini['INPUTS']['cloud_cover'])
min_pixel_count = float(ini['TCORR']['min_pixel_count'])
min_scene_count = float(ini['TCORR']['min_scene_count'])
if (tmax_name.upper() == 'CIMIS'
and ini['INPUTS']['end_date'] < '2003-10-01'):
logging.error(
'\nCIMIS is not currently available before 2003-10-01, exiting\n')
sys.exit()
elif (tmax_name.upper() == 'DAYMET'
and ini['INPUTS']['end_date'] > '2018-12-31'):
logging.warning('\nDAYMET is not currently available past 2018-12-31, '
'using median Tmax values\n')
# sys.exit()
# elif (tmax_name.upper() == 'TOPOWX' and
# ini['INPUTS']['end_date'] > '2017-12-31'):
# logging.warning(
# '\nDAYMET is not currently available past 2017-12-31, '
# 'using median Tmax values\n')
# # sys.exit()
# Extract the model keyword arguments from the INI
# Set the property name to lower case and try to cast values to numbers
model_args = {
k.lower(): float(v) if utils.is_number(v) else v
for k, v in dict(ini[model_name]).items()
}
# et_reference_args = {
# k: model_args.pop(k)
# for k in [k for k in model_args.keys() if k.startswith('et_reference_')]}
logging.info('\nInitializing Earth Engine')
if gee_key_file:
logging.info(
' Using service account key file: {}'.format(gee_key_file))
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('x', key_file=gee_key_file))
else:
ee.Initialize()
logging.debug('\nTmax properties')
tmax_source = tmax_name.split('_', 1)[0]
tmax_version = tmax_name.split('_', 1)[1]
tmax_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower())
tmax_coll = ee.ImageCollection(tmax_coll_id)
tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0)
logging.debug(' Collection: {}'.format(tmax_coll_id))
logging.debug(' Source: {}'.format(tmax_source))
logging.debug(' Version: {}'.format(tmax_version))
logging.debug('\nExport properties')
export_info = utils.get_info(ee.Image(tmax_mask))
if 'daymet' in tmax_name.lower():
# Custom smaller extent for DAYMET focused on CONUS
export_extent = [-1999750, -1890500, 2500250, 1109500]
export_shape = [4500, 3000]
export_geo = [1000, 0, -1999750, 0, -1000, 1109500]
# Custom medium extent for DAYMET of CONUS, Mexico, and southern Canada
# export_extent = [-2099750, -3090500, 2900250, 1909500]
# export_shape = [5000, 5000]
# export_geo = [1000, 0, -2099750, 0, -1000, 1909500]
export_crs = export_info['bands'][0]['crs']
else:
export_crs = export_info['bands'][0]['crs']
export_geo = export_info['bands'][0]['crs_transform']
export_shape = export_info['bands'][0]['dimensions']
# export_geo = ee.Image(tmax_mask).projection().getInfo()['transform']
# export_crs = ee.Image(tmax_mask).projection().getInfo()['crs']
# export_shape = ee.Image(tmax_mask).getInfo()['bands'][0]['dimensions']
export_extent = [
export_geo[2], export_geo[5] + export_shape[1] * export_geo[4],
export_geo[2] + export_shape[0] * export_geo[0], export_geo[5]
]
export_geom = ee.Geometry.Rectangle(export_extent,
proj=export_crs,
geodesic=False)
logging.debug(' CRS: {}'.format(export_crs))
logging.debug(' Extent: {}'.format(export_extent))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
if study_area_extent is None:
if 'daymet' in tmax_name.lower():
# CGM - For now force DAYMET to a slightly smaller "CONUS" extent
study_area_extent = [-125, 25, -65, 49]
# study_area_extent = [-125, 25, -65, 52]
elif 'cimis' in tmax_name.lower():
study_area_extent = [-124, 35, -119, 42]
else:
# TODO: Make sure output from bounds is in WGS84
study_area_extent = tmax_mask.geometry().bounds().getInfo()
logging.debug(f'\nStudy area extent not set in INI, '
f'default to {study_area_extent}')
study_area_geom = ee.Geometry.Rectangle(study_area_extent,
proj='EPSG:4326',
geodesic=False)
if not ee.data.getInfo(tcorr_annual_coll_id):
logging.info('\nExport collection does not exist and will be built'
'\n {}'.format(tcorr_annual_coll_id))
input('Press ENTER to continue')
ee.data.createAsset({'type': 'IMAGE_COLLECTION'}, tcorr_annual_coll_id)
# Get current asset list
logging.debug('\nGetting GEE asset list')
asset_list = utils.get_ee_assets(tcorr_annual_coll_id)
# if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
# pprint.pprint(asset_list[:10])
# Get current running tasks
tasks = utils.get_ee_tasks()
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
logging.debug(' Tasks: {}\n'.format(len(tasks)))
input('ENTER')
# if cron_flag:
# # CGM - This seems like a silly way of getting the date as a datetime
# # Why am I doing this and not using the commented out line?
# end_dt = datetime.date.today().strftime('%Y-%m-%d')
# end_dt = datetime.datetime.strptime(end_dt, '%Y-%m-%d')
# end_dt = end_dt + datetime.timedelta(days=-4)
# # end_dt = datetime.datetime.today() + datetime.timedelta(days=-1)
# start_dt = end_dt + datetime.timedelta(days=-64)
# else:
start_dt = datetime.datetime.strptime(ini['INPUTS']['start_date'],
'%Y-%m-%d')
end_dt = datetime.datetime.strptime(ini['INPUTS']['end_date'], '%Y-%m-%d')
start_date = start_dt.strftime('%Y-%m-%d')
end_date = end_dt.strftime('%Y-%m-%d')
next_date = (end_dt + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
logging.debug('Start Date: {}'.format(start_date))
logging.debug('End Date: {}\n'.format(end_date))
# Limit by year and month
# try:
# month_list = sorted(list(utils.parse_int_set(ini['TCORR']['months'])))
# except:
# logging.info('\nTCORR "months" parameter not set in the INI,'
# '\n Defaulting to all months (1-12)\n')
# month_list = list(range(1, 13))
try:
year_list = sorted(list(utils.parse_int_set(ini['TCORR']['years'])))
except:
logging.info('\nTCORR "years" parameter not set in the INI,'
'\n Defaulting to all available years\n')
year_list = []
# Get the list of WRS2 tiles that intersect the data area and study area
wrs2_coll = ee.FeatureCollection(wrs2_coll_id) \
.filterBounds(export_geom) \
.filterBounds(study_area_geom)
if wrs2_tiles:
wrs2_coll = wrs2_coll.filter(
ee.Filter.inList(wrs2_tile_field, wrs2_tiles))
wrs2_info = wrs2_coll.getInfo()['features']
for wrs2_ftr in sorted(wrs2_info,
key=lambda k: k['properties']['WRS2_TILE'],
reverse=reverse_flag):
wrs2_tile = wrs2_ftr['properties'][wrs2_tile_field]
logging.info('{}'.format(wrs2_tile))
wrs2_path = int(wrs2_tile[1:4])
wrs2_row = int(wrs2_tile[5:8])
# wrs2_path = wrs2_ftr['properites']['PATH']
# wrs2_row = wrs2_ftr['properites']['ROW']
export_id = export_id_fmt.format(product=tmax_name.lower(),
wrs2=wrs2_tile)
logging.debug(' Export ID: {}'.format(export_id))
asset_id = asset_id_fmt.format(coll_id=tcorr_annual_coll_id,
wrs2=wrs2_tile)
logging.debug(' Asset ID: {}'.format(asset_id))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id]['id'])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if asset_id in asset_list:
logging.debug(' Asset already exists, removing')
ee.data.deleteAsset(asset_id)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, exiting')
continue
elif asset_id in asset_list:
logging.debug(' Asset already exists, skipping')
continue
# CGM: I couldn't find a way to build this from the Collection class
# TODO: Will need to be changed/updated for SR collection
# TODO: Add code to handle real time collections
landsat_coll = ee.ImageCollection([])
if 'LANDSAT/LC08/C01/T1_TOA' in collections:
l8_coll = ee.ImageCollection('LANDSAT/LC08/C01/T1_TOA') \
.filterMetadata('WRS_PATH', 'equals', wrs2_path) \
.filterMetadata('WRS_ROW', 'equals', wrs2_row) \
.filterMetadata('CLOUD_COVER_LAND', 'less_than', cloud_cover) \
.filterMetadata('DATA_TYPE', 'equals', 'L1TP') \
.filter(ee.Filter.gt('system:time_start',
ee.Date('2013-03-24').millis()))
# .filterDate(start_date, next_date)
landsat_coll = landsat_coll.merge(l8_coll)
if 'LANDSAT/LE07/C01/T1_TOA' in collections:
l7_coll = ee.ImageCollection('LANDSAT/LE07/C01/T1_TOA') \
.filterMetadata('WRS_PATH', 'equals', wrs2_path) \
.filterMetadata('WRS_ROW', 'equals', wrs2_row) \
.filterMetadata('CLOUD_COVER_LAND', 'less_than', cloud_cover) \
.filterMetadata('DATA_TYPE', 'equals', 'L1TP')
# .filterDate(start_date, next_date)
landsat_coll = landsat_coll.merge(l7_coll)
if 'LANDSAT/LT05/C01/T1_TOA' in collections:
l5_coll = ee.ImageCollection('LANDSAT/LT05/C01/T1_TOA') \
.filterMetadata('WRS_PATH', 'equals', wrs2_path) \
.filterMetadata('WRS_ROW', 'equals', wrs2_row) \
.filterMetadata('CLOUD_COVER_LAND', 'less_than', cloud_cover) \
.filterMetadata('DATA_TYPE', 'equals', 'L1TP') \
.filter(ee.Filter.lt('system:time_start',
ee.Date('2011-12-31').millis()))
# .filterDate(start_date, next_date)
landsat_coll = landsat_coll.merge(l5_coll)
# if 'LANDSAT/LT04/C01/T1_TOA' in collections:
# l4_coll = ee.ImageCollection('LANDSAT/LT04/C01/T1_TOA') \
# .filterMetadata('WRS_PATH', 'equals', wrs2_path) \
# .filterMetadata('WRS_ROW', 'equals', wrs2_row) \
# .filterMetadata('CLOUD_COVER_LAND', 'less_than', cloud_cover) \
# .filterMetadata('DATA_TYPE', 'equals', 'L1TP')
# # .filterDate(start_date, next_date)
# landsat_coll = landsat_coll.merge(l4_coll)
def tcorr_img_func(landsat_img):
# TODO: Will need to be changed for SR
t_obj = ssebop.Image.from_landsat_c1_toa(landsat_img, **model_args)
t_stats = ee.Dictionary(t_obj.tcorr_stats) \
.combine({'tcorr_value': 0, 'tcorr_count': 0}, overwrite=False)
tcorr = ee.Number(t_stats.get('tcorr_value'))
count = ee.Number(t_stats.get('tcorr_count'))
return tmax_mask.add(tcorr) \
.rename(['tcorr']) \
.set({
'system:time_start': ee.Image(landsat_img).get('system:time_start'),
'tcorr_value': tcorr,
'tcorr_pixel_count': count,
})
# Filter the Tcorr image collection based on the pixel counts
tcorr_coll = ee.ImageCollection(landsat_coll.map(tcorr_img_func)) \
.filterMetadata('tcorr_pixel_count', 'not_less_than', min_pixel_count)
# Use a common reducer for the image and property stats
reducer = ee.Reducer.median() \
.combine(ee.Reducer.count(), sharedInputs=True)
# Compute stats from the collection images
# This might be used when Tcorr is spatial
# tcorr_img = tcorr_coll.reduce(reducer).rename(['tcorr', 'count'])
# Compute stats from the image properties
tcorr_stats = ee.List(tcorr_coll.aggregate_array('tcorr_value')) \
.reduce(reducer)
tcorr_stats = ee.Dictionary(tcorr_stats) \
.combine({'median': 0, 'count': 0}, overwrite=False)
tcorr = ee.Number(tcorr_stats.get('median'))
count = ee.Number(tcorr_stats.get('count'))
index = count.lt(min_scene_count)\
.multiply(NODATA_TCORR_INDEX - ANNUAL_TCORR_INDEX)\
.add(ANNUAL_TCORR_INDEX)
# index = ee.Algorithms.If(count.gte(min_scene_count), 6, 9)
# Clip the mask image to the Landsat footprint
# Change mask values to 1 if count >= threshold
# Mask values of 0 will be set to nodata
mask_img = tmax_mask.add(count.gte(min_scene_count)) \
.clip(ee.Geometry(wrs2_ftr['geometry']))
output_img = ee.Image(
[mask_img.multiply(tcorr), mask_img.multiply(count)]) \
.rename(['tcorr', 'count']) \
.updateMask(mask_img.unmask(0))
# # Write an empty image if the pixel count is too low
# # CGM: Check/test if this can be combined into a single If()
# tcorr_img = ee.Algorithms.If(
# count.gte(min_scene_count),
# tmax_mask.add(tcorr), tmax_mask.updateMask(0))
# count_img = ee.Algorithms.If(
# count.gte(min_scene_count),
# tmax_mask.add(count), tmax_mask.updateMask(0))
#
# # Clip to the Landsat image footprint
# output_img = ee.Image([tcorr_img, count_img]) \
# .rename(['tcorr', 'count']) \
# .clip(ee.Geometry(wrs2_ftr['geometry']))
# # Clear the transparency mask
# output_img = output_img.updateMask(output_img.unmask(0))
output_img = output_img.set({
'date_ingested':
datetime.datetime.today().strftime('%Y-%m-%d'),
'model_name':
model_name,
'model_version':
ssebop.__version__,
# 'system:time_start': utils.millis(start_dt),
'tcorr_value':
tcorr,
'tcorr_index':
index,
'tcorr_scene_count':
count,
'tmax_source':
tmax_source.upper(),
'tmax_version':
tmax_version.upper(),
'wrs2_path':
wrs2_path,
'wrs2_row':
wrs2_row,
'wrs2_tile':
wrs2_tile,
'years':
','.join(map(str, year_list)),
# 'year_start': year_list[0],
# 'year_end': year_list[-1],
})
# pprint.pprint(output_img.getInfo())
# input('ENTER')
logging.debug(' Building export task')
task = ee.batch.Export.image.toAsset(
image=output_img,
description=export_id,
assetId=asset_id,
crs=export_crs,
crsTransform='[' + ','.join(list(map(str, export_geo))) + ']',
dimensions='{0}x{1}'.format(*export_shape),
)
logging.info(' Starting export task')
utils.ee_task_start(task)
# Pause before starting the next export task
utils.delay_task(delay_time, max_ready)
logging.debug('')
def main(ini_path=None,
overwrite_flag=False,
delay_time=0,
gee_key_file=None,
max_ready=-1,
cron_flag=False,
reverse_flag=False):
"""Compute daily Tcorr images
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files if the export dates are the same and
generate new images (but with different export dates) even if the tile
lists are the same. The default is False.
delay_time : float, optional
Delay time in seconds between starting export tasks (or checking the
number of queued tasks, see "max_ready" parameter). The default is 0.
gee_key_file : str, None, optional
Earth Engine service account JSON key file (the default is None).
max_ready: int, optional
Maximum number of queued "READY" tasks. The default is -1 which is
implies no limit to the number of tasks that will be submitted.
cron_flag : bool, optional
If True, only compute Tcorr daily image if existing image does not have
all available image (using the 'wrs2_tiles' property) and limit the
date range to the last 64 days (~2 months).
reverse_flag : bool, optional
If True, process dates in reverse order.
"""
logging.info('\nCompute daily Tcorr images')
ini = utils.read_ini(ini_path)
model_name = 'SSEBOP'
# model_name = ini['INPUTS']['et_model'].upper()
tmax_name = ini[model_name]['tmax_source']
export_id_fmt = 'tcorr_image_{product}_{date}_{export}'
asset_id_fmt = '{coll_id}/{date}_{export}'
tcorr_daily_coll_id = '{}/{}_daily'.format(ini['EXPORT']['export_coll'],
tmax_name.lower())
if (tmax_name.upper() == 'CIMIS'
and ini['INPUTS']['end_date'] < '2003-10-01'):
logging.error(
'\nCIMIS is not currently available before 2003-10-01, exiting\n')
sys.exit()
elif (tmax_name.upper() == 'DAYMET'
and ini['INPUTS']['end_date'] > '2018-12-31'):
logging.warning('\nDAYMET is not currently available past 2018-12-31, '
'using median Tmax values\n')
# sys.exit()
# elif (tmax_name.upper() == 'TOPOWX' and
# ini['INPUTS']['end_date'] > '2017-12-31'):
# logging.warning(
# '\nDAYMET is not currently available past 2017-12-31, '
# 'using median Tmax values\n')
# # sys.exit()
# Extract the model keyword arguments from the INI
# Set the property name to lower case and try to cast values to numbers
model_args = {
k.lower(): float(v) if utils.is_number(v) else v
for k, v in dict(ini[model_name]).items()
}
# et_reference_args = {
# k: model_args.pop(k)
# for k in [k for k in model_args.keys() if k.startswith('et_reference_')]}
logging.info('\nInitializing Earth Engine')
if gee_key_file:
logging.info(
' Using service account key file: {}'.format(gee_key_file))
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('x', key_file=gee_key_file))
else:
ee.Initialize()
# Get a Tmax image to set the Tcorr values to
logging.debug('\nTmax properties')
tmax_source = tmax_name.split('_', 1)[0]
tmax_version = tmax_name.split('_', 1)[1]
if 'MEDIAN' in tmax_name.upper():
tmax_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower())
tmax_coll = ee.ImageCollection(tmax_coll_id)
tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0)
else:
# TODO: Add support for non-median tmax sources
raise ValueError('unsupported tmax_source: {}'.format(tmax_name))
logging.debug(' Collection: {}'.format(tmax_coll_id))
logging.debug(' Source: {}'.format(tmax_source))
logging.debug(' Version: {}'.format(tmax_version))
logging.debug('\nExport properties')
export_info = utils.get_info(ee.Image(tmax_mask))
if 'daymet' in tmax_name.lower():
# Custom smaller extent for DAYMET focused on CONUS
export_extent = [-1999750, -1890500, 2500250, 1109500]
export_shape = [4500, 3000]
export_geo = [1000, 0, -1999750, 0, -1000, 1109500]
# Custom medium extent for DAYMET of CONUS, Mexico, and southern Canada
# export_extent = [-2099750, -3090500, 2900250, 1909500]
# export_shape = [5000, 5000]
# export_geo = [1000, 0, -2099750, 0, -1000, 1909500]
export_crs = export_info['bands'][0]['crs']
else:
export_crs = export_info['bands'][0]['crs']
export_geo = export_info['bands'][0]['crs_transform']
export_shape = export_info['bands'][0]['dimensions']
# export_geo = ee.Image(tmax_mask).projection().getInfo()['transform']
# export_crs = ee.Image(tmax_mask).projection().getInfo()['crs']
# export_shape = ee.Image(tmax_mask).getInfo()['bands'][0]['dimensions']
export_extent = [
export_geo[2], export_geo[5] + export_shape[1] * export_geo[4],
export_geo[2] + export_shape[0] * export_geo[0], export_geo[5]
]
logging.debug(' CRS: {}'.format(export_crs))
logging.debug(' Extent: {}'.format(export_extent))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
# This extent will limit the WRS2 tiles that are included
# This is needed especially for non-median DAYMET Tmax since the default
# extent is huge but we are only processing a subset
if 'daymet' in tmax_name.lower():
export_geom = ee.Geometry.Rectangle(export_extent,
proj=export_crs,
geodesic=False)
# export_geom = ee.Geometry.Rectangle(
# [-125, 25, -65, 53], proj='EPSG:4326', geodesic=False)
# export_geom = ee.Geometry.Rectangle(
# [-135, 15, -55, 60], proj='EPSG:4326', geodesic=False)
elif 'cimis' in tmax_name.lower():
export_geom = ee.Geometry.Rectangle([-124, 35, -119, 42],
proj='EPSG:4326',
geodesic=False)
else:
export_geom = tmax_mask.geometry()
# If cell_size parameter is set in the INI,
# adjust the output cellsize and recompute the transform and shape
try:
export_cs = float(ini['EXPORT']['cell_size'])
export_shape = [
int(math.ceil(abs((export_shape[0] * export_geo[0]) / export_cs))),
int(math.ceil(abs((export_shape[1] * export_geo[4]) / export_cs)))
]
export_geo = [
export_cs, 0.0, export_geo[2], 0.0, -export_cs, export_geo[5]
]
logging.debug(' Custom export cell size: {}'.format(export_cs))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
except KeyError:
pass
if not ee.data.getInfo(tcorr_daily_coll_id):
logging.info('\nExport collection does not exist and will be built'
'\n {}'.format(tcorr_daily_coll_id))
input('Press ENTER to continue')
ee.data.createAsset({'type': 'IMAGE_COLLECTION'}, tcorr_daily_coll_id)
# Get current asset list
logging.debug('\nGetting GEE asset list')
asset_list = utils.get_ee_assets(tcorr_daily_coll_id)
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
pprint.pprint(asset_list[:10])
# Get current running tasks
tasks = utils.get_ee_tasks()
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
logging.debug(' Tasks: {}\n'.format(len(tasks)))
input('ENTER')
collections = [x.strip() for x in ini['INPUTS']['collections'].split(',')]
# Limit by year and month
try:
month_list = sorted(list(utils.parse_int_set(ini['TCORR']['months'])))
except:
logging.info('\nTCORR "months" parameter not set in the INI,'
'\n Defaulting to all months (1-12)\n')
month_list = list(range(1, 13))
try:
year_list = sorted(list(utils.parse_int_set(ini['TCORR']['years'])))
except:
logging.info('\nTCORR "years" parameter not set in the INI,'
'\n Defaulting to all available years\n')
year_list = []
# Key is cycle day, value is a reference date on that cycle
# Data from: https://landsat.usgs.gov/landsat_acq
# I only need to use 8 cycle days because of 5/7 and 7/8 are offset
cycle_dates = {
7: '1970-01-01',
8: '1970-01-02',
1: '1970-01-03',
2: '1970-01-04',
3: '1970-01-05',
4: '1970-01-06',
5: '1970-01-07',
6: '1970-01-08',
}
# cycle_dates = {
# 1: '2000-01-06',
# 2: '2000-01-07',
# 3: '2000-01-08',
# 4: '2000-01-09',
# 5: '2000-01-10',
# 6: '2000-01-11',
# 7: '2000-01-12',
# 8: '2000-01-13',
# # 9: '2000-01-14',
# # 10: '2000-01-15',
# # 11: '2000-01-16',
# # 12: '2000-01-01',
# # 13: '2000-01-02',
# # 14: '2000-01-03',
# # 15: '2000-01-04',
# # 16: '2000-01-05',
# }
cycle_base_dt = datetime.datetime.strptime(cycle_dates[1], '%Y-%m-%d')
if cron_flag:
# CGM - This seems like a silly way of getting the date as a datetime
# Why am I doing this and not using the commented out line?
iter_end_dt = datetime.date.today().strftime('%Y-%m-%d')
iter_end_dt = datetime.datetime.strptime(iter_end_dt, '%Y-%m-%d')
iter_end_dt = iter_end_dt + datetime.timedelta(days=-4)
# iter_end_dt = datetime.datetime.today() + datetime.timedelta(days=-1)
iter_start_dt = iter_end_dt + datetime.timedelta(days=-64)
else:
iter_start_dt = datetime.datetime.strptime(ini['INPUTS']['start_date'],
'%Y-%m-%d')
iter_end_dt = datetime.datetime.strptime(ini['INPUTS']['end_date'],
'%Y-%m-%d')
logging.debug('Start Date: {}'.format(iter_start_dt.strftime('%Y-%m-%d')))
logging.debug('End Date: {}\n'.format(iter_end_dt.strftime('%Y-%m-%d')))
for export_dt in sorted(utils.date_range(iter_start_dt, iter_end_dt),
reverse=reverse_flag):
export_date = export_dt.strftime('%Y-%m-%d')
next_date = (export_dt +
datetime.timedelta(days=1)).strftime('%Y-%m-%d')
if month_list and export_dt.month not in month_list:
logging.debug(f'Date: {export_date} - month not in INI - skipping')
continue
elif year_list and export_dt.year not in year_list:
logging.debug(f'Date: {export_date} - year not in INI - skipping')
continue
elif export_date >= datetime.datetime.today().strftime('%Y-%m-%d'):
logging.debug(f'Date: {export_date} - unsupported date - skipping')
continue
elif export_date < '1984-03-23':
logging.debug(f'Date: {export_date} - no Landsat 5+ images before '
'1984-03-16 - skipping')
continue
logging.info(f'Date: {export_date}')
export_id = export_id_fmt.format(
product=tmax_name.lower(),
date=export_dt.strftime('%Y%m%d'),
export=datetime.datetime.today().strftime('%Y%m%d'))
logging.debug(' Export ID: {}'.format(export_id))
asset_id = asset_id_fmt.format(
coll_id=tcorr_daily_coll_id,
date=export_dt.strftime('%Y%m%d'),
export=datetime.datetime.today().strftime('%Y%m%d'))
logging.debug(' Asset ID: {}'.format(asset_id))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id]['id'])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if asset_id in asset_list:
logging.debug(' Asset already exists, removing')
ee.data.deleteAsset(asset_id)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, exiting')
continue
elif asset_id in asset_list:
logging.debug(' Asset already exists, skipping')
continue
# Build and merge the Landsat collections
model_obj = ssebop.Collection(
collections=collections,
start_date=export_dt.strftime('%Y-%m-%d'),
end_date=(export_dt +
datetime.timedelta(days=1)).strftime('%Y-%m-%d'),
cloud_cover_max=float(ini['INPUTS']['cloud_cover']),
geometry=export_geom,
model_args=model_args,
# filter_args=filter_args,
)
landsat_coll = model_obj.overpass(variables=['ndvi'])
# wrs2_tiles_all = model_obj.get_image_ids()
# pprint.pprint(landsat_coll.aggregate_array('system:id').getInfo())
# input('ENTER')
logging.debug(' Getting available WRS2 tile list')
landsat_id_list = utils.get_info(
landsat_coll.aggregate_array('system:id'))
if not landsat_id_list:
logging.info(' No available images - skipping')
continue
wrs2_tiles_all = set([id.split('_')[-2] for id in landsat_id_list])
# print(wrs2_tiles_all)
# print('\n')
def tile_set_2_str(tiles):
"""Trying to build a more compact version of the WRS2 tile list"""
tile_dict = defaultdict(list)
for tile in tiles:
tile_dict[int(tile[:3])].append(int(tile[3:]))
tile_dict = {k: sorted(v) for k, v in tile_dict.items()}
tile_str = json.dumps(tile_dict, sort_keys=True) \
.replace('"', '').replace(' ', '')\
.replace('{', '').replace('}', '')
return tile_str
wrs2_tiles_all_str = tile_set_2_str(wrs2_tiles_all)
# pprint.pprint(wrs2_tiles_all_str)
# print('\n')
def tile_str_2_set(tile_str):
# tile_dict = eval(tile_str)
tile_set = set()
for t in tile_str.replace('[', '').split('],'):
path = int(t.split(':')[0])
for row in t.split(':')[1].replace(']', '').split(','):
tile_set.add('{:03d}{:03d}'.format(path, int(row)))
return tile_set
# wrs2_tiles_all_dict = tile_str_2_set(wrs2_tiles_all_str)
# pprint.pprint(wrs2_tiles_all_dict)
# If overwriting, start a new export no matter what
# The default is to no overwrite, so this mode will not be used often
if not overwrite_flag:
# Check if there are any previous images for this date
# If so, only build a new Tcorr image if there are new wrs2_tiles
# that were not used in the previous image.
# Should this code only be run in cron mode or is this the expected
# operation when (re)running for any date range?
# Should we only test the last image
# or all previous images for the date?
logging.debug(
' Checking for previous exports/versions of daily image')
tcorr_daily_coll = ee.ImageCollection(tcorr_daily_coll_id)\
.filterDate(export_date, next_date)\
.limit(1, 'date_ingested', False)
tcorr_daily_info = utils.get_info(tcorr_daily_coll)
# pprint.pprint(tcorr_daily_info)
# input('ENTER')
if tcorr_daily_info['features']:
# Assume we won't be building a new image and only set flag
# to True if the WRS2 tile lists are different
export_flag = False
# The ".limit(1, ..." on the tcorr_daily_coll above makes this
# for loop and break statement unnecessary, but leaving for now
for tcorr_img in tcorr_daily_info['features']:
# If the full WRS2 list is not present, rebuild the image
# This should only happen for much older Tcorr images
if 'wrs2_available' not in tcorr_img['properties'].keys():
logging.debug(
' "wrs2_available" property not present in '
'previous export')
export_flag = True
break
# DEADBEEF - The wrs2_available property is now a string
# wrs2_tiles_old = set(tcorr_img['properties']['wrs2_available'].split(','))
# Convert available dict str to a list of path/rows
wrs2_tiles_old_str = tcorr_img['properties'][
'wrs2_available']
wrs2_tiles_old = tile_str_2_set(wrs2_tiles_old_str)
if wrs2_tiles_all != wrs2_tiles_old:
logging.debug(' Tile Lists')
logging.debug(' Previous: {}'.format(', '.join(
sorted(wrs2_tiles_old))))
logging.debug(' Available: {}'.format(', '.join(
sorted(wrs2_tiles_all))))
logging.debug(' New: {}'.format(', '.join(
sorted(
wrs2_tiles_all.difference(wrs2_tiles_old)))))
logging.debug(' Dropped: {}'.format(', '.join(
sorted(
wrs2_tiles_old.difference(wrs2_tiles_all)))))
export_flag = True
break
if not export_flag:
logging.debug(' No new WRS2 tiles/images - skipping')
continue
# else:
# logging.debug(' Building new version')
else:
logging.debug(' No previous exports')
def tcorr_img_func(image):
t_obj = ssebop.Image.from_landsat_c1_toa(ee.Image(image),
**model_args)
t_stats = ee.Dictionary(t_obj.tcorr_stats) \
.combine({'tcorr_value': 0, 'tcorr_count': 0}, overwrite=False)
tcorr = ee.Number(t_stats.get('tcorr_value'))
count = ee.Number(t_stats.get('tcorr_count'))
# Remove the merged collection indices from the system:index
scene_id = ee.List(
ee.String(image.get('system:index')).split('_')).slice(-3)
scene_id = ee.String(scene_id.get(0)).cat('_') \
.cat(ee.String(scene_id.get(1))).cat('_') \
.cat(ee.String(scene_id.get(2)))
return tmax_mask.add(tcorr) \
.rename(['tcorr']) \
.clip(image.geometry()) \
.set({
'system:time_start': image.get('system:time_start'),
'scene_id': scene_id,
'wrs2_path': ee.Number.parse(scene_id.slice(5, 8)),
'wrs2_row': ee.Number.parse(scene_id.slice(8, 11)),
'wrs2_tile': scene_id.slice(5, 11),
'spacecraft_id': image.get('SPACECRAFT_ID'),
'tcorr': tcorr,
'count': count,
})
# Test for one image
# pprint.pprint(tcorr_img_func(ee.Image(landsat_coll \
# .filterMetadata('WRS_PATH', 'equals', 36) \
# .filterMetadata('WRS_ROW', 'equals', 33).first())).getInfo())
# input('ENTER')
# (Re)build the Landsat collection from the image IDs
landsat_coll = ee.ImageCollection(landsat_id_list)
tcorr_img_coll = ee.ImageCollection(landsat_coll.map(tcorr_img_func)) \
.filterMetadata('count', 'not_less_than',
float(ini['TCORR']['min_pixel_count']))
# If there are no Tcorr values, return an empty image
tcorr_img = ee.Algorithms.If(tcorr_img_coll.size().gt(0),
tcorr_img_coll.median(),
tmax_mask.updateMask(0))
# Build the tile list as a string of a dictionary of paths and rows
def tile_dict(path):
# Get the row list for each path
rows = tcorr_img_coll\
.filterMetadata('wrs2_path', 'equals', path)\
.aggregate_array('wrs2_row')
# Convert rows to integers (otherwise they come back as floats)
rows = ee.List(rows).sort().map(lambda row: ee.Number(row).int())
return ee.Number(path).format('%d').cat(':[')\
.cat(ee.List(rows).join(',')).cat(']')
path_list = ee.List(tcorr_img_coll.aggregate_array('wrs2_path'))\
.distinct().sort()
wrs2_tile_str = ee.List(path_list.map(tile_dict)).join(',')
# pprint.pprint(wrs2_tile_str.getInfo())
# input('ENTER')
# # DEADBEEF - This works but is really slow because of the getInfo
# logging.debug(' Getting Tcorr collection tile list')
# wrs2_tile_list = utils.get_info(
# tcorr_img_coll.aggregate_array('wrs2_tile'))
# wrs2_tile_str = tile_set_2_str(wrs2_tile_list)
# pprint.pprint(wrs2_tile_list)
# pprint.pprint(wrs2_tile_str)
# input('ENTER')
# DEADBEEF - Old approach, tile lists for big areas are too long
# def unique_properties(coll, property):
# return ee.String(ee.List(ee.Dictionary(
# coll.aggregate_histogram(property)).keys()).join(','))
# wrs2_tile_list = ee.String('').cat(unique_properties(
# tcorr_img_coll, 'wrs2_tile'))
# wrs2_tile_list = set([id.split('_')[-2] for id in wrs2_tile_list])
def unique_properties(coll, property):
return ee.String(
ee.List(
ee.Dictionary(
coll.aggregate_histogram(property)).keys()).join(','))
landsat_list = ee.String('').cat(
unique_properties(tcorr_img_coll, 'spacecraft_id'))
# Cast to float and set properties
tcorr_img = ee.Image(tcorr_img).rename(['tcorr']).double() \
.set({
'system:time_start': utils.millis(export_dt),
'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'),
'date': export_dt.strftime('%Y-%m-%d'),
'year': int(export_dt.year),
'month': int(export_dt.month),
# 'day': int(export_dt.day),
'doy': int(export_dt.strftime('%j')),
'cycle_day': ((export_dt - cycle_base_dt).days % 8) + 1,
'landsat': landsat_list,
'model_name': model_name,
'model_version': ssebop.__version__,
'tmax_source': tmax_source.upper(),
'tmax_version': tmax_version.upper(),
'wrs2_tiles': wrs2_tile_str,
'wrs2_available': wrs2_tiles_all_str,
})
# pprint.pprint(tcorr_img.getInfo()['properties'])
# input('ENTER')
logging.debug(' Building export task')
task = ee.batch.Export.image.toAsset(
image=ee.Image(tcorr_img),
description=export_id,
assetId=asset_id,
crs=export_crs,
crsTransform='[' + ','.join(list(map(str, export_geo))) + ']',
dimensions='{0}x{1}'.format(*export_shape),
)
logging.info(' Starting export task')
utils.ee_task_start(task)
# Pause before starting the next export task
utils.delay_task(delay_time, max_ready)
logging.debug('')
def main(ini_path=None,
overwrite_flag=False,
delay_time=0,
gee_key_file=None,
max_ready=-1,
cron_flag=False,
reverse_flag=False,
update_flag=False):
"""Compute scene Tcorr images by WRS2 tile
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files if the export dates are the same and
generate new images (but with different export dates) even if the tile
lists are the same. The default is False.
delay_time : float, optional
Delay time in seconds between starting export tasks (or checking the
number of queued tasks, see "max_ready" parameter). The default is 0.
gee_key_file : str, None, optional
Earth Engine service account JSON key file (the default is None).
max_ready: int, optional
Maximum number of queued "READY" tasks. The default is -1 which is
implies no limit to the number of tasks that will be submitted.
cron_flag: bool, optional
Not currently implemented.
reverse_flag : bool, optional
If True, process WRS2 tiles and dates in reverse order.
update_flag : bool, optional
If True, only overwrite scenes with an older model version.
"""
logging.info('\nCompute scene Tcorr images by WRS2 tile')
ini = utils.read_ini(ini_path)
model_name = 'SSEBOP'
# model_name = ini['INPUTS']['et_model'].upper()
tmax_name = ini[model_name]['tmax_source']
export_id_fmt = 'tcorr_scene_{product}_{scene_id}'
asset_id_fmt = '{coll_id}/{scene_id}'
tcorr_scene_coll_id = '{}/{}_scene'.format(ini['EXPORT']['export_coll'],
tmax_name.lower())
wrs2_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/wrs2_descending_custom'
wrs2_tile_field = 'WRS2_TILE'
wrs2_path_field = 'ROW'
wrs2_row_field = 'PATH'
try:
wrs2_tiles = str(ini['INPUTS']['wrs2_tiles'])
wrs2_tiles = sorted([x.strip() for x in wrs2_tiles.split(',')])
except KeyError:
wrs2_tiles = []
logging.debug(' wrs2_tiles: not set in INI, defaulting to []')
except Exception as e:
raise e
try:
study_area_extent = str(ini['INPUTS']['study_area_extent']) \
.replace('[', '').replace(']', '').split(',')
study_area_extent = [float(x.strip()) for x in study_area_extent]
except KeyError:
study_area_extent = None
logging.debug(' study_area_extent: not set in INI')
except Exception as e:
raise e
# TODO: Add try/except blocks and default values?
collections = [x.strip() for x in ini['INPUTS']['collections'].split(',')]
cloud_cover = float(ini['INPUTS']['cloud_cover'])
min_pixel_count = float(ini['TCORR']['min_pixel_count'])
# min_scene_count = float(ini['TCORR']['min_scene_count'])
if (tmax_name.upper() == 'CIMIS'
and ini['INPUTS']['end_date'] < '2003-10-01'):
logging.error(
'\nCIMIS is not currently available before 2003-10-01, exiting\n')
sys.exit()
elif (tmax_name.upper() == 'DAYMET'
and ini['INPUTS']['end_date'] > '2018-12-31'):
logging.warning('\nDAYMET is not currently available past 2018-12-31, '
'using median Tmax values\n')
# sys.exit()
# elif (tmax_name.upper() == 'TOPOWX' and
# ini['INPUTS']['end_date'] > '2017-12-31'):
# logging.warning(
# '\nDAYMET is not currently available past 2017-12-31, '
# 'using median Tmax values\n')
# # sys.exit()
# Extract the model keyword arguments from the INI
# Set the property name to lower case and try to cast values to numbers
model_args = {
k.lower(): float(v) if utils.is_number(v) else v
for k, v in dict(ini[model_name]).items()
}
# et_reference_args = {
# k: model_args.pop(k)
# for k in [k for k in model_args.keys() if k.startswith('et_reference_')]}
logging.info('\nInitializing Earth Engine')
if gee_key_file:
logging.info(
' Using service account key file: {}'.format(gee_key_file))
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('x', key_file=gee_key_file),
use_cloud_api=True)
else:
ee.Initialize(use_cloud_api=True)
# Get a Tmax image to set the Tcorr values to
logging.debug('\nTmax properties')
tmax_source = tmax_name.split('_', 1)[0]
tmax_version = tmax_name.split('_', 1)[1]
if 'MEDIAN' in tmax_name.upper():
tmax_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower())
tmax_coll = ee.ImageCollection(tmax_coll_id)
tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0)
else:
# TODO: Add support for non-median tmax sources
raise ValueError('unsupported tmax_source: {}'.format(tmax_name))
logging.debug(' Collection: {}'.format(tmax_coll_id))
logging.debug(' Source: {}'.format(tmax_source))
logging.debug(' Version: {}'.format(tmax_version))
logging.debug('\nExport properties')
export_info = utils.get_info(ee.Image(tmax_mask))
if 'daymet' in tmax_name.lower():
# Custom smaller extent for DAYMET focused on CONUS
export_extent = [-1999750, -1890500, 2500250, 1109500]
export_shape = [4500, 3000]
export_geo = [1000, 0, -1999750, 0, -1000, 1109500]
# Custom medium extent for DAYMET of CONUS, Mexico, and southern Canada
# export_extent = [-2099750, -3090500, 2900250, 1909500]
# export_shape = [5000, 5000]
# export_geo = [1000, 0, -2099750, 0, -1000, 1909500]
export_crs = export_info['bands'][0]['crs']
else:
export_crs = export_info['bands'][0]['crs']
export_geo = export_info['bands'][0]['crs_transform']
export_shape = export_info['bands'][0]['dimensions']
# export_geo = ee.Image(tmax_mask).projection().getInfo()['transform']
# export_crs = ee.Image(tmax_mask).projection().getInfo()['crs']
# export_shape = ee.Image(tmax_mask).getInfo()['bands'][0]['dimensions']
export_extent = [
export_geo[2], export_geo[5] + export_shape[1] * export_geo[4],
export_geo[2] + export_shape[0] * export_geo[0], export_geo[5]
]
export_geom = ee.Geometry.Rectangle(export_extent,
proj=export_crs,
geodesic=False)
logging.debug(' CRS: {}'.format(export_crs))
logging.debug(' Extent: {}'.format(export_extent))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
if study_area_extent is None:
if 'daymet' in tmax_name.lower():
# CGM - For now force DAYMET to a slightly smaller "CONUS" extent
study_area_extent = [-125, 25, -65, 49]
# study_area_extent = [-125, 25, -65, 52]
elif 'cimis' in tmax_name.lower():
study_area_extent = [-124, 35, -119, 42]
else:
# TODO: Make sure output from bounds is in WGS84
study_area_extent = tmax_mask.geometry().bounds().getInfo()
logging.debug(f'\nStudy area extent not set in INI, '
f'default to {study_area_extent}')
study_area_geom = ee.Geometry.Rectangle(study_area_extent,
proj='EPSG:4326',
geodesic=False)
# For now define the study area from an extent
if study_area_extent:
study_area_geom = ee.Geometry.Rectangle(study_area_extent,
proj='EPSG:4326',
geodesic=False)
export_geom = export_geom.intersection(study_area_geom, 1)
# logging.debug(' Extent: {}'.format(export_geom.bounds().getInfo()))
# If cell_size parameter is set in the INI,
# adjust the output cellsize and recompute the transform and shape
try:
export_cs = float(ini['EXPORT']['cell_size'])
export_shape = [
int(math.ceil(abs((export_shape[0] * export_geo[0]) / export_cs))),
int(math.ceil(abs((export_shape[1] * export_geo[4]) / export_cs)))
]
export_geo = [
export_cs, 0.0, export_geo[2], 0.0, -export_cs, export_geo[5]
]
logging.debug(' Custom export cell size: {}'.format(export_cs))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
except KeyError:
pass
if not ee.data.getInfo(tcorr_scene_coll_id):
logging.info('\nExport collection does not exist and will be built'
'\n {}'.format(tcorr_scene_coll_id))
input('Press ENTER to continue')
ee.data.createAsset({'type': 'IMAGE_COLLECTION'}, tcorr_scene_coll_id)
# Get current asset list
logging.debug('\nGetting GEE asset list')
asset_list = utils.get_ee_assets(tcorr_scene_coll_id)
# if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
# pprint.pprint(asset_list[:10])
# Get current running tasks
tasks = utils.get_ee_tasks()
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
logging.debug(' Tasks: {}\n'.format(len(tasks)))
input('ENTER')
# TODO: Decide if month and year lists should be applied to scene exports
# # Limit by year and month
# try:
# month_list = sorted(list(utils.parse_int_set(ini['TCORR']['months'])))
# except:
# logging.info('\nTCORR "months" parameter not set in the INI,'
# '\n Defaulting to all months (1-12)\n')
# month_list = list(range(1, 13))
# try:
# year_list = sorted(list(utils.parse_int_set(ini['TCORR']['years'])))
# except:
# logging.info('\nTCORR "years" parameter not set in the INI,'
# '\n Defaulting to all available years\n')
# year_list = []
if cron_flag:
# CGM - This seems like a silly way of getting the date as a datetime
# Why am I doing this and not using the commented out line?
end_dt = datetime.date.today().strftime('%Y-%m-%d')
end_dt = datetime.datetime.strptime(end_dt, '%Y-%m-%d')
end_dt = end_dt + datetime.timedelta(days=-4)
# end_dt = datetime.datetime.today() + datetime.timedelta(days=-1)
start_dt = end_dt + datetime.timedelta(days=-64)
else:
start_dt = datetime.datetime.strptime(ini['INPUTS']['start_date'],
'%Y-%m-%d')
end_dt = datetime.datetime.strptime(ini['INPUTS']['end_date'],
'%Y-%m-%d')
if end_dt >= datetime.datetime.today():
logging.debug('End Date: {} - setting end date to current '
'date'.format(end_dt.strftime('%Y-%m-%d')))
end_dt = datetime.datetime.today()
if start_dt < datetime.datetime(1984, 3, 23):
logging.debug('Start Date: {} - no Landsat 5+ images before '
'1984-03-23'.format(start_dt.strftime('%Y-%m-%d')))
start_dt = datetime.datetime(1984, 3, 23)
start_date = start_dt.strftime('%Y-%m-%d')
end_date = end_dt.strftime('%Y-%m-%d')
# next_date = (start_dt + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
logging.debug('Start Date: {}'.format(start_date))
logging.debug('End Date: {}\n'.format(end_date))
if start_dt > end_dt:
raise ValueError('start date must be before end date')
# Get the list of WRS2 tiles that intersect the data area and study area
wrs2_coll = ee.FeatureCollection(wrs2_coll_id) \
.filterBounds(export_geom) \
.filterBounds(study_area_geom)
if wrs2_tiles:
wrs2_coll = wrs2_coll.filter(
ee.Filter.inList(wrs2_tile_field, wrs2_tiles))
wrs2_info = wrs2_coll.getInfo()['features']
# pprint.pprint(wrs2_info)
# input('ENTER')
# Iterate over WRS2 tiles (default is from west to east)
for wrs2_ftr in sorted(wrs2_info,
key=lambda k: k['properties']['WRS2_TILE'],
reverse=not (reverse_flag)):
wrs2_tile = wrs2_ftr['properties'][wrs2_tile_field]
logging.info('{}'.format(wrs2_tile))
wrs2_path = int(wrs2_tile[1:4])
wrs2_row = int(wrs2_tile[5:8])
# wrs2_path = wrs2_ftr['properties']['PATH']
# wrs2_row = wrs2_ftr['properties']['ROW']
wrs2_filter = [{
'type': 'equals',
'leftField': 'WRS_PATH',
'rightValue': wrs2_path
}, {
'type': 'equals',
'leftField': 'WRS_ROW',
'rightValue': wrs2_row
}]
filter_args = {c: wrs2_filter for c in collections}
# Build and merge the Landsat collections
model_obj = ssebop.Collection(
collections=collections,
start_date=start_date,
end_date=end_date,
cloud_cover_max=cloud_cover,
geometry=ee.Geometry(wrs2_ftr['geometry']),
model_args=model_args,
filter_args=filter_args,
)
landsat_coll = model_obj.overpass(variables=['ndvi'])
# pprint.pprint(landsat_coll.aggregate_array('system:id').getInfo())
# input('ENTER')
try:
image_id_list = landsat_coll.aggregate_array('system:id').getInfo()
except Exception as e:
logging.warning(' Error getting image ID list, skipping tile')
logging.debug(f' {e}')
continue
if update_flag:
assets_info = utils.get_info(
ee.ImageCollection(tcorr_scene_coll_id).filterMetadata(
'wrs2_tile', 'equals',
wrs2_tile).filterDate(start_date, end_date))
asset_props = {
f'{tcorr_scene_coll_id}/{x["properties"]["system:index"]}':
x['properties']
for x in assets_info['features']
}
else:
asset_props = {}
# Sort by date
for image_id in sorted(image_id_list,
key=lambda k: k.split('/')[-1].split('_')[-1],
reverse=reverse_flag):
coll_id, scene_id = image_id.rsplit('/', 1)
logging.info(f'{scene_id}')
export_dt = datetime.datetime.strptime(
scene_id.split('_')[-1], '%Y%m%d')
export_date = export_dt.strftime('%Y-%m-%d')
# next_date = (export_dt + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
# # Uncomment to apply month and year list filtering
# if month_list and export_dt.month not in month_list:
# logging.debug(f' Date: {export_date} - month not in INI - skipping')
# continue
# elif year_list and export_dt.year not in year_list:
# logging.debug(f' Date: {export_date} - year not in INI - skipping')
# continue
logging.debug(f' Date: {export_date}')
export_id = export_id_fmt.format(product=tmax_name.lower(),
scene_id=scene_id)
logging.debug(f' Export ID: {export_id}')
asset_id = asset_id_fmt.format(coll_id=tcorr_scene_coll_id,
scene_id=scene_id)
logging.debug(f' Asset ID: {asset_id}')
if update_flag:
def version_number(version_str):
return list(map(int, version_str.split('.')))
if export_id in tasks.keys():
logging.info(' Task already submitted, skipping')
continue
# In update mode only overwrite if the version is old
if asset_props and asset_id in asset_props.keys():
model_ver = version_number(ssebop.__version__)
asset_ver = version_number(
asset_props[asset_id]['model_version'])
if asset_ver < model_ver:
logging.info(' Asset model version is old, removing')
try:
ee.data.deleteAsset(asset_id)
except:
logging.info(' Error removing asset, skipping')
continue
else:
logging.info(' Asset is up to date, skipping')
continue
elif overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id]['id'])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if asset_id in asset_list:
logging.debug(' Asset already exists, removing')
ee.data.deleteAsset(asset_id)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, exiting')
continue
elif asset_id in asset_list:
logging.debug(' Asset already exists, skipping')
continue
image = ee.Image(image_id)
# TODO: Will need to be changed for SR or use from_image_id()
t_obj = ssebop.Image.from_landsat_c1_toa(image_id, **model_args)
t_stats = ee.Dictionary(t_obj.tcorr_stats) \
.combine({'tcorr_p5': 0, 'tcorr_count': 0}, overwrite=False)
tcorr = ee.Number(t_stats.get('tcorr_p5'))
count = ee.Number(t_stats.get('tcorr_count'))
index = ee.Algorithms.If(count.gte(min_pixel_count), 0, 9)
# Write an empty image if the pixel count is too low
tcorr_img = ee.Algorithms.If(count.gte(min_pixel_count),
tmax_mask.add(tcorr),
tmax_mask.updateMask(0))
# Clip to the Landsat image footprint
output_img = ee.Image(tcorr_img).clip(image.geometry())
# Clear the transparency mask
output_img = output_img.updateMask(output_img.unmask(0)) \
.rename(['tcorr']) \
.set({
'CLOUD_COVER': image.get('CLOUD_COVER'),
'CLOUD_COVER_LAND': image.get('CLOUD_COVER_LAND'),
# 'SPACECRAFT_ID': image.get('SPACECRAFT_ID'),
'coll_id': coll_id,
'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'),
'date': export_dt.strftime('%Y-%m-%d'),
'doy': int(export_dt.strftime('%j')),
'image_id': image_id,
'model_name': model_name,
'model_version': ssebop.__version__,
'month': int(export_dt.month),
'scene_id': scene_id,
'system:time_start': image.get('system:time_start'),
'tcorr_value': tcorr,
'tcorr_index': index,
'tcorr_pixel_count': count,
'tmax_source': tmax_source.upper(),
'tmax_version': tmax_version.upper(),
'wrs2_path': wrs2_path,
'wrs2_row': wrs2_row,
'wrs2_tile': wrs2_tile,
'year': int(export_dt.year),
})
# pprint.pprint(output_img.getInfo()['properties'])
# input('ENTER')
logging.debug(' Building export task')
task = ee.batch.Export.image.toAsset(
image=output_img,
description=export_id,
assetId=asset_id,
crs=export_crs,
crsTransform='[' + ','.join(list(map(str, export_geo))) + ']',
dimensions='{0}x{1}'.format(*export_shape),
)
logging.info(' Starting export task')
utils.ee_task_start(task)
# Pause before starting the next date (not export task)
utils.delay_task(delay_time, max_ready)
logging.debug('')
def main(ini_path=None, overwrite_flag=False, delay_time=0, gee_key_file=None,
max_ready=-1, reverse_flag=False):
"""Compute default Tcorr images by WRS2 tile
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
delay_time : float, optional
Delay time in seconds between starting export tasks (or checking the
number of queued tasks, see "max_ready" parameter). The default is 0.
gee_key_file : str, None, optional
Earth Engine service account JSON key file (the default is None).
max_ready: int, optional
Maximum number of queued "READY" tasks. The default is -1 which is
implies no limit to the number of tasks that will be submitted.
reverse_flag : bool, optional
If True, process WRS2 tiles in reverse order.
"""
logging.info('\nCompute default Tcorr images by WRS2 tile')
ini = utils.read_ini(ini_path)
model_name = 'SSEBOP'
# model_name = ini['INPUTS']['et_model'].upper()
tmax_name = ini[model_name]['tmax_source']
export_id_fmt = 'tcorr_scene_{product}_{wrs2}_default'
asset_id_fmt = '{coll_id}/{wrs2}'
tcorr_default_coll_id = '{}/{}_default'.format(
ini['EXPORT']['export_coll'], tmax_name.lower())
wrs2_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/wrs2_descending_custom'
wrs2_tile_field = 'WRS2_TILE'
# wrs2_path_field = 'ROW'
# wrs2_row_field = 'PATH'
try:
wrs2_tiles = str(ini['INPUTS']['wrs2_tiles'])
wrs2_tiles = [x.strip() for x in wrs2_tiles.split(',')]
wrs2_tiles = sorted([x.lower() for x in wrs2_tiles if x])
except KeyError:
wrs2_tiles = []
logging.debug(' wrs2_tiles: not set in INI, defaulting to []')
except Exception as e:
raise e
try:
study_area_extent = str(ini['INPUTS']['study_area_extent']) \
.replace('[', '').replace(']', '').split(',')
study_area_extent = [float(x.strip()) for x in study_area_extent]
except KeyError:
study_area_extent = None
logging.debug(' study_area_extent: not set in INI')
except Exception as e:
raise e
try:
tcorr_default = ini[model_name]['tcorr_default']
except:
tcorr_default = 0.978
logging.info('\nInitializing Earth Engine')
if gee_key_file:
logging.info(' Using service account key file: {}'.format(gee_key_file))
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('x', key_file=gee_key_file))
else:
ee.Initialize()
logging.debug('\nTmax properties')
tmax_source = tmax_name.split('_', 1)[0]
tmax_version = tmax_name.split('_', 1)[1]
tmax_coll_id = 'projects/earthengine-legacy/assets/' \
'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower())
tmax_coll = ee.ImageCollection(tmax_coll_id)
tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0)
logging.debug(' Collection: {}'.format(tmax_coll_id))
logging.debug(' Source: {}'.format(tmax_source))
logging.debug(' Version: {}'.format(tmax_version))
# # Get the Tcorr scene image collection properties
# logging.debug('\nTcorr scene collection')
# tcorr_scene_coll_id = '{}/{}_scene'.format(
# ini['EXPORT']['export_coll'], tmax_name.lower())
logging.debug('\nExport properties')
export_info = utils.get_info(ee.Image(tmax_mask))
if 'daymet' in tmax_name.lower():
# Custom smaller extent for DAYMET focused on CONUS
export_extent = [-1999750, -1890500, 2500250, 1109500]
export_shape = [4500, 3000]
export_geo = [1000, 0, -1999750, 0, -1000, 1109500]
# Custom medium extent for DAYMET of CONUS, Mexico, and southern Canada
# export_extent = [-2099750, -3090500, 2900250, 1909500]
# export_shape = [5000, 5000]
# export_geo = [1000, 0, -2099750, 0, -1000, 1909500]
export_crs = export_info['bands'][0]['crs']
else:
export_crs = export_info['bands'][0]['crs']
export_geo = export_info['bands'][0]['crs_transform']
export_shape = export_info['bands'][0]['dimensions']
# export_geo = ee.Image(tmax_mask).projection().getInfo()['transform']
# export_crs = ee.Image(tmax_mask).projection().getInfo()['crs']
# export_shape = ee.Image(tmax_mask).getInfo()['bands'][0]['dimensions']
export_extent = [
export_geo[2], export_geo[5] + export_shape[1] * export_geo[4],
export_geo[2] + export_shape[0] * export_geo[0], export_geo[5]]
export_geom = ee.Geometry.Rectangle(
export_extent, proj=export_crs, geodesic=False)
logging.debug(' CRS: {}'.format(export_crs))
logging.debug(' Extent: {}'.format(export_extent))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
if study_area_extent is None:
if 'daymet' in tmax_name.lower():
# CGM - For now force DAYMET to a slightly smaller "CONUS" extent
study_area_extent = [-125, 25, -65, 50]
# study_area_extent = [-125, 25, -65, 49]
# study_area_extent = [-125, 25, -65, 52]
elif 'cimis' in tmax_name.lower():
study_area_extent = [-124, 35, -119, 42]
else:
# TODO: Make sure output from bounds is in WGS84
study_area_extent = tmax_mask.geometry().bounds().getInfo()
logging.debug(f'\nStudy area extent not set in INI, '
f'default to {study_area_extent}')
study_area_geom = ee.Geometry.Rectangle(
study_area_extent, proj='EPSG:4326', geodesic=False)
if not ee.data.getInfo(tcorr_default_coll_id):
logging.info('\nExport collection does not exist and will be built'
'\n {}'.format(tcorr_default_coll_id))
input('Press ENTER to continue')
ee.data.createAsset({'type': 'IMAGE_COLLECTION'}, tcorr_default_coll_id)
# Get current asset list
logging.debug('\nGetting GEE asset list')
asset_list = utils.get_ee_assets(tcorr_default_coll_id)
# if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
# pprint.pprint(asset_list[:10])
# Get current running tasks
tasks = utils.get_ee_tasks()
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
logging.debug(' Tasks: {}\n'.format(len(tasks)))
input('ENTER')
# Get the list of WRS2 tiles that intersect the data area and study area
wrs2_coll = ee.FeatureCollection(wrs2_coll_id)\
.filterBounds(export_geom)\
.filterBounds(study_area_geom)
if wrs2_tiles:
wrs2_coll = wrs2_coll.filter(ee.Filter.inList(wrs2_tile_field, wrs2_tiles))
wrs2_info = wrs2_coll.getInfo()['features']
for wrs2_ftr in sorted(wrs2_info,
key=lambda k: k['properties']['WRS2_TILE'],
reverse=reverse_flag):
wrs2_tile = wrs2_ftr['properties'][wrs2_tile_field]
logging.info('{}'.format(wrs2_tile))
wrs2_path = int(wrs2_tile[1:4])
wrs2_row = int(wrs2_tile[5:8])
# wrs2_path = wrs2_ftr['properties'][wrs2_path_field]
# wrs2_row = wrs2_ftr['properties'][wrs2_row_field]
export_id = export_id_fmt.format(
product=tmax_name.lower(), wrs2=wrs2_tile)
logging.debug(' Export ID: {}'.format(export_id))
asset_id = asset_id_fmt.format(
coll_id=tcorr_default_coll_id, wrs2=wrs2_tile)
logging.debug(' Asset ID: {}'.format(asset_id))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id]['id'])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if asset_id in asset_list:
logging.debug(' Asset already exists, removing')
ee.data.deleteAsset(asset_id)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, exiting')
continue
elif asset_id in asset_list:
logging.debug(' Asset already exists, skipping')
continue
# Clip the mask image to the Landsat footprint
mask_img = tmax_mask.add(1).clip(ee.Geometry(wrs2_ftr['geometry']))
# Apply the default Tcorr value and then clear the transparency mask
output_img = mask_img.multiply(0.978).rename(['tcorr'])\
.updateMask(mask_img.unmask(0))
# # Clip to the Landsat image footprint
# output_img = tmax_mask.add(tcorr_default) \
# .rename(['tcorr']) \
# .clip(ee.Geometry(wrs2_ftr['geometry']))
# # Clear the transparency mask
# output_img = output_img.updateMask(output_img.unmask(0))
output_img = output_img.set({
'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'),
'model_name': model_name,
'model_version': ssebop.__version__,
# 'system:time_start': utils.millis(start_dt),
'tcorr_value': tcorr_default,
'tcorr_index': DEFAULT_TCORR_INDEX,
'tmax_source': tmax_source.upper(),
'tmax_version': tmax_version.upper(),
'wrs2_path': wrs2_path,
'wrs2_row': wrs2_row,
'wrs2_tile': wrs2_tile,
})
# pprint.pprint(output_img.getInfo())
# input('ENTER')
logging.debug(' Building export task')
task = ee.batch.Export.image.toAsset(
image=output_img,
description=export_id,
assetId=asset_id,
crs=export_crs,
crsTransform='[' + ','.join(list(map(str, export_geo))) + ']',
dimensions='{0}x{1}'.format(*export_shape),
)
logging.info(' Starting export task')
utils.ee_task_start(task)
# Pause before starting the next export task
utils.delay_task(delay_time, max_ready)
logging.debug('')
def main(ini_path=None,
overwrite_flag=False,
delay=0,
key=None,
cron_flag=False,
reverse_flag=False):
"""Compute daily Tcorr images
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files if the export dates are the same and
generate new images (but with different export dates) even if the tile
lists are the same. The default is False.
delay : float, optional
Delay time between each export task (the default is 0).
key : str, optional
File path to an Earth Engine json key file (the default is None).
cron_flag : bool, optional
If True, only compute Tcorr daily image if existing image does not have
all available image (using the 'wrs2_tiles' property) and limit the
date range to the last 64 days (~2 months).
reverse_flag : bool, optional
If True, process dates in reverse order.
"""
logging.info('\nCompute daily Tcorr images')
ini = utils.read_ini(ini_path)
model_name = 'SSEBOP'
# model_name = ini['INPUTS']['et_model'].upper()
if (ini[model_name]['tmax_source'].upper() == 'CIMIS'
and ini['INPUTS']['end_date'] < '2003-10-01'):
logging.error(
'\nCIMIS is not currently available before 2003-10-01, exiting\n')
sys.exit()
elif (ini[model_name]['tmax_source'].upper() == 'DAYMET'
and ini['INPUTS']['end_date'] > '2017-12-31'):
logging.warning('\nDAYMET is not currently available past 2017-12-31, '
'using median Tmax values\n')
# sys.exit()
# elif (ini[model_name]['tmax_source'].upper() == 'TOPOWX' and
# ini['INPUTS']['end_date'] > '2017-12-31'):
# logging.warning(
# '\nDAYMET is not currently available past 2017-12-31, '
# 'using median Tmax values\n')
# # sys.exit()
logging.info('\nInitializing Earth Engine')
if key:
logging.info(' Using service account key file: {}'.format(key))
# The "EE_ACCOUNT" parameter is not used if the key file is valid
ee.Initialize(ee.ServiceAccountCredentials('deadbeef', key_file=key))
else:
ee.Initialize()
# Output Tcorr daily image collection
tcorr_daily_coll_id = '{}/{}_daily'.format(
ini['EXPORT']['export_coll'], ini[model_name]['tmax_source'].lower())
# Get a Tmax image to set the Tcorr values to
logging.debug('\nTmax properties')
tmax_name = ini[model_name]['tmax_source']
tmax_source = tmax_name.split('_', 1)[0]
tmax_version = tmax_name.split('_', 1)[1]
tmax_coll_id = 'projects/usgs-ssebop/tmax/{}'.format(tmax_name.lower())
tmax_coll = ee.ImageCollection(tmax_coll_id)
tmax_mask = ee.Image(tmax_coll.first()).select([0]).multiply(0)
logging.debug(' Collection: {}'.format(tmax_coll_id))
logging.debug(' Source: {}'.format(tmax_source))
logging.debug(' Version: {}'.format(tmax_version))
logging.debug('\nExport properties')
export_geo = ee.Image(tmax_mask).projection().getInfo()['transform']
export_crs = ee.Image(tmax_mask).projection().getInfo()['crs']
export_shape = ee.Image(tmax_mask).getInfo()['bands'][0]['dimensions']
export_extent = [
export_geo[2], export_geo[5] + export_shape[1] * export_geo[4],
export_geo[2] + export_shape[0] * export_geo[0], export_geo[5]
]
logging.debug(' CRS: {}'.format(export_crs))
logging.debug(' Extent: {}'.format(export_extent))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
# # Limit export to a user defined study area or geometry?
# export_geom = ee.Geometry.Rectangle(
# [-125, 24, -65, 50], proj='EPSG:4326', geodesic=False) # CONUS
# export_geom = ee.Geometry.Rectangle(
# [-124, 35, -119, 42], proj='EPSG:4326', geodesic=False) # California
# If cell_size parameter is set in the INI,
# adjust the output cellsize and recompute the transform and shape
try:
export_cs = float(ini['EXPORT']['cell_size'])
export_shape = [
int(math.ceil(abs((export_shape[0] * export_geo[0]) / export_cs))),
int(math.ceil(abs((export_shape[1] * export_geo[4]) / export_cs)))
]
export_geo = [
export_cs, 0.0, export_geo[2], 0.0, -export_cs, export_geo[5]
]
logging.debug(' Custom export cell size: {}'.format(export_cs))
logging.debug(' Geo: {}'.format(export_geo))
logging.debug(' Shape: {}'.format(export_shape))
except KeyError:
pass
# Get current asset list
if ini['EXPORT']['export_dest'].upper() == 'ASSET':
logging.debug('\nGetting asset list')
# DEADBEEF - daily is hardcoded in the asset_id for now
asset_list = utils.get_ee_assets(tcorr_daily_coll_id)
else:
raise ValueError('invalid export destination: {}'.format(
ini['EXPORT']['export_dest']))
# Get current running tasks
tasks = utils.get_ee_tasks()
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
logging.debug(' Tasks: {}\n'.format(len(tasks)))
input('ENTER')
collections = [x.strip() for x in ini['INPUTS']['collections'].split(',')]
# Limit by year and month
try:
month_list = sorted(list(utils.parse_int_set(ini['TCORR']['months'])))
except:
logging.info('\nTCORR "months" parameter not set in the INI,'
'\n Defaulting to all months (1-12)\n')
month_list = list(range(1, 13))
try:
year_list = sorted(list(utils.parse_int_set(ini['TCORR']['years'])))
except:
logging.info('\nTCORR "years" parameter not set in the INI,'
'\n Defaulting to all available years\n')
year_list = []
# Key is cycle day, value is a reference date on that cycle
# Data from: https://landsat.usgs.gov/landsat_acq
# I only need to use 8 cycle days because of 5/7 and 7/8 are offset
cycle_dates = {
7: '1970-01-01',
8: '1970-01-02',
1: '1970-01-03',
2: '1970-01-04',
3: '1970-01-05',
4: '1970-01-06',
5: '1970-01-07',
6: '1970-01-08',
}
# cycle_dates = {
# 1: '2000-01-06',
# 2: '2000-01-07',
# 3: '2000-01-08',
# 4: '2000-01-09',
# 5: '2000-01-10',
# 6: '2000-01-11',
# 7: '2000-01-12',
# 8: '2000-01-13',
# # 9: '2000-01-14',
# # 10: '2000-01-15',
# # 11: '2000-01-16',
# # 12: '2000-01-01',
# # 13: '2000-01-02',
# # 14: '2000-01-03',
# # 15: '2000-01-04',
# # 16: '2000-01-05',
# }
cycle_base_dt = datetime.datetime.strptime(cycle_dates[1], '%Y-%m-%d')
if cron_flag:
# CGM - This seems like a silly way of getting the date as a datetime
# Why am I doing this and not using the commented out line?
iter_end_dt = datetime.date.today().strftime('%Y-%m-%d')
iter_end_dt = datetime.datetime.strptime(iter_end_dt, '%Y-%m-%d')
iter_end_dt = iter_end_dt + datetime.timedelta(days=-4)
# iter_end_dt = datetime.datetime.today() + datetime.timedelta(days=-1)
iter_start_dt = iter_end_dt + datetime.timedelta(days=-64)
else:
iter_start_dt = datetime.datetime.strptime(ini['INPUTS']['start_date'],
'%Y-%m-%d')
iter_end_dt = datetime.datetime.strptime(ini['INPUTS']['end_date'],
'%Y-%m-%d')
logging.debug('Start Date: {}'.format(iter_start_dt.strftime('%Y-%m-%d')))
logging.debug('End Date: {}\n'.format(iter_end_dt.strftime('%Y-%m-%d')))
for export_dt in sorted(utils.date_range(iter_start_dt, iter_end_dt),
reverse=reverse_flag):
export_date = export_dt.strftime('%Y-%m-%d')
next_date = (export_dt +
datetime.timedelta(days=1)).strftime('%Y-%m-%d')
# if ((month_list and export_dt.month not in month_list) or
# (year_list and export_dt.year not in year_list)):
if month_list and export_dt.month not in month_list:
logging.debug(f'Date: {export_date} - month not in INI - skipping')
continue
elif export_date >= datetime.datetime.today().strftime('%Y-%m-%d'):
logging.debug(f'Date: {export_date} - unsupported date - skipping')
continue
elif export_date < '1984-03-23':
logging.debug(f'Date: {export_date} - no Landsat 5+ images before '
'1984-03-16 - skipping')
continue
logging.info(f'Date: {export_date}')
export_id = ini['EXPORT']['export_id_fmt'] \
.format(
product=tmax_name.lower(),
date=export_dt.strftime('%Y%m%d'),
export=datetime.datetime.today().strftime('%Y%m%d'),
dest=ini['EXPORT']['export_dest'].lower())
logging.debug(' Export ID: {}'.format(export_id))
if ini['EXPORT']['export_dest'] == 'ASSET':
asset_id = '{}/{}_{}'.format(
tcorr_daily_coll_id, export_dt.strftime('%Y%m%d'),
datetime.datetime.today().strftime('%Y%m%d'))
logging.debug(' Asset ID: {}'.format(asset_id))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if (ini['EXPORT']['export_dest'].upper() == 'ASSET'
and asset_id in asset_list):
logging.debug(' Asset already exists, removing')
ee.data.deleteAsset(asset_id)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, exiting')
continue
elif (ini['EXPORT']['export_dest'].upper() == 'ASSET'
and asset_id in asset_list):
logging.debug(' Asset already exists, skipping')
continue
# Build and merge the Landsat collections
model_obj = ssebop.Collection(
collections=collections,
start_date=export_dt.strftime('%Y-%m-%d'),
end_date=(export_dt +
datetime.timedelta(days=1)).strftime('%Y-%m-%d'),
cloud_cover_max=float(ini['INPUTS']['cloud_cover']),
geometry=tmax_mask.geometry(),
# model_args=model_args,
# filter_args=filter_args,
)
landsat_coll = model_obj.overpass(variables=['ndvi'])
# wrs2_tiles_all = model_obj.get_image_ids()
# pprint.pprint(landsat_coll.aggregate_array('system:id').getInfo())
# input('ENTER')
logging.debug(' Getting available WRS2 tile list')
landsat_id_list = landsat_coll.aggregate_array('system:id').getInfo()
wrs2_tiles_all = set([id.split('_')[-2] for id in landsat_id_list])
if not wrs2_tiles_all:
logging.info(' No available images - skipping')
continue
# If overwriting, start a new export no matter what
# The default is to no overwrite, so this mode will not be used often
if not overwrite_flag:
# Check if there are any previous images for this date
# If so, only build a new Tcorr image if there are new wrs2_tiles
# that were not used in the previous image.
# Should this code only be run in cron mode or is this the expected
# operation when (re)running for any date range?
# Should we only test the last image
# or all previous images for the date?
logging.debug(
' Checking for previous exports/versions of daily image')
tcorr_daily_coll = ee.ImageCollection(tcorr_daily_coll_id)\
.filterDate(export_date, next_date)\
.limit(1, 'date_ingested', False)
tcorr_daily_info = tcorr_daily_coll.getInfo()
if tcorr_daily_info['features']:
# Assume we won't be building a new image and only set flag
# to True if the WRS2 tile lists are different
export_flag = False
# The ".limit(1, ..." on the tcorr_daily_coll above makes this
# for loop and break statement unnecessary, but leaving for now
for tcorr_img in tcorr_daily_info['features']:
# If the full WRS2 list is not present, rebuild the image
# This should only happen for much older Tcorr images
if 'wrs2_available' not in tcorr_img['properties'].keys():
logging.debug(
' "wrs2_available" property not present in '
'previous export')
export_flag = True
break
wrs2_tiles_old = set(
tcorr_img['properties']['wrs2_available'].split(','))
if wrs2_tiles_all != wrs2_tiles_old:
logging.debug(' Tile Lists')
logging.debug(' Previous: {}'.format(', '.join(
sorted(wrs2_tiles_old))))
logging.debug(' Available: {}'.format(', '.join(
sorted(wrs2_tiles_all))))
logging.debug(' New: {}'.format(', '.join(
sorted(
wrs2_tiles_all.difference(wrs2_tiles_old)))))
logging.debug(' Dropped: {}'.format(', '.join(
sorted(
wrs2_tiles_old.difference(wrs2_tiles_all)))))
export_flag = True
break
if not export_flag:
logging.debug(' No new WRS2 tiles/images - skipping')
continue
# else:
# logging.debug(' Building new version')
else:
logging.debug(' No previous exports')
def tcorr_img_func(image):
t_stats = ssebop.Image.from_landsat_c1_toa(
ee.Image(image),
tdiff_threshold=float(ini[model_name]['tdiff_threshold'])) \
.tcorr_stats
t_stats = ee.Dictionary(t_stats) \
.combine({'tcorr_p5': 0, 'tcorr_count': 0},
overwrite=False)
tcorr = ee.Number(t_stats.get('tcorr_p5'))
count = ee.Number(t_stats.get('tcorr_count'))
# Remove the merged collection indices from the system:index
scene_id = ee.List(
ee.String(image.get('system:index')).split('_')).slice(-3)
scene_id = ee.String(scene_id.get(0)).cat('_') \
.cat(ee.String(scene_id.get(1))).cat('_') \
.cat(ee.String(scene_id.get(2)))
return tmax_mask.add(tcorr) \
.rename(['tcorr']) \
.clip(image.geometry()) \
.set({
'system:time_start': image.get('system:time_start'),
'scene_id': scene_id,
'wrs2_tile': scene_id.slice(5, 11),
'spacecraft_id': image.get('SPACECRAFT_ID'),
'tcorr': tcorr,
'count': count,
})
# Test for one image
# pprint.pprint(tcorr_img_func(ee.Image(landsat_coll \
# .filterMetadata('WRS_PATH', 'equals', 36) \
# .filterMetadata('WRS_ROW', 'equals', 33).first())).getInfo())
# input('ENTER')
# (Re)build the Landsat collection from the image IDs
landsat_coll = ee.ImageCollection(landsat_id_list)
tcorr_img_coll = ee.ImageCollection(landsat_coll.map(tcorr_img_func)) \
.filterMetadata('count', 'not_less_than',
float(ini['TCORR']['min_pixel_count']))
# If there are no Tcorr values, return an empty image
tcorr_img = ee.Algorithms.If(tcorr_img_coll.size().gt(0),
tcorr_img_coll.median(),
tmax_mask.updateMask(0))
def unique_properties(coll, property):
return ee.String(
ee.List(
ee.Dictionary(
coll.aggregate_histogram(property)).keys()).join(','))
wrs2_tile_list = ee.String('').cat(
unique_properties(tcorr_img_coll, 'wrs2_tile'))
landsat_list = ee.String('').cat(
unique_properties(tcorr_img_coll, 'spacecraft_id'))
# Cast to float and set properties
tcorr_img = ee.Image(tcorr_img).rename(['tcorr']).double() \
.set({
'system:time_start': utils.millis(export_dt),
'date_ingested': datetime.datetime.today().strftime('%Y-%m-%d'),
'date': export_dt.strftime('%Y-%m-%d'),
'year': int(export_dt.year),
'month': int(export_dt.month),
'day': int(export_dt.day),
'doy': int(export_dt.strftime('%j')),
'cycle_day': ((export_dt - cycle_base_dt).days % 8) + 1,
'landsat': landsat_list,
'model_name': model_name,
'model_version': ssebop.__version__,
'tmax_source': tmax_source.upper(),
'tmax_version': tmax_version.upper(),
'wrs2_tiles': wrs2_tile_list,
'wrs2_available': ','.join(sorted(wrs2_tiles_all)),
})
# Build export tasks
if ini['EXPORT']['export_dest'] == 'ASSET':
logging.debug(' Building export task')
task = ee.batch.Export.image.toAsset(
image=ee.Image(tcorr_img),
description=export_id,
assetId=asset_id,
crs=export_crs,
crsTransform='[' + ','.join(list(map(str, export_geo))) + ']',
dimensions='{0}x{1}'.format(*export_shape),
)
logging.info(' Starting export task')
utils.ee_task_start(task)
# Pause before starting next task
utils.delay_task(delay)
logging.debug('')
def main(ini_path=None, overwrite_flag=False, tile_i='', tile_j='', delay=0):
"""Export annual ET/ETrF/ETr/count image tiles
Parameters
----------
ini_path : str
Input file path.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
tile_i : str
Comma separated list and/or range of tile row indices.
tile_j : str
Comma separated list and/or range of tile columns indices.
delay : float, optional
Delay time between each export task (the default is 0).
Returns
-------
None
"""
logging.info('\nExport annual ET/ETrF/ETr/count image tiles')
# Read config file
ini = inputs.read(ini_path)
inputs.parse_section(ini, section='INPUTS')
inputs.parse_section(ini, section='INTERPOLATE')
inputs.parse_section(ini, section='EXPORT')
inputs.parse_section(ini, section='TILE')
inputs.parse_section(ini, section=ini['INPUTS']['et_model'])
if os.name == 'posix':
shell_flag = False
else:
shell_flag = True
if ini['EXPORT']['export_dest'] == 'ASSET':
logging.error('\nERROR: ASSET tile export is not supported')
return False
# Limit tile ranges from command line
# Eventually move to config file?
try:
tile_i_list = list(utils.parse_int_set(tile_i))
except:
tile_i_list = []
try:
tile_j_list = list(utils.parse_int_set(tile_j))
except:
tile_j_list = []
# Use study area spatial reference if not set explicitly in INI
if ini['EXPORT']['output_osr'] is None:
# Get output coordinate system from study area shapefile
study_area_ds = ogr.Open(ini['INPUTS']['study_area_path'], 0)
study_area_lyr = study_area_ds.GetLayer()
ini['EXPORT']['output_osr'] = osr.SpatialReference()
ini['EXPORT']['output_osr'] = study_area_lyr.GetSpatialRef()
ini['EXPORT']['output_crs'] = str(
ini['EXPORT']['output_osr'].ExportToWkt())
study_area_ds = None
del study_area_lyr, study_area_ds
logging.debug('\n {:16s} {}'.format('Output crs:',
ini['EXPORT']['output_crs']))
logging.debug('\nInitializing Earth Engine')
ee.Initialize()
# Get current running tasks
tasks = utils.get_ee_tasks()
# Get list of existing images/files
if ini['EXPORT']['export_dest'] == 'ASSET':
logging.debug('\nGetting GEE asset list')
asset_list = utils.get_ee_assets(ini['EXPORT']['output_ws'],
shell_flag=shell_flag)
logging.debug(asset_list)
elif ini['EXPORT']['export_dest'] == 'CLOUD':
logging.debug('\nGetting cloud storage file list')
cloud_list = utils.get_bucket_files(ini['EXPORT']['project_name'],
ini['EXPORT']['output_ws'],
shell_flag=shell_flag)
# It may be necessary to remove image tile notation
elif ini['EXPORT']['export_dest'] == 'GDRIVE':
logging.debug('\nGetting Google drive file list')
gdrive_list = [
os.path.join(ini['EXPORT']['output_ws'], x)
for x in os.listdir(ini['EXPORT']['output_ws'])
]
# It may be necessary to remove image tile notation
# Very large tiles may get split up automatically by EE
# Strip the EE tile notation data from the image list
# gdrive_list = list(set([
# re.sub('-\d{10}-\d{10}.tif', '.tif', x)
# for x in os.listdir(ini['EXPORT']['output_ws'])]))
# logging.debug(gdrive_list)
# Get list of tiles that intersect the study area
logging.debug('\nBuilding export list')
export_list = list(
tile_export_generator(ini['INPUTS']['study_area_path'],
wrs2_coll=ini['INPUTS']['wrs2_coll'],
cell_size=ini['EXPORT']['cell_size'],
output_crs=ini['EXPORT']['output_crs'],
output_osr=ini['EXPORT']['output_osr'],
wrs2_tile_list=ini['INPUTS']['wrs2_tiles'],
wrs2_tile_field=ini['INPUTS']['wrs2_tile_field'],
snap_x=ini['EXPORT']['snap_x'],
snap_y=ini['EXPORT']['snap_y'],
tile_cells=ini['TILE']['tile_cells'],
wrs2_buffer=ini['INPUTS']['wrs2_buffer']))
if not export_list:
logging.error('\nEmpty export list, exiting')
return False
# Save export list to json
with open('export_tiles.json', 'w') as json_f:
json.dump(export_list, json_f)
# Process each tile separately
logging.info('\nImage Exports')
for export_n, export_info in enumerate(export_list):
logging.info('Tile: {} ({}/{})'.format(export_info['index'],
export_n + 1,
len(export_list)))
tile_i, tile_j = map(int, export_info['index'].split('_'))
if tile_i_list and int(tile_i) not in tile_i_list:
logging.debug(' Skipping tile')
continue
elif tile_j_list and int(tile_j) not in tile_j_list:
logging.debug(' Skipping tile')
continue
# # DEADBEEF - Only test 1 block
# if [int(tile_i), int(tile_j)] != [3, 3]:
# continue
logging.debug(' Shape: {}'.format(export_info['shape']))
logging.debug(' Transform: {}'.format(export_info['geo']))
logging.debug(' Extent: {}'.format(export_info['extent']))
logging.debug(' MaxPixels: {}'.format(export_info['maxpixels']))
logging.debug(' WRS2 tiles: {}'.format(', '.join(
export_info['wrs2_tiles'])))
if ini['INPUTS']['et_model'] == 'EEFLUX':
# Get the Landsat collection
landsat_coll = landsat.get_landsat_coll(
wrs2_tile_list=export_info['wrs2_tiles'],
cloud_cover=ini['INPUTS']['cloud_cover'],
start_date=ini['INTERPOLATE']['start_date'],
end_date=ini['INTERPOLATE']['end_date'],
landsat5_flag=ini['INPUTS']['landsat5_flag'],
landsat7_flag=ini['INPUTS']['landsat7_flag'],
landsat8_flag=ini['INPUTS']['landsat8_flag'],
landsat_type='RAD')
# Compute ETf for each Landsat scene
# The 'BQA' band is also being returned by the etrf method
def apply_et_fraction(image):
etrf_obj = eeflux.EEFlux(ee.Image(image)).etrf
etrf_img = ee.Image(etrf_obj.select(['etrf'], ['etf'])) \
.clamp(-1, 2)
cloud_mask = landsat.landsat_bqa_cloud_mask_func(
ee.Image(etrf_obj.select(['BQA'])))
return etrf_img.updateMask(cloud_mask) \
.copyProperties(image, ['system:time_start'])
scene_et_fraction_coll = ee.ImageCollection(
landsat_coll.map(apply_et_fraction))
else:
logging.error('\nInvalid/unsupported ET Model: {}'.format(
ini['INPUTS']['et_model']))
return False
# Daily reference ET collection
# Is the "refet_source" a function of the model, interpolation, or other?
# The "refet_type" parameter is currently being ignored
if ini[ini['INPUTS']['et_model']]['refet_source'] == 'GRIDMET':
daily_et_reference_coll = ee.ImageCollection('IDAHO_EPSCOR/GRIDMET') \
.filterDate(ini['INPUTS']['start_date'], ini['INPUTS']['end_date']) \
.select(['etr'], ['et_reference'])
# elif ini[ini['INPUTS']['et_model']]['refet_source'] == 'CIMIS':
# daily_et_reference_coll = ee.ImageCollection('projects/climate-engine/cimis/daily') \
# .filterDate(ini['INPUTS']['start_date'],
# ini['INPUTS']['end_date']) \
# .select(['etr_asce'], ['et_reference'])
# Compute composite/mosaic images for each image date
daily_et_fraction_coll = ee.ImageCollection(
interpolate.aggregate_daily(
image_coll=scene_et_fraction_coll,
start_date=ini['INTERPOLATE']['start_date'],
end_date=ini['INTERPOLATE']['end_date']))
# Interpolate daily ETf, multiply by daily ETr, and sum to ET
daily_et_actual_coll = ee.ImageCollection(
interpolate.interp_et_coll(
et_reference_coll=daily_et_reference_coll,
et_fraction_coll=daily_et_fraction_coll,
interp_days=ini['INTERPOLATE']['interp_days'],
interp_type=ini['INTERPOLATE']['interp_type']))
# Export products
for product in ini['EXPORT']['products']:
logging.debug('\n Product: {}'.format(product))
export_id = ini['EXPORT']['export_id_fmt'].format(
model=ini['INPUTS']['et_model'].lower(),
product=product.lower(),
study_area=ini['INPUTS']['study_area_name'],
index=export_info['index'],
start=ini['INPUTS']['start_date'],
end=ini['INPUTS']['end_date'],
export=ini['EXPORT']['export_dest'].lower())
export_id = export_id.replace('-', '')
logging.debug(' Export ID: {}'.format(export_id))
if product == 'scene_id':
# Export the scene list CSV to Google Drive
if ini['EXPORT']['export_dest'] == 'GDRIVE':
export_path = os.path.join(ini['EXPORT']['output_ws'],
export_id + '.csv')
elif ini['EXPORT']['export_dest'] == 'CLOUD':
export_path = '{}/{}/{}'.format(ini['EXPORT']['output_ws'],
product,
export_id + '.csv')
elif ini['EXPORT']['export_dest'] == 'CLOUD':
# Write each product to a separate folder
export_path = '{}/{}/{}'.format(ini['EXPORT']['output_ws'],
product, export_id + '.tif')
elif ini['EXPORT']['export_dest'] == 'GDRIVE':
export_path = os.path.join(ini['EXPORT']['output_ws'],
export_id + '.tif')
else:
logging.warning(' Unsupported product type, skipping')
continue
logging.debug(' Export folder: {}'.format(
os.path.dirname(export_path)))
logging.debug(' Export file: {}'.format(
os.path.basename(export_path)))
if overwrite_flag:
if export_id in tasks.keys():
logging.debug(' Task already submitted, cancelling')
ee.data.cancelTask(tasks[export_id])
# This is intentionally not an "elif" so that a task can be
# cancelled and an existing image/file/asset can be removed
if (ini['EXPORT']['export_dest'] == 'CLOUD'
and export_path in cloud_list):
logging.debug(' Export image already exists')
# Files in cloud storage are easily overwritten
# so it is unneccesary to manually remove them
# # This would remove an existing file
# subprocess.check_output(
# ['gsutil', 'rm', export_path], shell=shell_flag)
elif (ini['EXPORT']['export_dest'] == 'GDRIVE'
and export_path in gdrive_list):
logging.debug(' Export image already exists, removing')
os.remove(export_path)
# Remove automatically generated image tiles
# for f in glob.glob(export_path.replace('.tif', '*.tif')):
# os.remove(f)
else:
if export_id in tasks.keys():
logging.debug(' Task already submitted, skipping')
continue
elif (ini['EXPORT']['export_dest'] == 'CLOUD'
and export_path in cloud_list):
logging.debug(' Export file already exists, skipping')
continue
elif (ini['EXPORT']['export_dest'] == 'GDRIVE'
and os.path.isfile(export_path)):
logging.debug(' Export file already exists, skipping')
continue
# Compute target product
if product == 'scene_id':
def scene_id_extract(image):
return ee.Feature(None).setMulti(
{'SCENE_ID': ee.String(image.get('SCENE_ID'))})
scene_id_coll = ee.FeatureCollection(
scene_et_fraction_coll.map(scene_id_extract)).sort(
'SCENE_ID')
elif product == 'et_actual':
# Sum daily ET to total ET
output_image = ee.Image(daily_et_actual_coll.sum())
elif product == 'et_reference':
# Sum daily reference ET to total reference ET
output_image = ee.Image(daily_et_reference_coll.sum())
elif product == 'et_fraction':
# Compute mean ETf (ET / ETr)
output_image = ee.Image(daily_et_actual_coll.sum()) \
.divide(ee.Image(daily_et_reference_coll.sum()))
elif product == 'count':
# Filter count date range to same period as reference ET
output_image = ee.Image(
daily_et_fraction_coll.filterDate(
ini['INPUTS']['start_dt'], ini['INPUTS']['end_dt'] +
datetime.timedelta(days=1)).count())
# elif product == 'count_monthly':
# output_image = interpolate.aggregate_monthly(
# composite_etf_coll.filterDate(
# ini['INPUTS']['start_dt'],
# ini['INPUTS']['end_dt'] + datetime.timedelta(days=1)))
# Convert data types for export to Google Drive or Cloud Storage
if (product in ['et_actual', 'et_reference', 'et_fraction']
and ini['EXPORT']['export_dest'] in ['CLOUD', 'GDRIVE']):
output_image = output_image.unmask(-9999, False).toFloat()
# elif (product in ['count', 'count_monthly'] and
elif (product in ['count']
and ini['EXPORT']['export_dest'] in ['CLOUD', 'GDRIVE']):
output_image = output_image.unmask(255, False).toUint8()
elif ini['EXPORT']['export_dest'] in ['ASSET']:
pass
# Build export tasks
if product == 'scene_id':
if ini['EXPORT']['export_dest'] == 'CLOUD':
task = ee.batch.Export.table.toCloudStorage(
scene_id_coll,
description=export_id,
bucket=ini['EXPORT']['bucket_name'],
fileNamePrefix='{}/{}/{}'.format(
ini['EXPORT']['bucket_folder'], product,
export_id),
fileFormat='CSV')
elif ini['EXPORT']['export_dest'] == 'GDRIVE':
# Export the scene list CSV to Google Drive
task = ee.batch.Export.table.toDrive(
scene_id_coll,
description=export_id,
folder=os.path.basename(ini['EXPORT']['output_ws']),
fileNamePrefix=export_id,
fileFormat='CSV')
if ini['EXPORT']['export_dest'] == 'asset':
# Export the image to cloud storage
task = ee.batch.Export.image.toAsset(
output_image,
description=export_id,
bucket=ini['EXPORT']['bucket_name'],
fileNamePrefix='{}/{}/{}'.format(
ini['EXPORT']['bucket_folder'], product, export_id),
dimensions=export_info['shape'],
crs=export_info['crs'],
crsTransform=export_info['geo'],
maxPixels=export_info['maxpixels'])
elif ini['EXPORT']['export_dest'] == 'CLOUD':
# Export the image to cloud storage
task = ee.batch.Export.image.toCloudStorage(
output_image,
description=export_id,
bucket=ini['EXPORT']['bucket_name'],
fileNamePrefix='{}/{}/{}'.format(
ini['EXPORT']['bucket_folder'], product, export_id),
dimensions=export_info['shape'],
crs=export_info['crs'],
crsTransform=export_info['geo'],
# shardSize=,
# fileDimensions=,
maxPixels=export_info['maxpixels'])
elif ini['EXPORT']['export_dest'] == 'GDRIVE':
# Export the images to your Google Drive
task = ee.batch.Export.image.toDrive(
output_image,
description=export_id,
folder=os.path.basename(ini['EXPORT']['output_ws']),
fileNamePrefix=export_id,
dimensions=export_info['shape'],
crs=export_info['crs'],
crsTransform=export_info['geo'],
maxPixels=export_info['maxpixels'])
else:
logging.debug(' Export task not built, skipping')
continue
# Try to start the export task a few times
logging.debug(' Starting export task')
for i in range(1, 10):
try:
task.start()
break
except Exception as e:
logging.error(' Error: {}\n Retrying ({}/10)'.format(
e, i))
time.sleep(i**2)
i += 1
# logging.debug(' Active: {}'.format(task.active()))
# logging.debug(' Status: {}'.format(task.status()))
if delay and delay > 0:
time.sleep(delay)
elif delay and delay == -1:
input('ENTER')