def main(ini_path=None, overwrite_flag=False): """Export annual ET/ETrF/ETr/count WRS2 tile images Parameters ---------- ini_path : str Input file path. overwrite_flag : bool, optional If True, overwrite existing files (the default is False) Returns ------- None """ logging.info('\nExport annual ET/ETrF/ETr/count WRS2 tile images') # Read config file ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='INTERPOLATE') inputs.parse_section(ini, section='EXPORT') inputs.parse_section(ini, section=ini['INPUTS']['et_model']) if ini['EXPORT']['export_dest'] == 'ASSET': logging.error('\nERROR: ASSET tile export is not supported') return False # The study area cannot be used to set the output spatial reference # when exporting by WRS2 tile # if ini['EXPORT']['output_osr'] is None: # # Get output coordinate system from study area shapefile # study_area_ds = ogr.Open(ini['INPUTS']['study_area_path'], 0) # study_area_lyr = study_area_ds.GetLayer() # ini['EXPORT']['output_osr'] = osr.SpatialReference() # ini['EXPORT']['output_osr'] = study_area_lyr.GetSpatialRef() # ini['EXPORT']['output_crs'] = str( # ini['EXPORT']['output_osr'].ExportToWkt()) # study_area_ds = None # del study_area_lyr, study_area_ds # logging.debug('\n {:16s} {}'.format( # 'Output crs:', ini['EXPORT']['output_crs'])) logging.debug('\nInitializing Earth Engine') ee.Initialize() # Get current running tasks tasks = utils.get_ee_tasks() # Get list of existing images/files if ini['EXPORT']['export_dest'] == 'CLOUD': logging.debug('\nGetting cloud storage file list') cloud_list = utils.get_bucket_files( ini['EXPORT']['project_name'], ini['EXPORT']['output_ws']) # It may be necessary to remove image tile notation elif ini['EXPORT']['export_dest'] == 'GDRIVE': logging.debug('\nGetting Google drive file list') gdrive_list = [ os.path.join(ini['EXPORT']['output_ws'], x) for x in os.listdir(ini['EXPORT']['output_ws'])] # It may be necessary to remove image tile notation # Very large tiles may get split up automatically by EE # Strip the EE tile notation data from the image list # gdrive_list = list(set([ # re.sub('-\d{10}-\d{10}.tif', '.tif', x) # for x in os.listdir(ini['EXPORT']['output_ws'])])) # logging.debug(gdrive_list) # Get list of WRS2 tiles that intersect the study area logging.debug('\nBuilding export list') export_list = list(wrs2_tile_export_generator( ini['INPUTS']['study_area_path'], wrs2_coll=ini['INPUTS']['wrs2_coll'], cell_size=ini['EXPORT']['cell_size'], output_crs=ini['EXPORT']['output_crs'], output_osr=ini['EXPORT']['output_osr'], wrs2_tile_list=ini['INPUTS']['wrs2_tiles'], wrs2_tile_field=ini['INPUTS']['wrs2_tile_field'], snap_x=ini['EXPORT']['snap_x'], snap_y=ini['EXPORT']['snap_y'], wrs2_buffer=ini['INPUTS']['wrs2_buffer'])) if not export_list: logging.error('\nEmpty export list, exiting') return False # Save export list to json with open('export_wrs2_tile.json', 'w') as json_f: json.dump(export_list, json_f) # Process each WRS2 tile separately logging.info('\nImage Exports') for export_n, export_info in enumerate(export_list): # path, row = map(int, wrs2_tile_re.findall(export_info['index'])[0]) logging.info('WRS2 tile: {} ({}/{})'.format( export_info['index'], export_n + 1, len(export_list))) logging.debug(' Shape: {}'.format(export_info['shape'])) logging.debug(' Transform: {}'.format(export_info['geo'])) logging.debug(' Extent: {}'.format(export_info['extent'])) logging.debug(' MaxPixels: {}'.format(export_info['maxpixels'])) if ini['INPUTS']['et_model'] == 'EEFLUX': # Get the Landsat collection landsat_coll = landsat.get_landsat_coll( wrs2_tile_list=export_info['wrs2_tiles'], cloud_cover=ini['INPUTS']['cloud_cover'], start_date=ini['INTERPOLATE']['start_date'], end_date=ini['INTERPOLATE']['end_date'], landsat5_flag=ini['INPUTS']['landsat5_flag'], landsat7_flag=ini['INPUTS']['landsat7_flag'], landsat8_flag=ini['INPUTS']['landsat8_flag'], landsat_type='RAD') # Compute ETf for each Landsat scene # The 'BQA' band is also being returned by the etrf method def apply_et_fraction(image): etrf_obj = eeflux.EEFlux(ee.Image(image)).etrf etrf_img = ee.Image(etrf_obj.select(['etrf'], ['etf'])) \ .clamp(-1, 2) cloud_mask = landsat.landsat_bqa_cloud_mask_func( ee.Image(etrf_obj. select(['BQA']))) return etrf_img.updateMask(cloud_mask) \ .copyProperties(image, ['system:time_start']) scene_et_fraction_coll = ee.ImageCollection( landsat_coll.map(apply_et_fraction)) else: logging.error('\nInvalid/unsupported ET Model: {}'.format( ini['INPUTS']['et_model'])) return False # Daily reference ET collection # DEADBEEF - Hard coding to GRIDMET for now # Should this be retrieved from the model? daily_et_reference_coll = ee.ImageCollection('IDAHO_EPSCOR/GRIDMET') \ .filterDate(ini['INPUTS']['start_date'], ini['INPUTS']['end_date']) \ .select(['etr'], ['et_reference']) # Compute composite/mosaic images for each image date daily_et_fraction_coll = ee.ImageCollection(interpolate.aggregate_daily( image_coll=scene_et_fraction_coll, start_date=ini['INTERPOLATE']['start_date'], end_date=ini['INTERPOLATE']['end_date'])) # Interpolate daily ETf, multiply by daily ETr, and sum to ET daily_et_actual_coll = ee.ImageCollection(interpolate.interp_et_coll( et_reference_coll=daily_et_reference_coll, et_fraction_coll=daily_et_fraction_coll, interp_days=ini['INTERPOLATE']['interp_days'], interp_type=ini['INTERPOLATE']['interp_type'])) # Export products for product in ini['EXPORT']['products']: logging.debug('\n Product: {}'.format(product)) export_id = ini['EXPORT']['export_id_fmt'].format( model=ini['INPUTS']['et_model'].lower(), product=product.lower(), study_area=ini['INPUTS']['study_area_name'], index=export_info['index'], start=ini['INPUTS']['start_date'], end=ini['INPUTS']['end_date'], export=ini['EXPORT']['export_dest'].lower()) export_id = export_id.replace('-', '') logging.debug(' Export ID: {}'.format(export_id)) if product == 'scene_id': # Export the scene list CSV to Google Drive if ini['EXPORT']['export_dest'] == 'GDRIVE': export_path = os.path.join( ini['EXPORT']['output_ws'], export_id + '.csv') elif ini['EXPORT']['export_dest'] == 'CLOUD': export_path = '{}/{}/{}'.format( ini['EXPORT']['output_ws'], product, export_id + '.csv') elif ini['EXPORT']['export_dest'] == 'CLOUD': # Write each product to a separate folder export_path = '{}/{}/{}'.format( ini['EXPORT']['output_ws'], product, export_id + '.tif') elif ini['EXPORT']['export_dest'] == 'GDRIVE': export_path = os.path.join( ini['EXPORT']['output_ws'], export_id + '.tif') logging.debug(' Export folder: {}'.format( os.path.dirname(export_path))) logging.debug(' Export file: {}'.format( os.path.basename(export_path))) if overwrite_flag: if export_id in tasks.keys(): logging.debug(' Task already submitted, cancelling') ee.data.cancelTask(tasks[export_id]) # This is intentionally not an "elif" so that a task can be # cancelled and an existing image/file/asset can be removed if (ini['EXPORT']['export_dest'] == 'CLOUD' and export_path in cloud_list): logging.debug(' Export image already exists') # Files in cloud storage are easily overwritten # so it is unneccesary to manually remove them # # This would remove an existing file # subprocess.call(['gsutil', 'rm', export_path]) elif (ini['EXPORT']['export_dest'] == 'GDRIVE' and export_path in gdrive_list): logging.debug(' Export image already exists, removing') os.remove(export_path) # Remove automatically generated image tiles # for f in glob.glob(export_path.replace('.tif', '*.tif')): # os.remove(f) else: if export_id in tasks.keys(): logging.debug(' Task already submitted, skipping') continue elif (ini['EXPORT']['export_dest'] == 'CLOUD' and export_path in cloud_list): logging.debug(' Export file already exists, skipping') continue elif (ini['EXPORT']['export_dest'] == 'GDRIVE' and os.path.isfile(export_path)): logging.debug(' Export file already exists, skipping') continue # Compute target product if product == 'scene_id': def scene_id_extract(image): return ee.Feature(None).setMulti({ 'SCENE_ID': ee.String(image.get('SCENE_ID'))}) scene_id_coll = ee.FeatureCollection( scene_et_fraction_coll.map(scene_id_extract)).sort('SCENE_ID') elif product == 'et_actual': # Sum daily ET to total ET output_image = ee.Image(daily_et_actual_coll.sum()) elif product == 'et_reference': # Sum daily reference ET to total reference ET output_image = ee.Image(daily_et_reference_coll.sum()) elif product == 'et_fraction': # Compute mean ETf (ET / ETr) output_image = ee.Image(daily_et_actual_coll.sum()) \ .divide(ee.Image(daily_et_reference_coll.sum())) elif product == 'count': # Filter count date range to same period as reference ET output_image = ee.Image(daily_et_fraction_coll.filterDate( ini['INPUTS']['start_dt'], ini['INPUTS']['end_dt'] + datetime.timedelta(days=1)).count()) # elif product == 'count_monthly': # output_image = interpolate.aggregate_monthly( # composite_etf_coll.filterDate( # ini['INPUTS']['start_dt'], # ini['INPUTS']['end_dt'] + datetime.timedelta(days=1))) else: logging.warning(' Unsupported product type, skipping') continue # Convert data types for export to Google Drive or Cloud Storage if (product in ['et_actual', 'et_reference', 'et_fraction'] and ini['EXPORT']['export_dest'] in ['CLOUD', 'GDRIVE']): output_image = output_image.unmask(-9999, False).toFloat() # elif (product in ['count', 'count_monthly'] and elif (product in ['count'] and ini['EXPORT']['export_dest'] in ['CLOUD', 'GDRIVE']): output_image = output_image.unmask(255, False).toUint8() elif ini['EXPORT']['export_dest'] in ['ASSET']: pass # Build export tasks if product == 'scene_id': if ini['EXPORT']['export_dest'] == 'CLOUD': task = ee.batch.Export.table.toCloudStorage( collection=scene_id_coll, description=export_id, bucket=ini['EXPORT']['bucket_name'], fileNamePrefix='{}/{}'.format(product, export_id), fileFormat='CSV') elif ini['EXPORT']['export_dest'] == 'GDRIVE': # Export the scene list CSV to Google Drive task = ee.batch.Export.table.toDrive( collection=scene_id_coll, description=export_id, folder=os.path.basename(ini['EXPORT']['output_ws']), fileNamePrefix=export_id, fileFormat='CSV') elif ini['EXPORT']['export_dest'] == 'CLOUD': # Export the image to cloud storage task = ee.batch.Export.image.toCloudStorage( image=output_image, description=export_id, bucket=ini['EXPORT']['bucket_name'], fileNamePrefix='{}/{}'.format(product, export_id), dimensions=export_info['shape'], crs=export_info['crs'], crsTransform=export_info['geo'], # shardSize=, # fileDimensions=, maxPixels=export_info['maxpixels']) elif ini['EXPORT']['export_dest'] == 'GDRIVE': # Export the images to your Google Drive task = ee.batch.Export.image.toDrive( image=output_image, description=export_id, folder=os.path.basename(ini['EXPORT']['output_ws']), fileNamePrefix=export_id, dimensions=export_info['shape'], crs=export_info['crs'], crsTransform=export_info['geo'], maxPixels=export_info['maxpixels']) else: logging.debug(' Export task not built, skipping') continue # Try to start the export task a few times logging.debug(' Starting export task') for i in range(1, 10): try: task.start() break except Exception as e: logging.error( ' Error: {}\n Retrying ({}/10)'.format(e, i)) time.sleep(i ** 2) i += 1
def main(ini_path=None, overwrite_flag=False, tile_cols='', tile_rows='', delay=0): """Export annual ET/ETrF/ETr/count image ARG grid tiles Parameters ---------- ini_path : str Input file path. overwrite_flag : bool, optional If True, overwrite existing files (the default is False). tile_cols : str Comma separated list and/or range of ARD tile columns indices. tile_rows : str Comma separated list and/or range of ARD tile row indices. delay : float, optional Delay time between each export task (the default is 0). Returns ------- None """ logging.info('\nExport annual ET/ETrF/ETr/count image tiles') # Read config file ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='INTERPOLATE') inputs.parse_section(ini, section='EXPORT') inputs.parse_section(ini, section=ini['INPUTS']['et_model']) if os.name == 'posix': shell_flag = False else: shell_flag = True # Limit tile ranges from command line # Eventually move to config file? try: tile_cols_list = list(utils.parse_int_set(tile_cols)) except: tile_cols_list = [] try: tile_rows_list = list(utils.parse_int_set(tile_rows)) except: tile_rows_list = [] logging.debug('\nInitializing Earth Engine') ee.Initialize() # Get current running tasks tasks = utils.get_ee_tasks() # Get list of existing images/files if ini['EXPORT']['export_dest'] == 'ASSET': logging.debug('\nGetting GEE asset list') asset_list = utils.get_ee_assets( ini['EXPORT']['output_ws'], shell_flag=shell_flag) logging.debug(asset_list) # elif ini['EXPORT']['export_dest'] == 'CLOUD': # logging.debug('\nGetting cloud storage file list') # cloud_list = utils.get_bucket_files( # ini['EXPORT']['project_name'], ini['EXPORT']['output_ws'], # shell_flag=shell_flag) # # It may be necessary to remove image tile notation # elif ini['EXPORT']['export_dest'] == 'GDRIVE': # logging.debug('\nGetting Google drive file list') # gdrive_list = [ # os.path.join(ini['EXPORT']['output_ws'], x) # for x in os.listdir(ini['EXPORT']['output_ws'])] # # It may be necessary to remove image tile notation # # Very large tiles may get split up automatically by EE # # Strip the EE tile notation data from the image list # # gdrive_list = list(set([ # # re.sub('-\d{10}-\d{10}.tif', '.tif', x) # # for x in os.listdir(ini['EXPORT']['output_ws'])])) # # logging.debug(gdrive_list) # Get list of tiles that intersect the study area logging.debug('\nBuilding export list') export_list = list(ard_tile_export_generator( ini['INPUTS']['study_area_path'], wrs2_coll=ini['INPUTS']['wrs2_coll'], cell_size=ini['EXPORT']['cell_size'], wrs2_tile_list=ini['INPUTS']['wrs2_tiles'], wrs2_tile_field=ini['INPUTS']['wrs2_tile_field'], wrs2_buffer=ini['INPUTS']['wrs2_buffer'])) if not export_list: logging.error('\nEmpty export list, exiting') return False # Save export list to json with open('export_tiles.json', 'w') as json_f: json.dump(export_list, json_f) # Process each tile separately logging.info('\nImage Exports') for export_n, export_info in enumerate(export_list): tile_col = int(export_info['index'][1:4]) tile_row = int(export_info['index'][5:8]) if tile_cols_list and int(tile_col) not in tile_cols_list: logging.debug('ARD Tile: {} ({}/{}), skipping'.format( export_info['index'], export_n + 1, len(export_list))) continue elif tile_rows_list and int(tile_row) not in tile_rows_list: logging.debug('ARD Tile: {} ({}/{}), skipping'.format( export_info['index'], export_n + 1, len(export_list))) continue else: logging.info('ARD Tile: {} ({}/{})'.format( export_info['index'], export_n + 1, len(export_list))) logging.debug(' Shape: {}'.format(export_info['shape'])) logging.debug(' Transform: {}'.format(export_info['geo'])) logging.debug(' Extent: {}'.format(export_info['extent'])) logging.debug(' MaxPixels: {}'.format(export_info['maxpixels'])) logging.debug(' WRS2 tiles: {}'.format( ', '.join(export_info['wrs2_tiles']))) if ini['INPUTS']['et_model'] == 'EEFLUX': # Get the Landsat collection landsat_coll = landsat.get_landsat_coll( wrs2_tile_list=export_info['wrs2_tiles'], cloud_cover=ini['INPUTS']['cloud_cover'], start_date=ini['INTERPOLATE']['start_date'], end_date=ini['INTERPOLATE']['end_date'], landsat5_flag=ini['INPUTS']['landsat5_flag'], landsat7_flag=ini['INPUTS']['landsat7_flag'], landsat8_flag=ini['INPUTS']['landsat8_flag'], landsat_type='RAD') # Compute ETf for each Landsat scene # The 'BQA' band is also being returned by the etrf method def apply_et_fraction(image): etrf_obj = eeflux.EEFlux(ee.Image(image)).etrf etrf_img = ee.Image(etrf_obj.select(['etrf'], ['etf'])) \ .clamp(-1, 2) cloud_mask = landsat.landsat_bqa_cloud_mask_func( ee.Image(etrf_obj. select(['BQA']))) return etrf_img.updateMask(cloud_mask) \ .copyProperties(image, ['system:time_start']) scene_et_fraction_coll = ee.ImageCollection( landsat_coll.map(apply_et_fraction)) else: logging.error('\nInvalid/unsupported ET Model: {}'.format( ini['INPUTS']['et_model'])) return False # Daily reference ET collection # Is the "refet_source" a function of the model, interpolation, or other? # The "refet_type" parameter is currently being ignored if ini[ini['INPUTS']['et_model']]['refet_source'] == 'GRIDMET': daily_et_reference_coll = ee.ImageCollection('IDAHO_EPSCOR/GRIDMET') \ .filterDate(ini['INPUTS']['start_date'], ini['INPUTS']['end_date']) \ .select(['etr'], ['et_reference']) elif ini[ini['INPUTS']['et_model']]['refet_source'] == 'CIMIS': daily_et_reference_coll = ee.ImageCollection('projects/climate-engine/cimis/daily') \ .filterDate(ini['INPUTS']['start_date'], ini['INPUTS']['end_date']) \ .select(['etr_asce'], ['et_reference']) # Compute composite/mosaic images for each image date daily_et_fraction_coll = ee.ImageCollection(interpolate.aggregate_daily( image_coll=scene_et_fraction_coll, start_date=ini['INTERPOLATE']['start_date'], end_date=ini['INTERPOLATE']['end_date'])) # Interpolate daily ETf, multiply by daily ETr, and sum to ET daily_et_actual_coll = ee.ImageCollection(interpolate.interp_et_coll( et_reference_coll=daily_et_reference_coll, et_fraction_coll=daily_et_fraction_coll, interp_days=ini['INTERPOLATE']['interp_days'], interp_type=ini['INTERPOLATE']['interp_type'])) # Export products # for product in ini['EXPORT']['products']: # logging.debug('\n Product: {}'.format(product)) export_id = ini['EXPORT']['export_id_fmt'].format( model=ini['INPUTS']['et_model'].lower(), # product=product.lower(), study_area=ini['INPUTS']['study_area_name'], index=export_info['index'], start=ini['INPUTS']['start_date'], end=ini['INPUTS']['end_date'], export=ini['EXPORT']['export_dest'].lower()) export_id = export_id.replace('-', '') logging.debug(' Export ID: {}'.format(export_id)) # if product == 'scene_id': # # Export the scene list CSV to Google Drive # if ini['EXPORT']['export_dest'] == 'GDRIVE': # export_path = os.path.join( # ini['EXPORT']['output_ws'], export_id + '.csv') # elif ini['EXPORT']['export_dest'] == 'CLOUD': # export_path = '{}/{}/{}'.format( # ini['EXPORT']['output_ws'], product, export_id + '.csv') # if ini['EXPORT']['export_dest'] == 'CLOUD': # # Write each product to a separate folder # export_path = '{}/{}/{}'.format( # ini['EXPORT']['output_ws'], product, export_id + '.tif') # elif ini['EXPORT']['export_dest'] == 'GDRIVE': # export_path = os.path.join( # ini['EXPORT']['output_ws'], export_id + '.tif') if ini['EXPORT']['export_dest'] == 'ASSET': # Write each product to a separate folder export_path = '{}/{}'.format( ini['EXPORT']['output_ws'], export_id) else: logging.warning(' Unsupported product type, skipping') continue logging.debug(' Export folder: {}'.format( os.path.dirname(export_path))) logging.debug(' Export file: {}'.format( os.path.basename(export_path))) if overwrite_flag: if export_id in tasks.keys(): logging.debug(' Task already submitted, cancelling') ee.data.cancelTask(tasks[export_id]) # This is intentionally not an "elif" so that a task can be # cancelled and an existing image/file/asset can be removed if (ini['EXPORT']['export_dest'] == 'ASSET' and export_path in asset_list): logging.debug(' Asset already exists') subprocess.check_output( ['earthengine', 'rm', export_path], shell=shell_flag) # Files in cloud storage are easily overwritten # so it is unneccesary to manually remove them # # This would remove an existing file # subprocess.call(['gsutil', 'rm', export_path]) # if (ini['EXPORT']['export_dest'] == 'CLOUD' and # export_path in cloud_list): # logging.debug(' Export image already exists') # # Files in cloud storage are easily overwritten # # so it is unneccesary to manually remove them # # # This would remove an existing file # # subprocess.check_output(['gsutil', 'rm', export_path]) # elif (ini['EXPORT']['export_dest'] == 'GDRIVE' and # export_path in gdrive_list): # logging.debug(' Export image already exists, removing') # os.remove(export_path) # # Remove automatically generated image tiles # # for f in glob.glob(export_path.replace('.tif', '*.tif')): # # os.remove(f) else: if export_id in tasks.keys(): logging.debug(' Task already submitted, skipping') continue if (ini['EXPORT']['export_dest'] == 'ASSET' and export_path in asset_list): logging.debug(' Asset already exists, skipping') continue # elif (ini['EXPORT']['export_dest'] == 'CLOUD' and # export_path in cloud_list): # logging.debug(' Export file already exists, skipping') # continue # elif (ini['EXPORT']['export_dest'] == 'GDRIVE' and # os.path.isfile(export_path)): # logging.debug(' Export file already exists, skipping') # continue # Compute target product # if product == 'scene_id': # def scene_id_extract(image): # return ee.Feature(None).setMulti({ # 'SCENE_ID': ee.String(image.get('SCENE_ID'))}) # scene_id_coll = ee.FeatureCollection( # scene_et_fraction_coll.map(scene_id_extract)).sort('SCENE_ID') output_images = [] for product_i, product in enumerate(ini['EXPORT']['products']): logging.debug(' Product: {}'.format(product)) if product == 'et_actual': # Sum daily ET to total ET output_images.append( ee.Image(daily_et_actual_coll.sum()).toFloat()) elif product == 'et_reference': # Sum daily reference ET to total reference ET output_images.append( ee.Image(daily_et_reference_coll.sum()).toFloat()) elif product == 'et_fraction': # Compute mean ETf (ET / ETr) output_images.append( ee.Image(daily_et_actual_coll.sum()) \ .divide(ee.Image(daily_et_reference_coll.sum())).toFloat()) elif product == 'count': # Filter count date range to same period as reference ET output_images.append(ee.Image( daily_et_fraction_coll.filterDate( ini['INPUTS']['start_dt'], ini['INPUTS']['end_dt'] + datetime.timedelta(days=1)).count())\ .toUint8()) # DEADEEF - Consider saving other input parameters # CLOUD_COVER_LAND, number of interpolation days, ? output_image = ee.Image(ee.Image(output_images) \ .rename(ini['EXPORT']['products']) \ .setMulti({ 'system:time_start': ini['INPUTS']['start_date'], 'index': export_info['index']})) # print(output_image.get('system:time_start').getInfo()) # input('ENTER') # Build export tasks # if product == 'scene_id': # if ini['EXPORT']['export_dest'] == 'CLOUD': # task = ee.batch.Export.table.toCloudStorage( # scene_id_coll, # description=export_id, # bucket=ini['EXPORT']['bucket_name'], # fileNamePrefix='{}/{}/{}'.format( # ini['EXPORT']['bucket_folder'], product, export_id), # fileFormat='CSV') # elif ini['EXPORT']['export_dest'] == 'GDRIVE': # # Export the scene list CSV to Google Drive # task = ee.batch.Export.table.toDrive( # scene_id_coll, # description=export_id, # folder=os.path.basename(ini['EXPORT']['output_ws']), # fileNamePrefix=export_id, # fileFormat='CSV') # elif ini['EXPORT']['export_dest'] == 'CLOUD': # # Export the image to cloud storage # task = ee.batch.Export.image.toCloudStorage( # output_image, # description=export_id, # bucket=ini['EXPORT']['bucket_name'], # fileNamePrefix='{}/{}/{}'.format( # ini['EXPORT']['bucket_folder'], product, export_id), # dimensions=export_info['shape'], # crs=export_info['crs'], # crsTransform=export_info['geo'], # # shardSize=, # # fileDimensions=, # maxPixels=export_info['maxpixels']) # elif ini['EXPORT']['export_dest'] == 'GDRIVE': # # Export the images to your Google Drive # task = ee.batch.Export.image.toDrive( # output_image, # description=export_id, # folder=os.path.basename(ini['EXPORT']['output_ws']), # fileNamePrefix=export_id, # dimensions=export_info['shape'], # crs=export_info['crs'], # crsTransform=export_info['geo'], # maxPixels=export_info['maxpixels']) if ini['EXPORT']['export_dest'] == 'ASSET': # Export the image to cloud storage task = ee.batch.Export.image.toAsset( output_image, description=export_id, assetId='{}/{}'.format(ini['EXPORT']['output_ws'], export_id), # pyramidingPolicy='mean', dimensions=export_info['shape'], crs=export_info['crs'], crsTransform=export_info['geo'], maxPixels=export_info['maxpixels']) else: logging.debug(' Export task not built, skipping') # continue # Try to start the export task a few times logging.debug(' Starting export task') for i in range(1, 10): try: task.start() break except Exception as e: logging.error( ' Error: {}\n Retrying ({}/10)'.format(e, i)) time.sleep(i ** 2) i += 1 # logging.debug(' Active: {}'.format(task.active())) # logging.debug(' Status: {}'.format(task.status())) if delay and delay > 0: time.sleep(delay) elif delay and delay == -1: input('ENTER')
def main(ini_path=None, overwrite_flag=False, tile_i='', tile_j=''): """Export annual ET/ETrF/ETr/count image tiles Parameters ---------- ini_path : str Input file path. overwrite_flag : bool, optional If True, overwrite existing files (the default is False). tile_i : str Comma separated list and/or range of tile row indices. tile_j : str Comma separated list and/or range of tile columns indices. Returns ------- None """ logging.info('\nGenerate tile shapefile') # Read config file ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='EXPORT') output_path = ini['INPUTS']['study_area_path'].replace( '.shp', '_tiles.shp') if os.path.isfile(output_path) and not overwrite_flag: logging.info( '\nOutput shapefile already exists and overwrite_flag is False\n') return False # Limit tile ranges from command line # Eventually move to config file? try: tile_i_list = list(utils.parse_int_set(tile_i)) except: tile_i_list = [] try: tile_j_list = list(utils.parse_int_set(tile_j)) except: tile_j_list = [] # Use study area spatial reference if not set explicitly in INI if ini['EXPORT']['output_osr'] is None: # Get output coordinate system from study area shapefile study_area_ds = ogr.Open(ini['INPUTS']['study_area_path'], 0) study_area_lyr = study_area_ds.GetLayer() ini['EXPORT']['output_osr'] = osr.SpatialReference() ini['EXPORT']['output_osr'] = study_area_lyr.GetSpatialRef() ini['EXPORT']['output_crs'] = str( ini['EXPORT']['output_osr'].ExportToWkt()) study_area_ds = None del study_area_lyr, study_area_ds logging.debug('\n {:16s} {}'.format('Output crs:', ini['EXPORT']['output_crs'])) # Get list of tiles that intersect the study area logging.debug('\nBuilding export list') export_list = list( tile_export_generator(ini['INPUTS']['study_area_path'], cell_size=ini['EXPORT']['cell_size'], output_osr=ini['EXPORT']['output_osr'], snap_x=ini['EXPORT']['snap_x'], snap_y=ini['EXPORT']['snap_y'], tile_cells=ini['TILE']['tile_cells'])) if not export_list: logging.error('\nEmpty export list, exiting') return False # Build the output shapefile # Write the scene Tcorr values to the shapefile logging.info('\nWriting tiles to the shapefile') logging.debug(' {}'.format(output_path)) shp_driver = ogr.GetDriverByName("ESRI Shapefile") output_ds = shp_driver.CreateDataSource(output_path) output_lyr = output_ds.CreateLayer(output_path, ini['EXPORT']['output_osr'], ogr.wkbPolygon) field_name = ogr.FieldDefn('COL', ogr.OFTInteger) field_name.SetWidth(3) output_lyr.CreateField(field_name) field_name = ogr.FieldDefn('ROW', ogr.OFTInteger) field_name.SetWidth(3) output_lyr.CreateField(field_name) # Write each tile separately for export_n, export_info in enumerate(export_list): logging.info('Tile: {} ({}/{})'.format(export_info['index'], export_n + 1, len(export_list))) logging.debug(' Extent: {}'.format(export_info['extent'])) tile_i, tile_j = map(int, export_info['index'].split('_')) if tile_i_list and int(tile_i) not in tile_i_list: logging.debug(' Skipping tile') continue elif tile_j_list and int(tile_j) not in tile_j_list: logging.debug(' Skipping tile') continue feature = ogr.Feature(output_lyr.GetLayerDefn()) feature.SetField('COL', tile_j) feature.SetField('ROW', tile_i) polygon = ogr.CreateGeometryFromWkt( "POLYGON(({0} {1}, {2} {1}, {2} {3}, {0} {3}, {0} {1}))".format( *export_info['extent'])) feature.SetGeometry(polygon) output_lyr.CreateFeature(feature) feature = None output_ds = None
def main(ini_path=None, overwrite_flag=False): """Export daily ET/ETrF/ETr/count images as EE assets or to Google Drive Parameters ---------- ini_path : str Input file path. overwrite_flag : bool, optional If True, overwrite existing files (the default is False) Returns ------- None """ logging.info('\nComputing daily ET/ETrF/ETr/count images') # Read config file ini = inputs.read(ini_path) inputs.parse_section(ini, section='INPUTS') inputs.parse_section(ini, section='EXPORT') inputs.parse_section(ini, section=ini['INPUTS']['et_model']) # # Use study area spatial reference if not set explicitly in INI # if ini['EXPORT']['output_osr'] is None: # # Get output coordinate system from study area shapefile # study_area_ds = ogr.Open(ini['INPUTS']['study_area_path'], 0) # study_area_lyr = study_area_ds.GetLayer() # ini['EXPORT']['output_osr'] = osr.SpatialReference() # ini['EXPORT']['output_osr'] = study_area_lyr.GetSpatialRef() # ini['EXPORT']['output_crs'] = str( # ini['EXPORT']['output_osr'].ExportToWkt()) # study_area_ds = None # del study_area_lyr, study_area_ds # logging.debug('\n {:16s} {}'.format( # 'Output crs:', ini['EXPORT']['output_crs'])) logging.info('\nInitializing Earth Engine') ee.Initialize() # Get current running tasks tasks = utils.get_ee_tasks() # Get list of existing images/files # if ini['EXPORT']['export_dest'] == 'ASSET': # logging.debug('\nGetting EE asset list') # try: # asset_list = subprocess.check_output( # ['earthengine', 'ls', ini['EXPORT']['output_ws']], # universal_newlines=True) # asset_list = [x.strip() for x in asset_list.split('\n') if x] # # logging.debug(asset_list) # except ValueError as e: # logging.info(' Collection doesn\'t exist') # logging.debug(' {}'.format(str(e))) # asset_list = [] # except Exception as e: # logging.error('\n Unknown error, returning False') # logging.error(e) # return False if ini['EXPORT']['export_dest'] == 'CLOUD': logging.debug('\nGetting cloud storage file list') cloud_list = utils.get_bucket_files(ini['EXPORT']['project_name'], ini['EXPORT']['output_ws']) # It may be necessary to remove image tile notation elif ini['EXPORT']['export_dest'] == 'GDRIVE': logging.debug('\nGetting Google drive file list') gdrive_list = [ os.path.join(ini['EXPORT']['output_ws'], x) for x in os.listdir(ini['EXPORT']['output_ws']) ] # Very large tiles may get split up automatically by EE # Strip the EE tile notation data from the image list gdrive_list = list( set([re.sub('-\d{10}-\d{10}.tif', '.tif', x) for x in gdrive_list])) # logging.debug(gdrive_list) # Remove scene_id product if 'scene_id' in ini['EXPORT']['products']: ini['EXPORT']['products'].remove('scene_id') # Change "count" product to "count_mask" for single image exports # This block would need to be removed if a separate mask product was added ini['EXPORT']['products'] = [ p if p != 'count' else 'count_mask' for p in ini['EXPORT']['products'] ] # Get list of WRS2 tiles that intersect the study area logging.debug('\nBuilding export list') export_list = list( wrs2_tile_export_generator( ini['INPUTS']['study_area_path'], wrs2_coll=ini['INPUTS']['wrs2_coll'], cell_size=ini['EXPORT']['cell_size'], output_crs=ini['EXPORT']['output_crs'], output_osr=ini['EXPORT']['output_osr'], wrs2_tile_list=ini['INPUTS']['wrs2_tiles'], wrs2_tile_field=ini['INPUTS']['wrs2_tile_field'], snap_x=ini['EXPORT']['snap_x'], snap_y=ini['EXPORT']['snap_y'], wrs2_buffer=ini['INPUTS']['wrs2_buffer'])) if not export_list: logging.error('\nEmpty export list, exiting') return False # # Save export list to json # with open('export_wrs2_tile.json', 'w') as json_f: # json.dump(export_list, json_f) # Process each WRS2 tile separately logging.info('\nImage Exports') for export_n, export_info in enumerate(export_list): # path, row = map(int, path_row_re.findall(export_info['index'])[0]) logging.info('WRS2 tile: {} ({}/{})'.format(export_info['index'], export_n + 1, len(export_list))) logging.debug(' Shape: {}'.format(export_info['shape'])) logging.debug(' Transform: {}'.format(export_info['geo'])) logging.debug(' Extent: {}'.format(export_info['extent'])) logging.debug(' MaxPixels: {}'.format(export_info['maxpixels'])) # Get the full Landsat collection logging.debug(' Getting image IDs from EarthEngine') landsat_coll = landsat.get_landsat_coll( wrs2_tile_list=export_info['wrs2_tiles'], cloud_cover=ini['INPUTS']['cloud_cover'], start_date=ini['INPUTS']['start_date'], end_date=ini['INPUTS']['end_date'], landsat5_flag=ini['INPUTS']['landsat5_flag'], landsat7_flag=ini['INPUTS']['landsat7_flag'], landsat8_flag=ini['INPUTS']['landsat8_flag'], ) scene_id_list = landsat_coll.aggregate_histogram('SCENE_ID')\ .getInfo().keys() if not scene_id_list: logging.info('\nNo Landsat images in date range, exiting') sys.exit() # Process each image in the collection by date # image_id is the full Earth Engine ID to the asset for scene_id in scene_id_list: logging.info('{}'.format(scene_id)) l, p, r, year, month, day = landsat.parse_landsat_id(scene_id) image_dt = datetime.datetime.strptime( '{:04d}{:02d}{:02d}'.format(year, month, day), '%Y%m%d') image_date = image_dt.date().isoformat() # logging.debug(' Date: {0}'.format(image_date)) # logging.debug(' DOY: {0}'.format(doy)) # Export products for product in ini['EXPORT']['products']: export_id = ini['EXPORT']['export_id_fmt'] \ .replace('_{start}', '') \ .replace('_{end}', '') \ .format( model=ini['INPUTS']['et_model'].lower(), product=product.lower(), study_area=ini['INPUTS']['study_area_name'], index=scene_id, export=ini['EXPORT']['export_dest'].lower()) export_id = export_id.replace('-', '') logging.debug(' Export ID: {0}'.format(export_id)) if ini['EXPORT']['export_dest'] == 'CLOUD': # Write each product to a separate folder export_path = '{}/{}/{}'.format(ini['EXPORT']['output_ws'], product, export_id + '.tif') elif ini['EXPORT']['export_dest'] == 'GDRIVE': export_path = os.path.join(ini['EXPORT']['output_ws'], export_id + '.tif') logging.debug(' {}'.format(export_path)) if overwrite_flag: if export_id in tasks.keys(): logging.debug(' Task already submitted, cancelling') ee.data.cancelTask(tasks[export_id]) # This is intentionally not an "elif" so that a task can be # cancelled and an existing image/file/asset can be removed if (ini['EXPORT']['export_dest'] == 'CLOUD' and export_path in cloud_list): logging.debug(' Export image already exists') # Files in cloud storage are easily overwritten # so it is unneccesary to manually remove them # # This would remove an existing file # subprocess.call(['gsutil', 'rm', export_path]) elif (ini['EXPORT']['export_dest'] == 'GDRIVE' and export_path in gdrive_list): logging.debug( ' Export image already exists, removing') os.remove(export_path) # Remove automatically generated image tiles # for f in glob.glob(export_path.replace('.tif', '*.tif')): # os.remove(f) else: if export_id in tasks.keys(): logging.debug(' Task already submitted, skipping') continue elif (ini['EXPORT']['export_dest'] == 'CLOUD' and export_path in cloud_list): logging.debug( ' Export file already exists, skipping') continue elif (ini['EXPORT']['export_dest'] == 'GDRIVE' and os.path.isfile(export_path)): logging.debug( ' Export file already exists, skipping') continue if ini['INPUTS']['et_model'] == 'EEFLUX': # Get a Landsat collection with only the target image landsat_image = landsat.get_landsat_image( wrs2_tile_list=export_info['wrs2_tiles'], cloud_cover=ini['INPUTS']['cloud_cover'], start_date=image_date, landsat_type='RAD') # The 'BQA' band is also being returned by the etrf method etrf_obj = eeflux.EEFlux(ee.Image(landsat_image)).etrf etrf_img = ee.Image(etrf_obj.select(['etrf'], ['etf'])) \ .clamp(-1, 2) cloud_mask = landsat.landsat_bqa_cloud_mask_func( ee.Image(etrf_obj.select(['BQA']))) daily_et_fraction_image = etrf_img.updateMask(cloud_mask) \ .copyProperties(etrf_obj, ['system:time_start']) # Image date reference ET # DEADBEEF - Hard coding to GRIDMET for now # Should this be retrieved from the model? daily_et_reference_image = 'IDAHO_EPSCOR/GRIDMET/{}'.format( image_dt.date().strftime('%Y%m%d')) daily_et_reference_image = ee.Image(daily_et_reference_image) \ .select(['etr'], ['et_reference']) else: logging.error('\nInvalid/unsupported ET Model: {}'.format( ini['INPUTS']['et_model'])) return False # Compute target product if product == 'et_actual': output_image = daily_et_fraction_image \ .multiply(daily_et_reference_image) elif product == 'et_reference': output_image = daily_et_reference_image elif product == 'et_fraction': output_image = daily_et_fraction_image elif product in ['count', 'count_mask']: output_image = daily_et_fraction_image.mask() # elif product == 'scene_id': # continue else: logging.debug( ' Unsupported product {}, skipping'.format(product)) continue # Convert data types for export to Google Drive or Cloud Storage if (product in ['et_actual', 'et_reference', 'et_fraction'] and ini['EXPORT']['export_dest'] in ['CLOUD', 'GDRIVE']): output_image = output_image.unmask(-9999, False).toFloat() elif (product in ['count'] and ini['EXPORT']['export_dest'] in ['CLOUD', 'GDRIVE']): output_image = output_image.unmask(255, False).toUint8() # output_image = output_image.toUint8() elif ini['EXPORT']['export_dest'] in ['ASSET']: pass # Build export tasks if ini['EXPORT']['export_dest'] == 'CLOUD': # Export the image to cloud storage task = ee.batch.Export.image.toCloudStorage( output_image, description=export_id, bucket=ini['EXPORT']['bucket_name'], fileNamePrefix='{}/{}'.format(product, export_id), dimensions=export_info['shape'], crs=export_info['crs'], crsTransform=export_info['geo'], # shardSize=, # fileDimensions=, maxPixels=export_info['maxpixels']) elif ini['EXPORT']['export_dest'] == 'GDRIVE': # Export the images to your Google Drive task = ee.batch.Export.image.toDrive( output_image, description=export_id, folder=os.path.basename(ini['EXPORT']['output_ws']), fileNamePrefix=export_id, dimensions=export_info['shape'], crs=export_info['crs'], crsTransform=export_info['geo'], maxPixels=export_info['maxpixels']) else: logging.debug(' Export task not built, skipping') continue # Try to start the export task a few times logging.debug(' Starting export task') for i in range(1, 10): try: task.start() break except Exception as e: logging.error( ' Error: {}\n Retrying ({}/10)'.format(e, i)) time.sleep(i**2) i += 1