def remap_mask_func(input_path, output_path, value_list): """Remap the input raster to all 1's and return a mask raster Parameters ---------- input_path : str File path of the input raster. output_path : str File path of the output (mask) raster. value_list : list Values that will be set to 1. Returns ------- True if sucessful """ input_ds = gdal.Open(input_path) input_geo = drigo.raster_ds_geo(input_ds) input_proj = drigo.raster_ds_proj(input_ds) input_array = drigo.raster_ds_to_array(input_ds)[0] input_mask = np.zeros(input_array.shape, dtype=np.bool) for input_value in value_list: input_mask[input_array == input_value] = 1 drigo.array_to_raster(input_mask, output_path, input_geo, input_proj) return True
def metric_weather_func(output_raster, input_ws, input_re, prev_dt, next_dt, resample_method=gdal.GRA_NearestNeighbour, rounding_flag=False): """Interpolate/project/clip METRIC hourly rasters""" logging.debug(' Output: {}'.format(output_raster)) if os.path.isfile(output_raster): if overwrite_flag: logging.debug(' Overwriting output') dripy.remove_file(output_raster) else: logging.debug(' Skipping, file already exists ' + 'and overwrite is False') return False prev_ws = os.path.join(input_ws, str(prev_dt.year)) next_ws = os.path.join(input_ws, str(next_dt.year)) # Technically previous and next could come from different days # or even years, although this won't happen in the U.S. try: prev_path = [ os.path.join(prev_ws, input_name) for input_name in os.listdir(prev_ws) for input_match in [input_re.match(input_name)] if (input_match and (prev_dt.strftime('%Y%m%d') == input_match.group('YYYYMMDD')))][0] logging.debug(' Input prev: {}'.format(prev_path)) except IndexError: logging.error(' No previous hourly file') logging.error(' {}'.format(prev_dt)) return False try: next_path = [ os.path.join(next_ws, input_name) for input_name in os.listdir(next_ws) for input_match in [input_re.match(input_name)] if (input_match and (next_dt.strftime('%Y%m%d') == input_match.group('YYYYMMDD')))][0] logging.debug(' Input next: {}'.format(next_path)) except IndexError: logging.error(' No next hourly file') logging.error(' {}'.format(next_dt)) return False # Band numbers are 1's based prev_band = int(prev_dt.strftime('%H')) + 1 next_band = int(next_dt.strftime('%H')) + 1 logging.debug(' Input prev band: {}'.format(prev_band)) logging.debug(' Input next band: {}'.format(next_band)) # Read arrays prev_array = drigo.raster_to_array( prev_path, band=prev_band, mask_extent=common_gcs_extent, return_nodata=False) next_array = drigo.raster_to_array( next_path, band=next_band, mask_extent=common_gcs_extent, return_nodata=False) if not np.any(prev_array) or not np.any(next_array): logging.warning('\nWARNING: Input NLDAS array is all nodata\n') return None output_array = hourly_interpolate_func( prev_array, next_array, prev_dt, next_dt, image.acq_datetime) output_array = drigo.project_array( output_array, resample_method, input_osr, input_cs, common_gcs_extent, common_osr, env.cellsize, common_extent, output_nodata=None) # Apply common area mask output_array[~common_array] = np.nan # Reduce the file size by rounding to the nearest n digits if rounding_flag: output_array = np.around(output_array, rounding_digits) # Force output to 32-bit float drigo.array_to_raster( output_array.astype(np.float32), output_raster, output_geo=common_geo, output_proj=common_proj, stats_flag=stats_flag) del output_array return True
def main(image_ws, ini_path, bs=2048, stats_flag=False, overwrite_flag=False): """Prep a Landsat scene for METRIC Parameters ---------- image_ws : str Landsat scene folder that will be prepped. ini_path : str File path of the input parameters file. bs : int, optional Processing block size (the default is 2048). stats_flag : bool, optional If True, compute raster statistics (the default is True). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). Returns ------- True is successful """ # Open config file config = dripy.open_ini(ini_path) # Get input parameters logging.debug(' Reading Input File') calc_refl_toa_flag = dripy.read_param( 'calc_refl_toa_flag', True, config, 'INPUTS') calc_refl_toa_qa_flag = dripy.read_param( 'calc_refl_toa_qa_flag', True, config, 'INPUTS') # calc_refl_sur_ledaps_flag = dripy.read_param( # 'calc_refl_sur_ledaps_flag', False, config, 'INPUTS') # calc_refl_sur_qa_flag = dripy.read_param( # 'calc_refl_sur_qa_flag', False, config, 'INPUTS') calc_ts_bt_flag = dripy.read_param( 'calc_ts_bt_flag', True, config, 'INPUTS') # Use QA band to set common area # Fmask cloud, shadow, & snow pixels will be removed from common area calc_fmask_common_flag = dripy.read_param( 'calc_fmask_common_flag', True, config, 'INPUTS') fmask_smooth_flag = dripy.read_param( 'fmask_smooth_flag', False, config, 'INPUTS') fmask_buffer_flag = dripy.read_param( 'fmask_buffer_flag', False, config, 'INPUTS') fmask_erode_flag = dripy.read_param( 'fmask_erode_flag', False, config, 'INPUTS') if fmask_smooth_flag: fmask_smooth_cells = int(dripy.read_param( 'fmask_smooth_cells', 1, config, 'INPUTS')) if fmask_smooth_cells == 0 and fmask_smooth_flag: fmask_smooth_flag = False if fmask_erode_flag: fmask_erode_cells = int(dripy.read_param( 'fmask_erode_cells', 1, config, 'INPUTS')) if fmask_erode_cells == 0 and fmask_erode_flag: fmask_erode_flag = False if fmask_buffer_flag: fmask_buffer_cells = int(dripy.read_param( 'fmask_buffer_cells', 1, config, 'INPUTS')) if fmask_buffer_cells == 0 and fmask_buffer_flag: fmask_buffer_flag = False # Remove edge (fringe) cells edge_smooth_flag = dripy.read_param( 'edge_smooth_flag', True, config, 'INPUTS') # Include hand made cloud masks cloud_mask_flag = dripy.read_param( 'cloud_mask_flag', False, config, 'INPUTS') cloud_mask_ws = "" if cloud_mask_flag: cloud_mask_ws = config.get('INPUTS', 'cloud_mask_ws') # Extract separate Fmask rasters calc_fmask_flag = dripy.read_param( 'calc_fmask_flag', True, config, 'INPUTS') calc_fmask_cloud_flag = dripy.read_param( 'calc_fmask_cloud_flag', True, config, 'INPUTS') calc_fmask_snow_flag = dripy.read_param( 'calc_fmask_snow_flag', True, config, 'INPUTS') calc_fmask_water_flag = dripy.read_param( 'calc_fmask_water_flag', True, config, 'INPUTS') # Keep Landsat DN, LEDAPS, and Fmask rasters keep_dn_flag = dripy.read_param( 'keep_dn_flag', True, config, 'INPUTS') # keep_sr_flag = dripy.read_param( # 'keep_sr_flag', True, config, 'INPUTS') # For this to work I would need to pass in the metric input file # calc_elev_flag = dripy.read_param( # 'calc_elev_flag', False, config, 'INPUTS') # calc_landuse_flag = dripy.read_param( # 'calc_landuse_flag', False, config, 'INPUTS') # calc_acca_cloud_flag = dripy.read_param( # 'calc_acca_cloud_flag', True, config, 'INPUTS') # calc_acca_snow_flag = dripy.read_param( # 'calc_acca_snow_flag', True, config, 'INPUTS') # calc_ledaps_dem_land_flag = dripy.read_param( # 'calc_ledaps_dem_land_flag', False, config, 'INPUTS') # calc_ledaps_veg_flag = dripy.read_param( # 'calc_ledaps_veg_flag', False, config, 'INPUTS') # calc_ledaps_snow_flag = dripy.read_param( # 'calc_ledaps_snow_flag', False, config, 'INPUTS') # calc_ledaps_land_flag = dripy.read_param( # 'calc_ledaps_land_flag', False, config, 'INPUTS') # calc_ledaps_cloud_flag = dripy.read_param( # 'calc_ledaps_cloud_flag', False, config, 'INPUTS') # Interpolate/clip/project hourly rasters for each Landsat scene # calc_metric_flag = dripy.read_param( # 'calc_metric_flag', False, config, 'INPUTS') calc_metric_ea_flag = dripy.read_param( 'calc_metric_ea_flag', False, config, 'INPUTS') calc_metric_wind_flag = dripy.read_param( 'calc_metric_wind_flag', False, config, 'INPUTS') calc_metric_etr_flag = dripy.read_param( 'calc_metric_etr_flag', False, config, 'INPUTS') calc_metric_tair_flag = dripy.read_param( 'calc_metric_tair_flag', False, config, 'INPUTS') # Interpolate/clip/project AWC and daily ETr/PPT rasters # to compute SWB Ke for each Landsat scene calc_swb_ke_flag = dripy.read_param( 'calc_swb_ke_flag', False, config, 'INPUTS') if cloud_mask_flag: spinup_days = dripy.read_param( 'swb_spinup_days', 30, config, 'INPUTS') min_spinup_days = dripy.read_param( 'swb_min_spinup_days', 5, config, 'INPUTS') # Round ea raster to N digits to save space rounding_digits = dripy.read_param( 'rounding_digits', 3, config, 'INPUTS') env = drigo.env image = et_image.Image(image_ws, env) np.seterr(invalid='ignore', divide='ignore') gdal.UseExceptions() # Input file paths dn_image_dict = et_common.landsat_band_image_dict( image.orig_data_ws, image.image_name_re) # # Open METRIC config file # if config_file: # logging.info( # log_f.format('METRIC INI File:', os.path.basename(config_file))) # config = configparser.ConfigParser() # try: # config.read(config_file) # except: # logging.error('\nERROR: Config file could not be read, ' + # 'is not an input file, or does not exist\n' + # 'ERROR: config_file = {}\n').format(config_file) # sys.exit() # # Overwrite # overwrite_flag = dripy.read_param('overwrite_flag', True, config) # # # Elevation and landuse parameters/flags from METRIC input file # calc_elev_flag = dripy.read_param('save_dem_raster_flag', True, config) # calc_landuse_flag = dripy.read_param( # 'save_landuse_raster_flag', True, config) # if calc_elev_flag: # elev_pr_path = config.get('INPUTS','dem_raster') # if calc_landuse_flag: # landuse_pr_path = config.get('INPUTS', 'landuse_raster') # else: # overwrite_flag = False # calc_elev_flag = False # calc_landuse_flag = False # # Elev raster must exist # if calc_elev_flag and not os.path.isfile(elev_pr_path): # logging.error('\nERROR: Elevation raster {} does not exist\n'.format( # elev_pr_path)) # return False # Landuse raster must exist # if calc_landuse_flag and not os.path.isfile(landuse_pr_path): # logging.error('\nERROR: Landuse raster {} does not exist\n'.format( # landuse_pr_path)) # return False # Removing ancillary files before checking for inputs if os.path.isdir(os.path.join(image.orig_data_ws, 'gap_mask')): shutil.rmtree(os.path.join(image.orig_data_ws, 'gap_mask')) for item in os.listdir(image.orig_data_ws): if (image.type == 'Landsat7' and (item.endswith('_B8.TIF') or item.endswith('_B6_VCID_2.TIF'))): os.remove(os.path.join(image.orig_data_ws, item)) elif (image.type == 'Landsat8' and (item.endswith('_B1.TIF') or item.endswith('_B8.TIF') or item.endswith('_B9.TIF') or item.endswith('_B11.TIF'))): os.remove(os.path.join(image.orig_data_ws, item)) elif (item.endswith('_VER.jpg') or item.endswith('_VER.txt') or item.endswith('_GCP.txt') or item == 'README.GTF'): os.remove(os.path.join(image.orig_data_ws, item)) # Check correction level (image must be L1T to process) if image.correction != 'L1TP': logging.debug(' Image is not L1TP corrected, skipping') return False # calc_fmask_common_flag = False # calc_refl_toa_flag = False # calc_ts_bt_flag = False # calc_metric_ea_flag = False # calc_metric_wind_flag = False # calc_metric_etr_flag = False # overwrite_flag = False # QA band must exist if calc_fmask_common_flag and image.qa_band not in dn_image_dict.keys(): logging.warning( '\nQA band does not exist but calc_fmask_common_flag=True' '\n Setting calc_fmask_common_flag=False\n {}'.format( image.qa_band)) calc_fmask_common_flag = False if cloud_mask_flag and not os.path.isdir(cloud_mask_ws): logging.warning( '\ncloud_mask_ws is not a directory but cloud_mask_flag=True.' '\n Setting cloud_mask_flag=False\n {}'.format(cloud_mask_ws)) cloud_mask_flag = False # Check for Landsat TOA images if (calc_refl_toa_flag and (set(list(image.band_toa_dict.keys()) + [image.thermal_band, image.qa_band]) != set(dn_image_dict.keys()))): logging.warning( '\nMissing Landsat images but calc_refl_toa_flag=True' '\n Setting calc_refl_toa_flag=False') calc_refl_toa_flag = False # Check for Landsat brightness temperature image if calc_ts_bt_flag and image.thermal_band not in dn_image_dict.keys(): logging.warning( '\nThermal band image does not exist but calc_ts_bt_flag=True' '\n Setting calc_ts_bt_flag=False') calc_ts_bt_flag = False # DEADBEEF - Should the function return False if Ts doesn't exist? # return False # Check for METRIC hourly/daily input folders if calc_metric_ea_flag: metric_ea_input_ws = config.get('INPUTS', 'metric_ea_input_folder') if not os.path.isdir(metric_ea_input_ws): logging.warning( '\nHourly Ea folder does not exist but calc_metric_ea_flag=True' '\n Setting calc_metric_ea_flag=False\n {}'.format( metric_ea_input_ws)) calc_metric_ea_flag = False if calc_metric_wind_flag: metric_wind_input_ws = config.get('INPUTS', 'metric_wind_input_folder') if not os.path.isdir(metric_wind_input_ws): logging.warning( '\nHourly wind folder does not exist but calc_metric_wind_flag=True' '\n Setting calc_metric_wind_flag=False\n {}'.format( metric_wind_input_ws)) calc_metric_wind_flag = False if calc_metric_etr_flag: metric_etr_input_ws = config.get('INPUTS', 'metric_etr_input_folder') if not os.path.isdir(metric_etr_input_ws): logging.warning( '\nHourly ETr folder does not exist but calc_metric_etr_flag=True' '\n Setting calc_metric_etr_flag=False\n {}'.format( metric_etr_input_ws)) calc_metric_etr_flag = False if calc_metric_tair_flag: metric_tair_input_ws = config.get('INPUTS', 'metric_tair_input_folder') if not os.path.isdir(metric_tair_input_ws): logging.warning( '\nHourly Tair folder does not exist but calc_metric_tair_flag=True' '\n Setting calc_metric_tair_flag=False\n {}'.format( metric_tair_input_ws)) calc_metric_tair_flag = False if (calc_metric_ea_flag or calc_metric_wind_flag or calc_metric_etr_flag or calc_metric_tair_flag): metric_hourly_re = re.compile(config.get('INPUTS', 'metric_hourly_re')) metric_daily_re = re.compile(config.get('INPUTS', 'metric_daily_re')) if calc_swb_ke_flag: awc_input_path = config.get('INPUTS', 'awc_input_path') etr_input_ws = config.get('INPUTS', 'etr_input_folder') ppt_input_ws = config.get('INPUTS', 'ppt_input_folder') etr_input_re = re.compile(config.get('INPUTS', 'etr_input_re')) ppt_input_re = re.compile(config.get('INPUTS', 'ppt_input_re')) if not os.path.isfile(awc_input_path): logging.warning( '\nAWC raster does not exist but calc_swb_ke_flag=True' '\n Setting calc_swb_ke_flag=False\n {}'.format( awc_input_path)) calc_swb_ke_flag = False if not os.path.isdir(etr_input_ws): logging.warning( '\nDaily ETr folder does not exist but calc_swb_ke_flag=True' '\n Setting calc_swb_ke_flag=False\n {}'.format( etr_input_ws)) calc_swb_ke_flag = False if not os.path.isdir(ppt_input_ws): logging.warning( '\nDaily PPT folder does not exist but calc_swb_ke_flag=True' '\n Setting calc_swb_ke_flag=False\n {}'.format( ppt_input_ws)) calc_swb_ke_flag = False # Build folders for support rasters if ((calc_fmask_common_flag or calc_refl_toa_flag or # calc_refl_sur_ledaps_flag or calc_ts_bt_flag or calc_metric_ea_flag or calc_metric_wind_flag or calc_metric_etr_flag or calc_metric_tair_flag or calc_swb_ke_flag) and not os.path.isdir(image.support_ws)): os.makedirs(image.support_ws) if calc_refl_toa_flag and not os.path.isdir(image.refl_toa_ws): os.makedirs(image.refl_toa_ws) # if calc_refl_sur_ledaps_flag and not os.path.isdir(image.refl_sur_ws): # os.makedirs(image.refl_sur_ws) # DEADBEEF - This is being further down just for the Fmask images # # Apply overwrite flag # if overwrite_flag: # overwrite_list = [ # image.fmask_cloud_raster, image.fmask_snow_raster, # image.fmask_water_raster # # image.elev_raster, image.landuse_raster # # image.common_area_raster # ] # for overwrite_path in overwrite_list: # try: # dripy.remove_file(image.fmask_cloud_raster) # except: # pass # Use QA band to build common area rasters logging.info('\nCommon Area Raster') qa_ds = gdal.Open(dn_image_dict[image.qa_band], 0) common_geo = drigo.raster_ds_geo(qa_ds) common_extent = drigo.raster_ds_extent(qa_ds) common_proj = drigo.raster_ds_proj(qa_ds) common_osr = drigo.raster_ds_osr(qa_ds) # Initialize common_area as all non-fill QA values qa_array = drigo.raster_ds_to_array(qa_ds, return_nodata=False) common_array = qa_array != 1 common_rows, common_cols = common_array.shape del qa_ds # Erode and dilate to remove fringe on edge # Default is to not smooth, but user can force smoothing # This needs to be applied before Fmask if edge_smooth_flag and image.prefix in ['LT05', 'LE07']: struct = ndimage.generate_binary_structure(2, 2).astype(np.uint8) if image.prefix == 'LT05': cells = 8 elif image.prefix == 'LE07': cells = 2 else: cells = 0 common_array = ndimage.binary_dilation( ndimage.binary_erosion(common_array, struct, cells), struct, cells) # Try applying user defined cloud masks to common_area cloud_mask_path = os.path.join( cloud_mask_ws, image.folder_id + '_mask.shp') if cloud_mask_flag and os.path.isfile(cloud_mask_path): logging.info(' Applying cloud mask shapefile') feature_path = os.path.join( cloud_mask_ws, (image.folder_id + '_mask.shp')) logging.info(' {}'.format(feature_path)) cloud_mask_memory_ds = drigo.polygon_to_raster_ds( feature_path, nodata_value=0, burn_value=1, output_osr=common_osr, output_cs=30, output_extent=common_extent) cloud_array = drigo.raster_ds_to_array( cloud_mask_memory_ds, return_nodata=False) # DEADBEEF - If user sets a cloud mask, # it is probably better than Fmask # Eventually change "if" calc_fmask_common_flag: to "elif" common_array[cloud_array == 1] = 0 del cloud_mask_memory_ds, cloud_array # Remove Fmask cloud, shadow, and snow pixels from common_area if calc_fmask_common_flag: logging.info(' Applying Fmask to common area') fmask_array = et_numpy.bqa_fmask_func(qa_array) fmask_mask = (fmask_array >= 2) & (fmask_array <= 4) if fmask_smooth_flag: logging.debug( ' Smoothing (dilate/erode/erode/dilate) Fmask clouds, shadows,' ' and snow pixels by {} cells'.format(fmask_smooth_cells)) # ArcGIS smoothing procedure fmask_mask = ndimage.binary_dilation( fmask_mask, iterations=fmask_smooth_cells, structure=ndimage.generate_binary_structure(2, 2)) fmask_mask = ndimage.binary_erosion( fmask_mask, iterations=fmask_smooth_cells, structure=ndimage.generate_binary_structure(2, 2)) fmask_mask = ndimage.binary_erosion( fmask_mask, iterations=fmask_smooth_cells, structure=ndimage.generate_binary_structure(2, 2)) fmask_mask = ndimage.binary_dilation( fmask_mask, iterations=fmask_smooth_cells, structure=ndimage.generate_binary_structure(2, 2)) if fmask_erode_flag: logging.debug( ' Eroding Fmask clouds, shadows, and snow pixels by ' '{} cells'.format(fmask_erode_cells)) fmask_mask = ndimage.binary_erosion( fmask_mask, iterations=fmask_erode_cells, structure=ndimage.generate_binary_structure(2, 2)) if fmask_buffer_flag: logging.debug( ' Dilating (buffering) Fmask clouds, shadows, and snow pixels ' 'by {} cells'.format(fmask_buffer_cells)) fmask_mask = ndimage.binary_dilation( fmask_mask, iterations=fmask_buffer_cells, structure=ndimage.generate_binary_structure(2, 2)) # Reset common_array for buffered cells common_array[fmask_mask] = 0 del fmask_array, fmask_mask # Check that there are some cloud free pixels if not np.any(common_array): logging.error( ' ERROR: There are no cloud/snow free pixels, returning False') return False # Always overwrite common area raster # if not os.path.isfile(image.common_area_raster): drigo.array_to_raster( common_array, image.common_area_raster, output_geo=common_geo, output_proj=common_proj, stats_flag=stats_flag) # Print common geo/extent logging.debug(' Common geo: {}'.format(common_geo)) logging.debug(' Common extent: {}'.format(common_extent)) # Extract Fmask components as separate rasters if (calc_fmask_flag or calc_fmask_cloud_flag or calc_fmask_snow_flag or calc_fmask_water_flag): logging.info('\nFmask') fmask_array = et_numpy.bqa_fmask_func(qa_array) # Remove existing Fmask rasters if (calc_fmask_flag and overwrite_flag and os.path.isfile(image.fmask_output_raster)): logging.debug(' Overwriting: {}'.format( image.fmask_output_raster)) dripy.remove_file(image.fmask_output_raster) if (calc_fmask_cloud_flag and overwrite_flag and os.path.isfile(image.fmask_cloud_raster)): logging.debug(' Overwriting: {}'.format( image.fmask_cloud_raster)) dripy.remove_file(image.fmask_cloud_raster) if (calc_fmask_snow_flag and overwrite_flag and os.path.isfile(image.fmask_snow_raster)): logging.debug(' Overwriting: {}'.format( image.fmask_snow_raster)) dripy.remove_file(image.fmask_snow_raster) if (calc_fmask_water_flag and overwrite_flag and os.path.isfile(image.fmask_water_raster)): logging.debug(' Overwriting: {}'.format( image.fmask_water_raster)) dripy.remove_file(image.fmask_water_raster) # Save Fmask data as separate rasters if (calc_fmask_flag and not os.path.isfile(image.fmask_output_raster)): logging.debug(' Saving Fmask raster') drigo.array_to_raster( fmask_array.astype(np.uint8), image.fmask_output_raster, output_geo=common_geo, output_proj=common_proj, mask_array=None, output_nodata=255, stats_flag=stats_flag) if (calc_fmask_cloud_flag and not os.path.isfile(image.fmask_cloud_raster)): logging.debug(' Saving Fmask cloud raster') fmask_cloud_array = (fmask_array == 2) | (fmask_array == 4) drigo.array_to_raster( fmask_cloud_array.astype(np.uint8), image.fmask_cloud_raster, output_geo=common_geo, output_proj=common_proj, mask_array=None, output_nodata=255, stats_flag=stats_flag) del fmask_cloud_array if (calc_fmask_snow_flag and not os.path.isfile(image.fmask_snow_raster)): logging.debug(' Saving Fmask snow raster') fmask_snow_array = (fmask_array == 3) drigo.array_to_raster( fmask_snow_array.astype(np.uint8), image.fmask_snow_raster, output_geo=common_geo, output_proj=common_proj, mask_array=None, output_nodata=255, stats_flag=stats_flag) del fmask_snow_array if (calc_fmask_water_flag and not os.path.isfile(image.fmask_water_raster)): logging.debug(' Saving Fmask water raster') fmask_water_array = (fmask_array == 1) drigo.array_to_raster( fmask_water_array.astype(np.uint8), image.fmask_water_raster, output_geo=common_geo, output_proj=common_proj, mask_array=None, output_nodata=255, stats_flag=stats_flag) del fmask_water_array del fmask_array # # Calculate elevation # if calc_elev_flag and not os.path.isfile(elev_path): # logging.info('Elevation') # elev_array, elev_nodata = drigo.raster_to_array( # elev_pr_path, 1, common_extent) # drigo.array_to_raster( # elev_array, elev_raster, # output_geo=common_geo, output_proj=env.snap_proj, # mask_array=common_array, stats_flag=stats_flag) # del elev_array, elev_nodata, elev_path # # # Calculate landuse # if calc_landuse_flag and not os.path.isfile(landuse_raster): # logging.info('Landuse') # landuse_array, landuse_nodata = drigo.raster_to_array( # landuse_pr_path, 1, common_extent) # drigo.array_to_raster( # landuse_array, landuse_raster, # output_geo=common_geo, output_proj=env.snap_proj, # mask_array=common_array, stats_flag=stats_flag) # del landuse_array, landuse_nodata, landuse_raster # Calculate toa reflectance # f32_gtype, f32_nodata = numpy_to_gdal_type(np.float32) if calc_refl_toa_flag: logging.info('Top-of-Atmosphere Reflectance') if os.path.isfile(image.refl_toa_raster) and overwrite_flag: logging.debug(' Overwriting: {}'.format( image.refl_toa_raster)) dripy.remove_file(image.refl_toa_raster) if not os.path.isfile(image.refl_toa_raster): # First build empty composite raster drigo.build_empty_raster( image.refl_toa_raster, image.band_toa_cnt, np.float32, None, env.snap_proj, env.cellsize, common_extent) # cos_theta_solar_flt = et_common.cos_theta_solar_func( # image.sun_elevation) # Process by block logging.info('Processing by block') logging.debug(' Mask cols/rows: {}/{}'.format( common_cols, common_rows)) for b_i, b_j in drigo.block_gen(common_rows, common_cols, bs): logging.debug(' Block y: {:5d} x: {:5d}'.format(b_i, b_j)) block_data_mask = drigo.array_to_block( common_array, b_i, b_j, bs).astype(np.bool) block_rows, block_cols = block_data_mask.shape block_geo = drigo.array_offset_geo(common_geo, b_j, b_i) block_extent = drigo.geo_extent( block_geo, block_rows, block_cols) logging.debug(' Block rows: {} cols: {}'.format( block_rows, block_cols)) logging.debug(' Block extent: {}'.format(block_extent)) logging.debug(' Block geo: {}'.format(block_geo)) # Process each TOA band # for band, band_i in sorted(image.band_toa_dict.items()): for band, dn_image in sorted(dn_image_dict.items()): if band not in image.band_toa_dict.keys(): continue # thermal_band_flag = (band == image.thermal_band) # Set 0 as nodata value drigo.raster_path_set_nodata(dn_image, 0) # Calculate TOA reflectance dn_array, dn_nodata = drigo.raster_to_array( dn_image, 1, block_extent) dn_array = dn_array.astype(np.float64) # dn_array = dn_array.astype(np.float32) dn_array[dn_array == 0] = np.nan # if image.type in ['Landsat4', 'Landsat5', 'Landsat7']: refl_toa_array = et_numpy.l457_refl_toa_band_func( dn_array, image.cos_theta_solar, image.dr, image.esun_dict[band], image.lmin_dict[band], image.lmax_dict[band], image.qcalmin_dict[band], image.qcalmax_dict[band]) elif image.type in ['Landsat8']: refl_toa_array = et_numpy.l8_refl_toa_band_func( dn_array, image.cos_theta_solar, image.refl_mult_dict[band], image.refl_add_dict[band]) # if (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and # not thermal_band_flag): # refl_toa_array = et_numpy.l457_refl_toa_band_func( # dn_array, image.cos_theta_solar, # image.dr, image.esun_dict[band], # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], # image.qcalmax_dict[band]) # # image.rad_mult_dict[band], # # image.rad_add_dict[band]) # elif (image.type in ['Landsat8'] and # not thermal_band_flag): # refl_toa_array = et_numpy.l8_refl_toa_band_func( # dn_array, image.cos_theta_solar, # image.refl_mult_dict[band], # image.refl_add_dict[band]) # elif (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and # thermal_band_flag): # refl_toa_array = et_numpy.l457_ts_bt_band_func( # dn_array, # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], # image.qcalmax_dict[band], # # image.rad_mult_dict[band], # # image.rad_add_dict[band], # image.k1_dict[band], image.k2_dict[band]) # elif (image.type in ['Landsat8'] and # thermal_band_flag): # refl_toa_array = et_numpy.l8_ts_bt_band_func( # dn_array, # image.rad_mult_dict[band], # image.rad_add_dict[band], # image.k1_dict[band], image.k2_dict[band]) # refl_toa_array = et_numpy.refl_toa_band_func( # dn_array, cos_theta_solar_flt, # image.dr, image.esun_dict[band], # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], image.qcalmax_dict[band], # thermal_band_flag) drigo.block_to_raster( refl_toa_array.astype(np.float32), image.refl_toa_raster, b_i, b_j, band=image.band_toa_dict[band]) # drigo.array_to_comp_raster( # refl_toa_array.astype(np.float32), # image.refl_toa_raster, # image.band_toa_dict[band], common_array) del refl_toa_array, dn_array if stats_flag: drigo.raster_statistics(image.refl_toa_raster) # # Process each TOA band # # for band, band_i in sorted(image.band_toa_dict.items()): # for band, dn_image in sorted(dn_image_dict.items()): # thermal_band_flag = (band == image.thermal_band) # # Set 0 as nodata value # drigo.raster_path_set_nodata(dn_image, 0) # # Calculate TOA reflectance # dn_array, dn_nodata = drigo.raster_to_array( # dn_image, 1, common_extent) # dn_array = dn_array.astype(np.float64) # # dn_array = dn_array.astype(np.float32) # dn_array[dn_array == 0] = np.nan # # # if (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and # not thermal_band_flag): # refl_toa_array = et_numpy.l457_refl_toa_band_func( # dn_array, image.cos_theta_solar, # image.dr, image.esun_dict[band], # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], image.qcalmax_dict[band]) # # image.rad_mult_dict[band], image.rad_add_dict[band]) # elif (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and # thermal_band_flag): # refl_toa_array = et_numpy.l457_ts_bt_band_func( # dn_array, image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], image.qcalmax_dict[band], # # image.rad_mult_dict[band], image.rad_add_dict[band], # image.k1_dict[band], image.k2_dict[band]) # elif (image.type in ['Landsat8'] and # not thermal_band_flag): # refl_toa_array = et_numpy.l8_refl_toa_band_func( # dn_array, image.cos_theta_solar, # image.refl_mult_dict[band], image.refl_add_dict[band]) # elif (image.type in ['Landsat8'] and # thermal_band_flag): # refl_toa_array = et_numpy.l8_ts_bt_band_func( # dn_array, # image.rad_mult_dict[band], image.rad_add_dict[band], # image.k1_dict[band], image.k2_dict[band]) # # refl_toa_array = et_numpy.refl_toa_band_func( # # dn_array, cos_theta_solar_flt, # # image.dr, image.esun_dict[band], # # image.lmin_dict[band], image.lmax_dict[band], # # image.qcalmin_dict[band], image.qcalmax_dict[band], # # thermal_band_flag) # drigo.array_to_comp_raster( # refl_toa_array.astype(np.float32), image.refl_toa_raster, # image.band_toa_dict[band], common_array) # del refl_toa_array, dn_array # Calculate brightness temperature if calc_ts_bt_flag: logging.info('Brightness Temperature') if os.path.isfile(image.ts_bt_raster) and overwrite_flag: logging.debug(' Overwriting: {}'.format(image.ts_bt_raster)) dripy.remove_file(image.ts_bt_raster) if not os.path.isfile(image.ts_bt_raster): band = image.thermal_band thermal_dn_path = dn_image_dict[band] drigo.raster_path_set_nodata(thermal_dn_path, 0) thermal_dn_array, thermal_dn_nodata = drigo.raster_to_array( thermal_dn_path, 1, common_extent, return_nodata=True) thermal_dn_mask = thermal_dn_array != thermal_dn_nodata if image.type in ['Landsat4', 'Landsat5', 'Landsat7']: ts_bt_array = et_numpy.l457_ts_bt_band_func( thermal_dn_array, image.lmin_dict[band], image.lmax_dict[band], image.qcalmin_dict[band], image.qcalmax_dict[band], # image.rad_mult_dict[band], image.rad_add_dict[band], image.k1_dict[band], image.k2_dict[band]) elif image.type in ['Landsat8']: ts_bt_array = et_numpy.l8_ts_bt_band_func( thermal_dn_array, image.rad_mult_dict[band], image.rad_add_dict[band], image.k1_dict[band], image.k2_dict[band]) # thermal_rad_array = et_numpy.refl_toa_band_func( # thermal_dn_array, image.cos_theta_solar, # image.dr, image.esun_dict[band], # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], image.qcalmax_dict[band], # thermal_band_flag=True) # ts_bt_array = et_numpy.ts_bt_func( # thermal_rad_array, image.k1_dict[image.thermal_band], # image.k2_dict[image.thermal_band]) ts_bt_array[~thermal_dn_mask] = np.nan drigo.array_to_raster( ts_bt_array, image.ts_bt_raster, output_geo=common_geo, output_proj=env.snap_proj, # mask_array=common_array, stats_flag=stats_flag) # del thermal_dn_array, thermal_rad_array del thermal_dn_path, thermal_dn_array, ts_bt_array # Interpolate/project/clip METRIC hourly/daily rasters if (calc_metric_ea_flag or calc_metric_wind_flag or calc_metric_etr_flag): logging.info('METRIC hourly/daily rasters') # Get bracketing hours from image acquisition time image_prev_dt = image.acq_datetime.replace( minute=0, second=0, microsecond=0) image_next_dt = image_prev_dt + timedelta(seconds=3600) # Get NLDAS properties from one of the images input_ws = os.path.join( metric_etr_input_ws, str(image_prev_dt.year)) try: input_path = [ os.path.join(input_ws, file_name) for file_name in os.listdir(input_ws) for match in [metric_hourly_re.match(file_name)] if (match and (image_prev_dt.strftime('%Y%m%d') == match.group('YYYYMMDD')))][0] except IndexError: logging.error(' No hourly file for {}'.format( image_prev_dt.strftime('%Y-%m-%d %H00'))) return False try: input_ds = gdal.Open(input_path) input_osr = drigo.raster_ds_osr(input_ds) # input_proj = drigo.osr_proj(input_osr) input_extent = drigo.raster_ds_extent(input_ds) input_cs = drigo.raster_ds_cellsize(input_ds, x_only=True) # input_geo = input_extent.geo(input_cs) input_x, input_y = input_extent.origin() input_ds = None except: logging.error(' Could not get default input image properties') logging.error(' {}'.format(input_path)) return False # Project Landsat scene extent to NLDAS GCS common_gcs_osr = common_osr.CloneGeogCS() common_gcs_extent = drigo.project_extent( common_extent, common_osr, common_gcs_osr, cellsize=env.cellsize) common_gcs_extent.buffer_extent(0.1) common_gcs_extent.adjust_to_snap( 'EXPAND', input_x, input_y, input_cs) # common_gcs_geo = common_gcs_extent.geo(input_cs) def metric_weather_func(output_raster, input_ws, input_re, prev_dt, next_dt, resample_method=gdal.GRA_NearestNeighbour, rounding_flag=False): """Interpolate/project/clip METRIC hourly rasters""" logging.debug(' Output: {}'.format(output_raster)) if os.path.isfile(output_raster): if overwrite_flag: logging.debug(' Overwriting output') dripy.remove_file(output_raster) else: logging.debug(' Skipping, file already exists ' + 'and overwrite is False') return False prev_ws = os.path.join(input_ws, str(prev_dt.year)) next_ws = os.path.join(input_ws, str(next_dt.year)) # Technically previous and next could come from different days # or even years, although this won't happen in the U.S. try: prev_path = [ os.path.join(prev_ws, input_name) for input_name in os.listdir(prev_ws) for input_match in [input_re.match(input_name)] if (input_match and (prev_dt.strftime('%Y%m%d') == input_match.group('YYYYMMDD')))][0] logging.debug(' Input prev: {}'.format(prev_path)) except IndexError: logging.error(' No previous hourly file') logging.error(' {}'.format(prev_dt)) return False try: next_path = [ os.path.join(next_ws, input_name) for input_name in os.listdir(next_ws) for input_match in [input_re.match(input_name)] if (input_match and (next_dt.strftime('%Y%m%d') == input_match.group('YYYYMMDD')))][0] logging.debug(' Input next: {}'.format(next_path)) except IndexError: logging.error(' No next hourly file') logging.error(' {}'.format(next_dt)) return False # Band numbers are 1's based prev_band = int(prev_dt.strftime('%H')) + 1 next_band = int(next_dt.strftime('%H')) + 1 logging.debug(' Input prev band: {}'.format(prev_band)) logging.debug(' Input next band: {}'.format(next_band)) # Read arrays prev_array = drigo.raster_to_array( prev_path, band=prev_band, mask_extent=common_gcs_extent, return_nodata=False) next_array = drigo.raster_to_array( next_path, band=next_band, mask_extent=common_gcs_extent, return_nodata=False) if not np.any(prev_array) or not np.any(next_array): logging.warning('\nWARNING: Input NLDAS array is all nodata\n') return None output_array = hourly_interpolate_func( prev_array, next_array, prev_dt, next_dt, image.acq_datetime) output_array = drigo.project_array( output_array, resample_method, input_osr, input_cs, common_gcs_extent, common_osr, env.cellsize, common_extent, output_nodata=None) # Apply common area mask output_array[~common_array] = np.nan # Reduce the file size by rounding to the nearest n digits if rounding_flag: output_array = np.around(output_array, rounding_digits) # Force output to 32-bit float drigo.array_to_raster( output_array.astype(np.float32), output_raster, output_geo=common_geo, output_proj=common_proj, stats_flag=stats_flag) del output_array return True # Ea - Project to Landsat scene after clipping if calc_metric_ea_flag: logging.info(' Hourly vapor pressure (Ea)') metric_weather_func( image.metric_ea_raster, metric_ea_input_ws, metric_hourly_re, image_prev_dt, image_next_dt, gdal.GRA_Bilinear, rounding_flag=True) # Wind - Project to Landsat scene after clipping if calc_metric_wind_flag: logging.info(' Hourly windspeed') metric_weather_func( image.metric_wind_raster, metric_wind_input_ws, metric_hourly_re, image_prev_dt, image_next_dt, gdal.GRA_NearestNeighbour, rounding_flag=False) # ETr - Project to Landsat scene after clipping if calc_metric_etr_flag: logging.info(' Hourly reference ET (ETr)') metric_weather_func( image.metric_etr_raster, metric_etr_input_ws, metric_hourly_re, image_prev_dt, image_next_dt, gdal.GRA_NearestNeighbour, rounding_flag=False) # ETr 24hr - Project to Landsat scene after clipping if calc_metric_etr_flag: logging.info(' Daily reference ET (ETr)') logging.debug(' Output: {}'.format( image.metric_etr_24hr_raster)) if (os.path.isfile(image.metric_etr_24hr_raster) and overwrite_flag): logging.debug(' Overwriting output') os.remove(image.metric_etr_24hr_raster) if not os.path.isfile(image.metric_etr_24hr_raster): etr_prev_ws = os.path.join( metric_etr_input_ws, str(image_prev_dt.year)) try: input_path = [ os.path.join(etr_prev_ws, file_name) for file_name in os.listdir(etr_prev_ws) for match in [metric_daily_re.match(file_name)] if (match and (image_prev_dt.strftime('%Y%m%d') == match.group('YYYYMMDD')))][0] logging.debug(' Input: {}'.format(input_path)) except IndexError: logging.error(' No daily file for {}'.format( image_prev_dt.strftime('%Y-%m-%d'))) return False output_array = drigo.raster_to_array( input_path, mask_extent=common_gcs_extent, return_nodata=False) output_array = drigo.project_array( output_array, gdal.GRA_NearestNeighbour, input_osr, input_cs, common_gcs_extent, common_osr, env.cellsize, common_extent, output_nodata=None) # Apply common area mask output_array[~common_array] = np.nan # Reduce the file size by rounding to the nearest n digits # output_array = np.around(output_array, rounding_digits) drigo.array_to_raster( output_array, image.metric_etr_24hr_raster, output_geo=common_geo, output_proj=common_proj, stats_flag=stats_flag) del output_array del input_path # Tair - Project to Landsat scene after clipping if calc_metric_tair_flag: logging.info(' Hourly air temperature (Tair)') metric_weather_func( image.metric_tair_raster, metric_tair_input_ws, metric_hourly_re, image_prev_dt, image_next_dt, gdal.GRA_NearestNeighbour, rounding_flag=False) # Cleanup del image_prev_dt, image_next_dt # Soil Water Balance if calc_swb_ke_flag: logging.info('Daily soil water balance') # Check if output file already exists logging.debug(' Ke: {}'.format(image.ke_raster)) if os.path.isfile(image.ke_raster): if overwrite_flag: logging.debug(' Overwriting output') dripy.remove_file(image.ke_raster) else: logging.debug(' Skipping, file already ' 'exists and overwrite is False') return False ke_array = et_common.raster_swb_func( image.acq_datetime, common_osr, env.cellsize, common_extent, awc_input_path, etr_input_ws, etr_input_re, ppt_input_ws, ppt_input_re, spinup_days=spinup_days, min_spinup_days=min_spinup_days) # Apply common area mask ke_array[~common_array] = np.nan # Reduce the file size by rounding to the nearest 2 digits np.around(ke_array, 2, out=ke_array) # Force output to 32-bit float drigo.array_to_raster( ke_array.astype(np.float32), image.ke_raster, output_geo=common_geo, output_proj=common_proj, stats_flag=stats_flag) # Remove Landsat TOA rasters if not keep_dn_flag: for landsat_item in dripy.build_file_list( image.orig_data_ws, image.image_name_re): os.remove(os.path.join(image.orig_data_ws, landsat_item)) return True
def main(grb_ws, ancillary_ws, output_ws, etr_flag=False, eto_flag=False, scene_list_path=None, start_dt=None, end_dt=None, times_str='', extent_path=None, output_extent=None, daily_flag=True, stats_flag=True, overwrite_flag=False): """Compute hourly ETr/ETo from NLDAS data Parameters ---------- grb_ws : str Folder of NLDAS GRB files. ancillary_ws : str Folder of ancillary rasters. output_ws : str Folder of output rasters. etr_flag : bool, optional If True, compute alfalfa reference ET (ETr). eto_flag : bool, optional If True, compute grass reference ET (ETo). scene_list_path : str, optional Landsat scene keep list file path. start_date : str, optional ISO format date (YYYY-MM-DD). end_date : str, optional ISO format date (YYYY-MM-DD). times : str, optional Comma separated values and/or ranges of UTC hours (i.e. "1, 2, 5-8"). Parsed with python_common.parse_int_set(). extent_path : str, optional File path defining the output extent. output_extent : list, optional Decimal degrees values defining output extent. daily_flag : bool, optional If True, save daily ETr/ETo sum raster (the default is True). stats_flag : bool, optional If True, compute raster statistics (the default is True). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). Returns ------- None """ logging.info('\nComputing NLDAS hourly ETr/ETo') np.seterr(invalid='ignore') # Compute ETr and/or ETo if not etr_flag and not eto_flag: logging.info(' ETo/ETr flag(s) not set, defaulting to ETr') etr_flag = True # Only process a specific hours if not times_str: time_list = range(0, 24, 1) else: time_list = list(_utils.parse_int_set(times_str)) time_list = ['{:02d}00'.format(t) for t in time_list] etr_folder = 'etr' eto_folder = 'eto' hour_fmt = '{}_{:04d}{:02d}{:02d}_hourly_nldas.img' # hour_fmt = '{}_{:04d}{:02d}{:02d}_{4:04d}_nldas.img' day_fmt = '{}_{:04d}{:02d}{:02d}_nldas.img' # input_fmt = 'NLDAS_FORA0125_H.A{:04d}{:02d}{:02d}.{}.002.grb' input_re = re.compile('NLDAS_FORA0125_H.A(?P<YEAR>\d{4})(?P<MONTH>\d{2})' + '(?P<DAY>\d{2}).(?P<TIME>\d{4}).002.grb$') # # Landsat Collection 1 Product ID # landsat_re = re.compile( # '^(?:LT04|LT05|LE07|LC08)_\w{4}_\d{3}\d{3}_(?P<DATE>\d{8})_' # '\w{8}_\w{2}_\w{2}') # Landsat Custom Scene ID landsat_re = re.compile('^(?:LT04|LT05|LE07|LC08)_\d{6}_(?P<DATE>\d{8})') # Assume NLDAS is NAD83 # input_epsg = 'EPSG:4269' # Ancillary raster paths mask_path = os.path.join(ancillary_ws, 'nldas_mask.img') elev_path = os.path.join(ancillary_ws, 'nldas_elev.img') lat_path = os.path.join(ancillary_ws, 'nldas_lat.img') lon_path = os.path.join(ancillary_ws, 'nldas_lon.img') # Process Landsat scene list and start/end input parameters if not scene_list_path and (not start_dt or not end_dt): logging.error( '\nERROR: A Landsat scene list or start/end dates must be set, ' 'exiting\n') return False if scene_list_path is not None and os.path.isfile(scene_list_path): # Build a date list from the Landsat scene keep list file logging.info('\nReading dates from scene keep list file') logging.info(' {}'.format(scene_list_path)) with open(scene_list_path) as input_f: keep_list = input_f.readlines() date_list = sorted([ dt.datetime.strptime(m.group('DATE'), '%Y%m%d').strftime('%Y-%m-%d') for image_id in keep_list for m in [landsat_re.match(image_id)] if m ]) logging.debug(' {}'.format(', '.join(date_list))) else: date_list = [] if start_dt and end_dt: logging.debug(' Start date: {}'.format(start_dt)) logging.debug(' End date: {}'.format(end_dt)) else: start_dt = dt.datetime.strptime(date_list[0], '%Y-%m-%d') end_dt = dt.datetime.strptime(date_list[-1], '%Y-%m-%d') # This allows GDAL to throw Python Exceptions # gdal.UseExceptions() # mem_driver = gdal.GetDriverByName('MEM') # Get the NLDAS spatial reference from the mask raster nldas_ds = gdal.Open(mask_path) nldas_osr = drigo.raster_ds_osr(nldas_ds) nldas_proj = drigo.osr_proj(nldas_osr) nldas_cs = drigo.raster_ds_cellsize(nldas_ds, x_only=True) nldas_extent = drigo.raster_ds_extent(nldas_ds) nldas_geo = nldas_extent.geo(nldas_cs) nldas_x, nldas_y = nldas_extent.origin() nldas_ds = None logging.debug(' Projection: {}'.format(nldas_proj)) logging.debug(' Cellsize: {}'.format(nldas_cs)) logging.debug(' Geo: {}'.format(nldas_geo)) logging.debug(' Extent: {}'.format(nldas_extent)) # Subset data to a smaller extent if output_extent is not None: logging.info('\nComputing subset extent & geo') logging.debug(' Extent: {}'.format(output_extent)) nldas_extent = drigo.Extent(output_extent) nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs) nldas_geo = nldas_extent.geo(nldas_cs) logging.debug(' Geo: {}'.format(nldas_geo)) logging.debug(' Extent: {}'.format(output_extent)) elif extent_path is not None: logging.info('\nComputing subset extent & geo') if not os.path.isfile(extent_path): logging.error('\nThe extent object does not exist, exiting\n' ' {}'.format(extent_path)) return False elif extent_path.lower().endswith('.shp'): nldas_extent = drigo.feature_path_extent(extent_path) extent_osr = drigo.feature_path_osr(extent_path) extent_cs = None else: nldas_extent = drigo.raster_path_extent(extent_path) extent_osr = drigo.raster_path_osr(extent_path) extent_cs = drigo.raster_path_cellsize(extent_path, x_only=True) nldas_extent = drigo.project_extent(nldas_extent, extent_osr, nldas_osr, extent_cs) nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs) nldas_geo = nldas_extent.geo(nldas_cs) logging.debug(' Geo: {}'.format(nldas_geo)) logging.debug(' Extent: {}'.format(nldas_extent)) logging.debug('') # Read the NLDAS mask array if present if mask_path and os.path.isfile(mask_path): mask_array, mask_nodata = drigo.raster_to_array( mask_path, mask_extent=nldas_extent, fill_value=0, return_nodata=True) mask_array = mask_array != mask_nodata else: mask_array = None # Read ancillary arrays (or subsets?) elev_array = drigo.raster_to_array(elev_path, mask_extent=nldas_extent, return_nodata=False) # pair_array = et_common.air_pressure_func(elev_array) lat_array = drigo.raster_to_array(lat_path, mask_extent=nldas_extent, return_nodata=False) lon_array = drigo.raster_to_array(lon_path, mask_extent=nldas_extent, return_nodata=False) # Hourly RefET functions expects lat/lon in radians lat_array *= (math.pi / 180) lon_array *= (math.pi / 180) # Build output folder etr_ws = os.path.join(output_ws, etr_folder) eto_ws = os.path.join(output_ws, eto_folder) if etr_flag and not os.path.isdir(etr_ws): os.makedirs(etr_ws) if eto_flag and not os.path.isdir(eto_ws): os.makedirs(eto_ws) # DEADBEEF - Instead of processing all available files, the following # code will process files for target dates # for input_dt in date_range(start_dt, end_dt + dt.timedelta(1)): # logging.info(input_dt.date()) # Iterate all available files and check dates if necessary # Each sub folder in the main folder has all imagery for 1 day # (in UTC time) # The path for each subfolder is the /YYYY/DOY errors = defaultdict(list) for root, folders, files in os.walk(grb_ws): root_split = os.path.normpath(root).split(os.sep) # If the year/doy is outside the range, skip if (re.match('\d{4}', root_split[-2]) and re.match('\d{3}', root_split[-1])): root_dt = dt.datetime.strptime( '{}_{}'.format(root_split[-2], root_split[-1]), '%Y_%j') logging.info('{}'.format(root_dt.date())) if ((start_dt is not None and root_dt < start_dt) or (end_dt is not None and root_dt > end_dt)): continue elif date_list and root_dt.date().isoformat() not in date_list: continue # If the year is outside the range, don't search subfolders elif re.match('\d{4}', root_split[-1]): root_year = int(root_split[-1]) logging.info('Year: {}'.format(root_year)) if ((start_dt is not None and root_year < start_dt.year) or (end_dt is not None and root_year > end_dt.year)): folders[:] = [] else: folders[:] = sorted(folders) continue else: continue logging.debug(' {}'.format(root)) # Start off assuming every file needs to be processed day_skip_flag = False # Build output folders if necessary etr_year_ws = os.path.join(etr_ws, str(root_dt.year)) eto_year_ws = os.path.join(eto_ws, str(root_dt.year)) if etr_flag and not os.path.isdir(etr_year_ws): os.makedirs(etr_year_ws) if eto_flag and not os.path.isdir(eto_year_ws): os.makedirs(eto_year_ws) # Build daily total paths etr_day_path = os.path.join( etr_year_ws, day_fmt.format('etr', root_dt.year, root_dt.month, root_dt.day)) eto_day_path = os.path.join( eto_year_ws, day_fmt.format('eto', root_dt.year, root_dt.month, root_dt.day)) etr_hour_path = os.path.join( etr_year_ws, hour_fmt.format('etr', root_dt.year, root_dt.month, root_dt.day)) eto_hour_path = os.path.join( eto_year_ws, hour_fmt.format('eto', root_dt.year, root_dt.month, root_dt.day)) # logging.debug(' {}'.format(etr_hour_path)) # If daily ETr/ETo files are present, day can be skipped if not overwrite_flag and daily_flag: if etr_flag and not os.path.isfile(etr_day_path): pass elif eto_flag and not os.path.isfile(eto_day_path): pass else: day_skip_flag = True # If the hour and daily files don't need to be made, skip the day if not overwrite_flag: if etr_flag and not os.path.isfile(etr_hour_path): pass elif eto_flag and not os.path.isfile(eto_hour_path): pass elif day_skip_flag: logging.debug(' File(s) already exist, skipping') continue # Create a single raster for each day with 24 bands # Each time step will be stored in a separate band if etr_flag: logging.debug(' {}'.format(etr_day_path)) drigo.build_empty_raster(etr_hour_path, band_cnt=24, output_dtype=np.float32, output_proj=nldas_proj, output_cs=nldas_cs, output_extent=nldas_extent, output_fill_flag=True) if eto_flag: logging.debug(' {}'.format(eto_day_path)) drigo.build_empty_raster(eto_hour_path, band_cnt=24, output_dtype=np.float32, output_proj=nldas_proj, output_cs=nldas_cs, output_extent=nldas_extent, output_fill_flag=True) # Sum all ETr/ETo images in each folder to generate a UTC day total etr_day_array = 0 eto_day_array = 0 # Process each hour file for input_name in sorted(files): logging.info(' {}'.format(input_name)) input_match = input_re.match(input_name) if input_match is None: logging.debug(' Regular expression didn\'t match, skipping') continue input_dt = dt.datetime(int(input_match.group('YEAR')), int(input_match.group('MONTH')), int(input_match.group('DAY'))) input_doy = int(input_dt.strftime('%j')) time_str = input_match.group('TIME') band_num = int(time_str[:2]) + 1 # if start_dt is not None and input_dt < start_dt: # continue # elif end_dt is not None and input_dt > end_dt: # continue # elif date_list and input_dt.date().isoformat() not in date_list: # continue if not daily_flag and time_str not in time_list: logging.debug(' Time not in list and not daily, skipping') continue input_path = os.path.join(root, input_name) logging.debug(' Time: {} {}'.format(input_dt.date(), time_str)) logging.debug(' Band: {}'.format(band_num)) # Determine band numbering/naming input_band_dict = grib_band_names(input_path) # Read input bands input_ds = gdal.Open(input_path) # Temperature should be in C for et_common.refet_hourly_func() if 'Temperature [K]' in input_band_dict.keys(): temp_band_units = 'K' temp_array = drigo.raster_ds_to_array( input_ds, band=input_band_dict['Temperature [K]'], mask_extent=nldas_extent, return_nodata=False) elif 'Temperature [C]' in input_band_dict.keys(): temp_band_units = 'C' temp_array = drigo.raster_ds_to_array( input_ds, band=input_band_dict['Temperature [C]'], mask_extent=nldas_extent, return_nodata=False) else: logging.error('Unknown Temperature units, skipping') logging.error(' {}'.format(input_band_dict.keys())) continue # DEADBEEF - Having issue with T appearing to be C but labeled as K # Try to determine temperature units from values temp_mean = float(np.nanmean(temp_array)) temp_units_dict = {20: 'C', 293: 'K'} temp_array_units = temp_units_dict[min( temp_units_dict, key=lambda x: abs(x - temp_mean))] if temp_array_units == 'K' and temp_band_units == 'K': logging.debug(' Converting temperature from K to C') temp_array -= 273.15 elif temp_array_units == 'C' and temp_band_units == 'C': pass elif temp_array_units == 'C' and temp_band_units == 'K': logging.debug(( ' Temperature units are K in the GRB band name, ' + 'but values appear to be C\n Mean temperature: {:.2f}\n' + ' Values will NOT be adjusted').format(temp_mean)) elif temp_array_units == 'K' and temp_band_units == 'C': logging.debug(( ' Temperature units are C in the GRB band name, ' + 'but values appear to be K\n Mean temperature: {:.2f}\n' + ' Values will be adjusted from K to C').format(temp_mean)) temp_array -= 273.15 try: sph_array = drigo.raster_ds_to_array( input_ds, band=input_band_dict['Specific humidity [kg/kg]'], mask_extent=nldas_extent, return_nodata=False) rs_array = drigo.raster_ds_to_array( input_ds, band=input_band_dict[ 'Downward shortwave radiation flux [W/m^2]'], mask_extent=nldas_extent, return_nodata=False) wind_u_array = drigo.raster_ds_to_array( input_ds, band=input_band_dict['u-component of wind [m/s]'], mask_extent=nldas_extent, return_nodata=False) wind_v_array = drigo.raster_ds_to_array( input_ds, band=input_band_dict['v-component of wind [m/s]'], mask_extent=nldas_extent, return_nodata=False) input_ds = None except KeyError as e: errors[input_path].append(e) logging.error(' KeyError: {} Skipping: {}'.format( e, input_ds.GetDescription())) continue rs_array *= 0.0036 # W m-2 to MJ m-2 hr-1 wind_array = np.sqrt(wind_u_array**2 + wind_v_array**2) del wind_u_array, wind_v_array # Compute vapor pressure from specific humidity pair_array = refet.calcs._air_pressure(elev=elev_array) ea_array = refet.calcs._actual_vapor_pressure(q=sph_array, pair=pair_array) refet_obj = refet.Hourly(tmean=temp_array, ea=ea_array, rs=rs_array, uz=wind_array, zw=10, elev=elev_array, lat=lat_array, lon=lon_array, doy=input_doy, time=int(time_str) / 100, method='asce') # ETr if etr_flag: etr_array = refet_obj.etr() if daily_flag: etr_day_array += etr_array if time_str in time_list: drigo.array_to_comp_raster(etr_array.astype(np.float32), etr_hour_path, band=band_num, stats_flag=False) del etr_array # ETo if eto_flag: eto_array = refet_obj.eto() if eto_flag and daily_flag: eto_day_array += eto_array if eto_flag and time_str in time_list: drigo.array_to_comp_raster(eto_array.astype(np.float32), eto_hour_path, band=band_num, stats_flag=False) del eto_array del temp_array, sph_array, rs_array, wind_array del pair_array, ea_array if stats_flag and etr_flag: drigo.raster_statistics(etr_hour_path) if stats_flag and eto_flag: drigo.raster_statistics(eto_hour_path) # Save the projected ETr/ETo as 32-bit floats if not day_skip_flag and daily_flag: if etr_flag: try: drigo.array_to_raster(etr_day_array.astype(np.float32), etr_day_path, output_geo=nldas_geo, output_proj=nldas_proj, stats_flag=stats_flag) except AttributeError: pass if eto_flag: try: drigo.array_to_raster(eto_day_array.astype(np.float32), eto_day_path, output_geo=nldas_geo, output_proj=nldas_proj, stats_flag=stats_flag) except AttributeError: pass del etr_day_array, eto_day_array if len(errors) > 0: logging.info('\nThe following errors were encountered:') for key, value in errors.items(): logging.error(' Filepath: {}, error: {}'.format(key, value)) logging.debug('\nScript Complete')
def main(netcdf_ws=os.getcwd(), ancillary_ws=os.getcwd(), output_ws=os.getcwd(), variables=['prcp'], daily_flag=False, monthly_flag=True, annual_flag=False, start_year=1981, end_year=2010, extent_path=None, output_extent=None, stats_flag=True, overwrite_flag=False): """Extract DAYMET temperature Parameters ---------- netcdf_ws : str Folder of DAYMET netcdf files. ancillary_ws : str Folder of ancillary rasters. output_ws : str Folder of output rasters. variables : list, optional DAYMET variables to download ('prcp', 'srad', 'vp', 'tmmn', 'tmmx'). Set as ['all'] to process all variables. daily_flag : bool, optional If True, compute daily (DOY) climatologies. monthly_flag : bool, optional If True, compute monthly climatologies. annual_flag : bool, optional If True, compute annual climatologies. start_year : int, optional Climatology start year. end_year : int, optional Climatology end year. extent_path : str, optional File path a raster defining the output extent. output_extent : list, optional Decimal degrees values defining output extent. stats_flag : bool, optional If True, compute raster statistics (the default is True). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). Returns ------- None """ logging.info('\nGenerating DAYMET climatologies') daily_fmt = 'daymet_{var}_30yr_normal_{doy:03d}.img' monthly_fmt = 'daymet_{var}_30yr_normal_{month:02d}.img' annual_fmt = 'daymet_{var}_30yr_normal.img' # daily_fmt = 'daymet_{var}_normal_{start}_{end}_{doy:03d}.img' # monthly_fmt = 'daymet_{var}_normal_{start}_{end}_{month:02d}.img' # annual_fmt = 'daymet_{var}_normal_{start}_{end}.img' # If a date is not set, process 1981-2010 climatology try: start_dt = dt.datetime(start_year, 1, 1) logging.debug(' Start date: {}'.format(start_dt)) except: start_dt = dt.datetime(1981, 1, 1) logging.info(' Start date: {}'.format(start_dt)) try: end_dt = dt.datetime(end_year, 12, 31) logging.debug(' End date: {}'.format(end_dt)) except: end_dt = dt.datetime(2010, 12, 31) logging.info(' End date: {}'.format(end_dt)) # Get DAYMET spatial reference from an ancillary raster mask_raster = os.path.join(ancillary_ws, 'daymet_mask.img') daymet_re = re.compile('daymet_v3_(?P<VAR>\w+)_(?P<YEAR>\d{4})_na.nc4$') # DAYMET rasters to extract var_full_list = ['prcp', 'tmmn', 'tmmx'] # data_full_list = ['prcp', 'srad', 'vp', 'tmmn', 'tmmx'] if not variables: logging.error('\nERROR: variables parameter is empty\n') sys.exit() elif type(variables) is not list: # DEADBEEF - I could try converting comma separated strings to lists? logging.warning('\nERROR: variables parameter must be a list\n') sys.exit() elif 'all' in variables: logging.error('\nDownloading all variables\n {}'.format( ','.join(var_full_list))) var_list = var_full_list[:] elif not set(variables).issubset(set(var_full_list)): logging.error('\nERROR: variables parameter is invalid\n {}'.format( variables)) sys.exit() else: var_list = variables[:] # Get extent/geo from mask raster daymet_ds = gdal.Open(mask_raster) daymet_osr = drigo.raster_ds_osr(daymet_ds) daymet_proj = drigo.osr_proj(daymet_osr) daymet_cs = drigo.raster_ds_cellsize(daymet_ds, x_only=True) daymet_extent = drigo.raster_ds_extent(daymet_ds) daymet_geo = daymet_extent.geo(daymet_cs) daymet_x, daymet_y = daymet_extent.origin() daymet_ds = None logging.debug(' Projection: {}'.format(daymet_proj)) logging.debug(' Cellsize: {}'.format(daymet_cs)) logging.debug(' Geo: {}'.format(daymet_geo)) logging.debug(' Extent: {}'.format(daymet_extent)) logging.debug(' Origin: {} {}'.format(daymet_x, daymet_y)) # Subset data to a smaller extent if output_extent is not None: logging.info('\nComputing subset extent & geo') logging.debug(' Extent: {}'.format(output_extent)) # Assume input extent is in decimal degrees output_extent = drigo.project_extent( drigo.Extent(output_extent), drigo.epsg_osr(4326), daymet_osr, 0.001) output_extent = drigo.intersect_extents([daymet_extent, output_extent]) output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs) output_geo = output_extent.geo(daymet_cs) logging.debug(' Geo: {}'.format(output_geo)) logging.debug(' Extent: {}'.format(output_extent)) elif extent_path is not None: logging.info('\nComputing subset extent & geo') output_extent = drigo.project_extent( drigo.raster_path_extent(extent_path), drigo.raster_path_osr(extent_path), daymet_osr, drigo.raster_path_cellsize(extent_path, x_only=True)) output_extent = drigo.intersect_extents([daymet_extent, output_extent]) output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs) output_geo = output_extent.geo(daymet_cs) logging.debug(' Geo: {}'.format(output_geo)) logging.debug(' Extent: {}'.format(output_extent)) else: output_extent = daymet_extent.copy() output_geo = daymet_geo[:] output_shape = output_extent.shape(cs=daymet_cs) xi, yi = drigo.array_geo_offsets(daymet_geo, output_geo, daymet_cs) output_rows, output_cols = output_extent.shape(daymet_cs) logging.debug(' Shape: {} {}'.format(output_rows, output_cols)) logging.debug(' Offsets: {} {} (x y)'.format(xi, yi)) # Process each variable for input_var in var_list: logging.info("\nVariable: {}".format(input_var)) # Rename variables to match cimis if input_var == 'prcp': output_var = 'ppt' else: output_var = input_var logging.debug("Output name: {}".format(output_var)) # Build output folder var_ws = os.path.join(output_ws, output_var) if not os.path.isdir(var_ws): os.makedirs(var_ws) # Build output arrays logging.debug(' Building arrays') if daily_flag: daily_sum = np.full( (365, output_shape[0], output_shape[1]), 0, np.float64) daily_count = np.full( (365, output_shape[0], output_shape[1]), 0, np.uint8) if monthly_flag: monthly_sum = np.full( (12, output_shape[0], output_shape[1]), 0, np.float64) monthly_count = np.full( (12, output_shape[0], output_shape[1]), 0, np.uint8) if monthly_flag: annual_sum = np.full( (output_shape[0], output_shape[1]), 0, np.float64) annual_count = np.full( (output_shape[0], output_shape[1]), 0, np.uint8) # Process each file/year separately for input_name in sorted(os.listdir(netcdf_ws)): logging.debug(" {}".format(input_name)) input_match = daymet_re.match(input_name) if not input_match: logging.debug(' Regular expression didn\'t match, skipping') continue elif input_match.group('VAR') != input_var: logging.debug(' Variable didn\'t match, skipping') continue year_str = input_match.group('YEAR') logging.info(" Year: {}".format(year_str)) year_int = int(year_str) year_days = int(dt.datetime(year_int, 12, 31).strftime('%j')) if start_dt is not None and year_int < start_dt.year: logging.debug(' Before start date, skipping') continue elif end_dt is not None and year_int > end_dt.year: logging.debug(' After end date, skipping') continue # Build input file path input_raster = os.path.join(netcdf_ws, input_name) if not os.path.isfile(input_raster): logging.debug( ' Input raster doesn\'t exist, skipping {}'.format( input_raster)) continue # Build output folder if daily_flag: daily_ws = os.path.join(var_ws, 'daily') if not os.path.isdir(daily_ws): os.makedirs(daily_ws) if monthly_flag: monthly_temp_sum = np.full( (12, output_shape[0], output_shape[1]), 0, np.float64) monthly_temp_count = np.full( (12, output_shape[0], output_shape[1]), 0, np.uint8) # Read in the DAYMET NetCDF file input_nc_f = netCDF4.Dataset(input_raster, 'r') # logging.debug(input_nc_f.variables) # Check all valid dates in the year year_dates = _utils.date_range( dt.datetime(year_int, 1, 1), dt.datetime(year_int + 1, 1, 1)) for date_dt in year_dates: logging.debug(' {}'.format(date_dt.date())) # if start_dt is not None and date_dt < start_dt: # logging.debug( # ' {} - before start date, skipping'.format( # date_dt.date())) # continue # elif end_dt is not None and date_dt > end_dt: # logging.debug(' {} - after end date, skipping'.format( # date_dt.date())) # continue # else: # logging.info(' {}'.format(date_dt.date())) doy = int(date_dt.strftime('%j')) doy_i = range(1, year_days + 1).index(doy) month_i = date_dt.month - 1 # Arrays are being read as masked array with a -9999 fill value # Convert to basic numpy array arrays with nan values try: input_ma = input_nc_f.variables[input_var][ doy_i, yi: yi + output_rows, xi: xi + output_cols] except IndexError: logging.info(' date not in netcdf, skipping') continue input_nodata = float(input_ma.fill_value) output_array = input_ma.data.astype(np.float32) output_array[output_array == input_nodata] = np.nan output_mask = np.isfinite(output_array) # Convert Kelvin to Celsius if input_var in ['tmax', 'tmin']: output_array -= 273.15 # Save values if daily_flag: daily_sum[doy_i, :, :] += output_array daily_count[doy_i, :, :] += output_mask if monthly_flag: monthly_temp_sum[month_i, :, :] += output_array monthly_temp_count[month_i, :, :] += output_mask if annual_flag: annual_sum[:, :] += output_array annual_count[:, :] += output_mask # Cleanup # del input_ds, input_array del input_ma, output_array, output_mask # Compute mean monthly for the year if monthly_flag: # Sum precipitation if input_var == 'prcp': monthly_sum += monthly_temp_sum else: monthly_sum += monthly_temp_sum / monthly_temp_count # Is this the right count? monthly_count += np.any(monthly_temp_count, axis=0) del monthly_temp_sum, monthly_temp_count input_nc_f.close() del input_nc_f # Save the projected climatology arrays if daily_flag: for doy_i in range(daily_sum.shape[0]): daily_name = daily_fmt.format( var=output_var, start=start_year, end=end_year, doy=doy_i + 1) daily_path = os.path.join(daily_ws, daily_name) drigo.array_to_raster( daily_sum[doy_i, :, :] / daily_count[doy_i, :, :], daily_path, output_geo=output_geo, output_proj=daymet_proj, stats_flag=stats_flag) del daily_sum, daily_count if monthly_flag: for month_i in range(monthly_sum.shape[0]): monthly_name = monthly_fmt.format( var=output_var, start=start_year, end=end_year, month=month_i + 1) monthly_path = os.path.join(var_ws, monthly_name) drigo.array_to_raster( monthly_sum[month_i, :, :] / monthly_count[month_i, :, :], monthly_path, output_geo=output_geo, output_proj=daymet_proj, stats_flag=stats_flag) del monthly_sum, monthly_count if annual_flag: annual_name = annual_fmt.format( var=output_var, start=start_year, end=end_year) annual_path = os.path.join(var_ws, annual_name) drigo.array_to_raster( annual_sum / annual_count, annual_path, output_geo=output_geo, output_proj=daymet_proj, stats_flag=stats_flag) del annual_sum, annual_count logging.debug('\nScript Complete')
def main(netcdf_ws=os.getcwd(), ancillary_ws=os.getcwd(), output_ws=os.getcwd(), variables=['prcp'], start_date=None, end_date=None, extent_path=None, output_extent=None, stats_flag=True, overwrite_flag=False): """Extract DAYMET temperature Parameters ---------- netcdf_ws : str Folder of DAYMET netcdf files. ancillary_ws : str Folder of ancillary rasters. output_ws : str Folder of output rasters. variables : list, optional DAYMET variables to download ('prcp', 'srad', 'vp', 'tmmn', 'tmmx'). Set as ['all'] to process all variables. start_date : str, optional ISO format date (YYYY-MM-DD). end_date : str, optional ISO format date (YYYY-MM-DD). extent_path : str, optional File path defining the output extent. output_extent : list, optional Decimal degrees values defining output extent. stats_flag : bool, optional If True, compute raster statistics (the default is True). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). Returns ------- None """ logging.info('\nExtracting DAYMET variables') # If a date is not set, process 2015 try: start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d') logging.debug(' Start date: {}'.format(start_dt)) except: start_dt = dt.datetime(2015, 1, 1) logging.info(' Start date: {}'.format(start_dt)) try: end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d') logging.debug(' End date: {}'.format(end_dt)) except: end_dt = dt.datetime(2015, 12, 31) logging.info(' End date: {}'.format(end_dt)) # Get DAYMET spatial reference from an ancillary raster mask_raster = os.path.join(ancillary_ws, 'daymet_mask.img') daymet_re = re.compile('daymet_v3_(?P<VAR>\w+)_(?P<YEAR>\d{4})_na.nc4$') # DAYMET rasters to extract var_full_list = ['prcp', 'srad', 'vp', 'tmmn', 'tmmx'] if not variables: logging.error('\nERROR: variables parameter is empty\n') sys.exit() elif type(variables) is not list: # DEADBEEF - I could try converting comma separated strings to lists? logging.warning('\nERROR: variables parameter must be a list\n') sys.exit() elif 'all' in variables: logging.error('\nDownloading all variables\n {}'.format( ','.join(var_full_list))) var_list = var_full_list[:] elif not set(variables).issubset(set(var_full_list)): logging.error( '\nERROR: variables parameter is invalid\n {}'.format(variables)) sys.exit() else: var_list = variables[:] # DAYMET band name dictionary # daymet_band_dict = dict() # daymet_band_dict['prcp'] = 'precipitation_amount' # daymet_band_dict['srad'] = 'surface_downwelling_shortwave_flux_in_air' # daymet_band_dict['sph'] = 'specific_humidity' # daymet_band_dict['tmin'] = 'air_temperature' # daymet_band_dict['tmax'] = 'air_temperature' # Get extent/geo from mask raster daymet_ds = gdal.Open(mask_raster) daymet_osr = drigo.raster_ds_osr(daymet_ds) daymet_proj = drigo.osr_proj(daymet_osr) daymet_cs = drigo.raster_ds_cellsize(daymet_ds, x_only=True) daymet_extent = drigo.raster_ds_extent(daymet_ds) daymet_geo = daymet_extent.geo(daymet_cs) daymet_x, daymet_y = daymet_extent.origin() daymet_ds = None logging.debug(' Projection: {}'.format(daymet_proj)) logging.debug(' Cellsize: {}'.format(daymet_cs)) logging.debug(' Geo: {}'.format(daymet_geo)) logging.debug(' Extent: {}'.format(daymet_extent)) logging.debug(' Origin: {} {}'.format(daymet_x, daymet_y)) # Subset data to a smaller extent if output_extent is not None: logging.info('\nComputing subset extent & geo') logging.debug(' Extent: {}'.format(output_extent)) # Assume input extent is in decimal degrees output_extent = drigo.project_extent(drigo.Extent(output_extent), drigo.epsg_osr(4326), daymet_osr, 0.001) output_extent = drigo.intersect_extents([daymet_extent, output_extent]) output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs) output_geo = output_extent.geo(daymet_cs) logging.debug(' Geo: {}'.format(output_geo)) logging.debug(' Extent: {}'.format(output_extent)) elif extent_path is not None: logging.info('\nComputing subset extent & geo') if extent_path.lower().endswith('.shp'): output_extent = drigo.feature_path_extent(extent_path) extent_osr = drigo.feature_path_osr(extent_path) extent_cs = None else: output_extent = drigo.raster_path_extent(extent_path) extent_osr = drigo.raster_path_osr(extent_path) extent_cs = drigo.raster_path_cellsize(extent_path, x_only=True) output_extent = drigo.project_extent(output_extent, extent_osr, daymet_osr, extent_cs) output_extent = drigo.intersect_extents([daymet_extent, output_extent]) output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs) output_geo = output_extent.geo(daymet_cs) logging.debug(' Geo: {}'.format(output_geo)) logging.debug(' Extent: {}'.format(output_extent)) else: output_extent = daymet_extent.copy() output_geo = daymet_geo[:] # output_shape = output_extent.shape(cs=daymet_cs) xi, yi = drigo.array_geo_offsets(daymet_geo, output_geo, daymet_cs) output_rows, output_cols = output_extent.shape(daymet_cs) logging.debug(' Shape: {} {}'.format(output_rows, output_cols)) logging.debug(' Offsets: {} {} (x y)'.format(xi, yi)) # Process each variable for input_var in var_list: logging.info("\nVariable: {}".format(input_var)) # Rename variables to match cimis if input_var == 'prcp': output_var = 'ppt' else: output_var = input_var # Build output folder var_ws = os.path.join(output_ws, output_var) if not os.path.isdir(var_ws): os.makedirs(var_ws) # Process each file in the input workspace for input_name in sorted(os.listdir(netcdf_ws)): logging.debug("{}".format(input_name)) input_match = daymet_re.match(input_name) if not input_match: logging.debug(' Regular expression didn\'t match, skipping') continue elif input_match.group('VAR') != input_var: logging.debug(' Variable didn\'t match, skipping') continue year_str = input_match.group('YEAR') logging.info(" Year: {}".format(year_str)) year_int = int(year_str) year_days = int(dt.datetime(year_int, 12, 31).strftime('%j')) if start_dt is not None and year_int < start_dt.year: logging.debug(' Before start date, skipping') continue elif end_dt is not None and year_int > end_dt.year: logging.debug(' After end date, skipping') continue # Build input file path input_raster = os.path.join(netcdf_ws, input_name) # if not os.path.isfile(input_raster): # logging.debug( # ' Input raster doesn\'t exist, skipping {}'.format( # input_raster)) # continue # Build output folder output_year_ws = os.path.join(var_ws, year_str) if not os.path.isdir(output_year_ws): os.makedirs(output_year_ws) # Read in the DAYMET NetCDF file input_nc_f = netCDF4.Dataset(input_raster, 'r') # logging.debug(input_nc_f.variables) # Check all valid dates in the year year_dates = _utils.date_range(dt.datetime(year_int, 1, 1), dt.datetime(year_int + 1, 1, 1)) for date_dt in year_dates: if start_dt is not None and date_dt < start_dt: logging.debug(' {} - before start date, skipping'.format( date_dt.date())) continue elif end_dt is not None and date_dt > end_dt: logging.debug(' {} - after end date, skipping'.format( date_dt.date())) continue else: logging.info(' {}'.format(date_dt.date())) output_path = os.path.join( output_year_ws, '{}_{}_daymet.img'.format(output_var, date_dt.strftime('%Y%m%d'))) if os.path.isfile(output_path): logging.debug(' {}'.format(output_path)) if not overwrite_flag: logging.debug(' File already exists, skipping') continue else: logging.debug( ' File already exists, removing existing') os.remove(output_path) doy = int(date_dt.strftime('%j')) doy_i = range(1, year_days + 1).index(doy) # Arrays are being read as masked array with a fill value of -9999 # Convert to basic numpy array arrays with nan values try: input_ma = input_nc_f.variables[input_var][doy_i, yi:yi + output_rows, xi:xi + output_cols] except IndexError: logging.info(' date not in netcdf, skipping') continue input_nodata = float(input_ma.fill_value) output_array = input_ma.data.astype(np.float32) output_array[output_array == input_nodata] = np.nan # Convert Kelvin to Celsius if input_var in ['tmax', 'tmin']: output_array -= 273.15 # Save the array as 32-bit floats drigo.array_to_raster(output_array.astype(np.float32), output_path, output_geo=output_geo, output_proj=daymet_proj, stats_flag=stats_flag) del input_ma, output_array input_nc_f.close() del input_nc_f logging.debug('\nScript Complete')
def pixel_rating(image_ws, ini_path, bs=None, stats_flag=False, overwrite_flag=None): """Calculate pixel rating Parameters ---------- image_ws : str Image folder path. ini_path : str Pixel regions config file path. bs : int, optional Processing block size (the default is None). If set, this blocksize parameter will be used instead of the value in the INI file. stats_flag : bool, optional if True, compute raster statistics (the default is False). ovewrite_flag : bool, optional If True, overwrite existing files (the default is None). Returns ------- None """ logging.info('Generating suggested hot/cold pixel regions') log_fmt = ' {:<18s} {}' env = drigo.env image = et_image.Image(image_ws, env) np.seterr(invalid='ignore') # # Check that image_ws is valid # image_id_re = re.compile( # '^(LT04|LT05|LE07|LC08)_(?:\w{4})_(\d{3})(\d{3})_' # '(\d{4})(\d{2})(\d{2})_(?:\d{8})_(?:\d{2})_(?:\w{2})$') # if not os.path.isdir(image_ws) or not image_id_re.match(image_id): # logging.error('\nERROR: Image folder is invalid or does not exist\n') # return False # Folder Paths region_ws = os.path.join(image_ws, 'PIXEL_REGIONS') # Open config file config = dripy.open_ini(ini_path) # Get input parameters logging.debug(' Reading Input File') # Arrays are processed by block if bs is None: bs = dripy.read_param('block_size', 1024, config) logging.info(log_fmt.format('Block Size:', bs)) # Raster pyramids/statistics pyramids_flag = dripy.read_param('pyramids_flag', False, config) if pyramids_flag: gdal.SetConfigOption('HFA_USE_RRD', 'YES') if stats_flag is None: stats_flag = dripy.read_param('statistics_flag', False, config) # Overwrite if overwrite_flag is None: overwrite_flag = dripy.read_param('overwrite_flag', True, config) # Check that common_area raster exists if not os.path.isfile(image.common_area_raster): logging.error( '\nERROR: A common area raster was not found.' + '\nERROR: Please rerun prep tool to build these files.\n' + ' {}\n'.format(image.common_area_raster)) sys.exit() # Use common_area to set mask parameters common_ds = gdal.Open(image.common_area_raster) # env.mask_proj = raster_ds_proj(common_ds) env.mask_geo = drigo.raster_ds_geo(common_ds) env.mask_rows, env.mask_cols = drigo.raster_ds_shape(common_ds) env.mask_extent = drigo.geo_extent(env.mask_geo, env.mask_rows, env.mask_cols) env.mask_array = drigo.raster_ds_to_array(common_ds)[0] env.mask_path = image.common_area_raster env.snap_osr = drigo.raster_path_osr(image.common_area_raster) env.snap_proj = env.snap_osr.ExportToWkt() env.cellsize = drigo.raster_path_cellsize(image.common_area_raster)[0] common_ds = None logging.debug(' {:<18s} {}'.format('Mask Extent:', env.mask_extent)) # Read Pixel Regions config file # Currently there is no code to support applying an NLCD mask apply_nlcd_mask = False # apply_nlcd_mask = dripy.read_param('apply_nlcd_mask', False, config) apply_cdl_ag_mask = dripy.read_param('apply_cdl_ag_mask', False, config) apply_field_mask = dripy.read_param('apply_field_mask', False, config) apply_ndwi_mask = dripy.read_param('apply_ndwi_mask', True, config) apply_ndvi_mask = dripy.read_param('apply_ndvi_mask', True, config) # Currently the code to apply a study area mask is commented out # apply_study_area_mask = dripy.read_param( # 'apply_study_area_mask', False, config) albedo_rating_flag = dripy.read_param('albedo_rating_flag', True, config) nlcd_rating_flag = dripy.read_param('nlcd_rating_flag', True, config) ndvi_rating_flag = dripy.read_param('ndvi_rating_flag', True, config) ts_rating_flag = dripy.read_param('ts_rating_flag', True, config) ke_rating_flag = dripy.read_param('ke_rating_flag', False, config) # if apply_study_area_mask: # study_area_path = config.get('INPUTS', 'study_area_path') if apply_nlcd_mask or nlcd_rating_flag: nlcd_raster = config.get('INPUTS', 'landuse_raster') if apply_cdl_ag_mask: cdl_ag_raster = config.get('INPUTS', 'cdl_ag_raster') cdl_buffer_cells = dripy.read_param('cdl_buffer_cells', 0, config) cdl_ag_eroded_name = dripy.read_param('cdl_ag_eroded_name', 'cdl_ag_eroded_{}.img', config) if apply_field_mask: field_raster = config.get('INPUTS', 'fields_raster') cold_rating_pct = dripy.read_param('cold_percentile', 99, config) hot_rating_pct = dripy.read_param('hot_percentile', 99, config) # min_cold_rating_score = dripy.read_param('min_cold_rating_score', 0.3, config) # min_hot_rating_score = dripy.read_param('min_hot_rating_score', 0.3, config) ts_bin_count = int(dripy.read_param('ts_bin_count', 10, config)) if 100 % ts_bin_count != 0: logging.warning( 'WARNING: ts_bins_count of {} is not a divisor ' + 'of 100. Using default ts_bins_count = 4'.format(ts_bin_count)) ts_bin_count = 10 bin_size = 1. / (ts_bin_count - 1) hot_rating_values = np.arange(0., 1. + bin_size, step=bin_size) cold_rating_values = hot_rating_values[::-1] # Input raster paths r_fmt = '.img' if 'Landsat' in image.type: albedo_raster = image.albedo_sur_raster ndvi_raster = image.ndvi_toa_raster ndwi_raster = image.ndwi_toa_raster ts_raster = image.ts_raster ke_raster = image.ke_raster # Check config file input paths # if apply_study_area_mask and not os.path.isfile(study_area_path): # logging.error( # ('\nERROR: The study area shapefile {} does ' + # 'not exist\n').format(study_area_path)) # sys.exit() if ((apply_nlcd_mask or nlcd_rating_flag) and not os.path.isfile(nlcd_raster)): logging.error(('\nERROR: The NLCD raster {} does ' + 'not exist\n').format(nlcd_raster)) sys.exit() if apply_cdl_ag_mask and not os.path.isfile(cdl_ag_raster): logging.error(('\nERROR: The CDL Ag raster {} does ' + 'not exist\n').format(cdl_ag_raster)) sys.exit() if apply_field_mask and not os.path.isfile(field_raster): logging.error(('\nERROR: The field raster {} does ' + 'not exist\n').format(field_raster)) sys.exit() if (not (isinstance(cold_rating_pct, (int, float)) and (0 <= cold_rating_pct <= 100))): logging.error( '\nERROR: cold_percentile must be a value between 0 and 100\n') sys.exit() if (not (isinstance(hot_rating_pct, (int, float)) and (0 <= hot_rating_pct <= 100))): logging.error( '\nERROR: hot_percentile must be a value between 0 and 100\n') sys.exit() # Set raster names raster_dict = dict() # Output Rasters raster_dict['region_mask'] = os.path.join(region_ws, 'region_mask' + r_fmt) raster_dict['cold_rating'] = os.path.join(region_ws, 'cold_pixel_rating' + r_fmt) raster_dict['hot_rating'] = os.path.join(region_ws, 'hot_pixel_rating' + r_fmt) raster_dict['cold_sugg'] = os.path.join(region_ws, 'cold_pixel_suggestion' + r_fmt) raster_dict['hot_sugg'] = os.path.join(region_ws, 'hot_pixel_suggestion' + r_fmt) # Read pixel region raster flags save_dict = dict() save_dict['region_mask'] = dripy.read_param('save_region_mask_flag', False, config) save_dict['cold_rating'] = dripy.read_param('save_rating_rasters_flag', False, config) save_dict['hot_rating'] = dripy.read_param('save_rating_rasters_flag', False, config) save_dict['cold_sugg'] = dripy.read_param('save_suggestion_rasters_flag', True, config) save_dict['hot_sugg'] = dripy.read_param('save_suggestion_rasters_flag', True, config) # Output folder if not os.path.isdir(region_ws): os.mkdir(region_ws) # Remove existing files if necessary region_ws_file_list = [ os.path.join(region_ws, item) for item in os.listdir(region_ws) ] if overwrite_flag and region_ws_file_list: for raster_path in raster_dict.values(): if raster_path in region_ws_file_list: dripy.remove_file(raster_path) # Check scene specific input paths if apply_ndwi_mask and not os.path.isfile(ndwi_raster): logging.error( 'ERROR: NDWI raster does not exist\n {}'.format(ndwi_raster)) sys.exit() elif apply_ndvi_mask and not os.path.isfile(ndvi_raster): logging.error( 'ERROR: NDVI raster does not exist\n {}'.format(ndvi_raster)) sys.exit() elif ke_rating_flag and not os.path.isfile(ke_raster): logging.error( ('ERROR: The Ke raster does not exist\n {}').format(ke_raster)) sys.exit() # Remove existing and build new empty rasters if necessary # If processing by block, rating rasters must be built logging.debug('\nBuilding empty rasters') for name, save_flag in sorted(save_dict.items()): if save_flag and 'rating' in name: drigo.build_empty_raster(raster_dict[name], 1, np.float32) elif save_flag: drigo.build_empty_raster(raster_dict[name], 1, np.uint8, output_nodata=0) if apply_cdl_ag_mask: logging.info('Building CDL ag mask') cdl_array = drigo.raster_to_array(cdl_ag_raster, mask_extent=env.mask_extent, return_nodata=False) if cdl_buffer_cells > 0: logging.info(' Eroding CDL by {} cells'.format(cdl_buffer_cells)) structure_array = np.ones((cdl_buffer_cells, cdl_buffer_cells), dtype=np.int) # Deadbeef - This could blow up in memory on bigger rasters cdl_array = ndimage.binary_erosion( cdl_array, structure_array).astype(structure_array.dtype) cdl_ag_eroded_raster = os.path.join( image.support_ws, cdl_ag_eroded_name.format(cdl_buffer_cells)) drigo.array_to_raster(cdl_array, cdl_ag_eroded_raster, output_geo=env.mask_geo, output_proj=env.snap_proj, mask_array=env.mask_array, output_nodata=0, stats_flag=False) cdl_array = None del cdl_array # Build region mask logging.debug('Building region mask') region_mask = np.copy(env.mask_array).astype(np.bool) if apply_field_mask: field_mask, field_nodata = drigo.raster_to_array( field_raster, mask_extent=env.mask_extent, return_nodata=True) region_mask &= field_mask != field_nodata del field_mask, field_nodata if apply_ndwi_mask: ndwi_array = drigo.raster_to_array(ndwi_raster, 1, mask_extent=env.mask_extent, return_nodata=False) region_mask &= ndwi_array > 0.0 del ndwi_array if apply_ndvi_mask: ndvi_array = drigo.raster_to_array(ndvi_raster, 1, mask_extent=env.mask_extent, return_nodata=False) region_mask &= ndvi_array > 0.12 del ndvi_array if apply_cdl_ag_mask: cdl_array, cdl_nodata = drigo.raster_to_array( cdl_ag_eroded_raster, mask_extent=env.mask_extent, return_nodata=True) region_mask &= cdl_array != cdl_nodata del cdl_array, cdl_nodata if save_dict['region_mask']: drigo.array_to_raster(region_mask, raster_dict['region_mask'], stats_flag=False) # Initialize rating arrays # This needs to be done before the ts_rating if block cold_rating_array = np.ones(env.mask_array.shape, dtype=np.float32) hot_rating_array = np.ones(env.mask_array.shape, dtype=np.float32) cold_rating_array[~region_mask] = np.nan hot_rating_array[~region_mask] = np.nan # Temperature pixel rating - grab the max and min value for the entire # Ts image in a memory safe way by using gdal_common blocks # The following is a percentile based approach if ts_rating_flag: logging.debug('Computing Ts percentile rating') ts_array = drigo.raster_to_array(ts_raster, mask_extent=env.mask_extent, return_nodata=False) ts_array[~region_mask] = np.nan percentiles = list( range(0, (100 + ts_bin_count), int(100 / ts_bin_count))) ts_score_value = 1. / (ts_bin_count - 1) hot_rating_values = np.arange(0, (1. + ts_score_value), step=ts_score_value)[:ts_bin_count] cold_rating_values = hot_rating_values[::-1] ts_percentile_array = np.percentile(ts_array[np.isfinite(ts_array)], percentiles) # Deprecated - per SciPy help this function will become obsolete # ts_percentile_array = stats.scoreatpercentile( # ts_array[np.isfinite(ts_array)], percentiles) for bins_i in range(len(ts_percentile_array))[:-1]: bool_array = ((ts_array > ts_percentile_array[bins_i]) & (ts_array <= ts_percentile_array[bins_i + 1])) cold_rating_array[bool_array] = cold_rating_values[bins_i] hot_rating_array[bool_array] = hot_rating_values[bins_i] # drigo.array_to_raster(cold_rating_array, raster_dict['cold_rating']) # drigo.array_to_raster(hot_rating_array, raster_dict['hot_rating']) # Cleanup del ts_array, ts_percentile_array del cold_rating_values, hot_rating_values del ts_score_value, percentiles # Process by block logging.info('\nProcessing by block') logging.debug(' Mask cols/rows: {}/{}'.format(env.mask_cols, env.mask_rows)) for b_i, b_j in drigo.block_gen(env.mask_rows, env.mask_cols, bs): logging.debug(' Block y: {:5d} x: {:5d}'.format(b_i, b_j)) block_data_mask = drigo.array_to_block(env.mask_array, b_i, b_j, bs).astype(np.bool) # block_nodata_mask = ~block_data_mask block_rows, block_cols = block_data_mask.shape block_geo = drigo.array_offset_geo(env.mask_geo, b_j, b_i) block_extent = drigo.geo_extent(block_geo, block_rows, block_cols) logging.debug(' Block rows: {} cols: {}'.format( block_rows, block_cols)) # logging.debug(' Block extent: {}'.format(block_extent)) # logging.debug(' Block geo: {}'.format(block_geo)) # Don't skip empty blocks since block rating needs to be written # back to the array at the end of the block loop block_region_mask = drigo.array_to_block(region_mask, b_i, b_j, bs) if not np.any(block_region_mask): logging.debug(' Empty block') block_empty_flag = True else: block_empty_flag = False # New style continuous pixel weighting cold_rating_block = drigo.array_to_block(cold_rating_array, b_i, b_j, bs) hot_rating_block = drigo.array_to_block(hot_rating_array, b_i, b_j, bs) # Rating arrays already have region_mask set # cold_rating_block = np.ones(block_region_mask.shape, dtype=np.float32) # hot_rating_block = np.ones(block_region_mask.shape, dtype=np.float32) # cold_rating_block[~block_region_mask] = np.nan # hot_rating_block[~block_region_mask] = np.nan # del block_region_mask if ndvi_rating_flag and not block_empty_flag: # NDVI based rating ndvi_array = drigo.raster_to_array(ndvi_raster, 1, mask_extent=block_extent, return_nodata=False) # Don't let NDVI be negative ndvi_array.clip(0., 0.833, out=ndvi_array) # ndvi_array.clip(0.001, 0.833, out=ndvi_array) cold_rating_block *= ndvi_array cold_rating_block *= 1.20 ndvi_mask = (ndvi_array > 0) # DEADBEEF - Can this calculation be masked to only NDVI > 0? ndvi_mask = ndvi_array > 0 hot_rating_block[ndvi_mask] *= stats.norm.pdf( np.log(ndvi_array[ndvi_mask]), math.log(0.15), 0.5) hot_rating_block[ndvi_mask] *= 1.25 del ndvi_mask # hot_rating_block *= stats.norm.pdf( # np.log(ndvi_array), math.log(0.15), 0.5) # hot_rating_block *= 1.25 # cold_rating_block.clip(0., 1., out=cold_rating_block) # hot_rating_block.clip(0., 1., out=hot_rating_block) del ndvi_array if albedo_rating_flag and not block_empty_flag: # Albdo based rating albedo_array = drigo.raster_to_array(albedo_raster, 1, mask_extent=block_extent, return_nodata=False) albedo_cold_pdf = stats.norm.pdf(albedo_array, 0.21, 0.03) albedo_hot_pdf = stats.norm.pdf(albedo_array, 0.21, 0.06) del albedo_array cold_rating_block *= albedo_cold_pdf cold_rating_block *= 0.07 hot_rating_block *= albedo_hot_pdf hot_rating_block *= 0.15 # cold_rating_block.clip(0., 1., out=cold_rating_block) # hot_rating_block.clip(0., 1., out=hot_rating_block) del albedo_cold_pdf, albedo_hot_pdf if nlcd_rating_flag and not block_empty_flag: # NLCD based weighting, this could be CDL instead? nlcd_array = nlcd_rating( drigo.raster_to_array(nlcd_raster, 1, mask_extent=block_extent, return_nodata=False)) cold_rating_block *= nlcd_array hot_rating_block *= nlcd_array del nlcd_array if ke_rating_flag and not block_empty_flag: # SWB Ke based rating ke_array = drigo.raster_to_array(ke_raster, 1, mask_extent=block_extent, return_nodata=False) # Don't let Ke be negative ke_array.clip(0., 1., out=ke_array) # Assumption, lower Ke is better for selecting the hot pixel # As the power (2) decreases and approaches 1, # the relationship gets more linear # cold_rating_block *= (1 - ke_array ** 2) # hot_rating_block *= (1 - ke_array ** 1.5) # Linear inverse # cold_rating_block *= (1. - ke_array) hot_rating_block *= (1. - ke_array) # cold_rating_block.clip(0., 1., out=cold_rating_block) # hot_rating_block.clip(0., 1., out=hot_rating_block) del ke_array # Clearness # clearness = 1.0 # cold_rating *= clearness # hot_rating *= clearness # Reset nan values # cold_rating_block[~region_mask] = np.nan # hot_rating_block[~region_mask] = np.nan # Save rating values cold_rating_array = drigo.block_to_array(cold_rating_block, cold_rating_array, b_i, b_j, bs) hot_rating_array = drigo.block_to_array(hot_rating_block, hot_rating_array, b_i, b_j, bs) # Save rating rasters if save_dict['cold_rating']: drigo.block_to_raster(cold_rating_block, raster_dict['cold_rating'], b_i, b_j, bs) if save_dict['hot_rating']: drigo.block_to_raster(hot_rating_block, raster_dict['hot_rating'], b_i, b_j, bs) # Save rating values cold_rating_array = drigo.block_to_array(cold_rating_block, cold_rating_array, b_i, b_j, bs) hot_rating_array = drigo.block_to_array(hot_rating_block, hot_rating_array, b_i, b_j, bs) del cold_rating_block, hot_rating_block # Select pixels above target percentile # Only build suggestion arrays if saving logging.debug('Building suggested pixel rasters') if save_dict['cold_sugg']: cold_rating_score = float( np.percentile(cold_rating_array[np.isfinite(cold_rating_array)], cold_rating_pct)) # Deprecated - per SciPy help this function will become obsolete # cold_rating_score = float(stats.scoreatpercentile( # cold_rating_array[np.isfinite(cold_rating_array)], # cold_rating_pct)) # cold_rating_array, cold_rating_nodata = drigo.raster_to_array( # raster_dict['cold_rating'], 1, mask_extent=env.mask_extent) # if cold_rating_score < float(min_cold_rating_score): # logging.error(('ERROR: The cold_rating_score ({}) is less ' + # 'than the min_cold_rating_score ({})').format( # cold_rating_score, min_cold_rating_score)) # sys.exit() cold_sugg_mask = cold_rating_array >= cold_rating_score drigo.array_to_raster(cold_sugg_mask, raster_dict['cold_sugg'], stats_flag=stats_flag) logging.debug(' Cold Percentile: {}'.format(cold_rating_pct)) logging.debug(' Cold Score: {:.6f}'.format(cold_rating_score)) logging.debug(' Cold Pixels: {}'.format(np.sum(cold_sugg_mask))) del cold_sugg_mask, cold_rating_array if save_dict['hot_sugg']: hot_rating_score = float( np.percentile(hot_rating_array[np.isfinite(hot_rating_array)], hot_rating_pct)) # Deprecated - per SciPy help this function will become obsolete # hot_rating_score = float(stats.scoreatpercentile( # hot_rating_array[np.isfinite(hot_rating_array)], # hot_rating_pct)) # hot_rating_array, hot_rating_nodata = drigo.raster_to_array( # raster_dict['hot_rating'], 1, mask_extent=env.mask_extent) # if hot_rating_score < float(min_hot_rating_score): # logging.error(('ERROR: The hot_rating_array ({}) is less ' + # 'than the min_hot_rating_score ({})').format( # hot_rating_array, min_hot_rating_score)) # sys.exit() hot_sugg_mask = hot_rating_array >= hot_rating_score drigo.array_to_raster(hot_sugg_mask, raster_dict['hot_sugg'], stats_flag=stats_flag) logging.debug(' Hot Percentile: {}'.format(hot_rating_pct)) logging.debug(' Hot Score: {:.6f}'.format(hot_rating_score)) logging.debug(' Hot Pixels: {}'.format(np.sum(hot_sugg_mask))) del hot_sugg_mask, hot_rating_array # Raster Statistics if stats_flag: logging.info('Calculating Statistics') for name, save_flag in save_dict.items(): if save_flag: drigo.raster_statistics(raster_dict[name]) # Raster Pyramids if pyramids_flag: logging.info('Building Pyramids') for name, save_flag in save_dict.items(): if save_flag: drigo.raster_pyramids(raster_dict[name])
def main(ancillary_ws=os.getcwd(), zero_elev_nodata_flag=False, overwrite_flag=False): """Process DAYMET ancillary data Parameters ---------- ancillary_ws : str Folder of ancillary rasters. zero_elev_nodata_flag : bool, optional If True, set elevation nodata values to 0 (the default is False). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). Returns ------- None """ logging.info('\nProcess DAYMET ancillary rasters') # Site URL # ancillary_url = 'http://daymet.ornl.gov/files/ancillary_files.tgz' # Build output workspace if it doesn't exist if not os.path.isdir(ancillary_ws): os.makedirs(ancillary_ws) # Input paths # ancillary_targz = os.path.join( # ancillary_ws, os.path.basename(ancillary_url)) # dem_nc = os.path.join(ancillary_ws, 'dem_data.nc') # mask_nc = os.path.join(ancillary_ws, 'mask_data.nc') # Output paths dem_raster = os.path.join(ancillary_ws, 'daymet_elev.img') lat_raster = os.path.join(ancillary_ws, 'daymet_lat.img') lon_raster = os.path.join(ancillary_ws, 'daymet_lon.img') # mask_raster = os.path.join(ancillary_ws, 'daymet_mask.img') # Spatial reference parameters daymet_proj4 = ( "+proj=lcc +datum=WGS84 +lat_1=25 n " "+lat_2=60n +lat_0=42.5n +lon_0=100w") daymet_osr = drigo.proj4_osr(daymet_proj4) daymet_osr.MorphToESRI() daymet_proj = daymet_osr.ExportToWkt() daymet_cs = 1000 # daymet_nodata = -9999 # For now, hardcode the DAYMET extent/geo snap_xmin, snap_ymin = -4560750, -3090500 daymet_rows, daymet_cols = 8075, 7814 # snap_xmin, snap_ymin = -4659000, -3135000 # daymet_rows, daymet_cols = 8220, 8011 # daymet_geo = ( # snap_xmin, daymet_cs, 0., # snap_ymin + daymet_cs * daymet_rows, 0., -daymet_cs) daymet_extent = drigo.Extent([ snap_xmin, snap_ymin, snap_xmin + daymet_cs * daymet_cols, snap_ymin + daymet_cs * daymet_rows]) daymet_geo = daymet_extent.geo(daymet_cs) logging.debug(" Extent: {}".format(daymet_extent)) logging.debug(" Geo: {}".format(daymet_geo)) # logging.debug(" Cellsize: {}".format(daymet_cs)) # logging.debug(" Shape: {}".format(daymet_extent.shape(daymet_cs))) # # Download the ancillary raster tar.gz # if overwrite_flag or not os.path.isfile(ancillary_targz): # logging.info('\nDownloading ancillary tarball files') # logging.info(" {}".format(os.path.basename(ancillary_url))) # logging.debug(" {}".format(ancillary_url)) # logging.debug(" {}".format(ancillary_targz)) # url_download(ancillary_url, ancillary_targz) # try: # urllib.urlretrieve(ancillary_url, ancillary_targz) # except: # logging.error(" ERROR: {}\n FILE: {}".format( # sys.exc_info()[0], ancillary_targz)) # os.remove(ancillary_targz) # # Extract the ancillary rasters # ancillary_list = [dem_nc] # # ancillary_list = [dem_nc, mask_nc] # if (os.path.isfile(ancillary_targz) and # (overwrite_flag or # not all([os.path.isfile(os.path.join(ancillary_ws, x)) # for x in ancillary_list]))): # logging.info('\nExtracting ancillary rasters') # logging.debug(" {}".format(ancillary_targz)) # tar = tarfile.open(ancillary_targz) # for member in tar.getmembers(): # print member.name # member.name = os.path.basename(member.name) # # Strip off leading numbers from ancillary raster name # member.name = member.name.split('_', 1)[1] # member_path = os.path.join(ancillary_ws, member.name) # if not member.name.endswith('.nc'): # continue # elif member_path not in ancillary_list: # continue # elif os.path.isfile(member_path): # continue # logging.debug(" {}".format(member.name)) # tar.extract(member, ancillary_ws) # tar.close() # # Mask # if ((overwrite_flag or # not os.path.isfile(mask_raster)) and # os.path.isfile(mask_nc)): # logging.info('\nExtracting mask raster') # mask_nc_f = netCDF4.Dataset(mask_nc, 'r') # logging.debug(mask_nc_f) # # logging.debug(mask_nc_f.variables['image']) # mask_array = mask_nc_f.variables['image'][:] # mask_array[mask_array == daymet_nodata] = 255 # drigo.array_to_raster( # mask_array, mask_raster, # output_geo=daymet_geo, output_proj=daymet_proj, # output_nodata=255) # mask_nc_f.close() # # DEM # if ((overwrite_flag or not os.path.isfile(dem_raster)) and # os.path.isfile(dem_nc)): # logging.info('\nExtracting DEM raster') # dem_nc_f = netCDF4.Dataset(dem_nc, 'r') # logging.debug(dem_nc_f) # # logging.debug(dem_nc_f.variables['image']) # dem_array = dem_nc_f.variables['image'][:] # # Rounding issues of the nodata value when converting to float32 # dem_array[dem_array == daymet_nodata] -= 1 # dem_array = dem_array.astype(np.float32) # if zero_elev_nodata_flag: # dem_array[dem_array <= daymet_nodata] = 0 # else: # dem_array[dem_array <= daymet_nodata] = np.nan # drigo.array_to_raster( # dem_array, dem_raster, # output_geo=daymet_geo, output_proj=daymet_proj) # dem_nc_f.close() # Latitude/Longitude if (os.path.isfile(dem_raster) and (overwrite_flag or not os.path.isfile(lat_raster) or not os.path.isfile(lon_raster))): logging.info('\nDAYMET Latitude/Longitude') logging.debug(' {}'.format(lat_raster)) lat_array, lon_array = drigo.raster_lat_lon_func( dem_raster, gcs_cs=0.05) drigo.array_to_raster( lat_array.astype(np.float32), lat_raster, output_geo=daymet_geo, output_proj=daymet_proj) logging.debug(' {}'.format(lon_raster)) drigo.array_to_raster( lon_array.astype(np.float32), lon_raster, output_geo=daymet_geo, output_proj=daymet_proj) del lat_array, lon_array logging.debug('\nScript Complete')
def main(start_dt, end_dt, netcdf_ws, ancillary_ws, output_ws, extent_path=None, output_extent=None, stats_flag=True, overwrite_flag=False): """Extract DAYMET temperature Parameters ---------- start_dt : datetime Start date. end_dt : datetime End date. netcdf_ws : str Folder of DAYMET netcdf files. ancillary_ws : str Folder of ancillary rasters. output_ws : str Folder of output rasters. extent_path : str, optional File path defining the output extent. output_extent : list, optional Decimal degrees values defining output extent. stats_flag : bool, optional If True, compute raster statistics (the default is True). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). Returns ------- None """ logging.info('\nExtracting DAYMET vapor pressure') logging.debug(' Start date: {}'.format(start_dt)) logging.debug(' End date: {}'.format(end_dt)) # Get DAYMET spatial reference from an ancillary raster mask_raster = os.path.join(ancillary_ws, 'daymet_mask.img') elev_raster = os.path.join(ancillary_ws, 'daymet_elev.img') daymet_re = re.compile('daymet_v3_(?P<VAR>\w+)_(?P<YEAR>\d{4})_na.nc4$') # DAYMET band name dictionary # daymet_band_dict = dict() # daymet_band_dict['prcp'] = 'precipitation_amount' # daymet_band_dict['srad'] = 'surface_downwelling_shortwave_flux_in_air' # daymet_band_dict['sph'] = 'specific_humidity' # daymet_band_dict['tmin'] = 'air_temperature' # daymet_band_dict['tmax'] = 'air_temperature' # Get extent/geo from mask raster daymet_ds = gdal.Open(mask_raster) daymet_osr = drigo.raster_ds_osr(daymet_ds) daymet_proj = drigo.osr_proj(daymet_osr) daymet_cs = drigo.raster_ds_cellsize(daymet_ds, x_only=True) daymet_extent = drigo.raster_ds_extent(daymet_ds) daymet_geo = daymet_extent.geo(daymet_cs) daymet_x, daymet_y = daymet_extent.origin() daymet_ds = None logging.debug(' Projection: {}'.format(daymet_proj)) logging.debug(' Cellsize: {}'.format(daymet_cs)) logging.debug(' Geo: {}'.format(daymet_geo)) logging.debug(' Extent: {}'.format(daymet_extent)) logging.debug(' Origin: {} {}'.format(daymet_x, daymet_y)) # Subset data to a smaller extent if output_extent is not None: logging.info('\nComputing subset extent & geo') logging.debug(' Extent: {}'.format(output_extent)) # Assume input extent is in decimal degrees output_extent = drigo.project_extent(drigo.Extent(output_extent), drigo.epsg_osr(4326), daymet_osr, 0.001) output_extent = drigo.intersect_extents([daymet_extent, output_extent]) output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs) output_geo = output_extent.geo(daymet_cs) logging.debug(' Geo: {}'.format(output_geo)) logging.debug(' Extent: {}'.format(output_extent)) elif extent_path is not None: logging.info('\nComputing subset extent & geo') if extent_path.lower().endswith('.shp'): output_extent = drigo.feature_path_extent(extent_path) extent_osr = drigo.feature_path_osr(extent_path) extent_cs = None else: output_extent = drigo.raster_path_extent(extent_path) extent_osr = drigo.raster_path_osr(extent_path) extent_cs = drigo.raster_path_cellsize(extent_path, x_only=True) output_extent = drigo.project_extent(output_extent, extent_osr, daymet_osr, extent_cs) output_extent = drigo.intersect_extents([daymet_extent, output_extent]) output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs) output_geo = output_extent.geo(daymet_cs) logging.debug(' Geo: {}'.format(output_geo)) logging.debug(' Extent: {}'.format(output_extent)) else: output_extent = daymet_extent.copy() output_geo = daymet_geo[:] # output_shape = output_extent.shape(cs=daymet_cs) xi, yi = drigo.array_geo_offsets(daymet_geo, output_geo, daymet_cs) output_rows, output_cols = output_extent.shape(daymet_cs) logging.debug(' Shape: {} {}'.format(output_rows, output_cols)) logging.debug(' Offsets: {} {} (x y)'.format(xi, yi)) # Read the elevation array elev_array = drigo.raster_to_array(elev_raster, mask_extent=output_extent, return_nodata=False) pair_array = refet.calcs._air_pressure_func(elev_array) del elev_array # Process each variable input_var = 'vp' output_var = 'ea' logging.info("\nVariable: {}".format(input_var)) # Build output folder var_ws = os.path.join(output_ws, output_var) if not os.path.isdir(var_ws): os.makedirs(var_ws) # Process each file in the input workspace for input_name in sorted(os.listdir(netcdf_ws)): logging.debug("{}".format(input_name)) input_match = daymet_re.match(input_name) if not input_match: logging.debug(' Regular expression didn\'t match, skipping') continue elif input_match.group('VAR') != input_var: logging.debug(' Variable didn\'t match, skipping') continue year_str = input_match.group('YEAR') logging.info(" Year: {}".format(year_str)) year_int = int(year_str) year_days = int(dt.datetime(year_int, 12, 31).strftime('%j')) if start_dt is not None and year_int < start_dt.year: logging.debug(' Before start date, skipping') continue elif end_dt is not None and year_int > end_dt.year: logging.debug(' After end date, skipping') continue # Build input file path input_raster = os.path.join(netcdf_ws, input_name) # if not os.path.isfile(input_raster): # logging.debug( # ' Input raster doesn\'t exist, skipping {}'.format( # input_raster)) # continue # Build output folder output_year_ws = os.path.join(var_ws, year_str) if not os.path.isdir(output_year_ws): os.makedirs(output_year_ws) # Read in the DAYMET NetCDF file input_nc_f = netCDF4.Dataset(input_raster, 'r') # logging.debug(input_nc_f.variables) # Check all valid dates in the year year_dates = _utils.date_range(dt.datetime(year_int, 1, 1), dt.datetime(year_int + 1, 1, 1)) for date_dt in year_dates: if start_dt is not None and date_dt < start_dt: logging.debug(' {} - before start date, skipping'.format( date_dt.date())) continue elif end_dt is not None and date_dt > end_dt: logging.debug(' {} - after end date, skipping'.format( date_dt.date())) continue else: logging.info(' {}'.format(date_dt.date())) output_path = os.path.join( output_year_ws, '{}_{}_daymet.img'.format(output_var, date_dt.strftime('%Y%m%d'))) if os.path.isfile(output_path): logging.debug(' {}'.format(output_path)) if not overwrite_flag: logging.debug(' File already exists, skipping') continue else: logging.debug(' File already exists, removing existing') os.remove(output_path) doy = int(date_dt.strftime('%j')) doy_i = range(1, year_days + 1).index(doy) # Arrays are being read as masked array with a fill value of -9999 # Convert to basic numpy array arrays with nan values try: input_ma = input_nc_f.variables[input_var][doy_i, yi:yi + output_rows, xi:xi + output_cols] except IndexError: logging.info(' date not in netcdf, skipping') continue input_nodata = float(input_ma.fill_value) sph_array = input_ma.data.astype(np.float32) sph_array[sph_array == input_nodata] = np.nan # Compute ea [kPa] from specific humidity [kg/kg] ea_array = (sph_array * pair_array) / (0.622 + 0.378 * sph_array) # Save the array as 32-bit floats drigo.array_to_raster(ea_array.astype(np.float32), output_path, output_geo=output_geo, output_proj=daymet_proj, stats_flag=stats_flag) del input_ma, ea_array, sph_array input_nc_f.close() del input_nc_f logging.debug('\nScript Complete')
def main(ancillary_ws, overwrite_flag=False): """Process CIMIS ancillary data Parameters ---------- ancillary_ws : str Folder of ancillary rasters. overwrite_flag : bool, optional If True, overwrite existing files (the default is False). Returns ------- None """ logging.info('\nProcess CIMIS ancillary data') # Site URL site_url = 'http://cimis.casil.ucdavis.edu/cimis/' # DEM for air pressure calculation # http://topotools.cr.usgs.gov/gmted_viewer/gmted2010_global_grids.php elev_full_url = 'http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Grid_ZipFiles/mn30_grd.zip' elev_full_zip = os.path.join(ancillary_ws, 'mn30_grd.zip') elev_full_raster = os.path.join(ancillary_ws, 'mn30_grd') # Get CIMIS grid properties from 2010/01/01 ETo raster # Grid of the spatial cimis input rasters # cimis_extent = drigo.Extent((-410000, -660000, 610000, 460000)) # cimis_cs = 2000 # cimis_geo = drigo.extent_geo(cimis_extent, cimis_cs) # Spatial reference parameters cimis_proj4 = ( '+proj=aea +lat_1=34 +lat_2=40.5 +lat_0=0 +lon_0=-120 +x_0=0 ' '+y_0=-4000000 +ellps=GRS80 +datum=NAD83 +units=m +no_defs') cimis_osr = drigo.proj4_osr(cimis_proj4) # cimis_epsg = 3310 # NAD_1983_California_Teale_Albers # cimis_osr = drigo.epsg_osr(cimis_epsg) # Comment this line out if building GeoTIFF instead of IMG cimis_osr.MorphToESRI() cimis_proj = cimis_osr.ExportToWkt() # snap_xmin, snap_ymin = (0, 0) # Build output workspace if it doesn't exist if not os.path.isdir(ancillary_ws): os.makedirs(ancillary_ws) # File paths mask_url = site_url + '/2010/01/01/ETo.asc.gz' # mask_gz = os.path.join(ancillary_ws, 'cimis_mask.asc.gz') mask_ascii = os.path.join(ancillary_ws, 'cimis_mask.asc') mask_raster = os.path.join(ancillary_ws, 'cimis_mask.img') elev_raster = os.path.join(ancillary_ws, 'cimis_elev.img') lat_raster = os.path.join(ancillary_ws, 'cimis_lat.img') lon_raster = os.path.join(ancillary_ws, 'cimis_lon.img') # Download an ETo ASCII raster to generate the mask raster if overwrite_flag or not os.path.isfile(mask_raster): logging.info('\nCIMIS mask') logging.debug(' Downloading') logging.debug(" {}".format(mask_url)) logging.debug(" {}".format(mask_ascii)) _utils.url_download(mask_url, mask_ascii) # DEADBEEF - The files do not appeared to be compressed even though # they have a .asc.gz file extension on the server. # logging.debug(" {}".format(mask_gz)) # _utils.url_download(mask_url, mask_gz) # # they are named .asc.gz # # Uncompress '.gz' file to a new file # logging.debug(' Uncompressing') # logging.debug(' {}'.format(mask_ascii)) # try: # input_f = gzip.open(mask_gz, 'rb') # output_f = open(mask_ascii, 'wb') # output_f.write(input_f.read()) # output_f.close() # input_f.close() # del input_f, output_f # except: # logging.error(" ERROR EXTRACTING FILE") # os.remove(mask_gz) # # Set spatial reference of the ASCII files # if build_prj_flag: # prj_file = open(mask_asc.replace('.asc','.prj'), 'w') # prj_file.write(output_proj) # prj_file.close() # Convert the ASCII raster to a IMG raster logging.debug(' Computing mask') logging.debug(' {}'.format(mask_raster)) mask_array = drigo.raster_to_array(mask_ascii, return_nodata=False) cimis_geo = drigo.raster_path_geo(mask_ascii) # cimis_extent = drigo.raster_path_extent(mask_ascii) logging.debug(' {}'.format(cimis_geo)) mask_array = np.isfinite(mask_array).astype(np.uint8) drigo.array_to_raster(mask_array, mask_raster, output_geo=cimis_geo, output_proj=cimis_proj, output_nodata=0) # drigo.ascii_to_raster( # mask_ascii, mask_raster, np.float32, cimis_proj) os.remove(mask_ascii) # Compute latitude/longitude rasters if ((overwrite_flag or not os.path.isfile(lat_raster) or not os.path.isfile(lat_raster)) and os.path.isfile(mask_raster)): logging.info('\nCIMIS latitude/longitude') logging.debug(' {}'.format(lat_raster)) lat_array, lon_array = drigo.raster_lat_lon_func(mask_raster) drigo.array_to_raster(lat_array, lat_raster, output_geo=cimis_geo, output_proj=cimis_proj) logging.debug(' {}'.format(lon_raster)) drigo.array_to_raster(lon_array, lon_raster, output_geo=cimis_geo, output_proj=cimis_proj) # Compute DEM raster if overwrite_flag or not os.path.isfile(elev_raster): logging.info('\nCIMIS DEM') logging.debug(' Downloading GMTED2010 DEM') logging.debug(" {}".format(elev_full_url)) logging.debug(" {}".format(elev_full_zip)) if overwrite_flag or not os.path.isfile(elev_full_zip): _utils.url_download(elev_full_url, elev_full_zip) # Uncompress '.gz' file to a new file logging.debug(' Uncompressing') logging.debug(' {}'.format(elev_full_raster)) if overwrite_flag or not os.path.isfile(elev_full_raster): try: with zipfile.ZipFile(elev_full_zip, "r") as z: z.extractall(ancillary_ws) except: logging.error(" ERROR EXTRACTING FILE") os.remove(elev_full_zip) # Get the extent and cellsize from the mask logging.debug(' Projecting to CIMIS grid') cimis_cs = drigo.raster_path_cellsize(mask_raster)[0] cimis_extent = drigo.raster_path_extent(mask_raster) logging.debug(' Extent: {}'.format(cimis_extent)) logging.debug(' Cellsize: {}'.format(cimis_cs)) logging.info(' {}'.format(mask_ascii)) if overwrite_flag and os.path.isfile(elev_raster): subprocess.call(['gdalmanage', 'delete', elev_raster]) if not os.path.isfile(elev_raster): subprocess.call([ 'gdalwarp', '-r', 'average', '-t_srs', cimis_proj4, '-te', str(cimis_extent.xmin), str(cimis_extent.ymin), str(cimis_extent.xmax), str(cimis_extent.ymax), '-tr', str(cimis_cs), str(cimis_cs), '-of', 'HFA', '-co', 'COMPRESSED=TRUE', elev_full_raster, elev_raster ], cwd=ancillary_ws) logging.debug('\nScript Complete')
def main(ancillary_ws=os.getcwd(), zero_elev_nodata_flag=False, overwrite_flag=False): """Process GRIDMET ancillary data Parameters ---------- ancillary_ws : str Folder of ancillary rasters. zero_elev_nodata_flag : bool, optional If True, set elevation nodata values to 0 (the default is False). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). Returns ------- None """ logging.info('\nProcess GRIDMET ancillary rasters') # Site URL elev_url = 'https://climate.northwestknowledge.net/METDATA/data/metdata_elevationdata.nc' # Manually define the spatial reference and extent of the GRIDMET data # This could be read in from a raster gridmet_osr = osr.SpatialReference() # Assume GRIDMET data is in WGS84 not NAD83 (need to check with John) gridmet_osr.ImportFromEPSG(4326) # gridmet_osr.ImportFromEPSG(4326) gridmet_proj = drigo.osr_proj(gridmet_osr) gridmet_cs = 1. / 24 # 0.041666666666666666 gridmet_x = -125 + gridmet_cs * 5 gridmet_y = 49 + gridmet_cs * 10 # gridmet_y = lon_array[0,0] - 0.5 * gridmet_cs # gridmet_y = lat_array[0,0] + 0.5 * gridmet_cs # gridmet_rows, gridmet_cols = elev_array.shape gridmet_geo = (gridmet_x, gridmet_cs, 0., gridmet_y, 0., -gridmet_cs) # gridmet_extent = drigo.geo_extent( # gridmet_geo, gridmet_rows, gridmet_cols) # Keep track of the original/full geo-transform and extent # gridmet_full_geo = ( # gridmet_x, gridmet_cs, 0., gridmet_y, 0., -gridmet_cs) # gridmet_full_extent = drigo.geo_extent( # gridmet_geo, gridmet_rows, gridmet_cols) logging.debug(' X/Y: {} {}'.format(gridmet_x, gridmet_y)) logging.debug(' Geo: {}'.format(gridmet_geo)) logging.debug(' Cellsize: {}'.format(gridmet_cs)) # Build output workspace if it doesn't exist if not os.path.isdir(ancillary_ws): os.makedirs(ancillary_ws) # Output paths elev_nc = os.path.join(ancillary_ws, os.path.basename(elev_url)) elev_raster = os.path.join(ancillary_ws, 'gridmet_elev.img') lat_raster = os.path.join(ancillary_ws, 'gridmet_lat.img') lon_raster = os.path.join(ancillary_ws, 'gridmet_lon.img') # Compute DEM raster if overwrite_flag or not os.path.isfile(elev_raster): logging.info('\nGRIDMET DEM') logging.info(' Downloading') logging.debug(' {}'.format(elev_url)) logging.debug(' {}'.format(elev_nc)) _utils.url_download(elev_url, elev_nc) # try: # urllib.urlretrieve(elev_url, elev_nc) # except: # logging.error(" ERROR: {}\n FILE: {}".format( # sys.exc_info()[0], elev_nc)) # # Try to remove the file since it may not have completely downloaded # os.remove(elev_nc) logging.info(' Extracting') logging.debug(' {}'.format(elev_raster)) elev_nc_f = netCDF4.Dataset(elev_nc, 'r') elev_ma = elev_nc_f.variables['elevation'][0, :, :] elev_array = elev_ma.data.astype(np.float32) # elev_nodata = float(elev_ma.fill_value) elev_array[(elev_array == elev_ma.fill_value) | (elev_array <= -300)] = np.nan if zero_elev_nodata_flag: elev_array[np.isnan(elev_array)] = 0 if np.all(np.isnan(elev_array)): logging.error( '\nERROR: The elevation array is all nodata, exiting\n') sys.exit() drigo.array_to_raster(elev_array, elev_raster, output_geo=gridmet_geo, output_proj=gridmet_proj) elev_nc_f.close() # del elev_nc_f, elev_ma, elev_array, elev_nodata del elev_nc_f, elev_ma, elev_array os.remove(elev_nc) # Compute latitude/longitude rasters if ((overwrite_flag or not os.path.isfile(lat_raster) or not os.path.isfile(lat_raster)) and os.path.isfile(elev_raster)): logging.info('\nGRIDMET Latitude/Longitude') logging.debug(' {}'.format(lat_raster)) lat_array, lon_array = drigo.raster_lat_lon_func(elev_raster) # Handle the conversion to radians in the other GRIDMET scripts # lat_array *= (math.pi / 180) drigo.array_to_raster(lat_array, lat_raster, output_geo=gridmet_geo, output_proj=gridmet_proj) logging.debug(' {}'.format(lon_raster)) drigo.array_to_raster(lon_array, lon_raster, output_geo=gridmet_geo, output_proj=gridmet_proj) del lat_array, lon_array logging.debug('\nScript Complete')