def main(grb_ws=os.getcwd(), ancillary_ws=os.getcwd(), output_ws=os.getcwd(), variables=['pr'], landsat_ws=None, start_date=None, end_date=None, times_str='', extent_path=None, output_extent=None, stats_flag=True, overwrite_flag=False): """Extract NLDAS target variable(s) Args: grb_ws (str): folder of NLDAS GRB files ancillary_ws (str): folder of ancillary rasters output_ws (str): folder of output rasters variable (list): NLDAS variables to download ('ppt', 'srad', 'sph', 'tair', tmmn', 'tmmx', 'vs') landsat_ws (str): folder of Landsat scenes or tar.gz files start_date (str): ISO format date (YYYY-MM-DD) end_date (str): ISO format date (YYYY-MM-DD) times (str): comma separated values and/or ranges of UTC hours (i.e. "1, 2, 5-8") Parsed with python_common.parse_int_set() extent_path (str): file path defining the output extent output_extent (list): decimal degrees values defining output extent stats_flag (bool): if True, compute raster statistics. Default is True. overwrite_flag (bool): if True, overwrite existing files Returns: None """ logging.info('\nExtract NLDAS target variable(s)') # input_fmt = 'NLDAS_FORA0125_H.A{:04d}{:02d}{:02d}.{}.002.grb' input_re = re.compile( 'NLDAS_FORA0125_H.A(?P<YEAR>\d{4})(?P<MONTH>\d{2})' + '(?P<DAY>\d{2}).(?P<TIME>\d{4}).002.grb$') output_fmt = '{}_{:04d}{:02d}{:02d}_hourly_nldas.img' # output_fmt = '{}_{:04d}{:02d}{:02d}_{:04d}_nldas.img' # If a date is not set, process 2017 try: start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d') logging.debug(' Start date: {}'.format(start_dt)) except: start_dt = dt.datetime(2017, 1, 1) logging.info(' Start date: {}'.format(start_dt)) try: end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d') logging.debug(' End date: {}'.format(end_dt)) except: end_dt = dt.datetime(2017, 12, 31) logging.info(' End date: {}'.format(end_dt)) # Only process a specific hours if not times_str: time_list = range(0, 24, 1) else: time_list = list(parse_int_set(times_str)) time_list = ['{:02d}00'.format(t) for t in time_list] # Assume NLDAS is NAD83 # input_epsg = 'EPSG:4269' # NLDAS rasters to extract data_full_list = ['pr', 'srad', 'sph', 'tair', 'tmmn', 'tmmx', 'vs'] if not variables: logging.error('\nERROR: variables parameter is empty\n') sys.exit() elif type(variables) is not list: # DEADBEEF - I could try converting comma separated strings to lists? logging.warning('\nERROR: variables parameter must be a list\n') sys.exit() elif not set(variables).issubset(set(data_full_list)): logging.error('\nERROR: variables parameter is invalid\n {}'.format( variables)) sys.exit() # Ancillary raster paths mask_path = os.path.join(ancillary_ws, 'nldas_mask.img') # Build a date list from landsat_ws scene folders or tar.gz files date_list = [] if landsat_ws is not None and os.path.isdir(landsat_ws): logging.info('\nReading dates from Landsat IDs') logging.info(' {}'.format(landsat_ws)) landsat_re = re.compile( '^(?:LT04|LT05|LE07|LC08)_(?:\d{3})(?:\d{3})_' + '(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})') for root, dirs, files in os.walk(landsat_ws, topdown=True): # If root matches, don't explore subfolders try: landsat_match = landsat_re.match(os.path.basename(root)) date_list.append(dt.datetime.strptime( '_'.join(landsat_match.groups()), '%Y_%m_%d').date().isoformat()) dirs[:] = [] except: pass for file in files: try: landsat_match = landsat_re.match(file) date_list.append(dt.datetime.strptime( '_'.join(landsat_match.groups()), '%Y_%m_%d').date().isoformat()) except: pass date_list = sorted(list(set(date_list))) # elif landsat_ws is not None and os.path.isfile(landsat_ws): # with open(landsat_ws) as landsat_f: # This allows GDAL to throw Python Exceptions # gdal.UseExceptions() # mem_driver = gdal.GetDriverByName('MEM') # Get the NLDAS spatial reference from the mask raster nldas_ds = gdal.Open(mask_path) nldas_osr = gdc.raster_ds_osr(nldas_ds) nldas_proj = gdc.osr_proj(nldas_osr) nldas_cs = gdc.raster_ds_cellsize(nldas_ds, x_only=True) nldas_extent = gdc.raster_ds_extent(nldas_ds) nldas_geo = nldas_extent.geo(nldas_cs) nldas_x, nldas_y = nldas_extent.origin() nldas_ds = None logging.debug(' Projection: {}'.format(nldas_proj)) logging.debug(' Cellsize: {}'.format(nldas_cs)) logging.debug(' Geo: {}'.format(nldas_geo)) logging.debug(' Extent: {}'.format(nldas_extent)) # Subset data to a smaller extent if output_extent is not None: logging.info('\nComputing subset extent & geo') logging.debug(' Extent: {}'.format(output_extent)) nldas_extent = gdc.Extent(output_extent) nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs) nldas_geo = nldas_extent.geo(nldas_cs) logging.debug(' Geo: {}'.format(nldas_geo)) logging.debug(' Extent: {}'.format(output_extent)) elif extent_path is not None: logging.info('\nComputing subset extent & geo') if extent_path.lower().endswith('.shp'): nldas_extent = gdc.feature_path_extent(extent_path) extent_osr = gdc.feature_path_osr(extent_path) extent_cs = None else: nldas_extent = gdc.raster_path_extent(extent_path) extent_osr = gdc.raster_path_osr(extent_path) extent_cs = gdc.raster_path_cellsize(extent_path, x_only=True) nldas_extent = gdc.project_extent( nldas_extent, extent_osr, nldas_osr, extent_cs) nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs) nldas_geo = nldas_extent.geo(nldas_cs) logging.debug(' Geo: {}'.format(nldas_geo)) logging.debug(' Extent: {}'.format(nldas_extent)) logging.debug('') # Read the NLDAS mask array if present if mask_path and os.path.isfile(mask_path): mask_array, mask_nodata = gdc.raster_to_array( mask_path, mask_extent=nldas_extent, fill_value=0, return_nodata=True) mask_array = mask_array != mask_nodata else: mask_array = None # NLDAS band name dictionary nldas_band_dict = dict() nldas_band_dict['pr'] = 'Total precipitation [kg/m^2]' nldas_band_dict['srad'] = 'Downward shortwave radiation flux [W/m^2]' nldas_band_dict['sph'] = 'Specific humidity [kg/kg]' nldas_band_dict['tair'] = 'Temperature [C]' nldas_band_dict['tmmn'] = 'Temperature [C]' nldas_band_dict['tmmx'] = 'Temperature [C]' nldas_band_dict['vs'] = [ 'u-component of wind [m/s]', 'v-component of wind [m/s]'] # NLDAS band name dictionary # nldas_band_dict = dict() # nldas_band_dict['pr'] = 'precipitation_amount' # nldas_band_dict['srad'] = 'surface_downwelling_shortwave_flux_in_air' # nldas_band_dict['sph'] = 'specific_humidity' # nldas_band_dict['tmmn'] = 'air_temperature' # nldas_band_dict['tmmx'] = 'air_temperature' # nldas_band_dict['vs'] = 'wind_speed' # NLDAS band name dictionary (EarthEngine keys, GRID_ELEMENT values) # nldas_band_dict = dict() # nldas_band_dict['total_precipitation'] = 'Total precipitation [kg/m^2]' # nldas_band_dict['shortwave_radiation'] = 'Downward shortwave radiation flux [W/m^2]' # nldas_band_dict['specific_humidity'] = 'Specific humidity [kg/kg]' # nldas_band_dict['pressure'] = 'Pressure [Pa]' # nldas_band_dict['temperature'] = 'Temperature [C]' # nldas_band_dict['wind_u'] = 'u-component of wind [m/s]' # nldas_band_dict['wind_v'] = 'v-component of wind [m/s]' # Process each variable logging.info('\nReading NLDAS GRIBs') for input_var in variables: logging.info("Variable: {}".format(input_var)) # Build output folder var_ws = os.path.join(output_ws, input_var) if not os.path.isdir(var_ws): os.makedirs(var_ws) # Each sub folder in the main folde has all imagery for 1 day # The path for each subfolder is the /YYYY/DOY # This approach will process files for target dates # for input_dt in date_range(start_dt, end_dt + dt.timedelta(1)): # logging.info(input_dt.date()) # Iterate all available files and check dates if necessary for root, folders, files in os.walk(grb_ws): root_split = os.path.normpath(root).split(os.sep) # If the year/doy is outside the range, skip if (re.match('\d{4}', root_split[-2]) and re.match('\d{3}', root_split[-1])): root_dt = dt.datetime.strptime('{}_{}'.format( root_split[-2], root_split[-1]), '%Y_%j') logging.info('{}-{:02d}-{:02d}'.format( root_dt.year, root_dt.month, root_dt.day)) if ((start_dt is not None and root_dt < start_dt) or (end_dt is not None and root_dt > end_dt)): continue elif date_list and root_dt.date().isoformat() not in date_list: continue # If the year is outside the range, don't search subfolders elif re.match('\d{4}', root_split[-1]): root_year = int(root_split[-1]) logging.info('Year: {}'.format(root_year)) if ((start_dt is not None and root_year < start_dt.year) or (end_dt is not None and root_year > end_dt.year)): folders[:] = [] else: folders[:] = sorted(folders) continue else: continue # Create a single raster for each day with 24 bands # Each time step will be stored in a separate band output_name = output_fmt.format( input_var, root_dt.year, root_dt.month, root_dt.day) output_path = os.path.join( var_ws, str(root_dt.year), output_name) logging.debug(' {}'.format(output_path)) if os.path.isfile(output_path): if not overwrite_flag: logging.debug(' File already exists, skipping') continue else: logging.debug(' File already exists, removing existing') os.remove(output_path) logging.debug(' {}'.format(root)) if not os.path.isdir(os.path.dirname(output_path)): os.makedirs(os.path.dirname(output_path)) gdc.build_empty_raster( output_path, band_cnt=24, output_dtype=np.float32, output_proj=nldas_proj, output_cs=nldas_cs, output_extent=nldas_extent, output_fill_flag=True) # Iterate through hourly files for input_name in sorted(files): logging.info(' {}'.format(input_name)) input_path = os.path.join(root, input_name) input_match = input_re.match(input_name) if input_match is None: logging.debug( ' Regular expression didn\'t match, skipping') continue input_dt = dt.datetime( int(input_match.group('YEAR')), int(input_match.group('MONTH')), int(input_match.group('DAY'))) time_str = input_match.group('TIME') band_num = int(time_str[:2]) + 1 # if start_dt is not None and input_dt < start_dt: # continue # elif end_dt is not None and input_dt > end_dt: # continue # elif date_list and input_dt.date().isoformat() not in date_list: # continue if time_str not in time_list: logging.debug(' Time not in list, skipping') continue logging.debug(' Time: {} {}'.format( input_dt.date(), time_str)) logging.debug(' Band: {}'.format(band_num)) # Determine band numbering/naming input_band_dict = grib_band_names(input_path) # Extract array and save input_ds = gdal.Open(input_path) # Convert Kelvin to Celsius (old NLDAS files were in K i think) if input_var in ['tair', 'tmmx', 'tmmn']: # Temperature should be in C for et_common.refet_hourly_func() if 'Temperature [K]' in input_band_dict.keys(): temp_band_units = 'K' output_array = gdc.raster_ds_to_array( input_ds, band=input_band_dict['Temperature [K]'], mask_extent=nldas_extent, return_nodata=False) elif 'Temperature [C]' in input_band_dict.keys(): temp_band_units = 'C' output_array = gdc.raster_ds_to_array( input_ds, band=input_band_dict['Temperature [C]'], mask_extent=nldas_extent, return_nodata=False) else: logging.error('Unknown Temperature units, skipping') logging.error(' {}'.format(input_band_dict.keys())) continue # DEADBEEF - Having issue with T appearing to be C but labeled as K # Try to determine temperature units from values temp_mean = float(np.nanmean(output_array)) temp_units_dict = {20: 'C', 293: 'K'} temp_array_units = temp_units_dict[ min(temp_units_dict, key=lambda x:abs(x - temp_mean))] if temp_array_units == 'K' and temp_band_units == 'K': logging.debug(' Converting temperature from K to C') output_array -= 273.15 elif temp_array_units == 'C' and temp_band_units == 'C': pass elif temp_array_units == 'C' and temp_band_units == 'K': logging.debug( (' Temperature units are K in the GRB band name, ' + 'but values appear to be C\n Mean temperature: {:.2f}\n' + ' Values will NOT be adjusted').format(temp_mean)) elif temp_array_units == 'K' and temp_band_units == 'C': logging.debug( (' Temperature units are C in the GRB band name, ' + 'but values appear to be K\n Mean temperature: {:.2f}\n' + ' Values will be adjusted from K to C').format(temp_mean)) output_array -= 273.15 # Compute wind speed from vectors elif input_var == 'vs': wind_u_array = gdc.raster_ds_to_array( input_ds, band=input_band_dict['u-component of wind [m/s]'], mask_extent=nldas_extent, return_nodata=False) wind_v_array = gdc.raster_ds_to_array( input_ds, band=input_band_dict['v-component of wind [m/s]'], mask_extent=nldas_extent, return_nodata=False) output_array = np.sqrt( wind_u_array ** 2 + wind_v_array ** 2) # Read all other variables directly else: output_array = gdc.raster_ds_to_array( input_ds, band=input_band_dict[nldas_band_dict[input_var]], mask_extent=nldas_extent, return_nodata=False) # Save the projected array as 32-bit floats gdc.array_to_comp_raster( output_array.astype(np.float32), output_path, band=band_num) # gdc.block_to_raster( # ea_array.astype(np.float32), output_path, band=band) # gdc.array_to_raster( # output_array.astype(np.float32), output_path, # output_geo=nldas_geo, output_proj=nldas_proj, # stats_flag=stats_flag) del output_array input_ds = None if stats_flag: gdc.raster_statistics(output_path) logging.debug('\nScript Complete')
def main(cimis_ws=os.getcwd(), gridmet_ws=None, ancillary_ws=os.getcwd(), etr_flag=False, eto_flag=False, start_date=None, end_date=None, stats_flag=True, overwrite_flag=False): """Fill missing CIMIS days with projected data from GRIDMET Currently missing (CGM 2014-08-15) 2010-11-16 -> 2010-11-23 Args: cimis_ws (str): root folder path of CIMIS data gridmet_ws (str): root folder path of GRIDMET data ancillary_ws (str): folder of ancillary rasters etr_flag (bool): if True, compute alfalfa reference ET (ETr) eto_flag (bool): if True, compute grass reference ET (ETo) start_date (str): ISO format date (YYYY-MM-DD) end_date (str): ISO format date (YYYY-MM-DD) stats_flag (bool): if True, compute raster statistics. Default is True. overwrite_flag (bool): if True, overwrite existing files Returns: None """ logging.info('\nFilling CIMIS with GRIDMET') cimis_re = re.compile( '(?P<VAR>etr)_(?P<YYYY>\d{4})_daily_(?P<GRID>\w+).img$') # gridmet_re = re.compile( # '(?P<VAR>ppt)_(?P<YYY>\d{4})_daily_(?P<GRID>\w+).img$') gridmet_fmt = 'etr_{}_daily_gridmet.img' # Compute ETr and/or ETo if not etr_flag and not eto_flag: logging.info(' ETo/ETr flag(s) not set, defaulting to ETr') etr_flag = True logging.debug(' CIMIS: {}'.format(cimis_ws)) logging.debug(' GRIDMET: {}'.format(gridmet_ws)) # If a date is not set, process 2017 try: start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d') logging.debug(' Start date: {}'.format(start_dt)) except: start_dt = dt.datetime(2017, 1, 1) logging.info(' Start date: {}'.format(start_dt)) try: end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d') logging.debug(' End date: {}'.format(end_dt)) except: end_dt = dt.datetime(2017, 12, 31) logging.info(' End date: {}'.format(end_dt)) # Get GRIDMET spatial reference and cellsize from elevation raster # gridmet_elev_raster = os.path.join(ancillary_ws, 'gridmet_elev.img') # Get CIMIS spatial reference and cellsize from mask raster cimis_mask_raster = os.path.join(ancillary_ws, 'cimis_mask.img') # Resample type # 0 = GRA_NearestNeighbour, Nearest neighbour (select on one input pixel) # 1 = GRA_Bilinear,Bilinear (2x2 kernel) # 2 = GRA_Cubic, Cubic Convolution Approximation (4x4 kernel) # 3 = GRA_CubicSpline, Cubic B-Spline Approximation (4x4 kernel) # 4 = GRA_Lanczos, Lanczos windowed sinc interpolation (6x6 kernel) # 5 = GRA_Average, Average (computes the average of all non-NODATA contributing pixels) # 6 = GRA_Mode, Mode (selects the value which appears most often of all the sampled points) resample_type = gdal.GRA_Bilinear # ETo/ETr workspaces cimis_eto_ws = os.path.join(cimis_ws, 'eto') cimis_etr_ws = os.path.join(cimis_ws, 'etr') gridmet_eto_ws = os.path.join(gridmet_ws, 'eto') gridmet_etr_ws = os.path.join(gridmet_ws, 'etr') # This allows GDAL to throw Python Exceptions # gdal.UseExceptions() # mem_driver = gdal.GetDriverByName('MEM') # Get CIMIS grid properties from mask logging.info('\nCIMIS Properties') cimis_mask_ds = gdal.Open(cimis_mask_raster) cimis_osr = gdc.raster_ds_osr(cimis_mask_ds) cimis_proj = gdc.osr_proj(cimis_osr) cimis_cs = gdc.raster_ds_cellsize(cimis_mask_ds, x_only=True) cimis_extent = gdc.raster_ds_extent(cimis_mask_ds) cimis_geo = cimis_extent.geo(cimis_cs) cimis_mask_ds = None logging.debug(' Projection: {}'.format(cimis_proj)) logging.debug(' Cellsize: {}'.format(cimis_cs)) logging.debug(' Geo: {}'.format(cimis_geo)) logging.debug(' Extent: {}'.format(cimis_extent)) # Read the CIMIS mask array if present cimis_mask, cimis_mask_nodata = gdc.raster_to_array(cimis_mask_raster) cimis_mask = cimis_mask != cimis_mask_nodata # # Get extent/geo from elevation raster # logging.info('\nGRIDMET Properties') # gridmet_ds = gdal.Open(gridmet_elev_raster) # gridmet_osr = gdc.raster_ds_osr(gridmet_ds) # gridmet_proj = gdc.osr_proj(gridmet_osr) # gridmet_cs = gdc.raster_ds_cellsize(gridmet_ds, x_only=True) # gridmet_full_extent = gdc.raster_ds_extent(gridmet_ds) # gridmet_full_geo = gridmet_full_extent.geo(gridmet_cs) # gridmet_x, gridmet_y = gridmet_full_extent.origin() # gridmet_ds = None # logging.debug(' Projection: {}'.format(gridmet_proj)) # logging.debug(' Cellsize: {}'.format(gridmet_cs)) # logging.debug(' Geo: {}'.format(gridmet_full_geo)) # logging.debug(' Extent: {}'.format(gridmet_full_extent)) # # Project CIMIS extent to the GRIDMET spatial reference # logging.info('\nGet CIMIS extent in GRIDMET spat. ref.') # gridmet_sub_extent = gdc.project_extent( # cimis_extent, cimis_osr, gridmet_osr, cimis_cs) # gridmet_sub_extent.buffer_extent(4 * gridmet_cs) # gridmet_sub_extent.adjust_to_snap( # 'EXPAND', gridmet_x, gridmet_y, gridmet_cs) # gridmet_sub_geo = gridmet_sub_extent.geo(gridmet_cs) # logging.debug(' Geo: {}'.format(gridmet_sub_geo)) # logging.debug(' Extent: {}'.format(gridmet_sub_extent)) # Process Missing ETo if eto_flag: logging.info('\nETo') for cimis_name in sorted(os.listdir(cimis_eto_ws)): logging.debug("\n{}".format(cimis_name)) cimis_match = cimis_re.match(cimis_name) if not cimis_match: logging.debug(' Regular expression didn\'t match, skipping') continue year = int(cimis_match.group('YYYY')) logging.info(" {}".format(str(year))) if start_dt is not None and year < start_dt.year: logging.debug(' Before start date, skipping') continue elif end_dt is not None and year > end_dt.year: logging.debug(' After end date, skipping') continue cimis_path = os.path.join(cimis_eto_ws, cimis_name) gridmet_path = os.path.join(gridmet_eto_ws, gridmet_fmt.format(str(year))) if not os.path.isfile(gridmet_path): logging.debug(' GRIDMET raster does not exist, skipping') continue if not os.path.isfile(cimis_path): logging.error(' CIMIS raster does not exist, skipping') continue # Check all valid dates in the year year_dates = date_range(dt.datetime(year, 1, 1), dt.datetime(year + 1, 1, 1)) for date_dt in year_dates: if start_dt is not None and date_dt < start_dt: continue elif end_dt is not None and date_dt > end_dt: continue doy = int(date_dt.strftime('%j')) # Look for arrays that don't have data eto_array = gdc.raster_to_array(cimis_path, band=doy, return_nodata=False) if np.any(np.isfinite(eto_array)): logging.debug(' {} - no missing data, skipping'.format( date_dt.strftime('%Y-%m-%d'))) continue else: logging.info(' {}'.format(date_dt.strftime('%Y-%m-%d'))) # # This is much faster but doesn't apply the CIMIS mask # # Create an in memory dataset of the full ETo array # eto_full_rows, eto_full_cols = eto_full_array[:,:,doy_i].shape # eto_full_type, eto_full_nodata = numpy_to_gdal_type(np.float32) # eto_full_ds = mem_driver.Create( # '', eto_full_cols, eto_full_rows, 1, eto_full_type) # eto_full_ds.SetProjection(gridmet_proj) # eto_full_ds.SetGeoTransform(gridmet_full_geo) # eto_full_band = eto_full_ds.GetRasterBand(1) # # eto_full_band.Fill(eto_full_nodata) # eto_full_band.SetNoDataValue(eto_full_nodata) # eto_full_band.WriteArray(eto_full_array[:,:,doy_i], 0, 0) # # # Extract the subset # eto_sub_array, eto_sub_nodata = gdc.raster_ds_to_array( # eto_full_ds, 1, gridmet_sub_extent) # eto_sub_rows, eto_sub_cols = eto_sub_array.shape # eto_full_ds = None # # # Create projected raster # eto_sub_ds = mem_driver.Create( # '', eto_sub_cols, eto_sub_rows, 1, eto_full_type) # eto_sub_ds.SetProjection(gridmet_proj) # eto_sub_ds.SetGeoTransform(gridmet_sub_geo) # eto_sub_band = eto_sub_ds.GetRasterBand(1) # eto_sub_band.Fill(eto_sub_nodata) # eto_sub_band.SetNoDataValue(eto_sub_nodata) # eto_sub_band.WriteArray(eto_sub_array, 0, 0) # eto_sub_ds.FlushCache() # # # Project input DEM to CIMIS spat. ref. # gdc.project_raster_ds( # eto_sub_ds, gridmet_path, resample_type, # env.snap_proj, env.cellsize, cimis_extent) # eto_sub_ds = None # Extract the subset gridmet_ds = gdal.Open(gridmet_path) gridmet_extent = gdc.raster_ds_extent(gridmet_ds) gridmet_cs = gdc.raster_ds_cellsize(gridmet_ds, x_only=True) gridmet_osr = gdc.raster_ds_osr(gridmet_ds) eto_full_array = gdc.raster_ds_to_array(gridmet_ds, band=doy, return_nodata=False) gridmet_ds = None # Get the projected subset of the full ETo array # This is slower than projecting the subset above eto_sub_array = gdc.project_array(eto_full_array, resample_type, gridmet_osr, gridmet_cs, gridmet_extent, cimis_osr, cimis_cs, cimis_extent) # Save the projected array gdc.array_to_comp_raster(eto_sub_array, cimis_path, band=doy, stats_flag=False) # gdc.array_to_raster( # eto_sub_array, output_path, output_geo=cimis_geo, # output_proj=cimis_proj, stats_flag=False) # gdc.array_to_raster( # eto_sub_array, output_path, # output_geo=cimis_geo, output_proj=cimis_proj, # mask_array=cimis_mask, stats_flag=False) del eto_sub_array, eto_full_array if stats_flag: gdc.raster_statistics(cimis_path) # Process Missing ETr if etr_flag: logging.info('\nETr') for cimis_name in sorted(os.listdir(cimis_etr_ws)): cimis_match = cimis_re.match(cimis_name) if not cimis_match: continue year = int(cimis_match.group('YYYY')) if start_dt is not None and year < start_dt.year: continue elif end_dt is not None and year > end_dt.year: continue logging.info("{}".format(str(year))) cimis_path = os.path.join(cimis_etr_ws, cimis_name) gridmet_path = os.path.join(gridmet_etr_ws, gridmet_fmt.format(str(year))) if not os.path.isfile(gridmet_path): continue if not os.path.isfile(cimis_path): logging.error(' CIMIS raster does not exist') continue # Check all valid dates in the year year_dates = date_range(dt.datetime(year, 1, 1), dt.datetime(year + 1, 1, 1)) for date_dt in year_dates: if start_dt is not None and date_dt < start_dt: continue elif end_dt is not None and date_dt > end_dt: continue doy = int(date_dt.strftime('%j')) # Look for arrays that don't have data etr_array = gdc.raster_to_array(cimis_path, band=doy, return_nodata=False) if np.any(np.isfinite(etr_array)): logging.debug(' {} - skipping'.format( date_dt.strftime('%Y-%m-%d'))) continue else: logging.info(' {}'.format(date_dt.strftime('%Y-%m-%d'))) # Extract the subset gridmet_ds = gdal.Open(gridmet_path) gridmet_extent = gdc.raster_ds_extent(gridmet_ds) gridmet_cs = gdc.raster_ds_cellsize(gridmet_ds, x_only=True) gridmet_osr = gdc.raster_ds_osr(gridmet_ds) etr_full_array = gdc.raster_ds_to_array(gridmet_ds, band=doy, return_nodata=False) gridmet_ds = None # Get the projected subset of the full ETr array # This is slower than projecting the subset etr_sub_array = gdc.project_array(etr_full_array, resample_type, gridmet_osr, gridmet_cs, gridmet_extent, cimis_osr, cimis_cs, cimis_extent) # # Save the projected array gdc.array_to_comp_raster(etr_sub_array, cimis_path, band=doy, stats_flag=False) # gdc.array_to_raster( # etr_sub_array, output_path, # output_geo=cimis_geo, output_proj=cimis_proj, # mask_array=cimis_mask, stats_flag=False) del etr_sub_array, etr_full_array if stats_flag: gdc.raster_statistics(cimis_path) logging.debug('\nScript Complete')
def main(gis_ws, cdl_input_ws, cdl_year='', overwrite_flag=False, pyramids_flag=False, stats_flag=False): """Clip CDL rasters to a target extent and rebuild color table Args: gis_ws (str): Folder/workspace path of the GIS data for the project cdl_input_ws (str): Folder/workspace path of the GIS data for the project cdl_year (str): Comma separated list and/or range of years overwrite_flag (bool): If True, overwrite output rasters pyramids_flag (bool): If True, build pyramids/overviews for the output rasters Returns: None """ cdl_format = '{0}_30m_cdls.img' cdl_output_ws = os.path.join(gis_ws, 'cdl') scratch_ws = os.path.join(gis_ws, 'scratch') zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img') if pyramids_flag: levels = '2 4 8 16 32 64 128' # gdal.SetConfigOption('USE_RRD', 'YES') # gdal.SetConfigOption('HFA_USE_RRD', 'YES') # Check input folders if not os.path.isdir(gis_ws): logging.error(('\nERROR: The GIS workspace {} ' + 'does not exist').format(gis_ws)) sys.exit() elif not os.path.isfile(zone_raster_path): logging.error(('\nERROR: The zone raster {} does not exist' + '\n Try re-running "build_study_area_raster.py"' ).format(zone_raster_path)) sys.exit() if not os.path.isdir(cdl_output_ws): os.makedirs(cdl_output_ws) if not os.path.isdir(scratch_ws): os.makedirs(scratch_ws) logging.info('\nGIS Workspace: {}'.format(gis_ws)) logging.info('CDL Workspace: {}'.format(cdl_output_ws)) logging.info('Scratch Workspace: {}'.format(scratch_ws)) # Process each CDL year separately for cdl_year in list(util.parse_int_set(cdl_year)): logging.info('{0}'.format(cdl_year)) cdl_input_path = os.path.join(cdl_input_ws, cdl_format.format(cdl_year)) cdl_output_path = os.path.join(cdl_output_ws, cdl_format.format(cdl_year)) if not os.path.isfile(cdl_input_path): logging.error(('\nERROR: The input CDL raster {} ' + 'does not exist').format(cdl_input_path)) continue # Reference all output rasters zone raster zone_raster_ds = gdal.Open(zone_raster_path) output_osr = gdc.raster_ds_osr(zone_raster_ds) # output_wkt = gdc.raster_ds_proj(zone_raster_ds) output_cs = gdc.raster_ds_cellsize(zone_raster_ds)[0] output_x, output_y = gdc.raster_ds_origin(zone_raster_ds) output_extent = gdc.raster_ds_extent(zone_raster_ds) output_ullr = output_extent.ul_lr_swap() zone_raster_ds = None logging.debug('\nStudy area properties') logging.debug(' Output OSR: {}'.format(output_osr)) logging.debug(' Output Extent: {}'.format(output_extent)) logging.debug(' Output cellsize: {}'.format(output_cs)) logging.debug(' Output UL/LR: {}'.format(output_ullr)) # Overwrite if os.path.isfile(cdl_output_path) or overwrite_flag: subprocess.call(['gdalmanage', 'delete', cdl_output_path]) # remove_file(cdl_output_path) # Clip if not os.path.isfile(cdl_output_path): subprocess.call( ['gdal_translate', '-of', 'HFA', '-co', 'COMPRESSED=YES'] + ['-projwin'] + str(output_ullr).split() + ['-a_ullr'] + str(output_ullr).split() + [cdl_input_path, cdl_output_path]) # , '-a_srs', 'output_proj' # subprocess.call( # 'gdalwarp', '-overwrite', '-of', 'HFA']+ # '-te'] + str(output_extent).split() + # '-tr', '{}'.format(input_cs), '{}'.format(input_cs), # _cdl_path, cdl_output_path]) # Get class names from CDL raster logging.info('Read RAT') input_ds = gdal.Open(cdl_input_path, 0) input_band = input_ds.GetRasterBand(1) input_rat = input_band.GetDefaultRAT() classname_dict = dict() for row_i in range(input_rat.GetRowCount()): classname_dict[row_i] = input_rat.GetValueAsString( row_i, input_rat.GetColOfUsage(2)) input_ds = None del input_ds, input_band, input_rat # Set class names in the clipped CDL raster logging.info('Write RAT', ) clip_ds = gdal.Open(cdl_output_path, 1) clip_band = clip_ds.GetRasterBand(1) clip_rat = clip_band.GetDefaultRAT() clip_usage_list = [ clip_rat.GetUsageOfCol(col_i) for col_i in range(clip_rat.GetColumnCount()) ] logging.debug(clip_usage_list) # if 1 not in clip_usage_list: # _rat.CreateColumn('Count', 1, 1) if 2 not in clip_usage_list: clip_rat.CreateColumn('Class_Name', 2, 2) for row_i in range(clip_rat.GetRowCount()): clip_rat.SetValueAsString(row_i, clip_rat.GetColOfUsage(2), classname_dict[row_i]) clip_band.SetDefaultRAT(clip_rat) # Check that they were written to the RAT # clip_rat = clip_band.GetDefaultRAT() # for row_i in range(clip_rat.GetRowCount()): # clip_rat.GetValueAsString(row_i, clip_rat.GetColOfUsage(2)) clip_ds = None # Statistics if stats_flag and os.path.isfile(cdl_output_path): logging.info('Computing statistics') logging.debug(' {}'.format(cdl_output_path)) subprocess.call([ 'gdalinfo', '-stats', '-nomd', '-noct', '-norat', cdl_output_path ]) # Pyramids if pyramids_flag and os.path.isfile(cdl_output_path): logging.info('Building statistics') logging.debug(' {}'.format(cdl_output_path)) subprocess.call(['gdaladdo', '-ro', cdl_output_path] + levels.split())
def main(netcdf_ws=os.getcwd(), ancillary_ws=os.getcwd(), output_ws=os.getcwd(), start_date=None, end_date=None, extent_path=None, output_extent=None, stats_flag=True, overwrite_flag=False): """Extract DAYMET precipitation Args: netcdf_ws (str): folder of DAYMET netcdf files ancillary_ws (str): folder of ancillary rasters output_ws (str): folder of output rasters start_date (str): ISO format date (YYYY-MM-DD) end_date (str): ISO format date (YYYY-MM-DD) extent_path (str): file path defining the output extent output_extent (list): decimal degrees values defining output extent stats_flag (bool): if True, compute raster statistics. Default is True. overwrite_flag (bool): if True, overwrite existing files Returns: None """ logging.info('\nExtracting DAYMET precipitation') # If a date is not set, process 2015 try: start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d') logging.debug(' Start date: {}'.format(start_dt)) except: start_dt = dt.datetime(2015, 1, 1) logging.info(' Start date: {}'.format(start_dt)) try: end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d') logging.debug(' End date: {}'.format(end_dt)) except: end_dt = dt.datetime(2015, 12, 31) logging.info(' End date: {}'.format(end_dt)) # Save DAYMET lat, lon, and elevation arrays mask_raster = os.path.join(ancillary_ws, 'daymet_mask.img') daymet_re = re.compile('daymet_v3_(?P<VAR>\w+)_(?P<YEAR>\d{4})_na.nc4$') # DAYMET band name dictionary # daymet_band_dict = dict() # daymet_band_dict['prcp'] = 'precipitation_amount' # daymet_band_dict['srad'] = 'surface_downwelling_shortwave_flux_in_air' # daymet_band_dict['sph'] = 'specific_humidity' # daymet_band_dict['tmin'] = 'air_temperature' # daymet_band_dict['tmax'] = 'air_temperature' # Get extent/geo from mask raster daymet_ds = gdal.Open(mask_raster) daymet_osr = gdc.raster_ds_osr(daymet_ds) daymet_proj = gdc.osr_proj(daymet_osr) daymet_cs = gdc.raster_ds_cellsize(daymet_ds, x_only=True) daymet_extent = gdc.raster_ds_extent(daymet_ds) daymet_geo = daymet_extent.geo(daymet_cs) daymet_x, daymet_y = daymet_extent.origin() daymet_ds = None logging.debug(' Projection: {}'.format(daymet_proj)) logging.debug(' Cellsize: {}'.format(daymet_cs)) logging.debug(' Geo: {}'.format(daymet_geo)) logging.debug(' Extent: {}'.format(daymet_extent)) logging.debug(' Origin: {} {}'.format(daymet_x, daymet_y)) # Subset data to a smaller extent if output_extent is not None: logging.info('\nComputing subset extent & geo') logging.debug(' Extent: {}'.format(output_extent)) # Assume input extent is in decimal degrees output_extent = gdc.project_extent( gdc.Extent(output_extent), gdc.epsg_osr(4326), daymet_osr, 0.001) output_extent = gdc.intersect_extents([daymet_extent, output_extent]) output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs) output_geo = output_extent.geo(daymet_cs) logging.debug(' Geo: {}'.format(output_geo)) logging.debug(' Extent: {}'.format(output_extent)) elif extent_path is not None: logging.info('\nComputing subset extent & geo') if extent_path.lower().endswith('.shp'): output_extent = gdc.feature_path_extent(extent_path) extent_osr = gdc.feature_path_osr(extent_path) extent_cs = None else: output_extent = gdc.raster_path_extent(extent_path) extent_osr = gdc.raster_path_osr(extent_path) extent_cs = gdc.raster_path_cellsize(extent_path, x_only=True) output_extent = gdc.project_extent( output_extent, extent_osr, daymet_osr, extent_cs) output_extent = gdc.intersect_extents([daymet_extent, output_extent]) output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs) output_geo = output_extent.geo(daymet_cs) logging.debug(' Geo: {}'.format(output_geo)) logging.debug(' Extent: {}'.format(output_extent)) else: output_extent = daymet_extent.copy() output_geo = daymet_geo[:] # output_shape = output_extent.shape(cs=daymet_cs) xi, yi = gdc.array_geo_offsets(daymet_geo, output_geo, daymet_cs) output_rows, output_cols = output_extent.shape(daymet_cs) logging.debug(' Shape: {} {}'.format(output_rows, output_cols)) logging.debug(' Offsets: {} {} (x y)'.format(xi, yi)) # Process each variable input_var = 'prcp' output_var = 'ppt' logging.info("\nVariable: {}".format(input_var)) # Build output folder var_ws = os.path.join(output_ws, output_var) if not os.path.isdir(var_ws): os.makedirs(var_ws) # Process each file in the input workspace for input_name in sorted(os.listdir(netcdf_ws)): logging.debug("{}".format(input_name)) input_match = daymet_re.match(input_name) if not input_match: logging.debug(' Regular expression didn\'t match, skipping') continue elif input_match.group('VAR') != input_var: logging.debug(' Variable didn\'t match, skipping') continue year_str = input_match.group('YEAR') logging.info(" Year: {}".format(year_str)) year_int = int(year_str) year_days = int(dt.datetime(year_int, 12, 31).strftime('%j')) if start_dt is not None and year_int < start_dt.year: logging.debug(' Before start date, skipping') continue elif end_dt is not None and year_int > end_dt.year: logging.debug(' After end date, skipping') continue # Build input file path input_raster = os.path.join(netcdf_ws, input_name) # if not os.path.isfile(input_raster): # logging.debug( # ' Input raster doesn\'t exist, skipping {}'.format( # input_raster)) # continue # Build output folder output_year_ws = os.path.join(var_ws, year_str) if not os.path.isdir(output_year_ws): os.makedirs(output_year_ws) # Read in the DAYMET NetCDF file input_nc_f = netCDF4.Dataset(input_raster, 'r') # logging.debug(input_nc_f.variables) # Check all valid dates in the year year_dates = date_range( dt.datetime(year_int, 1, 1), dt.datetime(year_int + 1, 1, 1)) for date_dt in year_dates: if start_dt is not None and date_dt < start_dt: logging.debug(' {} - before start date, skipping'.format( date_dt.date())) continue elif end_dt is not None and date_dt > end_dt: logging.debug(' {} - after end date, skipping'.format( date_dt.date())) continue else: logging.info(' {}'.format(date_dt.date())) output_path = os.path.join( output_year_ws, '{}_{}_daymet.img'.format( output_var, date_dt.strftime('%Y%m%d'))) if os.path.isfile(output_path): logging.debug(' {}'.format(output_path)) if not overwrite_flag: logging.debug(' File already exists, skipping') continue else: logging.debug(' File already exists, removing existing') os.remove(output_path) doy = int(date_dt.strftime('%j')) doy_i = range(1, year_days + 1).index(doy) # Arrays are being read as masked array with a fill value of -9999 # Convert to basic numpy array arrays with nan values try: input_ma = input_nc_f.variables[input_var][ doy_i, yi: yi + output_rows, xi: xi + output_cols] except IndexError: logging.info(' date not in netcdf, skipping') continue input_nodata = float(input_ma.fill_value) output_array = input_ma.data.astype(np.float32) output_array[output_array == input_nodata] = np.nan # Save the array as 32-bit floats gdc.array_to_raster( output_array.astype(np.float32), output_path, output_geo=output_geo, output_proj=daymet_proj, stats_flag=stats_flag) del input_ma, output_array input_nc_f.close() del input_nc_f logging.debug('\nScript Complete')
def main(netcdf_ws=os.getcwd(), ancillary_ws=os.getcwd(), output_ws=os.getcwd(), start_date=None, end_date=None, extent_path=None, output_extent=None, stats_flag=True, overwrite_flag=False): """Extract GRIDMET temperature Args: netcdf_ws (str): folder of GRIDMET netcdf files ancillary_ws (str): folder of ancillary rasters output_ws (str): folder of output rasters start_date (str): ISO format date (YYYY-MM-DD) end_date (str): ISO format date (YYYY-MM-DD) extent_path (str): filepath a raster defining the output extent output_extent (list): decimal degrees values defining output extent stats_flag (bool): if True, compute raster statistics. Default is True. overwrite_flag (bool): if True, overwrite existing files Returns: None """ logging.info('\nExtracting GRIDMET vapor pressure') # If a date is not set, process 2017 try: start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d') logging.debug(' Start date: {}'.format(start_dt)) except: start_dt = dt.datetime(2017, 1, 1) logging.info(' Start date: {}'.format(start_dt)) try: end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d') logging.debug(' End date: {}'.format(end_dt)) except: end_dt = dt.datetime(2017, 12, 31) logging.info(' End date: {}'.format(end_dt)) # Save GRIDMET lat, lon, and elevation arrays elev_raster = os.path.join(ancillary_ws, 'gridmet_elev.img') output_fmt = '{}_{}_daily_gridmet.img' gridmet_re = re.compile('(?P<VAR>\w+)_(?P<YEAR>\d{4}).nc$') # GRIDMET band name dictionary gridmet_band_dict = dict() gridmet_band_dict['pr'] = 'precipitation_amount' gridmet_band_dict['srad'] = 'surface_downwelling_shortwave_flux_in_air' gridmet_band_dict['sph'] = 'specific_humidity' gridmet_band_dict['tmmn'] = 'air_temperature' gridmet_band_dict['tmmx'] = 'air_temperature' gridmet_band_dict['vs'] = 'wind_speed' # Get extent/geo from elevation raster gridmet_ds = gdal.Open(elev_raster) gridmet_osr = gdc.raster_ds_osr(gridmet_ds) gridmet_proj = gdc.osr_proj(gridmet_osr) gridmet_cs = gdc.raster_ds_cellsize(gridmet_ds, x_only=True) gridmet_extent = gdc.raster_ds_extent(gridmet_ds) gridmet_full_geo = gridmet_extent.geo(gridmet_cs) gridmet_x, gridmet_y = gridmet_extent.origin() gridmet_ds = None logging.debug(' Projection: {}'.format(gridmet_proj)) logging.debug(' Cellsize: {}'.format(gridmet_cs)) logging.debug(' Geo: {}'.format(gridmet_full_geo)) logging.debug(' Extent: {}'.format(gridmet_extent)) # Subset data to a smaller extent if output_extent is not None: logging.info('\nComputing subset extent & geo') logging.debug(' Extent: {}'.format(output_extent)) gridmet_extent = gdc.Extent(output_extent) gridmet_extent.adjust_to_snap('EXPAND', gridmet_x, gridmet_y, gridmet_cs) gridmet_geo = gridmet_extent.geo(gridmet_cs) logging.debug(' Geo: {}'.format(gridmet_geo)) logging.debug(' Extent: {}'.format(gridmet_extent)) elif extent_path is not None: logging.info('\nComputing subset extent & geo') gridmet_extent = gdc.raster_path_extent(extent_path) extent_osr = gdc.raster_path_osr(extent_path) extent_cs = gdc.raster_path_cellsize(extent_path, x_only=True) gridmet_extent = gdc.project_extent(gridmet_extent, extent_osr, gridmet_osr, extent_cs) gridmet_extent.adjust_to_snap('EXPAND', gridmet_x, gridmet_y, gridmet_cs) gridmet_geo = gridmet_extent.geo(gridmet_cs) logging.debug(' Geo: {}'.format(gridmet_geo)) logging.debug(' Extent: {}'.format(gridmet_extent)) else: gridmet_geo = gridmet_full_geo # Get indices for slicing/clipping input arrays g_i, g_j = gdc.array_geo_offsets(gridmet_full_geo, gridmet_geo, cs=gridmet_cs) g_rows, g_cols = gridmet_extent.shape(cs=gridmet_cs) # Read the elevation array elev_array = gdc.raster_to_array(elev_raster, mask_extent=gridmet_extent, return_nodata=False) pair_array = et_common.air_pressure_func(elev_array) del elev_array # Process each variable input_var = 'sph' output_var = 'ea' logging.info("\nVariable: {}".format(input_var)) # Build output folder var_ws = os.path.join(output_ws, output_var) if not os.path.isdir(var_ws): os.makedirs(var_ws) # Process each file in the input workspace for input_name in sorted(os.listdir(netcdf_ws)): input_match = gridmet_re.match(input_name) if not input_match: logging.debug("{}".format(input_name)) logging.debug(' Regular expression didn\'t match, skipping') continue elif input_match.group('VAR') != input_var: logging.debug("{}".format(input_name)) logging.debug(' Variable didn\'t match, skipping') continue else: logging.info("{}".format(input_name)) year_str = input_match.group('YEAR') logging.info(" {}".format(year_str)) year_int = int(year_str) year_days = int(dt.datetime(year_int, 12, 31).strftime('%j')) if start_dt is not None and year_int < start_dt.year: logging.debug(' Before start date, skipping') continue elif end_dt is not None and year_int > end_dt.year: logging.debug(' After end date, skipping') continue # Build input file path input_raster = os.path.join(netcdf_ws, input_name) # if not os.path.isfile(input_raster): # logging.debug( # ' Input NetCDF doesn\'t exist, skipping {}'.format( # input_raster)) # continue # Create a single raster for each year with 365 bands # Each day will be stored in a separate band output_path = os.path.join(var_ws, output_fmt.format(output_var, year_str)) logging.debug(' {}'.format(output_path)) if os.path.isfile(output_path): if not overwrite_flag: logging.debug(' File already exists, skipping') continue else: logging.debug(' File already exists, removing existing') os.remove(output_path) gdc.build_empty_raster(output_path, band_cnt=366, output_dtype=np.float32, output_proj=gridmet_proj, output_cs=gridmet_cs, output_extent=gridmet_extent, output_fill_flag=True) # Read in the GRIDMET NetCDF file # Immediatly clip input array to save memory input_nc_f = netCDF4.Dataset(input_raster, 'r') input_nc = input_nc_f.variables[ gridmet_band_dict[input_var]][:, g_i:g_i + g_cols, g_j:g_j + g_rows].copy() input_nc = np.transpose(input_nc, (0, 2, 1)) # A numpy array is returned when slicing a masked array # if there are no masked pixels # This is a hack to force the numpy array back to a masked array if type(input_nc) != np.ma.core.MaskedArray: input_nc = np.ma.core.MaskedArray( input_nc, np.zeros(input_nc.shape, dtype=bool)) # Check all valid dates in the year year_dates = date_range(dt.datetime(year_int, 1, 1), dt.datetime(year_int + 1, 1, 1)) for date_dt in year_dates: if start_dt is not None and date_dt < start_dt: # logging.debug(' before start date, skipping') continue elif end_dt is not None and date_dt > end_dt: # logging.debug(' after end date, skipping') continue logging.info(' {}'.format(date_dt.strftime('%Y_%m_%d'))) doy = int(date_dt.strftime('%j')) doy_i = range(1, year_days + 1).index(doy) # Arrays are being read as masked array with a fill value of -9999 # Convert to basic numpy array arrays with nan values try: input_full_ma = input_nc[doy_i, :, :] except IndexError: logging.info(' date not in netcdf, skipping') continue input_full_array = input_full_ma.data.astype(np.float32) input_full_nodata = float(input_full_ma.fill_value) input_full_array[input_full_array == input_full_nodata] = np.nan # Since inputs are netcdf, need to create GDAL raster # datasets in order to use gdal_common functions # Create an in memory dataset of the full ETo array input_full_ds = gdc.array_to_mem_ds(input_full_array, output_geo=gridmet_full_geo, output_proj=gridmet_proj) # Then extract the subset from the in memory dataset sph_array = gdc.raster_ds_to_array(input_full_ds, 1, mask_extent=gridmet_extent, return_nodata=False) # Compute ea [kPa] from specific humidity [kg/kg] ea_array = (sph_array * pair_array) / (0.622 + 0.378 * sph_array) # Save the projected array as 32-bit floats gdc.array_to_comp_raster(ea_array.astype(np.float32), output_path, band=doy, stats_flag=False) # gdc.array_to_raster( # ea_array.astype(np.float32), output_path, # output_geo=gridmet_geo, output_proj=gridmet_proj, # stats_flag=False) del sph_array, ea_array input_nc_f.close() del input_nc_f if stats_flag: gdc.raster_statistics(output_path) logging.debug('\nScript Complete')
def main(grb_ws=os.getcwd(), ancillary_ws=os.getcwd(), output_ws=os.getcwd(), etr_flag=False, eto_flag=False, landsat_ws=None, start_date=None, end_date=None, times_str='', extent_path=None, output_extent=None, daily_flag=True, stats_flag=True, overwrite_flag=False): """Compute hourly ETr/ETo from NLDAS data Args: grb_ws (str): folder of NLDAS GRB files ancillary_ws (str): folder of ancillary rasters output_ws (str): folder of output rasters etr_flag (bool): if True, compute alfalfa reference ET (ETr) eto_flag (bool): if True, compute grass reference ET (ETo) landsat_ws (str): folder of Landsat scenes or tar.gz files start_date (str): ISO format date (YYYY-MM-DD) end_date (str): ISO format date (YYYY-MM-DD) times (str): comma separated values and/or ranges of UTC hours (i.e. "1, 2, 5-8") Parsed with python_common.parse_int_set() extent_path (str): file path defining the output extent output_extent (list): decimal degrees values defining output extent daily_flag (bool): if True, save daily ETr/ETo sum raster. Default is True stats_flag (bool): if True, compute raster statistics. Default is True. overwrite_flag (bool): if True, overwrite existing files Returns: None """ logging.info('\nComputing NLDAS hourly ETr/ETo') np.seterr(invalid='ignore') # Compute ETr and/or ETo if not etr_flag and not eto_flag: logging.info(' ETo/ETr flag(s) not set, defaulting to ETr') etr_flag = True # If a date is not set, process 2017 try: start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d') logging.debug(' Start date: {}'.format(start_dt)) except: start_dt = dt.datetime(2017, 1, 1) logging.info(' Start date: {}'.format(start_dt)) try: end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d') logging.debug(' End date: {}'.format(end_dt)) except: end_dt = dt.datetime(2017, 12, 31) logging.info(' End date: {}'.format(end_dt)) # Only process a specific hours if not times_str: time_list = range(0, 24, 1) else: time_list = list(parse_int_set(times_str)) time_list = ['{:02d}00'.format(t) for t in time_list] etr_folder = 'etr' eto_folder = 'eto' hour_fmt = '{}_{:04d}{:02d}{:02d}_hourly_nldas.img' # hour_fmt = '{}_{:04d}{:02d}{:02d}_{4:04d}_nldas.img' day_fmt = '{}_{:04d}{:02d}{:02d}_nldas.img' # input_fmt = 'NLDAS_FORA0125_H.A{:04d}{:02d}{:02d}.{}.002.grb' input_re = re.compile('NLDAS_FORA0125_H.A(?P<YEAR>\d{4})(?P<MONTH>\d{2})' + '(?P<DAY>\d{2}).(?P<TIME>\d{4}).002.grb$') # Assume NLDAS is NAD83 # input_epsg = 'EPSG:4269' # Ancillary raster paths mask_path = os.path.join(ancillary_ws, 'nldas_mask.img') elev_path = os.path.join(ancillary_ws, 'nldas_elev.img') lat_path = os.path.join(ancillary_ws, 'nldas_lat.img') lon_path = os.path.join(ancillary_ws, 'nldas_lon.img') # Build a date list from landsat_ws scene folders or tar.gz files date_list = [] if landsat_ws is not None and os.path.isdir(landsat_ws): logging.info('\nReading dates from Landsat IDs') logging.info(' {}'.format(landsat_ws)) landsat_re = re.compile( '^(?:LT04|LT05|LE07|LC08)_(?:\d{3})(?:\d{3})_' + '(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})') for root, dirs, files in os.walk(landsat_ws, topdown=True): # If root matches, don't explore subfolders try: landsat_match = landsat_re.match(os.path.basename(root)) date_list.append( dt.datetime.strptime('_'.join(landsat_match.groups()), '%Y_%m_%d').date().isoformat()) dirs[:] = [] except: pass for file in files: try: landsat_match = landsat_re.match(file) date_list.append( dt.datetime.strptime('_'.join(landsat_match.groups()), '%Y_%m_%d').date().isoformat()) except: pass date_list = sorted(list(set(date_list))) # elif landsat_ws is not None and os.path.isfile(landsat_ws): # with open(landsat_ws) as landsat_f: # This allows GDAL to throw Python Exceptions # gdal.UseExceptions() # mem_driver = gdal.GetDriverByName('MEM') # Get the NLDAS spatial reference from the mask raster nldas_ds = gdal.Open(mask_path) nldas_osr = gdc.raster_ds_osr(nldas_ds) nldas_proj = gdc.osr_proj(nldas_osr) nldas_cs = gdc.raster_ds_cellsize(nldas_ds, x_only=True) nldas_extent = gdc.raster_ds_extent(nldas_ds) nldas_geo = nldas_extent.geo(nldas_cs) nldas_x, nldas_y = nldas_extent.origin() nldas_ds = None logging.debug(' Projection: {}'.format(nldas_proj)) logging.debug(' Cellsize: {}'.format(nldas_cs)) logging.debug(' Geo: {}'.format(nldas_geo)) logging.debug(' Extent: {}'.format(nldas_extent)) # Subset data to a smaller extent if output_extent is not None: logging.info('\nComputing subset extent & geo') logging.debug(' Extent: {}'.format(output_extent)) nldas_extent = gdc.Extent(output_extent) nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs) nldas_geo = nldas_extent.geo(nldas_cs) logging.debug(' Geo: {}'.format(nldas_geo)) logging.debug(' Extent: {}'.format(output_extent)) elif extent_path is not None: logging.info('\nComputing subset extent & geo') if extent_path.lower().endswith('.shp'): nldas_extent = gdc.feature_path_extent(extent_path) extent_osr = gdc.feature_path_osr(extent_path) extent_cs = None else: nldas_extent = gdc.raster_path_extent(extent_path) extent_osr = gdc.raster_path_osr(extent_path) extent_cs = gdc.raster_path_cellsize(extent_path, x_only=True) nldas_extent = gdc.project_extent(nldas_extent, extent_osr, nldas_osr, extent_cs) nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs) nldas_geo = nldas_extent.geo(nldas_cs) logging.debug(' Geo: {}'.format(nldas_geo)) logging.debug(' Extent: {}'.format(nldas_extent)) logging.debug('') # Read the NLDAS mask array if present if mask_path and os.path.isfile(mask_path): mask_array, mask_nodata = gdc.raster_to_array(mask_path, mask_extent=nldas_extent, fill_value=0, return_nodata=True) mask_array = mask_array != mask_nodata else: mask_array = None # Read ancillary arrays (or subsets?) elev_array = gdc.raster_to_array(elev_path, mask_extent=nldas_extent, return_nodata=False) # pair_array = et_common.air_pressure_func(elev_array) lat_array = gdc.raster_to_array(lat_path, mask_extent=nldas_extent, return_nodata=False) lon_array = gdc.raster_to_array(lon_path, mask_extent=nldas_extent, return_nodata=False) # Hourly RefET functions expects lat/lon in radians lat_array *= (math.pi / 180) lon_array *= (math.pi / 180) # Build output folder etr_ws = os.path.join(output_ws, etr_folder) eto_ws = os.path.join(output_ws, eto_folder) if etr_flag and not os.path.isdir(etr_ws): os.makedirs(etr_ws) if eto_flag and not os.path.isdir(eto_ws): os.makedirs(eto_ws) # DEADBEEF - Instead of processing all available files, the following # code will process files for target dates # for input_dt in date_range(start_dt, end_dt + dt.timedelta(1)): # logging.info(input_dt.date()) # Iterate all available files and check dates if necessary # Each sub folder in the main folder has all imagery for 1 day # (in UTC time) # The path for each subfolder is the /YYYY/DOY errors = defaultdict(list) for root, folders, files in os.walk(grb_ws): root_split = os.path.normpath(root).split(os.sep) # If the year/doy is outside the range, skip if (re.match('\d{4}', root_split[-2]) and re.match('\d{3}', root_split[-1])): root_dt = dt.datetime.strptime( '{}_{}'.format(root_split[-2], root_split[-1]), '%Y_%j') logging.info('{}'.format(root_dt.date())) if ((start_dt is not None and root_dt < start_dt) or (end_dt is not None and root_dt > end_dt)): continue elif date_list and root_dt.date().isoformat() not in date_list: continue # If the year is outside the range, don't search subfolders elif re.match('\d{4}', root_split[-1]): root_year = int(root_split[-1]) logging.info('Year: {}'.format(root_year)) if ((start_dt is not None and root_year < start_dt.year) or (end_dt is not None and root_year > end_dt.year)): folders[:] = [] else: folders[:] = sorted(folders) continue else: continue logging.debug(' {}'.format(root)) # Start off assuming every file needs to be processed day_skip_flag = False # Build output folders if necessary etr_year_ws = os.path.join(etr_ws, str(root_dt.year)) eto_year_ws = os.path.join(eto_ws, str(root_dt.year)) if etr_flag and not os.path.isdir(etr_year_ws): os.makedirs(etr_year_ws) if eto_flag and not os.path.isdir(eto_year_ws): os.makedirs(eto_year_ws) # Build daily total paths etr_day_path = os.path.join( etr_year_ws, day_fmt.format('etr', root_dt.year, root_dt.month, root_dt.day)) eto_day_path = os.path.join( eto_year_ws, day_fmt.format('eto', root_dt.year, root_dt.month, root_dt.day)) etr_hour_path = os.path.join( etr_year_ws, hour_fmt.format('etr', root_dt.year, root_dt.month, root_dt.day)) eto_hour_path = os.path.join( eto_year_ws, hour_fmt.format('eto', root_dt.year, root_dt.month, root_dt.day)) # logging.debug(' {}'.format(etr_hour_path)) # If daily ETr/ETo files are present, day can be skipped if not overwrite_flag and daily_flag: if etr_flag and not os.path.isfile(etr_day_path): pass elif eto_flag and not os.path.isfile(eto_day_path): pass else: day_skip_flag = True # If the hour and daily files don't need to be made, skip the day if not overwrite_flag: if etr_flag and not os.path.isfile(etr_hour_path): pass elif eto_flag and not os.path.isfile(eto_hour_path): pass elif day_skip_flag: logging.debug(' File(s) already exist, skipping') continue # Create a single raster for each day with 24 bands # Each time step will be stored in a separate band if etr_flag: logging.debug(' {}'.format(etr_day_path)) gdc.build_empty_raster(etr_hour_path, band_cnt=24, output_dtype=np.float32, output_proj=nldas_proj, output_cs=nldas_cs, output_extent=nldas_extent, output_fill_flag=True) if eto_flag: logging.debug(' {}'.format(eto_day_path)) gdc.build_empty_raster(eto_hour_path, band_cnt=24, output_dtype=np.float32, output_proj=nldas_proj, output_cs=nldas_cs, output_extent=nldas_extent, output_fill_flag=True) # Sum all ETr/ETo images in each folder to generate a UTC day total etr_day_array = 0 eto_day_array = 0 # Process each hour file for input_name in sorted(files): logging.info(' {}'.format(input_name)) input_match = input_re.match(input_name) if input_match is None: logging.debug(' Regular expression didn\'t match, skipping') continue input_dt = dt.datetime(int(input_match.group('YEAR')), int(input_match.group('MONTH')), int(input_match.group('DAY'))) input_doy = int(input_dt.strftime('%j')) time_str = input_match.group('TIME') band_num = int(time_str[:2]) + 1 # if start_dt is not None and input_dt < start_dt: # continue # elif end_dt is not None and input_dt > end_dt: # continue # elif date_list and input_dt.date().isoformat() not in date_list: # continue if not daily_flag and time_str not in time_list: logging.debug(' Time not in list and not daily, skipping') continue input_path = os.path.join(root, input_name) logging.debug(' Time: {} {}'.format(input_dt.date(), time_str)) logging.debug(' Band: {}'.format(band_num)) # Determine band numbering/naming try: input_band_dict = grib_band_names(input_path) except RuntimeError as e: errors[input_path].append(e) logging.error(' RuntimeError: {} Skipping: {}'.format( e, input_path)) continue # Read input bands input_ds = gdal.Open(input_path) # Temperature should be in C for et_common.refet_hourly_func() if 'Temperature [K]' in input_band_dict.keys(): temp_band_units = 'K' temp_array = gdc.raster_ds_to_array( input_ds, band=input_band_dict['Temperature [K]'], mask_extent=nldas_extent, return_nodata=False) elif 'Temperature [C]' in input_band_dict.keys(): temp_band_units = 'C' temp_array = gdc.raster_ds_to_array( input_ds, band=input_band_dict['Temperature [C]'], mask_extent=nldas_extent, return_nodata=False) else: logging.error('Unknown Temperature units, skipping') logging.error(' {}'.format(input_band_dict.keys())) continue # DEADBEEF - Having issue with T appearing to be C but labeled as K # Try to determine temperature units from values temp_mean = float(np.nanmean(temp_array)) temp_units_dict = {20: 'C', 293: 'K'} temp_array_units = temp_units_dict[min( temp_units_dict, key=lambda x: abs(x - temp_mean))] if temp_array_units == 'K' and temp_band_units == 'K': logging.debug(' Converting temperature from K to C') temp_array -= 273.15 elif temp_array_units == 'C' and temp_band_units == 'C': pass elif temp_array_units == 'C' and temp_band_units == 'K': logging.debug(( ' Temperature units are K in the GRB band name, ' + 'but values appear to be C\n Mean temperature: {:.2f}\n' + ' Values will NOT be adjusted').format(temp_mean)) elif temp_array_units == 'K' and temp_band_units == 'C': logging.debug(( ' Temperature units are C in the GRB band name, ' + 'but values appear to be K\n Mean temperature: {:.2f}\n' + ' Values will be adjusted from K to C').format(temp_mean)) temp_array -= 273.15 try: sph_array = gdc.raster_ds_to_array( input_ds, band=input_band_dict['Specific humidity [kg/kg]'], mask_extent=nldas_extent, return_nodata=False) rs_array = gdc.raster_ds_to_array( input_ds, band=input_band_dict[ 'Downward shortwave radiation flux [W/m^2]'], mask_extent=nldas_extent, return_nodata=False) wind_u_array = gdc.raster_ds_to_array( input_ds, band=input_band_dict['u-component of wind [m/s]'], mask_extent=nldas_extent, return_nodata=False) wind_v_array = gdc.raster_ds_to_array( input_ds, band=input_band_dict['v-component of wind [m/s]'], mask_extent=nldas_extent, return_nodata=False) input_ds = None except KeyError as e: errors[input_path].append(e) logging.error(' KeyError: {} Skipping: {}'.format( e, input_ds.GetDescription())) continue rs_array *= 0.0036 # W m-2 to MJ m-2 hr-1 wind_array = np.sqrt(wind_u_array**2 + wind_v_array**2) del wind_u_array, wind_v_array # ETr if etr_flag: etr_array = et_common.refet_hourly_func(temp_array, sph_array, rs_array, wind_array, zw=10, elev=elev_array, lat=lat_array, lon=lon_array, doy=input_doy, time=int(time_str) / 100, ref_type='ETR') if daily_flag: etr_day_array += etr_array if time_str in time_list: gdc.array_to_comp_raster(etr_array.astype(np.float32), etr_hour_path, band=band_num, stats_flag=False) del etr_array # ETo if eto_flag: eto_array = et_common.refet_hourly_func(temp_array, sph_array, rs_array, wind_array, zw=10, elev=elev_array, lat=lat_array, lon=lon_array, doy=input_doy, time=int(time_str) / 100, ref_type='ETO') if eto_flag and daily_flag: eto_day_array += eto_array if eto_flag and time_str in time_list: gdc.array_to_comp_raster(eto_array.astype(np.float32), eto_hour_path, band=band_num, stats_flag=False) del eto_array del temp_array, sph_array, rs_array, wind_array if stats_flag and etr_flag: gdc.raster_statistics(etr_hour_path) if stats_flag and eto_flag: gdc.raster_statistics(eto_hour_path) # Save the projected ETr/ETo as 32-bit floats if not day_skip_flag and daily_flag: if etr_flag: try: gdc.array_to_raster(etr_day_array.astype(np.float32), etr_day_path, output_geo=nldas_geo, output_proj=nldas_proj, stats_flag=stats_flag) except AttributeError: pass if eto_flag: try: gdc.array_to_raster(eto_day_array.astype(np.float32), eto_day_path, output_geo=nldas_geo, output_proj=nldas_proj, stats_flag=stats_flag) except AttributeError: pass del etr_day_array, eto_day_array if len(errors) > 0: logging.info('\nThe following errors were encountered:') for key, value in errors.items(): logging.error(' Filepath: {}, error: {}'.format(key, value)) logging.debug('\nScript Complete')
def main(gis_ws, tile_ws, dem_cs, mask_flag=False, overwrite_flag=False): """Download NED tiles that intersect the study_area Script assumes DEM data is in 1x1 WGS84 degree tiles Download 10m (1/3 arc-second) or 30m (1 arc-second) versions from: 10m: rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/13/IMG 30m: rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/1/IMG For this example, only download 30m DEM Args: gis_ws (str): Folder/workspace path of the GIS data for the project tile_ws (str): Folder/workspace path of the DEM tiles dem_cs (int): DEM cellsize (10 or 30m) mask_flag (bool): If True, only download tiles intersecting zones mask overwrite_flag (bool): If True, overwrite existing files Returns: None """ logging.info('\nDownload DEM tiles') zip_fmt = 'n{0:02d}w{1:03d}.zip' if dem_cs == 10: site_url = 'ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/13/IMG' tile_fmt = 'imgn{0:02d}w{1:03d}_13.img' elif dem_cs == 30: site_url = 'ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/1/IMG' tile_fmt = 'imgn{0:02d}w{1:03d}_1.img' else: logging.error('\nERROR: The input cellsize must be 10 or 30\n') sys.exit() # Use 1 degree snap point and "cellsize" to get 1x1 degree tiles tile_osr = gdc.epsg_osr(4269) tile_buffer = 0.5 tile_x, tile_y, tile_cs = 0, 0, 1 scratch_ws = os.path.join(gis_ws, 'scratch') zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img') zone_polygon_path = os.path.join(scratch_ws, 'zone_polygon.shp') # Error checking if not os.path.isfile(zone_raster_path): logging.error(('\nERROR: The zone raster {} does not exist' + '\n Try re-running "build_study_area_raster.py"' ).format(zone_raster_path)) sys.exit() if mask_flag and not os.path.isfile(zone_polygon_path): logging.error( ('\nERROR: The zone polygon {} does not exist and mask_flag=True' + '\n Try re-running "build_study_area_raster.py"' ).format(zone_raster_path)) sys.exit() if not os.path.isdir(tile_ws): os.makedirs(tile_ws) # Reference all output rasters zone raster zone_raster_ds = gdal.Open(zone_raster_path) output_osr = gdc.raster_ds_osr(zone_raster_ds) # output_wkt = gdc.raster_ds_proj(zone_raster_ds) output_cs = gdc.raster_ds_cellsize(zone_raster_ds)[0] output_x, output_y = gdc.raster_ds_origin(zone_raster_ds) output_extent = gdc.raster_ds_extent(zone_raster_ds) output_ullr = output_extent.ul_lr_swap() zone_raster_ds = None logging.debug('\nStudy area properties') logging.debug(' Output OSR: {}'.format(output_osr)) logging.debug(' Output Extent: {}'.format(output_extent)) logging.debug(' Output cellsize: {}'.format(output_cs)) logging.debug(' Output UL/LR: {}'.format(output_ullr)) if mask_flag: # Keep tiles that intersect zone polygon lat_lon_list = polygon_tiles(zone_polygon_path, tile_osr, tile_x, tile_y, tile_cs, tile_buffer=0) else: # Keep tiles that intersect zone raster extent # Project study area extent to DEM tile coordinate system tile_extent = gdc.project_extent(output_extent, output_osr, tile_osr) logging.debug('Output Extent: {}'.format(tile_extent)) # Extent needed to select 1x1 degree tiles tile_extent.buffer_extent(tile_buffer) tile_extent.adjust_to_snap('EXPAND', tile_x, tile_y, tile_cs) logging.debug('Tile Extent: {}'.format(tile_extent)) # Get list of available tiles that intersect the extent lat_lon_list = sorted( list( set([(lat, -lon) for lon in range(int(tile_extent.xmin), int(tile_extent.xmax)) for lat in range(int(tile_extent.ymax), int(tile_extent.ymin), -1)]))) # Attempt to download the tiles logging.debug('Downloading') for lat_lon in lat_lon_list: logging.info(' {}'.format(lat_lon)) zip_name = zip_fmt.format(*lat_lon) zip_url = site_url + '/' + zip_name zip_path = os.path.join(tile_ws, zip_name) tile_name = tile_fmt.format(*lat_lon) tile_path = os.path.join(tile_ws, tile_name) logging.debug(zip_url) logging.debug(zip_path) if not os.path.isfile(tile_path) or overwrite_flag: try: urllib.urlretrieve(zip_url, zip_path) zip_f = zipfile.ZipFile(zip_path) zip_f.extract(tile_name, tile_ws) zip_f.close() except IOError: logging.debug(' IOError, skipping') try: os.remove(zip_path) except: pass
def main(grb_ws=os.getcwd(), ancillary_ws=os.getcwd(), output_ws=os.getcwd(), landsat_ws=None, start_date=None, end_date=None, times_str='', extent_path=None, output_extent=None, stats_flag=True, overwrite_flag=False): """Extract hourly NLDAS vapour pressure rasters Args: grb_ws (str): folder of NLDAS GRB files ancillary_ws (str): folder of ancillary rasters output_ws (str): folder of output rasters landsat_ws (str): folder of Landsat scenes or tar.gz files start_date (str): ISO format date (YYYY-MM-DD) end_date (str): ISO format date (YYYY-MM-DD) times (str): comma separated values and/or ranges of UTC hours (i.e. "1, 2, 5-8") Parsed with python_common.parse_int_set() extent_path (str): file path defining the output extent output_extent (list): decimal degrees values defining output extent stats_flag (bool): if True, compute raster statistics. Default is True. overwrite_flag (bool): if True, overwrite existing files Returns: None """ logging.info('\nExtracting NLDAS vapour pressure rasters') # input_fmt = 'NLDAS_FORA0125_H.A{:04d}{:02d}{:02d}.{}.002.grb' input_re = re.compile('NLDAS_FORA0125_H.A(?P<YEAR>\d{4})(?P<MONTH>\d{2})' + '(?P<DAY>\d{2}).(?P<TIME>\d{4}).002.grb$') output_folder = 'ea' output_fmt = 'ea_{:04d}{:02d}{:02d}_hourly_nldas.img' # output_fmt = 'ea_{:04d}{:02d}{:02d}_{:04d}_nldas.img' # If a date is not set, process 2017 try: start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d') logging.debug(' Start date: {}'.format(start_dt)) except: start_dt = dt.datetime(2017, 1, 1) logging.info(' Start date: {}'.format(start_dt)) try: end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d') logging.debug(' End date: {}'.format(end_dt)) except: end_dt = dt.datetime(2017, 12, 31) logging.info(' End date: {}'.format(end_dt)) # Only process a specific hours if not times_str: time_list = range(0, 24, 1) else: time_list = list(parse_int_set(times_str)) time_list = ['{:02d}00'.format(t) for t in time_list] # Assume NLDAS is NAD83 # input_epsg = 'EPSG:4269' # Ancillary raster paths mask_path = os.path.join(ancillary_ws, 'nldas_mask.img') elev_path = os.path.join(ancillary_ws, 'nldas_elev.img') # Build a date list from landsat_ws scene folders or tar.gz files date_list = [] if landsat_ws is not None and os.path.isdir(landsat_ws): logging.info('\nReading dates from Landsat IDs') logging.info(' {}'.format(landsat_ws)) landsat_re = re.compile( '^(?:LT04|LT05|LE07|LC08)_(?:\d{3})(?:\d{3})_' + '(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})') for root, dirs, files in os.walk(landsat_ws, topdown=True): # If root matches, don't explore subfolders try: landsat_match = landsat_re.match(os.path.basename(root)) date_list.append( dt.datetime.strptime('_'.join(landsat_match.groups()), '%Y_%m_%d').date().isoformat()) dirs[:] = [] except: pass for file in files: try: landsat_match = landsat_re.match(file) date_list.append( dt.datetime.strptime('_'.join(landsat_match.groups()), '%Y_%m_%d').date().isoformat()) except: pass date_list = sorted(list(set(date_list))) # elif landsat_ws is not None and os.path.isfile(landsat_ws): # with open(landsat_ws) as landsat_f: # This allows GDAL to throw Python Exceptions # gdal.UseExceptions() # mem_driver = gdal.GetDriverByName('MEM') # Get the NLDAS spatial reference from the mask raster nldas_ds = gdal.Open(mask_path) nldas_osr = gdc.raster_ds_osr(nldas_ds) nldas_proj = gdc.osr_proj(nldas_osr) nldas_cs = gdc.raster_ds_cellsize(nldas_ds, x_only=True) nldas_extent = gdc.raster_ds_extent(nldas_ds) nldas_geo = nldas_extent.geo(nldas_cs) nldas_x, nldas_y = nldas_extent.origin() nldas_ds = None logging.debug(' Projection: {}'.format(nldas_proj)) logging.debug(' Cellsize: {}'.format(nldas_cs)) logging.debug(' Geo: {}'.format(nldas_geo)) logging.debug(' Extent: {}'.format(nldas_extent)) # Subset data to a smaller extent if output_extent is not None: logging.info('\nComputing subset extent & geo') logging.debug(' Extent: {}'.format(output_extent)) nldas_extent = gdc.Extent(output_extent) nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs) nldas_geo = nldas_extent.geo(nldas_cs) logging.debug(' Geo: {}'.format(nldas_geo)) logging.debug(' Extent: {}'.format(output_extent)) elif extent_path is not None: logging.info('\nComputing subset extent & geo') if extent_path.lower().endswith('.shp'): nldas_extent = gdc.feature_path_extent(extent_path) extent_osr = gdc.feature_path_osr(extent_path) extent_cs = None else: nldas_extent = gdc.raster_path_extent(extent_path) extent_osr = gdc.raster_path_osr(extent_path) extent_cs = gdc.raster_path_cellsize(extent_path, x_only=True) nldas_extent = gdc.project_extent(nldas_extent, extent_osr, nldas_osr, extent_cs) nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs) nldas_geo = nldas_extent.geo(nldas_cs) logging.debug(' Geo: {}'.format(nldas_geo)) logging.debug(' Extent: {}'.format(nldas_extent)) logging.debug('') # Read the NLDAS mask array if present if mask_path and os.path.isfile(mask_path): mask_array, mask_nodata = gdc.raster_to_array(mask_path, mask_extent=nldas_extent, fill_value=0, return_nodata=True) mask_array = mask_array != mask_nodata else: mask_array = None # Read elevation arrays (or subsets?) elev_array = gdc.raster_to_array(elev_path, mask_extent=nldas_extent, return_nodata=False) pair_array = et_common.air_pressure_func(elev_array) # Build output folder var_ws = os.path.join(output_ws, output_folder) if not os.path.isdir(var_ws): os.makedirs(var_ws) # Each sub folder in the main folder has all imagery for 1 day # The path for each subfolder is the /YYYY/DOY # This approach will process files for target dates # for input_dt in date_range(start_dt, end_dt + dt.timedelta(1)): # logging.info(input_dt.date()) # Iterate all available files and check dates if necessary for root, folders, files in os.walk(grb_ws): root_split = os.path.normpath(root).split(os.sep) # If the year/doy is outside the range, skip if (re.match('\d{4}', root_split[-2]) and re.match('\d{3}', root_split[-1])): root_dt = dt.datetime.strptime( '{}_{}'.format(root_split[-2], root_split[-1]), '%Y_%j') logging.info('{}'.format(root_dt.date())) if ((start_dt is not None and root_dt < start_dt) or (end_dt is not None and root_dt > end_dt)): continue elif date_list and root_dt.date().isoformat() not in date_list: continue # If the year is outside the range, don't search subfolders elif re.match('\d{4}', root_split[-1]): root_year = int(root_split[-1]) logging.info('Year: {}'.format(root_year)) if ((start_dt is not None and root_year < start_dt.year) or (end_dt is not None and root_year > end_dt.year)): folders[:] = [] else: folders[:] = sorted(folders) continue else: continue # Create a single raster for each day with 24 bands # Each time step will be stored in a separate band output_name = output_fmt.format(root_dt.year, root_dt.month, root_dt.day) output_path = os.path.join(var_ws, str(root_dt.year), output_name) logging.debug(' {}'.format(output_path)) if os.path.isfile(output_path): if not overwrite_flag: logging.debug(' File already exists, skipping') continue else: logging.debug(' File already exists, removing existing') os.remove(output_path) logging.debug(' {}'.format(root)) if not os.path.isdir(os.path.dirname(output_path)): os.makedirs(os.path.dirname(output_path)) gdc.build_empty_raster(output_path, band_cnt=24, output_dtype=np.float32, output_proj=nldas_proj, output_cs=nldas_cs, output_extent=nldas_extent, output_fill_flag=True) # Iterate through hourly files for input_name in sorted(files): logging.info(' {}'.format(input_name)) input_path = os.path.join(root, input_name) input_match = input_re.match(input_name) if input_match is None: logging.debug(' Regular expression didn\'t match, skipping') continue input_dt = dt.datetime(int(input_match.group('YEAR')), int(input_match.group('MONTH')), int(input_match.group('DAY'))) time_str = input_match.group('TIME') band_num = int(time_str[:2]) + 1 # if start_dt is not None and input_dt < start_dt: # continue # elif end_dt is not None and input_dt > end_dt: # continue # elif date_list and input_dt.date().isoformat() not in date_list: # continue if time_str not in time_list: logging.debug(' Time not in list, skipping') continue logging.debug(' Time: {} {}'.format(input_dt.date(), time_str)) logging.debug(' Band: {}'.format(band_num)) # Determine band numbering/naming input_band_dict = grib_band_names(input_path) # Compute vapour pressure from specific humidity input_ds = gdal.Open(input_path) sph_array = gdc.raster_ds_to_array( input_ds, band=input_band_dict['Specific humidity [kg/kg]'], mask_extent=nldas_extent, return_nodata=False) ea_array = (sph_array * pair_array) / (0.622 + 0.378 * sph_array) # Save the projected array as 32-bit floats gdc.array_to_comp_raster(ea_array.astype(np.float32), output_path, band=band_num) # gdc.block_to_raster( # ea_array.astype(np.float32), output_path, band=band) # gdc.array_to_raster( # ea_array.astype(np.float32), output_path, # output_geo=nldas_geo, output_proj=nldas_proj, # stats_flag=stats_flag) del sph_array input_ds = None if stats_flag: gdc.raster_statistics(output_path) logging.debug('\nScript Complete')
def main(gis_ws, tile_ws, dem_cs, overwrite_flag=False, pyramids_flag=False, stats_flag=False): """Merge, project, and clip NED tiles Args: gis_ws (str): Folder/workspace path of the GIS data for the project tile_ws (str): Folder/workspace path of the DEM tiles dem_cs (int): DEM cellsize (10 or 30m) overwrite_flag (bool): If True, overwrite existing files pyramids_flag (bool): If True, build pyramids/overviews for the output rasters stats_flag (bool): If True, compute statistics for the output rasters Returns: None """ logging.info('\nPrepare DEM tiles') # Inputs output_units = 'METERS' dem_ws = os.path.join(gis_ws, 'dem') scratch_ws = os.path.join(gis_ws, 'scratch') zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img') # Use 1 degree snap point and "cellsize" to get 1x1 degree tiles tile_osr = gdc.epsg_osr(4269) tile_buffer = 0.5 tile_x, tile_y, tile_cs = 0, 0, 1 # Input error checking if not os.path.isdir(gis_ws): logging.error(('\nERROR: The GIS workspace {} ' + 'does not exist').format(gis_ws)) sys.exit() elif not os.path.isdir(tile_ws): logging.error(('\nERROR: The DEM tile workspace {} ' + 'does not exist').format(tile_ws)) sys.exit() elif not os.path.isfile(zone_raster_path): logging.error(('\nERROR: The zone raster {} does not exist' + '\n Try re-running "build_study_area_raster.py"' ).format(zone_raster_path)) sys.exit() elif output_units not in ['FEET', 'METERS']: logging.error('\nERROR: The output units must be FEET or METERS\n') sys.exit() logging.info('\nGIS Workspace: {}'.format(gis_ws)) logging.info('DEM Workspace: {}'.format(dem_ws)) logging.info('Tile Workspace: {}\n'.format(tile_ws)) # Input folder/files if dem_cs == 10: tile_fmt = 'imgn{0:02d}w{1:03d}_13.img' elif dem_cs == 30: tile_fmt = 'imgn{0:02d}w{1:03d}_1.img' # Output folder/files if not os.path.isdir(dem_ws): os.makedirs(dem_ws) # Output file names dem_fmt = 'ned_{0}m{1}.img' # dem_gcs = dem_fmt.format(dem_cs, '_nad83_meters') # dem_feet = dem_fmt.format(dem_cs, '_nad83_feet') # dem_proj = dem_fmt.format(dem_cs, '_albers') # dem_hs = dem_fmt.format(dem_cs, '_hs') dem_gcs_path = os.path.join(dem_ws, dem_fmt.format(dem_cs, '_nad83_meters')) dem_feet_path = os.path.join(dem_ws, dem_fmt.format(dem_cs, '_nad83_feet')) dem_proj_path = os.path.join(dem_ws, dem_fmt.format(dem_cs, '_albers')) dem_hs_path = os.path.join(dem_ws, dem_fmt.format(dem_cs, '_hs')) # f32_nodata = float(np.finfo(np.float32).min) if pyramids_flag: levels = '2 4 8 16 32 64 128' # gdal.SetConfigOption('USE_RRD', 'YES') # gdal.SetConfigOption('HFA_USE_RRD', 'YES') # Reference all output rasters zone raster zone_raster_ds = gdal.Open(zone_raster_path) output_osr = gdc.raster_ds_osr(zone_raster_ds) output_wkt = gdc.raster_ds_proj(zone_raster_ds) output_cs = gdc.raster_ds_cellsize(zone_raster_ds)[0] output_x, output_y = gdc.raster_ds_origin(zone_raster_ds) output_extent = gdc.raster_ds_extent(zone_raster_ds) zone_raster_ds = None logging.debug('\nStudy area properties') logging.debug(' Output OSR: {}'.format(output_osr)) logging.debug(' Output Extent: {}'.format(output_extent)) logging.debug(' Output cellsize: {}'.format(output_cs)) # Project study area extent to DEM tile coordinate system tile_extent = gdc.project_extent(output_extent, output_osr, tile_osr) logging.debug('Output Extent: {}'.format(tile_extent)) # Extent needed to select 1x1 degree tiles tile_extent.buffer_extent(tile_buffer) tile_extent.adjust_to_snap('EXPAND', tile_x, tile_y, tile_cs) logging.debug('Tile Extent: {}'.format(tile_extent)) # Get list of available tiles that intersect the extent input_path_list = sorted( list( set([ tile_fmt.format(lat, -lon) # os.path.join(tile_ws, tile_fmt.format(lat, -lon)) for lon in range(int(tile_extent.xmin), int(tile_extent.xmax)) for lat in range(int(tile_extent.ymax), int( tile_extent.ymin), -1) if os.path.isfile( os.path.join(tile_ws, tile_fmt.format(lat, -lon))) ]))) logging.debug('Tiles') # for input_path in input_path_list: # .debug(' {}'.format(input_path)) # Calculate using GDAL utilities if input_path_list: logging.info('Merging tiles') if os.path.isfile(dem_gcs_path) and overwrite_flag: util.remove_file(dem_gcs_path) # subprocess.call( # 'gdalmanage', 'delete', '-f', 'HFA', dem_gcs_path]) if not os.path.isfile(dem_gcs_path): # gdal_merge.py was only working if shell=True # It would also work to add the scripts folder to the path (in Pythong) # Or the scripts folder could be added to the system PYTHONPATH? args_list = [ 'python', '{}\scripts\gdal_merge.py'.format( sys.exec_prefix), '-o', dem_gcs_path, '-of', 'HFA', '-co', 'COMPRESSED=YES', '-a_nodata', str(f32_nodata) ] + input_path_list logging.debug(args_list) logging.debug('command length: {}'.format(len( ' '.join(args_list)))) subprocess.call(args_list, cwd=tile_ws) # subprocess.call( # 'set', 'GDAL_DATA={}\Lib\site-packages\osgeo\data\gdal'.format(sys.exec_prefix)], # =True) # subprocess.call( # 'gdal_merge.py', '-o', dem_gcs_path, '-of', 'HFA', # '-co', 'COMPRESSED=YES', '-a_nodata', # str(f32_nodata)] + input_path_list, # =True) # Convert DEM from meters to feet if output_units == 'FEET': # DEADBEEF - This won't run when called through subprocess? # subprocess.call( # 'gdal_calc.py', '-A', dem_gcs_path, # '--outfile={}'.format(dem_feet_path), '--calc="0.3048*A"', # '--format', 'HFA', '--co', 'COMPRESSED=YES', # '--NoDataValue={}'.format(str(f32_nodata)), # '--type', 'Float32', '--overwrite'], # =dem_ws, shell=True) # dem_gcs_path = dem_feet_path # Scale the values using custom function m2ft_func(dem_gcs_path) if os.path.isfile(dem_proj_path) and overwrite_flag: subprocess.call(['gdalmanage', 'delete', '-f', 'HFA', dem_proj_path]) if os.path.isfile(dem_hs_path) and overwrite_flag: subprocess.call(['gdalmanage', 'delete', '-f', 'HFA', dem_hs_path]) if (not os.path.isfile(dem_proj_path) and os.path.isfile(dem_gcs_path)): subprocess.call([ 'gdalwarp', '-r', 'bilinear', '-tr', str(output_cs), str(output_cs), '-s_srs', 'EPSG:4269', '-t_srs', output_wkt, '-ot', 'Float32' ] + ['-te'] + str(output_extent).split() + # ['-srcnodata', 'None', '-dstnodata', str(f32_nodata), [ '-of', 'HFA', '-co', 'COMPRESSED=YES', '-overwrite', '-multi', '-wm', '1024', '-wo', 'NUM_THREADS=ALL_CPUS', dem_gcs_path, dem_proj_path ]) if (not os.path.isfile(dem_hs_path) and os.path.isfile(dem_proj_path)): subprocess.call([ 'gdaldem', 'hillshade', dem_proj_path, dem_hs_path, '-of', 'HFA', '-co', 'COMPRESSED=YES' ]) if stats_flag: logging.info('Computing statistics') if os.path.isfile(dem_proj_path): logging.debug(' {}'.format(dem_proj_path)) subprocess.call(['gdalinfo', '-stats', '-nomd', dem_proj_path]) if os.path.isfile(dem_hs_path): logging.debug(' {}'.format(dem_hs_path)) subprocess.call(['gdalinfo', '-stats', '-nomd', dem_hs_path]) if pyramids_flag: logging.info('\nBuilding pyramids') if os.path.isfile(dem_proj_path): logging.debug(' {}'.format(dem_proj_path)) subprocess.call(['gdaladdo', '-ro', dem_proj_path] + levels.split()) if os.path.isfile(dem_hs_path): logging.debug(' {}'.format(dem_hs_path)) subprocess.call(['gdaladdo', '-ro', dem_hs_path] + levels.split()) # subprocess.call( # 'gdaladdo', '-ro', '--config', 'USE_RRD', 'YES', # '--config', 'HFA_USE_RRD', 'YES', dem_proj_path] + levels.split()]) # subprocess.call( # 'gdaladdo', '-ro', '--config', 'USE_RRD', 'YES', # '--config', 'HFA_USE_RRD', 'YES', dem_hs_path] + levels.split()]) if os.path.isfile(os.path.join(dem_ws, dem_gcs_path)): subprocess.call(['gdalmanage', 'delete', '-f', 'HFA', dem_gcs_path])
def main(netcdf_ws=os.getcwd(), ancillary_ws=os.getcwd(), output_ws=os.getcwd(), etr_flag=False, eto_flag=False, start_date=None, end_date=None, extent_path=None, output_extent=None, stats_flag=True, overwrite_flag=False): """Compute daily ETr/ETo from GRIDMET data Args: netcdf_ws (str): folder of GRIDMET netcdf files ancillary_ws (str): folder of ancillary rasters output_ws (str): folder of output rasters etr_flag (bool): if True, compute alfalfa reference ET (ETr) eto_flag (bool): if True, compute grass reference ET (ETo) start_date (str): ISO format date (YYYY-MM-DD) end_date (str): ISO format date (YYYY-MM-DD) extent_path (str): file path defining the output extent output_extent (list): decimal degrees values defining output extent stats_flag (bool): if True, compute raster statistics. Default is True. overwrite_flag (bool): if True, overwrite existing files Returns: None """ logging.info('\nComputing GRIDMET ETo/ETr') np.seterr(invalid='ignore') # Compute ETr and/or ETo if not etr_flag and not eto_flag: logging.info(' ETo/ETr flag(s) not set, defaulting to ETr') etr_flag = True # If a date is not set, process 2017 try: start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d') logging.debug(' Start date: {}'.format(start_dt)) except: start_dt = dt.datetime(2017, 1, 1) logging.info(' Start date: {}'.format(start_dt)) try: end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d') logging.debug(' End date: {}'.format(end_dt)) except: end_dt = dt.datetime(2017, 12, 31) logging.info(' End date: {}'.format(end_dt)) # Save GRIDMET lat, lon, and elevation arrays elev_raster = os.path.join(ancillary_ws, 'gridmet_elev.img') lat_raster = os.path.join(ancillary_ws, 'gridmet_lat.img') # Wind speed is measured at 2m zw = 10 etr_fmt = 'etr_{}_daily_gridmet.img' eto_fmt = 'eto_{}_daily_gridmet.img' # gridmet_re = re.compile('(?P<VAR>\w+)_(?P<YEAR>\d{4}).nc') # GRIDMET band name dictionary gridmet_band_dict = dict() gridmet_band_dict['pr'] = 'precipitation_amount' gridmet_band_dict['srad'] = 'surface_downwelling_shortwave_flux_in_air' gridmet_band_dict['sph'] = 'specific_humidity' gridmet_band_dict['tmmn'] = 'air_temperature' gridmet_band_dict['tmmx'] = 'air_temperature' gridmet_band_dict['vs'] = 'wind_speed' # Get extent/geo from elevation raster gridmet_ds = gdal.Open(elev_raster) gridmet_osr = gdc.raster_ds_osr(gridmet_ds) gridmet_proj = gdc.osr_proj(gridmet_osr) gridmet_cs = gdc.raster_ds_cellsize(gridmet_ds, x_only=True) gridmet_extent = gdc.raster_ds_extent(gridmet_ds) gridmet_full_geo = gridmet_extent.geo(gridmet_cs) gridmet_x, gridmet_y = gridmet_extent.origin() gridmet_ds = None logging.debug(' Projection: {}'.format(gridmet_proj)) logging.debug(' Cellsize: {}'.format(gridmet_cs)) logging.debug(' Geo: {}'.format(gridmet_full_geo)) logging.debug(' Extent: {}'.format(gridmet_extent)) # Subset data to a smaller extent if output_extent is not None: logging.info('\nComputing subset extent & geo') logging.debug(' Extent: {}'.format(output_extent)) gridmet_extent = gdc.Extent(output_extent) gridmet_extent.adjust_to_snap('EXPAND', gridmet_x, gridmet_y, gridmet_cs) gridmet_geo = gridmet_extent.geo(gridmet_cs) logging.debug(' Geo: {}'.format(gridmet_geo)) logging.debug(' Extent: {}'.format(output_extent)) elif extent_path is not None: logging.info('\nComputing subset extent & geo') if extent_path.lower().endswith('.shp'): gridmet_extent = gdc.feature_path_extent(extent_path) extent_osr = gdc.feature_path_osr(extent_path) extent_cs = None else: gridmet_extent = gdc.raster_path_extent(extent_path) extent_osr = gdc.raster_path_osr(extent_path) extent_cs = gdc.raster_path_cellsize(extent_path, x_only=True) gridmet_extent = gdc.project_extent(gridmet_extent, extent_osr, gridmet_osr, extent_cs) gridmet_extent.adjust_to_snap('EXPAND', gridmet_x, gridmet_y, gridmet_cs) gridmet_geo = gridmet_extent.geo(gridmet_cs) logging.debug(' Geo: {}'.format(gridmet_geo)) logging.debug(' Extent: {}'.format(gridmet_extent)) else: gridmet_geo = gridmet_full_geo # Get indices for slicing/clipping input arrays g_i, g_j = gdc.array_geo_offsets(gridmet_full_geo, gridmet_geo, cs=gridmet_cs) g_rows, g_cols = gridmet_extent.shape(cs=gridmet_cs) # Read the elevation and latitude arrays elev_array = gdc.raster_to_array(elev_raster, mask_extent=gridmet_extent, return_nodata=False) lat_array = gdc.raster_to_array(lat_raster, mask_extent=gridmet_extent, return_nodata=False) lat_array *= math.pi / 180 # Check elevation and latitude arrays if np.all(np.isnan(elev_array)): logging.error('\nERROR: The elevation array is all nodata, exiting\n') sys.exit() elif np.all(np.isnan(lat_array)): logging.error('\nERROR: The latitude array is all nodata, exiting\n') sys.exit() # Build output folder etr_ws = os.path.join(output_ws, 'etr') eto_ws = os.path.join(output_ws, 'eto') if etr_flag and not os.path.isdir(etr_ws): os.makedirs(etr_ws) if eto_flag and not os.path.isdir(eto_ws): os.makedirs(eto_ws) # By default, try to process all possible years if start_dt.year == end_dt.year: year_list = [str(start_dt.year)] year_list = sorted(map(str, range((start_dt.year), (end_dt.year + 1)))) # Process each year separately for year_str in year_list: logging.info("\nYear: {}".format(year_str)) year_int = int(year_str) year_days = int(dt.datetime(year_int, 12, 31).strftime('%j')) if start_dt is not None and year_int < start_dt.year: logging.debug(' Before start date, skipping') continue elif end_dt is not None and year_int > end_dt.year: logging.debug(' After end date, skipping') continue # Build input file path tmin_path = os.path.join(netcdf_ws, 'tmmn_{}.nc'.format(year_str)) tmax_path = os.path.join(netcdf_ws, 'tmmx_{}.nc'.format(year_str)) sph_path = os.path.join(netcdf_ws, 'sph_{}.nc'.format(year_str)) rs_path = os.path.join(netcdf_ws, 'srad_{}.nc'.format(year_str)) wind_path = os.path.join(netcdf_ws, 'vs_{}.nc'.format(year_str)) # Check that all input files are present missing_flag = False for input_path in [tmin_path, tmax_path, sph_path, rs_path, wind_path]: if not os.path.isfile(input_path): logging.debug( ' Input NetCDF doesn\'t exist\n {}'.format(input_path)) missing_flag = True if missing_flag: logging.debug(' skipping') continue logging.debug(" {}".format(tmin_path)) logging.debug(" {}".format(tmax_path)) logging.debug(" {}".format(sph_path)) logging.debug(" {}".format(rs_path)) logging.debug(" {}".format(wind_path)) # Create a single raster for each year with 365 bands # Each day will be stored in a separate band etr_raster = os.path.join(etr_ws, etr_fmt.format(year_str)) eto_raster = os.path.join(eto_ws, eto_fmt.format(year_str)) if etr_flag and (overwrite_flag or not os.path.isfile(etr_raster)): logging.debug(' {}'.format(etr_raster)) gdc.build_empty_raster(etr_raster, band_cnt=366, output_dtype=np.float32, output_proj=gridmet_proj, output_cs=gridmet_cs, output_extent=gridmet_extent, output_fill_flag=True) if eto_flag and (overwrite_flag or not os.path.isfile(eto_raster)): logging.debug(' {}'.format(eto_raster)) gdc.build_empty_raster(eto_raster, band_cnt=366, output_dtype=np.float32, output_proj=gridmet_proj, output_cs=gridmet_cs, output_extent=gridmet_extent, output_fill_flag=True) # DEADBEEF - Need to find a way to test if both of these conditionals # did not pass and pass logging debug message to user # Read in the GRIDMET NetCDF file tmin_nc_f = netCDF4.Dataset(tmin_path, 'r') tmax_nc_f = netCDF4.Dataset(tmax_path, 'r') sph_nc_f = netCDF4.Dataset(sph_path, 'r') rs_nc_f = netCDF4.Dataset(rs_path, 'r') wind_nc_f = netCDF4.Dataset(wind_path, 'r') logging.info(' Reading NetCDFs into memory') # Immediatly clip input arrays to save memory tmin_nc = tmin_nc_f.variables[ gridmet_band_dict['tmmn']][:, g_i:g_i + g_cols, g_j:g_j + g_rows].copy() tmax_nc = tmax_nc_f.variables[ gridmet_band_dict['tmmx']][:, g_i:g_i + g_cols, g_j:g_j + g_rows].copy() sph_nc = sph_nc_f.variables[gridmet_band_dict['sph']][:, g_i:g_i + g_cols, g_j:g_j + g_rows].copy() rs_nc = rs_nc_f.variables[gridmet_band_dict['srad']][:, g_i:g_i + g_cols, g_j:g_j + g_rows].copy() wind_nc = wind_nc_f.variables[gridmet_band_dict['vs']][:, g_i:g_i + g_cols, g_j:g_j + g_rows].copy() # tmin_nc = tmin_nc_f.variables[gridmet_band_dict['tmmn']][:] # tmax_nc = tmax_nc_f.variables[gridmet_band_dict['tmmx']][:] # sph_nc = sph_nc_f.variables[gridmet_band_dict['sph']][:] # rs_nc = rs_nc_f.variables[gridmet_band_dict['srad']][:] # wind_nc = wind_nc_f.variables[gridmet_band_dict['vs']][:] # Transpose arrays back to row x col tmin_nc = np.transpose(tmin_nc, (0, 2, 1)) tmax_nc = np.transpose(tmax_nc, (0, 2, 1)) sph_nc = np.transpose(sph_nc, (0, 2, 1)) rs_nc = np.transpose(rs_nc, (0, 2, 1)) wind_nc = np.transpose(wind_nc, (0, 2, 1)) # A numpy array is returned when slicing a masked array # if there are no masked pixels # This is a hack to force the numpy array back to a masked array # For now assume all arrays need to be converted if type(tmin_nc) != np.ma.core.MaskedArray: tmin_nc = np.ma.core.MaskedArray( tmin_nc, np.zeros(tmin_nc.shape, dtype=bool)) if type(tmax_nc) != np.ma.core.MaskedArray: tmax_nc = np.ma.core.MaskedArray( tmax_nc, np.zeros(tmax_nc.shape, dtype=bool)) if type(sph_nc) != np.ma.core.MaskedArray: sph_nc = np.ma.core.MaskedArray(sph_nc, np.zeros(sph_nc.shape, dtype=bool)) if type(rs_nc) != np.ma.core.MaskedArray: rs_nc = np.ma.core.MaskedArray(rs_nc, np.zeros(rs_nc.shape, dtype=bool)) if type(wind_nc) != np.ma.core.MaskedArray: wind_nc = np.ma.core.MaskedArray( wind_nc, np.zeros(wind_nc.shape, dtype=bool)) # Check all valid dates in the year year_dates = date_range(dt.datetime(year_int, 1, 1), dt.datetime(year_int + 1, 1, 1)) for date_dt in year_dates: if start_dt is not None and date_dt < start_dt: logging.debug(' {} - before start date, skipping'.format( date_dt.date())) continue elif end_dt is not None and date_dt > end_dt: logging.debug(' {} - after end date, skipping'.format( date_dt.date())) continue else: logging.info(' {}'.format(date_dt.date())) doy = int(date_dt.strftime('%j')) doy_i = range(1, year_days + 1).index(doy) # Arrays are being read as masked array with a fill value of -9999 # Convert to basic numpy array arrays with nan values try: tmin_ma = tmin_nc[doy_i, :, :] except IndexError: logging.info(' date not in netcdf, skipping') continue tmin_array = tmin_ma.data.astype(np.float32) tmin_nodata = float(tmin_ma.fill_value) tmin_array[tmin_array == tmin_nodata] = np.nan try: tmax_ma = tmax_nc[doy_i, :, :] except IndexError: logging.info(' date not in netcdf, skipping') continue tmax_array = tmax_ma.data.astype(np.float32) tmax_nodata = float(tmax_ma.fill_value) tmax_array[tmax_array == tmax_nodata] = np.nan try: sph_ma = sph_nc[doy_i, :, :] except IndexError: logging.info(' date not in netcdf, skipping') continue sph_array = sph_ma.data.astype(np.float32) sph_nodata = float(sph_ma.fill_value) sph_array[sph_array == sph_nodata] = np.nan try: rs_ma = rs_nc[doy_i, :, :] except IndexError: logging.info(' date not in netcdf, skipping') continue rs_array = rs_ma.data.astype(np.float32) rs_nodata = float(rs_ma.fill_value) rs_array[rs_array == rs_nodata] = np.nan try: wind_ma = wind_nc[doy_i, :, :] except IndexError: logging.info(' date not in netcdf, skipping') continue wind_array = wind_ma.data.astype(np.float32) wind_nodata = float(wind_ma.fill_value) wind_array[wind_array == wind_nodata] = np.nan del tmin_ma, tmax_ma, sph_ma, rs_ma, wind_ma # Since inputs are netcdf, need to create GDAL raster # datasets in order to use gdal_common functions # Create an in memory dataset of the full ETo array tmin_ds = gdc.array_to_mem_ds( tmin_array, output_geo=gridmet_geo, # tmin_array, output_geo=gridmet_full_geo, output_proj=gridmet_proj) tmax_ds = gdc.array_to_mem_ds( tmax_array, output_geo=gridmet_geo, # tmax_array, output_geo=gridmet_full_geo, output_proj=gridmet_proj) sph_ds = gdc.array_to_mem_ds( sph_array, output_geo=gridmet_geo, # sph_array, output_geo=gridmet_full_geo, output_proj=gridmet_proj) rs_ds = gdc.array_to_mem_ds( rs_array, output_geo=gridmet_geo, # rs_array, output_geo=gridmet_full_geo, output_proj=gridmet_proj) wind_ds = gdc.array_to_mem_ds( wind_array, output_geo=gridmet_geo, # wind_array, output_geo=gridmet_full_geo, output_proj=gridmet_proj) # Then extract the subset from the in memory dataset tmin_array = gdc.raster_ds_to_array(tmin_ds, 1, mask_extent=gridmet_extent, return_nodata=False) tmax_array = gdc.raster_ds_to_array(tmax_ds, 1, mask_extent=gridmet_extent, return_nodata=False) sph_array = gdc.raster_ds_to_array(sph_ds, 1, mask_extent=gridmet_extent, return_nodata=False) rs_array = gdc.raster_ds_to_array(rs_ds, 1, mask_extent=gridmet_extent, return_nodata=False) wind_array = gdc.raster_ds_to_array(wind_ds, 1, mask_extent=gridmet_extent, return_nodata=False) del tmin_ds, tmax_ds, sph_ds, rs_ds, wind_ds # Adjust units tmin_array -= 273.15 tmax_array -= 273.15 rs_array *= 0.0864 # ETr/ETo if etr_flag: etr_array = et_common.refet_daily_func(tmin_array, tmax_array, sph_array, rs_array, wind_array, zw, elev_array, lat_array, doy, 'ETR') if eto_flag: eto_array = et_common.refet_daily_func(tmin_array, tmax_array, sph_array, rs_array, wind_array, zw, elev_array, lat_array, doy, 'ETO') # del tmin_array, tmax_array, sph_array, rs_array, wind_array # Save the projected array as 32-bit floats if etr_flag: gdc.array_to_comp_raster(etr_array.astype(np.float32), etr_raster, band=doy, stats_flag=False) # gdc.array_to_raster( # etr_array.astype(np.float32), etr_raster, # output_geo=gridmet_geo, output_proj=gridmet_proj, # stats_flag=stats_flag) del etr_array if eto_flag: gdc.array_to_comp_raster(eto_array.astype(np.float32), eto_raster, band=doy, stats_flag=False) # gdc.array_to_raster( # eto_array.astype(np.float32), eto_raster, # output_geo=gridmet_geo, output_proj=gridmet_proj, # stats_flag=stats_flag) del eto_array del tmin_nc del tmax_nc del sph_nc del rs_nc del wind_nc tmin_nc_f.close() tmax_nc_f.close() sph_nc_f.close() rs_nc_f.close() wind_nc_f.close() del tmin_nc_f, tmax_nc_f, sph_nc_f, rs_nc_f, wind_nc_f if stats_flag and etr_flag: gdc.raster_statistics(etr_raster) if stats_flag and eto_flag: gdc.raster_statistics(eto_raster) logging.debug('\nScript Complete')
def main(image_ws, ini_path, blocksize=2048, smooth_flag=False, stats_flag=False, overwrite_flag=False): """Prep a Landsat scene for METRIC Args: image_ws (str): the landsat scene folder that will be prepped ini_path (str): file path of the input parameters file blocksize (int): defines the size of blocks to process smooth_flag (bool): If True, dilate/erode image to remove fringe/edge pixels stats_flag (bool): if True, compute raster statistics overwrite_flag (bool): If True, overwrite existing files Returns: True is successful """ # Open config file config = python_common.open_ini(ini_path) # Get input parameters logging.debug(' Reading Input File') calc_refl_toa_flag = python_common.read_param( 'calc_refl_toa_flag', True, config, 'INPUTS') calc_refl_toa_qa_flag = python_common.read_param( 'calc_refl_toa_qa_flag', True, config, 'INPUTS') # calc_refl_sur_ledaps_flag = python_common.read_param( # 'calc_refl_sur_ledaps_flag', False, config, 'INPUTS') # calc_refl_sur_qa_flag = python_common.read_param( # 'calc_refl_sur_qa_flag', False, config, 'INPUTS') calc_ts_bt_flag = python_common.read_param( 'calc_ts_bt_flag', True, config, 'INPUTS') # Use QA band to set common area # Fmask cloud, shadow, & snow pixels will be removed from common area calc_fmask_common_flag = python_common.read_param( 'calc_fmask_common_flag', True, config, 'INPUTS') fmask_buffer_flag = python_common.read_param( 'fmask_buffer_flag', False, config, 'INPUTS') fmask_erode_flag = python_common.read_param( 'fmask_erode_flag', False, config, 'INPUTS') if fmask_erode_flag: fmask_erode_cells = int(python_common.read_param( 'fmask_erode_cells', 10, config, 'INPUTS')) if fmask_erode_cells == 0 and fmask_erode_flag: fmask_erode_flag = False # Number of cells to buffer Fmask clouds # For now use the same buffer radius and apply to if fmask_buffer_flag: fmask_buffer_cells = int(python_common.read_param( 'fmask_buffer_cells', 25, config, 'INPUTS')) if fmask_buffer_cells == 0 and fmask_buffer_flag: fmask_buffer_flag = False # Include hand made cloud masks cloud_mask_flag = python_common.read_param( 'cloud_mask_flag', False, config, 'INPUTS') cloud_mask_ws = "" if cloud_mask_flag: cloud_mask_ws = config.get('INPUTS', 'cloud_mask_ws') # Extract separate Fmask rasters calc_fmask_flag = python_common.read_param( 'calc_fmask_flag', True, config, 'INPUTS') calc_fmask_cloud_flag = python_common.read_param( 'calc_fmask_cloud_flag', True, config, 'INPUTS') calc_fmask_snow_flag = python_common.read_param( 'calc_fmask_snow_flag', True, config, 'INPUTS') calc_fmask_water_flag = python_common.read_param( 'calc_fmask_water_flag', True, config, 'INPUTS') # Keep Landsat DN, LEDAPS, and Fmask rasters keep_dn_flag = python_common.read_param( 'keep_dn_flag', True, config, 'INPUTS') # keep_sr_flag = python_common.read_param( # 'keep_sr_flag', True, config, 'INPUTS') # For this to work I would need to pass in the metric input file # calc_elev_flag = python_common.read_param( # 'calc_elev_flag', False, config, 'INPUTS') # calc_landuse_flag = python_common.read_param( # 'calc_landuse_flag', False, config, 'INPUTS') # calc_acca_cloud_flag = python_common.read_param( # 'calc_acca_cloud_flag', True, config, 'INPUTS') # calc_acca_snow_flag = python_common.read_param( # 'calc_acca_snow_flag', True, config, 'INPUTS') # calc_ledaps_dem_land_flag = python_common.read_param( # 'calc_ledaps_dem_land_flag', False, config, 'INPUTS') # calc_ledaps_veg_flag = python_common.read_param( # 'calc_ledaps_veg_flag', False, config, 'INPUTS') # calc_ledaps_snow_flag = python_common.read_param( # 'calc_ledaps_snow_flag', False, config, 'INPUTS') # calc_ledaps_land_flag = python_common.read_param( # 'calc_ledaps_land_flag', False, config, 'INPUTS') # calc_ledaps_cloud_flag = python_common.read_param( # 'calc_ledaps_cloud_flag', False, config, 'INPUTS') # Interpolate/clip/project hourly rasters for each Landsat scene # calc_metric_flag = python_common.read_param( # 'calc_metric_flag', False, config, 'INPUTS') calc_metric_ea_flag = python_common.read_param( 'calc_metric_ea_flag', False, config, 'INPUTS') calc_metric_wind_flag = python_common.read_param( 'calc_metric_wind_flag', False, config, 'INPUTS') calc_metric_etr_flag = python_common.read_param( 'calc_metric_etr_flag', False, config, 'INPUTS') calc_metric_tair_flag = python_common.read_param( 'calc_metric_tair_flag', False, config, 'INPUTS') # Interpolate/clip/project AWC and daily ETr/PPT rasters # to compute SWB Ke for each Landsat scene calc_swb_ke_flag = python_common.read_param( 'calc_swb_ke_flag', False, config, 'INPUTS') if cloud_mask_flag: spinup_days = python_common.read_param( 'swb_spinup_days', 30, config, 'INPUTS') min_spinup_days = python_common.read_param( 'swb_min_spinup_days', 5, config, 'INPUTS') # Round ea raster to N digits to save space rounding_digits = python_common.read_param( 'rounding_digits', 3, config, 'INPUTS') env = gdc.env image = et_image.Image(image_ws, env) np.seterr(invalid='ignore', divide='ignore') gdal.UseExceptions() # Input file paths dn_image_dict = et_common.landsat_band_image_dict( image.orig_data_ws, image.image_re) # # Open METRIC config file # if config_file: # logging.info( # log_f.format('METRIC INI File:', os.path.basename(config_file))) # config = configparser.ConfigParser() # try: # config.read(config_file) # except: # logging.error('\nERROR: Config file could not be read, ' + # 'is not an input file, or does not exist\n' + # 'ERROR: config_file = {}\n').format(config_file) # sys.exit() # # Overwrite # overwrite_flag = read_param('overwrite_flag', True, config) # # # Elevation and landuse parameters/flags from METRIC input file # calc_elev_flag = read_param('save_dem_raster_flag', True, config) # calc_landuse_flag = read_param( # 'save_landuse_raster_flag', True, config) # if calc_elev_flag: # elev_pr_path = config.get('INPUTS','dem_raster') # if calc_landuse_flag: # landuse_pr_path = config.get('INPUTS', 'landuse_raster') # else: # overwrite_flag = False # calc_elev_flag = False # calc_landuse_flag = False # # Elev raster must exist # if calc_elev_flag and not os.path.isfile(elev_pr_path): # logging.error('\nERROR: Elevation raster {} does not exist\n'.format( # elev_pr_path)) # return False # Landuse raster must exist # if calc_landuse_flag and not os.path.isfile(landuse_pr_path): # logging.error('\nERROR: Landuse raster {} does not exist\n'.format( # landuse_pr_path)) # return False # Removing ancillary files before checking for inputs if os.path.isdir(os.path.join(image.orig_data_ws, 'gap_mask')): shutil.rmtree(os.path.join(image.orig_data_ws, 'gap_mask')) for item in os.listdir(image.orig_data_ws): if (image.type == 'Landsat7' and (item.endswith('_B8.TIF') or item.endswith('_B6_VCID_2.TIF'))): os.remove(os.path.join(image.orig_data_ws, item)) elif (image.type == 'Landsat8' and (item.endswith('_B1.TIF') or item.endswith('_B8.TIF') or item.endswith('_B9.TIF') or item.endswith('_B11.TIF'))): os.remove(os.path.join(image.orig_data_ws, item)) elif (item.endswith('_VER.jpg') or item.endswith('_VER.txt') or item.endswith('_GCP.txt') or item == 'README.GTF'): os.remove(os.path.join(image.orig_data_ws, item)) # Check correction level (image must be L1T to process) if image.correction != 'L1TP': logging.debug(' Image is not L1TP corrected, skipping') return False # calc_fmask_common_flag = False # calc_refl_toa_flag = False # calc_ts_bt_flag = False # calc_metric_ea_flag = False # calc_metric_wind_flag = False # calc_metric_etr_flag = False # overwrite_flag = False # QA band must exist if (calc_fmask_common_flag and image.qa_band not in dn_image_dict.keys()): logging.warning( ('\nQA band does not exist but calc_fmask_common_flag=True' + '\n Setting calc_fmask_common_flag=False\n {}').format( os.path.basename(image.qa_input_raster))) calc_fmask_common_flag = False if cloud_mask_flag and not os.path.isdir(cloud_mask_ws): logging.warning( ('\ncloud_mask_ws is not a directory but cloud_mask_flag=True.' + '\n Setting cloud_mask_flag=False\n {}').format(cloud_mask_ws)) cloud_mask_flag = False # Check for Landsat TOA images if (calc_refl_toa_flag and (set(list(image.band_toa_dict.keys()) + [image.thermal_band, image.qa_band]) != set(dn_image_dict.keys()))): logging.warning( '\nMissing Landsat images but calc_refl_toa_flag=True' + '\n Setting calc_refl_toa_flag=False') calc_refl_toa_flag = False # Check for Landsat brightness temperature image if calc_ts_bt_flag and image.thermal_band not in dn_image_dict.keys(): logging.warning( '\nThermal band image does not exist but calc_ts_bt_flag=True' + '\n Setting calc_ts_bt_flag=False') calc_ts_bt_flag = False # DEADBEEF - Should the function return False if Ts doesn't exist? # return False # Check for METRIC hourly/daily input folders if calc_metric_ea_flag: metric_ea_input_ws = config.get('INPUTS', 'metric_ea_input_folder') if not os.path.isdir(metric_ea_input_ws): logging.warning( ('\nHourly Ea folder does not exist but calc_metric_ea_flag=True' + '\n Setting calc_metric_ea_flag=False\n {}').format( metric_ea_input_ws)) calc_metric_ea_flag = False if calc_metric_wind_flag: metric_wind_input_ws = config.get('INPUTS', 'metric_wind_input_folder') if not os.path.isdir(metric_wind_input_ws): logging.warning( ('\nHourly wind folder does not exist but calc_metric_wind_flag=True' + '\n Setting calc_metric_wind_flag=False\n {}').format( metric_wind_input_ws)) calc_metric_wind_flag = False if calc_metric_etr_flag: metric_etr_input_ws = config.get('INPUTS', 'metric_etr_input_folder') if not os.path.isdir(metric_etr_input_ws): logging.warning( ('\nHourly ETr folder does not exist but calc_metric_etr_flag=True' + '\n Setting calc_metric_etr_flag=False\n {}').format( metric_etr_input_ws)) calc_metric_etr_flag = False if calc_metric_tair_flag: metric_tair_input_ws = config.get('INPUTS', 'metric_tair_input_folder') if not os.path.isdir(metric_tair_input_ws): logging.warning( ('\nHourly Tair folder does not exist but calc_metric_tair_flag=True' + '\n Setting calc_metric_tair_flag=False\n {}').format( metric_tair_input_ws)) calc_metric_tair_flag = False if (calc_metric_ea_flag or calc_metric_wind_flag or calc_metric_etr_flag or calc_metric_tair_flag): metric_hourly_re = re.compile(config.get('INPUTS', 'metric_hourly_re')) metric_daily_re = re.compile(config.get('INPUTS', 'metric_daily_re')) if calc_swb_ke_flag: awc_input_path = config.get('INPUTS', 'awc_input_path') etr_input_ws = config.get('INPUTS', 'etr_input_folder') ppt_input_ws = config.get('INPUTS', 'ppt_input_folder') etr_input_re = re.compile(config.get('INPUTS', 'etr_input_re')) ppt_input_re = re.compile(config.get('INPUTS', 'ppt_input_re')) if not os.path.isfile(awc_input_path): logging.warning( ('\nAWC raster does not exist but calc_swb_ke_flag=True' + '\n Setting calc_swb_ke_flag=False\n {}').format( awc_input_path)) calc_swb_ke_flag = False if not os.path.isdir(etr_input_ws): logging.warning( ('\nDaily ETr folder does not exist but calc_swb_ke_flag=True' + '\n Setting calc_swb_ke_flag=False\n {}').format( etr_input_ws)) calc_swb_ke_flag = False if not os.path.isdir(ppt_input_ws): logging.warning( ('\nDaily PPT folder does not exist but calc_swb_ke_flag=True' + '\n Setting calc_swb_ke_flag=False\n {}').format( ppt_input_ws)) calc_swb_ke_flag = False # Build folders for support rasters if ((calc_fmask_common_flag or calc_refl_toa_flag or # calc_refl_sur_ledaps_flag or calc_ts_bt_flag or calc_metric_ea_flag or calc_metric_wind_flag or calc_metric_etr_flag or calc_metric_tair_flag or calc_swb_ke_flag) and not os.path.isdir(image.support_ws)): os.makedirs(image.support_ws) if calc_refl_toa_flag and not os.path.isdir(image.refl_toa_ws): os.makedirs(image.refl_toa_ws) # if calc_refl_sur_ledaps_flag and not os.path.isdir(image.refl_sur_ws): # os.makedirs(image.refl_sur_ws) # Apply overwrite flag if overwrite_flag: overwrite_list = [ image.fmask_cloud_raster, image.fmask_snow_raster, image.fmask_water_raster # image.elev_raster, image.landuse_raster # image.common_area_raster ] for overwrite_path in overwrite_list: try: python_common.remove_file(image.fmask_cloud_raster) except: pass # Use QA band to build common area rasters logging.info('\nCommon Area Raster') qa_ds = gdal.Open(dn_image_dict[image.qa_band], 0) common_geo = gdc.raster_ds_geo(qa_ds) common_extent = gdc.raster_ds_extent(qa_ds) common_proj = gdc.raster_ds_proj(qa_ds) common_osr = gdc.raster_ds_osr(qa_ds) # Initialize common_area as all non-fill QA values qa_array = gdc.raster_ds_to_array(qa_ds, return_nodata=False) common_array = qa_array != 1 common_rows, common_cols = common_array.shape del qa_ds # First try applying user defined cloud masks cloud_mask_path = os.path.join( cloud_mask_ws, image.folder_id + '_mask.shp') if cloud_mask_flag and os.path.isfile(cloud_mask_path): logging.info(' Applying cloud mask shapefile') feature_path = os.path.join( cloud_mask_ws, (image.folder_id + '_mask.shp')) logging.info(' {}'.format(feature_path)) cloud_mask_memory_ds = gdc.polygon_to_raster_ds( feature_path, nodata_value=0, burn_value=1, output_osr=common_osr, output_cs=30, output_extent=common_extent) cloud_array = gdc.raster_ds_to_array( cloud_mask_memory_ds, return_nodata=False) # DEADBEEF - If user sets a cloud mask, # it is probably better than Fmask # Eventually change "if" calc_fmask_common_flag: to "elif" common_array[cloud_array == 1] = 0 del cloud_mask_memory_ds, cloud_array if calc_fmask_common_flag: fmask_array = et_numpy.bqa_fmask_func(qa_array) fmask_mask = (fmask_array >= 2) & (fmask_array <= 4) if fmask_erode_flag: logging.info( (' Eroding and dilating Fmask clouds, shadows, and snow ' + '{} cells\n to remove errantly masked pixels.').format( fmask_erode_cells)) fmask_mask = ndimage.binary_erosion( fmask_mask, iterations=fmask_erode_cells, structure=ndimage.generate_binary_structure(2, 2)) fmask_mask = ndimage.binary_dilation( fmask_mask, iterations=fmask_erode_cells, structure=ndimage.generate_binary_structure(2, 2)) if fmask_buffer_flag: logging.info( (' Buffering Fmask clouds, shadows, and snow ' + '{} cells').format(fmask_buffer_cells)) # Only buffer clouds, shadow, and snow (not water or nodata) if fmask_mask is None: fmask_mask = (fmask_array >= 2) & (fmask_array <= 4) fmask_mask = ndimage.binary_dilation( fmask_mask, iterations=fmask_buffer_cells, structure=ndimage.generate_binary_structure(2, 2)) # Reset common_array for buffered cells common_array[fmask_mask] = 0 del fmask_array, fmask_mask if common_array is not None: # Erode and dilate to remove fringe on edge # Default is to not smooth, but user can force smoothing if smooth_flag: common_array = smooth_func(common_array) # Check that there are some cloud free pixels if not np.any(common_array): logging.error(' ERROR: There are no cloud/snow free pixels') return False # Always overwrite common area raster # if not os.path.isfile(image.common_area_raster): gdc.array_to_raster( common_array, image.common_area_raster, output_geo=common_geo, output_proj=common_proj, stats_flag=stats_flag) # Print common geo/extent logging.debug(' Common geo: {}'.format(common_geo)) logging.debug(' Common extent: {}'.format(common_extent)) # Extract Fmask components as separate rasters if (calc_fmask_flag or calc_fmask_cloud_flag or calc_fmask_snow_flag or calc_fmask_water_flag): logging.info('\nFmask') fmask_array = et_numpy.bqa_fmask_func(qa_array) # Save Fmask data as separate rasters if (calc_fmask_flag and not os.path.isfile(image.fmask_output_raster)): gdc.array_to_raster( fmask_array.astype(np.uint8), image.fmask_output_raster, output_geo=common_geo, output_proj=common_proj, mask_array=None, output_nodata=255, stats_flag=stats_flag) if (calc_fmask_cloud_flag and not os.path.isfile(image.fmask_cloud_raster)): fmask_cloud_array = (fmask_array == 2) | (fmask_array == 4) gdc.array_to_raster( fmask_cloud_array.astype(np.uint8), image.fmask_cloud_raster, output_geo=common_geo, output_proj=common_proj, mask_array=None, output_nodata=255, stats_flag=stats_flag) del fmask_cloud_array if (calc_fmask_snow_flag and not os.path.isfile(image.fmask_snow_raster)): fmask_snow_array = (fmask_array == 3) gdc.array_to_raster( fmask_snow_array.astype(np.uint8), image.fmask_snow_raster, output_geo=common_geo, output_proj=common_proj, mask_array=None, output_nodata=255, stats_flag=stats_flag) del fmask_snow_array if (calc_fmask_water_flag and not os.path.isfile(image.fmask_water_raster)): fmask_water_array = (fmask_array == 1) gdc.array_to_raster( fmask_water_array.astype(np.uint8), image.fmask_water_raster, output_geo=common_geo, output_proj=common_proj, mask_array=None, output_nodata=255, stats_flag=stats_flag) del fmask_water_array del fmask_array # # Calculate elevation # if calc_elev_flag and not os.path.isfile(elev_path): # logging.info('Elevation') # elev_array, elev_nodata = gdc.raster_to_array( # elev_pr_path, 1, common_extent) # gdc.array_to_raster( # elev_array, elev_raster, # output_geo=common_geo, output_proj=env.snap_proj, # mask_array=common_array, stats_flag=stats_flag) # del elev_array, elev_nodata, elev_path # # # Calculate landuse # if calc_landuse_flag and not os.path.isfile(landuse_raster): # logging.info('Landuse') # landuse_array, landuse_nodata = gdc.raster_to_array( # landuse_pr_path, 1, common_extent) # gdc.array_to_raster( # landuse_array, landuse_raster, # output_geo=common_geo, output_proj=env.snap_proj, # mask_array=common_array, stats_flag=stats_flag) # del landuse_array, landuse_nodata, landuse_raster # Calculate toa reflectance # f32_gtype, f32_nodata = numpy_to_gdal_type(np.float32) if calc_refl_toa_flag: logging.info('Top-of-Atmosphere Reflectance') if os.path.isfile(image.refl_toa_raster) and overwrite_flag: logging.debug(' Overwriting: {}'.format( image.refl_toa_raster)) python_common.remove_file(image.refl_toa_raster) if not os.path.isfile(image.refl_toa_raster): # First build empty composite raster gdc.build_empty_raster( image.refl_toa_raster, image.band_toa_cnt, np.float32, None, env.snap_proj, env.cellsize, common_extent) # cos_theta_solar_flt = et_common.cos_theta_solar_func( # image.sun_elevation) # Process by block logging.info('Processing by block') logging.debug(' Mask cols/rows: {}/{}'.format( common_cols, common_rows)) for b_i, b_j in gdc.block_gen(common_rows, common_cols, blocksize): logging.debug(' Block y: {:5d} x: {:5d}'.format(b_i, b_j)) block_data_mask = gdc.array_to_block( common_array, b_i, b_j, blocksize).astype(np.bool) block_rows, block_cols = block_data_mask.shape block_geo = gdc.array_offset_geo(common_geo, b_j, b_i) block_extent = gdc.geo_extent( block_geo, block_rows, block_cols) logging.debug(' Block rows: {} cols: {}'.format( block_rows, block_cols)) logging.debug(' Block extent: {}'.format(block_extent)) logging.debug(' Block geo: {}'.format(block_geo)) # Process each TOA band # for band, band_i in sorted(image.band_toa_dict.items()): for band, dn_image in sorted(dn_image_dict.items()): if band not in image.band_toa_dict.keys(): continue # thermal_band_flag = (band == image.thermal_band) # Set 0 as nodata value gdc.raster_path_set_nodata(dn_image, 0) # Calculate TOA reflectance dn_array, dn_nodata = gdc.raster_to_array( dn_image, 1, block_extent) dn_array = dn_array.astype(np.float64) # dn_array = dn_array.astype(np.float32) dn_array[dn_array == 0] = np.nan # if image.type in ['Landsat4', 'Landsat5', 'Landsat7']: refl_toa_array = et_numpy.l457_refl_toa_band_func( dn_array, image.cos_theta_solar, image.dr, image.esun_dict[band], image.lmin_dict[band], image.lmax_dict[band], image.qcalmin_dict[band], image.qcalmax_dict[band]) elif image.type in ['Landsat8']: refl_toa_array = et_numpy.l8_refl_toa_band_func( dn_array, image.cos_theta_solar, image.refl_mult_dict[band], image.refl_add_dict[band]) # if (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and # not thermal_band_flag): # refl_toa_array = et_numpy.l457_refl_toa_band_func( # dn_array, image.cos_theta_solar, # image.dr, image.esun_dict[band], # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], # image.qcalmax_dict[band]) # # image.rad_mult_dict[band], # # image.rad_add_dict[band]) # elif (image.type in ['Landsat8'] and # not thermal_band_flag): # refl_toa_array = et_numpy.l8_refl_toa_band_func( # dn_array, image.cos_theta_solar, # image.refl_mult_dict[band], # image.refl_add_dict[band]) # elif (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and # thermal_band_flag): # refl_toa_array = et_numpy.l457_ts_bt_band_func( # dn_array, # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], # image.qcalmax_dict[band], # # image.rad_mult_dict[band], # # image.rad_add_dict[band], # image.k1_dict[band], image.k2_dict[band]) # elif (image.type in ['Landsat8'] and # thermal_band_flag): # refl_toa_array = et_numpy.l8_ts_bt_band_func( # dn_array, # image.rad_mult_dict[band], # image.rad_add_dict[band], # image.k1_dict[band], image.k2_dict[band]) # refl_toa_array = et_numpy.refl_toa_band_func( # dn_array, cos_theta_solar_flt, # image.dr, image.esun_dict[band], # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], image.qcalmax_dict[band], # thermal_band_flag) gdc.block_to_raster( refl_toa_array.astype(np.float32), image.refl_toa_raster, b_i, b_j, band=image.band_toa_dict[band]) # gdc.array_to_comp_raster( # refl_toa_array.astype(np.float32), # image.refl_toa_raster, # image.band_toa_dict[band], common_array) del refl_toa_array, dn_array if stats_flag: gdc.raster_statistics(image.refl_toa_raster) # # Process each TOA band # # for band, band_i in sorted(image.band_toa_dict.items()): # for band, dn_image in sorted(dn_image_dict.items()): # thermal_band_flag = (band == image.thermal_band) # # Set 0 as nodata value # gdc.raster_path_set_nodata(dn_image, 0) # # Calculate TOA reflectance # dn_array, dn_nodata = gdc.raster_to_array( # dn_image, 1, common_extent) # dn_array = dn_array.astype(np.float64) # # dn_array = dn_array.astype(np.float32) # dn_array[dn_array == 0] = np.nan # # # if (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and # not thermal_band_flag): # refl_toa_array = et_numpy.l457_refl_toa_band_func( # dn_array, image.cos_theta_solar, # image.dr, image.esun_dict[band], # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], image.qcalmax_dict[band]) # # image.rad_mult_dict[band], image.rad_add_dict[band]) # elif (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and # thermal_band_flag): # refl_toa_array = et_numpy.l457_ts_bt_band_func( # dn_array, image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], image.qcalmax_dict[band], # # image.rad_mult_dict[band], image.rad_add_dict[band], # image.k1_dict[band], image.k2_dict[band]) # elif (image.type in ['Landsat8'] and # not thermal_band_flag): # refl_toa_array = et_numpy.l8_refl_toa_band_func( # dn_array, image.cos_theta_solar, # image.refl_mult_dict[band], image.refl_add_dict[band]) # elif (image.type in ['Landsat8'] and # thermal_band_flag): # refl_toa_array = et_numpy.l8_ts_bt_band_func( # dn_array, # image.rad_mult_dict[band], image.rad_add_dict[band], # image.k1_dict[band], image.k2_dict[band]) # # refl_toa_array = et_numpy.refl_toa_band_func( # # dn_array, cos_theta_solar_flt, # # image.dr, image.esun_dict[band], # # image.lmin_dict[band], image.lmax_dict[band], # # image.qcalmin_dict[band], image.qcalmax_dict[band], # # thermal_band_flag) # gdc.array_to_comp_raster( # refl_toa_array.astype(np.float32), image.refl_toa_raster, # image.band_toa_dict[band], common_array) # del refl_toa_array, dn_array # Calculate brightness temperature if calc_ts_bt_flag: logging.info('Brightness Temperature') if os.path.isfile(image.ts_bt_raster) and overwrite_flag: logging.debug(' Overwriting: {}'.format(image.ts_bt_raster)) python_common.remove_file(image.ts_bt_raster) if not os.path.isfile(image.ts_bt_raster): band = image.thermal_band thermal_dn_path = dn_image_dict[band] gdc.raster_path_set_nodata(thermal_dn_path, 0) thermal_dn_array, thermal_dn_nodata = gdc.raster_to_array( thermal_dn_path, 1, common_extent, return_nodata=True) thermal_dn_mask = thermal_dn_array != thermal_dn_nodata if image.type in ['Landsat4', 'Landsat5', 'Landsat7']: ts_bt_array = et_numpy.l457_ts_bt_band_func( thermal_dn_array, image.lmin_dict[band], image.lmax_dict[band], image.qcalmin_dict[band], image.qcalmax_dict[band], # image.rad_mult_dict[band], image.rad_add_dict[band], image.k1_dict[band], image.k2_dict[band]) elif image.type in ['Landsat8']: ts_bt_array = et_numpy.l8_ts_bt_band_func( thermal_dn_array, image.rad_mult_dict[band], image.rad_add_dict[band], image.k1_dict[band], image.k2_dict[band]) # thermal_rad_array = et_numpy.refl_toa_band_func( # thermal_dn_array, image.cos_theta_solar, # image.dr, image.esun_dict[band], # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], image.qcalmax_dict[band], # thermal_band_flag=True) # ts_bt_array = et_numpy.ts_bt_func( # thermal_rad_array, image.k1_dict[image.thermal_band], # image.k2_dict[image.thermal_band]) ts_bt_array[~thermal_dn_mask] = np.nan gdc.array_to_raster( ts_bt_array, image.ts_bt_raster, output_geo=common_geo, output_proj=env.snap_proj, # mask_array=common_array, stats_flag=stats_flag) # del thermal_dn_array, thermal_rad_array del thermal_dn_path, thermal_dn_array, ts_bt_array # Interpolate/project/clip METRIC hourly/daily rasters if (calc_metric_ea_flag or calc_metric_wind_flag or calc_metric_etr_flag): logging.info('METRIC hourly/daily rasters') # Get bracketing hours from image acquisition time image_prev_dt = image.acq_datetime.replace( minute=0, second=0, microsecond=0) image_next_dt = image_prev_dt + timedelta(seconds=3600) # Get NLDAS properties from one of the images input_ws = os.path.join( metric_etr_input_ws, str(image_prev_dt.year)) try: input_path = [ os.path.join(input_ws, file_name) for file_name in os.listdir(input_ws) for match in [metric_hourly_re.match(file_name)] if (match and (image_prev_dt.strftime('%Y%m%d') == match.group('YYYYMMDD')))][0] except IndexError: logging.error(' No hourly file for {}'.format( image_prev_dt.strftime('%Y-%m-%d %H00'))) return False try: input_ds = gdal.Open(input_path) input_osr = gdc.raster_ds_osr(input_ds) # input_proj = gdc.osr_proj(input_osr) input_extent = gdc.raster_ds_extent(input_ds) input_cs = gdc.raster_ds_cellsize(input_ds, x_only=True) # input_geo = input_extent.geo(input_cs) input_x, input_y = input_extent.origin() input_ds = None except: logging.error(' Could not get default input image properties') logging.error(' {}'.format(input_path)) return False # Project Landsat scene extent to NLDAS GCS common_gcs_osr = common_osr.CloneGeogCS() common_gcs_extent = gdc.project_extent( common_extent, common_osr, common_gcs_osr, cellsize=env.cellsize) common_gcs_extent.buffer_extent(0.1) common_gcs_extent.adjust_to_snap( 'EXPAND', input_x, input_y, input_cs) # common_gcs_geo = common_gcs_extent.geo(input_cs) def metric_weather_func(output_raster, input_ws, input_re, prev_dt, next_dt, resample_method=gdal.GRA_NearestNeighbour, rounding_flag=False): """Interpolate/project/clip METRIC hourly rasters""" logging.debug(' Output: {}'.format(output_raster)) if os.path.isfile(output_raster): if overwrite_flag: logging.debug(' Overwriting output') python_common.remove_file(output_raster) else: logging.debug(' Skipping, file already exists ' + 'and overwrite is False') return False prev_ws = os.path.join(input_ws, str(prev_dt.year)) next_ws = os.path.join(input_ws, str(next_dt.year)) # Technically previous and next could come from different days # or even years, although this won't happen in the U.S. try: prev_path = [ os.path.join(prev_ws, input_name) for input_name in os.listdir(prev_ws) for input_match in [input_re.match(input_name)] if (input_match and (prev_dt.strftime('%Y%m%d') == input_match.group('YYYYMMDD')))][0] logging.debug(' Input prev: {}'.format(prev_path)) except IndexError: logging.error(' No previous hourly file') logging.error(' {}'.format(prev_dt)) return False try: next_path = [ os.path.join(next_ws, input_name) for input_name in os.listdir(next_ws) for input_match in [input_re.match(input_name)] if (input_match and (next_dt.strftime('%Y%m%d') == input_match.group('YYYYMMDD')))][0] logging.debug(' Input next: {}'.format(next_path)) except IndexError: logging.error(' No next hourly file') logging.error(' {}'.format(next_dt)) return False # Band numbers are 1's based prev_band = int(prev_dt.strftime('%H')) + 1 next_band = int(next_dt.strftime('%H')) + 1 logging.debug(' Input prev band: {}'.format(prev_band)) logging.debug(' Input next band: {}'.format(next_band)) # Read arrays prev_array = gdc.raster_to_array( prev_path, band=prev_band, mask_extent=common_gcs_extent, return_nodata=False) next_array = gdc.raster_to_array( next_path, band=next_band, mask_extent=common_gcs_extent, return_nodata=False) if not np.any(prev_array) or not np.any(next_array): logging.warning('\nWARNING: Input NLDAS array is all nodata\n') return None output_array = hourly_interpolate_func( prev_array, next_array, prev_dt, next_dt, image.acq_datetime) output_array = gdc.project_array( output_array, resample_method, input_osr, input_cs, common_gcs_extent, common_osr, env.cellsize, common_extent, output_nodata=None) # Apply common area mask output_array[~common_array] = np.nan # Reduce the file size by rounding to the nearest n digits if rounding_flag: output_array = np.around(output_array, rounding_digits) # Force output to 32-bit float gdc.array_to_raster( output_array.astype(np.float32), output_raster, output_geo=common_geo, output_proj=common_proj, stats_flag=stats_flag) del output_array return True # Ea - Project to Landsat scene after clipping if calc_metric_ea_flag: logging.info(' Hourly vapor pressure (Ea)') metric_weather_func( image.metric_ea_raster, metric_ea_input_ws, metric_hourly_re, image_prev_dt, image_next_dt, gdal.GRA_Bilinear, rounding_flag=True) # Wind - Project to Landsat scene after clipping if calc_metric_wind_flag: logging.info(' Hourly windspeed') metric_weather_func( image.metric_wind_raster, metric_wind_input_ws, metric_hourly_re, image_prev_dt, image_next_dt, gdal.GRA_NearestNeighbour, rounding_flag=False) # ETr - Project to Landsat scene after clipping if calc_metric_etr_flag: logging.info(' Hourly reference ET (ETr)') metric_weather_func( image.metric_etr_raster, metric_etr_input_ws, metric_hourly_re, image_prev_dt, image_next_dt, gdal.GRA_NearestNeighbour, rounding_flag=False) # ETr 24hr - Project to Landsat scene after clipping if calc_metric_etr_flag: logging.info(' Daily reference ET (ETr)') logging.debug(' Output: {}'.format( image.metric_etr_24hr_raster)) if (os.path.isfile(image.metric_etr_24hr_raster) and overwrite_flag): logging.debug(' Overwriting output') os.remove(image.metric_etr_24hr_raster) if not os.path.isfile(image.metric_etr_24hr_raster): etr_prev_ws = os.path.join( metric_etr_input_ws, str(image_prev_dt.year)) try: input_path = [ os.path.join(etr_prev_ws, file_name) for file_name in os.listdir(etr_prev_ws) for match in [metric_daily_re.match(file_name)] if (match and (image_prev_dt.strftime('%Y%m%d') == match.group('YYYYMMDD')))][0] logging.debug(' Input: {}'.format(input_path)) except IndexError: logging.error(' No daily file for {}'.format( image_prev_dt.strftime('%Y-%m-%d'))) return False output_array = gdc.raster_to_array( input_path, mask_extent=common_gcs_extent, return_nodata=False) output_array = gdc.project_array( output_array, gdal.GRA_NearestNeighbour, input_osr, input_cs, common_gcs_extent, common_osr, env.cellsize, common_extent, output_nodata=None) # Apply common area mask output_array[~common_array] = np.nan # Reduce the file size by rounding to the nearest n digits # output_array = np.around(output_array, rounding_digits) gdc.array_to_raster( output_array, image.metric_etr_24hr_raster, output_geo=common_geo, output_proj=common_proj, stats_flag=stats_flag) del output_array del input_path # Tair - Project to Landsat scene after clipping if calc_metric_tair_flag: logging.info(' Hourly air temperature (Tair)') metric_weather_func( image.metric_tair_raster, metric_tair_input_ws, metric_hourly_re, image_prev_dt, image_next_dt, gdal.GRA_NearestNeighbour, rounding_flag=False) # Cleanup del image_prev_dt, image_next_dt # Soil Water Balance if calc_swb_ke_flag: logging.info('Daily soil water balance') # Check if output file already exists logging.debug(' Ke: {}'.format(image.ke_raster)) if os.path.isfile(image.ke_raster): if overwrite_flag: logging.debug(' Overwriting output') python_common.remove_file(image.ke_raster) else: logging.debug(' Skipping, file already ' + 'exists and overwrite is False') return False ke_array = et_common.raster_swb_func( image.acq_datetime, common_osr, env.cellsize, common_extent, awc_input_path, etr_input_ws, etr_input_re, ppt_input_ws, ppt_input_re, spinup_days=spinup_days, min_spinup_days=min_spinup_days) # Apply common area mask ke_array[~common_array] = np.nan # Reduce the file size by rounding to the nearest 2 digits np.around(ke_array, 2, out=ke_array) # Force output to 32-bit float gdc.array_to_raster( ke_array.astype(np.float32), image.ke_raster, output_geo=common_geo, output_proj=common_proj, stats_flag=stats_flag) # Remove Landsat TOA rasters if not keep_dn_flag: for landsat_item in python_common.build_file_list( image.orig_data_ws, image.image_re): os.remove(os.path.join(image.orig_data_ws, landsat_item)) return True
def main(netcdf_ws=os.getcwd(), ancillary_ws=os.getcwd(), output_ws=os.getcwd(), variables=['prcp'], daily_flag=False, monthly_flag=True, annual_flag=False, start_year=1981, end_year=2010, extent_path=None, output_extent=None, stats_flag=True, overwrite_flag=False): """Extract DAYMET temperature Args: netcdf_ws (str): folder of DAYMET netcdf files ancillary_ws (str): folder of ancillary rasters output_ws (str): folder of output rasters variables (list): DAYMET variables to download ('prcp', 'srad', 'vp', 'tmmn', 'tmmx') Set as ['all'] to process all variables daily_flag (bool): if True, compute daily (DOY) climatologies monthly_flag (bool): if True, compute monthly climatologies annual_flag (bool): if True, compute annual climatologies start_year (int): YYYY end_year (int): YYYY extent_path (str): filepath a raster defining the output extent output_extent (list): decimal degrees values defining output extent stats_flag (bool): if True, compute raster statistics. Default is True. overwrite_flag (bool): if True, overwrite existing files Returns: None """ logging.info('\nGenerating DAYMET climatologies') daily_fmt = 'daymet_{var}_30yr_normal_{doy:03d}.img' monthly_fmt = 'daymet_{var}_30yr_normal_{month:02d}.img' annual_fmt = 'daymet_{var}_30yr_normal.img' # daily_fmt = 'daymet_{var}_normal_{start}_{end}_{doy:03d}.img' # monthly_fmt = 'daymet_{var}_normal_{start}_{end}_{month:02d}.img' # annual_fmt = 'daymet_{var}_normal_{start}_{end}.img' # If a date is not set, process 1981-2010 climatology try: start_dt = dt.datetime(start_year, 1, 1) logging.debug(' Start date: {}'.format(start_dt)) except: start_dt = dt.datetime(1981, 1, 1) logging.info(' Start date: {}'.format(start_dt)) try: end_dt = dt.datetime(end_year, 12, 31) logging.debug(' End date: {}'.format(end_dt)) except: end_dt = dt.datetime(2010, 12, 31) logging.info(' End date: {}'.format(end_dt)) # Get DAYMET spatial reference from an ancillary raster mask_raster = os.path.join(ancillary_ws, 'daymet_mask.img') daymet_re = re.compile('daymet_v3_(?P<VAR>\w+)_(?P<YEAR>\d{4})_na.nc4$') # DAYMET rasters to extract var_full_list = ['prcp', 'tmmn', 'tmmx'] # data_full_list = ['prcp', 'srad', 'vp', 'tmmn', 'tmmx'] if not variables: logging.error('\nERROR: variables parameter is empty\n') sys.exit() elif type(variables) is not list: # DEADBEEF - I could try converting comma separated strings to lists? logging.warning('\nERROR: variables parameter must be a list\n') sys.exit() elif 'all' in variables: logging.error('\nDownloading all variables\n {}'.format( ','.join(var_full_list))) var_list = var_full_list[:] elif not set(variables).issubset(set(var_full_list)): logging.error( '\nERROR: variables parameter is invalid\n {}'.format(variables)) sys.exit() else: var_list = variables[:] # Get extent/geo from mask raster daymet_ds = gdal.Open(mask_raster) daymet_osr = gdc.raster_ds_osr(daymet_ds) daymet_proj = gdc.osr_proj(daymet_osr) daymet_cs = gdc.raster_ds_cellsize(daymet_ds, x_only=True) daymet_extent = gdc.raster_ds_extent(daymet_ds) daymet_geo = daymet_extent.geo(daymet_cs) daymet_x, daymet_y = daymet_extent.origin() daymet_ds = None logging.debug(' Projection: {}'.format(daymet_proj)) logging.debug(' Cellsize: {}'.format(daymet_cs)) logging.debug(' Geo: {}'.format(daymet_geo)) logging.debug(' Extent: {}'.format(daymet_extent)) logging.debug(' Origin: {} {}'.format(daymet_x, daymet_y)) # Subset data to a smaller extent if output_extent is not None: logging.info('\nComputing subset extent & geo') logging.debug(' Extent: {}'.format(output_extent)) # Assume input extent is in decimal degrees output_extent = gdc.project_extent(gdc.Extent(output_extent), gdc.epsg_osr(4326), daymet_osr, 0.001) output_extent = gdc.intersect_extents([daymet_extent, output_extent]) output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs) output_geo = output_extent.geo(daymet_cs) logging.debug(' Geo: {}'.format(output_geo)) logging.debug(' Extent: {}'.format(output_extent)) elif extent_path is not None: logging.info('\nComputing subset extent & geo') output_extent = gdc.project_extent( gdc.raster_path_extent(extent_path), gdc.raster_path_osr(extent_path), daymet_osr, gdc.raster_path_cellsize(extent_path, x_only=True)) output_extent = gdc.intersect_extents([daymet_extent, output_extent]) output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs) output_geo = output_extent.geo(daymet_cs) logging.debug(' Geo: {}'.format(output_geo)) logging.debug(' Extent: {}'.format(output_extent)) else: output_extent = daymet_extent.copy() output_geo = daymet_geo[:] output_shape = output_extent.shape(cs=daymet_cs) xi, yi = gdc.array_geo_offsets(daymet_geo, output_geo, daymet_cs) output_rows, output_cols = output_extent.shape(daymet_cs) logging.debug(' Shape: {} {}'.format(output_rows, output_cols)) logging.debug(' Offsets: {} {} (x y)'.format(xi, yi)) # Process each variable for input_var in var_list: logging.info("\nVariable: {}".format(input_var)) # Rename variables to match cimis if input_var == 'prcp': output_var = 'ppt' else: output_var = input_var logging.debug("Output name: {}".format(output_var)) # Build output folder var_ws = os.path.join(output_ws, output_var) if not os.path.isdir(var_ws): os.makedirs(var_ws) # Build output arrays logging.debug(' Building arrays') if daily_flag: daily_sum = np.full((365, output_shape[0], output_shape[1]), 0, np.float64) daily_count = np.full((365, output_shape[0], output_shape[1]), 0, np.uint8) if monthly_flag: monthly_sum = np.full((12, output_shape[0], output_shape[1]), 0, np.float64) monthly_count = np.full((12, output_shape[0], output_shape[1]), 0, np.uint8) if monthly_flag: annual_sum = np.full((output_shape[0], output_shape[1]), 0, np.float64) annual_count = np.full((output_shape[0], output_shape[1]), 0, np.uint8) # Process each file/year separately for input_name in sorted(os.listdir(netcdf_ws)): logging.debug(" {}".format(input_name)) input_match = daymet_re.match(input_name) if not input_match: logging.debug(' Regular expression didn\'t match, skipping') continue elif input_match.group('VAR') != input_var: logging.debug(' Variable didn\'t match, skipping') continue year_str = input_match.group('YEAR') logging.info(" Year: {}".format(year_str)) year_int = int(year_str) year_days = int(dt.datetime(year_int, 12, 31).strftime('%j')) if start_dt is not None and year_int < start_dt.year: logging.debug(' Before start date, skipping') continue elif end_dt is not None and year_int > end_dt.year: logging.debug(' After end date, skipping') continue # Build input file path input_raster = os.path.join(netcdf_ws, input_name) if not os.path.isfile(input_raster): logging.debug( ' Input raster doesn\'t exist, skipping {}'.format( input_raster)) continue # Build output folder if daily_flag: daily_ws = os.path.join(var_ws, 'daily') if not os.path.isdir(daily_ws): os.makedirs(daily_ws) if monthly_flag: monthly_temp_sum = np.full( (12, output_shape[0], output_shape[1]), 0, np.float64) monthly_temp_count = np.full( (12, output_shape[0], output_shape[1]), 0, np.uint8) # Read in the DAYMET NetCDF file input_nc_f = netCDF4.Dataset(input_raster, 'r') # logging.debug(input_nc_f.variables) # Check all valid dates in the year year_dates = date_range(dt.datetime(year_int, 1, 1), dt.datetime(year_int + 1, 1, 1)) for date_dt in year_dates: logging.debug(' {}'.format(date_dt.date())) # if start_dt is not None and date_dt < start_dt: # logging.debug( # ' {} - before start date, skipping'.format( # date_dt.date())) # continue # elif end_dt is not None and date_dt > end_dt: # logging.debug(' {} - after end date, skipping'.format( # date_dt.date())) # continue # else: # logging.info(' {}'.format(date_dt.date())) doy = int(date_dt.strftime('%j')) doy_i = range(1, year_days + 1).index(doy) month_i = date_dt.month - 1 # Arrays are being read as masked array with a -9999 fill value # Convert to basic numpy array arrays with nan values try: input_ma = input_nc_f.variables[input_var][doy_i, yi:yi + output_rows, xi:xi + output_cols] except IndexError: logging.info(' date not in netcdf, skipping') continue input_nodata = float(input_ma.fill_value) output_array = input_ma.data.astype(np.float32) output_array[output_array == input_nodata] = np.nan output_mask = np.isfinite(output_array) # Convert Kelvin to Celsius if input_var in ['tmax', 'tmin']: output_array -= 273.15 # Save values if daily_flag: daily_sum[doy_i, :, :] += output_array daily_count[doy_i, :, :] += output_mask if monthly_flag: monthly_temp_sum[month_i, :, :] += output_array monthly_temp_count[month_i, :, :] += output_mask if annual_flag: annual_sum[:, :] += output_array annual_count[:, :] += output_mask # Cleanup # del input_ds, input_array del input_ma, output_array, output_mask # Compute mean monthly for the year if monthly_flag: # Sum precipitation if input_var == 'prcp': monthly_sum += monthly_temp_sum else: monthly_sum += monthly_temp_sum / monthly_temp_count # Is this the right count? monthly_count += np.any(monthly_temp_count, axis=0) del monthly_temp_sum, monthly_temp_count input_nc_f.close() del input_nc_f # Save the projected climatology arrays if daily_flag: for doy_i in range(daily_sum.shape[0]): daily_name = daily_fmt.format(var=output_var, start=start_year, end=end_year, doy=doy_i + 1) daily_path = os.path.join(daily_ws, daily_name) gdc.array_to_raster(daily_sum[doy_i, :, :] / daily_count[doy_i, :, :], daily_path, output_geo=output_geo, output_proj=daymet_proj, stats_flag=stats_flag) del daily_sum, daily_count if monthly_flag: for month_i in range(monthly_sum.shape[0]): monthly_name = monthly_fmt.format(var=output_var, start=start_year, end=end_year, month=month_i + 1) monthly_path = os.path.join(var_ws, monthly_name) gdc.array_to_raster(monthly_sum[month_i, :, :] / monthly_count[month_i, :, :], monthly_path, output_geo=output_geo, output_proj=daymet_proj, stats_flag=stats_flag) del monthly_sum, monthly_count if annual_flag: annual_name = annual_fmt.format(var=output_var, start=start_year, end=end_year) annual_path = os.path.join(var_ws, annual_name) gdc.array_to_raster(annual_sum / annual_count, annual_path, output_geo=output_geo, output_proj=daymet_proj, stats_flag=stats_flag) del annual_sum, annual_count logging.debug('\nScript Complete')
def main(gis_ws, input_soil_ws, prop_list=['all'], overwrite_flag=False, pyramids_flag=False, stats_flag=False): """Convert soil polygon shapefiles to raster Snap to latest CDL rasters (in CDL workspace) with an albers projection Args: gis_ws (str): Folder/workspace path of the GIS data for the project input_soil_ws (str): Folder/workspace path of the common soils data prop_list (list): String of the soil types to build (i.e. awc, clay, sand, all) overwrite_flag (bool): If True, overwrite output rasters pyramids_flag (bool): If True, build pyramids/overviews for the output rasters stats_flag (bool): If True, compute statistics for the output rasters Returns: None """ logging.info('\nRasterizing Soil Polygons') folder_fmt = 'gsmsoil_{}' polygon_fmt = 'gsmsoilmu_a_us_{}_albers.shp' output_soil_ws = os.path.join(gis_ws, 'soils') scratch_ws = os.path.join(gis_ws, 'scratch') zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img') # Soil polygons have a float and integer field field_fmt = '{}' # field_fmt = '{}_INT' raster_fmt = '{}_30m_albers.img' # raster_fmt = '{}_2010_30m_cdls.img' # raster_fmt = 'gsmsoil_{}_integer.img' output_format = 'HFA' output_type = 'Float32' output_nodata = float(np.finfo(np.float32).min) # output_type = 'Byte' # output_nodata = 255 if pyramids_flag: levels = '2 4 8 16 32 64 128' # gdal.SetConfigOption('USE_RRD', 'YES') # gdal.SetConfigOption('HFA_USE_RRD', 'YES') logging.info('Soil Property: {}'.format(', '.join(prop_list))) if prop_list == ['all']: prop_list = ['awc', 'clay', 'sand'] # Check input folders if not os.path.isdir(gis_ws): logging.error('\nERROR: The GIS workspace {} ' + 'does not exist\n'.format(gis_ws)) sys.exit() elif not os.path.isdir(input_soil_ws): logging.error(('\nERROR: The input soil workspace {} ' + 'does not exist').format(input_soil_ws)) sys.exit() elif not os.path.isfile(zone_raster_path): logging.error(('\nERROR: The zone raster {} does not exist' + '\n Try re-running "build_study_area_raster.py"' ).format(zone_raster_path)) sys.exit() if not os.path.isdir(output_soil_ws): os.makedirs(output_soil_ws) logging.info('\nGIS Workspace: {}'.format(gis_ws)) logging.info('Soil Workspace: {}\n'.format(output_soil_ws)) temp_polygon_path = os.path.join(output_soil_ws, 'temp_polygon.shp') if os.path.isfile(temp_polygon_path): util.remove_file(temp_polygon_path) # subprocess.call( # ['gdalmanage', 'delete', '-f', '', temp_polygon_path]) # Reference all output rasters zone raster zone_raster_ds = gdal.Open(zone_raster_path) output_osr = gdc.raster_ds_osr(zone_raster_ds) output_wkt = gdc.raster_ds_proj(zone_raster_ds) output_cs = gdc.raster_ds_cellsize(zone_raster_ds)[0] output_x, output_y = gdc.raster_ds_origin(zone_raster_ds) output_extent = gdc.raster_ds_extent(zone_raster_ds) zone_raster_ds = None logging.debug('\nStudy area properties') logging.debug(' Output OSR: {}'.format(output_osr)) logging.debug(' Output Extent: {}'.format(output_extent)) logging.debug(' Output cellsize: {}'.format(output_cs)) # Process each soil property for prop_str in prop_list: input_polygon_path = os.path.join(input_soil_ws, folder_fmt.format(prop_str), polygon_fmt.format(prop_str)) output_raster_path = os.path.join(output_soil_ws, raster_fmt.format(prop_str)) if not os.path.isfile(input_polygon_path): logging.info(('The soil polygon {} does not ' + 'exist').format(input_polygon_path)) continue elif os.path.isfile(output_raster_path) and overwrite_flag: subprocess.call(['gdalmanage', 'delete', output_raster_path]) if not os.path.isfile(output_raster_path): soil_field = field_fmt.format(prop_str.upper()) logging.info('Projecting shapefile') # Project study area extent to the input/soil spatial reference input_osr = gdc.feature_path_osr(input_polygon_path) input_extent = gdc.project_extent(output_extent, output_osr, input_osr) logging.debug('Input Extent: {}'.format(input_extent)) subprocess.call([ 'ogr2ogr', '-overwrite', '-preserve_fid', '-t_srs', str(output_wkt), '-spat', str(input_extent.xmin), str(input_extent.ymin), str(input_extent.ymax), str(input_extent.ymax), temp_polygon_path, input_polygon_path ]) logging.info('Rasterizing shapefile') subprocess.call([ 'gdal_rasterize', '-of', output_format, '-a', soil_field, '-a_nodata', str(output_nodata), '-init', str(output_nodata), '-co', 'COMPRESSED=YES' ] + ['-te'] + str(output_extent).split() + [ '-tr', str(output_cs), str(output_cs), '-ot', output_type, temp_polygon_path, output_raster_path ]) if os.path.isfile(temp_polygon_path): util.remove_file(temp_polygon_path) # subprocess.call(['gdalmanage', 'delete', temp_polygon_path]) if stats_flag and os.path.isfile(output_raster_path): logging.info('Computing statistics') logging.debug(' {}'.format(output_raster_path)) subprocess.call( ['gdalinfo', '-stats', '-nomd', output_raster_path]) if pyramids_flag and os.path.isfile(output_raster_path): logging.info('Building pyramids') logging.debug(' {}'.format(output_raster_path)) subprocess.call(['gdaladdo', '-ro', output_raster_path] + levels.split())
def main(img_ws=os.getcwd(), ancillary_ws=os.getcwd(), output_ws=os.getcwd(), etr_flag=False, eto_flag=False, start_date=None, end_date=None, extent_path=None, output_extent=None, stats_flag=True, overwrite_flag=False, use_cimis_eto_flag=False): """Compute daily ETr/ETo from CIMIS data Args: img_ws (str): root folder of GRIDMET data ancillary_ws (str): folder of ancillary rasters output_ws (str): folder of output rasters etr_flag (bool): if True, compute alfalfa reference ET (ETr) eto_flag (bool): if True, compute grass reference ET (ETo) start_date (str): ISO format date (YYYY-MM-DD) end_date (str): ISO format date (YYYY-MM-DD) extent_path (str): file path defining the output extent output_extent (list): decimal degrees values defining output extent stats_flag (bool): if True, compute raster statistics. Default is True. overwrite_flag (bool): If True, overwrite existing files use_cimis_eto_flag (bool): if True, use CIMIS ETo raster if one of the component rasters is missing and ETo/ETr cannot be computed Returns: None """ logging.info('\nComputing CIMIS ETo/ETr') np.seterr(invalid='ignore') # Use CIMIS ETo raster directly instead of computing from components # Currently this will only be applied if one of the inputs is missing use_cimis_eto_flag = True # Compute ETr and/or ETo if not etr_flag and not eto_flag: logging.info(' ETo/ETr flag(s) not set, defaulting to ETr') etr_flag = True # If a date is not set, process 2017 try: start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d') logging.debug(' Start date: {}'.format(start_dt)) except: start_dt = dt.datetime(2017, 1, 1) logging.info(' Start date: {}'.format(start_dt)) try: end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d') logging.debug(' End date: {}'.format(end_dt)) except: end_dt = dt.datetime(2017, 12, 31) logging.info(' End date: {}'.format(end_dt)) etr_folder = 'etr' eto_folder = 'eto' etr_fmt = 'etr_{}_daily_cimis.img' eto_fmt = 'eto_{}_daily_cimis.img' # DEM for air pressure calculation mask_raster = os.path.join(ancillary_ws, 'cimis_mask.img') dem_raster = os.path.join(ancillary_ws, 'cimis_elev.img') lat_raster = os.path.join(ancillary_ws, 'cimis_lat.img') # lon_raster = os.path.join(ancillary_ws, 'cimis_lon.img') # Interpolate zero windspeed pixels # interpolate_zero_u2_flag = False # Interpolate edge and coastal cells # interpolate_edge_flag = False # Resample type # 0 = GRA_NearestNeighbour, Nearest neighbour (select on one input pixel) # 1 = GRA_Bilinear,Bilinear (2x2 kernel) # 2 = GRA_Cubic, Cubic Convolution Approximation (4x4 kernel) # 3 = GRA_CubicSpline, Cubic B-Spline Approximation (4x4 kernel) # 4 = GRA_Lanczos, Lanczos windowed sinc interpolation (6x6 kernel) # 5 = GRA_Average, Average (computes the average of all non-NODATA contributing pixels) # 6 = GRA_Mode, Mode (selects the value which appears most often of all the sampled points) resample_type = gdal.GRA_CubicSpline # Wind speed is measured at 2m zw = 2 # Output workspaces etr_ws = os.path.join(output_ws, etr_folder) eto_ws = os.path.join(output_ws, eto_folder) if etr_flag and not os.path.isdir(etr_ws): os.makedirs(etr_ws) if eto_flag and not os.path.isdir(eto_ws): os.makedirs(eto_ws) # Check ETr/ETo functions test_flag = False # Check that the daily_refet_func produces the correct values if test_flag: doy_test = 245 elev_test = 1050.0 lat_test = 39.9396 * math.pi / 180 tmin_test = 11.07 tmax_test = 34.69 rs_test = 22.38 u2_test = 1.94 zw_test = 2.5 tdew_test = -3.22 ea_test = et_common.saturation_vapor_pressure_func(tdew_test) pair_test = 101.3 * np.power((285 - 0.0065 * elev_test) / 285, 5.26) q_test = 0.622 * ea_test / (pair_test - (0.378 * ea_test)) etr = float( et_common.daily_refet_func(tmin_test, tmax_test, q_test, rs_test, u2_test, zw_test, elev_test, doy_test, lat_test, 'ETR')) eto = float( et_common.daily_refet_func(tmin_test, tmax_test, q_test, rs_test, u2_test, zw_test, elev_test, doy_test, lat_test, 'ETO')) print('ETr: 8.89', etr) print('ETo: 6.16', eto) sys.exit() # Get CIMIS grid properties from mask cimis_mask_ds = gdal.Open(mask_raster) cimis_osr = gdc.raster_ds_osr(cimis_mask_ds) cimis_proj = gdc.osr_proj(cimis_osr) cimis_cs = gdc.raster_ds_cellsize(cimis_mask_ds, x_only=True) cimis_extent = gdc.raster_ds_extent(cimis_mask_ds) cimis_full_geo = cimis_extent.geo(cimis_cs) cimis_x, cimis_y = cimis_extent.origin() cimis_mask_ds = None logging.debug(' Projection: {}'.format(cimis_proj)) logging.debug(' Cellsize: {}'.format(cimis_cs)) logging.debug(' Geo: {}'.format(cimis_full_geo)) logging.debug(' Extent: {}'.format(cimis_extent)) # Manually set CIMIS grid properties # cimis_extent = gdc.Extent((-400000, -650000, 600000, 454000)) # cimis_cs = 2000 # cimis_geo = gdc.extent_geo(cimis_extent, cellsize) # cimis_epsg = 3310 # NAD_1983_California_Teale_Albers # cimis_x, cimis_y = (0,0) # Subset data to a smaller extent if output_extent is not None: logging.info('\nComputing subset extent & geo') logging.debug(' Extent: {}'.format(output_extent)) cimis_extent = gdc.Extent(output_extent) cimis_extent.adjust_to_snap('EXPAND', cimis_x, cimis_y, cimis_cs) cimis_geo = cimis_extent.geo(cimis_cs) logging.debug(' Geo: {}'.format(cimis_geo)) logging.debug(' Extent: {}'.format(output_extent)) elif extent_path is not None: logging.info('\nComputing subset extent & geo') if extent_path.lower().endswith('.shp'): cimis_extent = gdc.feature_path_extent(extent_path) extent_osr = gdc.feature_path_osr(extent_path) extent_cs = None else: cimis_extent = gdc.raster_path_extent(extent_path) extent_osr = gdc.raster_path_osr(extent_path) extent_cs = gdc.raster_path_cellsize(extent_path, x_only=True) cimis_extent = gdc.project_extent(cimis_extent, extent_osr, cimis_osr, extent_cs) cimis_extent.adjust_to_snap('EXPAND', cimis_x, cimis_y, cimis_cs) cimis_geo = cimis_extent.geo(cimis_cs) logging.debug(' Geo: {}'.format(cimis_geo)) logging.debug(' Extent: {}'.format(cimis_extent)) else: cimis_geo = cimis_full_geo # Latitude lat_array = gdc.raster_to_array(lat_raster, mask_extent=cimis_extent, return_nodata=False) lat_array = lat_array.astype(np.float32) lat_array *= math.pi / 180 # Elevation data elev_array = gdc.raster_to_array(dem_raster, mask_extent=cimis_extent, return_nodata=False) elev_array = elev_array.astype(np.float32) # Process each year in the input workspace logging.info("") for year_str in sorted(os.listdir(img_ws)): logging.debug('{}'.format(year_str)) if not re.match('^\d{4}$', year_str): logging.debug(' Not a 4 digit year folder, skipping') continue year_ws = os.path.join(img_ws, year_str) year_int = int(year_str) # year_days = int(dt.datetime(year_int, 12, 31).strftime('%j')) if start_dt is not None and year_int < start_dt.year: logging.debug(' Before start date, skipping') continue elif end_dt is not None and year_int > end_dt.year: logging.debug(' After end date, skipping') continue logging.info('{}'.format(year_str)) # Output paths etr_raster = os.path.join(etr_ws, etr_fmt.format(year_str)) eto_raster = os.path.join(eto_ws, eto_fmt.format(year_str)) if etr_flag and (overwrite_flag or not os.path.isfile(etr_raster)): logging.debug(' {}'.format(etr_raster)) gdc.build_empty_raster(etr_raster, band_cnt=366, output_dtype=np.float32, output_proj=cimis_proj, output_cs=cimis_cs, output_extent=cimis_extent, output_fill_flag=True) if eto_flag and (overwrite_flag or not os.path.isfile(eto_raster)): logging.debug(' {}'.format(eto_raster)) gdc.build_empty_raster(eto_raster, band_cnt=366, output_dtype=np.float32, output_proj=cimis_proj, output_cs=cimis_cs, output_extent=cimis_extent, output_fill_flag=True) # Process each date in the year for date_str in sorted(os.listdir(year_ws)): logging.debug('{}'.format(date_str)) try: date_dt = dt.datetime.strptime(date_str, '%Y_%m_%d') except ValueError: logging.debug( ' Invalid folder date format (YYYY_MM_DD), skipping') continue if start_dt is not None and date_dt < start_dt: logging.debug(' Before start date, skipping') continue elif end_dt is not None and date_dt > end_dt: logging.debug(' After end date, skipping') continue logging.info(date_str) date_ws = os.path.join(year_ws, date_str) doy = int(date_dt.strftime('%j')) # Set file paths tmax_path = os.path.join(date_ws, 'Tx.img') tmin_path = os.path.join(date_ws, 'Tn.img') tdew_path = os.path.join(date_ws, 'Tdew.img') rso_path = os.path.join(date_ws, 'Rso.img') rs_path = os.path.join(date_ws, 'Rs.img') u2_path = os.path.join(date_ws, 'U2.img') eto_path = os.path.join(date_ws, 'ETo.img') # k_path = os.path.join(date_ws, 'K.img') # rnl_path = os.path.join(date_ws, 'Rnl.img') input_list = [ tmin_path, tmax_path, tdew_path, u2_path, rs_path, rso_path ] # If any input raster is missing, skip the day # Unless ETo is present (and use_cimis_eto_flag is True) day_skip_flag = False for t_path in input_list: if not os.path.isfile(t_path): logging.info(' {} is missing'.format(t_path)) day_skip_flag = True if (day_skip_flag and use_cimis_eto_flag and os.path.isfile(eto_path)): logging.info(' Using CIMIS ETo directly') eto_array = gdc.raster_to_array(eto_path, 1, cimis_extent, return_nodata=False) eto_array = eto_array.astype(np.float32) if not np.any(eto_array): logging.info(' {} is empty or missing'.format(eto_path)) logging.info(' Skipping date') continue # ETr if etr_flag: gdc.array_to_comp_raster(1.2 * eto_array, etr_raster, band=doy, stats_flag=False) # gdc.array_to_raster( # 1.2 * eto_array, etr_raster, # output_geo=cimis_geo, output_proj=cimis_proj, # stats_flag=stats_flag) # ETo if eto_flag: gdc.array_to_comp_raster(eto_array, eto_raster, band=doy, stats_flag=False) # gdc.array_to_raster( # eto_array, eto_raster, # output_geo=cimis_geo, output_proj=cimis_proj, # stats_flag=stats_flag) del eto_array continue elif not day_skip_flag: # Read in rasters # DEADBEEF - Read with extent since some arrays are too big # i.e. 2012-03-21, 2013-03-20, 2014-02-27 tmin_array = gdc.raster_to_array(tmin_path, 1, cimis_extent, return_nodata=False) tmax_array = gdc.raster_to_array(tmax_path, 1, cimis_extent, return_nodata=False) tdew_array = gdc.raster_to_array(tdew_path, 1, cimis_extent, return_nodata=False) rso_array = gdc.raster_to_array(rso_path, 1, cimis_extent, return_nodata=False) rs_array = gdc.raster_to_array(rs_path, 1, cimis_extent, return_nodata=False) u2_array = gdc.raster_to_array(u2_path, 1, cimis_extent, return_nodata=False) # k_array = gdc.raster_to_array( # k_path, 1, cimis_extent, return_nodata=False) # rnl_array = gdc.raster_to_array( # rnl_path, 1, cimis_extent, return_nodata=False) # Check that all input arrays have data for t_name, t_array in [[tmin_path, tmin_array], [tmax_path, tmax_array], [tdew_path, tdew_array], [u2_path, u2_array], [rs_path, rs_array]]: if not np.any(t_array): logging.warning( ' {} is empty or missing'.format(t_name)) day_skip_flag = True if day_skip_flag: logging.warning(' Skipping date') continue # DEADBEEF - Some arrays have a 500m cellsize # i.e. 2011-07-25, 2010-01-01 -> 2010-07-27 tmin_array = rescale_array_func(tmin_array, elev_array, 'tmin') tmax_array = rescale_array_func(tmax_array, elev_array, 'tmax') tdew_array = rescale_array_func(tdew_array, elev_array, 'tdew') rso_array = rescale_array_func(rso_array, elev_array, 'rso') rs_array = rescale_array_func(rs_array, elev_array, 'rs') u2_array = rescale_array_func(u2_array, elev_array, 'u2') # k_array = rescale_array_func(k_array, elev_array, 'k') # rnl_array = rescale_array_func(rnl_array, elev_array, 'rnl') # Back calculate q from tdew by first calculating ea from tdew es_array = et_common.saturation_vapor_pressure_func(tdew_array) pair_array = et_common.air_pressure_func(elev_array) q_array = 0.622 * es_array / (pair_array - (0.378 * es_array)) del es_array, pair_array, tdew_array # Back calculate rhmin/rhmax from tdew # ea_tmax = et_common.saturation_vapor_pressure_func(tmax_array) # ea_tmin = et_common.saturation_vapor_pressure_func(tmin_array) # rhmin = ea_tdew * 2 / (ea_tmax + ea_tmin); # rhmax = ea_tdew * 2 / (ea_tmax + ea_tmin); # del ea_tmax, ea_tmin # ETr if etr_flag: etr_array = et_common.refet_daily_func(tmin_array, tmax_array, q_array, rs_array, u2_array, zw, elev_array, lat_array, doy, ref_type='ETR', rso_type='ARRAY', rso=rso_array) gdc.array_to_comp_raster(etr_array.astype(np.float32), etr_raster, band=doy, stats_flag=False) # gdc.array_to_raster( # etr_array.astype(np.float32), etr_raster, # output_geo=cimis_geo, output_proj=cimis_proj, # stats_flag=stats_flag) del etr_array # ETo if eto_flag: eto_array = et_common.refet_daily_func(tmin_array, tmax_array, q_array, rs_array, u2_array, zw, elev_array, lat_array, doy, ref_type='ETO', rso_type='ARRAY', rso=rso_array) gdc.array_to_comp_raster(eto_array.astype(np.float32), eto_raster, band=doy, stats_flag=False) # gdc.array_to_raster( # eto_array.astype(np.float32), eto_raster, # output_geo=cimis_geo, output_proj=cimis_proj, # stats_flag=stats_flag) del eto_array # Cleanup del tmin_array, tmax_array, u2_array, rs_array, q_array # del rnl, rs, rso else: logging.info(' Skipping date') continue if stats_flag and etr_flag: gdc.raster_statistics(etr_raster) if stats_flag and eto_flag: gdc.raster_statistics(eto_raster) logging.debug('\nScript Complete')