def _read_cmd_args(): """Read command line arguments.""" # Check if argument count is correct if len(sys.argv) != 8: print("[ERR] Invalid number of command line arguments!") _usage() sys.exit(1) # Check if LDT parameter file can be opened. _ldtfile = sys.argv[1] ncid_ldt = nc4_dataset(_ldtfile, mode='r', format='NETCDF4_CLASSIC') ncid_ldt.close() # Check of LVT TS anomaly file can be opened _tsfile = sys.argv[2] ncid_lvt = nc4_dataset(_tsfile, mode='r', format='NETCDF4_CLASSIC') ncid_lvt.close() # Check of LVT FINAL anomaly file can be opened _finalfile = sys.argv[3] ncid_lvt = nc4_dataset(_finalfile, mode='r', format='NETCDF4_CLASSIC') ncid_lvt.close() _outfile_anomaly_prefix = sys.argv[4] _outfile_climo_prefix = sys.argv[5] _lsm = sys.argv[6] _yyyymmddhh = sys.argv[7] if _lsm not in ["noah39", "noahmp401", "jules50"]: print("[ERR] Unknown LSM %s" % (_lsm)) print("Options are noah39, noahmp401, jules50") sys.exit(1) return _ldtfile, _tsfile, _finalfile, \ _outfile_anomaly_prefix, _outfile_climo_prefix, _lsm, _yyyymmddhh
def calc_medians(self): """Calculate medians of the metrics.""" self.median_data = {} total_ens_size = 0 lead = 0 latitude = 0 longitude = 0 # We need to first count the total number of ensembles. nmme_models = self.config["s2smetric"]["nmme_models"].split() for nmme in nmme_models: if nmme not in self.nmme_metric_files: continue metric_file = self.nmme_metric_files[nmme] rootgrp = nc4_dataset(metric_file, 'r', format="NETCDF4_CLASSIC") total_ens_size += rootgrp.dimensions["ens"].size lead = rootgrp.dimensions["lead"].size latitude = rootgrp.dimensions["latitude"].size longitude = rootgrp.dimensions["longitude"].size # Find dimensions for a single month dims = (total_ens_size, latitude, longitude) # Now loop through each month, load a NMME contribution into the # total var array, and calculate the median. self.median_data["median"] = [] for itime in range(0, lead): var = np.zeros(dims) iens = 0 start_end_set = False for nmme in nmme_models: if nmme not in self.nmme_metric_files: continue metric_file = self.nmme_metric_files[nmme] rootgrp = nc4_dataset(metric_file, 'r', format="NETCDF4_CLASSIC") ens = rootgrp.dimensions["ens"].size var[iens:(iens+ens),:,:] = \ rootgrp.variables[self.metric][:,itime,:,:].data iens += ens if not start_end_set: self.median_data["num_months"] = \ rootgrp.dimensions["lead"].size self.median_data["latitudes"] = \ rootgrp.variables["latitude"][:] self.median_data["longitudes"] = \ rootgrp.variables["longitude"][:] self.median_data["units"] = \ rootgrp.variables[self.metric].units self.median_data["long_name"] = \ rootgrp.variables[self.metric].long_name self.set_startdates_enddates_by_month() start_end_set = True rootgrp.close() # Calculate the median for the current month self.median_data["median"].append(np.median(var, axis=0)) del var
def _create_firstguess_monthly_file(varlists, infile, outfile): """Read daily S2S file, and most fields copy to monthly S2S file. This allows us to cleanly copy dimensions and all attributes. The numerical values of the arrays in the monthly S2S file will be replaced later in the script.""" if not os.path.exists(infile): print(f"[ERR] {infile} does not exist!") sys.exit(1) ncid_in = nc4_dataset(infile, 'r', format='NETCDF4_CLASSIC') # Create monthly file, copying dimensions and global attributes. # NOTE: We clean up the global attributes later in the script. ncid_out = nc4_dataset(outfile, "w", format='NETCDF4_CLASSIC') _copy_dims_gattrs(ncid_in, ncid_out) # Copy the fields. varnames = [] for listname in [ "const_list", "var_inst_list", "var_acc_list", "var_tavg_land_list", "var_tavg_f_list", "var_tavg_twsgws_list", "var_tair_max_list", "var_tair_min_list" ]: varnames += varlists[listname] for varname in varnames: var_in = ncid_in.variables[varname] if "missing_value" in var_in.__dict__: var_out = \ ncid_out.createVariable(varname, var_in.datatype, dimensions=var_in.dimensions, zlib=True, complevel=1, shuffle=True, fill_value=var_in.missing_value) else: var_out = \ ncid_out.createVariable(varname, var_in.datatype, dimensions=var_in.dimensions, zlib=True, complevel=1, shuffle=True) for attrname in var_in.__dict__: if attrname == "_FillValue": continue var_out.setncattr(attrname, var_in.__dict__[attrname]) if len(var_out.shape) == 4: var_out[:, :, :, :] = var_in[:, :, :, :] elif len(var_out.shape) == 3: var_out[:, :, :] = var_in[:, :, :] elif len(var_out.shape) == 2: var_out[:, :] = var_in[:, :] elif len(var_out.shape) == 1: var_out[:] = var_in[:] ncid_out.close() ncid_in.close()
def write_3d_netcdf(infile, var, varname, description, source, \ var_units, lons, lats, sdate): """write netcdf files""" rootgrp = nc4_dataset(infile, 'w', format='NETCDF4') longitude = rootgrp.createDimension('lon', len(lons)) latitude = rootgrp.createDimension('lat', len(lats)) time = rootgrp.createDimension('time', None) longitudes = rootgrp.createVariable('lon', 'f4', ('lon', )) latitudes = rootgrp.createVariable('lat', 'f4', ('lat', )) times = rootgrp.createVariable('time', 'f8', ('time', )) # two dimensions unlimited. varname = rootgrp.createVariable(varname, 'f4', \ ('time', 'lat', 'lon'), \ fill_value=-9999., zlib=True) #import time rootgrp.description = description rootgrp.history = 'Created ' + t_ctime(t_time()) rootgrp.source = source latitudes.units = 'degrees_north' longitudes.units = 'degrees_east' varname.units = var_units string_date = datetime.strftime(sdate, "%Y-%m-%d") times.units = 'days since ' + string_date times.calendar = 'gregorian' latitudes[:] = lats longitudes[:] = lons varname[:, :, :] = var times[:] = nc4_date2num(sdate, units=times.units, calendar=times.calendar) rootgrp.close()
def __init__(self, metricfile): """Constructor""" self.metricfile = metricfile # Parse the name of the metric file, and save elements. Assumes # filename obeys Air Force Weather file naming convention. self.metric_filename_elements = {} element_list = self.metricfile.split("_") for element in element_list: key = element.split(".")[0] value = element.split(".")[1] self.metric_filename_elements[key] = value # Find number of months in metrics file, and save latitudes and # longitudes. rootgrp = nc4_dataset(self.metricfile, 'r', format="NETCDF4_CLASSIC") num_months = rootgrp.dimensions["lead"].size self.latitudes = rootgrp.variables["latitude"][:] self.longitudes = rootgrp.variables["longitude"][:] rootgrp.close() # Set start and end of each month of data in the metric file. self.startdates_by_month = [] self.enddates_by_month = [] for imonth in range(0, num_months): if imonth == 0: self.startdates_by_month.append(self.get_first_startdate()) newdate2 = _set_newdate(self.get_first_startdate()) self.enddates_by_month.append(newdate2) else: newdate1 = _set_newdate(self.startdates_by_month[imonth - 1]) newdate2 = _set_newdate(self.enddates_by_month[imonth - 1]) self.startdates_by_month.append(newdate1) self.enddates_by_month.append(newdate2)
def get_variable(self, varname): """Retrieve variable and select metadata from metric file.""" rootgrp = nc4_dataset(self.metricfile, 'r', format="NETCDF4_CLASSIC") var = rootgrp.variables[varname][:, :, :, :] units = rootgrp.variables[varname].units long_name = rootgrp.variables[varname].long_name rootgrp.close() return var, units, long_name
def _cleanup_global_attrs(outfile): """Clean-up global attributes.""" if not os.path.exists(outfile): print(f"[ERR] {outfile} does not exist!") sys.exit(1) ncid = nc4_dataset(outfile, 'a', format='NETCDF4_CLASSIC') ncid.history = f"created on date: {time.ctime()}" del ncid.NCO del ncid.history_of_appended_files
def _add_time_data(infile, outfile, startdate, enddate): """Add time information to outfile, matching CF convention. This requires pulling more data from the last daily file.""" if not os.path.exists(infile): print(f"[ERR] {infile} does not exist!") sys.exit(1) ncid_in = nc4_dataset(infile, 'r', format='NETCDF4_CLASSIC') if not os.path.exists(outfile): print(f"[ERR] {outfile} does not exist!") sys.exit(1) ncid_out = nc4_dataset(outfile, 'a', format='NETCDF4_CLASSIC') # Copy the time array from the last daily file. var_in = ncid_in.variables["time"] var_out = ncid_out.createVariable("time", var_in.datatype, dimensions=var_in.dimensions, zlib=True, complevel=1, shuffle=True) for attrname in var_in.__dict__: if attrname == "_FillValue": continue var_out.setncattr(attrname, var_in.__dict__[attrname]) var_out[:] = var_in[:] # Copy the time_bnds array from the last daily file. But, we will change # the value to span one month of data. var_in = ncid_in.variables["time_bnds"] var_out = ncid_out.createVariable("time_bnds", var_in.datatype, dimensions=var_in.dimensions, zlib=True, complevel=1, shuffle=True) var_out[:, :] = var_in[:, :] # Count number of minutes between start and end dates. var_out[0, 0] = ((enddate - startdate).days) * (-24 * 60) # Days to minutes var_out[0, 1] = 0 ncid_in.close() ncid_out.close()
def _update_cell_methods(varlists, outfile): """Update cell_method attributes for select variables.""" if not os.path.exists(outfile): print(f"[ERR] {outfile} does not exist!") sys.exit(1) ncid = nc4_dataset(outfile, 'a', format='NETCDF4_CLASSIC') # Elaborate on monthly calculations of most variables varnames = [] for listname in [ "var_acc_list", "var_tavg_land_list", "var_tavg_f_list", "var_tavg_twsgws_list", "var_tair_max_list", "var_tair_min_list" ]: varnames += varlists[listname] for varname in varnames: var = ncid.variables[varname] # Special handling for TWS_inst and GWS_inst -- we want the monthly # means. if varname in varlists["var_tavg_twsgws_list"]: var.cell_methods = \ "time: mean (interval: 1 day) area: point where land" # Special handling for Tair_f_max and Tair_f_min: We want the monthly # averages of the daily maxs and mins elif varname in varlists["var_tair_max_list"]: var.cell_methods = \ "time: mean (interval: 1 day comment: daily maxima)" elif varname in varlists["var_tair_min_list"]: var.cell_methods = \ "time: mean (interval: 1 day comment: daily minima)" # Clarify monthly mean of daily means over land elif varname in varlists["var_tavg_land_list"]: var.cell_methods = \ "time: mean (interval: 1 day comment: daily means)" + \ " area: point where land" # Clarify monthly mean of daily means everywhere elif varname in varlists["var_tavg_f_list"]: var.cell_methods = \ "time: mean (interval: 1 day comment: daily means)" # Clarify monthly accumulations elif varname in varlists["var_acc_list"]: var.cell_methods = \ "time: sum (interval: 1 day comment: daily sums)" ncid.close()
def _read_cmd_args(): """Read command line arguments.""" # Check if argument count is correct if len(sys.argv) != 2: print("[ERR] Invalid number of command line arguments!") _usage() sys.exit(1) # Check if metric file can be opened. metricfile = sys.argv[1] rootgrp = nc4_dataset(metricfile, mode="r", format="NETCDF4_CLASSIC") rootgrp.close() return metricfile
def _update_monthly_s2s_values(outfile, accs, tavgs): """Update the values in the monthly S2S file.""" if not os.path.exists(outfile): print(f"[ERR] {outfile} does not exist!") sys.exit(1) ncid = nc4_dataset(outfile, 'a', format='NETCDF4_CLASSIC') for dictionary in [accs, tavgs]: for varname in dictionary: var = ncid.variables[varname] if len(var.shape) == 4: var[:, :, :, :] = dictionary[varname][:, :, :, :] elif len(var.shape) == 3: var[:, :, :] = dictionary[varname][:, :, :] elif len(var.shape) == 2: var[:, :] = dictionary[varname][:, :] ncid.close()
def write_bc_netcdf(outfile, var, varname, description, source, var_units, \ var_standard_name, lons, lats, sdate, dates, sig_digit, north_east_corner_lat, \ north_east_corner_lon, south_west_corner_lat, south_west_corner_lon, \ resolution_x, resolution_y, time_increment): """write netcdf""" rootgrp = nc4_dataset(outfile, 'w', format='NETCDF4_CLASSIC') time = rootgrp.createDimension('time', None) longitude = rootgrp.createDimension('lon', len(lons)) latitude = rootgrp.createDimension('lat', len(lats)) longitudes = rootgrp.createVariable('lon', 'f4', ('lon', )) latitudes = rootgrp.createVariable('lat', 'f4', ('lat', )) times = rootgrp.createVariable('time', 'f4', ('time', )) # two dimensions unlimited. varname = rootgrp.createVariable(varname, 'f4', ('time', 'lat', \ 'lon',), fill_value=-9999, zlib=True, \ least_significant_digit=sig_digit) rootgrp.missing_value = -9999 rootgrp.description = description rootgrp.zenith_interp = "true,false," rootgrp.MAP_PROJECTION = "EQUIDISTANT CYLINDRICAL" rootgrp.conventions = "CF-1.6" rootgrp.south_west_corner_lat = float(south_west_corner_lat) rootgrp.south_west_corner_lon = float(south_west_corner_lon) rootgrp.north_east_corner_lat = float(north_east_corner_lat) rootgrp.north_east_corner_lon = float(north_east_corner_lon) rootgrp.DX = resolution_x rootgrp.DY = resolution_y rootgrp.history = 'Created ' + t_ctime(t_time()) rootgrp.source = source latitudes.units = 'degrees_north' longitudes.units = 'degrees_east' varname.units = var_units varname.standard_name = var_standard_name string_date = datetime.strftime(sdate, "%Y-%m-%d %H:%M:%S") times.units = 'minutes since ' + string_date times.time_increment = time_increment times.begin_date = datetime.strftime(sdate, "%Y%m%d") times.begin_time = '000000' times.calendar = 'gregorian' latitudes[:] = lats longitudes[:] = lons varname[:, :, :] = var times[:] = nc4_date2num(dates, units=times.units, calendar=times.calendar) rootgrp.close()
def _read_second_daily_file(varlists, infile): """Read the second daily S2S file and copy the acc and tavg fields in appropriate dictionaries. We use the second file to start, since acc and tavg are valid for the prior 24-hr period.""" if not os.path.exists(infile): print(f"[ERR] {infile} does not exist!") sys.exit(1) ncid_in = nc4_dataset(infile, 'r', format='NETCDF4_CLASSIC') accs = {} tavgs = {} tavgs["counter"] = 1 # Copy the values of the fields we will average or accumulate varnames = [] for listname in [ "var_tavg_land_list", "var_tavg_f_list", "var_tavg_twsgws_list", "var_tair_max_list", "var_tair_min_list" ]: varnames += varlists[listname] for varname in varnames: var_in = ncid_in.variables[varname] if len(var_in.shape) == 4: tavgs[varname] = var_in[:, :, :, :] elif len(var_in.shape) == 3: tavgs[varname] = var_in[:, :, :] elif len(var_in.shape) == 2: tavgs[varname] = var_in[:, :] for varname in varlists["var_acc_list"]: var_in = ncid_in.variables[varname] if len(var_in.shape) == 4: accs[varname] = var_in[:, :, :, :] elif len(var_in.shape) == 3: accs[varname] = var_in[:, :, :] elif len(var_in.shape) == 2: accs[varname] = var_in[:, :] ncid_in.close() return accs, tavgs
def _read_next_daily_file(varlists, infile, accs, tavgs): """Read next daily S2S file and copy the required variable values to appropriate dictionaries.""" if not os.path.exists(infile): print(f"[ERR] {infile} does not exist!") sys.exit(1) ncid_in = nc4_dataset(infile, 'r', format='NETCDF4_CLASSIC') tavgs["counter"] += 1 # Add the values of the fields we will average or accumulate varnames = [] for listname in [ "var_tavg_land_list", "var_tavg_f_list", "var_tavg_twsgws_list", "var_tair_max_list", "var_tair_min_list" ]: varnames += varlists[listname] for varname in varnames: var_in = ncid_in.variables[varname] if len(var_in.shape) == 4: tavgs[varname][:, :, :, :] += var_in[:, :, :, :] elif len(var_in.shape) == 3: tavgs[varname][:, :, :] += var_in[:, :, :] elif len(var_in.shape) == 2: tavgs[varname][:, :] += var_in[:, :] for varname in varlists["var_acc_list"]: var_in = ncid_in.variables[varname] if len(var_in.shape) == 4: accs[varname][:, :, :, :] = var_in[:, :, :, :] elif len(var_in.shape) == 3: accs[varname][:, :, :] = var_in[:, :, :] elif len(var_in.shape) == 2: accs[varname][:, :] = var_in[:, :] ncid_in.close() return accs, tavgs
def write_bc_netcdf(outfile, var, varname, description, source, var_units, \ var_standard_name, lons, lats, sdate, dates, sig_digit, north_east_corner_lat, \ north_east_corner_lon, south_west_corner_lat, south_west_corner_lon, \ resolution_x, resolution_y, time_increment): """write netcdf""" rootgrp = nc4_dataset(outfile, 'w', format='NETCDF4_CLASSIC') time = rootgrp.createDimension('time', None) longitude = rootgrp.createDimension('longitude', len(lons)) latitude = rootgrp.createDimension('latitude', len(lats)) longitudes = rootgrp.createVariable('longitude', 'f4', ('longitude', )) latitudes = rootgrp.createVariable('latitude', 'f4', ('latitude', )) times = rootgrp.createVariable('time', 'f4', ('time', )) # two dimensions unlimited. varname1 = rootgrp.createVariable(varname[0], 'f4', ('time', \ 'latitude', 'longitude',), fill_value=-9999, zlib=True, \ least_significant_digit=sig_digit) varname2 = rootgrp.createVariable(varname[1], 'f4', ('time', \ 'latitude', 'longitude',), fill_value=-9999, zlib=True, \ least_significant_digit=sig_digit) varname3 = rootgrp.createVariable(varname[2], 'f4', ('time', \ 'latitude', 'longitude',), fill_value=-9999, zlib=True, \ least_significant_digit=sig_digit) varname4 = rootgrp.createVariable(varname[3], 'f4', ('time', \ 'latitude', 'longitude',), fill_value=-9999, zlib=True, \ least_significant_digit=sig_digit) varname5 = rootgrp.createVariable(varname[4], 'f4', ('time', \ 'latitude', 'longitude',), fill_value=-9999, zlib=True, \ least_significant_digit=sig_digit) varname6 = rootgrp.createVariable(varname[5], 'f4', ('time', \ 'latitude', 'longitude',), fill_value=-9999, zlib=True, \ least_significant_digit=sig_digit) rootgrp.missing_value = -9999 rootgrp.description = description rootgrp.zenith_interp = "true,false," rootgrp.MAP_PROJECTION = "EQUIDISTANT CYLINDRICAL" rootgrp.conventions = "CF-1.6" rootgrp.SOUTH_WEST_CORNER_LAT = float(south_west_corner_lat) rootgrp.SOUTH_WEST_CORNER_LON = float(south_west_corner_lon) rootgrp.NORTH_EAST_CORNER_LAT = float(north_east_corner_lat) rootgrp.NORTH_EAST_CORNER_LON = float(north_east_corner_lon) rootgrp.DX = resolution_x rootgrp.DY = resolution_y #rootgrp.history = 'Created ' + time.ctime(time.time()) rootgrp.history = 'Created ' + t_ctime(t_time()) rootgrp.source = source latitudes.units = 'degrees_north' longitudes.units = 'degrees_east' ### Assigning units for each variables varname1.units = var_units[0] varname2.units = var_units[1] varname3.units = var_units[2] varname4.units = var_units[3] varname5.units = var_units[4] varname6.units = var_units[5] ### Assigning standard names for each variables varname1.standard_name = var_standard_name[0] varname2.standard_name = var_standard_name[1] varname3.standard_name = var_standard_name[2] varname4.standard_name = var_standard_name[3] varname5.standard_name = var_standard_name[4] varname6.standard_name = var_standard_name[5] string_date = datetime.strftime(sdate, "%Y-%m-%d %H:%M:%S") times.units = 'minutes since ' + string_date times.time_increment = time_increment times.begin_date = datetime.strftime(sdate, "%Y%m%d") times.begin_time = '000000' times.calendar = 'gregorian' latitudes[:] = lats longitudes[:] = lons ## Passing on values varname1[:, :, :] = var[0, ] varname2[:, :, :] = var[1, ] varname3[:, :, :] = var[2, ] varname4[:, :, :] = var[3, ] varname5[:, :, :] = var[4, ] varname6[:, :, :] = var[5, ] times[:] = nc4_date2num(dates, units=times.units, calendar=times.calendar) rootgrp.close()
def get_ensemble_size(self): """Fetch number of ensemble members in LIS output.""" rootgrp = nc4_dataset(self.metricfile, 'r', format="NETCDF4_CLASSIC") num_ens = rootgrp.dimensions["ens"].size rootgrp.close() return num_ens
def get_num_of_months(self): """Fetch number of months in LIS output.""" rootgrp = nc4_dataset(self.metricfile, 'r', format="NETCDF4_CLASSIC") num_months = rootgrp.dimensions["lead"].size rootgrp.close() return num_months
"noah39": ["0-0.1 m", "0.1-0.4 m", "0.4-1.0 m", "1.0-2.0 m"], "noahmp401": ["0-0.1 m", "0.1-0.4 m", "0.4-1.0 m", "1.0-2.0 m"], "jules50": ["0-0.1 m", "0.1-0.35 m", "0.35-1.0 m", "1.0-3.0 m"], } # Main driver if __name__ == "__main__": # Get the file names for this invocation. ldtfile, tsfile, finalfile, anomaly_gt_prefix, \ climo_gt_prefix, lsm, yyyymmddhh = \ _read_cmd_args() # First, fetch latitude/longitudes. This is pulled from the LDT parameter # file, since LVT output has data voids over water. ncid = nc4_dataset(ldtfile, 'r', format='NETCDF4') longitudes = ncid.variables["lon"][:, :] latitudes = ncid.variables["lat"][:, :] ncid.close() # Next, fetch the soil moisture anomalies from the LVT 'TS' file. ncid = nc4_dataset(tsfile, 'r', format='NETCDF4') for i in range(0, 4): # Loop across four LSM layers sm_anomalies = ncid.variables["SoilMoist"][i, :, :] nrows, ncols = sm_anomalies.shape _soil_layer = _SOIL_LAYERS[lsm][i] # Write soil moisture anomalies to GeoTIFF sm1 = sm_anomalies[::-1, :] geotransform = _make_geotransform(longitudes, latitudes, ncols, nrows)