def _read_cmd_args():
"""Read command line arguments."""
# Check if argument count is correct
if len(sys.argv) != 8:
print("[ERR] Invalid number of command line arguments!")
_usage()
sys.exit(1)
# Check if LDT parameter file can be opened.
_ldtfile = sys.argv[1]
ncid_ldt = nc4_dataset(_ldtfile, mode='r', format='NETCDF4_CLASSIC')
ncid_ldt.close()
# Check of LVT TS anomaly file can be opened
_tsfile = sys.argv[2]
ncid_lvt = nc4_dataset(_tsfile, mode='r', format='NETCDF4_CLASSIC')
ncid_lvt.close()
# Check of LVT FINAL anomaly file can be opened
_finalfile = sys.argv[3]
ncid_lvt = nc4_dataset(_finalfile, mode='r', format='NETCDF4_CLASSIC')
ncid_lvt.close()
_outfile_anomaly_prefix = sys.argv[4]
_outfile_climo_prefix = sys.argv[5]
_lsm = sys.argv[6]
_yyyymmddhh = sys.argv[7]
if _lsm not in ["noah39", "noahmp401", "jules50"]:
print("[ERR] Unknown LSM %s" % (_lsm))
print("Options are noah39, noahmp401, jules50")
sys.exit(1)
return _ldtfile, _tsfile, _finalfile, \
_outfile_anomaly_prefix, _outfile_climo_prefix, _lsm, _yyyymmddhh
def calc_medians(self):
"""Calculate medians of the metrics."""
self.median_data = {}
total_ens_size = 0
lead = 0
latitude = 0
longitude = 0
# We need to first count the total number of ensembles.
nmme_models = self.config["s2smetric"]["nmme_models"].split()
for nmme in nmme_models:
if nmme not in self.nmme_metric_files:
continue
metric_file = self.nmme_metric_files[nmme]
rootgrp = nc4_dataset(metric_file, 'r', format="NETCDF4_CLASSIC")
total_ens_size += rootgrp.dimensions["ens"].size
lead = rootgrp.dimensions["lead"].size
latitude = rootgrp.dimensions["latitude"].size
longitude = rootgrp.dimensions["longitude"].size
# Find dimensions for a single month
dims = (total_ens_size, latitude, longitude)
# Now loop through each month, load a NMME contribution into the
# total var array, and calculate the median.
self.median_data["median"] = []
for itime in range(0, lead):
var = np.zeros(dims)
iens = 0
start_end_set = False
for nmme in nmme_models:
if nmme not in self.nmme_metric_files:
continue
metric_file = self.nmme_metric_files[nmme]
rootgrp = nc4_dataset(metric_file,
'r',
format="NETCDF4_CLASSIC")
ens = rootgrp.dimensions["ens"].size
var[iens:(iens+ens),:,:] = \
rootgrp.variables[self.metric][:,itime,:,:].data
iens += ens
if not start_end_set:
self.median_data["num_months"] = \
rootgrp.dimensions["lead"].size
self.median_data["latitudes"] = \
rootgrp.variables["latitude"][:]
self.median_data["longitudes"] = \
rootgrp.variables["longitude"][:]
self.median_data["units"] = \
rootgrp.variables[self.metric].units
self.median_data["long_name"] = \
rootgrp.variables[self.metric].long_name
self.set_startdates_enddates_by_month()
start_end_set = True
rootgrp.close()
# Calculate the median for the current month
self.median_data["median"].append(np.median(var, axis=0))
del var
def _create_firstguess_monthly_file(varlists, infile, outfile):
"""Read daily S2S file, and most fields copy to monthly S2S file.
This allows us to cleanly copy dimensions and all attributes. The
numerical values of the arrays in the monthly S2S
file will be replaced later in the script."""
if not os.path.exists(infile):
print(f"[ERR] {infile} does not exist!")
sys.exit(1)
ncid_in = nc4_dataset(infile, 'r', format='NETCDF4_CLASSIC')
# Create monthly file, copying dimensions and global attributes.
# NOTE: We clean up the global attributes later in the script.
ncid_out = nc4_dataset(outfile, "w", format='NETCDF4_CLASSIC')
_copy_dims_gattrs(ncid_in, ncid_out)
# Copy the fields.
varnames = []
for listname in [
"const_list", "var_inst_list", "var_acc_list",
"var_tavg_land_list", "var_tavg_f_list", "var_tavg_twsgws_list",
"var_tair_max_list", "var_tair_min_list"
]:
varnames += varlists[listname]
for varname in varnames:
var_in = ncid_in.variables[varname]
if "missing_value" in var_in.__dict__:
var_out = \
ncid_out.createVariable(varname, var_in.datatype,
dimensions=var_in.dimensions,
zlib=True,
complevel=1,
shuffle=True,
fill_value=var_in.missing_value)
else:
var_out = \
ncid_out.createVariable(varname, var_in.datatype,
dimensions=var_in.dimensions,
zlib=True,
complevel=1,
shuffle=True)
for attrname in var_in.__dict__:
if attrname == "_FillValue":
continue
var_out.setncattr(attrname, var_in.__dict__[attrname])
if len(var_out.shape) == 4:
var_out[:, :, :, :] = var_in[:, :, :, :]
elif len(var_out.shape) == 3:
var_out[:, :, :] = var_in[:, :, :]
elif len(var_out.shape) == 2:
var_out[:, :] = var_in[:, :]
elif len(var_out.shape) == 1:
var_out[:] = var_in[:]
ncid_out.close()
ncid_in.close()
def write_3d_netcdf(infile, var, varname, description, source, \
var_units, lons, lats, sdate):
"""write netcdf files"""
rootgrp = nc4_dataset(infile, 'w', format='NETCDF4')
longitude = rootgrp.createDimension('lon', len(lons))
latitude = rootgrp.createDimension('lat', len(lats))
time = rootgrp.createDimension('time', None)
longitudes = rootgrp.createVariable('lon', 'f4', ('lon', ))
latitudes = rootgrp.createVariable('lat', 'f4', ('lat', ))
times = rootgrp.createVariable('time', 'f8', ('time', ))
# two dimensions unlimited.
varname = rootgrp.createVariable(varname, 'f4', \
('time', 'lat', 'lon'), \
fill_value=-9999., zlib=True)
#import time
rootgrp.description = description
rootgrp.history = 'Created ' + t_ctime(t_time())
rootgrp.source = source
latitudes.units = 'degrees_north'
longitudes.units = 'degrees_east'
varname.units = var_units
string_date = datetime.strftime(sdate, "%Y-%m-%d")
times.units = 'days since ' + string_date
times.calendar = 'gregorian'
latitudes[:] = lats
longitudes[:] = lons
varname[:, :, :] = var
times[:] = nc4_date2num(sdate, units=times.units, calendar=times.calendar)
rootgrp.close()
def __init__(self, metricfile):
"""Constructor"""
self.metricfile = metricfile
# Parse the name of the metric file, and save elements. Assumes
# filename obeys Air Force Weather file naming convention.
self.metric_filename_elements = {}
element_list = self.metricfile.split("_")
for element in element_list:
key = element.split(".")[0]
value = element.split(".")[1]
self.metric_filename_elements[key] = value
# Find number of months in metrics file, and save latitudes and
# longitudes.
rootgrp = nc4_dataset(self.metricfile, 'r', format="NETCDF4_CLASSIC")
num_months = rootgrp.dimensions["lead"].size
self.latitudes = rootgrp.variables["latitude"][:]
self.longitudes = rootgrp.variables["longitude"][:]
rootgrp.close()
# Set start and end of each month of data in the metric file.
self.startdates_by_month = []
self.enddates_by_month = []
for imonth in range(0, num_months):
if imonth == 0:
self.startdates_by_month.append(self.get_first_startdate())
newdate2 = _set_newdate(self.get_first_startdate())
self.enddates_by_month.append(newdate2)
else:
newdate1 = _set_newdate(self.startdates_by_month[imonth - 1])
newdate2 = _set_newdate(self.enddates_by_month[imonth - 1])
self.startdates_by_month.append(newdate1)
self.enddates_by_month.append(newdate2)
def get_variable(self, varname):
"""Retrieve variable and select metadata from metric file."""
rootgrp = nc4_dataset(self.metricfile, 'r', format="NETCDF4_CLASSIC")
var = rootgrp.variables[varname][:, :, :, :]
units = rootgrp.variables[varname].units
long_name = rootgrp.variables[varname].long_name
rootgrp.close()
return var, units, long_name
def _cleanup_global_attrs(outfile):
"""Clean-up global attributes."""
if not os.path.exists(outfile):
print(f"[ERR] {outfile} does not exist!")
sys.exit(1)
ncid = nc4_dataset(outfile, 'a', format='NETCDF4_CLASSIC')
ncid.history = f"created on date: {time.ctime()}"
del ncid.NCO
del ncid.history_of_appended_files
def _add_time_data(infile, outfile, startdate, enddate):
"""Add time information to outfile, matching CF convention. This
requires pulling more data from the last daily file."""
if not os.path.exists(infile):
print(f"[ERR] {infile} does not exist!")
sys.exit(1)
ncid_in = nc4_dataset(infile, 'r', format='NETCDF4_CLASSIC')
if not os.path.exists(outfile):
print(f"[ERR] {outfile} does not exist!")
sys.exit(1)
ncid_out = nc4_dataset(outfile, 'a', format='NETCDF4_CLASSIC')
# Copy the time array from the last daily file.
var_in = ncid_in.variables["time"]
var_out = ncid_out.createVariable("time",
var_in.datatype,
dimensions=var_in.dimensions,
zlib=True,
complevel=1,
shuffle=True)
for attrname in var_in.__dict__:
if attrname == "_FillValue":
continue
var_out.setncattr(attrname, var_in.__dict__[attrname])
var_out[:] = var_in[:]
# Copy the time_bnds array from the last daily file. But, we will change
# the value to span one month of data.
var_in = ncid_in.variables["time_bnds"]
var_out = ncid_out.createVariable("time_bnds",
var_in.datatype,
dimensions=var_in.dimensions,
zlib=True,
complevel=1,
shuffle=True)
var_out[:, :] = var_in[:, :]
# Count number of minutes between start and end dates.
var_out[0,
0] = ((enddate - startdate).days) * (-24 * 60) # Days to minutes
var_out[0, 1] = 0
ncid_in.close()
ncid_out.close()
def _update_cell_methods(varlists, outfile):
"""Update cell_method attributes for select variables."""
if not os.path.exists(outfile):
print(f"[ERR] {outfile} does not exist!")
sys.exit(1)
ncid = nc4_dataset(outfile, 'a', format='NETCDF4_CLASSIC')
# Elaborate on monthly calculations of most variables
varnames = []
for listname in [
"var_acc_list", "var_tavg_land_list", "var_tavg_f_list",
"var_tavg_twsgws_list", "var_tair_max_list", "var_tair_min_list"
]:
varnames += varlists[listname]
for varname in varnames:
var = ncid.variables[varname]
# Special handling for TWS_inst and GWS_inst -- we want the monthly
# means.
if varname in varlists["var_tavg_twsgws_list"]:
var.cell_methods = \
"time: mean (interval: 1 day) area: point where land"
# Special handling for Tair_f_max and Tair_f_min: We want the monthly
# averages of the daily maxs and mins
elif varname in varlists["var_tair_max_list"]:
var.cell_methods = \
"time: mean (interval: 1 day comment: daily maxima)"
elif varname in varlists["var_tair_min_list"]:
var.cell_methods = \
"time: mean (interval: 1 day comment: daily minima)"
# Clarify monthly mean of daily means over land
elif varname in varlists["var_tavg_land_list"]:
var.cell_methods = \
"time: mean (interval: 1 day comment: daily means)" + \
" area: point where land"
# Clarify monthly mean of daily means everywhere
elif varname in varlists["var_tavg_f_list"]:
var.cell_methods = \
"time: mean (interval: 1 day comment: daily means)"
# Clarify monthly accumulations
elif varname in varlists["var_acc_list"]:
var.cell_methods = \
"time: sum (interval: 1 day comment: daily sums)"
ncid.close()
def _read_cmd_args():
"""Read command line arguments."""
# Check if argument count is correct
if len(sys.argv) != 2:
print("[ERR] Invalid number of command line arguments!")
_usage()
sys.exit(1)
# Check if metric file can be opened.
metricfile = sys.argv[1]
rootgrp = nc4_dataset(metricfile, mode="r", format="NETCDF4_CLASSIC")
rootgrp.close()
return metricfile
def _update_monthly_s2s_values(outfile, accs, tavgs):
"""Update the values in the monthly S2S file."""
if not os.path.exists(outfile):
print(f"[ERR] {outfile} does not exist!")
sys.exit(1)
ncid = nc4_dataset(outfile, 'a', format='NETCDF4_CLASSIC')
for dictionary in [accs, tavgs]:
for varname in dictionary:
var = ncid.variables[varname]
if len(var.shape) == 4:
var[:, :, :, :] = dictionary[varname][:, :, :, :]
elif len(var.shape) == 3:
var[:, :, :] = dictionary[varname][:, :, :]
elif len(var.shape) == 2:
var[:, :] = dictionary[varname][:, :]
ncid.close()
def write_bc_netcdf(outfile, var, varname, description, source, var_units, \
var_standard_name, lons, lats, sdate, dates, sig_digit, north_east_corner_lat, \
north_east_corner_lon, south_west_corner_lat, south_west_corner_lon, \
resolution_x, resolution_y, time_increment):
"""write netcdf"""
rootgrp = nc4_dataset(outfile, 'w', format='NETCDF4_CLASSIC')
time = rootgrp.createDimension('time', None)
longitude = rootgrp.createDimension('lon', len(lons))
latitude = rootgrp.createDimension('lat', len(lats))
longitudes = rootgrp.createVariable('lon', 'f4', ('lon', ))
latitudes = rootgrp.createVariable('lat', 'f4', ('lat', ))
times = rootgrp.createVariable('time', 'f4', ('time', ))
# two dimensions unlimited.
varname = rootgrp.createVariable(varname, 'f4', ('time', 'lat', \
'lon',), fill_value=-9999, zlib=True, \
least_significant_digit=sig_digit)
rootgrp.missing_value = -9999
rootgrp.description = description
rootgrp.zenith_interp = "true,false,"
rootgrp.MAP_PROJECTION = "EQUIDISTANT CYLINDRICAL"
rootgrp.conventions = "CF-1.6"
rootgrp.south_west_corner_lat = float(south_west_corner_lat)
rootgrp.south_west_corner_lon = float(south_west_corner_lon)
rootgrp.north_east_corner_lat = float(north_east_corner_lat)
rootgrp.north_east_corner_lon = float(north_east_corner_lon)
rootgrp.DX = resolution_x
rootgrp.DY = resolution_y
rootgrp.history = 'Created ' + t_ctime(t_time())
rootgrp.source = source
latitudes.units = 'degrees_north'
longitudes.units = 'degrees_east'
varname.units = var_units
varname.standard_name = var_standard_name
string_date = datetime.strftime(sdate, "%Y-%m-%d %H:%M:%S")
times.units = 'minutes since ' + string_date
times.time_increment = time_increment
times.begin_date = datetime.strftime(sdate, "%Y%m%d")
times.begin_time = '000000'
times.calendar = 'gregorian'
latitudes[:] = lats
longitudes[:] = lons
varname[:, :, :] = var
times[:] = nc4_date2num(dates, units=times.units, calendar=times.calendar)
rootgrp.close()
def _read_second_daily_file(varlists, infile):
"""Read the second daily S2S file and copy the acc and tavg fields in
appropriate dictionaries. We use the second file to start, since acc
and tavg are valid for the prior 24-hr period."""
if not os.path.exists(infile):
print(f"[ERR] {infile} does not exist!")
sys.exit(1)
ncid_in = nc4_dataset(infile, 'r', format='NETCDF4_CLASSIC')
accs = {}
tavgs = {}
tavgs["counter"] = 1
# Copy the values of the fields we will average or accumulate
varnames = []
for listname in [
"var_tavg_land_list", "var_tavg_f_list", "var_tavg_twsgws_list",
"var_tair_max_list", "var_tair_min_list"
]:
varnames += varlists[listname]
for varname in varnames:
var_in = ncid_in.variables[varname]
if len(var_in.shape) == 4:
tavgs[varname] = var_in[:, :, :, :]
elif len(var_in.shape) == 3:
tavgs[varname] = var_in[:, :, :]
elif len(var_in.shape) == 2:
tavgs[varname] = var_in[:, :]
for varname in varlists["var_acc_list"]:
var_in = ncid_in.variables[varname]
if len(var_in.shape) == 4:
accs[varname] = var_in[:, :, :, :]
elif len(var_in.shape) == 3:
accs[varname] = var_in[:, :, :]
elif len(var_in.shape) == 2:
accs[varname] = var_in[:, :]
ncid_in.close()
return accs, tavgs
def _read_next_daily_file(varlists, infile, accs, tavgs):
"""Read next daily S2S file and copy the required variable values to
appropriate dictionaries."""
if not os.path.exists(infile):
print(f"[ERR] {infile} does not exist!")
sys.exit(1)
ncid_in = nc4_dataset(infile, 'r', format='NETCDF4_CLASSIC')
tavgs["counter"] += 1
# Add the values of the fields we will average or accumulate
varnames = []
for listname in [
"var_tavg_land_list", "var_tavg_f_list", "var_tavg_twsgws_list",
"var_tair_max_list", "var_tair_min_list"
]:
varnames += varlists[listname]
for varname in varnames:
var_in = ncid_in.variables[varname]
if len(var_in.shape) == 4:
tavgs[varname][:, :, :, :] += var_in[:, :, :, :]
elif len(var_in.shape) == 3:
tavgs[varname][:, :, :] += var_in[:, :, :]
elif len(var_in.shape) == 2:
tavgs[varname][:, :] += var_in[:, :]
for varname in varlists["var_acc_list"]:
var_in = ncid_in.variables[varname]
if len(var_in.shape) == 4:
accs[varname][:, :, :, :] = var_in[:, :, :, :]
elif len(var_in.shape) == 3:
accs[varname][:, :, :] = var_in[:, :, :]
elif len(var_in.shape) == 2:
accs[varname][:, :] = var_in[:, :]
ncid_in.close()
return accs, tavgs
def write_bc_netcdf(outfile, var, varname, description, source, var_units, \
var_standard_name, lons, lats, sdate, dates, sig_digit, north_east_corner_lat, \
north_east_corner_lon, south_west_corner_lat, south_west_corner_lon, \
resolution_x, resolution_y, time_increment):
"""write netcdf"""
rootgrp = nc4_dataset(outfile, 'w', format='NETCDF4_CLASSIC')
time = rootgrp.createDimension('time', None)
longitude = rootgrp.createDimension('longitude', len(lons))
latitude = rootgrp.createDimension('latitude', len(lats))
longitudes = rootgrp.createVariable('longitude', 'f4', ('longitude', ))
latitudes = rootgrp.createVariable('latitude', 'f4', ('latitude', ))
times = rootgrp.createVariable('time', 'f4', ('time', ))
# two dimensions unlimited.
varname1 = rootgrp.createVariable(varname[0], 'f4', ('time', \
'latitude', 'longitude',), fill_value=-9999, zlib=True, \
least_significant_digit=sig_digit)
varname2 = rootgrp.createVariable(varname[1], 'f4', ('time', \
'latitude', 'longitude',), fill_value=-9999, zlib=True, \
least_significant_digit=sig_digit)
varname3 = rootgrp.createVariable(varname[2], 'f4', ('time', \
'latitude', 'longitude',), fill_value=-9999, zlib=True, \
least_significant_digit=sig_digit)
varname4 = rootgrp.createVariable(varname[3], 'f4', ('time', \
'latitude', 'longitude',), fill_value=-9999, zlib=True, \
least_significant_digit=sig_digit)
varname5 = rootgrp.createVariable(varname[4], 'f4', ('time', \
'latitude', 'longitude',), fill_value=-9999, zlib=True, \
least_significant_digit=sig_digit)
varname6 = rootgrp.createVariable(varname[5], 'f4', ('time', \
'latitude', 'longitude',), fill_value=-9999, zlib=True, \
least_significant_digit=sig_digit)
rootgrp.missing_value = -9999
rootgrp.description = description
rootgrp.zenith_interp = "true,false,"
rootgrp.MAP_PROJECTION = "EQUIDISTANT CYLINDRICAL"
rootgrp.conventions = "CF-1.6"
rootgrp.SOUTH_WEST_CORNER_LAT = float(south_west_corner_lat)
rootgrp.SOUTH_WEST_CORNER_LON = float(south_west_corner_lon)
rootgrp.NORTH_EAST_CORNER_LAT = float(north_east_corner_lat)
rootgrp.NORTH_EAST_CORNER_LON = float(north_east_corner_lon)
rootgrp.DX = resolution_x
rootgrp.DY = resolution_y
#rootgrp.history = 'Created ' + time.ctime(time.time())
rootgrp.history = 'Created ' + t_ctime(t_time())
rootgrp.source = source
latitudes.units = 'degrees_north'
longitudes.units = 'degrees_east'
### Assigning units for each variables
varname1.units = var_units[0]
varname2.units = var_units[1]
varname3.units = var_units[2]
varname4.units = var_units[3]
varname5.units = var_units[4]
varname6.units = var_units[5]
### Assigning standard names for each variables
varname1.standard_name = var_standard_name[0]
varname2.standard_name = var_standard_name[1]
varname3.standard_name = var_standard_name[2]
varname4.standard_name = var_standard_name[3]
varname5.standard_name = var_standard_name[4]
varname6.standard_name = var_standard_name[5]
string_date = datetime.strftime(sdate, "%Y-%m-%d %H:%M:%S")
times.units = 'minutes since ' + string_date
times.time_increment = time_increment
times.begin_date = datetime.strftime(sdate, "%Y%m%d")
times.begin_time = '000000'
times.calendar = 'gregorian'
latitudes[:] = lats
longitudes[:] = lons
## Passing on values
varname1[:, :, :] = var[0, ]
varname2[:, :, :] = var[1, ]
varname3[:, :, :] = var[2, ]
varname4[:, :, :] = var[3, ]
varname5[:, :, :] = var[4, ]
varname6[:, :, :] = var[5, ]
times[:] = nc4_date2num(dates, units=times.units, calendar=times.calendar)
rootgrp.close()
def get_ensemble_size(self):
"""Fetch number of ensemble members in LIS output."""
rootgrp = nc4_dataset(self.metricfile, 'r', format="NETCDF4_CLASSIC")
num_ens = rootgrp.dimensions["ens"].size
rootgrp.close()
return num_ens
def get_num_of_months(self):
"""Fetch number of months in LIS output."""
rootgrp = nc4_dataset(self.metricfile, 'r', format="NETCDF4_CLASSIC")
num_months = rootgrp.dimensions["lead"].size
rootgrp.close()
return num_months
"noah39": ["0-0.1 m", "0.1-0.4 m", "0.4-1.0 m", "1.0-2.0 m"],
"noahmp401": ["0-0.1 m", "0.1-0.4 m", "0.4-1.0 m", "1.0-2.0 m"],
"jules50": ["0-0.1 m", "0.1-0.35 m", "0.35-1.0 m", "1.0-3.0 m"],
}
# Main driver
if __name__ == "__main__":
# Get the file names for this invocation.
ldtfile, tsfile, finalfile, anomaly_gt_prefix, \
climo_gt_prefix, lsm, yyyymmddhh = \
_read_cmd_args()
# First, fetch latitude/longitudes. This is pulled from the LDT parameter
# file, since LVT output has data voids over water.
ncid = nc4_dataset(ldtfile, 'r', format='NETCDF4')
longitudes = ncid.variables["lon"][:, :]
latitudes = ncid.variables["lat"][:, :]
ncid.close()
# Next, fetch the soil moisture anomalies from the LVT 'TS' file.
ncid = nc4_dataset(tsfile, 'r', format='NETCDF4')
for i in range(0, 4): # Loop across four LSM layers
sm_anomalies = ncid.variables["SoilMoist"][i, :, :]
nrows, ncols = sm_anomalies.shape
_soil_layer = _SOIL_LAYERS[lsm][i]
# Write soil moisture anomalies to GeoTIFF
sm1 = sm_anomalies[::-1, :]
geotransform = _make_geotransform(longitudes, latitudes, ncols, nrows)
