def loadFile(self, inputFile, varname):
"""
Load a variable from a netcdf file and return data as a masked array
:param str inputFile: path to a netcdf file containing hazard data.
:param str varname: name of the netcdf variable to plot.
:returns: lon, lat, years and data (as a masked array)
"""
try:
ncobj = nctools.ncLoadFile(inputFile)
lon = nctools.ncGetDims(ncobj, 'lon')
lat = nctools.ncGetDims(ncobj, 'lat')
years = nctools.ncGetDims(ncobj, 'years')
data = nctools.ncGetData(ncobj, varname)
mv = getattr(ncobj.variables[varname], '_FillValue')
ncobj.close()
del ncobj
except:
self.logger.critical("Cannot load input file: %s"%inputFile)
try:
ncobj.close()
except (IOError, KeyError, RuntimeError):
pass
raise
# Create a masked array:
mask = (data==mv)
mdata = ma.array(data, mask=mask)
return lon, lat, years, mdata
def loadFile(self, inputFile, varname):
"""
Load a variable from a netcdf file and return data as a masked array
:param str inputFile: path to a netcdf file containing hazard data.
:param str varname: name of the netcdf variable to plot.
:returns: lon, lat, years and data (as a masked array)
"""
try:
ncobj = nctools.ncLoadFile(inputFile)
lon = nctools.ncGetDims(ncobj, 'lon')
lat = nctools.ncGetDims(ncobj, 'lat')
years = nctools.ncGetDims(ncobj, 'ari')
data = nctools.ncGetData(ncobj, varname)
mv = getattr(ncobj.variables[varname], '_FillValue')
ncobj.close()
del ncobj
except:
log.critical("Cannot load input file: %s"%inputFile)
try:
ncobj.close()
except (IOError, KeyError, RuntimeError):
pass
raise
# Create a masked array:
mask = (data == mv)
mdata = ma.array(data, mask=mask)
return lon, lat, years, mdata
def setDomain(inputPath):
"""
Establish the full extent of input wind field files
Parameters:
-----------
:param inputPath: `str` path of folder containing wind field files
Returns:
--------
:param wf_lon: `numpy.ndarray` of longitudes of the wind field
:param wf_lat: `numpy.ndarray` of latitudes of the wind field
"""
fileList = os.listdir(inputPath)
inputFile = pjoin(inputPath, fileList[0])
ncobj = nctools.ncLoadFile(inputFile)
wf_lon = nctools.ncGetDims(ncobj, 'lon')
wf_lat = nctools.ncGetDims(ncobj, 'lat')
ncobj.close()
return wf_lon, wf_lat
def process(argv):
recdim = None
mv = -9999.
try:
opts, args = getopt.getopt(argv, "f:hm:r:v:",
["filename=",
"help",
"missingvalue=",
"record=",
"variable="])
except getopt.GetoptError:
usage()
sys.exit(2)
for opt, arg in opts:
if opt in ("-h", "--help"):
usage()
sys.exit(2)
elif opt in ("-f", "--filename"):
filename = arg
elif opt in ("-m", "--missingvalue"):
mv = arg
elif opt in ("-v", "--variable"):
variable = arg
elif opt in ("-r", "--record"):
recdim = arg
ncobj = nctools.ncLoadFile(filename)
lat = nctools.ncGetDims(ncobj, 'lat')
lon = nctools.ncGetDims(ncobj, 'lon')
delta = lon[1] - lon[0]
# Fix incorrectly reported corner of lower left pixel
lon = lon - delta/2.
lat = lat - delta/2.
if recdim:
recval = nctools.ncGetDims(ncobj, recdim)
data = nctools.ncGetData(ncobj, variable)
ncobj.close()
if recdim:
for i, v in enumerate(recval):
outputfile = "%s.%s.%s"%(os.path.splitext(filename)[0],
repr(recval[i]), 'txt')
print("Saving data to %s"%outputfile)
grid.grdSave(outputfile, np.flipud(data[i]), lon, lat,
delta, delimiter=' ', nodata=mv, fmt='%6.2f')
else:
outputfile = "%s.%s"%(os.path.splitext(filename)[0], 'txt')
print("Saving data to %s"%outputfile)
grid.grdSave(outputfile, np.flipud(data), lon, lat,
delta, delimiter=' ', nodata=mv, fmt='%6.2f')
def setUp(self):
self.filename = os.path.join(TEST_DIR, 'test_data', 'landmask.nc')
# Load the data using grid.grdRead:
self.lslon, self.lslat, self.lsgrid = grid.grdRead(self.filename)
# Load the data using nctools.ncLoadFile and nctools.ncGetData:
ncobj = nctools.ncLoadFile(self.filename)
self.nclon = nctools.ncGetDims(ncobj, 'lon')
self.nclat = nctools.ncGetDims(ncobj, 'lat')
self.ncgrid = nctools.ncGetData(ncobj, 'landmask')
ncobj.close()
# Set up an instance of SampleGrid:
self.sample = grid.SampleGrid(self.filename)
# Sample the land-sea mask at these points around the globe:
self.xlon = [100., 130., 180., 250., 300.]
self.ylat = [-80., -20., 40.]
# Known point values of the land-sea mask data:
self.ls = [3., 0., 3., 3., 3., 0., 3., 0., 0., 3., 0., 3., 3., 3., 0.]
def setUp(self):
self.filename = os.path.join(TEST_DIR, 'test_data', 'landmask.nc')
# Load the data using grid.grdRead:
self.lslon, self.lslat, self.lsgrid = grid.grdRead(self.filename)
# Load the data using nctools.ncLoadFile and nctools.ncGetData:
ncobj = nctools.ncLoadFile(self.filename)
self.nclon = nctools.ncGetDims(ncobj, 'lon')
self.nclat = nctools.ncGetDims(ncobj, 'lat')
self.ncgrid = nctools.ncGetData(ncobj, 'landmask')
ncobj.close()
# Set up an instance of SampleGrid:
self.sample = grid.SampleGrid(self.filename)
# Sample the land-sea mask at these points around the globe:
self.xlon = [100., 130., 180., 250., 300.]
self.ylat = [-80., -20., 40.]
# Known point values of the land-sea mask data:
self.ls = [3., 0., 3., 3., 3., 0., 3., 0., 0., 3., 0., 3., 3., 3., 0.]
def setDomain(inputPath):
"""
Establish the full extent of input wind field files
:param str inputPath: path of folder containing wind field files
:return: Longitudes and latitudes of the wind field grid.
:rtype: `numpy.ndarray`
"""
fileList = os.listdir(inputPath)
inputFile = pjoin(inputPath, fileList[0])
ncobj = nctools.ncLoadFile(inputFile)
wf_lon = nctools.ncGetDims(ncobj, 'lon')
wf_lat = nctools.ncGetDims(ncobj, 'lat')
ncobj.close()
return wf_lon, wf_lat
def setDomain(inputPath):
"""
Establish the full extent of input wind field files
:param str inputPath: path of folder containing wind field files
:return: Longitudes and latitudes of the wind field grid.
:rtype: `numpy.ndarray`
"""
fileList = os.listdir(inputPath)
inputFile = pjoin(inputPath, fileList[0])
ncobj = nctools.ncLoadFile(inputFile)
wf_lon = nctools.ncGetDims(ncobj, 'lon')
wf_lat = nctools.ncGetDims(ncobj, 'lat')
ncobj.close()
return wf_lon, wf_lat
def __init__(self, selected_months, filename=''):
if not os.path.isfile(filename):
tcrm_dir = pathLocator.getRootDirectory()
filename = os.path.join(tcrm_dir, 'MSLP', 'mslp_monthly_clim.nc')
if not os.path.isfile(filename):
error_msg = "MSLP data file not found"
raise IOError(error_msg)
selected_months = set(selected_months)
ncobj = nctools.ncLoadFile(filename)
mslp_all = nctools.ncGetData(ncobj, 'mslp')
self.lon = nctools.ncGetDims(ncobj, 'lon')
self.lat = nctools.ncGetDims(ncobj, 'lat')
dim0, dim1, dim2 = np.shape(mslp_all)
# Average over selected months
mslp_sum = np.zeros([dim1, dim2], dtype='float32')
for month in selected_months:
mslp_sum = mslp_sum + mslp_all[month - 1, :, :]
self.mslp_av = np.flipud(mslp_sum / len(selected_months))
def __init__(self, selected_months, filename=''):
if not os.path.isfile(filename):
tcrm_dir = pathLocator.getRootDirectory()
filename = os.path.join(tcrm_dir, 'MSLP', 'mslp_monthly_clim.nc')
if not os.path.isfile(filename):
error_msg = "MSLP data file not found"
raise IOError, error_msg
selected_months = set(selected_months)
ncobj = nctools.ncLoadFile(filename)
mslp_all = nctools.ncGetData(ncobj, 'mslp')
self.lon = nctools.ncGetDims(ncobj, 'lon')
self.lat = nctools.ncGetDims(ncobj, 'lat')
dim0,dim1,dim2 = shape(mslp_all)
# Average over selected months
mslp_sum = zeros([dim1,dim2],dtype='float32')
for month in selected_months:
mslp_sum = mslp_sum + mslp_all[month-1,:,:]
self.mslp_av = flipud(mslp_sum / len(selected_months))
def test_ncReadFile(self):
"""Test nctools functions for reading dimensions and variables"""
ncobj = nctools.ncLoadFile(self.ncfile)
lats_check = -25.0 + 5.0*np.arange(self.nlats, dtype='float')
lons_check = 125.0 + 5.0*np.arange(self.nlons, dtype='float')
press_check = 900. + np.arange(self.nlevs*self.nlats*self.nlons, dtype='float64') # 1d array
press_check.shape = (self.nlevs, self.nlats, self.nlons) # reshape to 2d array
temp_check = 9. + np.arange(self.nlevs*self.nlats*self.nlons, dtype='float64') # 1d array
temp_check.shape = (self.nlevs, self.nlats, self.nlons) # reshape to 2d array
lats = nctools.ncGetDims(ncobj, 'lat')
lons = nctools.ncGetDims(ncobj, 'lon')
self.numpyAssertAlmostEqual(lats_check, lats)
self.numpyAssertAlmostEqual(lons_check, lons)
press = nctools.ncGetData(ncobj, 'pressure')
temp = nctools.ncGetData(ncobj, 'temperature')
for nrec in range(self.nrecs):
self.numpyAssertEqual(press_check, press[nrec])
self.numpyAssertEqual(temp_check, temp[nrec])
ncobj.close()
def setUp(self):
self.filename = os.path.join(unittest_dir, 'test_data', 'mslp_ltm.nc')
self.ncobj = nctools.ncLoadFile(self.filename)
self.lon = nctools.ncGetDims(self.ncobj, 'lon')
self.lat = nctools.ncGetDims(self.ncobj, 'lat')
self.time = nctools.ncGetDims(self.ncobj, 'time')
self.data = nctools.ncGetData(self.ncobj, 'mslp')
self.ncobj.close()
self.scale = [365.,-180.,360.]
self.offset = [0., 90., 0.]
# Load data
pfile = open(os.path.join(unittest_dir, 'test_data',
'testinterp3d.pck'), 'r')
self.xlon = cPickle.load(pfile)
self.ylat = cPickle.load(pfile)
self.ztime = cPickle.load(pfile)
self.values = cPickle.load(pfile)
pfile.close()
self.coords = numpy.array([self.ztime, self.ylat, self.xlon])
def setUp(self):
self.filename = os.path.join(unittest_dir, 'test_data', 'mslp_ltm.nc')
self.ncobj = nctools.ncLoadFile(self.filename)
self.lon = nctools.ncGetDims(self.ncobj, 'lon')
self.lat = nctools.ncGetDims(self.ncobj, 'lat')
self.time = nctools.ncGetDims(self.ncobj, 'time')
self.data = nctools.ncGetData(self.ncobj, 'mslp')
self.ncobj.close()
self.scale = [365., -180., 360.]
self.offset = [0., 90., 0.]
# Load data
pfile = open(
os.path.join(unittest_dir, 'test_data', 'testinterp3d.pkl'), 'rb')
self.xlon = pickle.load(pfile)
self.ylat = pickle.load(pfile)
self.ztime = pickle.load(pfile)
self.values = pickle.load(pfile)
pfile.close()
self.coords = numpy.array([self.ztime, self.ylat, self.xlon])
def test_ncReadFile(self):
"""Test nctools functions for reading dimensions and variables"""
ncobj = nctools.ncLoadFile(self.ncfile)
lats_check = -25.0 + 5.0 * np.arange(self.nlats, dtype='float')
lons_check = 125.0 + 5.0 * np.arange(self.nlons, dtype='float')
press_check = 900. + np.arange(self.nlevs * self.nlats * \
self.nlons, dtype='float64')
press_check.shape = (self.nlevs, self.nlats, self.nlons)
temp_check = 9. + np.arange(self.nlevs * self.nlats * \
self.nlons, dtype='float64')
temp_check.shape = (self.nlevs, self.nlats, self.nlons)
lats = nctools.ncGetDims(ncobj, 'lat')
lons = nctools.ncGetDims(ncobj, 'lon')
self.numpyAssertAlmostEqual(lats_check, lats)
self.numpyAssertAlmostEqual(lons_check, lons)
press = nctools.ncGetData(ncobj, 'pressure')
temp = nctools.ncGetData(ncobj, 'temperature')
for nrec in range(self.nrecs):
self.numpyAssertEqual(press_check, press[nrec])
self.numpyAssertEqual(temp_check, temp[nrec])
ncobj.close()
data = var
data[ij] = lvar[ij]
return data
# Let's breifly look at the statistical data generated. Here, we
# load the 'pressure_rate_stats.nc' file, which contains the statistics
# for the pressure rate of change (hPa/hr) of obsersved TCs.
outputPath = config.get("Output", "Path")
processPath = pjoin(outputPath, "process")
plotsPath = pjoin(outputPath, "plots", "stats")
fname = pjoin(processPath, "pressure_rate_stats.nc")
ncobj = ncLoadFile(fname)
lon = ncGetDims(ncobj, 'lon')
lat = ncGetDims(ncobj, 'lat')
ncobj.close()
xgrid, ygrid = np.meshgrid(lon, lat)
ls = np.zeros(np.shape(xgrid))
for i in range(len(lon)):
for j in range(len(lat)):
if landmask.sampleGrid(lon[i], lat[j]) > 0.0:
ls[j, i] = 1
ij = np.where(ls == 1)
# Set the map keyword arguments that will help draw the basemap.
def plotHazardCurves(self, inputFile, plotPath):
"""
Plot the hazard values stored in hazardFile, at the stns
stored in stnFile.
"""
log.info(("Plotting return period curves for locations within the "
"model domain"))
# Open data file
try:
ncobj = nctools.ncLoadFile(inputFile)
lon = nctools.ncGetDims(ncobj, 'lon')
lat = nctools.ncGetDims(ncobj, 'lat')
years = nctools.ncGetDims(ncobj, 'ari')
except (IOError, RuntimeError, KeyError):
log.critical("Cannot load input file: %s"%inputFile)
raise
placeNames, placeID, placeLats, placeLons, locations = self.getLocations()
for name, plat, plon, pID in zip(placeNames, placeLats, placeLons, placeID):
pID = int(pID)
log.debug("Plotting return period curve for %s"%name)
i = find_index(lon, plon)
j = find_index(lat, plat)
xlabel = 'Average recurrence interval (years)'
ylabel = 'Wind speed (%s)'%self.plotUnits.label
title = "Return period wind speeds at " + name + ", \n(%5.1f,%5.1f)"%(plon, plat)
name.replace(' ', '')
log.debug("Working on {0}".format(name))
filename = pjoin(plotPath, 'ARI_curve_%s.%s'%(pID, "png"))
log.debug("Saving hazard curve for %s to %s"%(name, filename))
wspd = ncobj.variables['wspd'][:, j, i]
recs = database.queries.locationRecords(self.db, pID)
data = np.zeros(int(self.numsimulations * 365.25))
if len(recs) > 0:
data[-len(recs):] = recs['wspd']
allevents = np.sort(data)
log.debug("allevents length = {0}".format(len(allevents)))
placeWspd = metutils.convert(wspd, 'mps',
self.plotUnits.units)
if np.all(placeWspd.mask):
log.debug("All values for {0} are null".format(name))
continue
if self.ciBounds:
wspdLower = ncobj.variables['wspdlower'][:, j, i]
wspdUpper = ncobj.variables['wspdupper'][:, j, i]
placeWspdLower = metutils.convert(wspdLower, 'mps',
self.plotUnits.units)
placeWspdUpper = metutils.convert(wspdUpper, 'mps',
self.plotUnits.units)
else:
placeWspdUpper = np.zeros(len(placeWspd))
placeWspdLower = np.zeros(len(placeWspd))
saveHazardCurve(years, allevents, placeWspd, placeWspdUpper, placeWspdLower,
xlabel, ylabel, title, filename, self.fit)
ncobj.close()
def plotHazardCurves(self, inputFile, plotPath):
"""
Plot the hazard values stored in hazardFile, at the stns
stored in stnFile.
"""
log.info(("Plotting return period curves for locations within the "
"model domain"))
# Open data file
try:
ncobj = nctools.ncLoadFile(inputFile)
lon = nctools.ncGetDims(ncobj, 'lon')
lat = nctools.ncGetDims(ncobj, 'lat')
years = nctools.ncGetDims(ncobj, 'years')
except (IOError, RuntimeError, KeyError):
log.critical("Cannot load input file: %s"%inputFile)
raise
# Load data
wspd = nctools.ncGetData(ncobj, 'wspd')
try:
wLower = nctools.ncGetData(ncobj, 'wspdlower')
wUpper = nctools.ncGetData(ncobj, 'wspdupper')
ciBounds = True
except KeyError:
ciBounds = False
ncobj.close()
minLon = min(lon)
maxLon = max(lon)
minLat = min(lat)
maxLat = max(lat)
# Use the same maximum value for all localities to simplify
# intercomparisons:
defaultMax = np.ceil(metutils.convert(100.0, 'mps',
self.plotUnits.units)/10.0)*10.0
placeNames, parentCountries, placeLats, placeLons = \
self.getLocations(minLon, maxLon, minLat, maxLat)
for name, plat, plon, country in zip(placeNames, placeLats,
placeLons, parentCountries):
log.debug("Plotting return period curve for %s"%name)
i = find_index(lon, plon)
j = find_index(lat, plat)
xlabel = 'Average recurrence interval (years)'
ylabel = 'Wind speed (%s)'%self.plotUnits.label
title = "Return period wind speeds at " + name + ", " \
+ country + "\n(%5.1f,%5.1f)"%(plon, plat)
name = unicodedata.normalize('NFKD', name).encode('ascii', 'ignore')
name.replace(' ', '')
filename = pjoin(plotPath, 'ARI_curve_%s.%s'%(name,"png"))
log.debug("Saving hazard curve for %s to %s"%(name, filename))
placeWspd = metutils.convert(wspd[:, j, i], 'mps',
self.plotUnits.units)
maxWspd = placeWspd.max()
if ciBounds:
placeWspdLower = metutils.convert(wLower[:,j,i], 'mps',
self.plotUnits.units)
placeWspdUpper = metutils.convert(wUpper[:,j,i], 'mps',
self.plotUnits.units)
saveHazardCurve(years, placeWspd, placeWspdUpper, placeWspdLower,
xlabel, ylabel, title, filename)
def plotHazardCurves(self, inputFile, plotPath):
"""
Plot the hazard values stored in hazardFile, at the stns
stored in stnFile.
"""
log.info(("Plotting return period curves for locations within the "
"model domain"))
# Open data file
try:
ncobj = nctools.ncLoadFile(inputFile)
lon = nctools.ncGetDims(ncobj, 'lon')
lat = nctools.ncGetDims(ncobj, 'lat')
years = nctools.ncGetDims(ncobj, 'years')
except (IOError, RuntimeError, KeyError):
log.critical("Cannot load input file: %s"%inputFile)
raise
# Load data
#wspd = nctools.ncGetData(ncobj, 'wspd')
#try:
# wLower = nctools.ncGetData(ncobj, 'wspdlower')
# wUpper = nctools.ncGetData(ncobj, 'wspdupper')
ciBounds = True
#except KeyError:
# ciBounds = False
#ncobj.close()
minLon = min(lon)
maxLon = max(lon)
minLat = min(lat)
maxLat = max(lat)
# Use the same maximum value for all localities to simplify
# intercomparisons:
#defaultMax = np.ceil(metutils.convert(100.0, 'mps',
# self.plotUnits.units)/10.0)*10.0
placeNames, parentCountries, placeLats, placeLons = \
self.getLocations(minLon, maxLon, minLat, maxLat)
for name, plat, plon, country in zip(placeNames, placeLats,
placeLons, parentCountries):
log.debug("Plotting return period curve for %s"%name)
i = find_index(lon, plon)
j = find_index(lat, plat)
xlabel = 'Average recurrence interval (years)'
ylabel = 'Wind speed (%s)'%self.plotUnits.label
title = "Return period wind speeds at " + name + ", " \
+ country + "\n(%5.1f,%5.1f)"%(plon, plat)
name = unicodedata.normalize('NFKD', name).encode('ascii', 'ignore')
name.replace(' ', '')
filename = pjoin(plotPath, 'ARI_curve_%s.%s'%(name, "png"))
log.debug("Saving hazard curve for %s to %s"%(name, filename))
wspd = ncobj.variables['wspd'][:, j, i]
placeWspd = metutils.convert(wspd, 'mps',
self.plotUnits.units)
if np.all(placeWspd.mask):
log.debug("All values for {0} are null".format(name))
continue
if ciBounds:
wspdLower = ncobj.variables['wspdlower'][:, j, i]
wspdUpper = ncobj.variables['wspdupper'][:, j, i]
placeWspdLower = metutils.convert(wspdLower, 'mps',
self.plotUnits.units)
placeWspdUpper = metutils.convert(wspdUpper, 'mps',
self.plotUnits.units)
saveHazardCurve(years, placeWspd, placeWspdUpper, placeWspdLower,
xlabel, ylabel, title, filename)
ncobj.close()
