def _process(argv):
recdim = None
mv = -9999.
try:
opts, args = getopt.getopt(argv,"f:hm:r:v:",["filename=","help","missingvalue=","record=","variable="])
except getopt.GetoptError:
_usage()
sys.exit(2)
for opt,arg in opts:
if opt in ("-h","--help"):
usage()
sys.exit(2)
elif opt in ("-f","--filename"):
filename = arg
elif opt in ("-m","--missingvalue"):
mv = arg
elif opt in ("-v","--variable"):
variable = arg
elif opt in ("-r,--record"):
recdim = arg
ncobj = nctools.ncLoadFile(filename)
lat = nctools.ncGetDims(ncobj, 'lat')
lon = nctools.ncGetDims(ncobj, 'lon')
delta = lon[1] - lon[0]
# Fix incorrectly reported corner of lower left pixel
lon = lon - delta/2
lat = lat - delta/2
if recdim:
recval = nctools.ncGetDims(ncobj,recdim)
#data = nctools.ncGetData(ncobj, variable, missingValue=mv)
data = nctools.ncGetData(ncobj, variable)
ncobj.close()
if recdim:
for i,v in enumerate(recval):
outputfile = "%s.%s.%s"%(os.path.splitext(filename)[0],repr(recval[i]),'txt')
print "Saving data to %s"%outputfile
grid.grdSave(outputfile, numpy.flipud(data[i]), lon, lat, delta, delimiter=' ', nodata=mv, fmt='%6.2f')
else:
outputfile="%s.%s"%(os.path.splitext(filename)[0],'txt')
print "Saving data to %s"%outputfile
grid.grdSave(outputfile, numpy.flipud(data), lon, lat, delta, delimiter=' ', nodata=mv, fmt='%6.2f')
def main():
"""
Handle command line arguments and call processing functions
"""
p = argparse.ArgumentParser()
p.add_argument('-c', '--config_file', help="Configuration file")
p.add_argument('-v',
'--verbose',
help="Verbose output",
action='store_true')
p.add_argument('-y', '--year', help="Year to process (1979-2020)")
args = p.parse_args()
configFile = args.config_file
config = ConfigParser()
config.read(configFile)
logFile = config.get('Logging', 'LogFile')
logdir = dirname(realpath(logFile))
# if log file directory does not exist, create it
if not isdir(logdir):
try:
os.makedirs(logdir)
except OSError:
logFile = pjoin(os.getcwd(), 'pcmin.log')
logLevel = config.get('Logging', 'LogLevel')
verbose = config.getboolean('Logging', 'Verbose')
datestamp = config.getboolean('Logging', 'Datestamp')
if args.verbose:
verbose = True
if comm.size > 1 and comm.rank > 0:
logFile += '-' + str(comm.rank)
verbose = False
if datestamp:
base, ext = splitext(logFile)
curdate = datetime.datetime.now()
curdatestr = curdate.strftime('%Y%m%d%H%M')
logfile = f"{base}.{curdatestr}.{ext.lstrip('.')}"
logging.basicConfig(level=logLevel,
format="%(asctime)s: %(funcName)s: %(message)s",
filename=logfile,
filemode='w',
datefmt="%Y-%m-%d %H:%M:%S")
if verbose:
console = logging.StreamHandler(sys.stdout)
console.setLevel(getattr(logging, logLevel))
formatter = logging.Formatter(
'%(asctime)s: %(funcName)s: %(message)s',
datefmt='%H:%M:%S',
)
console.setFormatter(formatter)
LOGGER.addHandler(console)
LOGGER.info(f"Started {sys.argv[0]} (pid {os.getpid()})")
LOGGER.info(f"Log file: {logfile} (detail level {logLevel})")
LOGGER.info(f"Code version: f{COMMIT}")
tpath = config.get('Input', 'Temp')
rpath = config.get('Input', 'Humidity')
sstpath = config.get('Input', 'SST')
slppath = config.get('Input', 'SLP')
if args.year:
year = int(args.year)
else:
year = 2015
minLon = config.getfloat('Domain', 'MinLon')
maxLon = config.getfloat('Domain', 'MaxLon')
minLat = config.getfloat('Domain', 'MinLat')
maxLat = config.getfloat('Domain', 'MaxLat')
LOGGER.info(f"Domain: {minLon}-{maxLon}, {minLat}-{maxLat}")
for month in range(1, 13):
LOGGER.info(f"Processing {year}-{month}")
startdate = datetime.datetime(year, month, 1)
enddate = datetime.datetime(year, month, monthrange(year, month)[1])
filedatestr = f"{startdate.strftime('%Y%m%d')}-{enddate.strftime('%Y%m%d')}"
tfile = pjoin(tpath, f'{year}', f't_era5_oper_pl_{filedatestr}.nc')
try:
assert (os.path.isfile(tfile))
except AssertionError:
LOGGER.warning(f"Input file is missing: {tfile}")
LOGGER.warning(f"Skipping month {month}")
continue
tobj = nctools.ncLoadFile(tfile)
tvar = nctools.ncGetVar(tobj, 't')
tvar.set_auto_maskandscale(True)
rfile = pjoin(rpath, f'{year}', f'r_era5_oper_pl_{filedatestr}.nc')
try:
assert (os.path.isfile(rfile))
except AssertionError:
LOGGER.warning(f"Input file is missing: {rfile}")
LOGGER.warning(f"Skipping month {month}")
continue
robj = nctools.ncLoadFile(rfile)
rvar = nctools.ncGetVar(robj, 'r')
rvar.set_auto_maskandscale(True)
# This is actually relative humidity, we need to convert to mixing ratio
# Calculate mixing ratio - this function returns mixing ratio in g/kg
# Dimensions need to come from the pressure files
# These have been clipped to the Australian region, so contain
# a subset of the global data. The SST and MSLP data
# are then clipped to the same domain
tlon = nctools.ncGetDims(tobj, 'longitude')
tlat = nctools.ncGetDims(tobj, 'latitude')
LOGGER.debug(f"Latitude extents: {tlat.min()} - {tlat.max()}")
LOGGER.debug(f"Longitude extents: {tlon.min()} - {tlon.max()}")
varidx = np.where((tlon >= minLon) & (tlon <= maxLon))[0]
varidy = np.where((tlat >= minLat) & (tlat <= maxLat))[0]
templon = tlon[varidx]
templat = tlat[varidy]
LOGGER.info(f"Loading SST data")
sstfile = pjoin(sstpath, f'{year}',
f'sst_era5_oper_sfc_{filedatestr}.nc')
try:
assert (os.path.isfile(sstfile))
except AssertionError:
LOGGER.warning(f"Input file is missing: {sstfile}")
LOGGER.warning(f"Skipping month {month}")
continue
sstobj = nctools.ncLoadFile(sstfile)
sstvar = nctools.ncGetVar(sstobj, 'sst')
sstvar.set_auto_maskandscale(True)
sstlon = nctools.ncGetDims(sstobj, 'longitude')
sstlat = nctools.ncGetDims(sstobj, 'latitude')
LOGGER.debug(f"SST latitude extents: {sstlat.min()} - {sstlat.max()}")
LOGGER.debug(f"SST longitude extents: {sstlon.min()} - {sstlon.max()}")
LOGGER.info("Loading SLP data")
slpfile = pjoin(slppath, f'{year}',
f'msl_era5_oper_sfc_{filedatestr}.nc')
try:
assert (os.path.isfile(slpfile))
except AssertionError:
LOGGER.warning(f"Input file is missing: {slpfile}")
LOGGER.warning(f"Skipping month {month}")
continue
slpobj = nctools.ncLoadFile(slpfile)
slpvar = nctools.ncGetVar(slpobj, 'msl')
slpvar.set_auto_maskandscale(True)
# In the ERA5 data on NCI, surface variables are global,
# pressure variables are only over Australian region
LOGGER.info("Getting intersection of grids")
lonx, sstidx, varidxx = np.intersect1d(sstlon,
templon,
return_indices=True)
laty, sstidy, varidyy = np.intersect1d(sstlat,
templat[::-1],
return_indices=True)
nx = len(varidx)
ny = len(varidy)
LOGGER.info("Loading and converting SST and SLP data")
sst = metutils.convert(sstvar[:, sstidy, sstidx], sstvar.units, 'C')
slp = metutils.convert(slpvar[:, sstidy, sstidx], slpvar.units, 'hPa')
times = nctools.ncGetTimes(nctools.ncLoadFile(tfile))
nt = len(times)
LOGGER.debug(f"There are {nt} times in the data file")
levels = nctools.ncGetDims(nctools.ncLoadFile(tfile), 'level')
nz = len(levels)
LOGGER.debug(f"There are {nz} vertical levels in the data file")
# Create an array of the pressure variable that
# matches the shape of the temperature and mixing ratio
# variables.
LOGGER.info("Creating temporary pressure array")
pp = np.ones((nz, ny, nx))
ppT = pp.T
ppT *= levels
pmin = np.zeros(sst.shape)
vmax = np.zeros(sst.shape)
status = MPI.Status()
work_tag = 0
result_tag = 1
LOGGER.info("Calculating potential intensity")
if (comm.rank == 0) and (comm.size > 1):
w = 0
p = comm.size - 1
for d in range(1, comm.size):
if w < nt:
LOGGER.debug(f"Sending time {w} to node {d}")
comm.send(w, dest=d, tag=work_tag)
w += 1
else:
comm.send(None, dest=d, tag=work_tag)
p = w
terminated = 0
while (terminated < p):
result, tdx = comm.recv(source=MPI.ANY_SOURCE,
status=status,
tag=MPI.ANY_TAG)
pmin[tdx, :, :], vmax[tdx, :, :] = result
LOGGER.debug(f"Mean PI: {np.nanmean(vmax[tdx, :, :]):.2f} m/s")
d = status.source
if w < nt:
LOGGER.debug(f"Sending time {times[w]} to node {d}")
comm.send(w, dest=d, tag=status.tag)
w += 1
else:
# Exhausted all times, send empty packet:
comm.send(None, dest=d, tag=status.tag)
terminated += 1
elif (comm.size > 1) and (comm.rank != 0):
status = MPI.Status()
W = None
while (True):
W = comm.recv(source=0, tag=work_tag, status=status)
if W is None:
# Received an empty packet, so no work required
LOGGER.debug(
"No work to be done on this processor: {0}".format(
comm.rank))
break
LOGGER.debug(f"Processing time {times[W]} on node {comm.rank}")
t = metutils.convert(tvar[W, :, varidy, varidx], tvar.units,
'C')
r = metutils.rHToMixRat(rvar[W, :, varidy, varidx], t, pp, 'C')
r = np.where(r < 0, 0, r)
results = calculate(sst[W, :, :], slp[W, :, :], pp, t, r,
levels)
LOGGER.debug(f"Finished time {times[W]} on node {comm.rank}")
comm.send((results, W), dest=0, tag=status.tag)
elif (comm.size == 1) and (comm.rank == 0):
# We're working on a single processor:
for tdx in range(nt):
LOGGER.debug(f"Processing time {times[W]}")
t = metutils.convert(tvar[tdx, :, varidy, varidx], tvar.units,
'C')
r = metutils.rHToMixRat(rvar[tdx, :, varidy, varidx], t, pp,
'C')
r = np.where(r < 0, 0, r)
pmin[tdx, :, :], vmax[tdx, :, :] = calculate(
sst[tdx, :, :], slp[tdx, :, :], pp, t, r, levels)
if comm.rank == 0:
sleep(5)
comm.Barrier()
LOGGER.info(f"Saving data for month: {month}")
outputPath = config.get('Output', 'Path')
try:
os.makedirs(outputPath)
except:
pass
outputFile = pjoin(outputPath, f'pcmin.{filedatestr}.nc')
saveData(outputFile, pmin, vmax, lonx, laty, times)
LOGGER.info("Finished calculating potential intensity")
def grdRead(filename, delimiter=None):
"""
Read formatted data from an ascii grid format file.
Returns the longitude and latitude of the grid and the data values
:param str filename: Path to an ascii grid format or netcdf file.
:param delimiter: Delimiter for the ascii format file (optional).
:returns: longitude, latitude, data
:rtype: :class:`numpy.ndarray`
Usage:
longitude, latitude, data = grdRead(filename, [delimiter])
"""
fileext = filename.rsplit('.')[-1]
# If file extention is '.nc' then load as netcdf file
# Otherwise load with grdRead
if fileext == 'nc':
nc_obj = nctools.ncLoadFile(filename)
lon = numpy.array(nctools.ncGetDims(nc_obj, 'lon'),dtype=float)
lat = numpy.array(nctools.ncGetDims(nc_obj, 'lat'),dtype=float)
#lat = numpy.flipud(lat)
data_varname = set.difference(set(nc_obj.variables.keys()),
set(nc_obj.dimensions.keys()))
if len(data_varname) != 1:
raise IOError, 'Cannot resolve data variable in netcdf file: ' + filename
data = numpy.array(nctools.ncGetData(nc_obj, data_varname.pop()),dtype=float)
nc_obj.close()
else:
try:
fh = open(filename, 'r')
except:
#g_logger.flLog("Cannot open %s"%filename)
raise IOError, "Cannot open %s"%filename
return
metadata = {}
metadata["ncols"] = []
metadata["nrows"] = []
metadata["xllcorner"] = []
metadata["yllcorner"] = []
metadata["cellsize"] = []
metadata["NODATA_value"] = []
for i in xrange(0,6):
line = fh.readline()
contents = line.split()
label = contents[0]
metadata[label] = float(contents[1])
lon0 = metadata["xllcorner"]
lon = numpy.array(range(int(metadata["ncols"])), dtype=float)
lon = lon*metadata["cellsize"]+lon0
lat0 = metadata["yllcorner"]
lat = numpy.array(range(int(metadata["nrows"])), dtype=float)
lat = lat*metadata["cellsize"]+lat0
lat = numpy.flipud(lat)
data = numpy.zeros([metadata["nrows"], metadata["ncols"]], dtype=float)
for i in xrange(int(metadata["nrows"])):
row = numpy.zeros([metadata["ncols"]], dtype=float)
line = fh.readline()
for j, val in enumerate(line.split(delimiter)):
value = float(val)
if value == metadata["NODATA_value"]:
value = Nan
row[j] = value
data[i,:] = row
fh.close()
log.debug('filename %s mem:: lon %i lat %i data %i' % (filename, lon.nbytes, lat.nbytes, data.nbytes))
return lon, lat, data
def grdRead(filename, delimiter=None):
"""
Read formatted data from an ascii grid format file.
Returns the longitude and latitude of the grid and the data values
:param str filename: Path to an ascii grid format or netcdf file.
:param delimiter: Delimiter for the ascii format file (optional).
:returns: longitude, latitude, data
:rtype: :class:`numpy.ndarray`
Usage:
longitude, latitude, data = grdRead(filename, [delimiter])
"""
fileext = filename.rsplit('.')[-1]
# If file extention is '.nc' then load as netcdf file
# Otherwise load with grdRead
if fileext == 'nc':
nc_obj = nctools.ncLoadFile(filename)
lon = numpy.array(nctools.ncGetDims(nc_obj, 'lon'), dtype=float)
lat = numpy.array(nctools.ncGetDims(nc_obj, 'lat'), dtype=float)
data_varname = set.difference(set(nc_obj.variables.keys()),
set(nc_obj.dimensions.keys()))
if len(data_varname) != 1:
raise IOError('Cannot resolve data variable in netcdf file: ' +
filename)
data = numpy.array(nctools.ncGetData(nc_obj, data_varname.pop()),
dtype=float)
nc_obj.close()
else:
try:
fh = open(filename, 'r')
except:
raise IOError("Cannot open %s" % filename)
return
metadata = {}
metadata["ncols"] = []
metadata["nrows"] = []
metadata["xllcorner"] = []
metadata["yllcorner"] = []
metadata["cellsize"] = []
metadata["NODATA_value"] = []
for i in xrange(0, 6):
line = fh.readline()
contents = line.split()
label = contents[0]
metadata[label] = float(contents[1])
lon0 = metadata["xllcorner"]
lon = numpy.array(range(int(metadata["ncols"])), dtype=float)
lon = lon * metadata["cellsize"] + lon0
lat0 = metadata["yllcorner"]
lat = numpy.array(range(int(metadata["nrows"])), dtype=float)
lat = lat * metadata["cellsize"] + lat0
lat = numpy.flipud(lat)
data = numpy.zeros([int(metadata["nrows"]),
int(metadata["ncols"])],
dtype=float)
for i in xrange(int(metadata["nrows"])):
row = numpy.zeros([int(metadata["ncols"])], dtype=float)
line = fh.readline()
for j, val in enumerate(line.split(delimiter)):
value = float(val)
if value == metadata["NODATA_value"]:
value = numpy.nan
row[j] = value
data[i, :] = row
fh.close()
log.debug('filename %s mem:: lon %i lat %i data %i' %
(filename, lon.nbytes, lat.nbytes, data.nbytes))
return lon, lat, data
def _process(argv):
"""
A wrapper function to provide an interface between the command line
args and the actual plotField function. This function reads settings
from a configuration file and then passes those arguments to the
plotField function.
"""
if len(argv)==0:
_usage()
sys.exit(2)
logLevel = 'INFO'
verbose = False
configFile = flConfigFile()
try:
opts, args = getopt.getopt(argv, "c:hl:v", ["config=", "help",
"loglevel=", "verbose"])
except getopt.GetoptError:
_usage()
sys.exit(2)
for opt,arg in opts:
if opt in ("-h", "--help"):
_usage()
sys.exit(2)
elif opt in ("-c", "--config"):
configFile = arg
elif opt in ("-l", "--loglevel"):
logLevel = arg
elif opt in ("-v", "--verbose"):
verbose = True
flStartLog(cnfGetIniValue(configFile, 'Logging', 'LogFile', flConfigFile('.log')),
cnfGetIniValue(configFile, 'Logging', 'LogLevel', logLevel),
cnfGetIniValue(configFile, 'Logging', 'Verbose', verbose))
# Input data:
inputFile = cnfGetIniValue(configFile, 'Input', 'File')
inputFormat = cnfGetIniValue(configFile, 'Input', 'Format', os.path.splitext(inputFile)[-1])
varname = cnfGetIniValue(configFile,'Input','Variable','')
record = cnfGetIniValue(configFile,'Input','Record',0)
lvl = cnfGetIniValue(configFile,'Input','Level',0)
# Output settings - the default is to use the input filename, with
# the extension replaced by the image format:
# The smoothing is optional. Set it to the number of grid points to
# smooth over (recommend the reciprocal of the data resolution in degrees).
imgfmt = cnfGetIniValue(configFile, 'Output', 'Format','png')
outputFile = cnfGetIniValue(configFile, 'Output', 'File',
"%s.%s" % (os.path.splitext(inputFile)[0], imgfmt))
smoothing = cnfGetIniValue(configFile, 'Output', 'Smoothing', False)
cmapName = cnfGetIniValue(configFile, 'Output', 'ColourMap', 'gist_ncar')
label = cnfGetIniValue(configFile, 'Output', 'Label', '')
mask = cnfGetIniValue(configFile, 'Output', 'MaskLand', False)
maskocean = cnfGetIniValue(configFile, 'Output', 'MaskOcean', False)
fill = cnfGetIniValue(configFile, 'Output', 'FillContours', True)
title = cnfGetIniValue(configFile,'Plot','Title',None)
# Load data:
if inputFormat == '.txt':
# Attempt to load the dataset:
try:
lon,lat,data = grid.grdRead(inputFile)
except:
logger.critical("Cannot load input file: %s"%inputFile)
raise
elif inputFormat == '.nc':
try:
ncobj = nctools.ncLoadFile(inputFile)
lon = nctools.ncGetDims(ncobj,'lon')
lat = nctools.ncGetDims(ncobj,'lat')
data = nctools.ncGetData(ncobj,varname)
mv = getattr(ncobj.variables[varname],'_FillValue')
ncobj.close()
except:
logger.critical("Cannot load input file: %s"%inputFile)
raise
if len(shape(data))==3:
data = data[record,:,:]
elif len(shape(data))==4:
data = data[record,lvl,:,:]
# Create a masked array:
datamask = (data==mv)
data = ma.array(data,mask=datamask)
else:
logger.critical("Unknown data format")
raise IOError
# Set defaults for the extent of the map to match the data in the
# input file:
llLon = min(lon)
urLon = max(lon)
llLat = min(lat)
urLat = max(lat)
res = 'l'
dl = 10.
# Domain settings - can override the default settings:
domain = cnfGetIniValue(configFile, 'Domain', 'Name', None)
if domain is not None:
llLon = cnfGetIniValue(configFile, domain, 'LowerLeftLon', min(lon))
llLat = cnfGetIniValue(configFile, domain, 'LowerLeftLat', min(lat))
urLon = cnfGetIniValue(configFile, domain, 'UpperRightLon', max(lon))
urLat = cnfGetIniValue(configFile, domain, 'UpperRightLat', max(lat))
res = cnfGetIniValue(configFile, domain, 'Resolution', res)
dl = cnfGetIniValue(configFile, domain, 'GridInterval', dl)
[x,y] = meshgrid(lon, lat)
# Set the scale:
scaleMin = cnfGetIniValue(configFile, 'Output', 'ScaleMin', 0)
scaleMax = cnfGetIniValue(configFile, 'Output', 'ScaleMax', 101)
scaleInt = cnfGetIniValue(configFile, 'Output', 'ScaleInt', 10)
levels = arange(scaleMin, scaleMax, scaleInt)
plotField(x,y,data, llLon, llLat, urLon, urLat, res, dl, levels,
cmapName, smoothing, title=title, xlab='Longitude',
ylab='Latitude', clab=label, maskland=mask,
maskocean=maskocean,outputFile=outputFile,fill=fill)
logger.info("Completed %s"%sys.argv[0])
