def _process(argv): recdim = None mv = -9999. try: opts, args = getopt.getopt(argv,"f:hm:r:v:",["filename=","help","missingvalue=","record=","variable="]) except getopt.GetoptError: _usage() sys.exit(2) for opt,arg in opts: if opt in ("-h","--help"): usage() sys.exit(2) elif opt in ("-f","--filename"): filename = arg elif opt in ("-m","--missingvalue"): mv = arg elif opt in ("-v","--variable"): variable = arg elif opt in ("-r,--record"): recdim = arg ncobj = nctools.ncLoadFile(filename) lat = nctools.ncGetDims(ncobj, 'lat') lon = nctools.ncGetDims(ncobj, 'lon') delta = lon[1] - lon[0] # Fix incorrectly reported corner of lower left pixel lon = lon - delta/2 lat = lat - delta/2 if recdim: recval = nctools.ncGetDims(ncobj,recdim) #data = nctools.ncGetData(ncobj, variable, missingValue=mv) data = nctools.ncGetData(ncobj, variable) ncobj.close() if recdim: for i,v in enumerate(recval): outputfile = "%s.%s.%s"%(os.path.splitext(filename)[0],repr(recval[i]),'txt') print "Saving data to %s"%outputfile grid.grdSave(outputfile, numpy.flipud(data[i]), lon, lat, delta, delimiter=' ', nodata=mv, fmt='%6.2f') else: outputfile="%s.%s"%(os.path.splitext(filename)[0],'txt') print "Saving data to %s"%outputfile grid.grdSave(outputfile, numpy.flipud(data), lon, lat, delta, delimiter=' ', nodata=mv, fmt='%6.2f')
def ltmPressure(jdays, time, lon, lat, ncfile): """ Extract pressure value from a daily long-term mean SLP dataset at the given day of year and lon,lat position To use this function (and hence some form of daily LTM SLP data) requires knowledge of the day of year. :param jdays: Julian day (day of year) values. :param time: Time of day for each observation (fraction of a day). :param lon: Longitude of TC position. :param lat: Latitude of TC position. :param str ncfile: Path to netCDF file containing daily long-term mean sea level pressure data. :type jdays: :class:`numpy.ndarray` :type time: :class:`numpy.ndarray` :type lon: :class:`numpy.ndarray` :type lat: :class:`numpy.ndarray` :returns: :class:`numpy.ndarray` of long-term mean sea level pressure values at the day of year and positions given. """ jtime = jdays + np.modf(time)[0] coords = np.array([jtime, lat, lon]) LOG.debug("Sampling data from MSLP data in {0}".format(ncfile)) ncobj = nctools.ncLoadFile(ncfile) slpunits = getattr(ncobj.variables['slp'], 'units') data = nctools.ncGetData(ncobj, 'slp') # Get the MSLP by interpolating to the location of the TC: penv = interp3d.interp3d(data, coords, scale=[365., 180., 360.], offset=[0., -90., 0.]) penv = metutils.convert(penv, slpunits, 'hPa') del data ncobj.close() del ncobj return penv
def ltmPressure(jdays, time, lon, lat, ncfile): """ Extract pressure value from a daily long-term mean SLP dataset at the given day of year and lon,lat position To use this function (and hence some form of daily LTM SLP data) requires knowledge of the day of year. :param jdays: Julian day (day of year) values. :param time: Time of day for each observation (fraction of a day). :param lon: Longitude of TC position. :param lat: Latitude of TC position. :param str ncfile: Path to netCDF file containing daily long-term mean sea level pressure data. :type jdays: :class:`numpy.ndarray` :type time: :class:`numpy.ndarray` :type lon: :class:`numpy.ndarray` :type lat: :class:`numpy.ndarray` :returns: :class:`numpy.ndarray` of long-term mean sea level pressure values at the day of year and positions given. """ jtime = jdays + np.modf(time)[0] coords = np.array([jtime, lat, lon]) logger.debug("Sampling data from MSLP data in {0}".format(ncfile)) ncobj = nctools.ncLoadFile(ncfile) slpunits = getattr(ncobj.variables['slp'], 'units') data = nctools.ncGetData(ncobj, 'slp') # Get the MSLP by interpolating to the location of the TC: penv = interp3d.interp3d(data, coords, scale=[365., 180., 360.], offset=[0., -90., 0.]) penv = metutils.convert(penv, slpunits, 'hPa') del data ncobj.close() del ncobj return penv
def ltmPressure(jdays, time, lon, lat, ncfile): """ Extract pressure value from a daily long-term mean SLP dataset at the given day of year and lon,lat position To use this function (and hence some form of daily LTM SLP data) requires knowledge of the day of year. """ jtime = jdays + np.modf(time)[0] coords = np.array([jtime, lat, lon]) logger.debug("Sampling data from MSLP data in {0}".format(ncfile)) ncobj = nctools.ncLoadFile(ncfile) slpunits = getattr(ncobj.variables['slp'], 'units') data = nctools.ncGetData(ncobj, 'slp') # Get the MSLP by interpolating to the location of the TC: penv = interp3d.interp3d(data, coords) penv = metutils.convert(penv, slpunits, 'hPa') del data ncobj.close() del ncobj return penv
def main(): """ Handle command line arguments and call processing functions """ p = argparse.ArgumentParser() p.add_argument('-c', '--config_file', help="Configuration file") p.add_argument('-v', '--verbose', help="Verbose output", action='store_true') p.add_argument('-y', '--year', help="Year to process (1979-2020)") args = p.parse_args() configFile = args.config_file config = ConfigParser() config.read(configFile) logFile = config.get('Logging', 'LogFile') logdir = dirname(realpath(logFile)) # if log file directory does not exist, create it if not isdir(logdir): try: os.makedirs(logdir) except OSError: logFile = pjoin(os.getcwd(), 'pcmin.log') logLevel = config.get('Logging', 'LogLevel') verbose = config.getboolean('Logging', 'Verbose') datestamp = config.getboolean('Logging', 'Datestamp') if args.verbose: verbose = True if comm.size > 1 and comm.rank > 0: logFile += '-' + str(comm.rank) verbose = False if datestamp: base, ext = splitext(logFile) curdate = datetime.datetime.now() curdatestr = curdate.strftime('%Y%m%d%H%M') logfile = f"{base}.{curdatestr}.{ext.lstrip('.')}" logging.basicConfig(level=logLevel, format="%(asctime)s: %(funcName)s: %(message)s", filename=logfile, filemode='w', datefmt="%Y-%m-%d %H:%M:%S") if verbose: console = logging.StreamHandler(sys.stdout) console.setLevel(getattr(logging, logLevel)) formatter = logging.Formatter( '%(asctime)s: %(funcName)s: %(message)s', datefmt='%H:%M:%S', ) console.setFormatter(formatter) LOGGER.addHandler(console) LOGGER.info(f"Started {sys.argv[0]} (pid {os.getpid()})") LOGGER.info(f"Log file: {logfile} (detail level {logLevel})") LOGGER.info(f"Code version: f{COMMIT}") tpath = config.get('Input', 'Temp') rpath = config.get('Input', 'Humidity') sstpath = config.get('Input', 'SST') slppath = config.get('Input', 'SLP') if args.year: year = int(args.year) else: year = 2015 minLon = config.getfloat('Domain', 'MinLon') maxLon = config.getfloat('Domain', 'MaxLon') minLat = config.getfloat('Domain', 'MinLat') maxLat = config.getfloat('Domain', 'MaxLat') LOGGER.info(f"Domain: {minLon}-{maxLon}, {minLat}-{maxLat}") for month in range(1, 13): LOGGER.info(f"Processing {year}-{month}") startdate = datetime.datetime(year, month, 1) enddate = datetime.datetime(year, month, monthrange(year, month)[1]) filedatestr = f"{startdate.strftime('%Y%m%d')}-{enddate.strftime('%Y%m%d')}" tfile = pjoin(tpath, f'{year}', f't_era5_oper_pl_{filedatestr}.nc') try: assert (os.path.isfile(tfile)) except AssertionError: LOGGER.warning(f"Input file is missing: {tfile}") LOGGER.warning(f"Skipping month {month}") continue tobj = nctools.ncLoadFile(tfile) tvar = nctools.ncGetVar(tobj, 't') tvar.set_auto_maskandscale(True) rfile = pjoin(rpath, f'{year}', f'r_era5_oper_pl_{filedatestr}.nc') try: assert (os.path.isfile(rfile)) except AssertionError: LOGGER.warning(f"Input file is missing: {rfile}") LOGGER.warning(f"Skipping month {month}") continue robj = nctools.ncLoadFile(rfile) rvar = nctools.ncGetVar(robj, 'r') rvar.set_auto_maskandscale(True) # This is actually relative humidity, we need to convert to mixing ratio # Calculate mixing ratio - this function returns mixing ratio in g/kg # Dimensions need to come from the pressure files # These have been clipped to the Australian region, so contain # a subset of the global data. The SST and MSLP data # are then clipped to the same domain tlon = nctools.ncGetDims(tobj, 'longitude') tlat = nctools.ncGetDims(tobj, 'latitude') LOGGER.debug(f"Latitude extents: {tlat.min()} - {tlat.max()}") LOGGER.debug(f"Longitude extents: {tlon.min()} - {tlon.max()}") varidx = np.where((tlon >= minLon) & (tlon <= maxLon))[0] varidy = np.where((tlat >= minLat) & (tlat <= maxLat))[0] templon = tlon[varidx] templat = tlat[varidy] LOGGER.info(f"Loading SST data") sstfile = pjoin(sstpath, f'{year}', f'sst_era5_oper_sfc_{filedatestr}.nc') try: assert (os.path.isfile(sstfile)) except AssertionError: LOGGER.warning(f"Input file is missing: {sstfile}") LOGGER.warning(f"Skipping month {month}") continue sstobj = nctools.ncLoadFile(sstfile) sstvar = nctools.ncGetVar(sstobj, 'sst') sstvar.set_auto_maskandscale(True) sstlon = nctools.ncGetDims(sstobj, 'longitude') sstlat = nctools.ncGetDims(sstobj, 'latitude') LOGGER.debug(f"SST latitude extents: {sstlat.min()} - {sstlat.max()}") LOGGER.debug(f"SST longitude extents: {sstlon.min()} - {sstlon.max()}") LOGGER.info("Loading SLP data") slpfile = pjoin(slppath, f'{year}', f'msl_era5_oper_sfc_{filedatestr}.nc') try: assert (os.path.isfile(slpfile)) except AssertionError: LOGGER.warning(f"Input file is missing: {slpfile}") LOGGER.warning(f"Skipping month {month}") continue slpobj = nctools.ncLoadFile(slpfile) slpvar = nctools.ncGetVar(slpobj, 'msl') slpvar.set_auto_maskandscale(True) # In the ERA5 data on NCI, surface variables are global, # pressure variables are only over Australian region LOGGER.info("Getting intersection of grids") lonx, sstidx, varidxx = np.intersect1d(sstlon, templon, return_indices=True) laty, sstidy, varidyy = np.intersect1d(sstlat, templat[::-1], return_indices=True) nx = len(varidx) ny = len(varidy) LOGGER.info("Loading and converting SST and SLP data") sst = metutils.convert(sstvar[:, sstidy, sstidx], sstvar.units, 'C') slp = metutils.convert(slpvar[:, sstidy, sstidx], slpvar.units, 'hPa') times = nctools.ncGetTimes(nctools.ncLoadFile(tfile)) nt = len(times) LOGGER.debug(f"There are {nt} times in the data file") levels = nctools.ncGetDims(nctools.ncLoadFile(tfile), 'level') nz = len(levels) LOGGER.debug(f"There are {nz} vertical levels in the data file") # Create an array of the pressure variable that # matches the shape of the temperature and mixing ratio # variables. LOGGER.info("Creating temporary pressure array") pp = np.ones((nz, ny, nx)) ppT = pp.T ppT *= levels pmin = np.zeros(sst.shape) vmax = np.zeros(sst.shape) status = MPI.Status() work_tag = 0 result_tag = 1 LOGGER.info("Calculating potential intensity") if (comm.rank == 0) and (comm.size > 1): w = 0 p = comm.size - 1 for d in range(1, comm.size): if w < nt: LOGGER.debug(f"Sending time {w} to node {d}") comm.send(w, dest=d, tag=work_tag) w += 1 else: comm.send(None, dest=d, tag=work_tag) p = w terminated = 0 while (terminated < p): result, tdx = comm.recv(source=MPI.ANY_SOURCE, status=status, tag=MPI.ANY_TAG) pmin[tdx, :, :], vmax[tdx, :, :] = result LOGGER.debug(f"Mean PI: {np.nanmean(vmax[tdx, :, :]):.2f} m/s") d = status.source if w < nt: LOGGER.debug(f"Sending time {times[w]} to node {d}") comm.send(w, dest=d, tag=status.tag) w += 1 else: # Exhausted all times, send empty packet: comm.send(None, dest=d, tag=status.tag) terminated += 1 elif (comm.size > 1) and (comm.rank != 0): status = MPI.Status() W = None while (True): W = comm.recv(source=0, tag=work_tag, status=status) if W is None: # Received an empty packet, so no work required LOGGER.debug( "No work to be done on this processor: {0}".format( comm.rank)) break LOGGER.debug(f"Processing time {times[W]} on node {comm.rank}") t = metutils.convert(tvar[W, :, varidy, varidx], tvar.units, 'C') r = metutils.rHToMixRat(rvar[W, :, varidy, varidx], t, pp, 'C') r = np.where(r < 0, 0, r) results = calculate(sst[W, :, :], slp[W, :, :], pp, t, r, levels) LOGGER.debug(f"Finished time {times[W]} on node {comm.rank}") comm.send((results, W), dest=0, tag=status.tag) elif (comm.size == 1) and (comm.rank == 0): # We're working on a single processor: for tdx in range(nt): LOGGER.debug(f"Processing time {times[W]}") t = metutils.convert(tvar[tdx, :, varidy, varidx], tvar.units, 'C') r = metutils.rHToMixRat(rvar[tdx, :, varidy, varidx], t, pp, 'C') r = np.where(r < 0, 0, r) pmin[tdx, :, :], vmax[tdx, :, :] = calculate( sst[tdx, :, :], slp[tdx, :, :], pp, t, r, levels) if comm.rank == 0: sleep(5) comm.Barrier() LOGGER.info(f"Saving data for month: {month}") outputPath = config.get('Output', 'Path') try: os.makedirs(outputPath) except: pass outputFile = pjoin(outputPath, f'pcmin.{filedatestr}.nc') saveData(outputFile, pmin, vmax, lonx, laty, times) LOGGER.info("Finished calculating potential intensity")
def grdRead(filename, delimiter=None): """ Read formatted data from an ascii grid format file. Returns the longitude and latitude of the grid and the data values :param str filename: Path to an ascii grid format or netcdf file. :param delimiter: Delimiter for the ascii format file (optional). :returns: longitude, latitude, data :rtype: :class:`numpy.ndarray` Usage: longitude, latitude, data = grdRead(filename, [delimiter]) """ fileext = filename.rsplit('.')[-1] # If file extention is '.nc' then load as netcdf file # Otherwise load with grdRead if fileext == 'nc': nc_obj = nctools.ncLoadFile(filename) lon = numpy.array(nctools.ncGetDims(nc_obj, 'lon'),dtype=float) lat = numpy.array(nctools.ncGetDims(nc_obj, 'lat'),dtype=float) #lat = numpy.flipud(lat) data_varname = set.difference(set(nc_obj.variables.keys()), set(nc_obj.dimensions.keys())) if len(data_varname) != 1: raise IOError, 'Cannot resolve data variable in netcdf file: ' + filename data = numpy.array(nctools.ncGetData(nc_obj, data_varname.pop()),dtype=float) nc_obj.close() else: try: fh = open(filename, 'r') except: #g_logger.flLog("Cannot open %s"%filename) raise IOError, "Cannot open %s"%filename return metadata = {} metadata["ncols"] = [] metadata["nrows"] = [] metadata["xllcorner"] = [] metadata["yllcorner"] = [] metadata["cellsize"] = [] metadata["NODATA_value"] = [] for i in xrange(0,6): line = fh.readline() contents = line.split() label = contents[0] metadata[label] = float(contents[1]) lon0 = metadata["xllcorner"] lon = numpy.array(range(int(metadata["ncols"])), dtype=float) lon = lon*metadata["cellsize"]+lon0 lat0 = metadata["yllcorner"] lat = numpy.array(range(int(metadata["nrows"])), dtype=float) lat = lat*metadata["cellsize"]+lat0 lat = numpy.flipud(lat) data = numpy.zeros([metadata["nrows"], metadata["ncols"]], dtype=float) for i in xrange(int(metadata["nrows"])): row = numpy.zeros([metadata["ncols"]], dtype=float) line = fh.readline() for j, val in enumerate(line.split(delimiter)): value = float(val) if value == metadata["NODATA_value"]: value = Nan row[j] = value data[i,:] = row fh.close() log.debug('filename %s mem:: lon %i lat %i data %i' % (filename, lon.nbytes, lat.nbytes, data.nbytes)) return lon, lat, data
def grdRead(filename, delimiter=None): """ Read formatted data from an ascii grid format file. Returns the longitude and latitude of the grid and the data values :param str filename: Path to an ascii grid format or netcdf file. :param delimiter: Delimiter for the ascii format file (optional). :returns: longitude, latitude, data :rtype: :class:`numpy.ndarray` Usage: longitude, latitude, data = grdRead(filename, [delimiter]) """ fileext = filename.rsplit('.')[-1] # If file extention is '.nc' then load as netcdf file # Otherwise load with grdRead if fileext == 'nc': nc_obj = nctools.ncLoadFile(filename) lon = numpy.array(nctools.ncGetDims(nc_obj, 'lon'), dtype=float) lat = numpy.array(nctools.ncGetDims(nc_obj, 'lat'), dtype=float) data_varname = set.difference(set(nc_obj.variables.keys()), set(nc_obj.dimensions.keys())) if len(data_varname) != 1: raise IOError('Cannot resolve data variable in netcdf file: ' + filename) data = numpy.array(nctools.ncGetData(nc_obj, data_varname.pop()), dtype=float) nc_obj.close() else: try: fh = open(filename, 'r') except: raise IOError("Cannot open %s" % filename) return metadata = {} metadata["ncols"] = [] metadata["nrows"] = [] metadata["xllcorner"] = [] metadata["yllcorner"] = [] metadata["cellsize"] = [] metadata["NODATA_value"] = [] for i in xrange(0, 6): line = fh.readline() contents = line.split() label = contents[0] metadata[label] = float(contents[1]) lon0 = metadata["xllcorner"] lon = numpy.array(range(int(metadata["ncols"])), dtype=float) lon = lon * metadata["cellsize"] + lon0 lat0 = metadata["yllcorner"] lat = numpy.array(range(int(metadata["nrows"])), dtype=float) lat = lat * metadata["cellsize"] + lat0 lat = numpy.flipud(lat) data = numpy.zeros([int(metadata["nrows"]), int(metadata["ncols"])], dtype=float) for i in xrange(int(metadata["nrows"])): row = numpy.zeros([int(metadata["ncols"])], dtype=float) line = fh.readline() for j, val in enumerate(line.split(delimiter)): value = float(val) if value == metadata["NODATA_value"]: value = numpy.nan row[j] = value data[i, :] = row fh.close() log.debug('filename %s mem:: lon %i lat %i data %i' % (filename, lon.nbytes, lat.nbytes, data.nbytes)) return lon, lat, data
def _process(argv): """ A wrapper function to provide an interface between the command line args and the actual plotField function. This function reads settings from a configuration file and then passes those arguments to the plotField function. """ if len(argv)==0: _usage() sys.exit(2) logLevel = 'INFO' verbose = False configFile = flConfigFile() try: opts, args = getopt.getopt(argv, "c:hl:v", ["config=", "help", "loglevel=", "verbose"]) except getopt.GetoptError: _usage() sys.exit(2) for opt,arg in opts: if opt in ("-h", "--help"): _usage() sys.exit(2) elif opt in ("-c", "--config"): configFile = arg elif opt in ("-l", "--loglevel"): logLevel = arg elif opt in ("-v", "--verbose"): verbose = True flStartLog(cnfGetIniValue(configFile, 'Logging', 'LogFile', flConfigFile('.log')), cnfGetIniValue(configFile, 'Logging', 'LogLevel', logLevel), cnfGetIniValue(configFile, 'Logging', 'Verbose', verbose)) # Input data: inputFile = cnfGetIniValue(configFile, 'Input', 'File') inputFormat = cnfGetIniValue(configFile, 'Input', 'Format', os.path.splitext(inputFile)[-1]) varname = cnfGetIniValue(configFile,'Input','Variable','') record = cnfGetIniValue(configFile,'Input','Record',0) lvl = cnfGetIniValue(configFile,'Input','Level',0) # Output settings - the default is to use the input filename, with # the extension replaced by the image format: # The smoothing is optional. Set it to the number of grid points to # smooth over (recommend the reciprocal of the data resolution in degrees). imgfmt = cnfGetIniValue(configFile, 'Output', 'Format','png') outputFile = cnfGetIniValue(configFile, 'Output', 'File', "%s.%s" % (os.path.splitext(inputFile)[0], imgfmt)) smoothing = cnfGetIniValue(configFile, 'Output', 'Smoothing', False) cmapName = cnfGetIniValue(configFile, 'Output', 'ColourMap', 'gist_ncar') label = cnfGetIniValue(configFile, 'Output', 'Label', '') mask = cnfGetIniValue(configFile, 'Output', 'MaskLand', False) maskocean = cnfGetIniValue(configFile, 'Output', 'MaskOcean', False) fill = cnfGetIniValue(configFile, 'Output', 'FillContours', True) title = cnfGetIniValue(configFile,'Plot','Title',None) # Load data: if inputFormat == '.txt': # Attempt to load the dataset: try: lon,lat,data = grid.grdRead(inputFile) except: logger.critical("Cannot load input file: %s"%inputFile) raise elif inputFormat == '.nc': try: ncobj = nctools.ncLoadFile(inputFile) lon = nctools.ncGetDims(ncobj,'lon') lat = nctools.ncGetDims(ncobj,'lat') data = nctools.ncGetData(ncobj,varname) mv = getattr(ncobj.variables[varname],'_FillValue') ncobj.close() except: logger.critical("Cannot load input file: %s"%inputFile) raise if len(shape(data))==3: data = data[record,:,:] elif len(shape(data))==4: data = data[record,lvl,:,:] # Create a masked array: datamask = (data==mv) data = ma.array(data,mask=datamask) else: logger.critical("Unknown data format") raise IOError # Set defaults for the extent of the map to match the data in the # input file: llLon = min(lon) urLon = max(lon) llLat = min(lat) urLat = max(lat) res = 'l' dl = 10. # Domain settings - can override the default settings: domain = cnfGetIniValue(configFile, 'Domain', 'Name', None) if domain is not None: llLon = cnfGetIniValue(configFile, domain, 'LowerLeftLon', min(lon)) llLat = cnfGetIniValue(configFile, domain, 'LowerLeftLat', min(lat)) urLon = cnfGetIniValue(configFile, domain, 'UpperRightLon', max(lon)) urLat = cnfGetIniValue(configFile, domain, 'UpperRightLat', max(lat)) res = cnfGetIniValue(configFile, domain, 'Resolution', res) dl = cnfGetIniValue(configFile, domain, 'GridInterval', dl) [x,y] = meshgrid(lon, lat) # Set the scale: scaleMin = cnfGetIniValue(configFile, 'Output', 'ScaleMin', 0) scaleMax = cnfGetIniValue(configFile, 'Output', 'ScaleMax', 101) scaleInt = cnfGetIniValue(configFile, 'Output', 'ScaleInt', 10) levels = arange(scaleMin, scaleMax, scaleInt) plotField(x,y,data, llLon, llLat, urLon, urLat, res, dl, levels, cmapName, smoothing, title=title, xlab='Longitude', ylab='Latitude', clab=label, maskland=mask, maskocean=maskocean,outputFile=outputFile,fill=fill) logger.info("Completed %s"%sys.argv[0])