def _process(argv): recdim = None mv = -9999. try: opts, args = getopt.getopt(argv,"f:hm:r:v:",["filename=","help","missingvalue=","record=","variable="]) except getopt.GetoptError: _usage() sys.exit(2) for opt,arg in opts: if opt in ("-h","--help"): usage() sys.exit(2) elif opt in ("-f","--filename"): filename = arg elif opt in ("-m","--missingvalue"): mv = arg elif opt in ("-v","--variable"): variable = arg elif opt in ("-r,--record"): recdim = arg ncobj = nctools.ncLoadFile(filename) lat = nctools.ncGetDims(ncobj, 'lat') lon = nctools.ncGetDims(ncobj, 'lon') delta = lon[1] - lon[0] # Fix incorrectly reported corner of lower left pixel lon = lon - delta/2 lat = lat - delta/2 if recdim: recval = nctools.ncGetDims(ncobj,recdim) #data = nctools.ncGetData(ncobj, variable, missingValue=mv) data = nctools.ncGetData(ncobj, variable) ncobj.close() if recdim: for i,v in enumerate(recval): outputfile = "%s.%s.%s"%(os.path.splitext(filename)[0],repr(recval[i]),'txt') print "Saving data to %s"%outputfile grid.grdSave(outputfile, numpy.flipud(data[i]), lon, lat, delta, delimiter=' ', nodata=mv, fmt='%6.2f') else: outputfile="%s.%s"%(os.path.splitext(filename)[0],'txt') print "Saving data to %s"%outputfile grid.grdSave(outputfile, numpy.flipud(data), lon, lat, delta, delimiter=' ', nodata=mv, fmt='%6.2f')
def main(): """ Handle command line arguments and call processing functions """ p = argparse.ArgumentParser() p.add_argument('-c', '--config_file', help="Configuration file") p.add_argument('-v', '--verbose', help="Verbose output", action='store_true') p.add_argument('-y', '--year', help="Year to process (1979-2020)") args = p.parse_args() configFile = args.config_file config = ConfigParser() config.read(configFile) logFile = config.get('Logging', 'LogFile') logdir = dirname(realpath(logFile)) # if log file directory does not exist, create it if not isdir(logdir): try: os.makedirs(logdir) except OSError: logFile = pjoin(os.getcwd(), 'pcmin.log') logLevel = config.get('Logging', 'LogLevel') verbose = config.getboolean('Logging', 'Verbose') datestamp = config.getboolean('Logging', 'Datestamp') if args.verbose: verbose = True if comm.size > 1 and comm.rank > 0: logFile += '-' + str(comm.rank) verbose = False if datestamp: base, ext = splitext(logFile) curdate = datetime.datetime.now() curdatestr = curdate.strftime('%Y%m%d%H%M') logfile = f"{base}.{curdatestr}.{ext.lstrip('.')}" logging.basicConfig(level=logLevel, format="%(asctime)s: %(funcName)s: %(message)s", filename=logfile, filemode='w', datefmt="%Y-%m-%d %H:%M:%S") if verbose: console = logging.StreamHandler(sys.stdout) console.setLevel(getattr(logging, logLevel)) formatter = logging.Formatter( '%(asctime)s: %(funcName)s: %(message)s', datefmt='%H:%M:%S', ) console.setFormatter(formatter) LOGGER.addHandler(console) LOGGER.info(f"Started {sys.argv[0]} (pid {os.getpid()})") LOGGER.info(f"Log file: {logfile} (detail level {logLevel})") LOGGER.info(f"Code version: f{COMMIT}") tpath = config.get('Input', 'Temp') rpath = config.get('Input', 'Humidity') sstpath = config.get('Input', 'SST') slppath = config.get('Input', 'SLP') if args.year: year = int(args.year) else: year = 2015 minLon = config.getfloat('Domain', 'MinLon') maxLon = config.getfloat('Domain', 'MaxLon') minLat = config.getfloat('Domain', 'MinLat') maxLat = config.getfloat('Domain', 'MaxLat') LOGGER.info(f"Domain: {minLon}-{maxLon}, {minLat}-{maxLat}") for month in range(1, 13): LOGGER.info(f"Processing {year}-{month}") startdate = datetime.datetime(year, month, 1) enddate = datetime.datetime(year, month, monthrange(year, month)[1]) filedatestr = f"{startdate.strftime('%Y%m%d')}-{enddate.strftime('%Y%m%d')}" tfile = pjoin(tpath, f'{year}', f't_era5_oper_pl_{filedatestr}.nc') try: assert (os.path.isfile(tfile)) except AssertionError: LOGGER.warning(f"Input file is missing: {tfile}") LOGGER.warning(f"Skipping month {month}") continue tobj = nctools.ncLoadFile(tfile) tvar = nctools.ncGetVar(tobj, 't') tvar.set_auto_maskandscale(True) rfile = pjoin(rpath, f'{year}', f'r_era5_oper_pl_{filedatestr}.nc') try: assert (os.path.isfile(rfile)) except AssertionError: LOGGER.warning(f"Input file is missing: {rfile}") LOGGER.warning(f"Skipping month {month}") continue robj = nctools.ncLoadFile(rfile) rvar = nctools.ncGetVar(robj, 'r') rvar.set_auto_maskandscale(True) # This is actually relative humidity, we need to convert to mixing ratio # Calculate mixing ratio - this function returns mixing ratio in g/kg # Dimensions need to come from the pressure files # These have been clipped to the Australian region, so contain # a subset of the global data. The SST and MSLP data # are then clipped to the same domain tlon = nctools.ncGetDims(tobj, 'longitude') tlat = nctools.ncGetDims(tobj, 'latitude') LOGGER.debug(f"Latitude extents: {tlat.min()} - {tlat.max()}") LOGGER.debug(f"Longitude extents: {tlon.min()} - {tlon.max()}") varidx = np.where((tlon >= minLon) & (tlon <= maxLon))[0] varidy = np.where((tlat >= minLat) & (tlat <= maxLat))[0] templon = tlon[varidx] templat = tlat[varidy] LOGGER.info(f"Loading SST data") sstfile = pjoin(sstpath, f'{year}', f'sst_era5_oper_sfc_{filedatestr}.nc') try: assert (os.path.isfile(sstfile)) except AssertionError: LOGGER.warning(f"Input file is missing: {sstfile}") LOGGER.warning(f"Skipping month {month}") continue sstobj = nctools.ncLoadFile(sstfile) sstvar = nctools.ncGetVar(sstobj, 'sst') sstvar.set_auto_maskandscale(True) sstlon = nctools.ncGetDims(sstobj, 'longitude') sstlat = nctools.ncGetDims(sstobj, 'latitude') LOGGER.debug(f"SST latitude extents: {sstlat.min()} - {sstlat.max()}") LOGGER.debug(f"SST longitude extents: {sstlon.min()} - {sstlon.max()}") LOGGER.info("Loading SLP data") slpfile = pjoin(slppath, f'{year}', f'msl_era5_oper_sfc_{filedatestr}.nc') try: assert (os.path.isfile(slpfile)) except AssertionError: LOGGER.warning(f"Input file is missing: {slpfile}") LOGGER.warning(f"Skipping month {month}") continue slpobj = nctools.ncLoadFile(slpfile) slpvar = nctools.ncGetVar(slpobj, 'msl') slpvar.set_auto_maskandscale(True) # In the ERA5 data on NCI, surface variables are global, # pressure variables are only over Australian region LOGGER.info("Getting intersection of grids") lonx, sstidx, varidxx = np.intersect1d(sstlon, templon, return_indices=True) laty, sstidy, varidyy = np.intersect1d(sstlat, templat[::-1], return_indices=True) nx = len(varidx) ny = len(varidy) LOGGER.info("Loading and converting SST and SLP data") sst = metutils.convert(sstvar[:, sstidy, sstidx], sstvar.units, 'C') slp = metutils.convert(slpvar[:, sstidy, sstidx], slpvar.units, 'hPa') times = nctools.ncGetTimes(nctools.ncLoadFile(tfile)) nt = len(times) LOGGER.debug(f"There are {nt} times in the data file") levels = nctools.ncGetDims(nctools.ncLoadFile(tfile), 'level') nz = len(levels) LOGGER.debug(f"There are {nz} vertical levels in the data file") # Create an array of the pressure variable that # matches the shape of the temperature and mixing ratio # variables. LOGGER.info("Creating temporary pressure array") pp = np.ones((nz, ny, nx)) ppT = pp.T ppT *= levels pmin = np.zeros(sst.shape) vmax = np.zeros(sst.shape) status = MPI.Status() work_tag = 0 result_tag = 1 LOGGER.info("Calculating potential intensity") if (comm.rank == 0) and (comm.size > 1): w = 0 p = comm.size - 1 for d in range(1, comm.size): if w < nt: LOGGER.debug(f"Sending time {w} to node {d}") comm.send(w, dest=d, tag=work_tag) w += 1 else: comm.send(None, dest=d, tag=work_tag) p = w terminated = 0 while (terminated < p): result, tdx = comm.recv(source=MPI.ANY_SOURCE, status=status, tag=MPI.ANY_TAG) pmin[tdx, :, :], vmax[tdx, :, :] = result LOGGER.debug(f"Mean PI: {np.nanmean(vmax[tdx, :, :]):.2f} m/s") d = status.source if w < nt: LOGGER.debug(f"Sending time {times[w]} to node {d}") comm.send(w, dest=d, tag=status.tag) w += 1 else: # Exhausted all times, send empty packet: comm.send(None, dest=d, tag=status.tag) terminated += 1 elif (comm.size > 1) and (comm.rank != 0): status = MPI.Status() W = None while (True): W = comm.recv(source=0, tag=work_tag, status=status) if W is None: # Received an empty packet, so no work required LOGGER.debug( "No work to be done on this processor: {0}".format( comm.rank)) break LOGGER.debug(f"Processing time {times[W]} on node {comm.rank}") t = metutils.convert(tvar[W, :, varidy, varidx], tvar.units, 'C') r = metutils.rHToMixRat(rvar[W, :, varidy, varidx], t, pp, 'C') r = np.where(r < 0, 0, r) results = calculate(sst[W, :, :], slp[W, :, :], pp, t, r, levels) LOGGER.debug(f"Finished time {times[W]} on node {comm.rank}") comm.send((results, W), dest=0, tag=status.tag) elif (comm.size == 1) and (comm.rank == 0): # We're working on a single processor: for tdx in range(nt): LOGGER.debug(f"Processing time {times[W]}") t = metutils.convert(tvar[tdx, :, varidy, varidx], tvar.units, 'C') r = metutils.rHToMixRat(rvar[tdx, :, varidy, varidx], t, pp, 'C') r = np.where(r < 0, 0, r) pmin[tdx, :, :], vmax[tdx, :, :] = calculate( sst[tdx, :, :], slp[tdx, :, :], pp, t, r, levels) if comm.rank == 0: sleep(5) comm.Barrier() LOGGER.info(f"Saving data for month: {month}") outputPath = config.get('Output', 'Path') try: os.makedirs(outputPath) except: pass outputFile = pjoin(outputPath, f'pcmin.{filedatestr}.nc') saveData(outputFile, pmin, vmax, lonx, laty, times) LOGGER.info("Finished calculating potential intensity")
def grdRead(filename, delimiter=None): """ Read formatted data from an ascii grid format file. Returns the longitude and latitude of the grid and the data values :param str filename: Path to an ascii grid format or netcdf file. :param delimiter: Delimiter for the ascii format file (optional). :returns: longitude, latitude, data :rtype: :class:`numpy.ndarray` Usage: longitude, latitude, data = grdRead(filename, [delimiter]) """ fileext = filename.rsplit('.')[-1] # If file extention is '.nc' then load as netcdf file # Otherwise load with grdRead if fileext == 'nc': nc_obj = nctools.ncLoadFile(filename) lon = numpy.array(nctools.ncGetDims(nc_obj, 'lon'),dtype=float) lat = numpy.array(nctools.ncGetDims(nc_obj, 'lat'),dtype=float) #lat = numpy.flipud(lat) data_varname = set.difference(set(nc_obj.variables.keys()), set(nc_obj.dimensions.keys())) if len(data_varname) != 1: raise IOError, 'Cannot resolve data variable in netcdf file: ' + filename data = numpy.array(nctools.ncGetData(nc_obj, data_varname.pop()),dtype=float) nc_obj.close() else: try: fh = open(filename, 'r') except: #g_logger.flLog("Cannot open %s"%filename) raise IOError, "Cannot open %s"%filename return metadata = {} metadata["ncols"] = [] metadata["nrows"] = [] metadata["xllcorner"] = [] metadata["yllcorner"] = [] metadata["cellsize"] = [] metadata["NODATA_value"] = [] for i in xrange(0,6): line = fh.readline() contents = line.split() label = contents[0] metadata[label] = float(contents[1]) lon0 = metadata["xllcorner"] lon = numpy.array(range(int(metadata["ncols"])), dtype=float) lon = lon*metadata["cellsize"]+lon0 lat0 = metadata["yllcorner"] lat = numpy.array(range(int(metadata["nrows"])), dtype=float) lat = lat*metadata["cellsize"]+lat0 lat = numpy.flipud(lat) data = numpy.zeros([metadata["nrows"], metadata["ncols"]], dtype=float) for i in xrange(int(metadata["nrows"])): row = numpy.zeros([metadata["ncols"]], dtype=float) line = fh.readline() for j, val in enumerate(line.split(delimiter)): value = float(val) if value == metadata["NODATA_value"]: value = Nan row[j] = value data[i,:] = row fh.close() log.debug('filename %s mem:: lon %i lat %i data %i' % (filename, lon.nbytes, lat.nbytes, data.nbytes)) return lon, lat, data
def grdRead(filename, delimiter=None): """ Read formatted data from an ascii grid format file. Returns the longitude and latitude of the grid and the data values :param str filename: Path to an ascii grid format or netcdf file. :param delimiter: Delimiter for the ascii format file (optional). :returns: longitude, latitude, data :rtype: :class:`numpy.ndarray` Usage: longitude, latitude, data = grdRead(filename, [delimiter]) """ fileext = filename.rsplit('.')[-1] # If file extention is '.nc' then load as netcdf file # Otherwise load with grdRead if fileext == 'nc': nc_obj = nctools.ncLoadFile(filename) lon = numpy.array(nctools.ncGetDims(nc_obj, 'lon'), dtype=float) lat = numpy.array(nctools.ncGetDims(nc_obj, 'lat'), dtype=float) data_varname = set.difference(set(nc_obj.variables.keys()), set(nc_obj.dimensions.keys())) if len(data_varname) != 1: raise IOError('Cannot resolve data variable in netcdf file: ' + filename) data = numpy.array(nctools.ncGetData(nc_obj, data_varname.pop()), dtype=float) nc_obj.close() else: try: fh = open(filename, 'r') except: raise IOError("Cannot open %s" % filename) return metadata = {} metadata["ncols"] = [] metadata["nrows"] = [] metadata["xllcorner"] = [] metadata["yllcorner"] = [] metadata["cellsize"] = [] metadata["NODATA_value"] = [] for i in xrange(0, 6): line = fh.readline() contents = line.split() label = contents[0] metadata[label] = float(contents[1]) lon0 = metadata["xllcorner"] lon = numpy.array(range(int(metadata["ncols"])), dtype=float) lon = lon * metadata["cellsize"] + lon0 lat0 = metadata["yllcorner"] lat = numpy.array(range(int(metadata["nrows"])), dtype=float) lat = lat * metadata["cellsize"] + lat0 lat = numpy.flipud(lat) data = numpy.zeros([int(metadata["nrows"]), int(metadata["ncols"])], dtype=float) for i in xrange(int(metadata["nrows"])): row = numpy.zeros([int(metadata["ncols"])], dtype=float) line = fh.readline() for j, val in enumerate(line.split(delimiter)): value = float(val) if value == metadata["NODATA_value"]: value = numpy.nan row[j] = value data[i, :] = row fh.close() log.debug('filename %s mem:: lon %i lat %i data %i' % (filename, lon.nbytes, lat.nbytes, data.nbytes)) return lon, lat, data
def _process(argv): """ A wrapper function to provide an interface between the command line args and the actual plotField function. This function reads settings from a configuration file and then passes those arguments to the plotField function. """ if len(argv)==0: _usage() sys.exit(2) logLevel = 'INFO' verbose = False configFile = flConfigFile() try: opts, args = getopt.getopt(argv, "c:hl:v", ["config=", "help", "loglevel=", "verbose"]) except getopt.GetoptError: _usage() sys.exit(2) for opt,arg in opts: if opt in ("-h", "--help"): _usage() sys.exit(2) elif opt in ("-c", "--config"): configFile = arg elif opt in ("-l", "--loglevel"): logLevel = arg elif opt in ("-v", "--verbose"): verbose = True flStartLog(cnfGetIniValue(configFile, 'Logging', 'LogFile', flConfigFile('.log')), cnfGetIniValue(configFile, 'Logging', 'LogLevel', logLevel), cnfGetIniValue(configFile, 'Logging', 'Verbose', verbose)) # Input data: inputFile = cnfGetIniValue(configFile, 'Input', 'File') inputFormat = cnfGetIniValue(configFile, 'Input', 'Format', os.path.splitext(inputFile)[-1]) varname = cnfGetIniValue(configFile,'Input','Variable','') record = cnfGetIniValue(configFile,'Input','Record',0) lvl = cnfGetIniValue(configFile,'Input','Level',0) # Output settings - the default is to use the input filename, with # the extension replaced by the image format: # The smoothing is optional. Set it to the number of grid points to # smooth over (recommend the reciprocal of the data resolution in degrees). imgfmt = cnfGetIniValue(configFile, 'Output', 'Format','png') outputFile = cnfGetIniValue(configFile, 'Output', 'File', "%s.%s" % (os.path.splitext(inputFile)[0], imgfmt)) smoothing = cnfGetIniValue(configFile, 'Output', 'Smoothing', False) cmapName = cnfGetIniValue(configFile, 'Output', 'ColourMap', 'gist_ncar') label = cnfGetIniValue(configFile, 'Output', 'Label', '') mask = cnfGetIniValue(configFile, 'Output', 'MaskLand', False) maskocean = cnfGetIniValue(configFile, 'Output', 'MaskOcean', False) fill = cnfGetIniValue(configFile, 'Output', 'FillContours', True) title = cnfGetIniValue(configFile,'Plot','Title',None) # Load data: if inputFormat == '.txt': # Attempt to load the dataset: try: lon,lat,data = grid.grdRead(inputFile) except: logger.critical("Cannot load input file: %s"%inputFile) raise elif inputFormat == '.nc': try: ncobj = nctools.ncLoadFile(inputFile) lon = nctools.ncGetDims(ncobj,'lon') lat = nctools.ncGetDims(ncobj,'lat') data = nctools.ncGetData(ncobj,varname) mv = getattr(ncobj.variables[varname],'_FillValue') ncobj.close() except: logger.critical("Cannot load input file: %s"%inputFile) raise if len(shape(data))==3: data = data[record,:,:] elif len(shape(data))==4: data = data[record,lvl,:,:] # Create a masked array: datamask = (data==mv) data = ma.array(data,mask=datamask) else: logger.critical("Unknown data format") raise IOError # Set defaults for the extent of the map to match the data in the # input file: llLon = min(lon) urLon = max(lon) llLat = min(lat) urLat = max(lat) res = 'l' dl = 10. # Domain settings - can override the default settings: domain = cnfGetIniValue(configFile, 'Domain', 'Name', None) if domain is not None: llLon = cnfGetIniValue(configFile, domain, 'LowerLeftLon', min(lon)) llLat = cnfGetIniValue(configFile, domain, 'LowerLeftLat', min(lat)) urLon = cnfGetIniValue(configFile, domain, 'UpperRightLon', max(lon)) urLat = cnfGetIniValue(configFile, domain, 'UpperRightLat', max(lat)) res = cnfGetIniValue(configFile, domain, 'Resolution', res) dl = cnfGetIniValue(configFile, domain, 'GridInterval', dl) [x,y] = meshgrid(lon, lat) # Set the scale: scaleMin = cnfGetIniValue(configFile, 'Output', 'ScaleMin', 0) scaleMax = cnfGetIniValue(configFile, 'Output', 'ScaleMax', 101) scaleInt = cnfGetIniValue(configFile, 'Output', 'ScaleInt', 10) levels = arange(scaleMin, scaleMax, scaleInt) plotField(x,y,data, llLon, llLat, urLon, urLat, res, dl, levels, cmapName, smoothing, title=title, xlab='Longitude', ylab='Latitude', clab=label, maskland=mask, maskocean=maskocean,outputFile=outputFile,fill=fill) logger.info("Completed %s"%sys.argv[0])