def historic(self):
"""Load historic data and calculate histogram"""
config = ConfigParser()
config.read(self.configFile)
inputFile = config.get('DataProcess', 'InputFile')
if len(os.path.dirname(inputFile)) == 0:
inputFile = pjoin(self.inputPath, inputFile)
source = config.get('DataProcess', 'Source')
timestep = config.getfloat('TrackGenerator', 'Timestep')
interpHistFile = pjoin(self.inputPath, "interp_tracks.csv")
try:
tracks = interpolateTracks.parseTracks(self.configFile,
inputFile,
source,
timestep,
interpHistFile, 'linear')
except (TypeError, IOError, ValueError):
log.critical("Cannot load historical track file: {0}".format(inputFile))
raise
else:
startYr = 9999
endYr = 0
for t in tracks:
startYr = min(startYr, min(t.Year))
endYr = max(endYr, max(t.Year))
numYears = endYr - startYr
self.hist = self.calculate(tracks) / numYears
def historic(self):
"""Calculate historical rates of longitude crossing"""
log.info("Processing historical tracks for longitude crossings")
config = ConfigParser()
config.read(self.configFile)
inputFile = config.get("DataProcess", "InputFile")
source = config.get("DataProcess", "Source")
timestep = config.getfloat("TrackGenerator", "Timestep")
if len(os.path.dirname(inputFile)) == 0:
inputFile = pjoin(self.inputPath, inputFile)
try:
tracks = interpolateTracks.parseTracks(
self.configFile, inputFile, source, timestep, interpolation_type="linear"
)
except (TypeError, IOError, ValueError):
log.critical("Cannot load historical track file: {0}".format(inputFile))
raise
else:
self.lonCrossingHist, self.lonCrossingEWHist, self.lonCrossingWEHist = self.findCrossings(tracks)
return
def historic(self):
"""Calculate historical rates of longitude crossing"""
LOG.info("Processing historical tracks for longitude crossings")
config = ConfigParser()
config.read(self.configFile)
inputFile = config.get('DataProcess', 'InputFile')
source = config.get('DataProcess', 'Source')
timestep = config.getfloat('TrackGenerator', 'Timestep')
if len(os.path.dirname(inputFile)) == 0:
inputFile = pjoin(self.inputPath, inputFile)
try:
tracks = interpolateTracks.parseTracks(self.configFile,
inputFile,
source,
timestep,
interpolation_type='linear')
except (TypeError, IOError, ValueError):
LOG.critical("Cannot load historical track file: {0}".\
format(inputFile))
raise
else:
self.lonCrossingHist, self.lonCrossingEWHist, \
self.lonCrossingWEHist = self.findCrossings(tracks)
return
def __init__(self, configFile, autoCalc_gridLimit=None,
progressbar=None):
"""
Initialize the data and variables required for the interface
"""
self.configFile = configFile
config = ConfigParser()
config.read(configFile)
self.progressbar = progressbar
log.info("Initialising StatInterface")
self.kdeType = config.get('StatInterface', 'kdeType')
self.kde2DType = config.get('StatInterface','kde2DType')
minSamplesCell = config.getint('StatInterface', 'minSamplesCell')
self.kdeStep = config.getfloat('StatInterface', 'kdeStep')
self.outputPath = config.get('Output', 'Path')
self.processPath = pjoin(self.outputPath, 'process')
missingValue = cnfGetIniValue(self.configFile, 'StatInterface',
'MissingValue', sys.maxint)
gridLimitStr = cnfGetIniValue(self.configFile, 'StatInterface',
'gridLimit', '')
if gridLimitStr is not '':
try:
self.gridLimit = eval(gridLimitStr)
except SyntaxError:
log.exception('Error! gridLimit is not a dictionary')
else:
self.gridLimit = autoCalc_gridLimit
log.info('No gridLimit specified - using automatic' +
' selection: ' + str(self.gridLimit))
try:
gridSpace = config.geteval('Region', 'gridSpace')
gridInc = config.geteval('Region', 'gridInc')
except SyntaxError:
log.exception('Error! gridSpace or gridInc not dictionaries')
raise
self.generateDist = GenerateDistributions(self.configFile,
self.gridLimit,
gridSpace, gridInc,
self.kdeType,
minSamplesCell,
missingValue)
self.gridSpace = gridSpace
self.gridInc = gridInc
def __init__(self, configFile, autoCalc_gridLimit=None, progressbar=None):
"""
Initialize the data and variables required for the interface
"""
self.configFile = configFile
config = ConfigParser()
config.read(configFile)
self.progressbar = progressbar
log.info("Initialising StatInterface")
self.kdeType = config.get('StatInterface', 'kdeType')
minSamplesCell = config.getint('StatInterface', 'minSamplesCell')
self.kdeStep = config.getfloat('StatInterface', 'kdeStep')
self.outputPath = config.get('Output', 'Path')
self.processPath = pjoin(self.outputPath, 'process')
missingValue = cnfGetIniValue(self.configFile, 'StatInterface',
'MissingValue', sys.maxsize)
gridLimitStr = cnfGetIniValue(self.configFile, 'StatInterface',
'gridLimit', '')
if gridLimitStr is not '':
try:
self.gridLimit = eval(gridLimitStr)
except SyntaxError:
log.exception('Error! gridLimit is not a dictionary')
else:
self.gridLimit = autoCalc_gridLimit
log.info('No gridLimit specified - using automatic' +
' selection: ' + str(self.gridLimit))
try:
gridSpace = config.geteval('Region', 'gridSpace')
gridInc = config.geteval('Region', 'gridInc')
except SyntaxError:
log.exception('Error! gridSpace or gridInc not dictionaries')
raise
self.generateDist = GenerateDistributions(self.configFile,
self.gridLimit, gridSpace,
gridInc, self.kdeType,
minSamplesCell, missingValue)
self.gridSpace = gridSpace
self.gridInc = gridInc
def historic(self):
"""Calculate historical rates of landfall"""
log.info("Processing landfall rates of historical tracks")
config = ConfigParser()
config.read(self.configFile)
inputFile = config.get('DataProcess', 'InputFile')
source = config.get('DataProcess', 'Source')
timestep = config.getfloat('TrackGenerator', 'Timestep')
if len(os.path.dirname(inputFile)) == 0:
inputFile = pjoin(self.inputPath, inputFile)
try:
tracks = loadTrackFile(self.configFile, inputFile, source)
except (TypeError, IOError, ValueError):
log.critical("Cannot load historical track file: {0}".format(inputFile))
raise
else:
self.historicLandfall, self.historicOffshore = self.processTracks(tracks)
return
def run(configFile, callback=None):
"""
Run the wind field calculations.
:param str configFile: path to a configuration file.
:param func callback: optional callback function to track progress.
"""
log.info('Loading wind field calculation settings')
# Get configuration
config = ConfigParser()
config.read(configFile)
profileType = config.get('WindfieldInterface', 'profileType')
windFieldType = config.get('WindfieldInterface', 'windFieldType')
beta = config.getfloat('WindfieldInterface', 'beta')
beta1 = config.getfloat('WindfieldInterface', 'beta1')
beta2 = config.getfloat('WindfieldInterface', 'beta2')
thetaMax = config.getfloat('WindfieldInterface', 'thetaMax')
margin = config.getfloat('WindfieldInterface', 'Margin')
resolution = config.getfloat('WindfieldInterface', 'Resolution')
domain = config.get('WindfieldInterface', 'Domain')
outputPath = config.get('Output', 'Path')
windfieldPath = pjoin(outputPath, 'windfield')
trackPath = pjoin(outputPath, 'tracks')
gridLimit = None
if config.has_option('Region', 'gridLimit'):
gridLimit = config.geteval('Region', 'gridLimit')
if config.has_option('WindfieldInterface', 'gridLimit'):
gridLimit = config.geteval('WindfieldInterface', 'gridLimit')
if config.getboolean('Timeseries', 'Extract', fallback=False):
from Utilities.timeseries import Timeseries
ts = Timeseries(configFile)
timestepCallback = ts.extract
else:
timestepCallback = None
multipliers = None
if config.has_option('Input', 'Multipliers'):
multipliers = config.get('Input', 'Multipliers')
thetaMax = math.radians(thetaMax)
# Attempt to start the track generator in parallel
global MPI
MPI = attemptParallel()
comm = MPI.COMM_WORLD
log.info('Running windfield generator')
wfg = WindfieldGenerator(config=config,
margin=margin,
resolution=resolution,
profileType=profileType,
windFieldType=windFieldType,
beta=beta,
beta1=beta1,
beta2=beta2,
thetaMax=thetaMax,
gridLimit=gridLimit,
domain=domain,
multipliers=multipliers,
windfieldPath=windfieldPath)
log.info(f'Dumping gusts to {windfieldPath}')
# Get the trackfile names and count
files = os.listdir(trackPath)
trackfiles = [pjoin(trackPath, f) for f in files if f.startswith('tracks')]
nfiles = len(trackfiles)
log.info('Processing {0} track files in {1}'.format(nfiles, trackPath))
# Do the work
comm.barrier()
wfg.dumpGustsFromTrackfiles(trackfiles, windfieldPath, timestepCallback)
try:
ts.shutdown()
except NameError:
pass
comm.barrier()
log.info('Completed windfield generator')
def run(configFile, callback=None):
"""
Run the wind field calculations.
:param str configFile: path to a configuration file.
:param func callback: optional callback function to track progress.
"""
log.info('Loading wind field calculation settings')
# Get configuration
config = ConfigParser()
config.read(configFile)
profileType = config.get('WindfieldInterface', 'profileType')
windFieldType = config.get('WindfieldInterface', 'windFieldType')
beta = config.getfloat('WindfieldInterface', 'beta')
beta1 = config.getfloat('WindfieldInterface', 'beta1')
beta2 = config.getfloat('WindfieldInterface', 'beta2')
thetaMax = config.getfloat('WindfieldInterface', 'thetaMax')
margin = config.getfloat('WindfieldInterface', 'Margin')
resolution = config.getfloat('WindfieldInterface', 'Resolution')
domain = config.get('WindfieldInterface', 'Domain')
outputPath = config.get('Output', 'Path')
windfieldPath = pjoin(outputPath, 'windfield')
trackPath = pjoin(outputPath, 'tracks')
gridLimit = None
if config.has_option('Region', 'gridLimit'):
gridLimit = config.geteval('Region', 'gridLimit')
if config.has_option('WindfieldInterface', 'gridLimit'):
gridLimit = config.geteval('WindfieldInterface', 'gridLimit')
if config.has_section('Timeseries'):
if config.has_option('Timeseries', 'Extract'):
if config.getboolean('Timeseries', 'Extract'):
from Utilities.timeseries import Timeseries
log.debug("Timeseries data will be extracted")
ts = Timeseries(configFile)
timestepCallback = ts.extract
else:
def timestepCallback(*args):
"""Dummy timestepCallback function"""
pass
else:
def timestepCallback(*args):
"""Dummy timestepCallback function"""
pass
thetaMax = math.radians(thetaMax)
# Attempt to start the track generator in parallel
global pp
pp = attemptParallel()
log.info('Running windfield generator')
wfg = WindfieldGenerator(config=config,
margin=margin,
resolution=resolution,
profileType=profileType,
windFieldType=windFieldType,
beta=beta,
beta1=beta1,
beta2=beta2,
thetaMax=thetaMax,
gridLimit=gridLimit,
domain=domain)
msg = 'Dumping gusts to %s' % windfieldPath
log.info(msg)
# Get the trackfile names and count
files = os.listdir(trackPath)
trackfiles = [pjoin(trackPath, f) for f in files if f.startswith('tracks')]
nfiles = len(trackfiles)
def progressCallback(i):
"""Define the callback function"""
callback(i, nfiles)
msg = 'Processing %d track files in %s' % (nfiles, trackPath)
log.info(msg)
# Do the work
pp.barrier()
wfg.dumpGustsFromTrackfiles(trackfiles, windfieldPath, timestepCallback)
try:
ts.shutdown()
except NameError:
pass
pp.barrier()
log.info('Completed windfield generator')
def loadTrackFile(configFile, trackFile, source, missingValue=0,
calculateWindSpeed=True):
"""
Load TC track data from the given input file, from a specified source.
The configFile is a configuration file that contains a section called
'source' that describes the data.
This returns a collection of :class:`Track` objects that contains
the details of the TC tracks in the input file.
:param str configFile: Configuration file with a section ``source``.
:param str trackFile: Path to a csv-formatted file containing TC data.
:pararm str source: Name of the source format of the TC data. There
*must* be a section in ``configFile`` matching
this string, containing the details of the format
of the data.
:param missingValue: Replace all null values in the input data with
this value (default=0).
:param boolean calculateWindSpeed: Calculate maximum wind speed using
a pressure-wind relation described
in :func:`maxWindSpeed`
:returns: A collection of :class:`Track` objects.
If any of the variables are not present in the input
dataset, they are (where possible) calculated
(date/time/windspeed), sampled from default datasets
(e.g. environmental pressure) or set to the missing value.
Example::
>>> tracks = loadTrackFile('tcrm.ini', 'IBTRaCS.csv', 'IBTrACS' )
"""
LOG.info("Loading %s" % trackFile)
inputData = colReadCSV(configFile, trackFile, source) #,
#nullValue=missingValue)
config = ConfigParser()
config.read(configFile)
inputSpeedUnits = config.get(source, 'SpeedUnits')
inputPressureUnits = config.get(source, 'PressureUnits')
inputLengthUnits = config.get(source, 'LengthUnits')
inputDateFormat = config.get(source, 'DateFormat')
if config.getboolean('DataProcess', 'FilterSeasons'):
startSeason = config.getint('DataProcess', 'StartSeason')
idx = np.where(inputData['season'] >= startSeason)[0]
inputData = inputData[idx]
# Determine the initial TC positions...
indicator = getInitialPositions(inputData)
# Sort date/time information
if 'age' in inputData.dtype.names:
year, month, day, hour, minute, datetimes = parseAge(inputData,
indicator)
timeElapsed = inputData['age']
else:
year, month, day, hour, minute, datetimes = parseDates(inputData,
indicator,
inputDateFormat)
timeElapsed = getTimeElapsed(indicator, year, month, day, hour, minute)
# Time between observations:
dt = getTimeDelta(year, month, day, hour, minute)
# Calculate julian days
jdays = julianDays(year, month, day, hour, minute)
lat = np.array(inputData['lat'], 'd')
lon = np.mod(np.array(inputData['lon'], 'd'), 360)
delta_lon = np.diff(lon)
delta_lat = np.diff(lat)
# Split into separate tracks if large jump occurs (delta_lon > 10 degrees
# or delta_lat > 5 degrees)
# This avoids two tracks being accidentally combined when seasons and track
# numbers match but basins are different as occurs in the IBTrACS dataset.
# This problem can also be prevented if the 'tcserialno' column is
# specified.
indicator[np.where(delta_lon > 10)[0] + 1] = 1
indicator[np.where(delta_lat > 5)[0] + 1] = 1
pressure = filterPressure(np.array(inputData['pressure'], 'd'),
inputPressureUnits, missingValue)
try:
windspeed = np.array(inputData['vmax'], 'd')
novalue_index = np.where(windspeed == sys.maxint)
windspeed = metutils.convert(windspeed, inputSpeedUnits, "mps")
windspeed[novalue_index] = missingValue
except (ValueError, KeyError):
LOG.debug("No max wind speed data - all values will be zero")
windspeed = np.zeros(indicator.size, 'f')
assert lat.size == indicator.size
assert lon.size == indicator.size
assert pressure.size == indicator.size
try:
rmax = np.array(inputData['rmax'])
novalue_index = np.where(rmax == missingValue)
rmax = metutils.convert(rmax, inputLengthUnits, "km")
rmax[novalue_index] = missingValue
except (ValueError, KeyError):
LOG.debug("No radius to max wind data - all values will be zero")
rmax = np.zeros(indicator.size, 'f')
if 'penv' in inputData.dtype.names:
penv = np.array(inputData['penv'], 'd')
else:
LOG.debug("No ambient MSLP data in this input file")
LOG.debug("Sampling data from MSLP data defined in "
"configuration file")
# Warning: using sampled data will likely lead to some odd behaviour
# near the boundary of the MSLP grid boundaries - higher resolution
# MSLP data will decrease this unusual behaviour.
try:
ncfile = cnfGetIniValue(configFile, 'Input', 'MSLPFile')
except:
LOG.exception("No input MSLP file specified in configuration")
raise
time = getTime(year, month, day, hour, minute)
penv = ltmPressure(jdays, time, lon, lat, ncfile)
if 'poci' in inputData.dtype.names:
poci = np.array(inputData['poci'], 'd')
else:
LOG.debug("Determining poci")
eps = np.random.normal(0, scale=2.5717)
poci = getPoci(penv, pressure, lat, jdays, eps)
if 'beta' in inputData.dtype.names:
beta = np.array(inputData['beta'], 'd')
LOG.debug("beta found in the input file")
else:
beta_vals = config.getfloat('WindfieldInterface','beta')
beta = np.ones(len(indicator)) * beta_vals
LOG.debug("no beta found in the input file")
speed, bearing = getSpeedBearing(indicator, lon, lat, dt,
missingValue=missingValue)
if calculateWindSpeed:
windspeed = maxWindSpeed(indicator, dt, lon, lat, pressure, poci, beta)
TCID = np.cumsum(indicator)
data = np.empty(len(indicator),
dtype={
'names': trackFields,
'formats': trackTypes
})
for key, value in zip(trackFields,
[indicator, TCID, year, month,
day, hour, minute, timeElapsed,
datetimes, lon, lat, speed, bearing,
pressure, windspeed, rmax, poci, beta]):
data[key] = value
tracks = []
n = np.max(TCID)
for i in range(1, n + 1):
track = Track(data[TCID == i])
track.trackId = (i, n)
track.trackfile = trackFile
getMinPressure(track, missingValue)
getMaxWind(track, missingValue)
tracks.append(track)
return tracks
def run(configFile, callback=None):
"""
Run the wind field calculations.
:param str configFile: path to a configuration file.
:param func callback: optional callback function to track progress.
"""
log.info('Loading wind field calculation settings')
# Get configuration
config = ConfigParser()
config.read(configFile)
outputPath = config.get('Output', 'Path')
profileType = config.get('WindfieldInterface', 'profileType')
windFieldType = config.get('WindfieldInterface', 'windFieldType')
beta = config.getfloat('WindfieldInterface', 'beta')
beta1 = config.getfloat('WindfieldInterface', 'beta1')
beta2 = config.getfloat('WindfieldInterface', 'beta2')
thetaMax = config.getfloat('WindfieldInterface', 'thetaMax')
margin = config.getfloat('WindfieldInterface', 'Margin')
resolution = config.getfloat('WindfieldInterface', 'Resolution')
domain = config.get('WindfieldInterface', 'Domain')
windfieldPath = pjoin(outputPath, 'windfield')
trackPath = pjoin(outputPath, 'tracks')
windfieldFormat = 'gust-%i-%04d.nc'
gridLimit = None
if config.has_option('Region','gridLimit'):
gridLimit = config.geteval('Region', 'gridLimit')
if config.has_option('WindfieldInterface', 'gridLimit'):
gridLimit = config.geteval('WindfieldInterface', 'gridLimit')
if config.has_section('Timeseries'):
if config.has_option('Timeseries', 'Extract'):
if config.getboolean('Timeseries', 'Extract'):
from Utilities.timeseries import Timeseries
log.debug("Timeseries data will be extracted")
ts = Timeseries(configFile)
timestepCallback = ts.extract
else:
def timestepCallback(*args):
"""Dummy timestepCallback function"""
pass
thetaMax = math.radians(thetaMax)
# Attempt to start the track generator in parallel
global pp
pp = attemptParallel()
log.info('Running windfield generator')
wfg = WindfieldGenerator(config=config,
margin=margin,
resolution=resolution,
profileType=profileType,
windFieldType=windFieldType,
beta=beta,
beta1=beta1,
beta2=beta2,
thetaMax=thetaMax,
gridLimit=gridLimit,
domain=domain)
msg = 'Dumping gusts to %s' % windfieldPath
log.info(msg)
# Get the trackfile names and count
files = os.listdir(trackPath)
trackfiles = [pjoin(trackPath, f) for f in files if f.startswith('tracks')]
nfiles = len(trackfiles)
def progressCallback(i):
"""Define the callback function"""
callback(i, nfiles)
msg = 'Processing %d track files in %s' % (nfiles, trackPath)
log.info(msg)
# Do the work
pp.barrier()
wfg.dumpGustsFromTrackfiles(trackfiles, windfieldPath, windfieldFormat,
progressCallback, timestepCallback)
try:
ts.shutdown()
except NameError:
pass
pp.barrier()
log.info('Completed windfield generator')
