def historic(self): """Load historic data and calculate histogram""" config = ConfigParser() config.read(self.configFile) inputFile = config.get('DataProcess', 'InputFile') if len(os.path.dirname(inputFile)) == 0: inputFile = pjoin(self.inputPath, inputFile) source = config.get('DataProcess', 'Source') timestep = config.getfloat('TrackGenerator', 'Timestep') interpHistFile = pjoin(self.inputPath, "interp_tracks.csv") try: tracks = interpolateTracks.parseTracks(self.configFile, inputFile, source, timestep, interpHistFile, 'linear') except (TypeError, IOError, ValueError): log.critical("Cannot load historical track file: {0}".format(inputFile)) raise else: startYr = 9999 endYr = 0 for t in tracks: startYr = min(startYr, min(t.Year)) endYr = max(endYr, max(t.Year)) numYears = endYr - startYr self.hist = self.calculate(tracks) / numYears
def historic(self): """Calculate historical rates of longitude crossing""" log.info("Processing historical tracks for longitude crossings") config = ConfigParser() config.read(self.configFile) inputFile = config.get("DataProcess", "InputFile") source = config.get("DataProcess", "Source") timestep = config.getfloat("TrackGenerator", "Timestep") if len(os.path.dirname(inputFile)) == 0: inputFile = pjoin(self.inputPath, inputFile) try: tracks = interpolateTracks.parseTracks( self.configFile, inputFile, source, timestep, interpolation_type="linear" ) except (TypeError, IOError, ValueError): log.critical("Cannot load historical track file: {0}".format(inputFile)) raise else: self.lonCrossingHist, self.lonCrossingEWHist, self.lonCrossingWEHist = self.findCrossings(tracks) return
def historic(self): """Calculate historical rates of longitude crossing""" LOG.info("Processing historical tracks for longitude crossings") config = ConfigParser() config.read(self.configFile) inputFile = config.get('DataProcess', 'InputFile') source = config.get('DataProcess', 'Source') timestep = config.getfloat('TrackGenerator', 'Timestep') if len(os.path.dirname(inputFile)) == 0: inputFile = pjoin(self.inputPath, inputFile) try: tracks = interpolateTracks.parseTracks(self.configFile, inputFile, source, timestep, interpolation_type='linear') except (TypeError, IOError, ValueError): LOG.critical("Cannot load historical track file: {0}".\ format(inputFile)) raise else: self.lonCrossingHist, self.lonCrossingEWHist, \ self.lonCrossingWEHist = self.findCrossings(tracks) return
def __init__(self, configFile, autoCalc_gridLimit=None, progressbar=None): """ Initialize the data and variables required for the interface """ self.configFile = configFile config = ConfigParser() config.read(configFile) self.progressbar = progressbar log.info("Initialising StatInterface") self.kdeType = config.get('StatInterface', 'kdeType') self.kde2DType = config.get('StatInterface','kde2DType') minSamplesCell = config.getint('StatInterface', 'minSamplesCell') self.kdeStep = config.getfloat('StatInterface', 'kdeStep') self.outputPath = config.get('Output', 'Path') self.processPath = pjoin(self.outputPath, 'process') missingValue = cnfGetIniValue(self.configFile, 'StatInterface', 'MissingValue', sys.maxint) gridLimitStr = cnfGetIniValue(self.configFile, 'StatInterface', 'gridLimit', '') if gridLimitStr is not '': try: self.gridLimit = eval(gridLimitStr) except SyntaxError: log.exception('Error! gridLimit is not a dictionary') else: self.gridLimit = autoCalc_gridLimit log.info('No gridLimit specified - using automatic' + ' selection: ' + str(self.gridLimit)) try: gridSpace = config.geteval('Region', 'gridSpace') gridInc = config.geteval('Region', 'gridInc') except SyntaxError: log.exception('Error! gridSpace or gridInc not dictionaries') raise self.generateDist = GenerateDistributions(self.configFile, self.gridLimit, gridSpace, gridInc, self.kdeType, minSamplesCell, missingValue) self.gridSpace = gridSpace self.gridInc = gridInc
def __init__(self, configFile, autoCalc_gridLimit=None, progressbar=None): """ Initialize the data and variables required for the interface """ self.configFile = configFile config = ConfigParser() config.read(configFile) self.progressbar = progressbar log.info("Initialising StatInterface") self.kdeType = config.get('StatInterface', 'kdeType') minSamplesCell = config.getint('StatInterface', 'minSamplesCell') self.kdeStep = config.getfloat('StatInterface', 'kdeStep') self.outputPath = config.get('Output', 'Path') self.processPath = pjoin(self.outputPath, 'process') missingValue = cnfGetIniValue(self.configFile, 'StatInterface', 'MissingValue', sys.maxsize) gridLimitStr = cnfGetIniValue(self.configFile, 'StatInterface', 'gridLimit', '') if gridLimitStr is not '': try: self.gridLimit = eval(gridLimitStr) except SyntaxError: log.exception('Error! gridLimit is not a dictionary') else: self.gridLimit = autoCalc_gridLimit log.info('No gridLimit specified - using automatic' + ' selection: ' + str(self.gridLimit)) try: gridSpace = config.geteval('Region', 'gridSpace') gridInc = config.geteval('Region', 'gridInc') except SyntaxError: log.exception('Error! gridSpace or gridInc not dictionaries') raise self.generateDist = GenerateDistributions(self.configFile, self.gridLimit, gridSpace, gridInc, self.kdeType, minSamplesCell, missingValue) self.gridSpace = gridSpace self.gridInc = gridInc
def historic(self): """Calculate historical rates of landfall""" log.info("Processing landfall rates of historical tracks") config = ConfigParser() config.read(self.configFile) inputFile = config.get('DataProcess', 'InputFile') source = config.get('DataProcess', 'Source') timestep = config.getfloat('TrackGenerator', 'Timestep') if len(os.path.dirname(inputFile)) == 0: inputFile = pjoin(self.inputPath, inputFile) try: tracks = loadTrackFile(self.configFile, inputFile, source) except (TypeError, IOError, ValueError): log.critical("Cannot load historical track file: {0}".format(inputFile)) raise else: self.historicLandfall, self.historicOffshore = self.processTracks(tracks) return
def run(configFile, callback=None): """ Run the wind field calculations. :param str configFile: path to a configuration file. :param func callback: optional callback function to track progress. """ log.info('Loading wind field calculation settings') # Get configuration config = ConfigParser() config.read(configFile) profileType = config.get('WindfieldInterface', 'profileType') windFieldType = config.get('WindfieldInterface', 'windFieldType') beta = config.getfloat('WindfieldInterface', 'beta') beta1 = config.getfloat('WindfieldInterface', 'beta1') beta2 = config.getfloat('WindfieldInterface', 'beta2') thetaMax = config.getfloat('WindfieldInterface', 'thetaMax') margin = config.getfloat('WindfieldInterface', 'Margin') resolution = config.getfloat('WindfieldInterface', 'Resolution') domain = config.get('WindfieldInterface', 'Domain') outputPath = config.get('Output', 'Path') windfieldPath = pjoin(outputPath, 'windfield') trackPath = pjoin(outputPath, 'tracks') gridLimit = None if config.has_option('Region', 'gridLimit'): gridLimit = config.geteval('Region', 'gridLimit') if config.has_option('WindfieldInterface', 'gridLimit'): gridLimit = config.geteval('WindfieldInterface', 'gridLimit') if config.getboolean('Timeseries', 'Extract', fallback=False): from Utilities.timeseries import Timeseries ts = Timeseries(configFile) timestepCallback = ts.extract else: timestepCallback = None multipliers = None if config.has_option('Input', 'Multipliers'): multipliers = config.get('Input', 'Multipliers') thetaMax = math.radians(thetaMax) # Attempt to start the track generator in parallel global MPI MPI = attemptParallel() comm = MPI.COMM_WORLD log.info('Running windfield generator') wfg = WindfieldGenerator(config=config, margin=margin, resolution=resolution, profileType=profileType, windFieldType=windFieldType, beta=beta, beta1=beta1, beta2=beta2, thetaMax=thetaMax, gridLimit=gridLimit, domain=domain, multipliers=multipliers, windfieldPath=windfieldPath) log.info(f'Dumping gusts to {windfieldPath}') # Get the trackfile names and count files = os.listdir(trackPath) trackfiles = [pjoin(trackPath, f) for f in files if f.startswith('tracks')] nfiles = len(trackfiles) log.info('Processing {0} track files in {1}'.format(nfiles, trackPath)) # Do the work comm.barrier() wfg.dumpGustsFromTrackfiles(trackfiles, windfieldPath, timestepCallback) try: ts.shutdown() except NameError: pass comm.barrier() log.info('Completed windfield generator')
def run(configFile, callback=None): """ Run the wind field calculations. :param str configFile: path to a configuration file. :param func callback: optional callback function to track progress. """ log.info('Loading wind field calculation settings') # Get configuration config = ConfigParser() config.read(configFile) profileType = config.get('WindfieldInterface', 'profileType') windFieldType = config.get('WindfieldInterface', 'windFieldType') beta = config.getfloat('WindfieldInterface', 'beta') beta1 = config.getfloat('WindfieldInterface', 'beta1') beta2 = config.getfloat('WindfieldInterface', 'beta2') thetaMax = config.getfloat('WindfieldInterface', 'thetaMax') margin = config.getfloat('WindfieldInterface', 'Margin') resolution = config.getfloat('WindfieldInterface', 'Resolution') domain = config.get('WindfieldInterface', 'Domain') outputPath = config.get('Output', 'Path') windfieldPath = pjoin(outputPath, 'windfield') trackPath = pjoin(outputPath, 'tracks') gridLimit = None if config.has_option('Region', 'gridLimit'): gridLimit = config.geteval('Region', 'gridLimit') if config.has_option('WindfieldInterface', 'gridLimit'): gridLimit = config.geteval('WindfieldInterface', 'gridLimit') if config.has_section('Timeseries'): if config.has_option('Timeseries', 'Extract'): if config.getboolean('Timeseries', 'Extract'): from Utilities.timeseries import Timeseries log.debug("Timeseries data will be extracted") ts = Timeseries(configFile) timestepCallback = ts.extract else: def timestepCallback(*args): """Dummy timestepCallback function""" pass else: def timestepCallback(*args): """Dummy timestepCallback function""" pass thetaMax = math.radians(thetaMax) # Attempt to start the track generator in parallel global pp pp = attemptParallel() log.info('Running windfield generator') wfg = WindfieldGenerator(config=config, margin=margin, resolution=resolution, profileType=profileType, windFieldType=windFieldType, beta=beta, beta1=beta1, beta2=beta2, thetaMax=thetaMax, gridLimit=gridLimit, domain=domain) msg = 'Dumping gusts to %s' % windfieldPath log.info(msg) # Get the trackfile names and count files = os.listdir(trackPath) trackfiles = [pjoin(trackPath, f) for f in files if f.startswith('tracks')] nfiles = len(trackfiles) def progressCallback(i): """Define the callback function""" callback(i, nfiles) msg = 'Processing %d track files in %s' % (nfiles, trackPath) log.info(msg) # Do the work pp.barrier() wfg.dumpGustsFromTrackfiles(trackfiles, windfieldPath, timestepCallback) try: ts.shutdown() except NameError: pass pp.barrier() log.info('Completed windfield generator')
def loadTrackFile(configFile, trackFile, source, missingValue=0, calculateWindSpeed=True): """ Load TC track data from the given input file, from a specified source. The configFile is a configuration file that contains a section called 'source' that describes the data. This returns a collection of :class:`Track` objects that contains the details of the TC tracks in the input file. :param str configFile: Configuration file with a section ``source``. :param str trackFile: Path to a csv-formatted file containing TC data. :pararm str source: Name of the source format of the TC data. There *must* be a section in ``configFile`` matching this string, containing the details of the format of the data. :param missingValue: Replace all null values in the input data with this value (default=0). :param boolean calculateWindSpeed: Calculate maximum wind speed using a pressure-wind relation described in :func:`maxWindSpeed` :returns: A collection of :class:`Track` objects. If any of the variables are not present in the input dataset, they are (where possible) calculated (date/time/windspeed), sampled from default datasets (e.g. environmental pressure) or set to the missing value. Example:: >>> tracks = loadTrackFile('tcrm.ini', 'IBTRaCS.csv', 'IBTrACS' ) """ LOG.info("Loading %s" % trackFile) inputData = colReadCSV(configFile, trackFile, source) #, #nullValue=missingValue) config = ConfigParser() config.read(configFile) inputSpeedUnits = config.get(source, 'SpeedUnits') inputPressureUnits = config.get(source, 'PressureUnits') inputLengthUnits = config.get(source, 'LengthUnits') inputDateFormat = config.get(source, 'DateFormat') if config.getboolean('DataProcess', 'FilterSeasons'): startSeason = config.getint('DataProcess', 'StartSeason') idx = np.where(inputData['season'] >= startSeason)[0] inputData = inputData[idx] # Determine the initial TC positions... indicator = getInitialPositions(inputData) # Sort date/time information if 'age' in inputData.dtype.names: year, month, day, hour, minute, datetimes = parseAge(inputData, indicator) timeElapsed = inputData['age'] else: year, month, day, hour, minute, datetimes = parseDates(inputData, indicator, inputDateFormat) timeElapsed = getTimeElapsed(indicator, year, month, day, hour, minute) # Time between observations: dt = getTimeDelta(year, month, day, hour, minute) # Calculate julian days jdays = julianDays(year, month, day, hour, minute) lat = np.array(inputData['lat'], 'd') lon = np.mod(np.array(inputData['lon'], 'd'), 360) delta_lon = np.diff(lon) delta_lat = np.diff(lat) # Split into separate tracks if large jump occurs (delta_lon > 10 degrees # or delta_lat > 5 degrees) # This avoids two tracks being accidentally combined when seasons and track # numbers match but basins are different as occurs in the IBTrACS dataset. # This problem can also be prevented if the 'tcserialno' column is # specified. indicator[np.where(delta_lon > 10)[0] + 1] = 1 indicator[np.where(delta_lat > 5)[0] + 1] = 1 pressure = filterPressure(np.array(inputData['pressure'], 'd'), inputPressureUnits, missingValue) try: windspeed = np.array(inputData['vmax'], 'd') novalue_index = np.where(windspeed == sys.maxint) windspeed = metutils.convert(windspeed, inputSpeedUnits, "mps") windspeed[novalue_index] = missingValue except (ValueError, KeyError): LOG.debug("No max wind speed data - all values will be zero") windspeed = np.zeros(indicator.size, 'f') assert lat.size == indicator.size assert lon.size == indicator.size assert pressure.size == indicator.size try: rmax = np.array(inputData['rmax']) novalue_index = np.where(rmax == missingValue) rmax = metutils.convert(rmax, inputLengthUnits, "km") rmax[novalue_index] = missingValue except (ValueError, KeyError): LOG.debug("No radius to max wind data - all values will be zero") rmax = np.zeros(indicator.size, 'f') if 'penv' in inputData.dtype.names: penv = np.array(inputData['penv'], 'd') else: LOG.debug("No ambient MSLP data in this input file") LOG.debug("Sampling data from MSLP data defined in " "configuration file") # Warning: using sampled data will likely lead to some odd behaviour # near the boundary of the MSLP grid boundaries - higher resolution # MSLP data will decrease this unusual behaviour. try: ncfile = cnfGetIniValue(configFile, 'Input', 'MSLPFile') except: LOG.exception("No input MSLP file specified in configuration") raise time = getTime(year, month, day, hour, minute) penv = ltmPressure(jdays, time, lon, lat, ncfile) if 'poci' in inputData.dtype.names: poci = np.array(inputData['poci'], 'd') else: LOG.debug("Determining poci") eps = np.random.normal(0, scale=2.5717) poci = getPoci(penv, pressure, lat, jdays, eps) if 'beta' in inputData.dtype.names: beta = np.array(inputData['beta'], 'd') LOG.debug("beta found in the input file") else: beta_vals = config.getfloat('WindfieldInterface','beta') beta = np.ones(len(indicator)) * beta_vals LOG.debug("no beta found in the input file") speed, bearing = getSpeedBearing(indicator, lon, lat, dt, missingValue=missingValue) if calculateWindSpeed: windspeed = maxWindSpeed(indicator, dt, lon, lat, pressure, poci, beta) TCID = np.cumsum(indicator) data = np.empty(len(indicator), dtype={ 'names': trackFields, 'formats': trackTypes }) for key, value in zip(trackFields, [indicator, TCID, year, month, day, hour, minute, timeElapsed, datetimes, lon, lat, speed, bearing, pressure, windspeed, rmax, poci, beta]): data[key] = value tracks = [] n = np.max(TCID) for i in range(1, n + 1): track = Track(data[TCID == i]) track.trackId = (i, n) track.trackfile = trackFile getMinPressure(track, missingValue) getMaxWind(track, missingValue) tracks.append(track) return tracks
def run(configFile, callback=None): """ Run the wind field calculations. :param str configFile: path to a configuration file. :param func callback: optional callback function to track progress. """ log.info('Loading wind field calculation settings') # Get configuration config = ConfigParser() config.read(configFile) outputPath = config.get('Output', 'Path') profileType = config.get('WindfieldInterface', 'profileType') windFieldType = config.get('WindfieldInterface', 'windFieldType') beta = config.getfloat('WindfieldInterface', 'beta') beta1 = config.getfloat('WindfieldInterface', 'beta1') beta2 = config.getfloat('WindfieldInterface', 'beta2') thetaMax = config.getfloat('WindfieldInterface', 'thetaMax') margin = config.getfloat('WindfieldInterface', 'Margin') resolution = config.getfloat('WindfieldInterface', 'Resolution') domain = config.get('WindfieldInterface', 'Domain') windfieldPath = pjoin(outputPath, 'windfield') trackPath = pjoin(outputPath, 'tracks') windfieldFormat = 'gust-%i-%04d.nc' gridLimit = None if config.has_option('Region','gridLimit'): gridLimit = config.geteval('Region', 'gridLimit') if config.has_option('WindfieldInterface', 'gridLimit'): gridLimit = config.geteval('WindfieldInterface', 'gridLimit') if config.has_section('Timeseries'): if config.has_option('Timeseries', 'Extract'): if config.getboolean('Timeseries', 'Extract'): from Utilities.timeseries import Timeseries log.debug("Timeseries data will be extracted") ts = Timeseries(configFile) timestepCallback = ts.extract else: def timestepCallback(*args): """Dummy timestepCallback function""" pass thetaMax = math.radians(thetaMax) # Attempt to start the track generator in parallel global pp pp = attemptParallel() log.info('Running windfield generator') wfg = WindfieldGenerator(config=config, margin=margin, resolution=resolution, profileType=profileType, windFieldType=windFieldType, beta=beta, beta1=beta1, beta2=beta2, thetaMax=thetaMax, gridLimit=gridLimit, domain=domain) msg = 'Dumping gusts to %s' % windfieldPath log.info(msg) # Get the trackfile names and count files = os.listdir(trackPath) trackfiles = [pjoin(trackPath, f) for f in files if f.startswith('tracks')] nfiles = len(trackfiles) def progressCallback(i): """Define the callback function""" callback(i, nfiles) msg = 'Processing %d track files in %s' % (nfiles, trackPath) log.info(msg) # Do the work pp.barrier() wfg.dumpGustsFromTrackfiles(trackfiles, windfieldPath, windfieldFormat, progressCallback, timestepCallback) try: ts.shutdown() except NameError: pass pp.barrier() log.info('Completed windfield generator')