def test_julianDays(self): """Test julianDays function""" jday = loadData.julianDays(self.inputYear, self.inputMonth, self.inputDay, self.inputHour, self.inputMinute) assert_almost_equal(jday, self.outputJdays)
def processData(self, restrictToWindfieldDomain=False): """ Process raw data into ASCII files that can be read by the main components of the system :param bool restrictToWindfieldDomain: if True, only process data within the wind field domain, otherwise, process data from across the track generation domain. """ config = ConfigParser() config.read(self.configFile) self.logger.info("Running {0}".format(flModuleName())) if config.has_option('DataProcess', 'InputFile'): inputFile = config.get('DataProcess', 'InputFile') if config.has_option('DataProcess', 'Source'): source = config.get('DataProcess', 'Source') self.logger.info('Loading %s dataset', source) fn = config.get(source, 'filename') path = config.get(source, 'path') inputFile = pjoin(path, fn) # If input file has no path information, default to tcrm input folder if len(os.path.dirname(inputFile)) == 0: inputFile = pjoin(self.tcrm_input_dir, inputFile) self.logger.info("Processing {0}".format(inputFile)) self.source = config.get('DataProcess', 'Source') inputData = colReadCSV(self.configFile, inputFile, self.source) inputSpeedUnits = config.get(self.source, 'SpeedUnits') inputPressureUnits = config.get(self.source, 'PressureUnits') inputLengthUnits = config.get(self.source, 'LengthUnits') startSeason = config.getint('DataProcess', 'StartSeason') indicator = loadData.getInitialPositions(inputData) lat = np.array(inputData['lat'], 'd') lon = np.mod(np.array(inputData['lon'], 'd'), 360) if restrictToWindfieldDomain: # Filter the input arrays to only retain the tracks that # pass through the windfield domain. CD = CalcTrackDomain(self.configFile) self.domain = CD.calcDomainFromTracks(indicator, lon, lat) domainIndex = self.extractTracks(indicator, lon, lat) inputData = inputData[domainIndex] indicator = indicator[domainIndex] lon = lon[domainIndex] lat = lat[domainIndex] if self.progressbar is not None: self.progressbar.update(0.125) # Sort date/time information try: dt = np.empty(indicator.size, 'f') dt[1:] = np.diff(inputData['age']) except (ValueError, KeyError): try: self.logger.info(("Filtering input data by season:" "season > {0}".format(startSeason))) # Find indicies that satisfy minimum season filter idx = np.where(inputData['season'] >= startSeason)[0] # Filter records: inputData = inputData[idx] indicator = indicator[idx] lon = lon[idx] lat = lat[idx] except (ValueError, KeyError): pass year, month, day, hour, minute, datetimes \ = loadData.parseDates(inputData, indicator) # Time between observations: dt = loadData.getTimeDelta(year, month, day, hour, minute) # Calculate julian days: jdays = loadData.julianDays(year, month, day, hour, minute) delta_lon = np.diff(lon) delta_lat = np.diff(lat) # Split into separate tracks if large jump occurs (delta_lon > # 15 degrees or delta_lat > 5 degrees) This avoids two tracks # being accidentally combined when seasons and track numbers # match but basins are different as occurs in the IBTrACS # dataset. This problem can also be prevented if the # 'tcserialno' column is specified. indicator[np.where(delta_lon > 15)[0] + 1] = 1 indicator[np.where(delta_lat > 5)[0] + 1] = 1 # Save information required for frequency auto-calculation try: origin_seasonOrYear = np.array(inputData['season'], 'i').compress(indicator) header = 'Season' except (ValueError, KeyError): origin_seasonOrYear = year.compress(indicator) header = 'Year' flSaveFile(self.origin_year, np.transpose(origin_seasonOrYear), header, ',', fmt='%d') pressure = np.array(inputData['pressure'], 'd') novalue_index = np.where(pressure == sys.maxint) pressure = metutils.convert(pressure, inputPressureUnits, "hPa") pressure[novalue_index] = sys.maxint # Convert any non-physical central pressure values to maximum integer # This is required because IBTrACS has a mix of missing value codes # (i.e. -999, 0, 9999) in the same global dataset. pressure = np.where((pressure < 600) | (pressure > 1100), sys.maxint, pressure) if self.progressbar is not None: self.progressbar.update(0.25) try: vmax = np.array(inputData['vmax'], 'd') except (ValueError, KeyError): self.logger.warning("No max wind speed data") vmax = np.empty(indicator.size, 'f') else: novalue_index = np.where(vmax == sys.maxint) vmax = metutils.convert(vmax, inputSpeedUnits, "mps") vmax[novalue_index] = sys.maxint assert lat.size == indicator.size assert lon.size == indicator.size assert pressure.size == indicator.size #assert vmax.size == indicator.size try: rmax = np.array(inputData['rmax']) novalue_index = np.where(rmax == sys.maxint) rmax = metutils.convert(rmax, inputLengthUnits, "km") rmax[novalue_index] = sys.maxint self._rmax(rmax, indicator) self._rmaxRate(rmax, dt, indicator) except (ValueError, KeyError): self.logger.warning("No rmax data available") if self.ncflag: self.data['index'] = indicator # ieast : parameter used in latLon2Azi # FIXME: should be a config setting describing the input data. ieast = 1 # Determine the index of initial cyclone observations, excluding # those cyclones that have only one observation. This is used # for calculating initial bearing and speed indicator2 = np.where(indicator > 0, 1, 0) initIndex = np.concatenate( [np.where(np.diff(indicator2) == -1, 1, 0), [0]]) # Calculate the bearing and distance (km) of every two # consecutive records using ll2azi bear_, dist_ = maputils.latLon2Azi(lat, lon, ieast, azimuth=0) assert bear_.size == indicator.size - 1 assert dist_.size == indicator.size - 1 bear = np.empty(indicator.size, 'f') bear[1:] = bear_ dist = np.empty(indicator.size, 'f') dist[1:] = dist_ self._lonLat(lon, lat, indicator, initIndex) self._bearing(bear, indicator, initIndex) self._bearingRate(bear, dt, indicator) if self.progressbar is not None: self.progressbar.update(0.375) self._speed(dist, dt, indicator, initIndex) self._speedRate(dist, dt, indicator) self._pressure(pressure, indicator) self._pressureRate(pressure, dt, indicator) self._windSpeed(vmax) try: self._frequency(year, indicator) self._juliandays(jdays, indicator, year) except (ValueError, KeyError): pass self.logger.info("Completed {0}".format(flModuleName())) if self.progressbar is not None: self.progressbar.update(0.5)
def processData(self, restrictToWindfieldDomain=False): """ Process raw data into ASCII files that can be read by the main components of the system :param bool restrictToWindfieldDomain: if True, only process data within the wind field domain, otherwise, process data from across the track generation domain. """ config = ConfigParser() config.read(self.configFile) self.logger.info("Running %s" % flModuleName()) if config.has_option('DataProcess', 'InputFile'): inputFile = config.get('DataProcess', 'InputFile') if config.has_option('DataProcess', 'Source'): source = config.get('DataProcess', 'Source') self.logger.info('Loading %s dataset', source) fn = config.get(source, 'filename') path = config.get(source, 'path') inputFile = pjoin(path, fn) # If input file has no path information, default to tcrm input folder if len(os.path.dirname(inputFile)) == 0: inputFile = pjoin(self.tcrm_input_dir, inputFile) self.logger.info("Processing %s" % inputFile) self.source = config.get('DataProcess', 'Source') inputData = colReadCSV(self.configFile, inputFile, self.source) inputSpeedUnits = config.get(self.source, 'SpeedUnits') inputPressureUnits = config.get(self.source, 'PressureUnits') inputLengthUnits = config.get(self.source, 'LengthUnits') startSeason = config.getint('DataProcess', 'StartSeason') indicator = loadData.getInitialPositions(inputData) lat = np.array(inputData['lat'], 'd') lon = np.mod(np.array(inputData['lon'], 'd'), 360) if restrictToWindfieldDomain: # Filter the input arrays to only retain the tracks that # pass through the windfield domain. CD = CalcTrackDomain(self.configFile) self.domain = CD.calcDomainFromTracks(indicator, lon, lat) domainIndex = self.extractTracks(indicator, lon, lat) inputData = inputData[domainIndex] indicator = indicator[domainIndex] lon = lon[domainIndex] lat = lat[domainIndex] if self.progressbar is not None: self.progressbar.update(0.125) # Sort date/time information try: dt = np.empty(indicator.size, 'f') dt[1:] = np.diff(inputData['age']) except (ValueError, KeyError): try: self.logger.info("Filtering input data by season: season > %d"%startSeason) # Find indicies that satisfy minimum season filter idx = np.where(inputData['season'] >= startSeason)[0] # Filter records: inputData = inputData[idx] indicator = indicator[idx] lon = lon[idx] lat = lat[idx] except (ValueError, KeyError): pass year, month, day, hour, minute, datetimes \ = loadData.parseDates(inputData, indicator) # Time between observations: dt = loadData.getTimeDelta(year, month, day, hour, minute) # Calculate julian days: jdays = loadData.julianDays(year, month, day, hour, minute) delta_lon = np.diff(lon) delta_lat = np.diff(lat) # Split into separate tracks if large jump occurs (delta_lon > # 15 degrees or delta_lat > 5 degrees) This avoids two tracks # being accidentally combined when seasons and track numbers # match but basins are different as occurs in the IBTrACS # dataset. This problem can also be prevented if the # 'tcserialno' column is specified. indicator[np.where(delta_lon > 15)[0] + 1] = 1 indicator[np.where(delta_lat > 5)[0] + 1] = 1 # Save information required for frequency auto-calculation try: origin_seasonOrYear = np.array( inputData['season'], 'i').compress(indicator) header = 'Season' except (ValueError, KeyError): origin_seasonOrYear = year.compress(indicator) header = 'Year' flSaveFile(self.origin_year, np.transpose(origin_seasonOrYear), header, ',', fmt='%d') pressure = np.array(inputData['pressure'], 'd') novalue_index = np.where(pressure == sys.maxint) pressure = metutils.convert(pressure, inputPressureUnits, "hPa") pressure[novalue_index] = sys.maxint # Convert any non-physical central pressure values to maximum integer # This is required because IBTrACS has a mix of missing value codes # (i.e. -999, 0, 9999) in the same global dataset. pressure = np.where((pressure < 600) | (pressure > 1100), sys.maxint, pressure) if self.progressbar is not None: self.progressbar.update(0.25) try: vmax = np.array(inputData['vmax'], 'd') except (ValueError, KeyError): self.logger.warning("No max wind speed data") vmax = np.empty(indicator.size, 'f') else: novalue_index = np.where(vmax == sys.maxint) vmax = metutils.convert(vmax, inputSpeedUnits, "mps") vmax[novalue_index] = sys.maxint assert lat.size == indicator.size assert lon.size == indicator.size assert pressure.size == indicator.size #assert vmax.size == indicator.size try: rmax = np.array(inputData['rmax']) novalue_index = np.where(rmax == sys.maxint) rmax = metutils.convert(rmax, inputLengthUnits, "km") rmax[novalue_index] = sys.maxint self._rmax(rmax, indicator) self._rmaxRate(rmax, dt, indicator) except (ValueError, KeyError): self.logger.warning("No rmax data available") if self.ncflag: self.data['index'] = indicator # ieast : parameter used in latLon2Azi # FIXME: should be a config setting describing the input data. ieast = 1 # Determine the index of initial cyclone observations, excluding # those cyclones that have only one observation. This is used # for calculating initial bearing and speed indicator2 = np.where(indicator > 0, 1, 0) initIndex = np.concatenate([np.where(np.diff(indicator2) == -1, 1, 0), [0]]) # Calculate the bearing and distance (km) of every two # consecutive records using ll2azi bear_, dist_ = maputils.latLon2Azi(lat, lon, ieast, azimuth=0) assert bear_.size == indicator.size - 1 assert dist_.size == indicator.size - 1 bear = np.empty(indicator.size, 'f') bear[1:] = bear_ dist = np.empty(indicator.size, 'f') dist[1:] = dist_ self._lonLat(lon, lat, indicator, initIndex) self._bearing(bear, indicator, initIndex) self._bearingRate(bear, dt, indicator) if self.progressbar is not None: self.progressbar.update(0.375) self._speed(dist, dt, indicator, initIndex) self._speedRate(dist, dt, indicator) self._pressure(pressure, indicator) self._pressureRate(pressure, dt, indicator) self._windSpeed(vmax) try: self._frequency(year, indicator) self._juliandays(jdays, indicator, year) except (ValueError, KeyError): pass self.logger.info("Completed %s" % flModuleName()) if self.progressbar is not None: self.progressbar.update(0.5)