Beispiel #1
0
    def __init__(self, parameter, lonLat, gridLimit,
                 gridSpace, gridInc, minSample=100, angular=False,
                 missingValue=sys.maxsize, progressbar=None,
                 prgStartValue=0, prgEndValue=1, calculateLater=False):

        self.logger = logging.getLogger()
        self.logger.debug('Initialising GenerateStats')

        self.gridLimit = gridLimit
        self.gridSpace = gridSpace
        self.gridInc = gridInc
        self.maxCell = stats.maxCellNum(self.gridLimit, self.gridSpace)
        self.minSample = minSample
        self.coeffs = parameters(self.maxCell+1)
        self.angular = angular
        self.missingValue = missingValue

        self.domain_warning_raised = False

        self.progressbar = progressbar
        self.prgStartValue = prgStartValue
        self.prgEndValue = prgEndValue

        if not calculateLater:
            if type(lonLat) is str:
                self.lonLat = np.array(flLoadFile(lonLat, delimiter=','))
            else:
                self.lonLat = lonLat
            if type(parameter) is str:
                self.param = np.array(flLoadFile(parameter))
            else:
                self.param = parameter

            self.calculateStatistics()
Beispiel #2
0
 def setKDEOrigins(self, kdeOriginX=None, kdeOriginY=None, kdeOriginZ=None,
                   outputPath=None):
     """set kde origin parameters
     """
     if outputPath:
         try:
             self.x, self.y. self.z = grdRead(os.path.join(self.outputPath,
                                                           'originPDF.txt'))
         except IOError:
             self.logger.critical('Error! Files relating to KDE of cyclone origins does not exist. Execute KDE of cyclone origins first.')
             raise
         self._calculateCDF() #calculate CDF of (x,Px) and (y,Py)
     elif type(kdeOriginZ) == str:
         try:
             self.x = flLoadFile(kdeOriginX)
             self.y = flLoadFile(kdeOriginY)
             self.z = flLoadFile(kdeOriginZ)
         except IOError:
             self.logger.critical('Error! Files relating to CDF of cyclone parameters do not exist. Generate KDE of cyclone parameters first.')
             raise
         self._calculateCDF()  # calculate CDF of (x,Px) and (y,Py)
     elif type(kdeOriginZ) == ndarray:
         self.x, self.y, self.z = grdRead(kdeOriginZ)
         self._calculateCDF()  # calculate CDF of (x,Px) and (y,Py)
     else:
         self.logger.error("No input arguments")
         raise
Beispiel #3
0
    def __init__(self, parameter, lonLat, gridLimit,
                 gridSpace, gridInc, minSample=100, angular=False,
                 missingValue=sys.maxint, progressbar=None,
                 prgStartValue=0, prgEndValue=1, calculateLater=False):
        
        self.logger = logging.getLogger()
        self.logger.debug('Initialising GenerateStats')
        
        self.gridLimit = gridLimit
        self.gridSpace = gridSpace
        self.gridInc = gridInc
        self.maxCell = stats.maxCellNum(self.gridLimit, self.gridSpace)
        self.minSample = minSample
        self.coeffs = parameters(self.maxCell+1)
        self.angular = angular
        self.missingValue = missingValue

        self.domain_warning_raised = False

        self.progressbar = progressbar
        self.prgStartValue = prgStartValue
        self.prgEndValue = prgEndValue

        if not calculateLater:
            if type(lonLat) is str:
                self.lonLat = np.array(flLoadFile(lonLat, delimiter=','))
            else:
                self.lonLat = lonLat
            if type(parameter) is str:
                self.param = np.array(flLoadFile(parameter))
            else:
                self.param = parameter

            self.calculateStatistics()
Beispiel #4
0
    def setKDEOrigins(self,
                      kdeOriginX=None,
                      kdeOriginY=None,
                      kdeOriginZ=None,
                      outputPath=None):
        """
        Set kernel density estimation origin parameters.

        :param kdeOriginX: x coordinates of kde result generated
                           from :class:`KDEOrigin`
        :param kdeOriginY: y coordinates of kde result generated
                           from :class:`KDEOrigin`
        :param kdeOriginZ: z coordinates of kde result generated
                           from :class:`KDEOrigin`
        :param outputPath: Path to output folder to load PDF file.

        :type  kdeOriginX: str or :class:`numpy.ndarray`
        :type  kdeOriginY: str or :class:`numpy.ndarray`
        :type  kdeOriginZ: str or :class:`numpy.ndarray`
        :type  outputPath: str

        """
        if outputPath:
            try:
                self.x, self.y, self.z = grdRead(
                    pjoin(self.outputPath, 'originPDF.txt'))
            except IOError:
                LOG.critical(('Error! Files relating to KDE of cyclone '
                              'origins does not exist. Execute KDE of '
                              'cyclone origins first.'))
                raise
            self._calculateCDF()
        elif isinstance(kdeOriginZ, str):
            try:
                self.x = flLoadFile(kdeOriginX)
                self.y = flLoadFile(kdeOriginY)
                self.z = flLoadFile(kdeOriginZ)
            except IOError:
                LOG.critical(('Error! Files relating to CDF of cyclone '
                              'parameters do not exist. Generate KDE of '
                              'cyclone parameters first.'))
                raise
            self._calculateCDF()
        elif isinstance(kdeOriginZ, np.ndarray):
            self.x, self.y, self.z = grdRead(kdeOriginZ)
            self._calculateCDF()
        else:
            LOG.error("No input arguments")
            raise
Beispiel #5
0
    def generateGenesisDateCDF(self, genDays, lonLat, bw=None, genesisKDE=None):
        """
        Calculate the PDF of genesis day using KDEs.
        Since the data is periodic, we use a simple method to include the 
        periodicity in estimating the PDF. We prepend and append the data
        to itself, then use the central third of the PDF and multiply by three to
        obtain the required PDF. Probably notquite exact, but it should be
        sufficient for our purposes. 
        """

        data = flLoadFile( genDays )
        days = np.arange( 1, 366 )
        ndays = np.concatenate( [days - 365, days, days + 365] )
        ndata = np.concatenate( [data - 365, data, data + 365] )

        if bw is None:
            bw = KPDF.UPDFOptimumBandwidth( ndata ) 

        try:
            kdeMethod = getattr(KPDF, "UPDF%s" %self.kdeType)
        except AttributeError:
            self.logger.exception("Invalid input on option: KDE method UPDF%s does not exist"%self.kdeType)
            raise
        pdf = kdeMethod( ndata, ndays, bw )
        # Actual PDF to return
        apdf = 3.0*pdf[365:730]
        cy = stats.cdf(days, apdf)
        if genesisKDE is None:
            return np.transpose(np.array(np.concatenate( [days, apdf, cy] ) ))
        else:
            # Assume both kdeParameters and cdfParameters are defined as files:
            self.logger.debug("Saving KDE and CDF data to files")
            #flSaveFile(genesisKDE, transpose(numpy.concatenate([days, pdf])))
            flSaveFile(genesisKDE, np.transpose(np.array([days, cy])))
Beispiel #6
0
    def __init__(self, configFile, kdeType, gridLimit, kdeStep, lonLat=None, progressbar=None):
        """
        
        """
        self.logger = logging.getLogger()
        self.progressbar = progressbar
        if self.progressbar:
            KPDF.set_callback(self.updateProgressBar)
        self.logger.info("Initialising KDEOrigins")
        self.configFile = configFile
        self.x = numpy.arange(gridLimit['xMin'], gridLimit['xMax'], kdeStep)
        self.y = numpy.arange(gridLimit['yMax'], gridLimit['yMin'], -kdeStep)

        self.kdeType = kdeType
        self.kdeStep = kdeStep

        config = ConfigParser()
        config.read(configFile)

        if lonLat is None:
            self.outputPath = config.get('Output', 'Path')
            self.processPath = os.path.join(self.outputPath, 'process')
            self.logger.debug("Loading "+os.path.join(self.processPath,
                                                  'init_lon_lat'))
            ll = flLoadFile(os.path.join(self.processPath, 'init_lon_lat'),
                            '%', ',')
            self.lonLat = ll[:,0:2]
        else:
            self.lonLat = lonLat[:,0:2]

        self.bw = KPDF.MPDFOptimumBandwidth(self.lonLat)
        self.logger.debug("Optimal bandwidth: %f"%self.bw)
Beispiel #7
0
    def generateGenesisDateCDF(self,
                               genDays,
                               lonLat,
                               bw=None,
                               genesisKDE=None):
        """
        Calculate the PDF of genesis day using KDEs.
        Since the data is periodic, we use a simple method to include
        the periodicity in estimating the PDF. We prepend and append
        the data to itself, then use the central third of the PDF and
        multiply by three to obtain the required PDF. Probably not
        quite exact, but it should be sufficient for our purposes.

        :param str genDays: Name of file containing genesis days
                            (as day of year).
        :param lonLat: Array of genesis longitudes and latitudes.
        :param float bw: Optional. Bandwidth of the KDE to use.
        :param str genesisKDE: Optional. File name to save resulting CDF to.
        :type  lonLat: :class:`numpy.ndarray`

        :returns: :class:`numpy.ndarray` containing the days, the PDF and CDF
                  of the genesis days.
        """

        data = flLoadFile(genDays)
        days = np.arange(1, 366)
        ndays = np.concatenate([days - 365, days, days + 365])
        ndata = np.concatenate([data - 365, data, data + 365])

        if bw is None:
            bw = stats.bandwidth(self.parameters)

        kde = sm.nonparametric.KDEUnivariate(self.parameters)
        kde.fit(kernel=self.kdeType,
                bw=bw,
                fft=False,
                gridsize=len(grid),
                clip=(min(grid), max(grid)),
                cut=0)
        #try:
        #    kdeMethod = getattr(KPDF, "UPDF%s" % self.kdeType)
        #except AttributeError:
        #    LOG.exception(("Invalid input on option: "
        #                   "KDE method UPDF%s does not exist"),
        #                  self.kdeType)
        #    raise

        veceval = np.vectorize(kde.evaluate)
        pdf = np.nan_to_num(veceval(grid))

        # Actual PDF to return
        apdf = 3.0 * pdf[365:730]
        cy = stats.cdf(days, apdf)
        if genesisKDE is None:
            return np.transpose(np.array(np.concatenate([days, apdf, cy])))
        else:
            # Assume both kdeParameters and cdfParameters are defined as files:
            LOG.debug("Saving KDE and CDF data to files")
            flSaveFile(genesisKDE, np.transpose(np.array([days, cy])))
Beispiel #8
0
    def __init__(self, lon_lat, bearing_list, speed_list, pressure_list,
                 bearing_rate_list, speed_rate_list, pressure_rate_list,
                 cellNum, gridLimit, gridSpace):
        """
        initialize the data needed for the plots including
        longitudes & latitudes of cyclone origins,
        cyclone bearings,
        cyclone speeds,
        cyclone pressures,
        as well as calculating and storing the
        cyclone index counts for a specific inputted cell number
        """
        self.gridLimit = gridLimit
        self.gridSpace = gridSpace
        self.lonlat = files.flLoadFile(lon_lat, '%', ',')

        self.bearing_list = files.flLoadFile(bearing_list)
        self.speed_list = files.flLoadFile(speed_list)
        self.pressure_list = files.flLoadFile(pressure_list)

        self.bearingRate_list = files.flLoadFile(bearing_rate_list)
        self.speedRate_list = files.flLoadFile(speed_rate_list)
        self.pressureRate_list = files.flLoadFile(pressure_rate_list)
        self.cellNum = cellNum
        self.extractParameters(cellNum)
Beispiel #9
0
    def __init__(self, lon_lat, bearing_list, speed_list, pressure_list,
                 bearing_rate_list, speed_rate_list, pressure_rate_list,
                 cellNum, gridLimit, gridSpace):
        """
        initialize the data needed for the plots including
        longitudes & latitudes of cyclone origins,
        cyclone bearings,
        cyclone speeds,
        cyclone pressures,
        as well as calculating and storing the
        cyclone index counts for a specific inputted cell number
        """
        self.gridLimit = gridLimit
        self.gridSpace = gridSpace
        self.lonlat = files.flLoadFile(lon_lat, '%', ',')

        self.bearing_list = files.flLoadFile(bearing_list)
        self.speed_list = files.flLoadFile(speed_list)
        self.pressure_list = files.flLoadFile(pressure_list)

        self.bearingRate_list = files.flLoadFile(bearing_rate_list)
        self.speedRate_list = files.flLoadFile(speed_rate_list)
        self.pressureRate_list = files.flLoadFile(pressure_rate_list)
        self.cellNum = cellNum
        self.extractParameters(cellNum)
Beispiel #10
0
    def calc(self):
        """
        Calculate the frequency of TC events in a pre-defined domain, based
        on the input dataset and the full range of years contained in the
        :attr:`origin_year` file.

        The :attr:`origin_year` file is created in
        :class:`DataProcess.processData` and restricts the range to a
        user-selected range of years.
        """

        logger.info("Calculating annual frequency of TC events")
        origin_year = np.array(flLoadFile(
            pjoin(self.outputPath, 'process', 'origin_year'), '%', ','),
                               dtype='int')

        origin_lon_lat = flLoadFile(
            pjoin(self.outputPath, 'process', 'origin_lon_lat'), '%', ',')
        origin_lon = origin_lon_lat[:, 0]
        origin_lat = origin_lon_lat[:, 1]
        min_year = origin_year.min()
        # Skip last year from average since may contain only partial year record
        max_year = origin_year.max() - 1

        freq_count = np.zeros(3000)

        for year in range(min_year, max_year + 1):
            freq_count[year] = sum((origin_year == year) & \
                                 (origin_lon > self.tg_domain['xMin']) & \
                                 (origin_lon < self.tg_domain['xMax']) & \
                                 (origin_lat > self.tg_domain['yMin']) & \
                                 (origin_lat < self.tg_domain['yMax']))

        freq = np.mean(freq_count[min_year:max_year + 1])
        freq = np.round(freq * 100) / 100

        fname = pjoin(self.outputPath, 'process', 'region_frequency')
        data = np.array([
            np.arange(min_year, max_year + 1),
            freq_count[min_year:max_year + 1]
        ])
        header = "Year,count"
        np.savetxt(fname, data.T, fmt="%d", delimiter=",", header=header)

        return freq
Beispiel #11
0
    def calc(self):
        """
        Calculate the frequency of TC events in a pre-defined domain, based 
        on the input dataset and the full range of years contained in the 
        :attr:`origin_year` file.

        The :attr:`origin_year` file is created in 
        :class:`DataProcess.processData` and restricts the range to a 
        user-selected range of years.
        """

        logger.info("Calculating annual frequency of TC events")
        origin_year = np.array(flLoadFile(pjoin(self.outputPath,
                                                'process', 'origin_year'),
                                                '%', ','), dtype='int')
        
        origin_lon_lat = flLoadFile(pjoin(self.outputPath,
                                          'process', 'origin_lon_lat'),
                                          '%', ',')
        origin_lon = origin_lon_lat[:, 0]
        origin_lat = origin_lon_lat[:, 1]
        min_year = origin_year.min()
        # Skip last year from average since may contain only partial year record
        max_year = origin_year.max() - 1

        freq_count = np.zeros(3000)
        
        for year in range(min_year, max_year + 1):
            freq_count[year] = sum((origin_year == year) & \
                                 (origin_lon > self.tg_domain['xMin']) & \
                                 (origin_lon < self.tg_domain['xMax']) & \
                                 (origin_lat > self.tg_domain['yMin']) & \
                                 (origin_lat < self.tg_domain['yMax']))

        freq = np.mean(freq_count[min_year:max_year + 1])
        freq = np.round(freq*100)/100

        fname = pjoin(self.outputPath, 'process', 'region_frequency')
        X = np.array([np.arange(min_year,max_year + 1), 
                      freq_count[min_year:max_year + 1]])
        header = "Year,count"
        np.savetxt(fname, X.T, fmt="%d", delimiter=",", header=header)

        return freq
Beispiel #12
0
    def setKDEOrigins(self, kdeOriginX=None, kdeOriginY=None, kdeOriginZ=None,
                      outputPath=None):
        """
        Set kernel density estimation origin parameters.

        :param kdeOriginX: x coordinates of kde result generated
                           from :class:`KDEOrigin`
        :param kdeOriginY: y coordinates of kde result generated
                           from :class:`KDEOrigin`
        :param kdeOriginZ: z coordinates of kde result generated
                           from :class:`KDEOrigin`
        :param outputPath: Path to output folder to load PDF file.

        :type  kdeOriginX: str or :class:`numpy.ndarray`
        :type  kdeOriginY: str or :class:`numpy.ndarray`
        :type  kdeOriginZ: str or :class:`numpy.ndarray`
        :type  outputPath: str
        
        """
        if outputPath:
            try:
                self.x, self.y. self.z = grdRead(os.path.join(self.outputPath,
                                                              'originPDF.txt'))
            except IOError:
                self.logger.critical('Error! Files relating to KDE of cyclone origins does not exist. Execute KDE of cyclone origins first.')
                raise
            self._calculateCDF() #calculate CDF of (x,Px) and (y,Py)
        elif type(kdeOriginZ) == str:
            try:
                self.x = flLoadFile(kdeOriginX)
                self.y = flLoadFile(kdeOriginY)
                self.z = flLoadFile(kdeOriginZ)
            except IOError:
                self.logger.critical('Error! Files relating to CDF of cyclone parameters do not exist. Generate KDE of cyclone parameters first.')
                raise
            self._calculateCDF()  # calculate CDF of (x,Px) and (y,Py)
        elif type(kdeOriginZ) == np.ndarray:
            self.x, self.y, self.z = grdRead(kdeOriginZ)
            self._calculateCDF()  # calculate CDF of (x,Px) and (y,Py)
        else:
            self.logger.error("No input arguments")
            raise
Beispiel #13
0
    def __init__(self, configFile):
        """
        Read configuration settings, load station data and set up
        output recarrays.

        :param str configFile: path to a configuration file.
        """

        config = ConfigParser()
        config.read(configFile)

        self.meta = False

        stnFile = config.get('Timeseries', 'LocationFile')
        self.outputPath = pjoin(config.get('Output', 'Path'),
                                'process', 'timeseries')

        self.maxfile = pjoin(config.get('Output', 'Path'),
                             'process', 'maxima.csv')
        self.minfile = pjoin(config.get('Output', 'Path'),
                             'process', 'minima.csv')


        log.info(f"Loading timeseries stations from {stnFile}")
        log.debug(f"Timeseries data will be written into {self.outputPath}")
        self.stations = []
        if stnFile.endswith("shp"):
            try:
                key_name = config.get('Timeseries', 'StationID')
            except NoOptionError:
                key_name = None

            vertices = shpGetVertices(stnFile, key_name=key_name)

            for stn in list(vertices.keys()):
                lat = vertices[stn][0][1]
                lon = vertices[stn][0][0]
                lon = np.where(lon < 0., lon + 360., lon)
                self.stations.append(Station(stn, lon, lat))


        else:
            stndata = flLoadFile(stnFile, delimiter=',')
            # If there are more than 3 columns, save the additional
            # columns as 'metadata'
            if stndata.shape[1] > 3:
                self.metadata = stndata[:, 3:]
                self.meta = True
            stnid = stndata[:, 0]
            stnlon = stndata[:, 1].astype(float)
            stnlat = stndata[:, 2].astype(float)
            for sid, lon, lat in zip(stnid, stnlon, stnlat):
                self.stations.append(Station(sid, lon, lat))
        log.info(f"There are {len(self.stations)} stations that will collect timeseries data")
Beispiel #14
0
    def __init__(self, configFile):
        """
        Read configuration settings, load station data and set up
        output recarrays.
        
        :param str configFile: path to a configuration file.
        """

        config = ConfigParser()
        config.read(configFile)

        self.meta = False

        stnFile = config.get('Timeseries', 'StationFile')
        self.outputPath = pjoin(config.get('Output', 'Path'), 
                                    'process', 'timeseries')

        self.maxfile = pjoin(config.get('Output', 'Path'), 
                                    'process', 'maxima.csv')
        self.minfile = pjoin(config.get('Output', 'Path'), 
                                    'process', 'minima.csv')


        log.debug("Loading stations from %s"%stnFile)
        log.debug("Timeseries data will be written into %s"%self.outputPath)
        self.stations = []
        if stnFile.endswith("shp"):
            try:
                key_name = config.get('Timeseries', 'StationID')
            except NoOptionError:
                key_name = None
                
            vertices = shpGetVertices(stnFile, key_name=key_name)

            for stn in vertices.keys():
                self.stations.append(Station(stn, vertices[stn][0][0], 
                                                  vertices[stn][0][1]))

        
        else:
            stndata = flLoadFile(stnFile, delimiter=',')
            # If there are more than 3 columns, save the additional 
            # columns as 'metadata'
            if stndata.shape[1] > 3:
                self.metadata = stndata[:, 3:]
                self.meta = True
            stnid = stndata[:, 0]
            stnlon = stndata[:, 1].astype(float)
            stnlat = stndata[:, 2].astype(float)
            for sid, lon, lat in zip(stnid, stnlon, stnlat):
                self.stations.append(Station(sid, lon, lat))
Beispiel #15
0
    def setParameters(self, cdfParameters):
        """Set parameters
        """

        if type(cdfParameters) == str:
            try:
                self.xacy = flLoadFile(cdfParameters)
            except IOError:
                self.logger.exception('Error! Files relating to cdf of cyclone parameters does not exist, please generate KDE of cyclone parameters first.')
                raise
        else:
            self.xacy = cdfParameters

        return self
Beispiel #16
0
    def generateGenesisDateCDF(self,
                               genDays,
                               lonLat,
                               bw=None,
                               genesisKDE=None):
        """
        Calculate the PDF of genesis day using KDEs.
        Since the data is periodic, we use a simple method to include
        the periodicity in estimating the PDF. We prepend and append
        the data to itself, then use the central third of the PDF and
        multiply by three to obtain the required PDF. Probably not
        quite exact, but it should be sufficient for our purposes.

        :param str genDays: Name of file containing genesis days
                            (as day of year).
        :param lonLat: Array of genesis longitudes and latitudes.
        :param float bw: Optional. Bandwidth of the KDE to use.
        :param str genesisKDE: Optional. File name to save resulting CDF to.
        :type  lonLat: :class:`numpy.ndarray`

        :returns: :class:`numpy.ndarray` containing the days, the PDF and CDF
                  of the genesis days.
        """

        data = flLoadFile(genDays)
        days = np.arange(1, 366)
        ndays = np.concatenate([days - 365, days, days + 365])
        ndata = np.concatenate([data - 365, data, data + 365])

        if bw is None:
            bw = KPDF.UPDFOptimumBandwidth(ndata)

        try:
            kdeMethod = getattr(KPDF, "UPDF%s" % self.kdeType)
        except AttributeError:
            LOG.exception(("Invalid input on option: "
                           "KDE method UPDF%s does not exist"), self.kdeType)
            raise
        pdf = kdeMethod(ndata, ndays, bw)
        # Actual PDF to return
        apdf = 3.0 * pdf[365:730]
        cy = stats.cdf(days, apdf)
        if genesisKDE is None:
            return np.transpose(np.array(np.concatenate([days, apdf, cy])))
        else:
            # Assume both kdeParameters and cdfParameters are defined as files:
            LOG.debug("Saving KDE and CDF data to files")
            flSaveFile(genesisKDE, np.transpose(np.array([days, cy])))
Beispiel #17
0
    def __init__(self, cdfParameters=None):
        """Initialize the data needed for the plots including CDF of
        cyclone parameters
        """

        self.logger = logging.getLogger()

        if type(cdfParameters) == str:
            try:
                self.xacy = flLoadFile(cdfParameters)
            except IOError:
                self.logger.exception('Error! Files relating to cdf of cyclone parameters does not exist, please generate KDE of cyclone parameters first.')
                raise
        elif type(cdfParameters) == ndarray:
            self.xacy = cdfParameters
        else:
            self.xacy = array([])
Beispiel #18
0
    def __init__(self, cdfParameters=None):
        """Initialize the data needed for the plots including CDF of
        cyclone parameters
        """

        self.logger = logging.getLogger()

        if type(cdfParameters) == str:
            try:
                self.xacy = flLoadFile(cdfParameters)
            except IOError:
                self.logger.exception('Error! Files relating to cdf of cyclone parameters does not exist, please generate KDE of cyclone parameters first.')
                raise
        elif type(cdfParameters) == np.ndarray:
            self.xacy = cdfParameters
        else:
            self.xacy = np.array([])
Beispiel #19
0
    def __init__(self,
                 configFile,
                 gridLimit,
                 kdeStep,
                 lonLat=None,
                 progressbar=None):
        """

        """
        self.progressbar = progressbar
        LOGGER.info("Initialising KDEOrigin")
        self.x = np.arange(gridLimit['xMin'], gridLimit['xMax'], kdeStep)
        self.y = np.arange(gridLimit['yMax'], gridLimit['yMin'], -kdeStep)

        self.kdeStep = kdeStep
        self.kde = None
        self.pdf = None
        self.cz = None

        self.configFile = configFile
        self.config = ConfigParser()
        self.config.read(configFile)

        if lonLat is None:
            # Load the data from file:
            self.outputPath = self.config.get('Output', 'Path')
            self.processPath = pjoin(self.outputPath, 'process')
            LOGGER.debug("Loading " + pjoin(self.processPath, 'init_lon_lat'))
            ll = flLoadFile(pjoin(self.processPath, 'init_lon_lat'), '%', ',')
            self.lonLat = ll[:, 0:2]
        else:
            self.lonLat = lonLat[:, 0:2]

        ii = np.where((self.lonLat[:, 0] >= gridLimit['xMin'])
                      & (self.lonLat[:, 0] <= gridLimit['xMax'])
                      & (self.lonLat[:, 1] >= gridLimit['yMin'])
                      & (self.lonLat[:, 1] <= gridLimit['yMax']))

        self.lonLat = self.lonLat[ii]

        self.bw = getOriginBandwidth(self.lonLat)
        LOGGER.info("Bandwidth: %s", repr(self.bw))
Beispiel #20
0
    def calcDomainFromFile(self):
        """
        Calculate track generation domain, using a file as the input source

        :rtype: dict
        :returns: the domain where the tracks will be generated.
                  The :class:`dict` should contain the keys :attr:`xMin`,
                  :attr:`xMax`, :attr:`yMin` and :attr:`yMax`. The *x*
                  variable bounds the longitude and the *y* variable
                  bounds the latitude.

        """

        # Load tracks from file
        cyclone_tracks = flLoadFile(
            pjoin(self.outputPath, 'process', 'cyclone_tracks'), '%', ',')
        tg_domain = self.calc(cyclone_tracks[:, 0], cyclone_tracks[:, 1],
                              cyclone_tracks[:, 2])

        return tg_domain
Beispiel #21
0
    def calcDomainFromFile(self):
        """
        Calculate track generation domain, using a file as the input source

        :rtype  tg_domain: :class:`dict`
        :return tg_domain: the domain where the tracks will be generated.
                           The :class:`dict` should contain the keys :attr:`xMin`,
                           :attr:`xMax`, :attr:`yMin` and :attr:`yMax`. The *x*
                           variable bounds the longitude and the *y* variable 
                           bounds the latitude.
        """

        # Load tracks from file
        cyclone_tracks = flLoadFile(pjoin(self.outputPath, 'process', 
                                          'cyclone_tracks'),
                                          '%', ',')
        tg_domain = self._calc(cyclone_tracks[:, 0],
                               cyclone_tracks[:, 1],
                               cyclone_tracks[:, 2])

        return tg_domain
Beispiel #22
0
    def setParameters(self, cdfParameters):
        """
        Set parameters.

        :param cdfParameters: Name of a file containing the CDF of
                              a parameter, or the actual CDF values.
        :type  cdfParamters: str or :class:`numpy.ndarray`

        :raises IOError: If the CDF files do not exist.
        """

        if type(cdfParameters) == str:
            try:
                self.xacy = flLoadFile(cdfParameters)
            except IOError:
                self.logger.exception('Error! Files relating to cdf of cyclone parameters does not exist, please generate KDE of cyclone parameters first.')
                raise
        else:
            self.xacy = cdfParameters

        return self
Beispiel #23
0
    def setParameters(self, cdfParameters):
        """
        Set parameters.

        :param cdfParameters: Name of a file containing the CDF of
                              a parameter, or the actual CDF values.
        :type  cdfParamters: str or :class:`numpy.ndarray`

        :raises IOError: If the CDF files do not exist. 
        """

        if type(cdfParameters) == str:
            try:
                self.xacy = flLoadFile(cdfParameters)
            except IOError:
                self.logger.exception(
                    "Error! Files relating to cdf of cyclone parameters does not exist, please generate KDE of cyclone parameters first."
                )
                raise
        else:
            self.xacy = cdfParameters

        return self
Beispiel #24
0
def main():
    import Utilities.files as files
    from os.path import join as pjoin, dirname, normpath
    baseDir = normpath(pjoin(dirname(__file__), '..'))
    inputPath = pjoin(baseDir, 'output', 'process')

    saveWindProfilesFigure(-12., 130., 100700., 95000., 30., 1.6)

    pressureRates = files.flLoadFile(pjoin(inputPath, 'pressure_rate'))
    pressures = files.flLoadFile(pjoin(inputPath, 'all_pressure'))
    savePressureFigure(pressures, pressureRates)

    speedRates = files.flLoadFile(pjoin(inputPath, 'speed_rate'))
    speeds = files.flLoadFile(pjoin(inputPath, 'all_speed'))
    saveSpeedFigures(speeds, speedRates)

    bearingRates = files.flLoadFile(pjoin(inputPath, 'bearing_rate'))
    bearings = files.flLoadFile(pjoin(inputPath, 'all_bearing'))
    saveBearingFigure(bearings, bearingRates)

    freq = files.flLoadFile(pjoin(inputPath, 'frequency'))
    saveFrequencyFigure(np.array(freq[:,0],int), freq[:,1])
Beispiel #25
0
def main():
    import Utilities.files as files
    from os.path import join as pjoin, dirname, normpath
    baseDir = normpath(pjoin(dirname(__file__), '..'))
    inputPath = pjoin(baseDir, 'output', 'process')

    saveWindProfilesFigure(-12., 130., 100700., 95000., 30., 1.6)

    pressureRates = files.flLoadFile(pjoin(inputPath, 'pressure_rate'))
    pressures = files.flLoadFile(pjoin(inputPath, 'all_pressure'))
    savePressureFigure(pressures, pressureRates)

    speedRates = files.flLoadFile(pjoin(inputPath, 'speed_rate'))
    speeds = files.flLoadFile(pjoin(inputPath, 'all_speed'))
    saveSpeedFigures(speeds, speedRates)

    bearingRates = files.flLoadFile(pjoin(inputPath, 'bearing_rate'))
    bearings = files.flLoadFile(pjoin(inputPath, 'all_bearing'))
    saveBearingFigure(bearings, bearingRates)

    freq = files.flLoadFile(pjoin(inputPath, 'frequency'))
    saveFrequencyFigure(np.array(freq[:, 0], int), freq[:, 1])
Beispiel #26
0
    def allDistributions(self, lonLat, parameterList, parameterName=None,
                         kdeStep=0.1, angular=False, periodic=False,
                         plotParam=False):
        """
        Calculate a distribution for each individual cell and store in a
        file or return the distribution.

        :param lonLat: The longitude/latitude of all observations in
                       the model domain. If a string is given, then
                       it is the path to a file containing the
                       longitude/latitude information. If an array
                       is given, then it should be a 2-d array
                       containing the data values.
        :type  lonLat: str or :class:`numpy.ndarray`
        :param parameterList: Parameter values. If a string is given,
                              then it is the path to a file containing
                              the values. If an array is passed, then it
                              should hold the parameter values.
        :type  parameterList: str or :class:`numpy.ndarray`
        :param str parameterName: Optional. If given, then the
                                  cell distributions will be saved to a
                                  file with this name. If absent,
                                  the distribution values are returned.
        :param kdeStep: Increment of the ordinate values at which
                        the distributions will be calculated.
        :type  kdeStep: float, default=`0.1`
        :param angular: Does the data represent an angular measure
                        (e.g. bearing).
        :type  angular: boolean, default=``False``
        :param periodic: Does the data represent some form of periodic
                         data (e.g. day of year). If given, it should
                         be the period of the data (e.g. for annual data,
                         ``periodic=365``).
        :type  periodic: boolean or float, default=``False``
        :param boolean plotParam: Plot the parameters. Default is ``False``.

        :returns: If no ``parameterName`` is given returns ``None``
                  (data are saved to file), otherwise
                  :class:`numpy.ndarray`.


        """
        if parameterName:
            self.logger.debug("Running allDistributions for %s",
                              parameterName)
        else:
            self.logger.debug("Running allDistributions")

        if isinstance(lonLat, str):
            self.logger.debug("Loading lat/lon data from file")
            self.lonLat = np.array(flLoadFile(lonLat, delimiter=','))
        else:
            self.lonLat = lonLat

        if isinstance(parameterList, str):
            self.logger.debug("Loading parameter data from file: %s",
                              parameterList)
            self.pList = np.array(flLoadFile(parameterList))
        else:
            self.pList = parameterList

        self.pName = parameterName

        if len(self.pList) != len(self.lonLat):
            errmsg = ("Parameter data and "
                      "Lon/Lat data are not the same length "
                      "for {}.".format(parameterName))
            self.logger.critical(errmsg)
            raise IndexError(errmsg)

        maxCellNum = stats.maxCellNum(self.gridLimit, self.gridSpace)

        # Writing CDF dataset for all individual cell number into files
        self.logger.debug(("Writing CDF dataset for all individual "
                           "cells into files"))

        for cellNum in range(0, maxCellNum + 1):
            self.logger.debug("Processing cell number %i", cellNum)

            # Generate cyclone parameter data for the cell number
            self.extractParameter(cellNum)

            # Estimate cyclone parameter data using KDE
            # The returned array contains the grid, the PDF and the CDF
            cdf = self.kdeParameter.generateKDE(self.parameter, kdeStep,
                                                angular=angular,
                                                periodic=periodic)
            if plotParam:
                self._plotParameter(cellNum, kdeStep)
            self.logger.debug(('size of parameter array = %d: '
                               'size of cdf array = %d'), 
                              self.parameter.size, cdf.size)

            cellNumlist = []
            for i in range(len(cdf)):
                cellNumlist.append(cellNum)
            if cellNum == 0:
                results = np.transpose(np.array([cellNumlist,
                                                 cdf[:, 0], cdf[:, 2]]))
            else:
                self.logger.debug('size of results = %s', str(results.size))
                results = np.concatenate((results,
                                          np.transpose(np.array([cellNumlist,
                                                                 cdf[:, 0],
                                                                 cdf[:, 2]]))))

        if parameterName == None:
            self.logger.debug(("Returning CDF dataset for all "
                               "individual cell numbers"))
            return results
        else:
            cdfHeader = "Cell_Number, CDF_" + self.pName + "_x, CDF_" + \
                        self.pName + "_y"
            allCellCdfOutput = pjoin(self.outputPath, 'process',
                                     'all_cell_cdf_' + self.pName)

            args = {"filename":allCellCdfOutput, "data":results,
                    "header":cdfHeader, "delimiter":",", "fmt":"%f"}

            self.logger.debug(("Writing CDF dataset for all individual "
                               "cell numbers into files"))
            flSaveFile(**args)

            # Save to netcdf too

            filename = allCellCdfOutput + '.nc'

            ncdf = Dataset(filename, 'w')

            ncdf.createDimension('cell', len(results[:, 0]))
            cell = ncdf.createVariable('cell', 'i', ('cell',))
            cell[:] = results[:, 0]

            x = ncdf.createVariable('x', 'f', ('cell',))
            x[:] = results[:, 1]

            y = ncdf.createVariable('CDF', 'f', ('cell',))
            y[:] = results[:, 2]

            ncdf.close()
Beispiel #27
0
    def allDistributions(self, lonLat, parameterList, parameterName=None,
                         kdeStep=0.1, angular=False, periodic=False,
                         plotParam=False):
        """
        Calculate a distribution for each individual cell and store in a
        file or return
        """
        if parameterName:
            self.logger.debug("Running allDistributions for %s"%parameterName)
        else:
            self.logger.debug("Running allDistributions")

        if type(lonLat) is str:
            self.logger.debug("Loading lat/lon data from file")
            self.lonLat = np.array(flLoadFile(lonLat, delimiter=','))
        else:
            self.lonLat = lonLat

        if type(parameterList) is str:
            self.logger.debug("Loading parameter data from file: %s" %
                          parameterList)
            self.pList = np.array(flLoadFile(parameterList))
        else:
            self.pList = parameterList

        self.pName = parameterName

        maxCellNum = stats.maxCellNum(self.gridLimit, self.gridSpace)

        # Writing CDF dataset for all individual cell number into files
        self.logger.debug("Writing CDF dataset for all individual cells into files")

        for cellNum in xrange(0, maxCellNum + 1):
            self.logger.debug("Processing cell number %i"%cellNum)

            # Generate cyclone parameter data for the cell number
            self.extractParameter(cellNum)

            # Estimate cyclone parameter data using KDE
            # The returned array contains the grid, the PDF and the CDF
            cdf = self.kdeParameter.generateKDE(self.parameter, kdeStep,
                                                angular=angular,
                                                periodic=periodic)
            if plotParam:
                self._plotParameter(cellNum, kdeStep)
            self.logger.debug('size of parameter array = %d: size of cdf array = %d'
                          % (self.parameter.size,cdf.size))

            cellNumlist = []
            for i in range(len(cdf)):
                cellNumlist.append(cellNum)
            if cellNum == 0:
                results = np.transpose(np.array([cellNumlist, cdf[:,0], cdf[:,2]]))
            else:
                self.logger.debug('size of results array = %s'%str(results.size))
                results = np.concatenate((results, np.transpose(np.array([cellNumlist,
                                                                          cdf[:,0],
                                                                     cdf[:,2]]))))

        if parameterName == None:
            self.logger.debug("Returning CDF dataset for all individual cell numbers")
            return results
        else:
            cdfHeader = "Cell_Number, CDF_" + self.pName + "_x, CDF_" + \
                        self.pName + "_y"
            allCellCdfOutput = pjoin(self.outputPath, 'process',
                                     'all_cell_cdf_' + self.pName)
                                     
            args = {"filename":allCellCdfOutput, "data":results,
                    "header":cdfHeader, "delimiter":",", "fmt":"%f"}
                    
            self.logger.debug("Writing CDF dataset for all individual cell numbers into files")
            flSaveFile(**args)

            # Save to netcdf too

            filename = allCellCdfOutput + '.nc'

            ncdf = Dataset(filename, 'w')
            
            ncdf.createDimension('cell', len(results[:,0]))
            cell = ncdf.createVariable('cell', 'i', ('cell',))
            cell[:] = results[:,0]
            
            x = ncdf.createVariable('x', 'f', ('cell',))
            x[:] = results[:,1]
            
            y = ncdf.createVariable('CDF', 'f', ('cell',))
            y[:] = results[:,2]
            
            ncdf.close()
Beispiel #28
0
    def generateKDE(self, parameters, kdeStep, kdeParameters=None,
                    cdfParameters=None, angular=False, periodic=False,
                    missingValue=sys.maxint):
        """
        Generate a PDF and CDF for a given parameter set using the
        method of kernel density estimators.
        Optionally return the PDF and CDF as an array, or write both
        to separate files.
        """

        self.logger.debug("Running generateKDE")
        if type(parameters) is str:
            self.parameters = stats.statRemoveNum(flLoadFile(parameters, '%', ','), missingValue)
        else:
            if parameters.size <= 1:
                self.logger.error("Insufficient members in parameter list")
                raise IndexError, "Insufficient members in parameter list"

            self.parameters = stats.statRemoveNum(parameters, missingValue)

        if angular:
            xmin = 0.0
            xmax = 360.0
        elif periodic:
            xmin = 0.0
            xmax = periodic
        else:
            xmin = self.parameters.min()
            xmax = self.parameters.max()

        self.logger.debug("xmin=%7.3f, xmax=%7.3f, kdeStep=%7.3f" %
                           (xmin, xmax, kdeStep))
        if periodic:
            x = np.arange(1, periodic + 1, kdeStep)
            self.grid = np.concatenate( [x - periodic, x, x + periodic] )
            self.parameters = np.concatenate([self.parameters - periodic, 
                                              self.parameters, 
                                              self.parameters + periodic])
        else:
            self.grid = np.arange(xmin, xmax, kdeStep)

        if self.grid.size<2:
            self.logger.critical("Grid for CDF generation is a single value")
            self.logger.critical("xmin=%7.3f, xmax=%7.3f, kdeStep=%7.3f" %
                                  (xmin, xmax,kdeStep))
            raise ValueError

        bw = KPDF.UPDFOptimumBandwidth(self.parameters)
        self.pdf = self._generatePDF(self.grid, bw, self.parameters)
        
        if periodic:
            self.pdf = 3.0*self.pdf[(periodic/kdeStep):2*(periodic/kdeStep)]
            self.grid = self.grid[(periodic/kdeStep):2*(periodic/kdeStep)]
            
        self.cy = stats.cdf(self.grid, self.pdf)
        if kdeParameters is None:
            return np.transpose(np.array([self.grid, self.pdf, self.cy]))
        else:
            # Assume both kdeParameters and cdfParameters are defined as files:
            self.logger.debug("Saving KDE and CDF data to files")
            flSaveFile(kdeParameters, np.transpose(np.array([self.grid, self.pdf])))
            flSaveFile(cdfParameters, np.transpose(np.array([self.grid, self.cy])))
Beispiel #29
0
 def testLoadFile(self):
     """Test flLoadFile loads data correctly"""
     data = files.flLoadFile(self.testfile, comments='%', delimiter=',')
     assert_almost_equal(self.lon, data[:, 0])
     assert_almost_equal(self.lat, data[:, 1])
Beispiel #30
0
    def generateKDE(self,
                    parameters,
                    kdeStep,
                    kdeParameters=None,
                    cdfParameters=None,
                    angular=False,
                    periodic=False,
                    missingValue=sys.maxsize):
        """
        Generate a PDF and CDF for a given parameter set using the
        method of kernel density estimators. Optionally return the PDF
        and CDF as an array, or write both to separate files.

        :param parameters: Parameter values. If a string is given,
                           then it is the path to a file containing
                           the values. If an array is passed, then it
                           should hold the parameter values.

        :param kdeStep: Increment of the ordinate values at which
                        the distributions will be calculated.
        :type  kdeStep: float, default=`0.1`
        :param str kdeParameters: Optional. If given, then the
                                  cell distributions will be saved to a
                                  file with this name. If absent,
                                  the distribution values are returned.
        :param str cdfParameters: Optional. If given, then the
                                  cell distributions will be saved to a
                                  file with this name. If absent,
                                  the distribution values are returned.
        :param angular: Does the data represent an angular measure
                        (e.g. bearing).
        :type  angular: boolean, default=``False``
        :param periodic: Does the data represent some form of periodic
                         data (e.g. day of year). If given, it should
                         be the period of the data (e.g. for annual data,
                         ``periodic=365``).
        :type  periodic: boolean or int, default=``False``
        :param missingValue: Missing values have this value (default
                         :attr:`sys.maxint`).

        returns: If ``kdeParameters`` is given, returns ``None``
                  (data are saved to file), otherwise
                  :class:`numpy.ndarray` of the parameter grid, the PDF and CDF.

        """

        LOG.debug("Running generateKDE")
        if type(parameters) is str:
            self.parameters = stats.statRemoveNum(
                flLoadFile(parameters, '%', ','), missingValue)
        else:
            if parameters.size <= 1:
                LOG.error("Insufficient members in parameter list")
                raise IndexError("Insufficient members in parameter list")

            self.parameters = stats.statRemoveNum(parameters, missingValue)

        if angular:
            xmin = 0.0
            xmax = 360.0
        elif periodic:
            xmin = 0.0
            xmax = periodic
        else:
            xmin = self.parameters.min()
            xmax = self.parameters.max()

        LOG.debug("xmin=%7.3f, xmax=%7.3f, kdeStep=%7.3f" %
                  (xmin, xmax, kdeStep))
        if periodic:
            x = np.arange(1, periodic + 1, kdeStep)
            self.grid = np.concatenate([x - periodic, x, x + periodic])
            self.parameters = np.concatenate([
                self.parameters - periodic, self.parameters,
                self.parameters + periodic
            ])
        else:
            self.grid = np.arange(xmin, xmax, kdeStep)

        if self.grid.size < 2:
            LOG.critical("Grid for CDF generation is a single value")
            LOG.critical("xmin=%7.3f, xmax=%7.3f, kdeStep=%7.3f", xmin, xmax,
                         kdeStep)
            raise ValueError

        #bw = KPDF.UPDFOptimumBandwidth(self.parameters)
        bw = stats.bandwidth(self.parameters)
        self.pdf = self._generatePDF(self.grid, bw, self.parameters)

        if periodic:
            idx = int(periodic / kdeStep)
            self.pdf = 3.0 * self.pdf[idx:2 * idx]
            self.grid = self.grid[idx:2 * idx]

        self.cy = stats.cdf(self.grid, self.pdf)
        if kdeParameters is None:
            return np.transpose(np.array([self.grid, self.pdf, self.cy]))
        else:
            # Assume both kdeParameters and cdfParameters are defined as files:
            LOG.debug("Saving KDE and CDF data to files")
            flSaveFile(kdeParameters,
                       np.transpose(np.array([self.grid, self.pdf])))
            flSaveFile(cdfParameters,
                       np.transpose(np.array([self.grid, self.cy])))
    def allDistributions(self, lonLat, parameterList, parameterName=None,
                         kdeStep=0.1, angular=False, periodic=False,
                         plotParam=False):
        """
        Calculate a distribution for each individual cell and store in a
        file or return the distribution.

        :param lonLat: The longitude/latitude of all observations in
                       the model domain. If a string is given, then
                       it is the path to a file containing the
                       longitude/latitude information. If an array
                       is given, then it should be a 2-d array
                       containing the data values. 
        :type  lonLat: str or :class:`numpy.ndarray`
        :param parameterList: Parameter values. If a string is given,
                              then it is the path to a file containing
                              the values. If an array is passed, then it
                              should hold the parameter values. 
        :type  parameterList: str or :class:`numpy.ndarray`
        :param str parameterName: Optional. If given, then the
                                  cell distributions will be saved to a
                                  file with this name. If absent,
                                  the distribution values are returned.
        :param kdeStep: Increment of the ordinate values at which
                        the distributions will be calculated.
        :type  kdeStep: float, default=`0.1`
        :param angular: Does the data represent an angular measure
                        (e.g. bearing).
        :type  angular: boolean, default=``False``
        :param periodic: Does the data represent some form of periodic
                         data (e.g. day of year). If given, it should
                         be the period of the data (e.g. for annual data,
                         ``periodic=365``).
        :type  periodic: boolean or float, default=``False``
        :param boolean plotParam: Plot the parameters. Default is ``False``.

        :returns: If no ``parameterName`` is given returns ``None``
                  (data are saved to file), otherwise
                  :class:`numpy.ndarray`.

        
        """
        if parameterName:
            self.logger.debug("Running allDistributions for %s"%parameterName)
        else:
            self.logger.debug("Running allDistributions")

        if type(lonLat) is str:
            self.logger.debug("Loading lat/lon data from file")
            self.lonLat = np.array(flLoadFile(lonLat, delimiter=','))
        else:
            self.lonLat = lonLat

        if type(parameterList) is str:
            self.logger.debug("Loading parameter data from file: %s" %
                          parameterList)
            self.pList = np.array(flLoadFile(parameterList))
        else:
            self.pList = parameterList

        self.pName = parameterName

        maxCellNum = stats.maxCellNum(self.gridLimit, self.gridSpace)

        # Writing CDF dataset for all individual cell number into files
        self.logger.debug("Writing CDF dataset for all individual cells into files")

        for cellNum in xrange(0, maxCellNum + 1):
            self.logger.debug("Processing cell number %i"%cellNum)

            # Generate cyclone parameter data for the cell number
            self.extractParameter(cellNum)

            # Estimate cyclone parameter data using KDE
            # The returned array contains the grid, the PDF and the CDF
            cdf = self.kdeParameter.generateKDE(self.parameter, kdeStep,
                                                angular=angular,
                                                periodic=periodic)
            if plotParam:
                self._plotParameter(cellNum, kdeStep)
            self.logger.debug('size of parameter array = %d: size of cdf array = %d'
                          % (self.parameter.size,cdf.size))

            cellNumlist = []
            for i in range(len(cdf)):
                cellNumlist.append(cellNum)
            if cellNum == 0:
                results = np.transpose(np.array([cellNumlist, cdf[:,0], cdf[:,2]]))
            else:
                self.logger.debug('size of results array = %s'%str(results.size))
                results = np.concatenate((results, np.transpose(np.array([cellNumlist,
                                                                          cdf[:,0],
                                                                     cdf[:,2]]))))

        if parameterName == None:
            self.logger.debug("Returning CDF dataset for all individual cell numbers")
            return results
        else:
            cdfHeader = "Cell_Number, CDF_" + self.pName + "_x, CDF_" + \
                        self.pName + "_y"
            allCellCdfOutput = pjoin(self.outputPath, 'process',
                                     'all_cell_cdf_' + self.pName)

            args = {"filename":allCellCdfOutput, "data":results,
                    "header":cdfHeader, "delimiter":",", "fmt":"%f"}

            self.logger.debug("Writing CDF dataset for all individual cell numbers into files")
            flSaveFile(**args)

            # Save to netcdf too

            filename = allCellCdfOutput + '.nc'

            ncdf = Dataset(filename, 'w')

            ncdf.createDimension('cell', len(results[:,0]))
            cell = ncdf.createVariable('cell', 'i', ('cell',))
            cell[:] = results[:,0]

            x = ncdf.createVariable('x', 'f', ('cell',))
            x[:] = results[:,1]

            y = ncdf.createVariable('CDF', 'f', ('cell',))
            y[:] = results[:,2]

            ncdf.close()
Beispiel #32
0
    def generateKDE(self, parameters, kdeStep, kdeParameters=None,
                    cdfParameters=None, angular=False, periodic=False,
                    missingValue=sys.maxint):
        """
        Generate a PDF and CDF for a given parameter set using the
        method of kernel density estimators. Optionally return the PDF
        and CDF as an array, or write both to separate files.

        :param parameters: Parameter values. If a string is given,
                           then it is the path to a file containing
                           the values. If an array is passed, then it
                           should hold the parameter values.
        
        :param kdeStep: Increment of the ordinate values at which
                        the distributions will be calculated.
        :type  kdeStep: float, default=`0.1`
        :param str kdeParameters: Optional. If given, then the
                                  cell distributions will be saved to a
                                  file with this name. If absent,
                                  the distribution values are returned.
        :param str cdfParameters: Optional. If given, then the
                                  cell distributions will be saved to a
                                  file with this name. If absent,
                                  the distribution values are returned.
        :param angular: Does the data represent an angular measure
                        (e.g. bearing).
        :type  angular: boolean, default=``False``
        :param periodic: Does the data represent some form of periodic
                         data (e.g. day of year). If given, it should
                         be the period of the data (e.g. for annual data,
                         ``periodic=365``).
        :type  periodic: boolean or float, default=``False``
        :param missingValue: Missing values have this value (default
                         :attr:`sys.maxint`).

        returns: If ``kdeParameters`` is given, returns ``None``
                  (data are saved to file), otherwise
                  :class:`numpy.ndarray` of the parameter grid, the PDF and CDF.
                 
        """

        self.logger.debug("Running generateKDE")
        if type(parameters) is str:
            self.parameters = stats.statRemoveNum(flLoadFile(parameters, '%', ','), missingValue)
        else:
            if parameters.size <= 1:
                self.logger.error("Insufficient members in parameter list")
                raise IndexError, "Insufficient members in parameter list"

            self.parameters = stats.statRemoveNum(parameters, missingValue)

        if angular:
            xmin = 0.0
            xmax = 360.0
        elif periodic:
            xmin = 0.0
            xmax = periodic
        else:
            xmin = self.parameters.min()
            xmax = self.parameters.max()

        self.logger.debug("xmin=%7.3f, xmax=%7.3f, kdeStep=%7.3f" %
                           (xmin, xmax, kdeStep))
        if periodic:
            x = np.arange(1, periodic + 1, kdeStep)
            self.grid = np.concatenate( [x - periodic, x, x + periodic] )
            self.parameters = np.concatenate([self.parameters - periodic,
                                              self.parameters,
                                              self.parameters + periodic])
        else:
            self.grid = np.arange(xmin, xmax, kdeStep)

        if self.grid.size<2:
            self.logger.critical("Grid for CDF generation is a single value")
            self.logger.critical("xmin=%7.3f, xmax=%7.3f, kdeStep=%7.3f" %
                                  (xmin, xmax,kdeStep))
            raise ValueError

        bw = KPDF.UPDFOptimumBandwidth(self.parameters)
        self.pdf = self._generatePDF(self.grid, bw, self.parameters)

        if periodic:
            self.pdf = 3.0*self.pdf[(periodic/kdeStep):2*(periodic/kdeStep)]
            self.grid = self.grid[(periodic/kdeStep):2*(periodic/kdeStep)]

        self.cy = stats.cdf(self.grid, self.pdf)
        if kdeParameters is None:
            return np.transpose(np.array([self.grid, self.pdf, self.cy]))
        else:
            # Assume both kdeParameters and cdfParameters are defined as files:
            self.logger.debug("Saving KDE and CDF data to files")
            flSaveFile(kdeParameters, np.transpose(np.array([self.grid, self.pdf])))
            flSaveFile(cdfParameters, np.transpose(np.array([self.grid, self.cy])))
Beispiel #33
0
def doDataPlotting(configFile):
    """
    Plot the pre-processed input data. Requires the Data Processing step to have
    been executed first (``Actions -- DataProcess = True``)

    :param str configFile: Name of configuration file.

    """

    config = ConfigParser()
    config.read(configFile)

    showProgressBar = config.get('Logging', 'ProgressBar')
    pbar = ProgressBar('Plotting results: ', showProgressBar)

    outputPath = config.get('Output', 'Path')

    statsPlotPath = pjoin(outputPath, 'plots', 'stats')
    processPath = pjoin(outputPath, 'process')

    pRateData = flLoadFile(pjoin(processPath, 'pressure_rate'))
    pAllData = flLoadFile(pjoin(processPath, 'all_pressure'))
    bRateData = flLoadFile(pjoin(processPath, 'bearing_rate'))
    bAllData = flLoadFile(pjoin(processPath, 'all_bearing'))
    sRateData = flLoadFile(pjoin(processPath, 'speed_rate'))
    sAllData = flLoadFile(pjoin(processPath, 'all_speed'))
    freq = flLoadFile(pjoin(processPath, 'frequency'))


    indLonLat = flLoadFile(pjoin(processPath, 'cyclone_tracks'),
                           delimiter=',')
    indicator = indLonLat[:, 0]
    lonData = indLonLat[:, 1]
    latData = indLonLat[:, 2]

    jdayobs = flLoadFile(pjoin(processPath, 'jday_obs'), delimiter=',')
    jdaygenesis = flLoadFile(pjoin(processPath, 'jday_genesis'), delimiter=',')


    from PlotInterface.plotStats import PlotPressure, PlotBearing, \
        PlotSpeed, PlotFrequency, PlotDays, PlotLonLat

    log.info('Plotting pressure data')
    pbar.update(0.05)
    PrsPlot = PlotPressure(statsPlotPath, "png")
    PrsPlot.plotPressure(pAllData)
    PrsPlot.plotPressureRate(pRateData)
    PrsPlot.plotMinPressure(indicator, pAllData)

    #FIXME: To be moved into `PlotPressure` class.
    #plotting.minPressureLat(pAllData, latData)

    log.info('Plotting bearing data')
    pbar.update(0.15)
    BearPlot = PlotBearing(statsPlotPath, "png")
    BearPlot.plotBearing(bAllData)
    BearPlot.plotBearingRate(bRateData)

    log.info('Plotting speed data')
    pbar.update(0.25)
    SpeedPlot = PlotSpeed(statsPlotPath, "png")
    SpeedPlot.plotSpeed(sAllData)
    SpeedPlot.plotSpeedRate(sRateData)

    log.info('Plotting longitude and latitude data')
    pbar.update(0.45)

    # FIXME: To be moved to it's own class in PlotStats
    LLPlot = PlotLonLat(statsPlotPath, "png")
    LLPlot.plotLonLat(lonData, latData, indicator)

    pbar.update(0.65)

    log.info('Plotting frequency data')
    pbar.update(0.85)
    FreqPlot = PlotFrequency(statsPlotPath, "png")
    FreqPlot.plotFrequency(freq[:, 0], freq[:, 1])

    DayPlot = PlotDays(statsPlotPath, "png")
    DayPlot.plotJulianDays(jdayobs, jdaygenesis)

    pbar.update(1.0)
Beispiel #34
0
def doDataPlotting(configFile):
    """
    Plot the data.

    :param str configFile: Name of configuration file.
    
    """
    import matplotlib
    matplotlib.use('Agg')  # Use matplotlib backend

    config = ConfigParser()
    config.read(configFile)

    showProgressBar = config.get('Logging', 'ProgressBar')
    pbar = ProgressBar('Plotting results: ', showProgressBar)

    outputPath = config.get('Output', 'Path')

    statsPlotPath = pjoin(outputPath, 'plots', 'stats')
    processPath = pjoin(outputPath, 'process')

    pRateData = flLoadFile(pjoin(processPath, 'pressure_rate'))
    pAllData = flLoadFile(pjoin(processPath, 'all_pressure'))
    bRateData = flLoadFile(pjoin(processPath, 'bearing_rate'))
    bAllData = flLoadFile(pjoin(processPath, 'all_bearing'))
    sRateData = flLoadFile(pjoin(processPath, 'speed_rate'))
    sAllData = flLoadFile(pjoin(processPath, 'all_speed'))

    indLonLat = flLoadFile(pjoin(processPath, 'cyclone_tracks'),
                           delimiter=',')
    indicator = indLonLat[:, 0]
    lonData = indLonLat[:, 1]
    latData = indLonLat[:, 2]

    from PlotInterface.plotStats import PlotData
    plotting = PlotData(statsPlotPath, "png")

    log.info('Plotting pressure data')
    pbar.update(0.05)

    plotting.plotPressure(pAllData, pRateData)
    plotting.scatterHistogram(
        pAllData[1:], pAllData[:-1], 'prs_scatterHist', allpos=True)
    plotting.scatterHistogram(
        pRateData[1:], pRateData[:-1], 'prsRate_scatterHist')
    plotting.minPressureHist(indicator, pAllData)
    plotting.minPressureLat(pAllData, latData)


    log.info('Plotting bearing data')
    pbar.update(0.15)

    plotting.plotBearing(bAllData, bRateData)

    log.info('Plotting speed data')
    pbar.update(0.25)

    plotting.plotSpeed(sAllData, sRateData)

    log.info('Plotting longitude and lattitude data')
    pbar.update(0.45)

    plotting.plotLonLat(lonData, latData, indicator)

    log.info('Plotting quantiles for pressure, bearing, and speed')
    pbar.update(0.65)

    plotting.quantile(pRateData, "Pressure", "logistic")
    plotting.quantile(bRateData, "Bearing", "logistic")
    plotting.quantile(sRateData, "Speed", "logistic")

    log.info('Plotting frequency data')
    pbar.update(0.85)

    try:
        freq = flLoadFile(pjoin(processPath, 'frequency'))
        years = freq[:, 0]
        frequency = freq[:, 1]
        plotting.plotFrequency(years, frequency)
    except IOError:
        log.warning("No frequency file available - skipping this stage")

    pbar.update(1.0)
Beispiel #35
0
 def testLoadFile(self):
     """Test flLoadFile loads data correctly"""
     data = files.flLoadFile(self.testfile, comments='%', delimiter=',')
     assert_almost_equal(self.lon, data[:, 0])
     assert_almost_equal(self.lat, data[:, 1])
Beispiel #36
0
        pos = (2, -4.8)
        pyplot.text(2, -4.9, r"$r^2=%1.4f$" % r, fontsize=12)

        self.savefig('qqplot_%s' % parameterName)
        #pyplot.rcdefaults()

if __name__=="__main__":
    from os.path import join as pjoin

    configFile = sys.argv[1]
    dataPath = config.cnfGetIniValue(configFile, 'Output', 'Path', os.getcwd())
    inputPath = pjoin(dataPath, 'process')
    outputPath = pjoin(dataPath, 'plots')
    pyplot.rcParams['figure.figsize'] = (7, 12)

    pRateData = files.flLoadFile(pjoin(inputPath, 'pressure_rate'))
    pAllData = files.flLoadFile(pjoin(inputPath, 'all_pressure'))
    bRateData = files.flLoadFile(pjoin(inputPath, 'bearing_rate'))
    bAllData = files.flLoadFile(pjoin(inputPath, 'all_bearing'))
    sRateData = files.flLoadFile(pjoin(inputPath, 'speed_rate'))
    sAllData = files.flLoadFile(pjoin(inputPath, 'all_speed'))
    freq = files.flLoadFile(pjoin(inputPath, 'frequency'))
    years = freq[:, 0]
    frequency = freq[:, 1]

    plotting = PlotData(outputPath, "png")
    plotting.plotPressure(pAllData, pRateData)
    plotting.plotBearing(bAllData, bRateData)
    plotting.plotSpeed(sAllData, sRateData)
    plotting.plotFrequency(years, frequency)
    plotting.plotSpeedBear(sAllData, bAllData)