Пример #1
0
    def read(self,
             var,
             x=None,
             y=None,
             radius=0.,
             tlim=None,
             ylim=None,
             xlim=None,
             missions=None,
             sort=True,
             profile=True):
        """Reads dataset.
        
        PARAMETERS
            var (string) :
                Variable to be read from dataset. It also accepts
                special naming conventions in order to rename the
                original dataset variable and to load alternative
                variables in case of invalid data according to the
                syntax '[new_var_name]:var[|other_var]'.
            x, y (array like, optional) :
                List of zonal and meridional point coordinate of 
                interest.
            radius (float, optional) :
                Search radius in degrees.
            tlim, ylim, xlim  (array like, optional) :
                The temporal, meridional and zonal limits (minimum,
                maximum) for which data will be read.
            missions (array like, optional) :
                List of missions to read data from. If omitted, defaults
                available missions on dataset class intialization.
            sort (boolean optional) :
                If true, sorts the data record in order of ascendant 
                time, latitude and longitude.
            profile (boolean, optional) :
                Sets whether the status is send to screen.
        
        RETURNS
            dat (record array) :
                Record time-series of 'time', 'latitude', 'longitude', 
                selected variable and 'mission'.
        
        """
        t0 = time()
        # Checks input parameters.
        T = self.variables['time'].data
        if var.find(':') >= 0:  # Checks spetial variable syntax
            var_name, var = var.split(':')
        else:
            var_name = var
        if tlim == None:
            tlim = (T.min(), T.max())
        if (x != None) | (y != None):
            x, y = asarray(x), asarray(y)
            if x.size != y.size:
                raise ValueError('Zonal and meridional coordinate dimensions '
                                 'do not match.')
            npoints = x.size
            radius2 = radius**2
        else:
            npoints = 0
            x = y = []
            #
            if ylim == None:
                ylim = (-90., 90.)
            if xlim == None:
                xlim = (0., 360.)
            else:
                # Make sure longitude limits are between 0 and 360.
                xlim = list(lon360(asarray(xlim)))
        if missions == None:
            missions = self.params['missions']

        # First we have to select which files will be loaded, which will
        # depend on the temporal limits given in $t$.
        sel_time = flatnonzero((T >= floor(min(tlim)))
                               & (T <= ceil(max(tlim))))
        N = len(sel_time)

        # Second we will walk through each of the selected time in the dataset
        # and load the correspondant file for the available missions.
        t1 = time()
        if profile:
            s = '\rLoading data...'
            stdout.write(s)
            stdout.flush()
        # Reset important variables
        TIME, LAT, LON, VAR, MISSION = [array([])] * 5
        #
        for i, tm in enumerate(T[sel_time]):
            t2 = time()
            for (mission, dset, fname, cycle,
                 orbit) in self.attributes['time_dataset'][tm]:
                # Skips mission not in missions list.
                if mission not in missions:
                    continue
                # Uncompresses gzipped file and opens NetCDF instance.
                data = self.read_file('%s/%s/%s' %
                                      (self.params['path'], mission, fname))
                # Reads variable from NetCDF file.
                raw_time = self.read_variable(data, 'time')
                raw_lat = self.read_variable(data, 'lat')
                raw_lon = self.read_variable(data, 'lon')
                raw_dat = self.read_variable(data, var)
                # Select relevant data range according to limit parameters
                sel_from_time = ((raw_time >= min(tlim)) &
                                 (raw_time <= max(tlim)))
                if (ylim != None) | (xlim != None):
                    sel_from_limits = ones(data.dimensions['time'], dtype=bool)
                else:
                    sel_from_limits = zeros(data.dimensions['time'],
                                            dtype=bool)
                if ylim != None:
                    sel_from_limits = (sel_from_limits &
                                       ((raw_lat >= min(ylim)) &
                                        (raw_lat <= max(ylim))))
                if xlim != None:
                    sel_from_limits = (sel_from_limits &
                                       ((raw_lon >= min(xlim)) &
                                        (raw_lon <= max(xlim))))
                # Select relevant data according to points and search radius.
                sel_from_radius = zeros(data.dimensions['time'], dtype=bool)
                for xx, yy in zip(x, y):
                    distance2 = ((raw_lat - yy)**2 + (raw_lon - lon360(xx))**2)
                    sel_from_radius = sel_from_radius | (distance2 <= radius2)
                #
                sel_data = flatnonzero(sel_from_time
                                       & (sel_from_limits | sel_from_radius)
                                       & (~isnan(raw_dat)))
                _time = raw_time[sel_data]
                _lat = raw_lat[sel_data]
                _lon = raw_lon[sel_data]
                _dat = raw_dat[sel_data]
                #
                TIME = append(TIME, _time)
                LAT = append(LAT, _lat)
                LON = append(LON, _lon)
                VAR = append(VAR, _dat)
                MISSION = append(MISSION, [mission] * len(sel_data))
                #
                self.close_file(data)
            #
            # Profiling
            if profile:
                s = '\rLoading data... %s ' % (profiler(N, i + 1, t0, t1,
                                                        t2), )
                stdout.write(s)
                stdout.flush()
        #
        if profile:
            stdout.write('\n')
            stdout.flush()

        # Converts the data a structured array
        DAT = rec.fromarrays((TIME, LAT, LON, VAR, MISSION),
                             dtype=[('time', float64), ('latitude', float64),
                                    ('longitude', float64),
                                    (var_name, float64), ('mission', '|S3')])

        # Some data sorting?
        if sort:
            DAT.sort(order=('time', 'latitude', 'longitude'), axis=0)

        return DAT
Пример #2
0
    def read(self, var, x=None, y=None, radius=0., tlim=None, ylim=None,
        xlim=None, missions=None, sort=True, profile=True):
        """Reads dataset.
        
        PARAMETERS
            var (string) :
                Variable to be read from dataset. It also accepts
                special naming conventions in order to rename the
                original dataset variable and to load alternative
                variables in case of invalid data according to the
                syntax '[new_var_name]:var[|other_var]'.
            x, y (array like, optional) :
                List of zonal and meridional point coordinate of 
                interest.
            radius (float, optional) :
                Search radius in degrees.
            tlim, ylim, xlim  (array like, optional) :
                The temporal, meridional and zonal limits (minimum,
                maximum) for which data will be read.
            missions (array like, optional) :
                List of missions to read data from. If omitted, defaults
                available missions on dataset class intialization.
            sort (boolean optional) :
                If true, sorts the data record in order of ascendant 
                time, latitude and longitude.
            profile (boolean, optional) :
                Sets whether the status is send to screen.
        
        RETURNS
            dat (record array) :
                Record time-series of 'time', 'latitude', 'longitude', 
                selected variable and 'mission'.
        
        """
        t0 = time()
        # Checks input parameters.
        T = self.variables['time'].data
        if var.find(':') >= 0:  # Checks spetial variable syntax
            var_name, var = var.split(':')
        else:
            var_name = var
        if tlim == None:
            tlim = (T.min(), T.max())
        if (x != None) | (y != None):
            x, y = asarray(x), asarray(y)
            if x.size != y.size:
                raise ValueError('Zonal and meridional coordinate dimensions '
                    'do not match.')
            npoints = x.size
            radius2 = radius ** 2
        else:
            npoints = 0
            x = y = []
            #
            if ylim == None:
                ylim = (-90., 90.)
            if xlim == None:
                xlim = (0., 360.)
            else:
                # Make sure longitude limits are between 0 and 360.
                xlim = list(lon360(asarray(xlim)))
        if missions == None:
            missions = self.params['missions']
        
        # First we have to select which files will be loaded, which will 
        # depend on the temporal limits given in $t$.
        sel_time = flatnonzero((T >= floor(min(tlim))) & 
            (T <= ceil(max(tlim))))
        N = len(sel_time)
        
        # Second we will walk through each of the selected time in the dataset
        # and load the correspondant file for the available missions.
        t1 = time()
        if profile:
            s = '\rLoading data...'
            stdout.write(s)
            stdout.flush()
        # Reset important variables
        TIME, LAT, LON, VAR, MISSION = [array([])] * 5
        #
        for i, tm in enumerate(T[sel_time]):
            t2 = time()
            for (mission, dset, fname, cycle,
                orbit) in self.attributes['time_dataset'][tm]:
                # Skips mission not in missions list.
                if mission not in missions:
                    continue
                # Uncompresses gzipped file and opens NetCDF instance.
                data = self.read_file('%s/%s/%s' % (self.params['path'], 
                    mission, fname))
                # Reads variable from NetCDF file.
                raw_time = self.read_variable(data, 'time')
                raw_lat = self.read_variable(data, 'lat')
                raw_lon = self.read_variable(data, 'lon')
                raw_dat = self.read_variable(data, var)
                # Select relevant data range according to limit parameters
                sel_from_time = (
                    (raw_time >= min(tlim)) & (raw_time <= max(tlim))
                )
                if (ylim != None) | (xlim !=None):
                    sel_from_limits = ones(data.dimensions['time'], dtype=bool)
                else:
                    sel_from_limits = zeros(data.dimensions['time'],
                        dtype=bool)
                if ylim != None:
                    sel_from_limits = (sel_from_limits & 
                        ((raw_lat >= min(ylim)) & (raw_lat <= max(ylim))))
                if xlim != None:
                    sel_from_limits = (sel_from_limits & 
                        ((raw_lon >= min(xlim)) & (raw_lon <= max(xlim))))
                # Select relevant data according to points and search radius.
                sel_from_radius =  zeros(data.dimensions['time'], dtype=bool)
                for xx, yy in zip(x, y):
                    distance2 = ((raw_lat - yy) ** 2 + 
                        (raw_lon - lon360(xx)) ** 2)
                    sel_from_radius = sel_from_radius | (distance2 <= radius2)
                #
                sel_data = flatnonzero(sel_from_time & 
                    (sel_from_limits | sel_from_radius) & (~isnan(raw_dat)))
                _time = raw_time[sel_data]
                _lat = raw_lat[sel_data]
                _lon = raw_lon[sel_data]
                _dat = raw_dat[sel_data]
                #
                TIME = append(TIME, _time)
                LAT = append(LAT, _lat)
                LON = append(LON, _lon)
                VAR = append(VAR, _dat)
                MISSION = append(MISSION, [mission] * len(sel_data))
                #
                self.close_file(data)
            #
            # Profiling
            if profile:
                s = '\rLoading data... %s ' % (profiler(N, i+1, t0, t1, t2),)
                stdout.write(s)
                stdout.flush()
        #
        if profile:
            stdout.write('\n')
            stdout.flush()

        # Converts the data a structured array
        DAT = rec.fromarrays((TIME, LAT, LON, VAR, MISSION), 
            dtype=[('time', float64), ('latitude', float64), 
            ('longitude', float64), (var_name, float64), ('mission', '|S3')])
        
        # Some data sorting?
        if sort:
            DAT.sort(order=('time', 'latitude', 'longitude'), axis=0)
        
        return DAT
Пример #3
0
    def read(self, x=None, y=None, radius=0., tlim=None, ylim=None, xlim=None,
            missions=None, sort=True, profile=True):
        """Reads dataset.
        
        PARAMETERS
            x, y (array like, optional) :
                List of zonal and meridional point coordinate of 
                interest.
            radius (float, optional) :
                Search radius in degrees.
            tlim, ylim, xlim  (array like, optional) :
                The temporal, meridional and zonal limits (minimum,
                maximum) for which data will be read.
            missions (array like, optional) :
                List of missions to read data from. If omitted, defaults
                available missions on dataset class intialization.
            sort (boolean optional) :
                If true, sorts the data record in order of ascendant 
                time, latitude and longitude.
            profile (boolean, optional) :
                Sets whether the status is send to screen.
        
        RETURNS
            dat (record array) :
                Record time-series of 'time', 'latitude', 'longitude', 
                selected variable and 'mission'.
        
        """
        t0 = time()
        # Checks input parameters.
        T = self.variables['time'].data
        if tlim == None:
            tlim = (T.min(), T.max())
        if (x != None) | (y != None):
            x, y = asarray(x), asarray(y)
            if x.size != y.size:
                raise ValueError('Zonal and meridional coordinate dimensions '
                    'do not match.')
            npoints = x.size
            radius2 = radius ** 2
        else:
            npoints = 0
            x = y = []
            #
            if ylim == None:
                ylim = (-90., 90.)
            if xlim == None:
                xlim = (0., 360.)
            else:
                # Make sure longitude limits are between 0 and 360.
                xlim = list(lon360(asarray(xlim)))
        if missions == None:
            missions = self.params['missions']
        
        # Aviso uses time in days since 1950-01-01 00:00:00 UTC, therefore
        # we have to calculate the initial time in matplotlib's format. We
        # also have to determine the proper variable using product name.
        T0 = dates.datestr2num('1950-01-01 00:00:00 UTC')
        var = self.params['product'].upper()
        
        # First we have to select which files will be loaded, which will 
        # depend on the temporal limits given in $t$.
        sel_time = flatnonzero((T >= floor(min(tlim))) & 
            (T <= ceil(max(tlim))))
        N = len(sel_time)
        
        # Second we will walk through each of the selected time in the dataset
        # and load the correspondant file for the available missions.
        t1 = time()
        if profile:
            s = '\rLoading data...'
            stdout.write(s)
            stdout.flush()
        # Reset important variables
        TIME, LAT, LON, VAR, MISSION = [array([])] * 5
        #
        for i, tm in enumerate(T[sel_time]):
            t2 = time()
            for (mission, fname) in self.attributes['time_dataset'][tm]:
                # Skips mission not in missions list.
                if mission not in missions:
                    continue
                # Uncompresses gzipped file and opens NetCDF instance.
                data = self.read_file('%s/%s/%s' % (self.params['path'], 
                    mission, fname))
                # Retrieve the scale factor for each variable
                scale_lat = data.variables['latitude'].scale_factor
                scale_lon = data.variables['latitude'].scale_factor
                scale_dat = data.variables[var].scale_factor
                # Get the raw time, latitude and longitude
                raw_time = data.variables['time'].data + T0
                raw_lat = data.variables['latitude'].data * scale_lat
                raw_lon = data.variables['longitude'].data * scale_lon
                # Select relevant data range according to limit parameters
                sel_from_time = (
                    (raw_time >= min(tlim)) & (raw_time <= max(tlim))
                )
                sel_from_limits = zeros(data.dimensions['time'], dtype=bool)
                if ylim != None:
                    sel_from_limits = (sel_from_limits | 
                        ((raw_lat >= min(ylim)) & (raw_lat <= max(ylim))))
                if xlim != None:
                    sel_from_limits = (sel_from_limits | 
                        ((raw_lon >= min(xlim)) & (raw_lon <= max(xlim))))
                # Select relevant data according to points and search radius.
                sel_from_radius =  zeros(data.dimensions['time'], dtype=bool)
                for xx, yy in zip(x, y):
                    distance2 = ((raw_lat - yy) ** 2 + 
                        (raw_lon - lon360(xx)) ** 2)
                    sel_from_radius = sel_from_radius | (distance2 <= radius2)
                #
                sel_data = flatnonzero(sel_from_time & 
                    (sel_from_limits | sel_from_radius))
                _time = raw_time[sel_data]
                _lat = raw_lat[sel_data]
                _lon = raw_lon[sel_data]
                _dat = data.variables[var].data[sel_data] * scale_dat
                #
                TIME = append(TIME, _time)
                LAT = append(LAT, _lat)
                LON = append(LON, _lon)
                VAR = append(VAR, _dat)
                MISSION = append(MISSION, [mission] * len(sel_data))
                #
                self.close_file(data)
            #
            # Profiling
            if profile:
                s = '\rLoading data... %s ' % (profiler(N, i+1, t0, t1, t2),)
                stdout.write(s)
                stdout.flush()
        #
        if profile:
            stdout.write('\n')
            stdout.flush()

        # Converts the data a structured array
        DAT = rec.fromarrays((TIME, LAT, LON, VAR, MISSION), 
            dtype=[('time', float64), ('latitude', float64), 
            ('longitude', float64), (self.params['product'], float64), 
            ('mission', '|S3')])
        #DAT = hstack((TIME[:, None], LAT[:, None], LON[:, None], 
        #    VAR[:, None], MISSION[:, None])).view(dtype=[('time', float64), 
        #    ('latitude', float64), ('longitude', float64), 
        #    (self.params['product'], float64), ('mission', '|S3')])
        
        # Some data sorting?
        if sort:
            DAT.sort(order=('time', 'latitude', 'longitude'), axis=0)
        
        return DAT