예제 #1
0
def test_resample_dtypes():
    """
    Test if dtypes stay the same when resampling.
    """

    data = {
        'testint8': np.array([5, 5], dtype=np.int8),
        'testfloat16': np.array([5, 5], dtype=np.float16)
    }

    fill_values = {'testint8': 0, 'testfloat16': 999.}
    lons = np.array([0, 0.1])
    lats = np.array([0, 0.1])
    # lets resample to a 0.1 degree grid
    # define the grid points in latitude and longitude
    lats_dim = np.arange(-1, 1, 0.1)
    lons_dim = np.arange(-1, 1, 0.1)
    # make 2d grid out the 1D grid spacing
    lons_grid, lats_grid = np.meshgrid(lons_dim, lats_dim)

    resampled_data = resample.resample_to_grid(data,
                                               lons,
                                               lats,
                                               lons_grid,
                                               lats_grid,
                                               fill_values=fill_values)

    for key in data:
        assert resampled_data[key].shape == lons_grid.shape
        assert resampled_data[key].dtype == data[key].dtype
예제 #2
0
def test_resample_dtypes():
    """
    Test if dtypes stay the same when resampling.
    """

    data = {'testint8': np.array([5, 5], dtype=np.int8),
            'testfloat16': np.array([5, 5], dtype=np.float16)}

    fill_values = {'testint8': 0,
                   'testfloat16': 999.}
    lons = np.array([0, 0.1])
    lats = np.array([0, 0.1])
    # lets resample to a 0.1 degree grid
    # define the grid points in latitude and longitude
    lats_dim = np.arange(-1, 1, 0.1)
    lons_dim = np.arange(-1, 1, 0.1)
    # make 2d grid out the 1D grid spacing
    lons_grid, lats_grid = np.meshgrid(lons_dim, lats_dim)

    resampled_data = resample.resample_to_grid(data, lons, lats,
                                               lons_grid, lats_grid,
                                               fill_values=fill_values)

    for key in data:
        assert resampled_data[key].shape == lons_grid.shape
        assert resampled_data[key].dtype == data[key].dtype
예제 #3
0
def resample_rho_LC(input_data,
                    src_lons,
                    src_lats,
                    target_lons,
                    target_lats,
                    search_rad=18000):

    if src_lons.shape != src_lats.shape:
        src_lons, src_lats = np.meshgrid(src_lons, src_lats)

    if target_lons.shape != target_lats.shape:
        target_lons, target_lats = np.meshgrid(target_lons, target_lats)

    data_resamp = resample_to_grid({'data': input_data},
                                   src_lons,
                                   src_lats,
                                   target_lons,
                                   target_lats,
                                   methods='nn',
                                   weight_funcs=None,
                                   min_neighbours=1,
                                   search_rad=search_rad,
                                   neighbours=8,
                                   fill_values=None)

    return data_resamp['data']
예제 #4
0
    def test_resample_to_zero_dot_one_deg(self):
        data, meta, timestamp, lons, lats, time_var = self.reader.read_img(
            datetime.datetime(2010, 5, 1, 8, 33, 1))
        # lets resample to a 0.1 degree grid
        # define the grid points in latitude and longitude
        lats_dim = np.arange(25, 75, 0.1)
        lons_dim = np.arange(-25, 45, 0.1)
        # make 2d grid out the 1D grid spacing
        lons_grid, lats_grid = np.meshgrid(lons_dim, lats_dim)

        resampled_data = resample.resample_to_grid(data, lons, lats,
                                                   lons_grid, lats_grid)

        for key in data:
            assert resampled_data[key].shape == lons_grid.shape
예제 #5
0
    def test_resample_to_zero_dot_one_deg(self):
        data, meta, timestamp, lons, lats, time_var = self.reader.read(
            datetime.datetime(2010, 5, 1, 8, 33, 1))
        # lets resample to a 0.1 degree grid
        # define the grid points in latitude and longitude
        lats_dim = np.arange(25, 75, 0.1)
        lons_dim = np.arange(-25, 45, 0.1)
        # make 2d grid out the 1D grid spacing
        lons_grid, lats_grid = np.meshgrid(lons_dim, lats_dim)

        resampled_data = resample.resample_to_grid(data, lons, lats, lons_grid,
                                                   lats_grid)

        for key in data:
            assert resampled_data[key].shape == lons_grid.shape
예제 #6
0
def test_resample_hamming():
    """
    Test if hamming window is applied correctly
    """
    # let's do 5 points with the highest value in the middle
    # -1-
    # 151
    # -1-

    data = {'testfloat16': np.array([1, 1, 5, 1, 1], dtype=np.float16)}

    fill_values = {'testfloat16': 999.}
    lons = np.array([0, -0.1, 0, 0.1, 0])
    lats = np.array([0.1, 0, 0, 0, -0.1])
    # lets resample to a 0.1 degree grid
    # define the grid points in latitude and longitude
    lats_dim = np.arange(-0.1, 0.11, 0.1)
    lons_dim = np.arange(-0.1, 0.11, 0.1)
    # make 2d grid out the 1D grid spacing
    lons_grid, lats_grid = np.meshgrid(lons_dim, lats_dim)
    # make partial function of the hamming window the radius of the hamming
    # window is in meters not in degrees
    hamm = functools.partial(resample.hamming_window, 15000)

    resampled_data = resample.resample_to_grid(data,
                                               lons,
                                               lats,
                                               lons_grid,
                                               lats_grid,
                                               fill_values=fill_values,
                                               methods='custom',
                                               weight_funcs=hamm)

    resampled_should = np.array([[1.640625, 1.64160156, 1.640625],
                                 [1.64160156, 3.11132812, 1.64160156],
                                 [1.640625, 1.64160156, 1.640625]])
    for key in data:
        assert resampled_data[key].shape == lons_grid.shape
        assert resampled_data[key].dtype == data[key].dtype
        nptest.assert_almost_equal(resampled_data[key], resampled_should)
예제 #7
0
def test_resample_hamming():
    """
    Test if hamming window is applied correctly
    """
    # let's do 5 points with the highest value in the middle
    # -1-
    # 151
    # -1-

    data = {'testfloat16': np.array([1, 1, 5, 1, 1], dtype=np.float16)}

    fill_values = {'testfloat16': 999.}
    lons = np.array([0, -0.1, 0, 0.1, 0])
    lats = np.array([0.1, 0, 0, 0, -0.1])
    # lets resample to a 0.1 degree grid
    # define the grid points in latitude and longitude
    lats_dim = np.arange(-0.1, 0.11, 0.1)
    lons_dim = np.arange(-0.1, 0.11, 0.1)
    # make 2d grid out the 1D grid spacing
    lons_grid, lats_grid = np.meshgrid(lons_dim, lats_dim)
    # make partial function of the hamming window the radius of the hamming
    # window is in meters not in degrees
    hamm = functools.partial(resample.hamming_window, 15000)

    resampled_data = resample.resample_to_grid(data, lons, lats,
                                               lons_grid, lats_grid,
                                               fill_values=fill_values,
                                               methods='custom', weight_funcs=hamm)

    resampled_should = np.array([[1.640625,  1.64160156,  1.640625],
                                 [1.64160156,  3.11132812,  1.64160156],
                                 [1.640625,  1.64160156,  1.640625]])
    for key in data:
        assert resampled_data[key].shape == lons_grid.shape
        assert resampled_data[key].dtype == data[key].dtype
        nptest.assert_almost_equal(resampled_data[key], resampled_should)
예제 #8
0
def resample_to_shape(source_file,
                      region,
                      sp_res,
                      grid,
                      prefix=None,
                      nan_value=None,
                      dest_nan_value=None,
                      variables=None,
                      shapefile=None):
    """
    Resamples images and clips country boundaries

    Parameters
    ----------
    source_file : str
        Path to source file.
    region : str
        Identifier of the region in the shapefile. If the default shapefile is
        used, this would be the FIPS country code.
    sp_res : int or float
        Spatial resolution of the shape-grid.
    grid : poets.grid.RegularGrid or poets.grid.ShapeGrid
        Grid to resample data to.
    prefix : str, optional
        Prefix for the variable in the NetCDF file, should be name of source
    nan_value : int, float, optional
        Not a number value of the original data as given by the data provider
    dest_nan_value : int or float, optional
        NaN value used in the final NetCDF file.
    variables : list of str, optional
        Variables to resample from original file.
    shapefile : str, optional
        Path to shape file, uses "world country admin boundary shapefile" by
        default.

    Returns
    -------
    res_data : dict of numpy.arrays
        resampled image
    dest_lon : numpy.array
        longitudes of the points in the resampled image
    dest_lat : numpy.array
        latitudes of the points in the resampled image
    gpis : numpy.array
        grid point indices
    timestamp : datetime.date
        date of the image
    metadata : dict
        Metadata derived from input file.
    """

    if prefix is not None:
        prefix += '_'

    fileExtension = os.path.splitext(source_file)[1].lower()

    if region == 'global':
        lon_min = -180
        lon_max = 180
        lat_min = -90
        lat_max = 90
    else:
        shp = Shape(region, shapefile)
        lon_min = shp.bbox[0]
        lon_max = shp.bbox[2]
        lat_min = shp.bbox[1]
        lat_max = shp.bbox[3]

    if fileExtension in ['.nc', '.nc3', '.nc4']:
        data_src, lon, lat, timestamp, metadata = nc.read_image(
            source_file, variables)
        if (lon_min >= lon.max() or lon_max <= lon.min()
                or lat_max <= lat.min() or lat_min >= lat.max()):
            return "No data"
        data, src_lon, src_lat = nc.clip_bbox(data_src, lon, lat, lon_min,
                                              lat_min, lon_max, lat_max)
    elif fileExtension in ['.h5']:
        data_src, lon, lat, timestamp, metadata = h5.read_image(
            source_file, variables)
        if (lon_min >= lon.max() or lon_max <= lon.min()
                or lat_max <= lat.min() or lat_min >= lat.max()):
            return "No data"
        data, src_lon, src_lat = nc.clip_bbox(data_src, lon, lat, lon_min,
                                              lat_min, lon_max, lat_max)
    elif fileExtension in imgfiletypes:
        data, src_lon, src_lat, timestamp, metadata = bbox_img(
            source_file, region, fileExtension, shapefile)

    if nan_value is not None:
        for key in data.keys():
            data[key] = np.ma.array(data[key], mask=(data[key] == nan_value))

    src_lon, src_lat = np.meshgrid(src_lon, src_lat)

    lons = grid.arrlon[0:grid.shape[0]]
    dest_lon, dest_lat = np.meshgrid(lons, np.unique(grid.arrlat)[::-1])

    gpis = grid.get_bbox_grid_points(grid.arrlat.min(), grid.arrlat.max(),
                                     grid.arrlon.min(), grid.arrlon.max())

    search_rad = 180000 * sp_res

    data = resample.resample_to_grid(data,
                                     src_lon,
                                     src_lat,
                                     dest_lon,
                                     dest_lat,
                                     search_rad=search_rad)

    res_data = {}
    path = []

    if region != 'global':
        _, _, multipoly = shp._get_shape()
        for ring in multipoly:
            poly_verts = list(ring.exterior.coords)
            path.append(matplotlib.path.Path(poly_verts))

        coords = [grid.arrlon, grid.arrlat[::-1]]
        coords2 = np.zeros((len(coords[0]), 2))

        for idx in range(0, len(coords[0])):
            coords2[idx] = [coords[0][idx], coords[1][idx]]

        mask_old = path[0].contains_points(coords2)

    for key in data.keys():
        if variables is not None:
            if key not in variables:
                del metadata[key]
                continue

        if region != 'global':
            for ring in path:
                mask_new = (ring.contains_points(coords2))
                mask_rev = scipy.logical_or(mask_old, mask_new)
                mask_old = mask_rev

            mask_rev = mask_rev.reshape(dest_lon.shape)
            mask = np.invert(mask_rev)

            mask[data[key].mask == True] = True
        else:
            mask = data[key].mask

        if prefix is None:
            var = key
        else:
            var = prefix + key

        if metadata is not None:
            metadata[var] = metadata[key]
            if var != key:
                del metadata[key]
        res_data[var] = np.ma.masked_array(data[key],
                                           mask=np.copy(mask),
                                           fill_value=dest_nan_value)

        dat = np.copy(res_data[var].data)
        dat[mask == True] = dest_nan_value

        res_data[var] = np.ma.masked_array(dat,
                                           mask=np.copy(mask),
                                           fill_value=dest_nan_value)

    return res_data, dest_lon, dest_lat, gpis, timestamp, metadata
예제 #9
0
파일: resampling.py 프로젝트: TUW-GEO/poets
def resample_to_shape(source_file, region, sp_res, grid, prefix=None,
                      nan_value=None, dest_nan_value=None, variables=None,
                      shapefile=None):
    """
    Resamples images and clips country boundaries

    Parameters
    ----------
    source_file : str
        Path to source file.
    region : str
        Identifier of the region in the shapefile. If the default shapefile is
        used, this would be the FIPS country code.
    sp_res : int or float
        Spatial resolution of the shape-grid.
    grid : poets.grid.RegularGrid or poets.grid.ShapeGrid
        Grid to resample data to.
    prefix : str, optional
        Prefix for the variable in the NetCDF file, should be name of source
    nan_value : int, float, optional
        Not a number value of the original data as given by the data provider
    dest_nan_value : int or float, optional
        NaN value used in the final NetCDF file.
    variables : list of str, optional
        Variables to resample from original file.
    shapefile : str, optional
        Path to shape file, uses "world country admin boundary shapefile" by
        default.

    Returns
    -------
    res_data : dict of numpy.arrays
        resampled image
    dest_lon : numpy.array
        longitudes of the points in the resampled image
    dest_lat : numpy.array
        latitudes of the points in the resampled image
    gpis : numpy.array
        grid point indices
    timestamp : datetime.date
        date of the image
    metadata : dict
        Metadata derived from input file.
    """

    if prefix is not None:
        prefix += '_'

    fileExtension = os.path.splitext(source_file)[1].lower()

    if region == 'global':
        lon_min = -180
        lon_max = 180
        lat_min = -90
        lat_max = 90
    else:
        shp = Shape(region, shapefile)
        lon_min = shp.bbox[0]
        lon_max = shp.bbox[2]
        lat_min = shp.bbox[1]
        lat_max = shp.bbox[3]

    if fileExtension in ['.nc', '.nc3', '.nc4']:
        data_src, lon, lat, timestamp, metadata = nc.read_image(source_file,
                                                                variables)
        if (lon_min >= lon.max() or lon_max <= lon.min() or
            lat_max <= lat.min() or lat_min >= lat.max()):
            return "No data"
        data, src_lon, src_lat = nc.clip_bbox(data_src, lon, lat, lon_min,
                                              lat_min, lon_max, lat_max)
    elif fileExtension in ['.h5']:
        data_src, lon, lat, timestamp, metadata = h5.read_image(source_file,
                                                                variables)
        if (lon_min >= lon.max() or lon_max <= lon.min() or
            lat_max <= lat.min() or lat_min >= lat.max()):
            return "No data"
        data, src_lon, src_lat = nc.clip_bbox(data_src, lon, lat, lon_min,
                                              lat_min, lon_max, lat_max)
    elif fileExtension in imgfiletypes:
        data, src_lon, src_lat, timestamp, metadata = bbox_img(source_file,
                                                               region,
                                                               fileExtension,
                                                               shapefile)

    if nan_value is not None:
        for key in data.keys():
            data[key] = np.ma.array(data[key], mask=(data[key] == nan_value))

    src_lon, src_lat = np.meshgrid(src_lon, src_lat)

    lons = grid.arrlon[0:grid.shape[0]]
    dest_lon, dest_lat = np.meshgrid(lons, np.unique(grid.arrlat)[::-1])

    gpis = grid.get_bbox_grid_points(grid.arrlat.min(), grid.arrlat.max(),
                                     grid.arrlon.min(), grid.arrlon.max())

    search_rad = 180000 * sp_res

    data = resample.resample_to_grid(data, src_lon, src_lat, dest_lon,
                                     dest_lat, search_rad=search_rad)

    res_data = {}
    path = []

    if region != 'global':
        _, _, multipoly = shp._get_shape()
        for ring in multipoly:
            poly_verts = list(ring.exterior.coords)
            path.append(matplotlib.path.Path(poly_verts))

        coords = [grid.arrlon, grid.arrlat[::-1]]
        coords2 = np.zeros((len(coords[0]), 2))

        for idx in range(0, len(coords[0])):
            coords2[idx] = [coords[0][idx], coords[1][idx]]

        mask_old = path[0].contains_points(coords2)

    for key in data.keys():
        if variables is not None:
            if key not in variables:
                del metadata[key]
                continue

        if region != 'global':
            for ring in path:
                mask_new = (ring.contains_points(coords2))
                mask_rev = scipy.logical_or(mask_old, mask_new)
                mask_old = mask_rev

            mask_rev = mask_rev.reshape(dest_lon.shape)
            mask = np.invert(mask_rev)

            mask[data[key].mask == True] = True
        else:
            mask = data[key].mask

        if prefix is None:
            var = key
        else:
            var = prefix + key

        if metadata is not None:
            metadata[var] = metadata[key]
            if var != key:
                del metadata[key]
        res_data[var] = np.ma.masked_array(data[key], mask=np.copy(mask),
                                           fill_value=dest_nan_value)

        dat = np.copy(res_data[var].data)
        dat[mask == True] = dest_nan_value

        res_data[var] = np.ma.masked_array(dat, mask=np.copy(mask),
                                           fill_value=dest_nan_value)

    return res_data, dest_lon, dest_lat, gpis, timestamp, metadata
예제 #10
0
# <rawcell>

# let's resample and plot the ssm data
# H07 data is not that easy to plot because it comes in orbit geometry and not on a fixed grid.

# <codecell>

# %matplotlib inline
# lets resample to a 0.1 degree grid
#define the grid points in latitude and logitude
lats_dim = np.arange(25, 75, 0.1)
lons_dim = np.arange(-25, 45, 0.1)
#make 2d grid out the 1D grid spacings
lons_grid, lats_grid = np.meshgrid(lons_dim, lats_dim)

resampled_data = resample.resample_to_grid(h07_data, lons, lats, lons_grid,
                                           lats_grid)

fig = plt.figure(figsize=(10, 10))
ax = fig.add_axes([0.1, 0.1, 0.8, 0.8])
# setup of basemap for europe
# simple mercator projection
m = Basemap(llcrnrlon=-25.0,llcrnrlat=25.0,urcrnrlon=45.0,urcrnrlat=75.0,\
            resolution='l',area_thresh=1000.,projection='merc',\
            lat_ts=50.,ax=ax)

# make a pseudocolor plot using the ASCAT SWI colormap
im = m.pcolormesh(lons_grid,
                  lats_grid,
                  resampled_data['ssm'],
                  latlon=True,
                  cmap=smcolormaps.load('SWI_ASCAT'))
예제 #11
0
# <rawcell>

# let's resample and plot the ssm data
# H07 data is not that easy to plot because it comes in orbit geometry and not on a fixed grid.

# <codecell>

# %matplotlib inline
# lets resample to a 0.1 degree grid
#define the grid points in latitude and logitude
lats_dim = np.arange(25,75,0.1)
lons_dim = np.arange(-25,45,0.1)
#make 2d grid out the 1D grid spacings
lons_grid, lats_grid = np.meshgrid(lons_dim,lats_dim)

resampled_data = resample.resample_to_grid(h07_data, lons, lats,
                                           lons_grid, lats_grid)

fig = plt.figure(figsize=(10,10))
ax = fig.add_axes([0.1,0.1,0.8,0.8])
# setup of basemap for europe
# simple mercator projection
m = Basemap(llcrnrlon=-25.0,llcrnrlat=25.0,urcrnrlon=45.0,urcrnrlat=75.0,\
            resolution='l',area_thresh=1000.,projection='merc',\
            lat_ts=50.,ax=ax)

# make a pseudocolor plot using the ASCAT SWI colormap
im = m.pcolormesh(lons_grid, lats_grid, resampled_data['Surface Soil Moisture (Ms)'], latlon=True,
         cmap=smcolormaps.load('SWI_ASCAT'))

m.drawcoastlines()
m.drawcountries()
예제 #12
0
    def img_bulk(self):
        """
        Yields numpy array of self.const.imgbuffer images,
        start and enddate until all dates have been read

        Returns
        -------
        img_stack_dict : dict of numpy.array
            stack of daily images for each variable
        startdate : date
            date of first image in stack
        enddate : date
            date of last image in stack
        datetimestack : np.array
            array of the timestamps of each image
        jd_stack : np.array or None
            if None all observations in an image have the same
            observation timestamp. Otherwise it gives the julian date
            of each observation in img_stack_dict
        """

        img_dict = {}
        datetimes = []
        jd_list = []
        # set start of current imgbulk to startdate
        bulkstart = self.startdate
        # image counter
        read_images = 0

        dates = self.imgin.tstamps_for_daterange(self.startdate,
                                                 self.enddate)
        for date in dates:
            try:
                (input_img, metadata,
                 image_datetime, lon,
                 lat, time_arr) = self.imgin.read(date, **self.input_kwargs)
            except IOError as e:
                msg = "I/O error({0}): {1}".format(e.errno,
                                                   e.strerror)
                logging.log(logging.INFO, msg)
                continue
            read_images += 1
            logging.log(logging.INFO, "read" + image_datetime.isoformat())
            if self.resample:

                if time_arr is not None:
                    input_img['jd'] = time_arr
                input_img = resamp.resample_to_grid(input_img, lon, lat,
                                                    self.target_grid.activearrlon,
                                                    self.target_grid.activearrlat,
                                                    methods=self.r_methods,
                                                    weight_funcs=self.r_weightf,
                                                    min_neighbours=self.r_min_n,
                                                    search_rad=self.r_radius,
                                                    neighbours=self.r_neigh,
                                                    fill_values=self.r_fill_values)
                time_arr = input_img.pop('jd')
            if time_arr is None:
                self.time_var = None

            else:
                self.time_var = 'jd'
            if time_arr is not None:
                # if time_var is not None means that each observation of the
                # image has its own observation time
                # this means that the resulting time series is not
                # regularly spaced in time
                if self.orthogonal is None:
                    self.orthogonal = False
                if self.orthogonal:
                    raise Img2TsError("Images can not switch between a fixed image "
                                      "timestamp and individual timestamps for each observation")
                jd_list.append(time_arr)
            if time_arr is None:
                if self.orthogonal is None:
                    self.orthogonal = True
                if not self.orthogonal:
                    raise Img2TsError(
                        "Images can not switch between a fixed image "
                        "timestamp and individual timestamps for each observation")

            for key in input_img:
                if key not in img_dict.keys():
                    img_dict[key] = []
                img_dict[key].append(input_img[key])

            datetimes.append(image_datetime)

            if read_images >= self.imgbuffer - 1:
                img_stack_dict = {}
                if len(jd_list) != 0:
                    jd_stack = np.ma.vstack(jd_list)
                    jd_list = None
                else:
                    jd_stack = None
                for key in img_dict:
                    img_stack_dict[key] = np.vstack(img_dict[key])
                    img_dict[key] = None
                datetimestack = np.array(datetimes)
                img_dict = {}
                datetimes = []
                jd_list = []
                yield (img_stack_dict, bulkstart, self.currentdate,
                       datetimestack, jd_stack)
                # reset image counter
                read_images = 0

        if len(datetimes) > 0:
            img_stack_dict = {}
            if len(jd_list) != 0:
                jd_stack = np.ma.vstack(jd_list)
            else:
                jd_stack = None
            for key in img_dict:
                img_stack_dict[key] = np.vstack(img_dict[key])
                img_dict[key] = None
            datetimestack = np.array(datetimes)
            yield (img_stack_dict, bulkstart, self.currentdate, datetimestack,
                   jd_stack)