def ERA_RegularImgGrid(res_lat=0.25, res_lon=0.25):
"""
Create ECMWF regular cell grid
Parameters
----------
res_lat : float, optional (default: 0.25)
Resolution in Y direction
res_lon : float, optional (default: 0.25)
Resolution in X direction
Returns
----------
CellGrid : pygeogrids.CellGrid
Regular, global CellGrid with 5DEG*5DEG cells
"""
lon = np.arange(0., 360. - res_lon / 2., res_lon)
lat = np.arange(90., -1. * 90. - res_lat / 2., -1. * res_lat)
lons_gt_180 = np.where(lon > 180.0)
lon[lons_gt_180] = lon[lons_gt_180] - 360.
lon, lat = np.meshgrid(lon, lat)
glob_basic_grid = BasicGrid(lon.flatten(), lat.flatten())
glob_cell_grid = glob_basic_grid.to_cell_grid(cellsize=5.)
return glob_cell_grid
def ERA_RegularImgGrid(res_lat=0.25, res_lon=0.25):
"""
Create ECMWF regular cell grid
Parameters
----------
res_lat : float, optional (default: 0.25)
Resolution in Y direction
res_lon : float, optional (default: 0.25)
Resolution in X direction
Returns
----------
CellGrid : pygeogrids.CellGrid
Regular, global CellGrid with 5DEG*5DEG cells
"""
# np.arange is not precise...
f_lon = (1. / res_lon)
f_lat = (1. / res_lat)
res_lon = res_lon * f_lon
res_lat = res_lat * f_lat
lon = np.arange(0., 360. * f_lon, res_lon)
lat = np.arange(90. * f_lat, -90 * f_lat - res_lat, -1 * res_lat)
lons_gt_180 = np.where(lon > (180. * f_lon))
lon[lons_gt_180] = lon[lons_gt_180] - (360. * f_lon)
lon, lat = np.meshgrid(lon, lat)
glob_basic_grid = BasicGrid(lon.flatten() / f_lon, lat.flatten() / f_lat)
glob_cell_grid = glob_basic_grid.to_cell_grid(cellsize=5.)
return glob_cell_grid
def cut(self) -> CellGrid:
# create a new grid from the active subset
return BasicGrid(lon=self.activearrlon,
lat=self.activearrlat,
gpis=self.activegpis,
subset=None,
shape=self.subset_shape).to_cell_grid(self.cellsize)
def ERA_RegularImgGrid(res_lat=0.25, res_lon=0.25):
'''
Create ECMWF regular cell grid
Parameters
----------
res_lat : float, optional (default: 0.25)
Resolution in Y direction
res_lon : float, optional (default: 0.25)
Resolution in X direction
Returns
----------
CellGrid : pygeogrids.CellGrid
Regular, global CellGrid with 5DEG*5DEG cells
'''
lon = np.arange(0, 360 - res_lon / 2, res_lon)
lat = np.arange(90, -1 * 90 - res_lat / 2, -1 * res_lat)
lons_gt_180 = np.where(lon > 180.0)
lon[lons_gt_180] = lon[lons_gt_180] - 360
lon, lat = np.meshgrid(lon, lat)
return BasicGrid(lon.flatten(), lat.flatten()).to_cell_grid(cellsize=5.)
def ERA_RegularImgGrid(
res_lat: float = 0.25,
res_lon: float = 0.25,
bbox: Tuple[float, float, float, float] = None,
) -> CellGrid:
"""
Create regular cell grid for bounding box with the selected
resolution.
Parameters
----------
res_lat: float, optional (default: 0.25)
Grid resolution (in degrees) in latitude direction.
res_lon: float, optional (default: 0.25)
Grid resolution (in degrees) in longitude direction.
bbox: tuple, optional (default: None)
(min_lon, min_lat, max_lon, max_lat) - wgs84
bbox to cut the global grid to.
Returns
----------
CellGrid : CellGrid
Regular, CellGrid with 5DEG*5DEG cells for the passed bounding box.
"""
# np.arange is not precise...
f_lon = 1.0 / res_lon
f_lat = 1.0 / res_lat
res_lon = res_lon * f_lon
res_lat = res_lat * f_lat
lon = np.arange(0.0, 360.0 * f_lon, res_lon)
lat = np.arange(90.0 * f_lat, -90 * f_lat - res_lat, -1 * res_lat)
lons_gt_180 = np.where(lon > (180.0 * f_lon))
lon[lons_gt_180] = lon[lons_gt_180] - (360.0 * f_lon)
lon, lat = np.meshgrid(lon, lat)
glob_basic_grid = BasicGrid(lon.flatten() / f_lon, lat.flatten() / f_lat)
glob_cell_grid = glob_basic_grid.to_cell_grid(cellsize=5.0)
if bbox is not None:
gpis = glob_cell_grid.get_bbox_grid_points(
lonmin=bbox[0], latmin=bbox[1], lonmax=bbox[2], latmax=bbox[3]
)
glob_cell_grid = glob_cell_grid.subgrid_from_gpis(gpis)
return glob_cell_grid
def study_area_gpis():
#=========STUDY AREA GRID POINTS ============
gpi_path = "C:\\Users\\s.hochstoger\\Desktop\\0_IWMI_DATASETS\\pointlist_India_warp.csv"
gpis = pd.read_csv(gpi_path)
gpi_path_p = "C:\\Users\\s.hochstoger\\Desktop\\0_IWMI_DATASETS\\pointlist_Pakistan_warp.csv"
gpis_p = pd.read_csv(gpi_path_p)
gpis = gpis.append(gpis_p)
grid = BasicGrid(gpis.lon, gpis.lat, gpis.point)
gpis = grid.get_bbox_grid_points(14.7148, 29.3655, 68.15, 81.8419)
lon, lat = grid.get_bbox_grid_points(14.7148, 29.3655, 68.15, 81.8419, coords=True)
gp = pd.DataFrame(gpis, columns=['gpi'])
lon = pd.DataFrame(lon, columns=['lon'])
lat = pd.DataFrame(lat, columns=['lat'])
gplon = pd.concat([gp, lon], axis=1)
gplonlat = pd.concat([gplon, lat], axis=1)
gplonlat.to_csv("C:\\Users\\s.hochstoger\\Desktop\\0_IWMI_DATASETS\\study_area_gp_lonlat_new.csv")
def globalCellgrid():
# symettrical grid
lon = (np.arange(360 * 4) * 0.25) - 179.875
lat = (np.arange(180 * 4) * 0.25) - 89.875
lon, lat = np.meshgrid(lon, lat)
return BasicGrid(lon.flatten(), lat.flatten()).to_cell_grid(cellsize=5.)
def _grid(self, gpis):
# create a pygeogrids object
lons, lats = np.meshgrid(self.ds.lon.values,
np.flipud(self.ds.lat.values))
lons, lats = lons.flatten(), lats.flatten()
grid = BasicGrid(lons, lats, gpis=gpis.flatten()).to_cell_grid(5.)
return grid
def GLDAS025Grids(only_land=False):
"""
Create global 0.25 DEG gldas grids (origin in bottom left)
Parameters
---------
only_land : bool, optional (default: False)
Uses the land mask to reduce the GLDAS 0.25DEG land grid to land points
only.
Returns
--------
grid : pygeogrids.CellGrid
Either a land grid or a global grid
"""
resolution = 0.25
glob_lons = np.arange(-180 + resolution / 2, 180 + resolution / 2,
resolution)
glob_lats = np.arange(-90 + resolution / 2, 90 + resolution / 2,
resolution)
lon, lat = np.meshgrid(glob_lons, glob_lats)
glob_grid = BasicGrid(lon.flatten(),
lat.flatten()).to_cell_grid(cellsize=5.0)
if only_land:
ds = Dataset(
os.path.join(
os.path.abspath(os.path.dirname(__file__)),
"GLDASp4_landmask_025d.nc4",
))
land_lats = ds.variables["lat"][:]
land_mask = ds.variables["GLDAS_mask"][:].flatten().filled() == 0.0
dlat = glob_lats.size - land_lats.size
land_mask = np.concatenate((np.ones(dlat * glob_lons.size), land_mask))
land_points = np.ma.masked_array(glob_grid.get_grid_points()[0],
land_mask)
land_grid = glob_grid.subgrid_from_gpis(
land_points[~land_points.mask].filled())
return land_grid
else:
return glob_grid
def init_0_1_grid(str):
'''
Parameters:
-----------
str : str
NDVI, SWI - which grid is needed
Returns:
--------
gird : BasicGrid
'''
if str == 'SWI':
fpath = "C:\\Users\\s.hochstoger\\Desktop\\0_IWMI_DATASETS\\SWI\\20070701"
fname = "g2_BIOPAR_SWI10_200707010000_GLOBE_ASCAT_V3_0_1.nc"
with Dataset(os.path.join(fpath, fname), mode='r') as ncfile:
lon = ncfile.variables['lon'][:]
lat = ncfile.variables['lat'][:]
mask = (ncfile.variables["SWI_010"][:]).mask
lons, lats = np.meshgrid(lon, lat)
grid = BasicGrid(lons[np.where(mask == False)],
lats[np.where(mask == False)])
elif str == 'IMD':
fpath = 'C:\\Users\\s.hochstoger\\Desktop\\0_IWMI_DATASETS\\Dataset_stacks\\IMD_RF_stack.nc'
with Dataset(fpath, mode='r') as ncfile:
lon = ncfile.variables['lon'][:]
lat = ncfile.variables['lat'][:]
mask = (ncfile.variables["IMD_RF"][:]).mask
lons, lats = np.meshgrid(lon, lat)
grid = BasicGrid(lons.flatten(), lats.flatten())
elif (str == 'NDVI') | (str == 'LAI'):
fpath = "C:\\Users\\s.hochstoger\\Desktop\\poets\\DATA"
fname = "West_SA_0.1_dekad_NDVI.nc"
with Dataset(os.path.join(fpath, fname), mode='r') as ncfile:
lon = ncfile.variables['lon'][:]
lat = ncfile.variables['lat'][:]
if str == 'NDVI':
mask = (ncfile.variables['NDVI_dataset'][10]).mask
else:
mask = (ncfile.variables['NDVI_dataset'][29]).mask
lons, lats = np.meshgrid(lon, lat)
grid = BasicGrid(lons[np.where(mask == False)],
lats[np.where(mask == False)])
return grid
def RZSMCellgrid(path_grid, file_grid='grid.nc'):
data_grid = Dataset(join(path_grid, file_grid))
lon_1d = data_grid['lon'][:]
lat_1d = data_grid['lat'][:]
lon_2d, lat_2d = np.meshgrid(lon_1d, lat_1d)
return BasicGrid(lon_2d.flatten(), lat_2d.flatten()).to_cell_grid(cellsize=5.)
def __init__(self, bbox=None):
"""
Parameters
----------
bbox: tuple, optional (default: None)
(min_lon, min_lat, max_lon, max_lat)
Bounding box to create subset for, if None is passed a global
grid is used.
"""
ease25 = EASE2_grid(25000)
lons, lats = ease25.londim, ease25.latdim
lons, lats = np.meshgrid(lons, lats)
assert lons.shape == lats.shape
shape = lons.shape
lats = np.flipud(lats) # flip lats, so that origin in bottom left
lons, lats = lons.flatten(), lats.flatten()
globgrid = BasicGrid(lons, lats, shape=shape)
sgpis = globgrid.activegpis
self.bbox = bbox
if self.bbox:
sgpis = globgrid.get_bbox_grid_points(latmin=self.bbox[1],
latmax=self.bbox[3],
lonmin=self.bbox[0],
lonmax=self.bbox[2])
self.cellsize = 5.
super(EASE25CellGrid,
self).__init__(lon=globgrid.arrlon,
lat=globgrid.arrlat,
subset=sgpis,
cells=lonlat2cell(globgrid.arrlon,
globgrid.arrlat, self.cellsize),
shape=shape)
self.subset_shape = (len(np.unique(self.activearrlat)),
len(np.unique(self.activearrlon)))
def oversample(lon, lat, data, extent, dx, dy):
other = BasicGrid(lon, lat)
reg_grid = genreg_grid(dx, dy, minlat=extent[2], maxlat=extent[3],
minlon=extent[0], maxlon=extent[1])
max_dist = dx * 111 * 1000 # a mean distance for one degree it's around 111 km
lut = reg_grid.calc_lut(other, max_dist=max_dist)
img = np.ma.masked_where(lut == -1, data[lut])
img[np.isnan(img)] = np.ma.masked
return img.reshape(-1, reg_grid.shape[1]), reg_grid
def ERA_IrregularImgGrid(
lons: np.ndarray,
lats: np.ndarray,
bbox: Tuple[float, float, float, float] = None,
) -> CellGrid:
"""
Create a irregular grid from the passed coordinates.
"""
lons_gt_180 = np.where(lons > 180.0)
lons[lons_gt_180] = lons[lons_gt_180] - 360
grid = BasicGrid(lons.flatten(), lats.flatten())\
.to_cell_grid(cellsize=5.0)
if bbox is not None:
gpis = grid.get_bbox_grid_points(
lonmin=bbox[0], latmin=bbox[1], lonmax=bbox[2], latmax=bbox[3]
)
grid = grid.subgrid_from_gpis(gpis)
return grid
def test_BasicGrid_transform_lon():
"""
Tests whether transforming longitudes works as expected.
"""
lat = np.asarray([10, -10, 5, 42])
lon_pos = np.asarray([0, 90, 180, 270])
lon_centered = np.asarray([0, 90, 180, -90])
# case 1: warning and transformation
with pytest.warns(UserWarning):
grid = BasicGrid(lon_pos, lat)
assert np.all(grid.arrlon == lon_centered)
# case 2: no warning and transform
grid = BasicGrid(lon_pos, lat, transform_lon=True)
assert np.all(grid.arrlon == lon_centered)
# case 3: no warning and no transform
grid = BasicGrid(lon_pos, lat, transform_lon=False)
assert np.all(grid.arrlon == lon_pos)
def CCI025Cellgrid():
"""
Class for the CCI Version 0.42 0.25deg cell grid.
"""
resolution = 0.25
lon, lat = np.meshgrid(
np.arange(-180 + resolution / 2, 180 + resolution / 2, resolution),
np.arange(-90 + resolution / 2, 90 + resolution / 2, resolution))
return BasicGrid(lon.flatten(), lat.flatten()).to_cell_grid(cellsize=5.)
def globalCellgrid():
# symettrical grid
lon_res = 0.25
lat_res = 0.25
offs_h = lon_res / 2.
offs_v = lat_res / 2.
# create meshgrid
lon, lat = np.meshgrid(np.arange(-180 + offs_h, 180 - offs_h, lon_res),
np.arange(-90 + offs_v, 90 - offs_v, lat_res))
return BasicGrid(lon.flatten(), lat.flatten()).to_cell_grid(cellsize=5.)
def ERA_RegularImgGrid(res_lat=0.3, res_lon=0.3):
'''
ECMWF 0.25deg cell grid.
:return: global QDEG-CellGrid
'''
lon = np.arange(0, 360 - res_lon / 2, res_lon)
lat = np.arange(90, -1 * 90 - res_lat / 2, -1 * res_lat)
lons_gt_180 = np.where(lon > 180.0)
lon[lons_gt_180] = lon[lons_gt_180] - 360
lon, lat = np.meshgrid(lon, lat)
return BasicGrid(lon.flatten(), lat.flatten()).to_cell_grid(cellsize=5.)
def test_add_station(self):
"""
Test adding stations.
"""
name = "station1"
self.network.add_station(name, 0, 0, 0)
assert self.network.stations[name].name == name
assert self.network.stations[name].lon == 0
assert self.network.stations[name].lat == 0
assert self.network.stations[name].elev == 0
assert self.network.grid == BasicGrid([0], [0])
def study_area_gpis():
#=========STUDY AREA GRID POINTS ============
gpi_path = "C:\\Users\\s.hochstoger\\Desktop\\0_IWMI_DATASETS\\pointlist_India_warp.csv"
gpis = pd.read_csv(gpi_path)
gpi_path_p = "C:\\Users\\s.hochstoger\\Desktop\\0_IWMI_DATASETS\\pointlist_Pakistan_warp.csv"
gpis_p = pd.read_csv(gpi_path_p)
gpis = gpis.append(gpis_p)
grid = BasicGrid(gpis.lon, gpis.lat, gpis.point)
gpis = grid.get_bbox_grid_points(14.7148, 29.3655, 68.15, 81.8419)
lon, lat = grid.get_bbox_grid_points(14.7148,
29.3655,
68.15,
81.8419,
coords=True)
gp = pd.DataFrame(gpis, columns=['gpi'])
lon = pd.DataFrame(lon, columns=['lon'])
lat = pd.DataFrame(lat, columns=['lat'])
gplon = pd.concat([gp, lon], axis=1)
gplonlat = pd.concat([gplon, lat], axis=1)
gplonlat.to_csv(
"C:\\Users\\s.hochstoger\\Desktop\\0_IWMI_DATASETS\\study_area_gp_lonlat_new.csv"
)
def init_0_4_grid(fpath=None):
if fpath is None:
fpath = "C:\\Users\\i.pfeil\\Desktop\\poets\\DATA\\alt\\"
fname = "West_SA_0.4_dekad.nc"
with Dataset(os.path.join(fpath, fname), mode='r') as ncfile:
lon = ncfile.variables['lon'][:]
lat = ncfile.variables['lat'][:]
mask = ncfile.variables['SWI_SWI_001'][0, :, :].mask
lons, lats = np.meshgrid(lon, lat)
grid = BasicGrid(lons[np.where(mask == False)],
lats[np.where(mask == False)])
return grid
def init_SWI_grid(fpath=None):
if fpath is None:
fpath = "C:\\Users\\i.pfeil\\Documents\\0_IWMI_DATASETS\\SWI\\20070701"
fname = "g2_BIOPAR_SWI10_200707010000_GLOBE_ASCAT_V3_0_1.nc"
with Dataset(os.path.join(fpath, fname), mode='r') as ncfile:
lon = ncfile.variables['lon'][:]
lat = ncfile.variables['lat'][:]
mask = (ncfile.variables["SWI_010"][:]).mask
lons, lats = np.meshgrid(lon, lat)
grid = BasicGrid(lons[np.where(mask == False)],
lats[np.where(mask == False)])
return grid
def ERA5025Cellgrid(path_grid, file_grid='grid.nc'):
data_grid = Dataset(join(path_grid, file_grid))
try:
lon_1d = data_grid['longitude'][:]
except BaseException:
lon_1d = data_grid['Longitude'][:]
try:
lat_1d = data_grid['latitude'][:]
except BaseException:
lat_1d = data_grid['Latitude'][:]
lon_2d, lat_2d = np.meshgrid(lon_1d, lat_1d)
return BasicGrid(lon_2d.flatten(), lat_2d.flatten()).to_cell_grid(cellsize=5.)
def create_merra_cell_grid():
"""
Function creates the asymmetrical GMAO 0.5 x 0.625 grid as a
BasicGrid instance.
Returns
-------
BasicGrid instance
"""
# define horizontal and vertical resolution of asymmetrical grid
lon_res = 0.625
lat_res = 0.5
# create 361 (lat) x 576 (lon) mesh grid
lon, lat = np.meshgrid(np.arange(-180, 180, lon_res),
np.arange(-90, 90 + lat_res / 2, lat_res))
return BasicGrid(lon.flatten(), lat.flatten()).to_cell_grid(cellsize=5.)
def get_grid(file_name):
data_handle = rasterio.open(file_name)
data_bbox = data_handle.bounds
data_res = data_handle.res
lon_1d = np.arange(data_bbox.left,
data_bbox.right + np.abs(data_res[0] / 2),
np.abs(data_res[0]), float)
lat_1d = np.arange(data_bbox.bottom,
data_bbox.top + np.abs(data_res[1] / 2),
np.abs(data_res[1]), float)
lon_2d, lat_2d = np.meshgrid(lon_1d, lat_1d)
grid = BasicGrid(lon_2d.flatten(),
lat_2d.flatten()).to_cell_grid(cellsize=5.)
return grid, lon_2d, lat_2d, data_bbox
def create_image(lon,
lat,
data,
minlat=-90.,
maxlat=90.,
minlon=-180.,
maxlon=180.):
other = BasicGrid(lon, lat)
reg_grid = genreg_grid(0.1,
0.1,
minlat=minlat,
maxlon=maxlon,
maxlat=maxlat,
minlon=minlon)
lons = np.arange(minlon, maxlon, 0.1)
lats = np.arange(minlat, maxlat, 0.1)
lut = reg_grid.calc_lut(other, max_dist=25000)
img = np.ma.masked_where(lut == -1, data[lut])
img = np.flipud(img.reshape(reg_grid.shape[0], -1))
return img, lons, lats
def __init__(
self,
fname,
parameters,
loading_func,
cellsize=5.0,
only_land=False,
bbox=None,
):
# input validation
if isinstance(parameters, str):
parameters = [parameters]
self.parameters = parameters
if isinstance(loading_func, str):
if loading_func not in loading_func_dict: # pragma: no cover
raise ValueError(
f"No loading function with the name '{loading_func}'"
" exists.")
loading_func = loading_func_dict[loading_func]
self.loading_func = loading_func
self.cellsize = cellsize
self.only_land = only_land
# load dataset
self.dataset = loading_func(str(fname), self.parameters)
# Img2Ts prefers flattened data
self.dataset = self.dataset.stack(
dimensions={"latlon": ("lat", "lon")})
# lons are between 0 and 360, they have to be remapped to (-180, 180)
self._lons = np.array(self.dataset.lon.values)
self._lons[self._lons > 180] -= 360
# setup grid
global_grid = BasicGrid(self.lon, self.lat)
# land mask
if self.only_land:
if "landmask" in self.dataset:
self.landmask = self.dataset.landmask.values
else: # pragma: no cover
raise ValueError("No landmask available!")
self.land_gpis = global_grid.get_grid_points()[0][self.landmask]
grid = global_grid.subgrid_from_gpis(self.land_gpis)
else:
grid = global_grid
# bounding box
if bbox is not None:
# given is: bbox = [lonmin, latmin, lonmax, latmax]
self.lonmin, self.latmin, self.lonmax, self.latmax = (*bbox, )
self.bbox_gpis = grid.get_bbox_grid_points(
lonmin=self.lonmin,
latmin=self.latmin,
lonmax=self.lonmax,
latmax=self.latmax,
)
grid = grid.subgrid_from_gpis(self.bbox_gpis)
self.grid = grid.to_cell_grid(cellsize=self.cellsize)
print(f"Number of active gpis: {len(self.grid.activegpis)}")
print(f"Number of grid cells: {len(self.grid.get_cells())}")
# create metadata dictionary from dataset attributes
# this copies the dataset metadata directly and appends metadata of the
# single variables with <param_name>_ as prefix to their metadata keys.
self.metadata = copy(self.dataset.attrs)
array_metadata = {}
for p in self.parameters:
md = copy(self.dataset[p].attrs)
for key in md:
array_metadata["_".join([p, key])] = md[key]
self.metadata.update(array_metadata)
def reshuffle(input_root,
outputpath,
startdate,
enddate,
parameters,
land_points=True,
imgbuffer=50):
"""
Reshuffle method applied to C3S data.
Parameters
----------
input_root: string
input path where c3s images were downloaded.
outputpath : string
Output path.
startdate : datetime
Start date.
enddate : datetime
End date.
parameters: list
parameters to read and convert
land_points : bool, optional (default: True)
Use the land grid to calculate time series on.
Leads to faster processing and smaller files.
imgbuffer: int, optional (default: 50)
How many images to read at once before writing time series.
"""
if land_points:
grid = C3SLandGrid()
else:
grid = C3SCellGrid()
gpis, lons, lats, cells = grid.get_grid_points()
grid_vars = {'gpis': gpis, 'lons': lons, 'lats': lats}
# repurpose cannot handle masked arrays
for k, v in grid_vars.items(): # type v: np.ma.MaskedArray
if isinstance(v, np.ma.MaskedArray):
grid_vars[k] = v.filled()
grid = BasicGrid(lon=grid_vars['lons'],
lat=grid_vars['lats'],
gpis=grid_vars['gpis']).to_cell_grid(5.)
if parameters is None:
file_args, file_vars = parse_filename(input_root)
parameters = [p for p in file_vars if p not in ['lat', 'lon', 'time']]
input_dataset = C3S_Nc_Img_Stack(data_path=input_root,
parameters=parameters,
subgrid=grid,
array_1D=True)
prod_args = input_dataset.fname_args
kwargs = {
'product_sensor_type': prod_args['sensor_type'].lower(),
'sub_version': '.' + prod_args['sub_version'],
'product_sub_type': prod_args['sub_prod']
}
class_str = "C3S_SM_TS_Attrs_%s" % (prod_args['version'])
subattr = getattr(metadata, class_str)
if prod_args['temp_res'] == 'DAILY':
attrs = C3S_daily_tsatt_nc(subattr, **kwargs)
else:
attrs = C3S_dekmon_tsatt_nc(subattr, **kwargs)
ts_attributes = {}
global_attributes = attrs.global_attr
# todo: attrs for all vars or only for the ones that TS were created for.
for var in parameters:
ts_attributes.update(attrs.ts_attributes[var])
if not os.path.exists(outputpath):
os.makedirs(outputpath)
reshuffler = Img2Ts(input_dataset=input_dataset,
outputpath=outputpath,
startdate=startdate,
enddate=enddate,
input_grid=grid,
imgbuffer=imgbuffer,
cellsize_lat=5.0,
cellsize_lon=5.0,
global_attr=global_attributes,
zlib=True,
unlim_chunksize=1000,
ts_attributes=ts_attributes)
reshuffler.calc()
def reshuffle(input_root,
outputpath,
startdate,
enddate,
parameters=None,
land_points=True,
ignore_meta=False,
imgbuffer=200):
"""
Reshuffle method applied to ESA CCI SM images.
Parameters
----------
input_root: string
input path where era interim data was downloaded
outputpath : string
Output path.
startdate : datetime
Start date.
enddate : datetime
End date.
parameters: list, optional (default: None)
parameters to read and convert
If none are passed, we read an image in the root path and use vars from
the image.
land_points : bool, optional (default: True)
Use the land grid to calculate time series on.
Leads to faster processing and smaller files.
imgbuffer: int, optional
How many images to read at once before writing time series.
"""
if land_points:
grid = CCILandGrid()
else:
grid = CCICellGrid()
gpis, lons, lats, cells = grid.get_grid_points()
grid_vars = {'gpis': gpis, 'lons': lons, 'lats': lats}
# repurpose cannot handle masked arrays
for k, v in grid_vars.items(): # type v: np.ma.MaskedArray
if isinstance(v, np.ma.MaskedArray):
grid_vars[k] = v.filled()
grid = BasicGrid(lon=grid_vars['lons'],
lat=grid_vars['lats'],
gpis=grid_vars['gpis']).to_cell_grid(5.)
if not os.path.exists(outputpath):
os.makedirs(outputpath)
file_args, file_vars = parse_filename(input_root)
if parameters is None:
parameters = [p for p in file_vars if p not in ['lat', 'lon', 'time']]
input_dataset = CCI_SM_025Ds(data_path=input_root,
parameter=parameters,
subgrid=grid,
array_1D=True)
if not ignore_meta:
global_attr, ts_attributes = read_metadata(
sensortype=file_args['sensor_type'],
version=int(file_args['version']),
varnames=parameters,
subversion=file_args['sub_version'])
else:
global_attr = {'product': 'ESA CCI SM'}
ts_attributes = None
reshuffler = Img2Ts(input_dataset=input_dataset,
outputpath=outputpath,
startdate=startdate,
enddate=enddate,
input_grid=grid,
imgbuffer=imgbuffer,
cellsize_lat=5.0,
cellsize_lon=5.0,
global_attr=global_attr,
zlib=True,
unlim_chunksize=1000,
ts_attributes=ts_attributes)
reshuffler.calc()
def __init__(
self,
filename_pattern,
parameter="mrsos",
cellsize=5.0,
only_land=False,
bbox=None,
):
self.parameter = parameter
self.cellsize = cellsize
self.only_land = only_land
# open dataset with xarray, using dask backend for parallel access
ds = xr.open_mfdataset(
str(filename_pattern), parallel=True, concat_dim="time"
)
self.dataset = ds.stack(dimensions={"latlon": ("lat", "lon")})
# lons are between 0 and 360, they have to be remapped to (-180, 180)
self._lons = np.array(self.dataset.lon.values)
self._lons[self._lons > 180] -= 360
# setup grid
global_grid = BasicGrid(self.lon, self.lat)
# land mask
self.landmask = ~np.isnan(self.dataset[parameter].isel(time=0).values)
self.land_gpis = global_grid.get_grid_points()[0][self.landmask]
if self.only_land:
grid = global_grid.subgrid_from_gpis(self.land_gpis)
else:
grid = global_grid
# bounding box
if bbox is not None:
# given is: bbox = [lonmin, latmin, lonmax, latmax]
self.lonmin, self.latmin, self.lonmax, self.latmax = (*bbox,)
self.bbox_gpis = grid.get_bbox_grid_points(
lonmin=self.lonmin,
latmin=self.latmin,
lonmax=self.lonmax,
latmax=self.latmax,
)
grid = grid.subgrid_from_gpis(self.bbox_gpis)
self.grid = grid.to_cell_grid(cellsize=self.cellsize)
print(f"Number of active gpis: {len(self.grid.activegpis)}")
print(f"Number of grid cells: {len(self.grid.get_cells())}")
# create metadata dictionary from dataset attributes
self.metadata = copy(self.dataset.attrs)
array_metadata = copy(self.dataset[self.parameter].attrs)
# merging history metadata (the only common keyword)
self.metadata["history"] = (
"Dataset: "
+ self.metadata["history"]
+ "; DataArray: "
+ array_metadata["history"]
)
del array_metadata["history"]
self.metadata.update(array_metadata)
def ERA_IrregularImgGrid(lons, lats):
lons_gt_180 = np.where(lons > 180.0)
lons[lons_gt_180] = lons[lons_gt_180] - 360
return BasicGrid(lons.flatten(), lats.flatten()).to_cell_grid(cellsize=5.)
