def test_xarray(self):
from salem import open_xr_dataset
go = get_demo_file("hef_srtm.tif")
gs = get_demo_file("hef_srtm_subset.tif")
geo = GeoTiff(go)
go = open_xr_dataset(go)
gs = open_xr_dataset(gs)
gos = go.salem.subset(grid=gs.salem.grid)
ref = gs["data"]
totest = gos["data"]
np.testing.assert_array_equal(ref.shape, (gos.salem.grid.ny, gos.salem.grid.nx))
np.testing.assert_array_equal(ref.shape, totest.shape)
np.testing.assert_array_equal(ref, totest)
rlon, rlat = geo.grid.ll_coordinates
tlon, tlat = go.salem.grid.ll_coordinates
assert_allclose(rlon, tlon)
assert_allclose(rlat, tlat)
def test_example_docs():
import salem
from salem.utils import get_demo_file
ds = salem.open_xr_dataset(get_demo_file('wrfout_d01.nc'))
t2 = ds.T2.isel(Time=2)
t2_sub = t2.salem.subset(corners=((77., 20.), (97., 35.)),
crs=salem.wgs84)
shdf = salem.read_shapefile(get_demo_file('world_borders.shp'))
shdf = shdf.loc[shdf['CNTRY_NAME'].isin(
['Nepal', 'Bhutan'])] # GeoPandas' GeoDataFrame
t2_sub = t2_sub.salem.subset(shape=shdf, margin=2) # add 2 grid points
t2_roi = t2_sub.salem.roi(shape=shdf)
smap = t2_roi.salem.get_map(data=t2_roi-273.15, cmap='RdYlBu_r', vmin=-14, vmax=18)
_ = smap.set_topography(get_demo_file('himalaya.tif'))
smap.set_shapefile(shape=shdf, color='grey', linewidth=3)
smap.set_points(91.1, 29.6)
smap.set_text(91.2, 29.7, 'Lhasa', fontsize=17)
smap.set_data(ds.T2.isel(Time=1)-273.15, crs=ds.salem.grid)
fig, ax = plt.subplots(1, 1)
smap.visualize(ax=ax)
plt.tight_layout()
return fig
def regrid(lakedepth, bathy):
'''
将obs数据插值到WRF网格上
'''
target = lakedepth.isel(Time=0)
regridder = xesmf.Regridder(bathy, target, 'bilinear')
bathy_regrid = regridder(bathy)
regridder.clean_weight_file() # 清理中间文件
return bathy_regrid
if __name__ == '__main__':
file_path = '/home/Public_Data/NamCo_Station_Data/NamCo_Bathymetry.nc'
bathy = salem.open_xr_dataset(file_path)['Bathy']
data_dir = wps_dir
for i in range(max_dom):
geo_file = f'geo_em.d0{i+1}.nc'
with xr.open_dataset(os.path.join(data_dir, geo_file)) as ds1:
with salem.open_xr_dataset(os.path.join(data_dir,
geo_file)) as ds2:
lakedepth = ds2['LAKE_DEPTH']
lakedepth_NamCo = lakedepth.salem.roi(
shape=load_NamCo_shp()).isel(Time=0)
mask = lakedepth_NamCo.notnull()
del mask.coords['west_east'], mask.coords['south_north']
### 不加坐标不给插值啊
lat, lon = ds2['XLAT_M'], ds2['XLONG_M']
if __name__ == '__main__':
# Прописываем пути к файлам
nc_path = r'..\..\storage\C3S-LC-L4-LCCS-Map-300m-P1Y-2018-v2.1.1.nc'
import cartopy.crs as ccrs
import cartopy.feature as cf
import matplotlib.pyplot as plt
import numpy as np
proj = ccrs.LambertConformal(central_latitude=60,
central_longitude=50,
standard_parallels=(25, 25))
print('subsetting')
dat = salem.open_xr_dataset(nc_path)
dat = dat['lccs_class'].isel(time=0)
#shapes = salem.read_shapefile("..\data\\regions\szfo\Arkhangelskaya_oblast.shp")
shapes = salem.read_shapefile(
"..\data\\regions\szfo\Pskovskaya_oblast.shp")
dat = dat.salem.subset(shape=shapes, margin=2)
print('set values')
lat = dat.coords['lat'].values
lon = dat.coords['lon'].values
print(dat)
#dat=dat.values
print(dat)
print('plot')
ax = plt.axes(projection=proj)
src = rasterio.open(vel_file)
# Retrieve the affine transformation
if isinstance(src.transform, Affine):
transform = src.transform
else:
transform = src.affine
N = src.width
M = src.height
dx = transform.a
dy = transform.e
minx = transform.c
maxy = transform.f
dvel = salem.open_xr_dataset(vel_file)
data = dvel.data.values
# Read the image data, flip upside down if necessary
data_in = data
if dy < 0:
dy = -dy
data_in = np.flip(data_in, 0)
# Generate X and Y grid locations
xdata = minx + dx/2 + dx*np.arange(N)
ydata = maxy - dy/2 - dy*np.arange(M-1,-1,-1)
# Scale the velocities by the log of the data.
d = np.log(np.clip(data_in, 1, 3000))
cached=True)
lakes = salem.read_shapefile(
'/users/global/cornkle/data/pythonWorkspace/proj_CEH/shapes/lakes/ne_10m_lakes.shp',
cached=True)
#map.set_lonlat_contours(interval=0)
map.set_shapefile(countries=False)
map.set_shapefile(rivers=True)
map.set_shapefile(lakes,
edgecolor='k',
facecolor='grey',
alpha=0.4,
linewidth=2,
linestyle='--')
srtm = open_xr_dataset(get_demo_file('hef_srtm.tif'))
srtm_on_ds = ds.salem.lookup_transform(srtm)
# t_on_ds = ds.salem.transform(t)
# t2_on_ds = ds.salem.transform(t2)
grid = ds.salem.grid
srtm_on_ds = ds.salem.lookup_transform(top)
#t_on_ds = ds.salem.lookup_transform(t, grid=grid)
#
#deforestation
# g = GeoTiff(lst)
# ex = grid.extent_in_crs(crs=wgs84) # l, r, b, t
# g.set_subset(corners=((ex[0], ex[2]), (ex[1], ex[3])),
# crs=wgs84, margin=10)
# ls = g.get_vardata()
add_lakes(ax)
### 去除地图边框
ax.background_patch.set_visible(False)
ax.outline_patch.set_visible(False)
fig.savefig(f'fig2/{figname}.jpg',
dpi=300,
bbox_inches='tight',
pad_inches=0.1)
#%%
if __name__ == '__main__':
data_path = '/home/zzhzhao/data/CMFD_Prec_TP/CMFD_Prec_TP_1979-2018.nc'
with salem.open_xr_dataset(data_path)['prec'].sel(
time=slice('1998', '2018')) as prec:
### 筛选出季节
# prec_daily = prec.resample(time='D').sum() # 转换为日平均
# prec_seasons = dict(prec_daily.groupby('time.season'))
prec_seasons = dict(prec.groupby('time.season'))
#%%
seasons = ['MAM', 'JJA', 'SON', 'DJF']
# seasons = ['JJA']
for season in seasons:
print(f'** {season} **')
### 提取青藏高原，各个季节平均
print('>> 开始计算降水气候态 <<')
prec_season_mean = prec_seasons[season].mean(dim='time')
prec_season_mean_TP = prec_season_mean.salem.roi(shape=load_TPshp())
var1 = prec_season_mean_TP * 24 * 92 # 季度降水
def main(agglom_extent_filepath, station_tair_filepath,
landsat_features_filepath, swiss_dem_filepath, regressor_filepath,
dst_filepath, dst_res, buffer_dist):
logger = logging.getLogger(__name__)
# Compute an air temperature array from the trained regressor
# 0. Preprocess the inputs
# get the agglomeration extent
agglom_extent_gdf = gpd.read_file(agglom_extent_filepath)
crs = agglom_extent_gdf.crs
data_geom = agglom_extent_gdf.loc[0]['geometry']
# add a buffer to compute the convolution features well
ref_geom = data_geom.buffer(buffer_dist)
# use the ref geometry to obtain the reference grid (data array) with the
# target resolution
ref_da = utils.get_ref_da(ref_geom, dst_res, dst_fill=0, dst_crs=crs)
# read the dates from the air temperature station measurements data frame
# we need at least series to groupby year and access the group series
# whose index will be the dates in its respective year
date_ser = pd.Series(0,
index=pd.to_datetime(
pd.read_csv(station_tair_filepath,
index_col=0).index))
# 1. Prepare the regression features
# 1.1-1.2 Landsat features
# open the dataset
landsat_features_ds = xr.open_dataset(landsat_features_filepath)
# note that we need to forward the dataset attributes to its data variables
for data_var in landsat_features_ds.data_vars:
landsat_features_ds[data_var].attrs = landsat_features_ds.attrs.copy()
# align it
landsat_features_ds = ref_da.salem.transform(landsat_features_ds,
interp='linear')
# spatial averaging
kernel_dict = regr_utils.get_kernel_dict(res=dst_res)
for landsat_feature in regr_utils.LANDSAT_BASE_FEATURES:
landsat_features_ds = landsat_features_ds.assign(
landsat_features_ds[landsat_feature].groupby('time').map(
get_savg_feature_ds,
args=(kernel_dict, ))).drop(landsat_feature)
landsat_features_ds = landsat_features_ds.salem.subset(geometry=data_geom,
crs=crs)
# 1.3 Elevation
# dem_s3_filepath = path.join(bucket_name, dem_file_key)
# with fs_s3.open(dem_s3_filepath) as dem_file_obj:
# dem_da = utils.salem_da_from_singleband(swiss_dem_filepath)
dem_da = salem.open_xr_dataset(swiss_dem_filepath)['data']
# align it
dem_arr = landsat_features_ds.salem.transform(dem_da,
interp='linear').values
# 2. Use the trained regressor to predict the air temperature at the
# target resolution
regr = jl.load(regressor_filepath)
T_pred_da = xr.DataArray(
[
predict_T(regr, landsat_features_ds.sel(time=date), dem_arr)
for date in date_ser.index
],
dims=('time', 'y', 'x'),
coords={
'time': date_ser.index,
'y': landsat_features_ds.y,
'x': landsat_features_ds.x
},
name='T',
attrs=landsat_features_ds.attrs)
# 3. Crop the data array to the valid data region and dump it to a file
T_pred_da.salem.roi(geometry=data_geom, crs=crs).to_netcdf(dst_filepath)
logger.info("dumped predicted air temperature data array to %s",
dst_filepath)
'''
target = tsk.isel(Time=0).rename({'XLONG': 'lon', 'XLAT': 'lat'})
regridder = xesmf.Regridder(modis_lst.isel(time=0), target, 'nearest_s2d')
modis_regrid = regridder(modis_lst)
regridder.clean_weight_file() # 清理中间文件
modis_regrid = modis_regrid.rename({'lon': 'XLONG', 'lat': 'XLAT'})
return modis_regrid
if __name__ == '__main__':
data_dir = '/home/zzhzhao/Model/tests/test-9.4/WRF/run/'
wrfinput_file = 'wrfinput_d01'
with xr.open_dataset(data_dir+wrfinput_file) as ds1:
with salem.open_xr_dataset(data_dir+wrfinput_file) as ds2:
shp = load_NamCo_shp()
tsk = ds2['TSK']
tsk_lake = tsk.salem.roi(shape=shp).isel(Time=0)
mask = tsk_lake.notnull()
del mask.coords['west_east'], mask.coords['south_north']
### 读取MOD11A1
modis = xr.open_dataset('/home/zzhzhao/code/python/replace_sh/MOD11A2.006_1km_aid0001.nc')
modis_lst = modis['LST_Day_1km']
### 将MODIS插值到WRF网格
modis_regrid = regrid(tsk, modis_lst)
### 替换
def __init__(self, nc_path, shape_path):
self.data_set = salem.open_xr_dataset(nc_path)
self.shape_path = salem.read_shapefile(shape_path)
pass
def print_types(self):
print(type(self.data_set), self.data_set)
nc_path = r'..\..\storage\C3S-LC-L4-LCCS-Map-300m-P1Y-2018-v2.1.1.nc'
shp_path = r"C:\NETCDF_PROJECT\storage\Belgorod\Adminbndy3.shp"
MC = MaskClipper(nc_path, shp_path)
MC.create_variable('lccs_class', 0)
MC.print_types()
MC.subset()
MC.clip()
MC.draw_map()
print('F**K')
ds = salem.open_xr_dataset(r'..\..\storage\C3S-LC-L4-LCCS-Map-300m-P1Y-2018-v2.1.1.nc')
#ds = salem.open_xr_dataset(get_demo_file('wrfout_d01.nc'))
#shdf = salem.read_shapefile(get_demo_file(r"C:\NETCDF_PROJECT\storage\Belgorod\Adminbndy3.shp"))
shdf_bel = salem.read_shapefile(r"C:\NETCDF_PROJECT\storage\Belgorod\Adminbndy3.shp")
#shdf = salem.read_shapefile(get_demo_file('world_borders.shp'))
#print(type(shdf), shdf)
print(type(shdf_bel), shdf_bel)
#shdf = shdf.loc[shdf['CNTRY_NAME'].isin(['Nepal', 'Bhutan'])]
#print(type(shdf), shdf)
t2 = ds.lccs_class.isel(time=0)
#t2 = ds.T2.isel(Time=2)
#t2 = t2.salem.subset(shape=shdf, margin=2)
from modify_nml2 import max_dom, wps_dir
import os
def load_mountain_shp():
f_in = '/home/zzhzhao/code/shpfiles/Nq_shp/Nq.shp'
shp = geopandas.read_file(f_in, encoding='gbk')
shp = shp.to_crs(epsg=4326)
return shp
if __name__ == '__main__':
data_dir = wps_dir
for i in range(max_dom):
geo_file = f'geo_em.d0{i+1}.nc'
with xr.open_dataset(os.path.join(data_dir, geo_file)) as ds1:
with salem.open_xr_dataset(os.path.join(data_dir, geo_file)) as ds2: # 只能以salem这种方式来读取，不然会损失grid信息
### 以两种方式读取的原因是修改salem（内层）读出来的数据，然后填充进纯净的xarray读出的数据（外层）中
shp = load_mountain_shp()
hgt = ds2['HGT_M']
hgt_mountain = hgt.salem.roi(shape=shp).isel(Time=0)
# mask = xr.where(lu_lake==21, True, False)
mask = hgt_mountain.notnull()
del mask.coords['west_east'], mask.coords['south_north']
### 变量的metadata和coords要跟原始一致
### 修改HGT_M
hgt = ds1['HGT_M']
hgt_new = hgt.copy()
scale = 0.1
base_hight = 4718
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_context('talk')
sns.set_style('whitegrid')
in_xls_dir = Path('output/xls/obs')
out_img_dir = Path('output/img/obs/compare_grids')
out_img_dir.mkdir(parents=True, exist_ok=True)
var_name = 'precip'
in_files = list(in_xls_dir.glob('*.xlsx'))
dem_ds = salem.open_xr_dataset(Path('input/nc/dem/dem.nc'))
for in_file in in_files:
print(in_file)
in_xlsx = pd.ExcelFile(in_file)
for ws_name in in_xlsx.sheet_names:
in_df = pd.read_excel(in_xlsx, sheet_name=ws_name)
in_hi_df = in_df.groupby(['lon', 'lat']).filter(lambda x: dem_ds.sel(
lon=x.name[0], lat=x.name[1])['Band1'].values > 200)
in_lo_df = in_df.groupby(['lon', 'lat']).filter(lambda x: dem_ds.sel(
lon=x.name[0], lat=x.name[1])['Band1'].values <= 200)
in_names = ['all', 'hi', 'lo']
for idx, _in_df in enumerate([in_df, in_hi_df, in_lo_df]):
if _in_df.size == 0:
continue
the topography from the SRTM v4.1 dataset (resolution 3 minutes of arc,
so ~ 90m). For this we use the :py:meth:`DataArrayAccessor.lookup_transform`
method.
From the plot below, we see that the model topography is smoother than the
aggregated SRTM (this is a good thing, as atmospheric models do not like
steep gradients too much). The highest peaks or lowest valley aren't resolved
by the 1km topography.
"""
import numpy as np
from salem import get_demo_file, open_xr_dataset
import matplotlib.pyplot as plt
# get the topography data
srtm = open_xr_dataset(get_demo_file('riosan_srtm_hgt.nc')).srtm
wrf = open_xr_dataset(get_demo_file('riosan_wrf_hgt.nc')).HGT
# transform the high-res topography onto the coarse grid
# we ask for the lookup table to speed up the second transform
srtm_on_wrf, lut = wrf.salem.lookup_transform(srtm, return_lut=True)
srtm_on_wrf_std = wrf.salem.lookup_transform(srtm, method=np.std, lut=lut)
# for fun we compute the max and min for each grid point
srtm_on_wrf_min = wrf.salem.lookup_transform(srtm, method=np.min, lut=lut)
srtm_on_wrf_max = wrf.salem.lookup_transform(srtm, method=np.max, lut=lut)
# then compute the max absolute difference to wrf
absdif = np.abs(np.dstack([srtm_on_wrf_min - wrf, srtm_on_wrf_max - wrf]))
maxabsdif = np.max(absdif, axis=2)
# Get the map defined by the WRF grid
and proposes an alternative based on cartopy.
The only problem at the pole is the plotting of the longitude contours.
There is probably no easy way so solve this problem in salem,
hence the suggestion to use cartopy in this case. Note that if the pole isn't
in the domain, salem works just fine.
"""
from salem import open_xr_dataset, get_demo_file
import matplotlib.pyplot as plt
# prepare the figure
f = plt.figure(figsize=(8, 7))
# WRF polar file(s)
d1 = open_xr_dataset(get_demo_file('geo_em_d01_polarstereo.nc'))
d2 = open_xr_dataset(get_demo_file('geo_em_d02_polarstereo.nc'))
# Plot with salem
ax = plt.subplot(2, 2, 1)
d1.HGT_M.salem.quick_map(ax=ax, cmap='Oranges')
ax = plt.subplot(2, 2, 3)
d2.HGT_M.salem.quick_map(ax=ax, cmap='Oranges')
# Now with cartopy
proj = d1.salem.cartopy()
ax = plt.subplot(2, 2, 2, projection=proj)
ax.coastlines()
ax.gridlines()
d1.HGT_M.plot(ax=ax, transform=proj, cmap='Oranges')
ax.set_extent(d1.salem.grid.extent, crs=proj)
#map.set_shapefile(oceans=True)
# read the ocean shapefile (data from http://www.naturalearthdata.com)
oceans = salem.read_shapefile(salem.get_demo_file('ne_50m_ocean.shp'),
cached=True)
river = salem.read_shapefile('/users/global/cornkle/data/pythonWorkspace/proj_CEH/shapes/rivers/ne_10m_rivers_lake_centerlines.shp', cached=True)
lakes = salem.read_shapefile('/users/global/cornkle/data/pythonWorkspace/proj_CEH/shapes/lakes/ne_10m_lakes.shp', cached=True)
#map.set_lonlat_contours(interval=0)
map.set_shapefile(countries=False)
#map.set_shapefile(rivers=True)
map.set_shapefile(lakes, edgecolor='k', facecolor='grey',alpha=0.4, linewidth=2, linestyle='--')
srtm = open_xr_dataset(get_demo_file('hef_srtm.tif'))
srtm_on_ds = ds.salem.lookup_transform(srtm)
t_on_ds = ds.salem.transform(t)
t2_on_ds = ds.salem.transform(t2)
grid = ds.salem.grid
srtm_on_ds = ds.salem.lookup_transform(top)
#t_on_ds = ds.salem.lookup_transform(t, grid=grid)
#
#deforestation
g = GeoTiff(lst)
ex = grid.extent_in_crs(crs=wgs84) # l, r, b, t
g.set_subset(corners=((ex[0], ex[2]), (ex[1], ex[3])),
crs=wgs84, margin=10)
ls = g.get_vardata()
ls = np.array(ls, dtype=float)
basin_shp.parent.name, exp_name, var_name)
out_csv_dir2.mkdir(parents=True, exist_ok=True)
# rcm loop
for rcm_name in RCMS:
print('Processing {} --> {} --> {} --> {}'.format(
basin_shp.parent.name, exp_name, var_name, rcm_name))
out_csv_file_name = out_csv_dir2 / '{}.csv'.format(rcm_name)
if not out_csv_file_name.is_file():
#region read rcm input
nc_file = list(
in_nc_dir.glob('{}_{}_{}_*.nc'.format(
rcm_name, exp_name, var_name)))
if len(nc_file) == 0:
continue
nc_file = nc_file[0]
in_ds = salem.open_xr_dataset(nc_file).sel(time=date_range)
#endregion read rcm input
#region get region of interest
out_ds = in_ds.salem.subset(shape=basin_gdf, margin=1)
out_ds = out_ds.salem.roi(shape=basin_gdf,
all_touched=True)
#endregion get region of interest
#region prepare data for writing
out_df = out_ds[var_name].to_dataframe().reset_index(
).groupby(['lat', 'lon'
]).filter(lambda x: not all(x[var_name].isna()))
if isinstance(out_df.iloc[0]['time'],
cftime._cftime.DatetimeNoLeap) or isinstance(
out_df.iloc[0]['time'],
