def smap_p_e_exact_downscale(doy_start, doy_end):
"""
smap_p_e usa 9km index range: lat: [171:507] lon: [568: 1241], starting from 0, included
corresponding 3km index range: lat: [513:1523] lon [1704:3725], starting from 0, included
"""
in_path = os.path.join("Data", "SMAP_P_E", "usa")
out_path = get_out_path(os.path.join("Data", "SMAP_P_E", "usa_3km_exact"))
cont_var_dic = [
"soil_moisture", "tb_v_corrected", "freeze_thaw_fraction",
"roughness_coefficient", "surface_temperature", "vegetation_opacity",
"vegetation_water_content", "albedo"
]
for doy in generate_doy(doy_start, doy_end, ""):
print(doy)
fh_in = Dataset(os.path.join(in_path, doy + ".nc"), "r")
fh_out = Dataset(os.path.join(out_path, doy + ".nc"), "w")
lats, lons = get_lat_lon("M03")
lats = lats[513:1524]
lons = lons[1704:3726]
fh_out.createDimension('lat', len(lats))
fh_out.createDimension('lon', len(lons))
outVar = fh_out.createVariable('lat', 'f4', ('lat', ))
outVar.setncatts({"units": "degree_north"})
outVar[:] = lats[:]
outVar = fh_out.createVariable('lon', 'f4', ('lon', ))
outVar.setncatts({"units": "degree_east"})
outVar[:] = lons[:]
datatype = None
tb_value, ts_value = None, None
for v_name, varin in fh_in.variables.items():
if v_name in cont_var_dic:
outVar = fh_out.createVariable(v_name, varin.datatype,
varin.dimensions)
outVar.setncatts(
{k: varin.getncattr(k)
for k in varin.ncattrs()})
varin_value = varin[:]
varin_value = np.repeat(varin_value, 3, axis=0)
varin_value = np.repeat(varin_value, 3, axis=1)
outVar[:] = varin_value[:]
if v_name == "tb_v_corrected":
datatype = varin.datatype
tb_value = varin_value[:]
if v_name == "surface_temperature":
ts_value = varin_value[:]
outVar = fh_out.createVariable("tb_divide_ts", datatype,
("lat", "lon"))
outVar[:] = ma.divide(tb_value, ts_value)
fh_in.close()
fh_out.close()
def generate_rb_hist_n(f_name, n, doy_start, doy_end):
fh_in = Dataset(os.path.join("Data", "Sentinel", f_name + ".nc"), "r")
out_path = get_out_path(
os.path.join("Data", "Sentinel", "usa_rb_hist_" + str(n)))
init_doy, final_doy = f_name.split("_")[1], f_name.split("_")[2]
init_doy = date(*map(int, [init_doy[:4], init_doy[4:6], init_doy[6:]]))
final_doy = date(*map(int, [final_doy[:4], final_doy[4:6], final_doy[6:]]))
doy_s = date(*map(int, [doy_start[:4], doy_start[4:6], doy_start[6:]]))
doy_e = date(*map(int, [doy_end[:4], doy_end[4:6], doy_end[6:]]))
assert ((doy_s - init_doy).days >= n)
assert ((final_doy - doy_e).days >= 0)
i_doy = (doy_s - init_doy).days
for doy in generate_doy(doy_start, doy_end, ""):
fh_out = Dataset(os.path.join(out_path, doy + ".nc"), "w")
for name, dim in fh_in.dimensions.items():
if name != "time":
fh_out.createDimension(name, len(dim))
for v_name, varin in fh_in.variables.items():
if v_name == 'lat' or v_name == 'lon':
outVar = fh_out.createVariable(v_name, varin.datatype,
varin.dimensions)
outVar.setncatts(
{k: varin.getncattr(k)
for k in varin.ncattrs()})
outVar[:] = varin[:]
elif v_name != "time":
outVar = fh_out.createVariable(v_name + "_hist_mean_" + str(n),
varin.datatype, (
"lat",
"lon",
))
outVar.setncatts(
{k: varin.getncattr(k)
for k in varin.ncattrs()})
s_doy = i_doy - n
print(s_doy, i_doy)
outVar[:] = ma.mean(varin[s_doy:i_doy, :, :], axis=0)
outVar = fh_out.createVariable(v_name + "_hist_std_" + str(n),
varin.datatype, (
"lat",
"lon",
))
outVar[:] = ma.std(varin[s_doy:i_doy, :, :], axis=0)
i_doy += 1
fh_out.close()
fh_in.close()
def exact_downscale_resize_to_match_sentinel(doy_start, doy_end):
"""
smap_p_e usa 3km index range: lat: [513:1523] lon [1704:3725], starting from 0, included
smap_ap usa 3km index range: lat: [566:1444] lon: [1767:3662], starting from 0, included
"""
print("Resize")
in_path = os.path.join("Data", "SMAP_P_E", "usa_3km_exact")
out_path = get_out_path(
os.path.join("Data", "SMAP_P_E", "usa_3km_exact_match_sentinel"))
for doy in generate_doy(doy_start, doy_end, ""):
print(doy)
fh_in = Dataset(os.path.join(in_path, doy + ".nc"), mode='r')
fh_out = Dataset(os.path.join(out_path, doy + ".nc"), 'w')
lats, lons = get_lat_lon("M03")
lats = lats[566:1445]
lons = lons[1767:3663]
fh_out.createDimension('lat', len(lats))
fh_out.createDimension('lon', len(lons))
outVar = fh_out.createVariable('lat', 'f4', ('lat', ))
outVar.setncatts({"units": "degree_north"})
outVar[:] = lats[:]
outVar = fh_out.createVariable('lon', 'f4', ('lon', ))
outVar.setncatts({"units": "degree_east"})
outVar[:] = lons[:]
for v_name, varin in fh_in.variables.items():
if v_name not in ["lat", "lon"]:
outVar = fh_out.createVariable(v_name, varin.datatype,
varin.dimensions)
outVar.setncatts(
{k: varin.getncattr(k)
for k in varin.ncattrs()})
outVar[:] = varin[53:932, 63:1959]
fh_in.close()
fh_out.close()
def generate_rb_hist_average_time_window_by_doy(doys, n, var_list):
out_path = get_out_path(
os.path.join("Data", "Sentinel", "usa_rb_hist_average_" + str(n)))
in_path = os.path.join("Data", "Sentinel", "usa_9km_all")
for doy in doys:
fh_in = Dataset(os.path.join(in_path, doy + ".nc"), "r")
fh_out = Dataset(os.path.join(out_path, doy + ".nc"), "w")
hist_vars = {}
datatype = None
n_lat, n_lon = len(fh_in.variables["lat"][:]), len(
fh_in.variables["lon"][:])
for name, dim in fh_in.dimensions.items():
fh_out.createDimension(name, len(dim))
for v_name, varin in fh_in.variables.items():
if v_name in var_list:
if "9km" not in v_name:
outVar = fh_out.createVariable(
v_name + "_hist_mean_" + str(n), varin.datatype, (
"lat",
"lon",
))
outVar.setncatts(
{k: varin.getncattr(k)
for k in varin.ncattrs()})
fh_out.createVariable(v_name + "_hist_std_" + str(n),
varin.datatype, (
"lat",
"lon",
))
hist_vars[v_name] = ma.empty([30, n_lat, n_lon])
else:
datatype = varin.datatype
outVar = fh_out.createVariable(v_name + "_hist_" + str(n),
varin.datatype, (
"lat",
"lon",
))
outVar.setncatts(
{k: varin.getncattr(k)
for k in varin.ncattrs()})
hist_vars[v_name] = ma.empty([30, n_lat, n_lon])
elif v_name == "lat" or v_name == "lon":
outVar = fh_out.createVariable(v_name, varin.datatype,
varin.dimensions)
outVar.setncatts(
{k: varin.getncattr(k)
for k in varin.ncattrs()})
outVar[:] = varin[:]
i_hist = 0
for hist_doy in generate_most_recent_doys(doy, n, ""):
fh_hist = Dataset(os.path.join(in_path, hist_doy + ".nc"), "r")
for v_name, varin in fh_hist.variables.items():
if v_name in var_list:
hist_vars[v_name][i_hist] = varin[:]
i_hist += 1
fh_hist.close()
for v_name in hist_vars.keys():
if "9km" in v_name:
fh_out.variables[v_name + "_hist_" + str(n)][:] = ma.mean(
hist_vars[v_name], axis=0)
else:
fh_out.variables[v_name + "_hist_mean_" + str(n)][:] = ma.mean(
hist_vars[v_name], axis=0)
fh_out.variables[v_name + "_hist_std_" + str(n)][:] = ma.std(
hist_vars[v_name], axis=0)
for v_name in ["sigma0_vh", "sigma0_vv"]:
outVar = fh_out.createVariable(v_name + "_diff_hist_" + str(n),
datatype, ("lat", "lon"))
outVar[:] = ma.mean(hist_vars[v_name+"_aggregated"], axis=0) - \
ma.mean(hist_vars[v_name+"_aggregated_9km_mean"], axis=0)
fh_in.close()
fh_out.close()
def landcover_upsample(lat1, lat2, lon1, lon2, reso, area_name):
fh_in = Dataset(os.path.join("n5eil01u.ecs.nsidc.org", "MCD12Q1", "15_12_31-16_12_31",
"MCD12Q1.006_500m_aid0001.nc"), "r")
fh_out = Dataset(os.path.join(get_out_path(os.path.join("Data", "LANDCOVER")),
"landcover_" + reso + "_" + area_name + "_2016.nc"), "w")
n_dim = str(int(reso[:-2])).zfill(2)
lats, lons, lat_bins, lon_bins = get_lat_lon_bins("M" + n_dim, lat1, lat2, lon1, lon2)
fill_value = -9999.0
dic_var = {}
for var in ['lat', 'lon']:
dic_var[var] = fh_in.variables[var]
dic_var['lat_value'] = dic_var['lat'][::-1]
dic_var['lon_value'] = dic_var['lon'][:]
fh_out.createDimension('lat', len(lats))
fh_out.createDimension('lon', len(lons))
for var in ['lat', 'lon']:
outVar = fh_out.createVariable(var, 'f4', (var,))
outVar.setncatts({k: dic_var[var].getncattr(k) for k in dic_var[var].ncattrs()})
outVar[:] = lats if var == "lat" else lons
lc_500m = fh_in.variables["LC_Type1"][1, :, :]
lc_resampled_dic = {}
for v in landcover_class_dic.values():
lc_resampled_dic[v] = np.full((len(lats), len(lons)), 0.0)
for s in ["1", "2", "3"]:
lc_resampled_dic["lc_" + s] = np.full((len(lats), len(lons)), fill_value)
lc_resampled_dic["lc_fraction_" + s] = np.full((len(lats), len(lons)), fill_value)
for id_lats in range(len(lats)):
for id_lons in range(len(lons)):
lats_index = np.searchsorted(dic_var['lat_value'],
[lat_bins[id_lats + 1], lat_bins[id_lats]])
lons_index = np.searchsorted(dic_var['lon_value'],
[lon_bins[id_lons], lon_bins[id_lons + 1]])
if lats_index[0] != lats_index[1] and lons_index[0] != lons_index[1]:
selected = lc_500m[np.array(range(-lats_index[1], -lats_index[0]))[:, None],
np.array(range(lons_index[0], lons_index[1]))]
selected_size = selected.shape[0] * selected.shape[1]
selected_compressed = selected.compressed()
lc_id, lc_count = np.unique(selected_compressed, return_counts=True)
for i, c in zip(lc_id, lc_count):
lc_resampled_dic[landcover_class_dic[i]][id_lats, id_lons] = c / selected_size
lc_count_sort_ind = np.argsort(-lc_count)
for i in range(3):
if len(lc_id) > i:
lc_resampled_dic["lc_" + str(i+1)][id_lats, id_lons] = \
lc_id[lc_count_sort_ind[i]]
lc_resampled_dic["lc_fraction_" + str(i+1)][id_lats, id_lons] = \
lc_count[lc_count_sort_ind[i]] / selected_size
else:
lc_resampled_dic["lc_" + str(i + 1)][id_lats, id_lons] = fill_value
lc_resampled_dic["lc_fraction_" + str(i + 1)][id_lats, id_lons] = fill_value
for v in landcover_class_dic.values():
outVar = fh_out.createVariable("lc_" + v, 'f4', ('lat', 'lon',))
outVar.setncatts({'_FillValue': np.array([-9999.0]).astype('f')})
outVar[:] = lc_resampled_dic[v][:]
outVar[:] = ma.masked_less(outVar, -1)
for s in ["1", "2", "3"]:
for t in ["lc_", "lc_fraction_"]:
outVar = fh_out.createVariable(t + s, 'f4', ('lat', 'lon',))
outVar.setncatts({'_FillValue': np.array([-9999.0]).astype('f')})
outVar[:] = lc_resampled_dic[t+s][:]
outVar[:] = ma.masked_less(outVar, -1)
fh_in.close()
fh_out.close()
def smap_sentinel_upscale(doy_start, doy_end, selected_vars, out_folder,
output_all):
"""
smap_ap usa 3km index range: lat: [566:1444] lon: [1767:3662], starting from 0, included
"""
in_path = os.path.join("Data", "Sentinel", "usa")
out_path = get_out_path(os.path.join("Data", "Sentinel", out_folder))
for doy in generate_doy(doy_start, doy_end, ""):
print(doy)
if doy + ".nc" in os.listdir(in_path):
fh_in = Dataset(os.path.join(in_path, doy + ".nc"), mode='r')
fh_out = Dataset(os.path.join(out_path, doy + ".nc"), 'w')
for name, dim in fh_in.dimensions.items():
fh_out.createDimension(name, len(dim))
for v_name, varin in fh_in.variables.items():
if output_all or v_name in ["lat", "lon"]:
outVar = fh_out.createVariable(v_name, varin.datatype,
varin.dimensions)
outVar.setncatts(
{k: varin.getncattr(k)
for k in varin.ncattrs()})
outVar[:] = varin[:]
if v_name in selected_vars:
outVar = fh_out.createVariable(v_name + "_9km_mean",
varin.datatype,
varin.dimensions)
outVar.setncatts(
{k: varin.getncattr(k)
for k in varin.ncattrs()})
varin_value = varin[1:-2, :]
n_lat, n_lon = varin_value.shape
aggregated_value = np.zeros((n_lat // 3, n_lon // 3))
for i in range(n_lat // 3):
for j in range(n_lon // 3):
aggregated_value[i, j] = ma.mean(
varin_value[i * 3:i * 3 + 3, j * 3:j * 3 + 3])
aggregated_value = np.repeat(aggregated_value, 3, axis=0)
aggregated_value = np.repeat(aggregated_value, 3, axis=1)
outVar[:] = ma.masked_invalid(
np.vstack((varin_value[:1, :], aggregated_value,
varin_value[-2:, :])))
fh_in.close()
fh_out.close()
else:
if output_all:
fh_in = Dataset(os.path.join(in_path, "20150401.nc"), mode='r')
fh_out = Dataset(os.path.join(out_path, doy + ".nc"), 'w')
for name, dim in fh_in.dimensions.items():
fh_out.createDimension(name, len(dim))
for v_name, varin in fh_in.variables.items():
if v_name in ["lat", "lon"]:
outVar = fh_out.createVariable(v_name, varin.datatype,
varin.dimensions)
outVar.setncatts(
{k: varin.getncattr(k)
for k in varin.ncattrs()})
outVar[:] = varin[:]
else:
outVar = fh_out.createVariable(v_name, varin.datatype,
varin.dimensions)
outVar.setncatts(
{k: varin.getncattr(k)
for k in varin.ncattrs()})
fh_in.close()
fh_out.close()