def interpolate_and_save(
var_name,
product,
date_str,
ds_cf,
ds_pf,
in_grid,
out_points,
data_export_dir,
data_BW,
):
file_path = os.path.join(data_export_dir,
f'{var_name}_{product}_{date_str}_bilinear.csv')
print('File save location: ' + file_path)
df_out = pd.DataFrame(columns=tuple(
['locNo', 'date', 'step_fwrd', f'{var_name}_ctrl'] +
[f'{var_name}_ptrb_{num:02}' for num in ds_pf.get_index('number')]))
for step in ds_cf.get_index('step'):
print('Time step: ' + str(step))
# NB: Is this the best way to deal with missings from on-land coordinates?
# NB: Axes of lat/lon are reversed between gridpp and xarray.
print('Filling')
gridpp.fill_missing(
np.transpose(ds_cf.variables[var_name][step.days - 1, :, :].data))
print('Done')
cf_values = gridpp.bilinear(
in_grid, out_points,
gridpp.fill_missing(
np.transpose(ds_cf.variables[var_name][step.days -
1, :, :].data)))
pf_values = np.empty((len(data_BW), len(ds_pf.get_index('number'))),
dtype=float)
for num in ds_pf.get_index('number'):
pf_values[:, num - 1] = gridpp.bilinear(
in_grid, out_points,
gridpp.fill_missing(
np.transpose(ds_pf.variables[var_name][num - 1, step.days -
1, :, :].data)))
for i in range(len(data_BW)):
row_dict = {
'locNo': data_BW['localityNo'][i],
'step_fwrd': step.days,
'date': date_str,
f'{var_name}_ctrl': cf_values[i],
}
for num in ds_pf.get_index('number'):
row_dict[f'{var_name}_ptrb_{num:02}'] = pf_values[i, num - 1]
df_out = df_out.append(pd.Series(row_dict), ignore_index=True)
df_out.to_csv(file_path)
def test_missing_on_y_edge(self):
"""Regression test for bug when X is wider than Y, and a y-slice has missing all the way to the upper edge"""
values0 = np.reshape(np.arange(24), [3, 8]).astype(float)
values = np.copy(values0)
values[1:, 1] = np.nan
output = gridpp.fill_missing(values)
np.testing.assert_array_equal(output, values0)
def test_linear(self):
"""Check that we are able to recover the missing values"""
values0 = np.reshape(np.arange(25), [5, 5]).astype(float)
# Add some missing values
values = np.copy(values0)
values[2, 1:4] = np.nan
values[1, 1] = np.nan
output = gridpp.fill_missing(values)
np.testing.assert_array_equal(output, values0)
def test_missing_on_both_edges(self):
"""Check that we are able to recover the missing values"""
values0 = np.reshape(np.arange(25), [5, 5]).astype(float)
# Add some missing values
values = np.copy(values0)
values[3:5, 3:5] = np.nan
output = gridpp.fill_missing(values)
np.testing.assert_array_equal(output[0:3, :], values0[0:3, :])
np.testing.assert_array_equal(output[:, 0:3], values0[:, 0:3])
np.testing.assert_array_equal(output[3:5, 3:5],
np.nan * np.zeros([2, 2]))
def test_missing_on_edge(self):
"""Check that we can recover when one dimension has missing all the way to the edge"""
values0 = np.reshape(np.arange(25), [5, 5]).astype(float)
# Add some missing values
values = np.copy(values0)
values[1, 1] = np.nan
values[1, 3:5] = np.nan
values[1, 4] = np.nan
values[1, 0:2] = np.nan
output = gridpp.fill_missing(values)
np.testing.assert_array_equal(output, values0)
var_name_abbr=var_name_abbr,
cast_type='cf',
date_str=curr_date,
)
with open(os.path.join(config['BW_DIR'], 'sites.json')) as json_file:
data_BW = pd.DataFrame(json.load(json_file))
BW_grid = gridpp.Points(data_BW.lat, data_BW.lon)
ECMWF_grid1_5deg = make_grid_object(SST_GRID1_5deg)
SST_1_5deg2point = gridpp.nearest(
ECMWF_grid1_5deg,
BW_grid,
gridpp.fill_missing(np.transpose(SST_GRID1_5deg.sst[0, :, :].data))
)
print('test 1 med grid2points')
print(SST_1_5deg2point)
sst = SST_GRID1_5deg.sst[0, :, :].data.flatten()
valid_points = np.isnan(sst) == 0 # Ocean points
ECMWF_points = make_points_flatten(SST_GRID1_5deg.latitude.data, SST_GRID1_5deg.longitude.data, valid_points)
SST_points = gridpp.nearest(ECMWF_points, BW_grid, sst[valid_points])
print('test 2 med points2points')
print(SST_points)
ECMWF_grid1_5deg_valid_points = make_grid_flatten(SST_GRID1_5deg.latitude.data, SST_GRID1_5deg.longitude.data, valid_points)
print('File save location: ' + file_path)
df_out = pd.DataFrame(
columns=tuple(['locNo', 'date', 'step_fwrd', 'sst_ctrl'] +
['sst_ptrb_%02d' % (i) for i in ds_pf.get_index('number')]))
for step in ds_cf.get_index('step'):
print('Time step: ' + str(step))
# NB: Is this the best way to deal with missings from on-land coordinates?
# NB: Axes of lat/lon are reversed between gridpp and xarray.
cf_ovalues_grid1deg = gridpp.bilinear(
ECMWF_grid, ECMWF_grid1deg,
np.transpose(ds_cf.sst[step.days - 1, :, :].data))
cf_values = gridpp.bilinear(ECMWF_grid1deg, BW_grid,
gridpp.fill_missing(cf_ovalues_grid1deg))
# cf_nearest= gridpp.nearest(
## ECMWF_grid,
# BW_grid,
# np.transpose(ds_cf.sst[step.days - 1,:,:].data)
# )
pf_values = np.empty((len(data_BW), len(ds_pf.get_index('number'))),
dtype=float)
for num in ds_pf.get_index('number'):
pf_ovalues_grid1deg = gridpp.bilinear(
ECMWF_grid, ECMWF_grid1deg,
np.transpose(ds_pf.sst[num - 1, step.days - 1, :, :].data))
pf_values[:, num - 1] = gridpp.bilinear(
SST_GRID1_5deg.longitude.data)
ECMWF_grid1deg = make_grid(SAL_GRID1deg.latitude.data,
SAL_GRID1deg.longitude.data)
with open("metadata_BW_sites.json") as json_file:
data_BW = pd.DataFrame(json.load(json_file))
BW_grid = gridpp.Points(data_BW.lat, data_BW.lon)
step = SST_GRID1_5deg.get_index('step')[0]
#for step in ds_cf.get_index('step'):
print('Time step: ' + str(step))
### Testing interpolating to 1 deg
SST_grid1deg = gridpp.bilinear(
ECMWF_grid1_5deg, ECMWF_grid1deg,
np.transpose(SST_GRID1_5deg.sst[step.days - 1, :, :].data))
SST_1deg2point = gridpp.bilinear(ECMWF_grid1deg, BW_grid,
gridpp.fill_missing(SST_grid1deg))
### Now it works when going from 1.5 deg to a point
SST_1_5deg2point = gridpp.bilinear(
ECMWF_grid1_5deg, BW_grid,
gridpp.fill_missing(
np.transpose(SST_GRID1_5deg.sst[step.days - 1, :, :].data)))
print('SST interpolated to 1 deg before to a point')
print(SST_1deg2point)
print('SST downscaled directly to a point')
print(SST_1_5deg2point)