def extrapolate_mar_daily(DIRECTORY,
EPSG,
VERSION,
tdec,
X,
Y,
XNAME=None,
YNAME=None,
TIMENAME='TIME',
VARIABLE='SMB',
SIGMA=1.5,
SEARCH='BallTree',
NN=10,
POWER=2.0,
FILL_VALUE=None,
EXTRAPOLATE=False):
#-- start and end years to read
SY = np.nanmin(np.floor(tdec)).astype(np.int)
EY = np.nanmax(np.floor(tdec)).astype(np.int)
YRS = '|'.join(['{0:4d}'.format(Y) for Y in range(SY, EY + 1)])
#-- regular expression pattern for MAR dataset
rx = re.compile(r'{0}-(.*?)-(\d+)(_subset)?.nc$'.format(VERSION, YRS))
#-- create list of files to read
input_files = sorted([f for f in os.listdir(DIRECTORY) if rx.match(f)])
#-- calculate number of time steps to read
nt = 0
for f, FILE in enumerate(input_files):
#-- Open the MAR NetCDF file for reading
with netCDF4.Dataset(os.path.join(DIRECTORY, FILE), 'r') as fileID:
nx = len(fileID.variables[XNAME][:])
ny = len(fileID.variables[YNAME][:])
TIME = fileID.variables[TIMENAME][:]
try:
nt += np.count_nonzero(TIME.data != TIME.fill_value)
except AttributeError:
nt += len(TIME)
#-- python dictionary with file variables
fd = {}
fd['TIME'] = np.zeros((nt))
#-- python dictionary with gaussian filtered variables
gs = {}
#-- calculate cumulative sum of gaussian filtered values
cumulative = np.zeros((ny, nx))
gs['CUMULATIVE'] = np.ma.zeros((nt, ny, nx), fill_value=FILL_VALUE)
gs['CUMULATIVE'].mask = np.ones((nt, ny, nx), dtype=np.bool)
#-- create a counter variable for filling variables
c = 0
#-- for each file in the list
for f, FILE in enumerate(input_files):
#-- Open the MAR NetCDF file for reading
with netCDF4.Dataset(os.path.join(DIRECTORY, FILE), 'r') as fileID:
#-- number of time variables within file
TIME = fileID.variables['TIME'][:]
try:
t = np.count_nonzero(TIME.data != TIME.fill_value)
except AttributeError:
t = len(TIME)
#-- create a masked array with all data
fd[VARIABLE] = np.ma.zeros((t, ny, nx), fill_value=FILL_VALUE)
fd[VARIABLE].mask = np.zeros((t, ny, nx), dtype=np.bool)
#-- surface type
SRF = fileID.variables['SRF'][:]
#-- indices of specified ice mask
i, j = np.nonzero(SRF == 4)
#-- ice fraction
FRA = fileID.variables['FRA'][:] / 100.0
#-- Get data from netCDF variable and remove singleton dimensions
tmp = np.squeeze(fileID.variables[VARIABLE][:])
#-- combine sectors for multi-layered data
if (np.ndim(tmp) == 4):
#-- create mask for combining data
MASK = np.zeros((t, ny, nx))
MASK[:, i, j] = FRA[:t, 0, i, j]
#-- combine data
fd[VARIABLE][:] = MASK * tmp[:t, 0, :, :] + (
1.0 - MASK) * tmp[:t, 1, :, :]
else:
#-- copy data
fd[VARIABLE][:] = tmp[:t, :, :].copy()
#-- verify mask object for interpolating data
surf_mask = np.broadcast_to(SRF, (t, ny, nx))
fd[VARIABLE].mask[:, :, :] |= (surf_mask != 4)
#-- combine mask object through time to create a single mask
fd[VARIABLE].mask = fd[VARIABLE].data == fd[VARIABLE].fill_value
fd['MASK'] = 1.0 - np.any(fd[VARIABLE].mask, axis=0).astype(
np.float)
#-- MAR coordinates
fd['LON'] = fileID.variables['LON'][:, :].copy()
fd['LAT'] = fileID.variables['LAT'][:, :].copy()
#-- convert x and y coordinates to meters
fd['x'] = 1000.0 * fileID.variables[XNAME][:].copy()
fd['y'] = 1000.0 * fileID.variables[YNAME][:].copy()
#-- extract delta time and epoch of time
delta_time = fileID.variables[TIMENAME][:t].astype(np.float)
units = fileID.variables[TIMENAME].units
#-- convert epoch of time to Julian days
Y1, M1, D1, h1, m1, s1 = [
float(d) for d in re.findall('\d+\.\d+|\d+', units)
]
epoch_julian = calc_julian_day(Y1,
M1,
D1,
HOUR=h1,
MINUTE=m1,
SECOND=s1)
#-- calculate time array in Julian days
Y2, M2, D2, h2, m2, s2 = convert_julian(epoch_julian + delta_time)
#-- calculate time in year-decimal
fd['TIME'][c:c + t] = convert_calendar_decimal(Y2,
M2,
D2,
HOUR=h2,
MINUTE=m2,
SECOND=s2)
#-- use a gaussian filter to smooth mask
gs['MASK'] = scipy.ndimage.gaussian_filter(fd['MASK'],
SIGMA,
mode='constant',
cval=0)
#-- indices of smoothed ice mask
ii, jj = np.nonzero(np.ceil(gs['MASK']) == 1.0)
#-- use a gaussian filter to smooth each model field
gs[VARIABLE] = np.ma.zeros((t, ny, nx), fill_value=FILL_VALUE)
gs[VARIABLE].mask = np.ones((t, ny, nx), dtype=np.bool)
#-- for each time
for tt in range(t):
#-- replace fill values before smoothing data
temp1 = np.zeros((ny, nx))
i, j = np.nonzero(~fd[VARIABLE].mask[tt, :, :])
temp1[i, j] = fd[VARIABLE][tt, i, j].copy()
#-- smooth spatial field
temp2 = scipy.ndimage.gaussian_filter(temp1,
SIGMA,
mode='constant',
cval=0)
#-- scale output smoothed field
gs[VARIABLE].data[tt, ii, jj] = temp2[ii, jj] / gs['MASK'][ii, jj]
#-- replace valid values with original
gs[VARIABLE].data[tt, i, j] = temp1[i, j]
#-- set mask variables for time
gs[VARIABLE].mask[tt, ii, jj] = False
#-- calculate cumulative
cumulative[ii, jj] += gs[VARIABLE][tt, ii, jj]
gs['CUMULATIVE'].data[c + tt, ii, jj] = np.copy(cumulative[ii, jj])
gs['CUMULATIVE'].mask[c + tt, ii, jj] = False
#-- add to counter
c += t
#-- convert MAR latitude and longitude to input coordinates (EPSG)
proj1 = pyproj.Proj("+init={0}".format(EPSG))
proj2 = pyproj.Proj("+init=EPSG:{0:d}".format(4326))
xg, yg = pyproj.transform(proj2, proj1, fd['LON'], fd['LAT'])
#-- construct search tree from original points
#-- can use either BallTree or KDTree algorithms
xy1 = np.concatenate((xg[i, j, None], yg[i, j, None]), axis=1)
tree = BallTree(xy1) if (SEARCH == 'BallTree') else KDTree(xy1)
#-- output interpolated arrays of output variable
npts = len(tdec)
extrap = np.ma.zeros((npts), fill_value=FILL_VALUE, dtype=np.float)
extrap.mask = np.ones((npts), dtype=np.bool)
#-- initially set all values to fill value
extrap.data[:] = extrap.fill_value
#-- type designating algorithm used (1:interpolate, 2:backward, 3:forward)
extrap.interpolation = np.zeros((npts), dtype=np.uint8)
#-- find days that can be interpolated
if np.any((tdec >= fd['TIME'].min()) & (tdec < fd['TIME'].max())):
#-- indices of dates for interpolated days
ind, = np.nonzero((tdec >= fd['TIME'].min())
& (tdec < fd['TIME'].max()))
#-- reduce x, y and t coordinates
xind, yind, tind = (X[ind], Y[ind], tdec[ind])
#-- find indices for linearly interpolating in time
f = scipy.interpolate.interp1d(fd['TIME'],
np.arange(nt),
kind='linear')
date_indice = f(tind).astype(np.int)
#-- for each unique model date
#-- linearly interpolate in time between two model maps
#-- then then inverse distance weighting to extrapolate in space
for k in np.unique(date_indice):
kk, = np.nonzero(date_indice == k)
count = np.count_nonzero(date_indice == k)
#-- query the search tree to find the NN closest points
xy2 = np.concatenate((xind[kk, None], yind[kk, None]), axis=1)
dist, indices = tree.query(xy2, k=NN, return_distance=True)
#-- normalized weights if POWER > 0 (typically between 1 and 3)
#-- in the inverse distance weighting
power_inverse_distance = dist**(-POWER)
s = np.sum(power_inverse_distance, axis=1)
w = power_inverse_distance / np.broadcast_to(
s[:, None], (count, NN))
#-- variable for times before and after tdec
var1 = gs['CUMULATIVE'][k, i, j]
var2 = gs['CUMULATIVE'][k + 1, i, j]
#-- linearly interpolate to date
dt = (tind[kk] - fd['TIME'][k]) / (fd['TIME'][k + 1] -
fd['TIME'][k])
#-- spatially extrapolate using inverse distance weighting
extrap.data[kk] = (1.0-dt)*np.sum(w*var1[indices],axis=1) + \
dt*np.sum(w*var2[indices], axis=1)
#-- set interpolation type (1: interpolated in time)
extrap.interpolation[ind] = 1
#-- check if needing to extrapolate backwards in time
count = np.count_nonzero(tdec < fd['TIME'].min())
if (count > 0) and EXTRAPOLATE:
#-- indices of dates before model
ind, = np.nonzero(tdec < fd['TIME'].min())
#-- query the search tree to find the NN closest points
xy2 = np.concatenate((X[ind, None], Y[ind, None]), axis=1)
dist, indices = tree.query(xy2, k=NN, return_distance=True)
#-- normalized weights if POWER > 0 (typically between 1 and 3)
#-- in the inverse distance weighting
power_inverse_distance = dist**(-POWER)
s = np.sum(power_inverse_distance, axis=1)
w = power_inverse_distance / np.broadcast_to(s[:, None], (count, NN))
#-- read the first year of data to create regression model
N = 365
#-- calculate a regression model for calculating values
#-- spatially interpolate variable to coordinates
DATA = np.zeros((count, N))
TIME = np.zeros((N))
#-- create interpolated time series for calculating regression model
for k in range(N):
#-- time at k
TIME[k] = fd['TIME'][k]
#-- spatially extrapolate variable
tmp = gs['CUMULATIVE'][k, i, j]
DATA[:, k] = np.sum(w * tmp[indices], axis=1)
#-- calculate regression model
for n, v in enumerate(ind):
extrap.data[v] = regress_model(TIME,
DATA[n, :],
tdec[v],
ORDER=2,
CYCLES=[0.25, 0.5, 1.0],
RELATIVE=TIME[0])
#-- set interpolation type (2: extrapolated backwards in time)
extrap.interpolation[ind] = 2
#-- check if needing to extrapolate forward in time
count = np.count_nonzero(tdec >= fd['TIME'].max())
if (count > 0) and EXTRAPOLATE:
#-- indices of dates after model
ind, = np.nonzero(tdec >= fd['TIME'].max())
#-- query the search tree to find the NN closest points
xy2 = np.concatenate((X[ind, None], Y[ind, None]), axis=1)
dist, indices = tree.query(xy2, k=NN, return_distance=True)
#-- normalized weights if POWER > 0 (typically between 1 and 3)
#-- in the inverse distance weighting
power_inverse_distance = dist**(-POWER)
s = np.sum(power_inverse_distance, axis=1)
w = power_inverse_distance / np.broadcast_to(s[:, None], (count, NN))
#-- read the last year of data to create regression model
N = 365
#-- calculate a regression model for calculating values
#-- spatially interpolate variable to coordinates
DATA = np.zeros((count, N))
TIME = np.zeros((N))
#-- create interpolated time series for calculating regression model
for k in range(N):
kk = nt - N + k
#-- time at kk
TIME[k] = fd['TIME'][kk]
#-- spatially extrapolate variable
tmp = gs['CUMULATIVE'][kk, i, j]
DATA[:, k] = np.sum(w * tmp[indices], axis=1)
#-- calculate regression model
for n, v in enumerate(ind):
extrap.data[v] = regress_model(TIME,
DATA[n, :],
tdec[v],
ORDER=2,
CYCLES=[0.25, 0.5, 1.0],
RELATIVE=TIME[-1])
#-- set interpolation type (3: extrapolated forward in time)
extrap.interpolation[ind] = 3
#-- complete mask if any invalid in data
invalid, = np.nonzero((extrap.data == extrap.fill_value)
| np.isnan(extrap.data))
extrap.mask[invalid] = True
#-- return the interpolated values
return extrap
def interpolate_mar_daily(DIRECTORY,
EPSG,
VERSION,
tdec,
X,
Y,
XNAME=None,
YNAME=None,
TIMENAME='TIME',
VARIABLE='SMB',
SIGMA=1.5,
FILL_VALUE=None,
EXTRAPOLATE=False):
#-- start and end years to read
SY = np.nanmin(np.floor(tdec)).astype(np.int)
EY = np.nanmax(np.floor(tdec)).astype(np.int)
YRS = '|'.join(['{0:4d}'.format(Y) for Y in range(SY, EY + 1)])
#-- regular expression pattern for MAR dataset
rx = re.compile('{0}-(.*?)-(\d+)(_subset)?.nc$'.format(VERSION, YRS))
#-- MAR model projection: Polar Stereographic (Oblique)
#-- Earth Radius: 6371229 m
#-- True Latitude: 0
#-- Center Longitude: -40
#-- Center Latitude: 70.5
proj4_params = ("+proj=sterea +lat_0=+70.5 +lat_ts=0 +lon_0=-40.0 "
"+a=6371229 +no_defs")
#-- create list of files to read
input_files = sorted([f for f in os.listdir(DIRECTORY) if rx.match(f)])
#-- calculate number of time steps to read
nt = 0
for f, FILE in enumerate(input_files):
#-- Open the MAR NetCDF file for reading
with netCDF4.Dataset(os.path.join(DIRECTORY, FILE), 'r') as fileID:
nx = len(fileID.variables[XNAME][:])
ny = len(fileID.variables[YNAME][:])
nt += len(fileID.variables[TIMENAME][:])
#-- python dictionary with file variables
fd = {}
fd['TIME'] = np.zeros((nt))
#-- python dictionary with gaussian filtered variables
gs = {}
#-- calculate cumulative sum of gaussian filtered values
cumulative = np.zeros((ny, nx))
gs['CUMULATIVE'] = np.ma.zeros((nt, ny, nx), fill_value=FILL_VALUE)
gs['CUMULATIVE'].mask = np.ones((nt, ny, nx), dtype=np.bool)
#-- create a counter variable for filling variables
c = 0
#-- for each file in the list
for f, FILE in enumerate(input_files):
#-- Open the MAR NetCDF file for reading
with netCDF4.Dataset(os.path.join(DIRECTORY, FILE), 'r') as fileID:
#-- number of time variables within file
t = len(fileID.variables['TIME'][:])
#-- create a masked array with all data
fd[VARIABLE] = np.ma.zeros((t, ny, nx), fill_value=FILL_VALUE)
fd[VARIABLE].mask = np.zeros((t, ny, nx), dtype=np.bool)
#-- surface type
SRF = fileID.variables['SRF'][:]
#-- indices of specified ice mask
i, j = np.nonzero(SRF == 4)
#-- ice fraction
FRA = fileID.variables['FRA'][:] / 100.0
#-- Get data from netCDF variable and remove singleton dimensions
tmp = np.squeeze(fileID.variables[VARIABLE][:])
#-- combine sectors for multi-layered data
if (np.ndim(tmp) == 4):
#-- create mask for combining data
MASK = np.zeros((nt, ny, nx))
MASK[:, i, j] = FRA[:, 0, i, j]
#-- combine data
fd[VARIABLE][:] = MASK * tmp[:, 0, :, :] + (
1.0 - MASK) * tmp[:, 1, :, :]
else:
#-- copy data
fd[VARIABLE][:] = tmp.copy()
#-- verify mask object for interpolating data
surf_mask = np.broadcast_to(SRF, (t, ny, nx))
fd[VARIABLE].mask[:, :, :] |= (surf_mask != 4)
#-- combine mask object through time to create a single mask
fd['MASK'] = 1.0 - np.any(fd[VARIABLE].mask, axis=0).astype(
np.float)
#-- MAR coordinates
fd['LON'] = fileID.variables['LON'][:, :].copy()
fd['LAT'] = fileID.variables['LAT'][:, :].copy()
#-- convert x and y coordinates to meters
fd['x'] = 1000.0 * fileID.variables[XNAME][:].copy()
fd['y'] = 1000.0 * fileID.variables[YNAME][:].copy()
#-- extract delta time and epoch of time
delta_time = fileID.variables[TIMENAME][:].astype(np.float)
units = fileID.variables[TIMENAME].units
#-- convert epoch of time to Julian days
Y1, M1, D1, h1, m1, s1 = [
float(d) for d in re.findall('\d+\.\d+|\d+', units)
]
epoch_julian = calc_julian_day(Y1,
M1,
D1,
HOUR=h1,
MINUTE=m1,
SECOND=s1)
#-- calculate time array in Julian days
Y2, M2, D2, h2, m2, s2 = convert_julian(epoch_julian + delta_time)
#-- calculate time in year-decimal
fd['TIME'][c:c + t] = convert_calendar_decimal(Y2,
M2,
D2,
HOUR=h2,
MINUTE=m2,
SECOND=s2)
#-- use a gaussian filter to smooth mask
gs['MASK'] = scipy.ndimage.gaussian_filter(fd['MASK'],
SIGMA,
mode='constant',
cval=0)
#-- indices of smoothed ice mask
ii, jj = np.nonzero(np.ceil(gs['MASK']) == 1.0)
#-- use a gaussian filter to smooth each model field
gs[VARIABLE] = np.ma.zeros((t, ny, nx), fill_value=FILL_VALUE)
gs[VARIABLE].mask = np.ones((t, ny, nx), dtype=np.bool)
#-- for each time
for tt in range(t):
#-- replace fill values before smoothing data
temp1 = np.zeros((ny, nx))
i, j = np.nonzero(~fd[VARIABLE].mask[tt, :, :])
temp1[i, j] = fd[VARIABLE][tt, i, j].copy()
#-- smooth spatial field
temp2 = scipy.ndimage.gaussian_filter(temp1,
SIGMA,
mode='constant',
cval=0)
#-- scale output smoothed field
gs[VARIABLE].data[tt, ii, jj] = temp2[ii, jj] / gs['MASK'][ii, jj]
#-- replace valid values with original
gs[VARIABLE].data[tt, i, j] = temp1[i, j]
#-- set mask variables for time
gs[VARIABLE].mask[tt, ii, jj] = False
#-- calculate cumulative
cumulative[ii, jj] += gs[VARIABLE][tt, ii, jj]
gs['CUMULATIVE'].data[c + tt, ii, jj] = np.copy(cumulative[ii, jj])
gs['CUMULATIVE'].mask[c + tt, ii, jj] = False
#-- add to counter
c += t
#-- convert projection from input coordinates (EPSG) to model coordinates
proj1 = pyproj.Proj("+init={0}".format(EPSG))
proj2 = pyproj.Proj(proj4_params)
#-- calculate projected coordinates of input coordinates
ix, iy = pyproj.transform(proj1, proj2, X, Y)
#-- check that input points are within convex hull of valid model points
gs['x'], gs['y'] = np.meshgrid(fd['x'], fd['y'])
points = np.concatenate((gs['x'][ii, jj, None], gs['y'][ii, jj, None]),
axis=1)
triangle = scipy.spatial.Delaunay(points.data,
qhull_options='Qt Qbb Qc Qz')
interp_points = np.concatenate((ix[:, None], iy[:, None]), axis=1)
valid = (triangle.find_simplex(interp_points) >= 0)
#-- output interpolated arrays of model variable
npts = len(tdec)
interp = np.ma.zeros((npts), fill_value=FILL_VALUE, dtype=np.float)
interp.mask = np.ones((npts), dtype=np.bool)
#-- initially set all values to fill value
interp.data[:] = interp.fill_value
#-- type designating algorithm used (1:interpolate, 2:backward, 3:forward)
interp.interpolation = np.zeros((npts), dtype=np.uint8)
#-- find days that can be interpolated
if np.any((tdec >= fd['TIME'].min()) & (tdec <= fd['TIME'].max()) & valid):
#-- indices of dates for interpolated days
ind, = np.nonzero((tdec >= fd['TIME'].min())
& (tdec <= fd['TIME'].max()) & valid)
#-- create an interpolator for model variable
RGI = scipy.interpolate.RegularGridInterpolator(
(fd['TIME'], fd['y'], fd['x']), gs['CUMULATIVE'].data)
#-- create an interpolator for input mask
MI = scipy.interpolate.RegularGridInterpolator(
(fd['TIME'], fd['y'], fd['x']), gs['CUMULATIVE'].mask)
#-- interpolate to points
interp.data[ind] = RGI.__call__(np.c_[tdec[ind], iy[ind], ix[ind]])
interp.mask[ind] = MI.__call__(np.c_[tdec[ind], iy[ind], ix[ind]])
#-- set interpolation type (1: interpolated)
interp.interpolation[ind] = 1
#-- check if needing to extrapolate backwards in time
count = np.count_nonzero((tdec < fd['TIME'].min()) & valid)
if (count > 0) and EXTRAPOLATE:
#-- indices of dates before model
ind, = np.nonzero((tdec < fd['TIME'].min()) & valid)
#-- read the first year of data to create regression model
N = 365
#-- calculate a regression model for calculating values
#-- spatially interpolate model variable to coordinates
DATA = np.zeros((count, N))
MASK = np.zeros((count, N), dtype=np.bool)
TIME = np.zeros((N))
#-- create interpolated time series for calculating regression model
for k in range(N):
#-- time at k
TIME[k] = fd['TIME'][k]
#-- spatially interpolate model variable
S1 = scipy.interpolate.RectBivariateSpline(
fd['x'], fd['y'], gs['CUMULATIVE'].data[k, :, :].T, kx=1, ky=1)
S2 = scipy.interpolate.RectBivariateSpline(
fd['x'], fd['y'], gs['CUMULATIVE'].mask[k, :, :].T, kx=1, ky=1)
#-- create numpy masked array of interpolated values
DATA[:, k] = S1.ev(ix[ind], iy[ind])
MASK[:, k] = S2.ev(ix[ind], iy[ind])
#-- calculate regression model
for n, v in enumerate(ind):
interp.data[v] = regress_model(TIME,
DATA[n, :],
tdec[v],
ORDER=2,
CYCLES=[0.25, 0.5, 1.0],
RELATIVE=TIME[0])
#-- mask any invalid points
interp.mask[ind] = np.any(MASK, axis=1)
#-- set interpolation type (2: extrapolated backward)
interp.interpolation[ind] = 2
#-- check if needing to extrapolate forward in time
count = np.count_nonzero((tdec > fd['TIME'].max()) & valid)
if (count > 0) and EXTRAPOLATE:
#-- indices of dates after model
ind, = np.nonzero((tdec > fd['TIME'].max()) & valid)
#-- read the last year of data to create regression model
N = 365
#-- calculate a regression model for calculating values
#-- spatially interpolate model variable to coordinates
DATA = np.zeros((count, N))
MASK = np.zeros((count, N), dtype=np.bool)
TIME = np.zeros((N))
#-- create interpolated time series for calculating regression model
for k in range(N):
kk = nt - N + k
#-- time at kk
TIME[k] = fd['TIME'][kk]
#-- spatially interpolate model variable
S1 = scipy.interpolate.RectBivariateSpline(
fd['x'],
fd['y'],
gs['CUMULATIVE'].data[kk, :, :].T,
kx=1,
ky=1)
S2 = scipy.interpolate.RectBivariateSpline(
fd['x'],
fd['y'],
gs['CUMULATIVE'].mask[kk, :, :].T,
kx=1,
ky=1)
#-- create numpy masked array of interpolated values
DATA[:, k] = S1.ev(ix[ind], iy[ind])
MASK[:, k] = S2.ev(ix[ind], iy[ind])
#-- calculate regression model
for n, v in enumerate(ind):
interp.data[v] = regress_model(TIME,
DATA[n, :],
tdec[v],
ORDER=2,
CYCLES=[0.25, 0.5, 1.0],
RELATIVE=TIME[-1])
#-- mask any invalid points
interp.mask[ind] = np.any(MASK, axis=1)
#-- set interpolation type (3: extrapolated forward)
interp.interpolation[ind] = 3
#-- complete mask if any invalid in data
invalid, = np.nonzero((interp.data == interp.fill_value)
| np.isnan(interp.data))
interp.mask[invalid] = True
#-- return the interpolated values
return interp
def interpolate_racmo_daily(base_dir,
EPSG,
MODEL,
tdec,
X,
Y,
VARIABLE='smb',
SIGMA=1.5,
FILL_VALUE=None,
EXTRAPOLATE=False):
#-- start and end years to read
SY = np.nanmin(np.floor(tdec)).astype(np.int)
EY = np.nanmax(np.floor(tdec)).astype(np.int)
YRS = '|'.join(['{0:4d}'.format(Y) for Y in range(SY, EY + 1)])
#-- input list of files
if (MODEL == 'FGRN055'):
#-- filename and directory for input FGRN055 files
file_pattern = 'RACMO2.3p2_FGRN055_{0}_daily_(\d+).nc'
DIRECTORY = os.path.join(base_dir, 'RACMO', 'GL', 'RACMO2.3p2_FGRN055')
#-- create list of files to read
rx = re.compile(file_pattern.format(VARIABLE, YRS), re.VERBOSE)
input_files = sorted([f for f in os.listdir(DIRECTORY) if rx.match(f)])
#-- calculate number of time steps to read
nt = 0
for f, FILE in enumerate(input_files):
#-- Open the RACMO NetCDF file for reading
with netCDF4.Dataset(os.path.join(DIRECTORY, FILE), 'r') as fileID:
nx = len(fileID.variables['rlon'][:])
ny = len(fileID.variables['rlat'][:])
nt += len(fileID.variables['time'][:])
#-- invalid data value
fv = np.float(fileID.variables[VARIABLE]._FillValue)
#-- scaling factor for converting units
if (VARIABLE == 'hgtsrf'):
scale_factor = 86400.0
elif (VARIABLE == 'smb'):
scale_factor = 1.0
#-- python dictionary with file variables
fd = {}
fd['time'] = np.zeros((nt))
#-- python dictionary with gaussian filtered variables
gs = {}
#-- calculate cumulative sum of gaussian filtered values
cumulative = np.zeros((ny, nx))
gs['cumulative'] = np.ma.zeros((nt, ny, nx), fill_value=fv)
gs['cumulative'].mask = np.zeros((nt, ny, nx), dtype=np.bool)
#-- create a counter variable for filling variables
c = 0
#-- for each file in the list
for f, FILE in enumerate(input_files):
#-- Open the RACMO NetCDF file for reading
with netCDF4.Dataset(os.path.join(DIRECTORY, FILE), 'r') as fileID:
#-- number of time variables within file
t = len(fileID.variables['time'][:])
fd[VARIABLE] = np.ma.zeros((t, ny, nx), fill_value=fv)
fd[VARIABLE].mask = np.ones((t, ny, nx), dtype=np.bool)
#-- Get data from netCDF variable and remove singleton dimensions
tmp = np.squeeze(fileID.variables[VARIABLE][:])
fd[VARIABLE][:] = scale_factor * tmp
#-- indices of specified ice mask
i, j = np.nonzero(tmp[0, :, :] != fv)
fd[VARIABLE].mask[:, i, j] = False
#-- combine mask object through time to create a single mask
fd['mask'] = 1.0 - np.any(fd[VARIABLE].mask, axis=0).astype(
np.float)
#-- racmo coordinates
fd['lon'] = fileID.variables['lon'][:, :].copy()
fd['lat'] = fileID.variables['lat'][:, :].copy()
fd['x'] = fileID.variables['rlon'][:].copy()
fd['y'] = fileID.variables['rlat'][:].copy()
#-- rotated pole parameters
proj4_params = fileID.variables['rotated_pole'].proj4_params
#-- extract delta time and epoch of time
delta_time = fileID.variables['time'][:].astype(np.float)
units = fileID.variables['time'].units
#-- convert epoch of time to Julian days
Y1, M1, D1, h1, m1, s1 = [
float(d) for d in re.findall('\d+\.\d+|\d+', units)
]
epoch_julian = calc_julian_day(Y1,
M1,
D1,
HOUR=h1,
MINUTE=m1,
SECOND=s1)
#-- calculate time array in Julian days
Y2, M2, D2, h2, m2, s2 = convert_julian(epoch_julian + delta_time)
#-- calculate time in year-decimal
fd['time'][c:c + t] = convert_calendar_decimal(Y2,
M2,
D2,
HOUR=h2,
MINUTE=m2,
SECOND=s2)
#-- use a gaussian filter to smooth mask
gs['mask'] = scipy.ndimage.gaussian_filter(fd['mask'],
SIGMA,
mode='constant',
cval=0)
#-- indices of smoothed ice mask
ii, jj = np.nonzero(np.ceil(gs['mask']) == 1.0)
#-- use a gaussian filter to smooth each model field
gs[VARIABLE] = np.ma.zeros((t, ny, nx), fill_value=fv)
gs[VARIABLE].mask = np.ones((t, ny, nx), dtype=np.bool)
#-- for each time
for tt in range(t):
#-- replace fill values before smoothing data
temp1 = np.zeros((ny, nx))
i, j = np.nonzero(~fd[VARIABLE].mask[tt, :, :])
temp1[i, j] = fd[VARIABLE][tt, i, j].copy()
#-- smooth spatial field
temp2 = scipy.ndimage.gaussian_filter(temp1,
SIGMA,
mode='constant',
cval=0)
#-- scale output smoothed field
gs[VARIABLE][tt, ii, jj] = temp2[ii, jj] / gs['mask'][ii, jj]
#-- replace valid values with original
gs[VARIABLE][tt, i, j] = temp1[i, j]
#-- set mask variables for time
gs[VARIABLE].mask[tt, ii, jj] = False
#-- calculate cumulative
cumulative[ii, jj] += gs[VARIABLE][tt, ii, jj]
gs['cumulative'].data[c + tt, ii, jj] = np.copy(cumulative[ii, jj])
gs['cumulative'].mask[c + tt, ii, jj] = False
#-- add to counter
c += t
#-- convert projection from input coordinates (EPSG) to model coordinates
#-- RACMO models are rotated pole latitude and longitude
proj1 = pyproj.Proj("+init={0}".format(EPSG))
proj2 = pyproj.Proj(proj4_params)
#-- calculate rotated pole coordinates of input coordinates
ix, iy = pyproj.transform(proj1, proj2, X, Y)
#-- check that input points are within convex hull of valid model points
gs['x'], gs['y'] = np.meshgrid(fd['x'], fd['y'])
v, triangle = find_valid_triangulation(gs['x'][ii, jj], gs['y'][ii, jj])
#-- check where points are within the complex hull of the triangulation
if v:
interp_points = np.concatenate((ix[:, None], iy[:, None]), axis=1)
valid = (triangle.find_simplex(interp_points) >= 0)
else:
#-- Check ix and iy against the bounds of x and y
valid = (ix >= fd['x'].min()) & (ix <= fd['x'].max()) & \
(iy >= fd['y'].min()) & (iy <= fd['y'].max())
#-- output interpolated arrays of model variable
npts = len(tdec)
interp = np.ma.zeros((npts), fill_value=fv, dtype=np.float)
interp.mask = np.ones((npts), dtype=np.bool)
#-- initially set all values to fill value
interp.data[:] = interp.fill_value
#-- type designating algorithm used (1:interpolate, 2:backward, 3:forward)
interp.interpolation = np.zeros((npts), dtype=np.uint8)
#-- find days that can be interpolated
if np.any((tdec >= fd['time'].min()) & (tdec <= fd['time'].max()) & valid):
#-- indices of dates for interpolated days
ind, = np.nonzero((tdec >= fd['time'].min())
& (tdec <= fd['time'].max()) & valid)
#-- create an interpolator for model variable
RGI = scipy.interpolate.RegularGridInterpolator(
(fd['time'], fd['y'], fd['x']), gs['cumulative'].data)
#-- create an interpolator for input mask
MI = scipy.interpolate.RegularGridInterpolator(
(fd['time'], fd['y'], fd['x']), gs['cumulative'].mask)
#-- interpolate to points
interp.data[ind] = RGI.__call__(np.c_[tdec[ind], iy[ind], ix[ind]])
interp.mask[ind] = MI.__call__(np.c_[tdec[ind], iy[ind], ix[ind]])
#-- set interpolation type (1: interpolated)
interp.interpolation[ind] = 1
#-- check if needing to extrapolate backwards in time
count = np.count_nonzero((tdec < fd['time'].min()) & valid)
if (count > 0) and EXTRAPOLATE:
#-- indices of dates before model
ind, = np.nonzero((tdec < fd['time'].min()) & valid)
#-- read the first year of data to create regression model
N = 365
#-- calculate a regression model for calculating values
#-- spatially interpolate model variable to coordinates
DATA = np.zeros((count, N))
MASK = np.zeros((count, N), dtype=np.bool)
TIME = np.zeros((N))
#-- create interpolated time series for calculating regression model
for k in range(N):
#-- time at k
TIME[k] = fd['time'][k]
#-- spatially interpolate model variable
S1 = scipy.interpolate.RectBivariateSpline(
fd['x'], fd['y'], gs['cumulative'].data[k, :, :].T, kx=1, ky=1)
S2 = scipy.interpolate.RectBivariateSpline(
fd['x'], fd['y'], gs['cumulative'].mask[k, :, :].T, kx=1, ky=1)
#-- create numpy masked array of interpolated values
DATA[:, k] = S1.ev(ix[ind], iy[ind])
MASK[:, k] = S2.ev(ix[ind], iy[ind])
#-- calculate regression model
for n, v in enumerate(ind):
interp.data[v] = regress_model(TIME,
DATA[n, :],
tdec[v],
ORDER=2,
CYCLES=[0.25, 0.5, 1.0],
RELATIVE=TIME[0])
#-- mask any invalid points
interp.mask[ind] = np.any(MASK, axis=1)
#-- set interpolation type (2: extrapolated backward)
interp.interpolation[ind] = 2
#-- check if needing to extrapolate forward in time
count = np.count_nonzero((tdec > fd['time'].max()) & valid)
if (count > 0) and EXTRAPOLATE:
#-- indices of dates after model
ind, = np.nonzero((tdec > fd['time'].max()) & valid)
#-- read the last year of data to create regression model
N = 365
#-- calculate a regression model for calculating values
#-- spatially interpolate model variable to coordinates
DATA = np.zeros((count, N))
MASK = np.zeros((count, N), dtype=np.bool)
TIME = np.zeros((N))
#-- create interpolated time series for calculating regression model
for k in range(N):
kk = nt - N + k
#-- time at kk
TIME[k] = fd['time'][kk]
#-- spatially interpolate model variable
S1 = scipy.interpolate.RectBivariateSpline(
fd['x'],
fd['y'],
gs['cumulative'].data[kk, :, :].T,
kx=1,
ky=1)
S2 = scipy.interpolate.RectBivariateSpline(
fd['x'],
fd['y'],
gs['cumulative'].mask[kk, :, :].T,
kx=1,
ky=1)
#-- create numpy masked array of interpolated values
DATA[:, k] = S1.ev(ix[ind], iy[ind])
MASK[:, k] = S2.ev(ix[ind], iy[ind])
#-- calculate regression model
for n, v in enumerate(ind):
interp.data[v] = regress_model(TIME,
DATA[n, :],
tdec[v],
ORDER=2,
CYCLES=[0.25, 0.5, 1.0],
RELATIVE=TIME[-1])
#-- mask any invalid points
interp.mask[ind] = np.any(MASK, axis=1)
#-- set interpolation type (3: extrapolated forward)
interp.interpolation[ind] = 3
#-- complete mask if any invalid in data
invalid, = np.nonzero((interp.data == interp.fill_value)
| np.isnan(interp.data))
interp.mask[invalid] = True
#-- replace fill value if specified
if FILL_VALUE:
interp.fill_value = FILL_VALUE
interp.data[interp.mask] = interp.fill_value
#-- return the interpolated values
return interp
def extrapolate_racmo_daily(base_dir,
EPSG,
MODEL,
tdec,
X,
Y,
VARIABLE='smb',
SIGMA=1.5,
SEARCH='BallTree',
NN=10,
POWER=2.0,
FILL_VALUE=None,
EXTRAPOLATE=False):
#-- start and end years to read
SY = np.nanmin(np.floor(tdec)).astype(np.int)
EY = np.nanmax(np.floor(tdec)).astype(np.int)
YRS = '|'.join(['{0:4d}'.format(Y) for Y in range(SY, EY + 1)])
#-- input list of files
if (MODEL == 'FGRN055'):
#-- filename and directory for input FGRN055 files
file_pattern = 'RACMO2.3p2_FGRN055_{0}_daily_(\d+).nc'
DIRECTORY = os.path.join(base_dir, 'RACMO', 'GL', 'RACMO2.3p2_FGRN055')
#-- create list of files to read
rx = re.compile(file_pattern.format(VARIABLE, YRS), re.VERBOSE)
input_files = sorted([f for f in os.listdir(DIRECTORY) if rx.match(f)])
#-- calculate number of time steps to read
nt = 0
for f, FILE in enumerate(input_files):
#-- Open the RACMO NetCDF file for reading
with netCDF4.Dataset(os.path.join(DIRECTORY, FILE), 'r') as fileID:
nx = len(fileID.variables['rlon'][:])
ny = len(fileID.variables['rlat'][:])
nt += len(fileID.variables['time'][:])
#-- invalid data value
fv = np.float(fileID.variables[VARIABLE]._FillValue)
#-- scaling factor for converting units
if (VARIABLE == 'hgtsrf'):
scale_factor = 86400.0
elif (VARIABLE == 'smb'):
scale_factor = 1.0
#-- python dictionary with file variables
fd = {}
fd['time'] = np.zeros((nt))
#-- python dictionary with gaussian filtered variables
gs = {}
#-- calculate cumulative sum of gaussian filtered values
cumulative = np.zeros((ny, nx))
gs['cumulative'] = np.ma.zeros((nt, ny, nx), fill_value=fv)
gs['cumulative'].mask = np.zeros((nt, ny, nx), dtype=np.bool)
#-- create a counter variable for filling variables
c = 0
#-- for each file in the list
for f, FILE in enumerate(input_files):
#-- Open the RACMO NetCDF file for reading
with netCDF4.Dataset(os.path.join(DIRECTORY, FILE), 'r') as fileID:
#-- number of time variables within file
t = len(fileID.variables['time'][:])
fd[VARIABLE] = np.ma.zeros((t, ny, nx), fill_value=fv)
fd[VARIABLE].mask = np.ones((t, ny, nx), dtype=np.bool)
#-- Get data from netCDF variable and remove singleton dimensions
tmp = np.squeeze(fileID.variables[VARIABLE][:])
fd[VARIABLE][:] = scale_factor * tmp
#-- indices of specified ice mask
i, j = np.nonzero(tmp[0, :, :] != fv)
fd[VARIABLE].mask[:, i, j] = False
#-- combine mask object through time to create a single mask
fd['mask'] = 1.0 - np.any(fd[VARIABLE].mask, axis=0).astype(
np.float)
#-- racmo coordinates
fd['lon'] = fileID.variables['lon'][:, :].copy()
fd['lat'] = fileID.variables['lat'][:, :].copy()
fd['x'] = fileID.variables['rlon'][:].copy()
fd['y'] = fileID.variables['rlat'][:].copy()
#-- rotated pole parameters
proj4_params = fileID.variables['rotated_pole'].proj4_params
#-- extract delta time and epoch of time
delta_time = fileID.variables['time'][:].astype(np.float)
units = fileID.variables['time'].units
#-- convert epoch of time to Julian days
Y1, M1, D1, h1, m1, s1 = [
float(d) for d in re.findall('\d+\.\d+|\d+', units)
]
epoch_julian = calc_julian_day(Y1,
M1,
D1,
HOUR=h1,
MINUTE=m1,
SECOND=s1)
#-- calculate time array in Julian days
Y2, M2, D2, h2, m2, s2 = convert_julian(epoch_julian + delta_time)
#-- calculate time in year-decimal
fd['time'][c:c + t] = convert_calendar_decimal(Y2,
M2,
D2,
HOUR=h2,
MINUTE=m2,
SECOND=s2)
#-- use a gaussian filter to smooth mask
gs['mask'] = scipy.ndimage.gaussian_filter(fd['mask'],
SIGMA,
mode='constant',
cval=0)
#-- indices of smoothed ice mask
ii, jj = np.nonzero(np.ceil(gs['mask']) == 1.0)
#-- use a gaussian filter to smooth each model field
gs[VARIABLE] = np.ma.zeros((t, ny, nx), fill_value=fv)
gs[VARIABLE].mask = np.ones((t, ny, nx), dtype=np.bool)
#-- for each time
for tt in range(t):
#-- replace fill values before smoothing data
temp1 = np.zeros((ny, nx))
i, j = np.nonzero(~fd[VARIABLE].mask[tt, :, :])
temp1[i, j] = fd[VARIABLE][tt, i, j].copy()
#-- smooth spatial field
temp2 = scipy.ndimage.gaussian_filter(temp1,
SIGMA,
mode='constant',
cval=0)
#-- scale output smoothed field
gs[VARIABLE][tt, ii, jj] = temp2[ii, jj] / gs['mask'][ii, jj]
#-- replace valid values with original
gs[VARIABLE][tt, i, j] = temp1[i, j]
#-- set mask variables for time
gs[VARIABLE].mask[tt, ii, jj] = False
#-- calculate cumulative
cumulative[ii, jj] += gs[VARIABLE][tt, ii, jj]
gs['cumulative'].data[c + tt, ii, jj] = np.copy(cumulative[ii, jj])
gs['cumulative'].mask[c + tt, ii, jj] = False
#-- add to counter
c += t
#-- convert RACMO latitude and longitude to input coordinates (EPSG)
proj1 = pyproj.Proj("+init={0}".format(EPSG))
proj2 = pyproj.Proj("+init=EPSG:{0:d}".format(4326))
xg, yg = pyproj.transform(proj2, proj1, fd['lon'], fd['lat'])
#-- construct search tree from original points
#-- can use either BallTree or KDTree algorithms
xy1 = np.concatenate((xg[i, j, None], yg[i, j, None]), axis=1)
tree = BallTree(xy1) if (SEARCH == 'BallTree') else KDTree(xy1)
#-- output interpolated arrays of variable
npts = len(tdec)
extrap = np.ma.zeros((npts), fill_value=fv, dtype=np.float)
extrap.mask = np.ones((npts), dtype=np.bool)
#-- initially set all values to fill value
extrap.data[:] = extrap.fill_value
#-- type designating algorithm used (1:interpolate, 2:backward, 3:forward)
extrap.interpolation = np.zeros((npts), dtype=np.uint8)
#-- find days that can be interpolated
if np.any((tdec >= fd['time'].min()) & (tdec < fd['time'].max())):
#-- indices of dates for interpolated days
ind, = np.nonzero((tdec >= fd['time'].min())
& (tdec < fd['time'].max()))
#-- reduce x, y and t coordinates
xind, yind, tind = (X[ind], Y[ind], tdec[ind])
#-- find indices for linearly interpolating in time
f = scipy.interpolate.interp1d(fd['time'],
np.arange(nt),
kind='linear')
date_indice = f(tind).astype(np.int)
#-- for each unique racmo date
#-- linearly interpolate in time between two racmo maps
#-- then then inverse distance weighting to extrapolate in space
for k in np.unique(date_indice):
kk, = np.nonzero(date_indice == k)
count = np.count_nonzero(date_indice == k)
#-- query the search tree to find the NN closest points
xy2 = np.concatenate((xind[kk, None], yind[kk, None]), axis=1)
dist, indices = tree.query(xy2, k=NN, return_distance=True)
#-- normalized weights if POWER > 0 (typically between 1 and 3)
#-- in the inverse distance weighting
power_inverse_distance = dist**(-POWER)
s = np.sum(power_inverse_distance, axis=1)
w = power_inverse_distance / np.broadcast_to(
s[:, None], (count, NN))
#-- variable for times before and after tdec
var1 = gs['cumulative'][k, i, j]
var2 = gs['cumulative'][k + 1, i, j]
#-- linearly interpolate to date
dt = (tind[kk] - fd['time'][k]) / (fd['time'][k + 1] -
fd['time'][k])
#-- spatially extrapolate using inverse distance weighting
extrap[kk] = (1.0-dt)*np.sum(w*var1[indices],axis=1) + \
dt*np.sum(w*var2[indices], axis=1)
#-- set interpolation type (1: interpolated in time)
extrap.interpolation[ind] = 1
#-- check if needing to extrapolate backwards in time
count = np.count_nonzero(tdec < fd['time'].min())
if (count > 0) and EXTRAPOLATE:
#-- indices of dates before model
ind, = np.nonzero(tdec < fd['time'].min())
#-- query the search tree to find the NN closest points
xy2 = np.concatenate((X[ind, None], Y[ind, None]), axis=1)
dist, indices = tree.query(xy2, k=NN, return_distance=True)
#-- normalized weights if POWER > 0 (typically between 1 and 3)
#-- in the inverse distance weighting
power_inverse_distance = dist**(-POWER)
s = np.sum(power_inverse_distance, axis=1)
w = power_inverse_distance / np.broadcast_to(s[:, None], (count, NN))
#-- read the first year of data to create regression model
N = 365
#-- calculate a regression model for calculating values
#-- spatially interpolate variable to coordinates
DATA = np.zeros((count, N))
TIME = np.zeros((N))
#-- create interpolated time series for calculating regression model
for k in range(N):
#-- time at k
TIME[k] = fd['time'][k]
#-- spatially extrapolate variable
tmp = gs['cumulative'][k, i, j]
DATA[:, k] = np.sum(w * tmp[indices], axis=1)
#-- calculate regression model
for n, v in enumerate(ind):
extrap[v] = regress_model(TIME,
DATA[n, :],
tdec[v],
ORDER=2,
CYCLES=[0.25, 0.5, 1.0],
RELATIVE=TIME[0])
#-- set interpolation type (2: extrapolated backwards in time)
extrap.interpolation[ind] = 2
#-- check if needing to extrapolate forward in time
count = np.count_nonzero(tdec >= fd['time'].max())
if (count > 0) and EXTRAPOLATE:
#-- indices of dates after racmo model
ind, = np.nonzero(tdec >= fd['time'].max())
#-- query the search tree to find the NN closest points
xy2 = np.concatenate((X[ind, None], Y[ind, None]), axis=1)
dist, indices = tree.query(xy2, k=NN, return_distance=True)
#-- normalized weights if POWER > 0 (typically between 1 and 3)
#-- in the inverse distance weighting
power_inverse_distance = dist**(-POWER)
s = np.sum(power_inverse_distance, axis=1)
w = power_inverse_distance / np.broadcast_to(s[:, None], (count, NN))
#-- read the last year of data to create regression model
N = 365
#-- calculate a regression model for calculating values
#-- spatially interpolate variable to coordinates
DATA = np.zeros((count, N))
TIME = np.zeros((N))
#-- create interpolated time series for calculating regression model
for k in range(N):
kk = nt - N + k
#-- time at kk
TIME[k] = fd['time'][kk]
#-- spatially extrapolate variable
tmp = gs['cumulative'][kk, i, j]
DATA[:, k] = np.sum(w * tmp[indices], axis=1)
#-- calculate regression model
for n, v in enumerate(ind):
extrap[v] = regress_model(TIME,
DATA[n, :],
tdec[v],
ORDER=2,
CYCLES=[0.25, 0.5, 1.0],
RELATIVE=TIME[-1])
#-- set interpolation type (3: extrapolated forward in time)
extrap.interpolation[ind] = 3
#-- complete mask if any invalid in data
invalid, = np.nonzero((extrap.data == extrap.fill_value)
| np.isnan(extrap.data))
extrap.mask[invalid] = True
#-- replace fill value if specified
if FILL_VALUE:
extrap.fill_value = FILL_VALUE
extrap.data[extrap.mask] = extrap.fill_value
#-- return the interpolated values
return extrap