def test_extrapolation(self):
""" Check that extrapolation filter works """
N = 5
y = np.linspace(0, 1000, N)
x = np.zeros(N)
grid = gridpp.Points(y, x, x, x, gridpp.Cartesian)
points = gridpp.Points([0, 100, 900, 1000], [0, 0, 0, 0], [0, 0, 0, 0],
[0, 0, 0, 0], gridpp.Cartesian)
Y = points.size()
pratios = 0.1 * np.ones(Y)
structure = gridpp.BarnesStructure(500)
pobs = [0, 1, 1, 0]
background = np.zeros(grid.size())
pbackground = np.zeros(Y)
max_points = 10
output0 = gridpp.optimal_interpolation(grid, background, points, pobs,
pratios, pbackground, structure,
max_points, False)
output1 = gridpp.optimal_interpolation(grid, background, points, pobs,
pratios, pbackground, structure,
max_points, True)
# Turning off extrapolation should mean we don't get increments greater than 1
self.assertTrue(np.max(output0) == 1)
self.assertTrue(np.max(output1) > 1)
I = np.where(output1 < 1)[0]
np.testing.assert_array_almost_equal(output0[I], output1[I])
def test_basic(self):
x = [0, 1000, 2000, 3000, np.nan]
structure_corr = dict()
barnes = gridpp.BarnesStructure(2000)
structure_corr[barnes] = [1, 0.8824968934059143, 0.6065306663513184, 0.32465246319770813, 0]
structure_corr[gridpp.CressmanStructure(2000)] = [1, 0.6, 0, 0, 0]
structure_corr[gridpp.CrossValidation(barnes, 1000)] = [0, 0, 0.6065306663513184, 0.32465246319770813, 0]
N = len(x)
for structure, corr in structure_corr.items():
is_cv = isinstance(structure, gridpp.CrossValidation)
for i in range(N):
with self.subTest(structure=type(structure), i=i):
p1 = gridpp.Point(0, 0, 0, 0, gridpp.Cartesian)
p2 = gridpp.Point(x[i], 0, 0, 0, gridpp.Cartesian)
funcs = [structure.corr, structure.corr_background]
if is_cv:
funcs = [structure.corr_background]
for func in funcs:
self.assertAlmostEqual(corr[i], func(p1, p2))
# Check that changing the order does not change the results
self.assertAlmostEqual(corr[i], func(p2, p1))
# Check identical points
if not is_cv and not np.isnan(x[i]):
self.assertAlmostEqual(1, func(p2, p2))
def test_invalid_arguments(self):
""" Check that exception is thrown on invalid input values """
# Set up struct with valid input arguments
ok_args = collections.OrderedDict({
'grid':
gridpp.Grid([[0, 0, 0]], [[0, 2500, 10000]], [[0, 0, 0]],
[[0, 0, 0]], gridpp.Cartesian),
'background':
np.zeros([1, 3]),
'points':
gridpp.Points([0], [2500], [0], [0], gridpp.Cartesian),
'pobs': [1],
'pratios': [0.1],
'pbackground': [0],
'structure':
gridpp.BarnesStructure(2500),
'max_points':
10
})
# Set up struct with invalid input arguments that will be substituted into ok_args one at a
# time in order to look for an exception being raised. Use an array of different invalid
# arguments for each key.
x = np.zeros([3, 2])
invalid_args = {
# Grid size mismatch, and coordinate-type mismatch
'grid': [
gridpp.Grid(x, x, x, x, gridpp.Cartesian),
gridpp.Grid([[0, 0, 0]], [[0, 2500, 10000]])
],
# Points size mismatch, and coordinate-type mismatch
'points': [
gridpp.Points([0, 1], [0, 2500], [0, 0], [0, 0],
gridpp.Cartesian),
gridpp.Points([0], [2500])
],
'pratios': [np.zeros(11)],
'pobs': [np.zeros([11])],
'background': [np.zeros([2, 11])],
'pbackground': [np.zeros(21)],
'max_points': [-1]
}
for key in invalid_args.keys():
for arg in invalid_args[key]:
args0 = ok_args.copy()
args0[key] = arg
q = [args0[f] for f in args0]
with self.subTest(key=key, arg=arg):
with self.assertRaises(ValueError) as e:
output = gridpp.optimal_interpolation(*q)
def test_barnes_hmax(self):
hmaxs = [0, 1000, 2000, 10000]
p0 = gridpp.Point(0, 0, 0, 0, gridpp.Cartesian)
dist_ans = {0:1, 1000:0.8824968934059143, 2000:0.6065306663513184, 3000:0.32465246319770813}
for hmax in hmaxs:
for dist, ans in dist_ans.items():
with self.subTest(hmax=hmax, dist=dist):
structure = gridpp.BarnesStructure(2000, 0, 0, hmax)
corr = structure.corr(p0, gridpp.Point(dist, 0, 0, 0, gridpp.Cartesian))
if dist > hmax:
self.assertEqual(0, corr)
else:
self.assertEqual(ans, corr)
def test_simple_1d(self):
N = 3
y = [[0, 0, 0]]
x = [[0, 2500, 10000]]
grid = gridpp.Grid(y, x, y, y, gridpp.Cartesian)
points = gridpp.Points([0], [2500], [0], [0], gridpp.Cartesian)
pratios = [0.1]
structure = gridpp.BarnesStructure(2500)
pobs = [1]
background = np.zeros([1, N])
pbackground = [0]
max_points = 10
output = gridpp.optimal_interpolation(grid, background, points, pobs,
pratios, pbackground, structure,
max_points)
np.testing.assert_array_almost_equal(
output,
np.array([[np.exp(-0.5) / 1.1, 1 / 1.1,
np.exp(-0.5 * 9) / 1.1]]))
def test_simple_1d_full(self):
N = 3
y = [[0, 0, 0]]
x = [[0, 2500, 10000]]
grid = gridpp.Grid(y, x, y, y, gridpp.Cartesian)
points = gridpp.Points([0], [2500], [0], [0], gridpp.Cartesian)
bvariance = np.ones([1, N])
obs_variance = [0.1]
bvariance_at_points = [1]
structure = gridpp.BarnesStructure(2500)
pobs = [1]
background = np.zeros([1, N])
background_at_points = [0]
max_points = 10
output, sigma = gridpp.optimal_interpolation_full(
grid, background, bvariance, points, pobs, obs_variance,
background_at_points, bvariance_at_points, structure, max_points)
# np.testing.assert_array_almost_equal(output, np.array([[np.exp(-0.5)/1.1, 1/1.1, np.exp(-0.5*9)/1.1]]))
# np.testing.assert_array_almost_equal(sigma, np.array([[0, np.sqrt(0.1/1.1), 1]]))
self.assertAlmostEqual(sigma[0, 1], 0.1 / 1.1)
def test_missing_values(self):
"""Check that missing values are not used in OI"""
obs = np.array([1, np.nan, 2, 3, np.nan, np.nan, 4, np.nan])
N = len(obs)
y = np.arange(0, N * 1000, 1000)
background = np.zeros(N)
points = gridpp.Points(y, np.zeros(N), np.zeros(N), np.zeros(N),
gridpp.Cartesian)
ratios = np.ones(N)
structure = gridpp.BarnesStructure(1000, 0)
analysis = gridpp.optimal_interpolation(points, background, points,
obs, ratios, background,
structure, 100)
I = np.where(np.isnan(y) == 0)[0]
points1 = gridpp.Points(y[I], np.zeros(len(I)), np.zeros(len(I)),
np.zeros(len(I)), gridpp.Cartesian)
analysis1 = gridpp.optimal_interpolation(points, background, points1,
obs[I], ratios[I],
background[I], structure, 100)
np.testing.assert_array_almost_equal(analysis, analysis1)
def test_cross_validation(self):
y = np.array([0, 1000, 2000, 3000])
N = len(y)
obs = np.array([0, 1, 2, 3])
background = np.zeros(N)
points = gridpp.Points(y, np.zeros(N), np.zeros(N), np.zeros(N),
gridpp.Cartesian)
ratios = np.ones(N)
Icv = [0, 2, 3]
points_cv = gridpp.Points(y[Icv], np.zeros(N - 1), np.zeros(N - 1),
np.zeros(N - 1), gridpp.Cartesian)
structure = gridpp.BarnesStructure(1000, 0)
structure_cv = gridpp.CrossValidation(structure, 750)
analysis = gridpp.optimal_interpolation(points, background, points_cv,
obs[Icv], ratios[Icv],
background[Icv], structure,
100)
analysis_cv = gridpp.optimal_interpolation(points, background, points,
obs, ratios, background,
structure_cv, 100)
def test_cross_validation_grid(self):
""" Check that the CV structure function works """
np.random.seed(1000)
y, x = np.meshgrid(np.arange(0, 3500, 500), np.arange(0, 3500, 500))
Y = y.shape[0]
X = y.shape[1]
grid = gridpp.Grid(y, x, np.zeros(x.shape), np.zeros(x.shape),
gridpp.Cartesian)
background = np.random.rand(Y, X) * 0
obs = np.array([10, 20, 30])
x_o = np.array([1000, 2000, 3000])
y_o = np.array([1000, 2000, 3000])
N = len(obs)
points = gridpp.Points(y_o, x_o, np.zeros(N), np.zeros(N),
gridpp.Cartesian)
background_o = gridpp.nearest(grid, points, background)
ratios = np.ones(N)
k = 0
ii = np.arange(N).astype('int') != k
points_cv = gridpp.Points(y_o[ii], x_o[ii], np.zeros(N - 1),
np.zeros(N - 1), gridpp.Cartesian)
structure = gridpp.BarnesStructure(1000, 0)
structure_cv = gridpp.CrossValidation(structure, 750)
analysis = gridpp.optimal_interpolation(grid, background, points_cv,
obs[ii], ratios[ii],
background_o[ii], structure,
100)
analysis_cv = gridpp.optimal_interpolation(points, background_o,
points, obs, ratios,
background_o, structure_cv,
100)
self.assertAlmostEqual(
gridpp.nearest(grid, points, analysis)[k], analysis_cv[k])
def test_invalid_hmax(self):
with self.assertRaises(Exception) as e:
structure = gridpp.BarnesStructure(2000, 100, 0, -1)
def test_barnes_h(self):
hs = [-1, np.nan]
for h in hs:
with self.subTest(h=h):
with self.assertRaises(Exception) as e:
structure = gridpp.BarnesStructure(h)
def test_invalid_cv(self):
barnes = gridpp.BarnesStructure(2000)
for dist in [-1, np.nan]:
with self.assertRaises(Exception) as e:
structure = gridpp.CrossValidation(barnes, dist)
def main():
parser = argparse.ArgumentParser(
description='Runs gridpp benchmarks for processing performance')
parser.add_argument('-j',
type=int,
help='Run multiple cores',
dest='num_cores')
parser.add_argument(
'-s',
type=int,
default=1,
help='Enlarge the inputs by this scaling factor to run a bigger test',
dest='scaling')
parser.add_argument('-n',
type=int,
default=1,
help='Number of iterations to average over',
dest='iterations')
parser.add_argument('-t', help='Run only this function', dest="function")
# if len(sys.argv) == 1:
# parser.print_help()
# sys.exit(1)
args = parser.parse_args()
input = dict()
grids = dict()
points = dict()
np.random.seed(1000)
for i in [10, 50, 100, 200, 500, 1000, 2000, 10000]:
input[i] = np.random.rand(i * args.scaling, i) * 10
for i in [10, 50, 100, 200, 500, 1000]:
grids[i] = gridpp.Grid(*np.meshgrid(
np.linspace(0, 1, i), np.linspace(0, 1, i * args.scaling)))
for i in [1000, 100000]:
# points[i] = gridpp.Points(np.linspace(0, 1, i), np.zeros(i))
points[i] = gridpp.Points(
np.random.rand(i) * 10,
np.random.rand(i) * 10)
structure = gridpp.BarnesStructure(10000)
radius = 7
quantile = 0.5
thresholds = np.linspace(0, 1, 11)
run = collections.OrderedDict()
run[(gridpp.Grid, "1000²")] = {
"expected":
0.74,
"args":
np.meshgrid(np.linspace(0, 1, 1000 * args.scaling),
np.linspace(0, 1, 1000))
}
run[(gridpp.neighbourhood, "10000²")] = {
"expected": 2.05,
"args": (np.zeros([10000, 10000]), radius, gridpp.Mean)
}
run[(gridpp.neighbourhood, "2000² max")] = {
"expected": 0.99,
"args": (input[2000], radius, gridpp.Max)
}
run[(gridpp.neighbourhood_quantile_fast, "2000²")] = {
"expected": 1.23,
"args": (input[2000], quantile, radius, thresholds)
}
run[(gridpp.neighbourhood_quantile, "500²")] = {
"expected": 1.70,
"args": (input[500], quantile, radius)
}
run[(gridpp.bilinear, "1000²")] = {
"expected": 1.68,
"args": (grids[1000], grids[1000], input[1000])
}
run[(gridpp.bilinear, "1000² x 50")] = {
"expected":
4.42,
"args": (grids[1000], grids[1000],
np.repeat(np.expand_dims(input[1000], 0), 50, axis=0))
}
run[(gridpp.nearest, "1000²")] = {
"expected": 1.52,
"args": (grids[1000], grids[1000], input[1000])
}
run[(gridpp.nearest, "1000² x 50")] = {
"expected":
2.30,
"args": (grids[1000], grids[1000],
np.repeat(np.expand_dims(input[1000], 0), 50, axis=0))
}
run[(gridpp.optimal_interpolation, "1000² 1000")] = {
"expected":
1.57,
"args": (grids[1000], input[1000], points[1000], np.zeros(1000),
np.ones(1000), np.ones(1000), structure, 20)
}
run[(gridpp.dewpoint, "1e7")] = {
"expected": 0.53,
"args": (np.zeros(10000000) + 273.15, np.zeros(10000000))
}
run[(gridpp.fill, "1e5")] = {
"expected":
0.52,
"args": (grids[1000], np.zeros([1000, 1000]), points[100000],
np.ones(100000) * 5000, 1, False)
}
run[(gridpp.doping_square, "1e5")] = {
"expected":
0.16,
"args": (grids[1000], np.zeros([1000, 1000]), points[100000],
np.ones(100000) * 1, np.ones(100000, 'int') * 5, False)
}
run[(gridpp.doping_circle, "1e5")] = {
"expected":
0.52,
"args": (grids[1000], np.zeros([1000, 1000]), points[100000],
np.ones(100000) * 1, np.ones(100000) * 5000, False)
}
print("Gridpp version %s" % gridpp.version())
if args.num_cores is not None:
print(
"Function Expected Time Diff Scaling"
)
else:
print(
"Function Expected Time Diff")
num_cores = [1]
if args.num_cores is not None and args.num_cores != 1:
num_cores += [args.num_cores]
for key in run.keys():
timings = dict()
for num_core in num_cores:
timings[num_core] = 0
if isinstance(key, tuple):
name = key[0].__name__ + " " + str(key[1])
func = key[0]
else:
name = key.__name__
func = key
if args.function is not None:
if func.__name__ != args.function:
continue
# Allow functions to fail (useful when benchmarking older versions of gridpp
# where functions may not be defined).
try:
for num_core in num_cores:
gridpp.set_omp_threads(num_core)
for it in range(args.iterations):
s_time = time.time()
func(*run[key]["args"])
e_time = time.time()
curr_time = e_time - s_time
timings[num_core] += curr_time
except Exception as e:
print("Could not run %s" % key)
continue
# print("%s() Expected: %.2f s Time: %.2f s" % (func.__name__, run[key]["expected"], e_time - s_time))
for num_core in num_cores:
timings[num_core] /= args.iterations
diff = (timings[1] - run[key]["expected"] * args.scaling) / (
run[key]["expected"] * args.scaling) * 100
string = "%-36s %8.2f %8.2f %8.2f %%" % (
name, run[key]["expected"] * args.scaling, timings[1], diff)
if args.num_cores is not None:
scaling = timings[1] / timings[args.num_cores] / args.num_cores
expected = timings[1] / args.num_cores
scaling = 1 - (timings[args.num_cores] - expected) / (timings[1] -
expected)
# scaling = (1 - timings[args.num_cores] / timings[1]) * (args.num_cores + 1)
string += " %8.2f %%" % (scaling * 100)
print(string)
def horizontal_oi(geo,
background,
observations,
gelevs,
glafs,
hlength=10000.,
vlength=10000.,
wlength=0.5,
elev_gradient=0,
structure_function="Barnes",
land_only=False,
max_locations=50,
epsilon=0.5,
minvalue=None,
maxvalue=None,
interpol="bilinear"):
if gridpp is None:
raise Exception("You need gridpp to perform OI")
glats = geo.lats
glons = geo.lons
def obs2vectors(my_obs):
return my_obs.lons, my_obs.lats, my_obs.stids, my_obs.elevs, \
my_obs.values, my_obs.cis, my_obs.lafs
vectors = np.vectorize(obs2vectors)
lons, lats, stids, elevs, values, pci, lafs = vectors(observations)
glats = np.transpose(glats)
glons = np.transpose(glons)
background = np.transpose(background)
gelevs = np.transpose(gelevs)
glafs = np.transpose(glafs)
bgrid = gridpp.Grid(glats, glons, gelevs)
points = gridpp.Points(lats, lons, elevs)
if interpol == "bilinear":
pbackground = gridpp.bilinear(bgrid, points, background)
elif interpol == "nearest":
pbackground = gridpp.nearest(bgrid, points, background, elev_gradient)
else:
raise NotImplementedError
variance_ratios = np.full(points.size(), epsilon)
if structure_function == "Barnes":
# No land/sea weight when land_only = True
if land_only:
wlength = 0.
structure = gridpp.BarnesStructure(hlength, vlength, wlength)
else:
raise NotImplementedError
field = gridpp.optimal_interpolation(bgrid, background, points, values,
variance_ratios, pbackground,
structure, max_locations)
field = np.asarray(field)
if minvalue is not None:
field[field < minvalue] = minvalue
if maxvalue is not None:
field[field > maxvalue] = maxvalue
if land_only:
no_land = np.where(glafs == 0)
field[no_land] = background[no_land]
return np.transpose(field)
blons = file.variables['longitude'][:]
belevs = file.variables['altitude'][:]
bgrid = gridpp.Grid(blats, blons, belevs)
# Downscaling
gradient = -0.0065
background = gridpp.simple_gradient(bgrid_2500m, bgrid, background_2500m, gradient)
points = gridpp.Points(obs_lats[index_valid_obs], obs_lons[index_valid_obs], obs_elevs[index_valid_obs])
pbackground = gridpp.bilinear(bgrid, points, background)
variance_ratios = np.full(points.size(), 0.1)
h = 100000
v = 200
structure = gridpp.BarnesStructure(h, v)
max_points = 50
analysis = gridpp.optimal_interpolation(bgrid, background, points,
obs[index_valid_obs], variance_ratios, pbackground, structure, max_points)
# Plotting the increments
diff = analysis - background
mpl.pcolormesh(blons, blats, diff, cmap="RdBu_r", vmin=-2, vmax=2)
mpl.xlim(0, 35)
mpl.ylim(55, 75)
mpl.gca().set_aspect(2)
cb = mpl.colorbar()
cb.set_label(r"Increment ($\degree C$)")