def test_extrapolation(self): """ Check that extrapolation filter works """ N = 5 y = np.linspace(0, 1000, N) x = np.zeros(N) grid = gridpp.Points(y, x, x, x, gridpp.Cartesian) points = gridpp.Points([0, 100, 900, 1000], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], gridpp.Cartesian) Y = points.size() pratios = 0.1 * np.ones(Y) structure = gridpp.BarnesStructure(500) pobs = [0, 1, 1, 0] background = np.zeros(grid.size()) pbackground = np.zeros(Y) max_points = 10 output0 = gridpp.optimal_interpolation(grid, background, points, pobs, pratios, pbackground, structure, max_points, False) output1 = gridpp.optimal_interpolation(grid, background, points, pobs, pratios, pbackground, structure, max_points, True) # Turning off extrapolation should mean we don't get increments greater than 1 self.assertTrue(np.max(output0) == 1) self.assertTrue(np.max(output1) > 1) I = np.where(output1 < 1)[0] np.testing.assert_array_almost_equal(output0[I], output1[I])
def test_basic(self): x = [0, 1000, 2000, 3000, np.nan] structure_corr = dict() barnes = gridpp.BarnesStructure(2000) structure_corr[barnes] = [1, 0.8824968934059143, 0.6065306663513184, 0.32465246319770813, 0] structure_corr[gridpp.CressmanStructure(2000)] = [1, 0.6, 0, 0, 0] structure_corr[gridpp.CrossValidation(barnes, 1000)] = [0, 0, 0.6065306663513184, 0.32465246319770813, 0] N = len(x) for structure, corr in structure_corr.items(): is_cv = isinstance(structure, gridpp.CrossValidation) for i in range(N): with self.subTest(structure=type(structure), i=i): p1 = gridpp.Point(0, 0, 0, 0, gridpp.Cartesian) p2 = gridpp.Point(x[i], 0, 0, 0, gridpp.Cartesian) funcs = [structure.corr, structure.corr_background] if is_cv: funcs = [structure.corr_background] for func in funcs: self.assertAlmostEqual(corr[i], func(p1, p2)) # Check that changing the order does not change the results self.assertAlmostEqual(corr[i], func(p2, p1)) # Check identical points if not is_cv and not np.isnan(x[i]): self.assertAlmostEqual(1, func(p2, p2))
def test_invalid_arguments(self): """ Check that exception is thrown on invalid input values """ # Set up struct with valid input arguments ok_args = collections.OrderedDict({ 'grid': gridpp.Grid([[0, 0, 0]], [[0, 2500, 10000]], [[0, 0, 0]], [[0, 0, 0]], gridpp.Cartesian), 'background': np.zeros([1, 3]), 'points': gridpp.Points([0], [2500], [0], [0], gridpp.Cartesian), 'pobs': [1], 'pratios': [0.1], 'pbackground': [0], 'structure': gridpp.BarnesStructure(2500), 'max_points': 10 }) # Set up struct with invalid input arguments that will be substituted into ok_args one at a # time in order to look for an exception being raised. Use an array of different invalid # arguments for each key. x = np.zeros([3, 2]) invalid_args = { # Grid size mismatch, and coordinate-type mismatch 'grid': [ gridpp.Grid(x, x, x, x, gridpp.Cartesian), gridpp.Grid([[0, 0, 0]], [[0, 2500, 10000]]) ], # Points size mismatch, and coordinate-type mismatch 'points': [ gridpp.Points([0, 1], [0, 2500], [0, 0], [0, 0], gridpp.Cartesian), gridpp.Points([0], [2500]) ], 'pratios': [np.zeros(11)], 'pobs': [np.zeros([11])], 'background': [np.zeros([2, 11])], 'pbackground': [np.zeros(21)], 'max_points': [-1] } for key in invalid_args.keys(): for arg in invalid_args[key]: args0 = ok_args.copy() args0[key] = arg q = [args0[f] for f in args0] with self.subTest(key=key, arg=arg): with self.assertRaises(ValueError) as e: output = gridpp.optimal_interpolation(*q)
def test_barnes_hmax(self): hmaxs = [0, 1000, 2000, 10000] p0 = gridpp.Point(0, 0, 0, 0, gridpp.Cartesian) dist_ans = {0:1, 1000:0.8824968934059143, 2000:0.6065306663513184, 3000:0.32465246319770813} for hmax in hmaxs: for dist, ans in dist_ans.items(): with self.subTest(hmax=hmax, dist=dist): structure = gridpp.BarnesStructure(2000, 0, 0, hmax) corr = structure.corr(p0, gridpp.Point(dist, 0, 0, 0, gridpp.Cartesian)) if dist > hmax: self.assertEqual(0, corr) else: self.assertEqual(ans, corr)
def test_simple_1d(self): N = 3 y = [[0, 0, 0]] x = [[0, 2500, 10000]] grid = gridpp.Grid(y, x, y, y, gridpp.Cartesian) points = gridpp.Points([0], [2500], [0], [0], gridpp.Cartesian) pratios = [0.1] structure = gridpp.BarnesStructure(2500) pobs = [1] background = np.zeros([1, N]) pbackground = [0] max_points = 10 output = gridpp.optimal_interpolation(grid, background, points, pobs, pratios, pbackground, structure, max_points) np.testing.assert_array_almost_equal( output, np.array([[np.exp(-0.5) / 1.1, 1 / 1.1, np.exp(-0.5 * 9) / 1.1]]))
def test_simple_1d_full(self): N = 3 y = [[0, 0, 0]] x = [[0, 2500, 10000]] grid = gridpp.Grid(y, x, y, y, gridpp.Cartesian) points = gridpp.Points([0], [2500], [0], [0], gridpp.Cartesian) bvariance = np.ones([1, N]) obs_variance = [0.1] bvariance_at_points = [1] structure = gridpp.BarnesStructure(2500) pobs = [1] background = np.zeros([1, N]) background_at_points = [0] max_points = 10 output, sigma = gridpp.optimal_interpolation_full( grid, background, bvariance, points, pobs, obs_variance, background_at_points, bvariance_at_points, structure, max_points) # np.testing.assert_array_almost_equal(output, np.array([[np.exp(-0.5)/1.1, 1/1.1, np.exp(-0.5*9)/1.1]])) # np.testing.assert_array_almost_equal(sigma, np.array([[0, np.sqrt(0.1/1.1), 1]])) self.assertAlmostEqual(sigma[0, 1], 0.1 / 1.1)
def test_missing_values(self): """Check that missing values are not used in OI""" obs = np.array([1, np.nan, 2, 3, np.nan, np.nan, 4, np.nan]) N = len(obs) y = np.arange(0, N * 1000, 1000) background = np.zeros(N) points = gridpp.Points(y, np.zeros(N), np.zeros(N), np.zeros(N), gridpp.Cartesian) ratios = np.ones(N) structure = gridpp.BarnesStructure(1000, 0) analysis = gridpp.optimal_interpolation(points, background, points, obs, ratios, background, structure, 100) I = np.where(np.isnan(y) == 0)[0] points1 = gridpp.Points(y[I], np.zeros(len(I)), np.zeros(len(I)), np.zeros(len(I)), gridpp.Cartesian) analysis1 = gridpp.optimal_interpolation(points, background, points1, obs[I], ratios[I], background[I], structure, 100) np.testing.assert_array_almost_equal(analysis, analysis1)
def test_cross_validation(self): y = np.array([0, 1000, 2000, 3000]) N = len(y) obs = np.array([0, 1, 2, 3]) background = np.zeros(N) points = gridpp.Points(y, np.zeros(N), np.zeros(N), np.zeros(N), gridpp.Cartesian) ratios = np.ones(N) Icv = [0, 2, 3] points_cv = gridpp.Points(y[Icv], np.zeros(N - 1), np.zeros(N - 1), np.zeros(N - 1), gridpp.Cartesian) structure = gridpp.BarnesStructure(1000, 0) structure_cv = gridpp.CrossValidation(structure, 750) analysis = gridpp.optimal_interpolation(points, background, points_cv, obs[Icv], ratios[Icv], background[Icv], structure, 100) analysis_cv = gridpp.optimal_interpolation(points, background, points, obs, ratios, background, structure_cv, 100)
def test_cross_validation_grid(self): """ Check that the CV structure function works """ np.random.seed(1000) y, x = np.meshgrid(np.arange(0, 3500, 500), np.arange(0, 3500, 500)) Y = y.shape[0] X = y.shape[1] grid = gridpp.Grid(y, x, np.zeros(x.shape), np.zeros(x.shape), gridpp.Cartesian) background = np.random.rand(Y, X) * 0 obs = np.array([10, 20, 30]) x_o = np.array([1000, 2000, 3000]) y_o = np.array([1000, 2000, 3000]) N = len(obs) points = gridpp.Points(y_o, x_o, np.zeros(N), np.zeros(N), gridpp.Cartesian) background_o = gridpp.nearest(grid, points, background) ratios = np.ones(N) k = 0 ii = np.arange(N).astype('int') != k points_cv = gridpp.Points(y_o[ii], x_o[ii], np.zeros(N - 1), np.zeros(N - 1), gridpp.Cartesian) structure = gridpp.BarnesStructure(1000, 0) structure_cv = gridpp.CrossValidation(structure, 750) analysis = gridpp.optimal_interpolation(grid, background, points_cv, obs[ii], ratios[ii], background_o[ii], structure, 100) analysis_cv = gridpp.optimal_interpolation(points, background_o, points, obs, ratios, background_o, structure_cv, 100) self.assertAlmostEqual( gridpp.nearest(grid, points, analysis)[k], analysis_cv[k])
def test_invalid_hmax(self): with self.assertRaises(Exception) as e: structure = gridpp.BarnesStructure(2000, 100, 0, -1)
def test_barnes_h(self): hs = [-1, np.nan] for h in hs: with self.subTest(h=h): with self.assertRaises(Exception) as e: structure = gridpp.BarnesStructure(h)
def test_invalid_cv(self): barnes = gridpp.BarnesStructure(2000) for dist in [-1, np.nan]: with self.assertRaises(Exception) as e: structure = gridpp.CrossValidation(barnes, dist)
def main(): parser = argparse.ArgumentParser( description='Runs gridpp benchmarks for processing performance') parser.add_argument('-j', type=int, help='Run multiple cores', dest='num_cores') parser.add_argument( '-s', type=int, default=1, help='Enlarge the inputs by this scaling factor to run a bigger test', dest='scaling') parser.add_argument('-n', type=int, default=1, help='Number of iterations to average over', dest='iterations') parser.add_argument('-t', help='Run only this function', dest="function") # if len(sys.argv) == 1: # parser.print_help() # sys.exit(1) args = parser.parse_args() input = dict() grids = dict() points = dict() np.random.seed(1000) for i in [10, 50, 100, 200, 500, 1000, 2000, 10000]: input[i] = np.random.rand(i * args.scaling, i) * 10 for i in [10, 50, 100, 200, 500, 1000]: grids[i] = gridpp.Grid(*np.meshgrid( np.linspace(0, 1, i), np.linspace(0, 1, i * args.scaling))) for i in [1000, 100000]: # points[i] = gridpp.Points(np.linspace(0, 1, i), np.zeros(i)) points[i] = gridpp.Points( np.random.rand(i) * 10, np.random.rand(i) * 10) structure = gridpp.BarnesStructure(10000) radius = 7 quantile = 0.5 thresholds = np.linspace(0, 1, 11) run = collections.OrderedDict() run[(gridpp.Grid, "1000²")] = { "expected": 0.74, "args": np.meshgrid(np.linspace(0, 1, 1000 * args.scaling), np.linspace(0, 1, 1000)) } run[(gridpp.neighbourhood, "10000²")] = { "expected": 2.05, "args": (np.zeros([10000, 10000]), radius, gridpp.Mean) } run[(gridpp.neighbourhood, "2000² max")] = { "expected": 0.99, "args": (input[2000], radius, gridpp.Max) } run[(gridpp.neighbourhood_quantile_fast, "2000²")] = { "expected": 1.23, "args": (input[2000], quantile, radius, thresholds) } run[(gridpp.neighbourhood_quantile, "500²")] = { "expected": 1.70, "args": (input[500], quantile, radius) } run[(gridpp.bilinear, "1000²")] = { "expected": 1.68, "args": (grids[1000], grids[1000], input[1000]) } run[(gridpp.bilinear, "1000² x 50")] = { "expected": 4.42, "args": (grids[1000], grids[1000], np.repeat(np.expand_dims(input[1000], 0), 50, axis=0)) } run[(gridpp.nearest, "1000²")] = { "expected": 1.52, "args": (grids[1000], grids[1000], input[1000]) } run[(gridpp.nearest, "1000² x 50")] = { "expected": 2.30, "args": (grids[1000], grids[1000], np.repeat(np.expand_dims(input[1000], 0), 50, axis=0)) } run[(gridpp.optimal_interpolation, "1000² 1000")] = { "expected": 1.57, "args": (grids[1000], input[1000], points[1000], np.zeros(1000), np.ones(1000), np.ones(1000), structure, 20) } run[(gridpp.dewpoint, "1e7")] = { "expected": 0.53, "args": (np.zeros(10000000) + 273.15, np.zeros(10000000)) } run[(gridpp.fill, "1e5")] = { "expected": 0.52, "args": (grids[1000], np.zeros([1000, 1000]), points[100000], np.ones(100000) * 5000, 1, False) } run[(gridpp.doping_square, "1e5")] = { "expected": 0.16, "args": (grids[1000], np.zeros([1000, 1000]), points[100000], np.ones(100000) * 1, np.ones(100000, 'int') * 5, False) } run[(gridpp.doping_circle, "1e5")] = { "expected": 0.52, "args": (grids[1000], np.zeros([1000, 1000]), points[100000], np.ones(100000) * 1, np.ones(100000) * 5000, False) } print("Gridpp version %s" % gridpp.version()) if args.num_cores is not None: print( "Function Expected Time Diff Scaling" ) else: print( "Function Expected Time Diff") num_cores = [1] if args.num_cores is not None and args.num_cores != 1: num_cores += [args.num_cores] for key in run.keys(): timings = dict() for num_core in num_cores: timings[num_core] = 0 if isinstance(key, tuple): name = key[0].__name__ + " " + str(key[1]) func = key[0] else: name = key.__name__ func = key if args.function is not None: if func.__name__ != args.function: continue # Allow functions to fail (useful when benchmarking older versions of gridpp # where functions may not be defined). try: for num_core in num_cores: gridpp.set_omp_threads(num_core) for it in range(args.iterations): s_time = time.time() func(*run[key]["args"]) e_time = time.time() curr_time = e_time - s_time timings[num_core] += curr_time except Exception as e: print("Could not run %s" % key) continue # print("%s() Expected: %.2f s Time: %.2f s" % (func.__name__, run[key]["expected"], e_time - s_time)) for num_core in num_cores: timings[num_core] /= args.iterations diff = (timings[1] - run[key]["expected"] * args.scaling) / ( run[key]["expected"] * args.scaling) * 100 string = "%-36s %8.2f %8.2f %8.2f %%" % ( name, run[key]["expected"] * args.scaling, timings[1], diff) if args.num_cores is not None: scaling = timings[1] / timings[args.num_cores] / args.num_cores expected = timings[1] / args.num_cores scaling = 1 - (timings[args.num_cores] - expected) / (timings[1] - expected) # scaling = (1 - timings[args.num_cores] / timings[1]) * (args.num_cores + 1) string += " %8.2f %%" % (scaling * 100) print(string)
def horizontal_oi(geo, background, observations, gelevs, glafs, hlength=10000., vlength=10000., wlength=0.5, elev_gradient=0, structure_function="Barnes", land_only=False, max_locations=50, epsilon=0.5, minvalue=None, maxvalue=None, interpol="bilinear"): if gridpp is None: raise Exception("You need gridpp to perform OI") glats = geo.lats glons = geo.lons def obs2vectors(my_obs): return my_obs.lons, my_obs.lats, my_obs.stids, my_obs.elevs, \ my_obs.values, my_obs.cis, my_obs.lafs vectors = np.vectorize(obs2vectors) lons, lats, stids, elevs, values, pci, lafs = vectors(observations) glats = np.transpose(glats) glons = np.transpose(glons) background = np.transpose(background) gelevs = np.transpose(gelevs) glafs = np.transpose(glafs) bgrid = gridpp.Grid(glats, glons, gelevs) points = gridpp.Points(lats, lons, elevs) if interpol == "bilinear": pbackground = gridpp.bilinear(bgrid, points, background) elif interpol == "nearest": pbackground = gridpp.nearest(bgrid, points, background, elev_gradient) else: raise NotImplementedError variance_ratios = np.full(points.size(), epsilon) if structure_function == "Barnes": # No land/sea weight when land_only = True if land_only: wlength = 0. structure = gridpp.BarnesStructure(hlength, vlength, wlength) else: raise NotImplementedError field = gridpp.optimal_interpolation(bgrid, background, points, values, variance_ratios, pbackground, structure, max_locations) field = np.asarray(field) if minvalue is not None: field[field < minvalue] = minvalue if maxvalue is not None: field[field > maxvalue] = maxvalue if land_only: no_land = np.where(glafs == 0) field[no_land] = background[no_land] return np.transpose(field)
blons = file.variables['longitude'][:] belevs = file.variables['altitude'][:] bgrid = gridpp.Grid(blats, blons, belevs) # Downscaling gradient = -0.0065 background = gridpp.simple_gradient(bgrid_2500m, bgrid, background_2500m, gradient) points = gridpp.Points(obs_lats[index_valid_obs], obs_lons[index_valid_obs], obs_elevs[index_valid_obs]) pbackground = gridpp.bilinear(bgrid, points, background) variance_ratios = np.full(points.size(), 0.1) h = 100000 v = 200 structure = gridpp.BarnesStructure(h, v) max_points = 50 analysis = gridpp.optimal_interpolation(bgrid, background, points, obs[index_valid_obs], variance_ratios, pbackground, structure, max_points) # Plotting the increments diff = analysis - background mpl.pcolormesh(blons, blats, diff, cmap="RdBu_r", vmin=-2, vmax=2) mpl.xlim(0, 35) mpl.ylim(55, 75) mpl.gca().set_aspect(2) cb = mpl.colorbar() cb.set_label(r"Increment ($\degree C$)")