def get_dataset(self, dataset):
points = self.get_points(dataset)
if "values" in dataset:
values = dataset["values"]
else:
values = np.nan * np.zeros(points.size())
dataset = titanlib.Dataset(points, values)
return dataset
def __init__(self, var, settings, tests, datasources, an_time, debug=False, corep=1, test_flags=None):
if tit is None:
raise ModuleNotFoundError("titanlib was not loaded properly")
self.var = var
self.tests = tests
self.settings = settings
self.test_flags = test_flags
self.debug = debug
obstimes = []
lons = []
lats = []
stids = []
elevs = []
values = []
varnames = []
providers = []
passed_tests = []
self.datasources = datasources
# Get global data
for obs_set in self.datasources:
lobstimes, llons, llats, lstids, lelevs, lvalues, lvarnames = obs_set.get_obs()
obstimes = obstimes + lobstimes
lons = lons + llons
lats = lats + llats
stids = stids + lstids
elevs = elevs + lelevs
values = values + lvalues
varnames = varnames + lvarnames
for i in range(0, len(llons)):
providers.append(obs_set.label)
for i in range(0, len(lons)):
passed_tests.append([])
flags = np.zeros(len(lons))
cis = np.ones(len(lons)) * corep
lafs = np.ones(len(lons))
stids = stids
observations = []
for i in range(0, len(lons)):
observations.append(surfex.Observation(obstimes[i], lons[i], lats[i], values[i], elev=elevs[i],
stid=stids[i], varname=varnames[i]))
self.titan_dataset = tit.Dataset(lats, lons, elevs, values)
if passed_tests is None:
passed_tests = []
for i in range(0, len(lons)):
passed_tests.append([])
self.passed_tests = passed_tests
else:
self.passed_tests = passed_tests
QCDataSet.__init__(self, an_time, observations, flags, cis, lafs, providers,
passed_tests=None, remove_invalid_elevs=False)
def get_dataset(self):
N = 5
np.random.seed(0)
lats = np.random.randn(N) * 10000;
lons = np.random.randn(N) * 10000;
elevs = np.random.rand(N) * 100;
points = titanlib.Points(lats, lons, elevs)
values = np.zeros(N);
dataset = titanlib.Dataset(points, values);
return dataset
def test_simple_dataset(self):
elevs0 = [0, 0, 0]
values0 = [0, 1, 100]
points0 = titanlib.Points(lats, lons, elevs0)
dataset = titanlib.Dataset(points0, values0)
sct, rep = dataset.sct(num_min, num_max, inner_radius,
outer_radius, num_iterations, num_min_prof, min_elev_diff, min_horizontal_scale,
vertical_scale, pos, neg, eps2)
np.testing.assert_array_equal(dataset.flags, [0, 0, 1])
def test_1(self):
"""Check that the test doesn't fail"""
N = 5
np.random.seed(0)
lats = np.random.randn(N) * 10000
lons = np.random.randn(N) * 10000
elevs = np.random.rand(N) * 100
values = np.random.randn(N) * 6
dataset = titanlib.Dataset(lats, lons, elevs, values)
dataset.range_check([-100], [-100], [0, 1, 2])
dataset.range_check([-100], [-100], [1, 2, 4])
self.assertListEqual([i for i in dataset.flags], [1, 1, 1, 0, 1])
def test_get_values(self):
N = 10
lats = [60]*N
lons = np.linspace(60, 60.001, N)
elevs = [0]*10
points = titanlib.Points(lats, lons, elevs)
values = [0, 0, 0, 0, 0, 0, 0, 0, 0.1, 1]
N = len(lats)
radius = [10000]
num_min = [1]
threshold = 1
elev_gradient = -0.0065
max_elev_diff = 200
min_std = 0.01
num_iterations = 2
dataset = titanlib.Dataset(points, values)
dataset.values[:] = [1]
print(dataset.get_values())
dataset.points = points
def test_buddy_check(self):
N = 10
lats = [60]*N
lons = np.linspace(60, 60.001, N)
elevs = [0]*10
points = titanlib.Points(lats, lons, elevs)
values = [0, 0, 0, 0, 0, 0, 0, 0, 0.1, 1]
N = len(lats)
radius = [10000]
num_min = [1]
threshold = 1
elev_gradient = -0.0065
max_elev_diff = 200
min_std = 0.01
num_iterations = 2
dataset = titanlib.Dataset(points, values)
dataset.buddy_check(radius, num_min, threshold,
max_elev_diff, elev_gradient, min_std, num_iterations)
np.testing.assert_array_equal(dataset.flags, [0]*8 + [1]*2)
def __init__(self,
var,
settings,
tests,
test_flags,
an_time,
debug=False,
corep=1):
self.var = var
self.tests = define_quality_control(tests, settings)
self.settings = settings
self.test_flags = test_flags
self.debug = debug
obstimes = []
lons = []
lats = []
stids = []
elevs = []
values = []
varnames = []
providers = []
passed_tests = []
self.datasources = surfex.obs.get_datasources(an_time,
settings["sets"])
# Get global data
for obs_set in self.datasources:
lobstimes, llons, llats, lstids, lelevs, lvalues, lvarnames = obs_set.get_obs(
)
obstimes = obstimes + lobstimes
lons = lons + llons
lats = lats + llats
stids = stids + lstids
elevs = elevs + lelevs
values = values + lvalues
varnames = varnames + lvarnames
for i in range(0, len(llons)):
providers.append(obs_set.label)
for i in range(0, len(lons)):
passed_tests.append([])
flags = np.zeros(len(lons))
cis = np.ones(len(lons)) * corep
lafs = np.ones(len(lons))
stids = stids
observations = []
for i in range(0, len(lons)):
observations.append(
surfex.Observation(obstimes[i],
lons[i],
lats[i],
values[i],
elev=elevs[i],
stid=stids[i],
varname=varnames[i]))
self.titan_dataset = tit.Dataset(lats, lons, elevs, values)
if passed_tests is None:
passed_tests = []
for i in range(0, len(lons)):
passed_tests.append([])
self.passed_tests = passed_tests
else:
self.passed_tests = passed_tests
QCDataSet.__init__(self,
an_time,
observations,
flags,
cis,
lafs,
providers,
passed_tests=None,
remove_invalid_elevs=False)