def test_UtcPeriod_trim(self):
utc = api.Calendar()
t0 = utc.time(2018, 1, 1, 0, 0, 0)
t1 = utc.time(2018, 2, 1, 0, 0, 0)
t2 = utc.time(2018, 3, 1, 0, 0, 0)
t3 = utc.time(2018, 4, 1, 0, 0, 0)
p_0_1 = api.UtcPeriod(t0, t1)
p_0_3 = api.UtcPeriod(t0, t3)
assert api.UtcPeriod(t0, t1).trim(utc, utc.MONTH, api.trim_policy.TRIM_IN) == p_0_1
assert api.UtcPeriod(t0, t1).trim(utc, 3600*24*30, api.trim_policy.TRIM_IN) == p_0_1
assert api.UtcPeriod(t0, t1 + 1).trim(utc, utc.MONTH) == p_0_1
assert api.UtcPeriod(t1 - 1, t2 + 1).trim(utc, utc.MONTH, api.trim_policy.TRIM_OUT) == p_0_3
# diff_units
assert api.UtcPeriod(t0, t3).diff_units(utc, utc.MONTH) == 3
# trim null period, should trow
p_null = api.UtcPeriod()
try:
p_null.trim(utc, utc.DAY)
did_raise = False
except RuntimeError:
did_raise = True
assert did_raise
assert p_null.diff_units(utc, utc.DAY) == 0
assert api.UtcPeriod(t3, t0).diff_units(utc, utc.MONTH) == -3
def test_read_forecast(self):
utc = api.Calendar()
ds = SmGTsRepository(PREPROD, FC_PREPROD)
nl = [u'/LTMS-Abisko........-T0000A5P_EC00_ENS']
t0 = utc.time(2016, 8, 1, 00, 00, 00)
t1 = utc.time(2016, 8, 10, 00, 00, 00)
p = api.UtcPeriod(t0, t1)
fclist = ds.read_forecast(nl, p)
self.assertIsNotNone(fclist)
fc1 = fclist[u'/LTMS-Abisko........-T0000A5P_EC00_ENS']
fc1_v = [fc1.value(i) for i in range(fc1.size())]
# Test times here, left for manual inspection fc1_t = [utc.to_string(fc1.time(i)) for i in range(fc1.size())]
self.assertIsNotNone(fc1_v)
# Values read from SMG PREPROD:
fc1_repeat = [
1.449, 1.001, 0.423, 0.249, 0.252, 0.126, 0.068, 0.067, 0.189,
0.309, 0.300, 0.086, 0.055, 0.121, 0.149, 0.020, 0.014, 0.055,
0.222, 0.070, 0.094, 0.196, 0.132, 0.085, 0.087, 0.159, 0.158,
0.150, 0.214, 0.254, 0.239, 0.099, 0.089, 0.140, 0.154
]
fc1_v_read = [0.0] + [x for x in fc1_repeat
for i in range(6)] + [0.079] * 5
[
self.assertLess(
fabs(fc1_v_read[i] - fc1_v[i]), 0.01,
"{}: {} != {}".format(i, fc1_v_read[i], fc1_v[i]))
for i in range(len(fc1_v_read))
]
def test_get_ensemble(self):
EPSG = 32633
upper_left_x = 436100.0
upper_left_y = 7417800.0
nx = 74
ny = 94
dx = 1000.0
dy = 1000.0
# Period start
year = 2015
month = 7
day = 26
hour = 0
n_hours = 30
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month, day, hour)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_hours))
t_c = utc.time(t0) + api.deltahours(1)
base_dir = path.join(shyftdata_dir, "netcdf", "arome")
pattern = "fc*.nc"
bbox = ([upper_left_x, upper_left_x + nx*dx,
upper_left_x + nx*dx, upper_left_x],
[upper_left_y, upper_left_y,
upper_left_y - ny*dy, upper_left_y - ny*dy])
try:
repos = AromeDataRepository(EPSG, base_dir, filename=pattern, bounding_box=bbox)
data_names = ("temperature", "wind_speed", "relative_humidity")
ensemble = repos.get_forecast_ensemble(data_names, period, t_c, None)
self.assertTrue(isinstance(ensemble, list))
self.assertEqual(len(ensemble), 10)
except AromeDataRepositoryError as adre:
self.skipTest("(test inconclusive- missing arome-data {0})".format(adre))
def test_get_forecast(self):
# Period start
year = 2015
month = 8
day = 24
hour = 6
n_hours = 65
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month, day, hour)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_hours))
t_c1 = utc.time(t0) + api.deltahours(1)
t_c2 = utc.time(t0) + api.deltahours(7)
base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository")
pattern = "arome_metcoop*default2_5km_*.nc"
EPSG, bbox = self.arome_epsg_bbox
repos = AromeDataRepository(EPSG, base_dir, filename=pattern, bounding_box=bbox)
data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity")
tc1_sources = repos.get_forecast(data_names, period, t_c1, None)
tc2_sources = repos.get_forecast(data_names, period, t_c2, None)
self.assertTrue(len(tc1_sources) == len(tc2_sources))
self.assertTrue(set(tc1_sources) == set(data_names))
self.assertTrue(tc1_sources["temperature"][0].ts.size() == n_hours + 1)
tc1_precip = tc1_sources["precipitation"][0].ts
tc2_precip = tc2_sources["precipitation"][0].ts
self.assertEqual(tc1_precip.size(), n_hours)
self.assertTrue(tc1_precip.time(0) != tc2_precip.time(0))
def test_subsets(self):
EPSG, bbox = self.arome_epsg_bbox
# Period start
year = 2015
month = 8
day = 24
hour = 6
n_hours = 30
date_str = "{}{:02}{:02}_{:02}".format(year, month, day, hour)
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month, day, hour)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_hours))
base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository")
filename = "arome_metcoop_red_default2_5km_{}.nc".format(date_str)
data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity", "radiation")
allow_subset = False
reader = AromeDataRepository(EPSG, base_dir, filename=filename,
bounding_box=bbox, allow_subset=allow_subset)
with self.assertRaises(AromeDataRepositoryError) as context:
reader.get_timeseries(data_names, period, None)
self.assertEqual("Could not find all data fields", context.exception.args[0])
allow_subset = True
reader = AromeDataRepository(EPSG, base_dir, filename=filename,
bounding_box=bbox, allow_subset=allow_subset)
try:
sources = reader.get_timeseries(data_names, period, None)
except AromeDataRepositoryError as e:
self.fail("AromeDataRepository.get_timeseries(data_names, period, None) "
"raised AromeDataRepositoryError unexpectedly.")
self.assertEqual(len(sources), len(data_names) - 1)
def test_tiny_bbox(self):
EPSG, _ = self.arome_epsg_bbox
x0 = 436250.0 # lower left
y0 = 6823250.0 # lower right
nx = 1
ny = 1
dx = 5.0
dy = 5.0
bbox = ([x0, x0 + nx*dx, x0 + nx*dx, x0], [y0, y0, y0 + ny*dy, y0 + ny*dy])
print(bbox)
# Period start
year = 2015
month = 8
day = 24
hour = 6
n_hours = 30
date_str = "{}{:02}{:02}_{:02}".format(year, month, day, hour)
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month, day, hour)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_hours))
base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository")
filename = "arome_metcoop_red_default2_5km_{}.nc".format(date_str)
reader = AromeDataRepository(EPSG, base_dir, filename=filename,
bounding_box=bbox, x_padding=0, y_padding=0)
data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity")
try:
tss = reader.get_timeseries(data_names, period, None)
except AromeDataRepository as err:
self.fail("reader.get_timeseries raised AromeDataRepositoryError('{}') "
"unexpectedly.".format(err.args[0]))
def test_get_ensemble(self):
"""
Simple ensemble regression test of OpenDAP data repository.
"""
epsg, bbox = self.epsg_bbox
dem_file = path.join(shyftdata_dir, "netcdf", "etopo180.nc")
# Period start
(year, month, day), hour = self.start_date, 9
n_hours = 30
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month, day, hour)
period = api.UtcPeriod(utc.time(t0),
utc.time(t0) + api.deltahours(n_hours))
t_c = utc.time(t0) + api.deltahours(7)
repos = GFSDataRepository(epsg, dem_file, bounding_box=bbox)
data_names = ("temperature", "wind_speed", "precipitation",
"relative_humidity", "radiation")
ensembles = repos.get_forecast_ensemble(data_names, period, t_c, None)
for sources in ensembles:
self.assertEqual(set(data_names), set(sources.keys()))
self.assertEqual(len(sources["temperature"]), 6)
data1 = sources["temperature"][0]
data2 = sources["temperature"][1]
self.assertNotEqual(data1.mid_point().x, data2.mid_point().x)
self.assertNotEqual(data1.mid_point().y, data2.mid_point().y)
self.assertNotEqual(data1.mid_point().z, data2.mid_point().z)
h_dt = (data1.ts.time(1) - data1.ts.time(0)) / 3600
self.assertEqual(data1.ts.size(), 30 // h_dt)
def _create_target_specvect(self):
print("Creating TargetSpecificationVector...")
tv = api.TargetSpecificationVector()
tst = api.TsTransform()
cid_map = self.region_model.catchment_id_map
for repo in self.target_repo:
tsp = repo['repository'].read([ts_info['uid'] for ts_info in repo['1D_timeseries']], self.time_axis.total_period())
for ts_info in repo['1D_timeseries']:
if np.count_nonzero(np.in1d(cid_map, ts_info['catch_id'])) != len(ts_info['catch_id']):
raise ConfigSimulatorError("Catchment ID {} for target series {} not found.".format(
','.join([str(val) for val in [i for i in ts_info['catch_id'] if i not in cid_map]]), ts_info['uid']))
period = api.UtcPeriod(ts_info['start_datetime'],
ts_info['start_datetime'] + ts_info['number_of_steps'] * ts_info['run_time_step'])
if not self.time_axis.total_period().contains(period):
raise ConfigSimulatorError(
"Period {} for target series {} is not within the calibration period {}.".format(
period.to_string(), ts_info['uid'], self.time_axis.total_period().to_string()))
#tsp = repo['repository'].read([ts_info['uid']], period)[ts_info['uid']]
t = api.TargetSpecificationPts()
t.uid = ts_info['uid']
t.catchment_indexes = api.IntVector(ts_info['catch_id'])
t.scale_factor = ts_info['weight']
t.calc_mode = self.obj_funcs[ts_info['obj_func']['name']]
[setattr(t, nm, ts_info['obj_func']['scaling_factors'][k]) for nm, k in zip(['s_r','s_a','s_b'], ['s_corr','s_var','s_bias'])]
t.ts = api.TimeSeries(tst.to_average(ts_info['start_datetime'], ts_info['run_time_step'], ts_info['number_of_steps'], tsp[ts_info['uid']]))
tv.append(t)
return tv
def test_no_point_inside_polygon_bounds(self):
EPSG, bbox, bpoly = self.arome_epsg_bbox
bounds = bpoly.bounds
bpoly = box(bounds[0], 6010000.0, bounds[2], 6035000.0)
# Period start
year = 2015
month = 8
day = 24
hour = 6
n_hours = 30
date_str = "{}{:02}{:02}_{:02}".format(year, month, day, hour)
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month, day, hour)
period = api.UtcPeriod(utc.time(t0),
utc.time(t0) + api.deltahours(n_hours))
base_dir = path.join(shyftdata_dir, "repository",
"arome_data_repository")
filename = "arome_metcoop_red_default2_5km_{}.nc".format(date_str)
reader = MetNetcdfDataRepository(EPSG,
base_dir,
filename=filename,
padding=0.0)
data_names = ("temperature", "wind_speed", "precipitation",
"relative_humidity")
with self.assertRaises(MetNetcdfDataRepositoryError) as context:
reader.get_timeseries(data_names,
period,
geo_location_criteria=bpoly)
self.assertEqual(
"No points in dataset which are within the bounding box of the geo_location_criteria polygon.",
context.exception.args[0])
def get_forecast(self,
input_source_types,
utc_period,
t_c,
geo_location_criteria=None):
"""Get shyft source vectors of time series for input_source_types
Parameters
----------
input_source_types: list
List of source types to retrieve (precipitation, temperature..)
geo_location_criteria: object, optional
Some type (to be decided), extent (bbox + coord.ref)
Returns
-------
geo_loc_ts: dictionary
dictionary keyed by time series name, where values are api vectors of geo
located timeseries.
"""
# filename = self._filename
# filename = self._get_files(t_c, "_(\d{8})([T_])(\d{2})(Z)?.nc$")
filename = self._get_files(t_c, "(\d{8})([T_])(\d{2})(.*.nc)")
with Dataset(filename) as dataset:
if utc_period is None:
time = dataset.variables.get("time", None)
conv_time = convert_netcdf_time(time.units, time)
start_t = conv_time[0]
last_t = conv_time[-1]
utc_period = api.UtcPeriod(int(start_t), int(last_t))
return self._get_data_from_dataset(dataset, input_source_types,
utc_period,
geo_location_criteria)
def test_get_dummy(self):
"""
#Simple regression test of WRF data repository.
"""
EPSG, bbox, bpoly = self.senorge_epsg_bbox
# Period start
n_days = 5
t0 = api.YMDhms(2015, 2)
date_str = "{}-{:02}".format(t0.year, t0.month)
utc = api.Calendar() # No offset gives Utc
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_days * 24))
base_dir = path.join(shyftdata_dir, "repository", "senorge_data_repository", "senorge2")
f1 = "seNorge2_PREC1d_grid_2015.nc"
senorge1 = SeNorgeDataRepository(EPSG, base_dir, filename=f1, allow_subset=True)
senorge1_data_names = ("radiation",)
sources = senorge1.get_timeseries(senorge1_data_names, period, geo_location_criteria=bpoly)
self.assertTrue(len(sources) > 0)
self.assertTrue(set(sources) == set(senorge1_data_names))
p0 = sources["radiation"][0].ts
self.assertTrue(p0.size() == n_days)
self.assertTrue(p0.time(0), period.start)
def _get_forecast(self):
"""
Simple forecast regression test of OpenDAP data repository.
"""
epsg, bbox, bpoly = self.epsg_bbox
dem_file = path.join(shyftdata_dir, "netcdf", "etopo180.nc")
n_hours = 30
t0 = self.start_date + api.deltahours(9)
period = api.UtcPeriod(t0, t0 + api.deltahours(n_hours))
t_c = self.start_date + api.deltahours(
7) # the beginning of the forecast criteria
repos = GFSDataRepository(epsg, dem_file)
data_names = (
"temperature",
) # the full set: "wind_speed", "precipitation", "relative_humidity", "radiation")
sources = repos.get_forecast(data_names,
period,
t_c,
geo_location_criteria=bpoly)
self.assertEqual(set(data_names), set(sources.keys()))
self.assertEqual(len(sources["temperature"]), 2)
data1 = sources["temperature"][0]
data2 = sources["temperature"][1]
self.assertNotEqual(data1.mid_point().x, data2.mid_point().x)
self.assertNotEqual(data1.mid_point().y, data2.mid_point().y)
self.assertNotEqual(data1.mid_point().z, data2.mid_point().z)
self.assertLessEqual(
data1.ts.time(0), period.start,
'expect returned fc ts to cover requested period')
self.assertGreaterEqual(
data1.ts.total_period().end, period.end,
'expect returned fc ts to cover requested period')
def test_tiny_bbox(self):
EPSG, _, _ = self.senorge_epsg_bbox
x0 = 374499.5 # lower left
y0 = 6451499.5 # lower right
nx = 1.0
ny = 1.0
dx = 1.0
dy = 1.0
bbox = ([x0, x0 + nx * dx, x0 + nx * dx, x0], [y0, y0, y0 + ny * dy, y0 + ny * dy])
bpoly = box(min(bbox[0]), min(bbox[1]), max(bbox[0]), max(bbox[1]))
# Period start
year = 2015
month = 1
n_hours = 30
date_str = "{}-{:02}".format(year, month)
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_hours))
base_dir = path.join(shyftdata_dir, "repository", "senorge_data_repository", "senorge2")
filename = "seNorge2_PREC1d_grid_2015.nc"
reader = SeNorgeDataRepository(EPSG, base_dir, filename=filename,
padding=0)
data_names = ("precipitation",)
tss = reader.get_timeseries(data_names, period, geo_location_criteria=bpoly)
for name, ts in tss.items():
self.assertTrue(len(ts) == 1)
def test_subsets(self):
EPSG, bbox, bpoly = self.senorge_epsg_bbox
# Period start
year = 2015
month = 1
n_hours = 30
date_str = "{}-{:02}".format(year, month)
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_hours))
base_dir = path.join(shyftdata_dir, "repository", "senorge_data_repository", "senorge2")
filename = "seNorge2_PREC1d_grid_2015.nc"
data_names = ("precipitation","foo")
allow_subset = False
reader = SeNorgeDataRepository(EPSG, base_dir, filename=filename,
allow_subset=allow_subset)
with self.assertRaises(SeNorgeDataRepositoryError) as context:
reader.get_timeseries(data_names, period, geo_location_criteria=bpoly)
self.assertEqual("Could not find all data fields", context.exception.args[0])
allow_subset = True
reader = SeNorgeDataRepository(EPSG, base_dir, filename=filename,
allow_subset=allow_subset)
try:
sources = reader.get_timeseries(data_names, period, geo_location_criteria=bpoly)
except SeNorgeDataRepositoryError as e:
self.fail("AromeDataRepository.get_timeseries(data_names, period, None) "
"raised AromeDataRepositoryError unexpectedly.")
self.assertEqual(len(sources), len(data_names)-1)
def test_store(self):
ds = SmGTsRepository(PREPROD)
nl = [u'/shyft/test/a', u'/shyft/test/b',
u'/shyft/test/c'] #[u'/ICC-test-v9.2']
t0 = 946684800 # time_t/unixtime 2000.01.01 00:00:00
dt = 3600 #one hour in seconds
values = np.array([1.0, 2.0, 3.0])
shyft_ts_factory = api.TsFactory()
shyft_result_ts = shyft_ts_factory.create_point_ts(
len(values), t0, dt, api.DoubleVector(values))
shyft_catchment_result = dict()
shyft_catchment_result[nl[0]] = shyft_result_ts
shyft_catchment_result[nl[1]] = shyft_result_ts
shyft_catchment_result[nl[2]] = shyft_result_ts
r = ds.store(shyft_catchment_result)
self.assertEqual(r, True)
# now read back the ts.. and verify it's there..
read_period = api.UtcPeriod(t0, t0 + 3 * dt)
rts_list = ds.read(nl, read_period)
self.assertIsNotNone(rts_list)
c2 = rts_list[nl[-1]]
[
self.assertAlmostEqual(c2.value(i), values[i])
for i in range(len(values))
]
def get_forecast_ensemble(self, input_source_types, utc_period, t_c, geo_location_criteria=None):
"""
Parameters
----------
input_source_types: list
List of source types to retrieve (precipitation, temperature, ...)
utc_period: api.UtcPeriod
The utc time period that should (as a minimum) be covered.
t_c: long
Forecast specification; return newest forecast older than t_c.
geo_location_criteria: object
Some type (to be decided), extent (bbox + coord.ref).
Returns
-------
ensemble: list of geo_loc_ts dictionaries
Dictionaries are keyed by time series type, with values
being api vectors of geo located timeseries.
"""
#filename = self._filename
filename = self._get_files(t_c, "_(\d{8})([T_])(\d{2})(Z)?.nc$")
with Dataset(filename) as dataset:
if utc_period is None:
time = dataset.variables.get("time", None)
conv_time = convert_netcdf_time(time.units, time)
start_t = conv_time[0]
last_t = conv_time[-1]
utc_period = api.UtcPeriod(int(start_t), int(last_t))
# for idx in range(dataset.dimensions["ensemble_member"].size):
res = self._get_data_from_dataset(dataset, input_source_types, utc_period,
geo_location_criteria,
ensemble_member='all')
return res
def test_get_ensemble(self):
EPSG = 32633
upper_left_x = 436100.0
upper_left_y = 7417800.0
nx = 74
ny = 94
dx = 1000.0
dy = 1000.0
# Period start
n_hours = 30
utc = api.Calendar() # No offset gives Utc
t0 = utc.time(2015, 7, 26)
period = api.UtcPeriod(t0, t0 + api.deltahours(n_hours))
t_c = t0 + api.deltahours(1)
base_dir = path.join(shyftdata_dir, "netcdf", "arome")
# pattern = "fc*.nc"
pattern = "fc_(\d{4})(\d{2})(\d{2})[T_](\d{2})Z?.nc$"
bpoly = box(upper_left_x, upper_left_y - ny * dy,
upper_left_x + nx * dx, upper_left_y)
try:
repos = MetNetcdfDataRepository(EPSG, base_dir, filename=pattern)
data_names = ("temperature", "wind_speed", "relative_humidity")
ensemble = repos.get_forecast_ensemble(data_names,
period,
t_c,
geo_location_criteria=bpoly)
self.assertTrue(isinstance(ensemble, list))
self.assertEqual(len(ensemble), 10)
except MetNetcdfDataRepositoryError as adre:
self.skipTest(
"(test inconclusive- missing arome-data {0})".format(adre))
def _get_timeseries(self):
"""
Simple regression test of OpenDAP data repository.
"""
epsg, bbox, bpoly = self.epsg_bbox
dem_file = path.join(shyftdata_dir, "netcdf", "etopo180.nc")
n_hours = 30
t0 = self.start_date + api.deltahours(7)
period = api.UtcPeriod(t0, t0 + api.deltahours(n_hours))
repos = GFSDataRepository(
epsg=epsg, dem_file=dem_file, padding=5000.0,
utc=t0) #//epsg, dem_file, padding=5000., utc=None
data_names = ("temperature", "wind_speed", "precipitation",
"relative_humidity", "radiation")
sources = repos.get_timeseries(data_names,
period,
geo_location_criteria=bpoly)
self.assertEqual(set(data_names), set(sources.keys()))
self.assertEqual(len(sources["temperature"]),
2) # TODO: this was 6 before common changes.
data1 = sources["temperature"][0]
data2 = sources["temperature"][1]
self.assertNotEqual(data1.mid_point().x, data2.mid_point().x)
self.assertNotEqual(data1.mid_point().y, data2.mid_point().y)
self.assertNotEqual(data1.mid_point().z, data2.mid_point().z)
self.assertLessEqual(
data1.ts.time(0), period.start,
'expect returned fc ts to cover requested period')
self.assertGreaterEqual(
data1.ts.total_period().end, period.end,
'expect returned fc ts to cover requested period')
def test_rel_hum_only(self):
print("rel hum test: ")
# relative humidity needs temperature and pressure to be calculated
EPSG, bbox = self.wrf_epsg_bbox
# Period start
year = 1999
month = 10
n_hours = 30
date_str = "{}-{:02}".format(year, month)
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month)
period = api.UtcPeriod(utc.time(t0),
utc.time(t0) + api.deltahours(n_hours))
base_dir = path.join(shyftdata_dir, "repository",
"wrf_data_repository")
filename = "wrfout_d03_{}".format(date_str)
data_names = ["relative_humidity"]
reader = WRFDataRepository(EPSG,
base_dir,
filename=filename,
bounding_box=bbox)
sources = reader.get_timeseries(data_names, period, None)
self.assertTrue(list(sources.keys()) == ["relative_humidity"])
def test_non_overlapping_bbox(self):
EPSG, bbox = self.wrf_epsg_bbox
bbox = list(bbox)
bbox[0] = [-100000.0, -90000.0, -90000.0, -100000]
# Period start
year = 1999
month = 10
n_hours = 30
date_str = "{}-{:02}".format(year, month)
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month)
period = api.UtcPeriod(utc.time(t0),
utc.time(t0) + api.deltahours(n_hours))
base_dir = path.join(shyftdata_dir, "repository",
"wrf_data_repository")
filename = "wrfout_d03_{}".format(date_str)
reader = WRFDataRepository(EPSG,
base_dir,
filename=filename,
bounding_box=bbox)
data_names = ("temperature", "wind_speed", "precipitation",
"relative_humidity")
with self.assertRaises(WRFDataRepositoryError) as context:
reader.get_timeseries(data_names, period, None)
self.assertEqual("Bounding box doesn't intersect with dataset.",
context.exception.args[0])
def test_get_ensemble(self):
"""
Simple ensemble regression test of OpenDAP data repository.
"""
epsg, bbox = self.epsg_bbox
dem_file = path.join(shyftdata_dir, "netcdf", "etopo180.nc")
n_hours = 30
t0 = self.start_date + api.deltahours(
9) # api.YMDhms(year, month, day, hour)
period = api.UtcPeriod(t0, t0 + api.deltahours(n_hours))
t_c = t0
repos = GFSDataRepository(epsg, dem_file, bounding_box=bbox)
data_names = (
"temperature",
) # this is the full set: "wind_speed", "precipitation", "relative_humidity", "radiation")
ensembles = repos.get_forecast_ensemble(data_names, period, t_c, None)
for sources in ensembles:
self.assertEqual(set(data_names), set(sources.keys()))
self.assertEqual(len(sources["temperature"]), 6)
data1 = sources["temperature"][0]
data2 = sources["temperature"][1]
self.assertNotEqual(data1.mid_point().x, data2.mid_point().x)
self.assertNotEqual(data1.mid_point().y, data2.mid_point().y)
self.assertNotEqual(data1.mid_point().z, data2.mid_point().z)
self.assertLessEqual(
data1.ts.time(0), period.start,
'expect returned fc ts to cover requested period')
self.assertGreaterEqual(
data1.ts.total_period().end, period.end,
'expect returned fc ts to cover requested period')
def test_wrong_file(self):
with self.assertRaises(SeNorgeDataRepositoryError) as context:
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(2015, 12, 25, 18)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(30))
ar1 = SeNorgeDataRepository(32632, shyftdata_dir, filename="plain_wrong.nc")
ar1.get_timeseries(("temperature",), period, geo_location_criteria=None)
self.assertTrue(all(x in context.exception.args[0] for x in ["File", "not found"]))
def test_UtcPeriod_str(self):
c1 = api.YMDhms(2000, 1, 2, 3, 4, 5)
t = self.utc.time(c1)
p = api.UtcPeriod(t, t + api.deltahours(1))
s = str(p)
self.assertEqual(s, "[2000-01-02T03:04:05Z,2000-01-02T04:04:05Z>")
s = repr(p)
self.assertEqual(s, "[2000-01-02T03:04:05Z,2000-01-02T04:04:05Z>")
def test_UtcPeriod_to_string(self):
c1 = api.YMDhms(2000, 1, 2, 3, 4, 5)
t = self.utc.time(c1)
p = api.UtcPeriod(t, t + api.deltahours(1))
s = p.to_string()
self.assertEqual(s, "[2000-01-02T03:04:05Z,2000-01-02T04:04:05Z>")
s2 = self.std.to_string(p)
self.assertEqual(s2, "[2000-01-02T04:04:05+01,2000-01-02T05:04:05+01>")
def test_wrong_forecast(self):
with self.assertRaises(AromeDataRepositoryError) as context:
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(2015, 12, 25, 18)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(30))
ar1 = AromeDataRepository(32632, shyftdata_dir, filename="plain_wrong_*.nc")
ar1.get_forecast(("temperature",), period, utc.time(t0), None)
self.assertTrue(all(x in context.exception.args[0] for x in
["No matches found for file_pattern = ", "and t_c = "]))
def test_get_dummy(self):
"""
#Simple regression test of dummy variables from `utils.dummy_var`
"""
EPSG, bbox, bpoly = self.senorge_epsg_bbox
# Period start
n_days = 5
t0 = api.YMDhms(1957, 1)
utc = api.Calendar() # No offset gives Utc
period = api.UtcPeriod(utc.time(t0),
utc.time(t0) + api.deltahours(n_days * 24))
base_dir = path.join(shyftdata_dir, "repository",
"senorge_data_repository")
f1 = "senorge_test.nc"
senorge1 = SeNorgeDataRepository(EPSG,
base_dir,
filename=f1,
allow_subset=True)
senorge1_data_names = ("radiation", "temperature", "wind_speed")
sources = senorge1.get_timeseries(senorge1_data_names,
period,
geo_location_criteria=bpoly)
self.assertTrue(set(sources) == set(senorge1_data_names))
r0 = sources["radiation"][0].ts
p0 = sources["temperature"][0].ts
self.assertTrue(p0.total_period().start <= period.start
and p0.total_period().end >= period.end)
self.assertTrue(r0.time(0), period.start)
t0 = api.YMDhms(1957, 1)
period = api.UtcPeriod(utc.time(t0),
utc.time(t0) + api.deltahours(n_days * 23))
sources = senorge1.get_timeseries(senorge1_data_names,
period,
geo_location_criteria=bpoly)
r0 = sources["radiation"][0].ts
p0 = sources["temperature"][0].ts
self.assertTrue(p0.total_period().start <= period.start
and p0.total_period().end >= period.end)
self.assertTrue(r0.time(0) == period.start)
self.assertTrue(r0.time_axis.time(1) - r0.time_axis.time(0) == 86400)
def _make_shyft_ts_from_xts(xts):
if not isinstance(xts, ITimeSeries):
raise SmgDataError("Supplied xts should be of type ITimeSeries")
t = api.UtcTimeVector()
v = api.DoubleVector()
for i in range(xts.Count):
t.push_back(xts.Time(i).ToUnixTime())
v.push_back(xts.Value(i).V)
shyft_ts = api.TsFactory().create_time_point_ts(api.UtcPeriod(t[0], t[-1]), t, v)
return shyft_ts
def test_create_TargetSpecificationPts(self):
t = api.TargetSpecificationPts();
t.scale_factor = 1.0
t.calc_mode = api.NASH_SUTCLIFFE
t.calc_mode = api.KLING_GUPTA;
t.s_r = 1.0 # KGEs scale-factors
t.s_a = 2.0
t.s_b = 3.0
self.assertAlmostEqual(t.scale_factor, 1.0)
# create a ts with some points
cal = api.Calendar();
start = cal.time(api.YMDhms(2015, 1, 1, 0, 0, 0))
dt = api.deltahours(1)
tsf = api.TsFactory();
times = api.UtcTimeVector()
times.push_back(start + 1 * dt);
times.push_back(start + 3 * dt);
times.push_back(start + 4 * dt)
values = api.DoubleVector()
values.push_back(1.0)
values.push_back(3.0)
values.push_back(np.nan)
tsp = tsf.create_time_point_ts(api.UtcPeriod(start, start + 24 * dt), times, values)
# convert it from a time-point ts( as returned from current smgrepository) to a fixed interval with timeaxis, needed by calibration
tst = api.TsTransform()
tsa = tst.to_average(start, dt, 24, tsp)
# tsa2 = tst.to_average(start,dt,24,tsp,False)
# tsa_staircase = tst.to_average_staircase(start,dt,24,tsp,False) # nans infects the complete interval to nan
# tsa_staircase2 = tst.to_average_staircase(start,dt,24,tsp,True) # skip nans, nans are 0
# stuff it into the target spec.
# also show how to specify snow-calibration
cids = api.IntVector([0, 2, 3])
t2 = api.TargetSpecificationPts(tsa,cids, 0.7, api.KLING_GUPTA, 1.0, 1.0, 1.0, api.SNOW_COVERED_AREA)
t2.catchment_property = api.SNOW_WATER_EQUIVALENT
self.assertEqual(t2.catchment_property, api.SNOW_WATER_EQUIVALENT)
self.assertIsNotNone(t2.catchment_indexes)
for i in range(len(cids)):
self.assertEqual(cids[i],t2.catchment_indexes[i])
t.ts = tsa
#TODO: Does not work, list of objects are not yet convertible tv = api.TargetSpecificationVector([t, t2])
tv=api.TargetSpecificationVector()
tv.append(t)
tv.append(t2)
# now verify we got something ok
self.assertEqual(2, tv.size())
self.assertAlmostEqual(tv[0].ts.value(1), 1.5) # average value 0..1 ->0.5
self.assertAlmostEqual(tv[0].ts.value(2), 2.5) # average value 0..1 ->0.5
self.assertAlmostEqual(tv[0].ts.value(3), 3.0) # average value 0..1 ->0.5
# and that the target vector now have its own copy of ts
tsa.set(1, 3.0)
self.assertAlmostEqual(tv[0].ts.value(1), 1.5) # make sure the ts passed onto target spec, is a copy
self.assertAlmostEqual(tsa.value(1), 3.0) # and that we really did change the source
def test_get_timeseries(self):
"""
Simple regression test of arome data respository.
"""
EPSG, bbox, bpoly = self.arome_epsg_bbox
# Period start
n_hours = 30
t0 = api.YMDhms(2015, 8, 24, 0)
date_str = "{}{:02}{:02}_{:02}".format(t0.year, t0.month, t0.day,
t0.hour)
utc = api.Calendar() # No offset gives Utc
period = api.UtcPeriod(utc.time(t0),
utc.time(t0) + api.deltahours(n_hours))
base_dir = path.join(shyftdata_dir, "repository",
"arome_data_repository")
f1 = "arome_metcoop_red_default2_5km_{}_diff_time_unit.nc".format(
date_str)
f2 = "arome_metcoop_red_test2_5km_{}.nc".format(date_str)
ar1 = MetNetcdfDataRepository(EPSG, base_dir, filename=f1)
ar2 = MetNetcdfDataRepository(EPSG,
base_dir,
filename=f2,
elevation_file=f1)
ar1_data_names = ("temperature", "wind_speed", "precipitation",
"relative_humidity")
ar2_data_names = ("radiation", )
sources = ar1.get_timeseries(ar1_data_names,
period,
geo_location_criteria=bpoly)
self.assertTrue(len(sources) > 0)
sources2 = ar2.get_timeseries(ar2_data_names,
period,
geo_location_criteria=bpoly)
self.assertTrue(set(sources) == set(ar1_data_names))
self.assertTrue(set(sources2) == set(ar2_data_names))
self.assertTrue(sources["temperature"][0].ts.size() == n_hours + 1)
r0 = sources2["radiation"][0].ts
p0 = sources["precipitation"][0].ts
temp0 = sources["temperature"][0].ts
self.assertTrue(r0.size() == n_hours)
self.assertTrue(p0.size() == n_hours)
self.assertTrue(r0.time(0) == temp0.time(0))
self.assertTrue(p0.time(0) == temp0.time(0))
self.assertTrue(
r0.time_axis.total_period().end == temp0.time(temp0.size() - 1))
self.assertTrue(
p0.time_axis.total_period().end == temp0.time(temp0.size() - 1))
self.assertTrue(p0.time(0), period.start)
def test_get_forecast(self):
# Period start
n_hours = 65
utc = api.Calendar() # No offset gives Utc
t0 = utc.time(2015, 8, 24, 6)
period1 = api.UtcPeriod(t0, t0 + api.deltahours(n_hours))
period2 = api.UtcPeriod(
t0 + api.deltahours(6),
t0 + api.deltahours(6) + api.deltahours(n_hours))
t_c1 = t0 + api.deltahours(1)
t_c2 = t0 + api.deltahours(7)
base_dir = path.join(shyftdata_dir, "repository",
"arome_data_repository")
# pattern = "arome_metcoop*default2_5km_*.nc"
pattern = "arome_metcoop_red_default2_5km_(\d{4})(\d{2})(\d{2})[T_](\d{2})Z?.nc$"
EPSG, bbox, bpoly = self.arome_epsg_bbox
repos = MetNetcdfDataRepository(EPSG, base_dir, filename=pattern)
data_names = ("temperature", "wind_speed", "precipitation",
"relative_humidity")
tc1_sources = repos.get_forecast(data_names,
period1,
t_c1,
geo_location_criteria=bpoly)
tc2_sources = repos.get_forecast(data_names,
period2,
t_c2,
geo_location_criteria=bpoly)
self.assertTrue(len(tc1_sources) == len(tc2_sources))
self.assertTrue(set(tc1_sources) == set(data_names))
self.assertTrue(tc1_sources["temperature"][0].ts.size() == n_hours + 1)
tc1_precip = tc1_sources["precipitation"][0].ts
tc2_precip = tc2_sources["precipitation"][0].ts
self.assertEqual(tc1_precip.size(), n_hours)
self.assertTrue(tc1_precip.time(0) != tc2_precip.time(0))
