def arome_repository(self):
"""
This shows how we
join together two arome repositories to a repository that gives us all the variabes that we need.
arome_4: all except radiation
arome_rad: ratiation
Return
------
Arome repository (inside a GeoTsRepositoryCollection), with all datafiles that matches
the filepattrns we currently use for arome downloads.
"""
base_dir = path.join(self.statkraft_data_dir, "repository",
"arome_data_repository")
epsg = self.grid_spec.epsg()
bbox = self.grid_spec.bounding_box(epsg)
f1 = "arome_metcoop_default2_5km_20151001_00.nc"
f2 = "arome_metcoop_test2_5km_20151001_00.nc"
arome_4 = AromeDataRepository(epsg,
base_dir,
filename=f1,
bounding_box=bbox,
allow_subset=True)
arome_rad = AromeDataRepository(epsg,
base_dir,
filename=f2,
elevation_file=f1,
bounding_box=bbox,
allow_subset=True)
return GeoTsRepositoryCollection([arome_4, arome_rad])
def test_get_ensemble(self):
EPSG = 32633
upper_left_x = 436100.0
upper_left_y = 7417800.0
nx = 74
ny = 94
dx = 1000.0
dy = 1000.0
# Period start
year = 2015
month = 7
day = 26
hour = 0
n_hours = 30
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month, day, hour)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_hours))
t_c = utc.time(t0) + api.deltahours(1)
base_dir = path.join(shyftdata_dir, "netcdf", "arome")
pattern = "fc*.nc"
bbox = ([upper_left_x, upper_left_x + nx*dx,
upper_left_x + nx*dx, upper_left_x],
[upper_left_y, upper_left_y,
upper_left_y - ny*dy, upper_left_y - ny*dy])
try:
repos = AromeDataRepository(EPSG, base_dir, filename=pattern, bounding_box=bbox)
data_names = ("temperature", "wind_speed", "relative_humidity")
ensemble = repos.get_forecast_ensemble(data_names, period, t_c, None)
self.assertTrue(isinstance(ensemble, list))
self.assertEqual(len(ensemble), 10)
except AromeDataRepositoryError as adre:
self.skipTest("(test inconclusive- missing arome-data {0})".format(adre))
def test_tiny_bbox(self):
EPSG, _ = self.arome_epsg_bbox
x0 = 436250.0 # lower left
y0 = 6823250.0 # lower right
nx = 1
ny = 1
dx = 5.0
dy = 5.0
bbox = ([x0, x0 + nx*dx, x0 + nx*dx, x0], [y0, y0, y0 + ny*dy, y0 + ny*dy])
print(bbox)
# Period start
year = 2015
month = 8
day = 24
hour = 6
n_hours = 30
date_str = "{}{:02}{:02}_{:02}".format(year, month, day, hour)
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month, day, hour)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_hours))
base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository")
filename = "arome_metcoop_red_default2_5km_{}.nc".format(date_str)
reader = AromeDataRepository(EPSG, base_dir, filename=filename,
bounding_box=bbox, x_padding=0, y_padding=0)
data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity")
try:
tss = reader.get_timeseries(data_names, period, None)
except AromeDataRepository as err:
self.fail("reader.get_timeseries raised AromeDataRepositoryError('{}') "
"unexpectedly.".format(err.args[0]))
def test_get_forecast(self):
# Period start
year = 2015
month = 8
day = 24
hour = 6
n_hours = 65
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month, day, hour)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_hours))
t_c1 = utc.time(t0) + api.deltahours(1)
t_c2 = utc.time(t0) + api.deltahours(7)
base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository")
pattern = "arome_metcoop*default2_5km_*.nc"
EPSG, bbox = self.arome_epsg_bbox
repos = AromeDataRepository(EPSG, base_dir, filename=pattern, bounding_box=bbox)
data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity")
tc1_sources = repos.get_forecast(data_names, period, t_c1, None)
tc2_sources = repos.get_forecast(data_names, period, t_c2, None)
self.assertTrue(len(tc1_sources) == len(tc2_sources))
self.assertTrue(set(tc1_sources) == set(data_names))
self.assertTrue(tc1_sources["temperature"][0].ts.size() == n_hours + 1)
tc1_precip = tc1_sources["precipitation"][0].ts
tc2_precip = tc2_sources["precipitation"][0].ts
self.assertEqual(tc1_precip.size(), n_hours)
self.assertTrue(tc1_precip.time(0) != tc2_precip.time(0))
def test_wrong_file(self):
with self.assertRaises(AromeDataRepositoryError) as context:
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(2015, 12, 25, 18)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(30))
ar1 = AromeDataRepository(32632, shyftdata_dir, filename="plain_wrong.nc")
ar1.get_timeseries(("temperature",), period, None)
self.assertTrue(all(x in context.exception.args[0] for x in ["File", "not found"]))
def test_subsets(self):
EPSG, bbox = self.arome_epsg_bbox
# Period start
year = 2015
month = 8
day = 24
hour = 6
n_hours = 30
date_str = "{}{:02}{:02}_{:02}".format(year, month, day, hour)
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month, day, hour)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_hours))
base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository")
filename = "arome_metcoop_red_default2_5km_{}.nc".format(date_str)
data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity", "radiation")
allow_subset = False
reader = AromeDataRepository(EPSG, base_dir, filename=filename,
bounding_box=bbox, allow_subset=allow_subset)
with self.assertRaises(AromeDataRepositoryError) as context:
reader.get_timeseries(data_names, period, None)
self.assertEqual("Could not find all data fields", context.exception.args[0])
allow_subset = True
reader = AromeDataRepository(EPSG, base_dir, filename=filename,
bounding_box=bbox, allow_subset=allow_subset)
try:
sources = reader.get_timeseries(data_names, period, None)
except AromeDataRepositoryError as e:
self.fail("AromeDataRepository.get_timeseries(data_names, period, None) "
"raised AromeDataRepositoryError unexpectedly.")
self.assertEqual(len(sources), len(data_names) - 1)
def test_missing_bbox(self):
EPSG, _ = self.arome_epsg_bbox
# Period start
year = 2015
month = 8
day = 24
hour = 6
n_hours = 30
date_str = "{}{:02}{:02}_{:02}".format(year, month, day, hour)
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month, day, hour)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_hours))
base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository")
filename = "arome_metcoop_red_default2_5km_{}.nc".format(date_str)
reader = AromeDataRepository(EPSG, base_dir, filename=filename)
data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity")
with self.assertRaises(AromeDataRepositoryError) as context:
reader.get_timeseries(data_names, period, None)
self.assertEqual("A bounding box must be provided.", context.exception.args[0])
def plot_region_model(self, catchment_type, identifier, x0, y0, dx, dy, nx, ny,
catch_indicies, station_ids, epsg_id):
grid_spec = GridSpecification(epsg_id, x0, y0, dx, dy, nx, ny)
cf = CellDataFetcher(catchment_type, identifier, grid_spec, id_list=catch_indicies)
print("Start fetching data")
cf.fetch()
print("Done, now preparing plot")
# Plot the extracted data
fig, ax = plt.subplots(1)
color_map = {"forest": 'g', "lake": 'b', "glacier": 'r', "cell": "0.75", "reservoir": "purple"}
extent = grid_spec.geometry[0], grid_spec.geometry[2], grid_spec.geometry[1], grid_spec.geometry[3]
ax.imshow(cf.elevation_raster, origin='upper', extent=extent, cmap=cm.gray)
for catchment_cells in iter(cf.cell_data.values()):
self.add_plot_polygons(ax, [cell["cell"] for cell in catchment_cells], color=color_map["cell"])
for catchment_land_type in iter(cf.catchment_land_types.values()):
for k, v in iter(catchment_land_type.items()):
self.add_plot_polygons(ax, v, color=color_map[k])
geometry = grid_spec.geometry
if station_ids is not None:
glr = GisLocationService()
stations = glr.get_locations(station_ids, epsg_id)
if stations:
points = stations.values()
ax.scatter([pt[0] for pt in points], [pt[1] for pt in points], alpha=0.5)
station_min_x = min([v[0] for v in points]) - 1000
station_max_x = max([v[0] for v in points]) + 1000
station_min_y = min([v[1] for v in points]) - 1000
station_max_y = max([v[1] for v in points]) + 1000
geometry[0] = min(station_min_x, geometry[0])
geometry[1] = min(station_min_y, geometry[1])
geometry[2] = max(station_max_x, geometry[2])
geometry[3] = max(station_max_y, geometry[3])
base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository")
EPSG = grid_spec.epsg()
bbox = grid_spec.bounding_box(EPSG)
arome4 = AromeDataRepository(EPSG, base_dir, filename="arome_metcoop_default2_5km_*.nc",
bounding_box=bbox, allow_subset=True)
# data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity","radiation")
# utc_period = api.UtcPeriod(
# api.Calendar().time(api.YMDhms(2015, 10, 1, 0, 0, 0)),
# api.Calendar().time(api.YMDhms(2015, 10, 2, 0, 0, 0))
# )
# arome_ts=arome4.get_timeseries(["temperature"],utc_period)
# arome_points=[gts.mid_point() for gts in arome_ts['temperature']]
# ax.scatter( [pt.x for pt in arome_points ],[pt.y for pt in arome_points],c=[pt.z for pt in arome_points],alpha=0.5,cmap='gray',s=100)#, facecolors=('r'))
ax.set_xlim(geometry[0], geometry[2])
ax.set_ylim(geometry[1], geometry[3])
plt.show()
def test_non_overlapping_y_bbox(self):
EPSG, bbox = self.arome_epsg_bbox
bbox = list(bbox)
bbox[1] = [6010000.0, 6010000.0, 6035000.0, 6035000]
# Period start
year = 2015
month = 8
day = 24
hour = 6
n_hours = 30
date_str = "{}{:02}{:02}_{:02}".format(year, month, day, hour)
utc = api.Calendar() # No offset gives Utc
t0 = api.YMDhms(year, month, day, hour)
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_hours))
base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository")
filename = "arome_metcoop_red_default2_5km_{}.nc".format(date_str)
reader = AromeDataRepository(EPSG, base_dir, filename=filename, bounding_box=bbox)
data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity")
with self.assertRaises(AromeDataRepositoryError) as context:
reader.get_timeseries(data_names, period, None)
self.assertEqual("Bounding box latitudes don't intersect with dataset.",
context.exception.args[0])
def test_get_timeseries(self):
"""
Simple regression test of arome data respository.
"""
EPSG, bbox = self.arome_epsg_bbox
# Period start
n_hours = 30
t0 = api.YMDhms(2015, 8, 24, 0)
date_str = "{}{:02}{:02}_{:02}".format(t0.year,t0.month, t0.day, t0.hour)
utc = api.Calendar() # No offset gives Utc
period = api.UtcPeriod(utc.time(t0), utc.time(t0) + api.deltahours(n_hours))
base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository")
f1 = "arome_metcoop_red_default2_5km_{}_diff_time_unit.nc".format(date_str)
f2 = "arome_metcoop_red_test2_5km_{}.nc".format(date_str)
ar1 = AromeDataRepository(EPSG, base_dir, filename=f1, bounding_box=bbox)
ar2 = AromeDataRepository(EPSG, base_dir, filename=f2, elevation_file=f1)
ar1_data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity")
ar2_data_names = ("radiation",)
sources = ar1.get_timeseries(ar1_data_names, period, None)
self.assertTrue(len(sources) > 0)
sources2 = ar2.get_timeseries(ar2_data_names, period, geo_location_criteria=bbox)
self.assertTrue(set(sources) == set(ar1_data_names))
self.assertTrue(set(sources2) == set(ar2_data_names))
self.assertTrue(sources["temperature"][0].ts.size() == n_hours + 1)
r0 = sources2["radiation"][0].ts
p0 = sources["precipitation"][0].ts
temp0 = sources["temperature"][0].ts
self.assertTrue(r0.size() == n_hours + 1)
self.assertTrue(p0.size() == n_hours + 1)
self.assertTrue(r0.time(0) == temp0.time(0))
self.assertTrue(p0.time(0) == temp0.time(0))
self.assertTrue(r0.time(r0.size() - 1) == temp0.time(temp0.size() - 1))
self.assertTrue(p0.time(r0.size() - 1) == temp0.time(temp0.size() - 1))
self.assertTrue(p0.time(0),period.start)
def test_wrong_elevation_file(self):
with self.assertRaises(AromeDataRepositoryError) as context:
AromeDataRepository(32632, shyftdata_dir, filename="", elevation_file="plain_wrong.nc")
self.assertTrue(all(x in context.exception.args[0] for x in ["Elevation file",
"not found"]))
def test_wrong_directory(self):
with self.assertRaises(AromeDataRepositoryError) as context:
AromeDataRepository(32632, "Foobar", filename="")
self.assertEqual("No such directory 'Foobar'", context.exception.args[0])