def test_gis_location_service(self):
glr=GisLocationService()
nea_nid=[402, 460]
# we test that for the same point, different projections, we get same heights and approx. different positions
utm32_loc=glr.get_locations(nea_nid, 32632)
utm33_loc=glr.get_locations(nea_nid, 32633)
self.assertIsNotNone(utm32_loc)
self.assertIsNotNone(utm33_loc)
for p in nea_nid:
self.assertAlmostEqual(utm32_loc[p][2],utm33_loc[p][2])
self.assertLess(fabs(utm32_loc[p][1]-utm33_loc[p][1]),10*1000,"expect y same")
self.assertGreater(fabs(utm32_loc[p][0]-utm33_loc[p][0]),30*1000,"expect x diff same")
def test_gis_location_service(self):
glr=GisLocationService()
nea_nid=[598, 574]
# we test that for the same point, different projections, we get same heights and approx. different positions
utm32_loc=glr.get_locations(nea_nid, 32632)
utm33_loc=glr.get_locations(nea_nid, 32633)
self.assertIsNotNone(utm32_loc)
self.assertIsNotNone(utm33_loc)
for p in nea_nid:
self.assertAlmostEqual(utm32_loc[p][2],utm33_loc[p][2])
self.assertGreater(fabs(utm32_loc[p][1]-utm33_loc[p][1]),10*1000,"expect y same")
self.assertGreater(fabs(utm32_loc[p][0]-utm33_loc[p][0]),30*1000,"expect x diff same")
def plot_region_model(self, catchment_type, identifier, x0, y0, dx, dy, nx, ny,
catch_indicies, station_ids, epsg_id):
grid_spec = GridSpecification(epsg_id, x0, y0, dx, dy, nx, ny)
cf = CellDataFetcher(catchment_type, identifier, grid_spec, id_list=catch_indicies)
print("Start fetching data")
cf.fetch()
print("Done, now preparing plot")
# Plot the extracted data
fig, ax = plt.subplots(1)
color_map = {"forest": 'g', "lake": 'b', "glacier": 'r', "cell": "0.75", "reservoir": "purple"}
extent = grid_spec.geometry[0], grid_spec.geometry[2], grid_spec.geometry[1], grid_spec.geometry[3]
ax.imshow(cf.elevation_raster, origin='upper', extent=extent, cmap=cm.gray)
for catchment_cells in iter(cf.cell_data.values()):
self.add_plot_polygons(ax, [cell["cell"] for cell in catchment_cells], color=color_map["cell"])
for catchment_land_type in iter(cf.catchment_land_types.values()):
for k, v in iter(catchment_land_type.items()):
self.add_plot_polygons(ax, v, color=color_map[k])
geometry = grid_spec.geometry
if station_ids is not None:
glr = GisLocationService()
stations = glr.get_locations(station_ids, epsg_id)
if stations:
points = stations.values()
ax.scatter([pt[0] for pt in points], [pt[1] for pt in points], alpha=0.5)
station_min_x = min([v[0] for v in points]) - 1000
station_max_x = max([v[0] for v in points]) + 1000
station_min_y = min([v[1] for v in points]) - 1000
station_max_y = max([v[1] for v in points]) + 1000
geometry[0] = min(station_min_x, geometry[0])
geometry[1] = min(station_min_y, geometry[1])
geometry[2] = max(station_max_x, geometry[2])
geometry[3] = max(station_max_y, geometry[3])
base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository")
EPSG = grid_spec.epsg()
bbox = grid_spec.bounding_box(EPSG)
arome4 = AromeDataRepository(EPSG, base_dir, filename="arome_metcoop_default2_5km_*.nc",
bounding_box=bbox, allow_subset=True)
# data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity","radiation")
# utc_period = api.UtcPeriod(
# api.Calendar().time(api.YMDhms(2015, 10, 1, 0, 0, 0)),
# api.Calendar().time(api.YMDhms(2015, 10, 2, 0, 0, 0))
# )
# arome_ts=arome4.get_timeseries(["temperature"],utc_period)
# arome_points=[gts.mid_point() for gts in arome_ts['temperature']]
# ax.scatter( [pt.x for pt in arome_points ],[pt.y for pt in arome_points],c=[pt.z for pt in arome_points],alpha=0.5,cmap='gray',s=100)#, facecolors=('r'))
ax.set_xlim(geometry[0], geometry[2])
ax.set_ylim(geometry[1], geometry[3])
plt.show()
def plot_region_model(self, catchment_type, identifier ,x0, y0, dx, dy, nx, ny,
catch_indicies, station_ids,epsg_id):
grid_spec = GridSpecification(epsg_id, x0, y0, dx, dy, nx, ny)
cf = CellDataFetcher(catchment_type, identifier, grid_spec, id_list=catch_indicies)
print( "Start fetching data")
cf.fetch()
print ("Done, now preparing plot")
# Plot the extracted data
fig, ax = plt.subplots(1)
color_map = {"forest": 'g', "lake": 'b', "glacier": 'r', "cell": "0.75", "reservoir": "purple"}
extent = grid_spec.geometry[0], grid_spec.geometry[2], grid_spec.geometry[1], grid_spec.geometry[3]
ax.imshow(cf.elevation_raster, origin='upper', extent=extent, cmap=cm.gray)
for catchment_cells in iter(cf.cell_data.values()):
self.add_plot_polygons(ax, [cell["cell"] for cell in catchment_cells], color=color_map["cell"])
for catchment_land_type in iter(cf.catchment_land_types.values()):
for k,v in iter(catchment_land_type.items()):
self.add_plot_polygons(ax, v, color=color_map[k])
geometry = grid_spec.geometry
if station_ids is not None:
glr = GisLocationService()
stations = glr.get_locations(station_ids, epsg_id)
if stations:
points = stations.values()
ax.scatter([pt[0] for pt in points], [pt[1] for pt in points], alpha=0.5)
station_min_x = min([v[0] for v in points]) - 1000
station_max_x = max([v[0] for v in points]) + 1000
station_min_y = min([v[1] for v in points]) - 1000
station_max_y = max([v[1] for v in points]) + 1000
geometry[0] = min(station_min_x, geometry[0])
geometry[1] = min(station_min_y, geometry[1])
geometry[2] = max(station_max_x, geometry[2])
geometry[3] = max(station_max_y, geometry[3])
base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository")
EPSG = grid_spec.epsg_id
bbox = grid_spec.bounding_box(EPSG)
arome4 = AromeDataRepository(EPSG, base_dir, filename="arome_metcoop_default2_5km_*.nc",
bounding_box=bbox, allow_subset=True)
#data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity","radiation")
utc_period = api.UtcPeriod(
api.Calendar().time(api.YMDhms(2015,10,1,0,0,0)),
api.Calendar().time(api.YMDhms(2015,10,2,0,0,0))
)
#arome_ts=arome4.get_timeseries(["temperature"],utc_period)
#arome_points=[gts.mid_point() for gts in arome_ts['temperature']]
#ax.scatter( [pt.x for pt in arome_points ],[pt.y for pt in arome_points],c=[pt.z for pt in arome_points],alpha=0.5,cmap='gray',s=100)#, facecolors=('r'))
ax.set_xlim(geometry[0], geometry[2])
ax.set_ylim(geometry[1], geometry[3])
plt.show()
def geo_ts_repo_constructor(cls, params): # ,region_config):
if cls_path(
cls
) == 'shyft.repository.service.ssa_geo_ts_repository.GeoTsRepository':
# from shyft.repository.service.ssa_geo_ts_repository import GeoTsRepository
from shyft.repository.service.ssa_geo_ts_repository import MetStationConfig
from shyft.repository.service.gis_location_service import GisLocationService
from shyft.repository.service.ssa_smg_db import SmGTsRepository, PROD, FC_PROD
#epsg = region_config.domain()["EPSG"]
epsg = params['epsg']
met_stations = [MetStationConfig(**s) for s in params['stations_met']]
gis_location_repository = GisLocationService(
server_name=params.get('server_name', None),
server_name_preprod=params.get(
'server_name_preprod',
None)) # this provides the gis locations for my stations
smg_ts_repository = SmGTsRepository(
PROD, FC_PROD) # this provide the read function for my time-series
# return GeoTsRepository(epsg_id=epsg, geo_location_repository=gis_location_repository,
# ts_repository=smg_ts_repository, met_station_list=met_stations,
# ens_config=None)
return cls(epsg_id=epsg,
geo_location_repository=gis_location_repository,
ts_repository=smg_ts_repository,
met_station_list=met_stations,
ens_config=None)
else:
#params.update({'epsg': region_config.domain()["EPSG"]})
return cls(**params)
def fc_geo_ts_repository(self):
"""
Returns
-------
- geo_ts_repository that have met-station-config relevant for tistel
"""
met_stations = [
# this is the list of MetStations, the gis_id tells the position, the remaining tells us what properties we observe/forecast/calculate at the metstation (smg-ts)
MetStationConfig(gis_id=218, # 0 midtpunkt
temperature=u'/Vikf-Tistel........-T0017A3P_MAN',
precipitation=u'/Vikf-Tistel........-T0000A5P_MAN',
radiation=u'/ENKI/STS/Radiation/Sim.-Hestvollan....-T0006V0B-0119-0.8',
wind_speed=u'/Dnmi-Fjærland.Bremu-T0016V3K-A55820-332',
relative_humidity=u'/SHFT-rel-hum-dummy.-T0002A3R-0103')
]
gis_location_repository = GisLocationService() # yaml... geo_location service..this provides the gis locations for my stations
smg_ts_repository = SmGTsRepository(PROD, FC_PROD) # this provide the read function for my time-series
return GeoTsRepository( # together, the location provider, ts-provider, and the station, we have
epsg_id=self.epsg_id,
geo_location_repository=gis_location_repository, # a complete geo_ts-repository
ts_repository=smg_ts_repository,
met_station_list=met_stations,
ens_config=None) # pass service info and met_stations
def smg_geo_ts_repo_constructor(params, region_config):
from shyft.repository.service.ssa_geo_ts_repository import GeoTsRepository
from shyft.repository.service.ssa_geo_ts_repository import MetStationConfig
from shyft.repository.service.gis_location_service import GisLocationService
from shyft.repository.service.ssa_smg_db import SmGTsRepository, PROD, FC_PROD
epsg = region_config.domain()["EPSG"]
met_stations = [MetStationConfig(**s) for s in params['stations_met']]
gis_location_repository = GisLocationService(
) # this provides the gis locations for my stations
smg_ts_repository = SmGTsRepository(
PROD, FC_PROD) # this provide the read function for my time-series
return GeoTsRepository(epsg_id=epsg,
geo_location_repository=gis_location_repository,
ts_repository=smg_ts_repository,
met_station_list=met_stations,
ens_config=None)
def test_get_timeseries_using_known_service_and_db_content(self):
utc = Calendar() # always use Calendar() stuff
met_stations = [ # this is the list of MetStations, the gis_id tells the position, the remaining tells us what properties we observe/forecast/calculate at the metstation (smg-ts)
MetStationConfig(
gis_id=598,
temperature=u'/NeNi-Sylsjøen......-T0017V3KI0114',
precipitation=u'/NeNi-Sylsjøen-2....-T0000D9BI0124'),
MetStationConfig(
gis_id=574,
temperature=u'/NeNi-Stuggusjøen...-T0017V3KI0114',
precipitation=u'/NeNi-Stuggusjøen...-T0000D9BI0124',
radiation=
u'/ENKI/STS/Radiation/Sim.-Stuggusjøen...-T0006A0B-0119')
]
#note: the MetStationConfig can be constructed from yaml-config
gis_location_repository = GisLocationService(
) # this provides the gis locations for my stations
smg_ts_repository = SmGTsRepository(
PROD,
FC_PROD) # this provide the read function for my time-series
geo_ts_repository = GeoTsRepository(
epsg_id=32633,
geo_location_repository=gis_location_repository,
ts_repository=smg_ts_repository,
met_station_list=met_stations,
ens_config=None) #pass service info and met_stations
self.assertIsNotNone(geo_ts_repository)
utc_period = UtcPeriod(utc.time(YMDhms(2010, 1, 1, 0, 0, 0)),
utc.time(YMDhms(2010, 1, 2, 0, 0, 0)))
ts_types = ['temperature', 'precipitation', 'radiation']
geo_ts_dict = geo_ts_repository.get_timeseries(
ts_types, utc_period=utc_period, geo_location_criteria=None)
self.assertIsNotNone(geo_ts_dict)
for ts_type in ts_types:
self.assertTrue(
ts_type in geo_ts_dict.keys(),
"we ecpect to find an entry for each requested type (it could be empty list though"
)
self.assertTrue(
len(geo_ts_dict[ts_type]) > 0,
"we expect to find the series that we pass in, given they have not changed the name in SmG PROD"
)
def test_get_ensemble_forecast_using_known_service_and_db_content(self):
utc = Calendar()
met_stations = [MetStationConfig(gis_id=402, temperature=u'/LTM5-Nea...........-T0017A3P_EC00_ENS', precipitation=u'/LTM5-Nea...........-T0000A5P_EC00_ENS')]
gis_location_repository = GisLocationService()
smg_ts_repository = SmGTsRepository(PREPROD, FC_PREPROD)
n_ensembles = 51
ens_station_list = [
EnsembleStation(402, n_ensembles,
temperature_ens = lambda i:u'/LTM5-Nea...........-T0017A3P_EC00_E{0:02}'.format(i),
precipitation_ens = lambda i:u'/LTM5-Nea...........-T0000A5P_EC00_E{0:02}'.format(i),
wind_speed_ens = None,
radiation_ens = None,
relative_humidity_ens = None
),
EnsembleStation(460, n_ensembles,
temperature_ens = lambda i:u'/LTM5-Tya...........-T0017A3P_EC00_E{0:02}'.format(i),
precipitation_ens = lambda i:u'/LTM5-Tya...........-T0000A5P_EC00_E{0:02}'.format(i),
wind_speed_ens = None,
radiation_ens = None,
relative_humidity_ens = None
)
]
ens_config = EnsembleConfig(n_ensembles, ens_station_list)
geo_ts_repository = GeoTsRepository(
epsg_id = 32633,
geo_location_repository = gis_location_repository,
ts_repository = smg_ts_repository,
met_station_list = met_stations,
ens_config = ens_config)
self.assertIsNotNone(geo_ts_repository)
utc_period = UtcPeriod(utc.time(YMDhms(2015, 10, 1, 0, 0, 0)),utc.time(YMDhms(2015, 10, 10, 0, 0, 0)))
ts_types= ['temperature', 'precipitation']
ens_geo_ts_dict = geo_ts_repository.get_forecast_ensemble(ts_types, utc_period=utc_period, t_c=None, geo_location_criteria=None)
self.assertIsNotNone(ens_geo_ts_dict)
self.assertEqual(ens_config.n_ensembles, len(ens_geo_ts_dict))
for i in range(ens_config.n_ensembles):
for ts_type in ts_types:
self.assertTrue(ts_type in ens_geo_ts_dict[i].keys(), "we expect to find an entry for each requested type (it could be empty list though)")
self.assertTrue(len(ens_geo_ts_dict[i][ts_type])>0, "we expect to find the series that we pass in by name in SmG PREPROD")
def test_get_forecast_using_known_service_and_db_content(self):
utc = Calendar()
met_stations=[ # this is the list of MetStations, the gis_id tells the position, the remaining tells us what properties we observe/forecast/calculate at the metstation (smg-ts)
MetStationConfig(gis_id=402, temperature=u'/LTM5-Nea...........-T0017A3P_EC00_ENS', precipitation=u'/LTM5-Nea...........-T0000A5P_EC00_ENS')
]
gis_location_repository = GisLocationService()
smg_ts_repository = SmGTsRepository(PREPROD, FC_PREPROD)
geo_ts_repository = GeoTsRepository(
epsg_id = 32633,
geo_location_repository = gis_location_repository,
ts_repository = smg_ts_repository,
met_station_list = met_stations,
ens_config = None)
self.assertIsNotNone(geo_ts_repository)
utc_period = UtcPeriod(utc.time(YMDhms(2015, 10, 1, 0, 0, 0)), utc.time(YMDhms(2015, 10, 10, 0, 0, 0)))
ts_types= ['temperature', 'precipitation']
geo_ts_dict = geo_ts_repository.get_forecast(ts_types, utc_period=utc_period, t_c=None, geo_location_criteria=None)
self.assertIsNotNone(geo_ts_dict)
for ts_type in ts_types:
self.assertTrue(ts_type in geo_ts_dict.keys(), "we expect to find an entry for each requested type (it could be empty list though)")
self.assertTrue(len(geo_ts_dict[ts_type])>0, "we expect to find the series that we pass in by name in SmG PREPROD")
def geo_ts_repository(self):
"""
Returns
-------
- geo_ts_repository with observed values, that have met-station-config relevant for tistel
"""
met_stations = [
# this is the list of MetStations, the gis_id tells the position, the remaining tells us what properties we observe/forecast/calculate at the metstation (smg-ts)
MetStationConfig(gis_id=129, # 0 Fjærland Bremu
temperature=None,
precipitation=None,
radiation=None,
wind_speed=u'/Dnmi-Fjærland.Bremu-T0016V3K-A55820-332'),
MetStationConfig(gis_id=619, # 1 Tistel
temperature=u'/Vikf-Tistel........-T0017A3KI0114',
precipitation=None,
radiation=None,
wind_speed=None,
relative_humidity=u'/SHFT-rel-hum-dummy.-T0002A3R-0103'),
MetStationConfig(gis_id=684, # 2 Vossevangen
temperature=None, # u'/Dnmi-Vossevangen...-T0017V3K-A51530-1337'
precipitation=u'/Dnmi-Vossevangen...-T0000D9B-A51530-1337',
radiation=None,
wind_speed=u'/Dnmi-Vossevangen...-T0016V3K-A51530-337'),
MetStationConfig(gis_id=654, # 2 Vossevangen
temperature=None,
precipitation=u'/Dnmi-Vangsnes......-T0000D9B-A53101-338',
radiation=None,
wind_speed=u'/Dnmi-Vangsnes......-T0016V3K-A53101-338'),
MetStationConfig(gis_id=218, # 4 Hestvollan
temperature=u'/Vikf-Hestvollan....-T0017A3KI0114',
precipitation=u'/Vikf-Hestvollan....-T0000D9BI0124',
radiation=u'/ENKI/STS/Radiation/Sim.-Hestvollan....-T0006V0B-0119-0.8', # clear sky,reduced to 0.8
wind_speed=u'/Vikf-Hestvollan....-T0015V3KI0120'),
MetStationConfig(gis_id=542, # 5 Sopandefjell
temperature=None, # u'/Vikf-Sopandefjell..-T0017V3KI0114'
precipitation=None,
radiation=None,
wind_speed=None),
MetStationConfig(gis_id=650, # 6 Ulldalsvatnet
temperature=None,
precipitation=None,
radiation=None,
wind_speed=u'/Hoey-Ulldalsvatnet.-T0015V3KI0120'),
]
gis_location_repository = GisLocationService() # this provides the gis locations for my stations
smg_ts_repository = SmGTsRepository(PROD, FC_PROD) # this provide the read function for my time-series
return GeoTsRepository( # together, the location provider, ts-provider, and the station, we have
epsg_id=self.epsg_id,
geo_location_repository=gis_location_repository, # a complete geo_ts-repository
ts_repository=smg_ts_repository,
met_station_list=met_stations,
ens_config=None) # pass service info and met_stations
def test_get_ensemble_forecast_using_known_service_and_db_content(
self):
utc = Calendar() # always use Calendar() stuff
met_stations = [ # this is the list of MetStations, the gis_id tells the position, the remaining tells us what properties we observe/forecast/calculate at the metstation (smg-ts)
MetStationConfig(
gis_id=598,
temperature=u'/LTM5-Nea...........-T0017A3P_EC00_ENS',
precipitation=u'/LTM5-Nea...........-T0000A5P_EC00_ENS')
]
#note: the MetStationConfig can be constructed from yaml-config
gis_location_repository = GisLocationService(
) # this provides the gis locations for my stations
smg_ts_repository = SmGTsRepository(
PROD,
FC_PROD) # this provide the read function for my time-series
n_ensembles = 51
ens_station_list = [
EnsembleStation(
598,
n_ensembles,
temperature_ens=lambda i:
u'/LTM5-Nea...........-T0017A3P_EC00_E{0:02}'.format(i),
precipitation_ens=lambda i:
u'/LTM5-Nea...........-T0000A5P_EC00_E{0:02}'.format(i),
wind_speed_ens=None,
radiation_ens=None,
relative_humidity_ens=None),
EnsembleStation(
574,
n_ensembles,
temperature_ens=lambda i:
u'/LTM5-Tya...........-T0017A3P_EC00_E{0:02}'.format(i),
precipitation_ens=lambda i:
u'/LTM5-Tya...........-T0000A5P_EC00_E{0:02}'.format(i),
wind_speed_ens=None,
radiation_ens=None,
relative_humidity_ens=None)
]
ens_config = EnsembleConfig(n_ensembles, ens_station_list)
geo_ts_repository = GeoTsRepository(
epsg_id=32633,
geo_location_repository=gis_location_repository,
ts_repository=smg_ts_repository,
met_station_list=met_stations,
ens_config=ens_config) #pass service info and met_stations
self.assertIsNotNone(geo_ts_repository)
utc_period = UtcPeriod(utc.time(YMDhms(2015, 10, 1, 0, 0, 0)),
utc.time(YMDhms(2015, 10, 10, 0, 0, 0)))
ts_types = ['temperature', 'precipitation']
ens_geo_ts_dict = geo_ts_repository.get_forecast_ensemble(
ts_types,
utc_period=utc_period,
t_c=None,
geo_location_criteria=None)
self.assertIsNotNone(ens_geo_ts_dict)
self.assertEqual(ens_config.n_ensembles, len(ens_geo_ts_dict))
for i in range(ens_config.n_ensembles):
for ts_type in ts_types:
self.assertTrue(
ts_type in ens_geo_ts_dict[i].keys(),
"we ecpect to find an entry for each requested type (it could be empty list though"
)
self.assertTrue(
len(ens_geo_ts_dict[i][ts_type]) > 0,
"we expect to find the series that we pass in, given they have not changed the name in SmG PROD"
)