class WeatherStationModelPrediction(Base):
""" The model prediction for a particular weather station.
Based on values from ModelRunGridSubsetPrediction, but captures linear interpolations based on weather
station's location within the grid_subset, and also captures time-based linear interpolations where
needed for certain Model types. """
__tablename__ = 'weather_station_model_predictions'
__table_args__ = (
UniqueConstraint(
'station_code', 'prediction_model_run_timestamp_id', 'prediction_timestamp'),
{'comment': 'The interpolated weather values for a weather station, weather date, and model run'}
)
id = Column(Integer, Sequence('weather_station_model_predictions_id_seq'),
primary_key=True, nullable=False, index=True)
# The 3-digit code for the weather station to which the prediction applies
station_code = Column(Integer, nullable=False, index=True)
# Which PredictionModelRunTimestamp is this station's prediction based on?
prediction_model_run_timestamp_id = Column(Integer, ForeignKey(
'prediction_model_run_timestamps.id'), nullable=False, index=True)
prediction_model_run_timestamp = relationship(
"PredictionModelRunTimestamp")
# The date and time to which the prediction applies. Will most often be copied directly from
# prediction_timestamp for the ModelRunGridSubsetPrediction, but is included again for cases
# when values are interpolated (e.g., noon interpolations on GDPS model runs)
prediction_timestamp = Column(TZTimeStamp, nullable=False, index=True)
# Temperature 2m above model layer - an interpolated value based on 4 values from
# model_run_grid_subset_prediction
tmp_tgl_2 = Column(Float, nullable=True)
# Temperature prediction using available data.
bias_adjusted_temperature = Column(Float, nullable=True)
# Relative Humidity 2m above model layer - an interpolated value based on 4 values
# from model_run_grid_subset_prediction
rh_tgl_2 = Column(Float, nullable=True)
# RH adjusted by bias
bias_adjusted_rh = Column(Float, nullable=True)
# Accumulated precipitation over calendar day measured in UTC (units kg.m^-2)
apcp_sfc_0 = Column(Float, nullable=True)
# Change in accumulated precipitation between current and previous prediction_timestamp
delta_precip = Column(Float, nullable=True)
# Wind direction 10m above ground.
wdir_tgl_10 = Column(Float, nullable=True)
# Wind speed 10m above ground.
wind_tgl_10 = Column(Float, nullable=True)
# Date this record was created.
create_date = Column(TZTimeStamp, nullable=False,
default=time_utils.get_utc_now())
# Date this record was updated.
update_date = Column(TZTimeStamp, nullable=False,
default=time_utils.get_utc_now())
def __str__(self):
return ('{self.station_code} {self.prediction_timestamp} {self.tmp_tgl_2} {self.apcp_sfc_0} '
'{self.delta_precip}').format(self=self)
def flag_file_as_processed(url: str, session: Session):
""" Flag the file as processed in the database """
processed_file = get_processed_file_record(session, url)
if processed_file:
logger.info('re-procesed %s', url)
else:
logger.info('file processed %s', url)
processed_file = ProcessedModelRunUrl(
url=url, create_date=time_utils.get_utc_now())
processed_file.update_date = time_utils.get_utc_now()
# pylint: disable=no-member
session.add(processed_file)
session.commit()
def mock_get_session(*args) -> UnifiedAlchemyMagicMock:
""" return a session with a bare minimum database that should be good for most unit tests. """
prediction_model = PredictionModel(
id=1,
abbreviation='GDPS',
projection='latlon.15x.15',
name='Global Deterministic Prediction System')
prediction_model_run = PredictionModelRunTimestamp(
id=1,
prediction_model_id=1,
prediction_run_timestamp=time_utils.get_utc_now(),
prediction_model=prediction_model,
complete=True)
session = UnifiedAlchemyMagicMock(
data=[(
[
mock.call.query(PredictionModel),
mock.call.filter(
PredictionModel.abbreviation == 'GDPS',
PredictionModel.projection == 'latlon.15x.15')
],
[prediction_model],
),
([mock.call.query(PredictionModelRunTimestamp)],
[prediction_model_run])])
return session
def parse_noon_forecast(station_code, forecast) -> NoonForecast:
""" Transform from the raw forecast json object returned by wf1, to our noon forecast object.
"""
timestamp = datetime.fromtimestamp(
int(forecast['weatherTimestamp']) / 1000, tz=timezone.utc).isoformat()
noon_forecast = NoonForecast(
weather_date=timestamp,
created_at=get_utc_now(),
wfwx_update_date=forecast.get('updateDate', None),
station_code=station_code,
temperature=forecast.get('temperature', math.nan),
relative_humidity=forecast.get('relativeHumidity', math.nan),
wind_speed=forecast.get('windSpeed', math.nan),
wind_direction=forecast.get('windDirection', math.nan),
precipitation=forecast.get('precipitation', math.nan),
gc=forecast.get('grasslandCuring', math.nan),
ffmc=forecast.get('fineFuelMoistureCode', math.nan),
dmc=forecast.get('duffMoistureCode', math.nan),
dc=forecast.get('droughtCode', math.nan),
isi=forecast.get('initialSpreadIndex', math.nan),
bui=forecast.get('buildUpIndex', math.nan),
fwi=forecast.get('fireWeatherIndex', math.nan),
)
temp_valid, rh_valid, wdir_valid, wspeed_valid, precip_valid = get_valid_flags(noon_forecast)
noon_forecast.temp_valid = temp_valid
noon_forecast.rh_valid = rh_valid
noon_forecast.wdir_valid = wdir_valid
noon_forecast.wspeed_valid = wspeed_valid
noon_forecast.precip_valid = precip_valid
return noon_forecast
async def get_detailed_stations(
response: Response,
toi: datetime = None,
source: StationSourceEnum = StationSourceEnum.WILDFIRE_ONE,
__=Depends(audit),
_=Depends(authentication_required)):
""" Returns a list of fire weather stations with detailed information.
-) Unspecified: Use configuration to establish source.
-) LocalStorage: Read from json file (ignore configuration).
-) WildfireOne: Use wildfire API (ignore configuration).
"""
try:
logger.info('/stations/details/')
response.headers[
"Cache-Control"] = "max-age=0" # don't let the browser cache this
if toi is None:
# NOTE: Don't be tempted to move this into the function definition. It's not possible
# to mock a function if it's part of the function definition, and will cause
# tests to fail.
toi = get_utc_now()
else:
toi = get_hour_20(toi)
weather_stations = await fetch_detailed_stations_as_geojson(
toi, source)
return DetailedWeatherStationsResponse(features=weather_stations)
except Exception as exception:
logger.critical(exception, exc_info=True)
raise
async def _get_payloads(self, temporary_path) -> Generator[EnvCanadaPayload, None, None]:
""" Iterator that yields the next to process. """
utc_now = time_utils.get_utc_now()
for model_hour in get_model_run_hours(self.model):
for prediction_hour in model_prediction_hour_iterator(self.model):
urls, model_run_timestamp, prediction_timestamp = make_model_run_download_urls(
self.model, utc_now, model_hour, prediction_hour)
# If the GeoJSON and the KML already exist, then we can skip this one.
if await self._assets_exist(self.model,
model_run_timestamp,
prediction_timestamp):
logger.info('%s: already processed %s-%s',
self.model,
model_run_timestamp, prediction_timestamp)
continue
payload = self._collect_payload(urls,
self.model,
model_run_timestamp,
prediction_timestamp,
temporary_path)
if payload:
yield payload
else:
# If you didn't get one of them - you probably won't get the rest either!
logger.info('Failed to download one of the model files - skipping the rest')
return
def test_get_gdps_download_urls():
""" test to see if get_download_urls methods give the correct number of urls """
# -1 because 000 hour has no APCP_SFC_0
total_num_of_urls = 81 * len(env_canada.GRIB_LAYERS) - 1
assert len(
list(
env_canada.get_global_model_run_download_urls(
time_utils.get_utc_now(), 0))) == total_num_of_urls
def mock_session(monkeypatch):
""" Mocked out sqlalchemy session object """
geom = (
"POLYGON ((-120.525 50.77500000000001, -120.375 50.77500000000001,-120.375 50.62500000000001,"
" -120.525 50.62500000000001, -120.525 50.77500000000001))")
shape = shapely.wkt.loads(geom)
gdps_url = (
'https://dd.weather.gc.ca/model_gem_global/15km/grib2/lat_lon/00/000/'
'CMC_glb_TMP_TGL_2_latlon.15x.15_2021020300_P000.grib2')
gdps_processed_model_run = ProcessedModelRunUrl(url=gdps_url)
gdps_prediction_model = PredictionModel(
id=1,
abbreviation='GDPS',
projection='latlon.15x.15',
name='Global Deterministic Prediction System')
gdps_prediction_model_run = PredictionModelRunTimestamp(
id=1,
prediction_model_id=1,
prediction_run_timestamp=time_utils.get_utc_now(),
prediction_model=gdps_prediction_model,
complete=True)
@contextmanager
def mock_get_session_gdps_scope(*args) -> Generator[Session, None, None]:
yield UnifiedAlchemyMagicMock(
data=[(
[
mock.call.query(PredictionModel),
mock.call.filter(
PredictionModel.abbreviation == 'GDPS',
PredictionModel.projection == 'latlon.15x.15')
],
[gdps_prediction_model],
),
([
mock.call.query(ProcessedModelRunUrl),
mock.call.filter(ProcessedModelRunUrl.url == gdps_url)
], [gdps_processed_model_run]),
([mock.call.query(PredictionModelRunTimestamp)],
[gdps_prediction_model_run]),
([mock.call.query(PredictionModelGridSubset)], [
PredictionModelGridSubset(
id=1,
prediction_model_id=gdps_prediction_model.id,
geom=from_shape(shape))
])])
def mock_get_gdps_prediction_model_run_timestamp_records(*args, **kwargs):
return [(gdps_prediction_model_run, gdps_prediction_model)]
monkeypatch.setattr(app.db.database, 'get_write_session_scope',
mock_get_session_gdps_scope)
monkeypatch.setattr(env_canada,
'get_prediction_model_run_timestamp_records',
mock_get_gdps_prediction_model_run_timestamp_records)
def mock_session(monkeypatch):
""" Mocked out sqlalchemy session object """
geom = (
"POLYGON ((-120.525 50.77500000000001, -120.375 50.77500000000001,-120.375 50.62500000000001,"
" -120.525 50.62500000000001, -120.525 50.77500000000001))")
shape = shapely.wkt.loads(geom)
rdps_url = (
'https://dd.weather.gc.ca/model_gem_regional/10km/grib2/00/034/'
'CMC_reg_RH_TGL_2_ps10km_2020110500_P034.grib2')
rdps_processed_model_run = ProcessedModelRunUrl(url=rdps_url)
rdps_prediction_model = PredictionModel(
id=2,
abbreviation='RDPS',
projection='ps10km',
name='Regional Deterministic Prediction System')
rdps_prediction_model_run = PredictionModelRunTimestamp(
id=1,
prediction_model_id=2,
prediction_run_timestamp=time_utils.get_utc_now(),
prediction_model=rdps_prediction_model,
complete=True)
@contextmanager
def mock_get_session_rdps_scope(*args):
yield UnifiedAlchemyMagicMock(
data=[(
[
mock.call.query(PredictionModel),
mock.call.filter(PredictionModel.abbreviation == 'RDPS',
PredictionModel.projection == 'ps10km')
],
[rdps_prediction_model],
),
([
mock.call.query(ProcessedModelRunUrl),
mock.call.filter(ProcessedModelRunUrl.url == rdps_url)
], [rdps_processed_model_run]),
([mock.call.query(PredictionModelRunTimestamp)],
[rdps_prediction_model_run]),
([mock.call.query(PredictionModelGridSubset)], [
PredictionModelGridSubset(
id=1,
prediction_model_id=rdps_prediction_model.id,
geom=from_shape(shape))
])])
def mock_get_rdps_prediction_model_run_timestamp_records(*args, **kwargs):
return [(rdps_prediction_model_run, rdps_prediction_model)]
monkeypatch.setattr(app.db.database, 'get_write_session_scope',
mock_get_session_rdps_scope)
monkeypatch.setattr(env_canada,
'get_prediction_model_run_timestamp_records',
mock_get_rdps_prediction_model_run_timestamp_records)
def mock_session(monkeypatch):
""" Mocked out sqlalchemy session object """
geom = (
"POLYGON ((-120.525 50.77500000000001, -120.375 50.77500000000001,-120.375 50.62500000000001,"
" -120.525 50.62500000000001, -120.525 50.77500000000001))")
shape = shapely.wkt.loads(geom)
hrdps_url = 'https://dd.weather.gc.ca/model_hrdps/continental/grib2/00/007/' \
+ 'CMC_hrdps_continental_TMP_TGL_2_ps2.5km_2020100700_P007-00.grib2'
hrdps_processed_model_run = ProcessedModelRunUrl(url=hrdps_url)
hrdps_prediction_model = PredictionModel(
id=3,
abbreviation='HRDPS',
projection='ps2.5km',
name='High Resolution Deterministic Prediction System')
hrdps_prediction_model_run = PredictionModelRunTimestamp(
id=1,
prediction_model_id=3,
prediction_run_timestamp=time_utils.get_utc_now(),
prediction_model=hrdps_prediction_model,
complete=True)
@contextmanager
def mock_get_session_hrdps_scope(*args):
yield UnifiedAlchemyMagicMock(
data=[(
[
mock.call.query(PredictionModel),
mock.call.filter(PredictionModel.abbreviation == 'HRDPS',
PredictionModel.projection == 'ps2.5km')
],
[hrdps_prediction_model],
),
([
mock.call.query(ProcessedModelRunUrl),
mock.call.filter(ProcessedModelRunUrl == hrdps_url)
], [hrdps_processed_model_run]),
([mock.call.query(PredictionModelRunTimestamp)],
[hrdps_prediction_model_run]),
([mock.call.query(PredictionModelGridSubset)], [
PredictionModelGridSubset(
id=1,
prediction_model_id=hrdps_prediction_model.id,
geom=from_shape(shape))
])])
def mock_get_hrdps_prediction_model_run_timestamp_records(*args, **kwargs):
return [(hrdps_prediction_model_run, hrdps_prediction_model)]
monkeypatch.setattr(app.db.database, 'get_write_session_scope',
mock_get_session_hrdps_scope)
monkeypatch.setattr(app.weather_models.env_canada,
'get_prediction_model_run_timestamp_records',
mock_get_hrdps_prediction_model_run_timestamp_records)
def store_hfi_request(session: Session, hfi_result_request: HFIResultRequest,
username: str):
""" Store the supplied hfi request """
hfi_request = HFIRequest(
fire_centre_id=hfi_result_request.selected_fire_center_id,
prep_start_day=hfi_result_request.start_date,
prep_end_day=hfi_result_request.end_date,
create_timestamp=get_utc_now(),
create_user=username,
request=hfi_result_request.json())
session.add(hfi_request)
def apply_data_retention_policy():
"""
We can't keep data forever, we just don't have the space.
"""
with app.db.database.get_write_session_scope() as session:
# The easiest target, is the 4 points surrounding a weather station, once it's interpolated
# and used for machine learning - it's no longer of use.
# It would be great to keep it forever. we could go back and use historic data to improve
# machine learning, but unfortunately takes a lot of space.
# Currently we're using 19 days of data for machine learning, so
# keeping 21 days (3 weeks) of historic data is sufficient.
oldest_to_keep = time_utils.get_utc_now() - datetime.timedelta(days=21)
delete_model_run_grid_subset_predictions(session, oldest_to_keep)
def create_api_access_audit_log(username: str, success: bool,
path: str) -> None:
""" Create an audit log. """
try:
with app.db.database.get_write_session_scope() as session:
now = get_utc_now()
audit_log = APIAccessAudit(create_user=username,
path=path,
success=success,
create_timestamp=now)
session.add(audit_log)
session.commit()
except Exception as exception:
logger.error(exception, exc_info=True)
raise
class NoonForecast(Base):
""" Class representing table structure of 'noon_forecasts' table in DB.
Default float values of math.nan are used for the weather variables that are
sometimes null (None), because Postgres evaluates None != None, so the unique
constraint doesn't work on records with >=1 None values. But math.nan == math.nan
"""
__tablename__ = 'noon_forecasts'
__table_args__ = (UniqueConstraint(
'weather_date', 'wfwx_update_date', 'station_code'), {
'comment':
'The noon_forecast for a weather station and weather date.'
})
id = Column(Integer, primary_key=True)
weather_date = Column(TZTimeStamp, nullable=False, index=True)
station_code = Column(Integer, nullable=False, index=True)
temp_valid = Column(Boolean, nullable=False, default=False)
temperature = Column(Float, nullable=False, default=math.nan)
rh_valid = Column(Boolean, nullable=False, default=False)
relative_humidity = Column(Float, nullable=False, default=math.nan)
wdir_valid = Column(Boolean, nullable=False, default=False)
# Set default wind_direction to NaN because some stations don't report it
wind_direction = Column(Float, nullable=False, default=math.nan)
wspeed_valid = Column(Boolean, nullable=False, default=False)
wind_speed = Column(Float, nullable=False, default=math.nan)
precip_valid = Column(Boolean, nullable=False, default=False)
precipitation = Column(Float, nullable=False, default=math.nan)
gc = Column(Float, nullable=False, default=math.nan)
ffmc = Column(Float, nullable=False, default=math.nan)
dmc = Column(Float, nullable=False, default=math.nan)
dc = Column(Float, nullable=False, default=math.nan)
isi = Column(Float, nullable=False, default=math.nan)
bui = Column(Float, nullable=False, default=math.nan)
fwi = Column(Float, nullable=False, default=math.nan)
created_at = Column(TZTimeStamp,
nullable=False,
default=time_utils.get_utc_now(),
index=True)
wfwx_update_date = Column(TZTimeStamp, nullable=False, index=True)
def __str__(self):
return ('station_code:{self.station_code}, '
'weather_date:{self.weather_date}, '
'created_at:{self.created_at}, '
'wfwx_update_date:{self.wfwx_update_date}, '
'temp={self.temperature}, '
'ffmc={self.ffmc}').format(self=self)
def get_session_with_data():
""" Create a session with some test data.
"""
session = UnifiedAlchemyMagicMock()
station_codes = [209, 322]
weather_values = []
for index, tmp in enumerate(mock_tmps):
weather_values.append({'tmp': tmp, 'rh': mock_rhs[index]})
for code in station_codes:
for value in weather_values:
session.add(
NoonForecast(station_code=code,
weather_date=weather_date,
created_at=time_utils.get_utc_now(),
temperature=value['tmp'],
relative_humidity=value['rh']))
return session
def __init__(self, model_type: ModelEnum):
""" Prep variables """
self.files_downloaded = 0
self.files_processed = 0
self.exception_count = 0
# We always work in UTC:
self.now = time_utils.get_utc_now()
self.grib_processor = GribFileProcessor()
self.model_type: ModelEnum = model_type
# set projection based on model_type
if self.model_type == ModelEnum.GDPS:
self.projection = ProjectionEnum.LATLON_15X_15
elif self.model_type == ModelEnum.HRDPS:
self.projection = ProjectionEnum.HIGH_RES_CONTINENTAL
elif self.model_type == ModelEnum.RDPS:
self.projection = ProjectionEnum.REGIONAL_PS
else:
raise UnhandledPredictionModelType(
f'Unknown model type: {self.model_type}')
def test_parse_hourly_actual():
""" Valid fields are set when values exist """
raw_actual = {
"weatherTimestamp": get_utc_now().timestamp(),
"temperature": 0.0,
"relativeHumidity": 0.0,
"windSpeed": 0.0,
"windDirection": 0.0,
"precipitation": 0.0,
"fineFuelMoistureCode": 0.0,
"initialSpreadIndex": 0.0,
"fireWeatherIndex": 0.0
}
hourly_actual = wfwx_api.parse_hourly_actual(1, raw_actual)
assert isinstance(hourly_actual, HourlyActual)
assert hourly_actual.rh_valid is True
assert hourly_actual.temp_valid is True
assert hourly_actual.wdir_valid is True
assert hourly_actual.precip_valid is True
assert hourly_actual.wspeed_valid is True
def test_invalid_metrics():
""" Metric valid flags should be false """
raw_actual = {
"weatherTimestamp": get_utc_now().timestamp(),
"temperature": 0.0,
"relativeHumidity": 101,
"windSpeed": -1,
"windDirection": 361,
"precipitation": -1,
"fineFuelMoistureCode": 0.0,
"initialSpreadIndex": 0.0,
"fireWeatherIndex": 0.0
}
hourly_actual = wfwx_api.parse_hourly_actual(1, raw_actual)
assert isinstance(hourly_actual, HourlyActual)
assert hourly_actual.temp_valid is True
assert hourly_actual.rh_valid is False
assert hourly_actual.precip_valid is False
assert hourly_actual.wspeed_valid is False
assert hourly_actual.wdir_valid is False
class HourlyActual(Base):
""" Class representing table structure of 'hourly_actuals' table in DB.
Default float values of math.nan are used for the weather variables that are
sometimes null (None), because Postgres evaluates None != None, so the unique
constraint doesn't work on records with >=1 None values. But math.nan == math.nan.
WFWX API returns None values when stations have an error reporting out >=1
weather variables (i.e., equipment malfunction).
"""
__tablename__ = 'hourly_actuals'
__table_args__ = (UniqueConstraint('weather_date', 'station_code'), {
'comment':
'The hourly_actuals for a weather station and weather date.'
})
id = Column(Integer, primary_key=True)
weather_date = Column(TZTimeStamp, nullable=False, index=True)
station_code = Column(Integer, nullable=False, index=True)
temp_valid = Column(Boolean, default=False, nullable=False, index=True)
temperature = Column(Float, nullable=False, default=math.nan)
dewpoint = Column(Float, nullable=True)
rh_valid = Column(Boolean, default=False, nullable=False, index=True)
relative_humidity = Column(Float, nullable=False, default=math.nan)
wdir_valid = Column(Boolean, default=False, nullable=False)
wind_direction = Column(Float, nullable=False, default=math.nan)
wspeed_valid = Column(Boolean, default=False, nullable=False)
wind_speed = Column(Float, nullable=False, default=math.nan)
precip_valid = Column(Boolean, default=False, nullable=False)
precipitation = Column(Float, nullable=False, default=math.nan)
ffmc = Column(Float, nullable=False, default=math.nan)
isi = Column(Float, nullable=False, default=math.nan)
fwi = Column(Float, nullable=False, default=math.nan)
created_at = Column(TZTimeStamp,
nullable=False,
default=time_utils.get_utc_now())
def __str__(self):
return ('station_code:{self.station_code}, '
'weather_date:{self.weather_date}, '
'temperature :{self.temperature}, '
'relative_humidity:{self.relative_humidity}').format(self=self)
import logging
from contextlib import contextmanager
from typing import List, Generator
from pytest_bdd import scenario, given, then, parsers
from fastapi.testclient import TestClient
from sqlalchemy.orm import Session
from alchemy_mock.mocking import UnifiedAlchemyMagicMock
import pytest
import app.main
from app.db.models.forecasts import NoonForecast
import app.utils.time as time_utils
logger = logging.getLogger(__name__)
noon = time_utils.get_utc_now().replace(hour=20,
minute=0,
second=0,
microsecond=0)
weather_date = noon - timedelta(days=2)
# they should have the same length
mock_tmps = [20, 21, 22]
mock_rhs = [50, 51, 52]
def get_session_with_data():
""" Create a session with some test data.
"""
session = UnifiedAlchemyMagicMock()
station_codes = [209, 322]
weather_values = []
for index, tmp in enumerate(mock_tmps):
class WeatherDataRequest(BaseModel):
""" A request for weather data for a given set of stations with a time of interest. """
stations: List[int]
time_of_interest: datetime = time_utils.get_utc_now()
def _process_prediction(
self, # pylint: disable=too-many-arguments
prediction: ModelRunGridSubsetPrediction,
station: WeatherStation,
model_run: PredictionModelRunTimestamp,
points: List,
coordinate: List,
machine: StationMachineLearning):
""" NOTE: Re. using griddata to interpolate:
We're interpolating using degrees, as such we're introducing a slight
innacuracy since degrees != distance. (i.e. This distance between two
points at the bottom of a grid, isn't the same as the distance at the
top.)
It would be more accurate to interpolate towards the point of interest
based on the distance of that point from the grid points.
One could:
a) convert all points from degrees to meters in UTM,
and then interpolate - that should be slightly more accurate, e.g.:
```
def transform(long, lat):
# Transform NAD83 long and lat into UTM meters.
zone = math.floor((long + 180) / 6) + 1
utmProjection = "+proj=utm +zone={zone} +ellps=GRS80 +datum=NAD83 +units=m +no_defs".format(
zone=zone)
proj = Proj(utmProjection)
return proj(long, lat)
```
b) use something else fancy like inverse distance weighting.
HOWEVER, the accuracy we gain is very little, it's adding an error of less than
100 meters on the global model. (Which typically results in the 3rd decimal value
of the interpolated value differing.)
More accuracy can be gained by taking into account altitude differences between
points and adjusting accordingly.
"""
# If there's already a prediction, we want to update it
station_prediction = get_weather_station_model_prediction(
self.session, station.code, model_run.id,
prediction.prediction_timestamp)
if station_prediction is None:
station_prediction = WeatherStationModelPrediction()
# Populate the weather station prediction object.
station_prediction.station_code = station.code
station_prediction.prediction_model_run_timestamp_id = model_run.id
station_prediction.prediction_timestamp = prediction.prediction_timestamp
# Calculate the interpolated values.
# 2020 Dec 15, Sybrand: Encountered situation where tmp_tgl_2 was None, add this workaround for it.
# NOTE: Not sure why this value would ever be None. This could happen if for whatever reason, the
# tmp_tgl_2 layer failed to download and process, while other layers did.
if prediction.tmp_tgl_2 is None:
logger.warning(
'tmp_tgl_2 is None for ModelRunGridSubsetPrediction.id == %s',
prediction.id)
else:
station_prediction.tmp_tgl_2 = griddata(points,
prediction.tmp_tgl_2,
coordinate,
method='linear')[0]
# 2020 Dec 10, Sybrand: Encountered situation where rh_tgl_2 was None, add this workaround for it.
# NOTE: Not sure why this value would ever be None. This could happen if for whatever reason, the
# rh_tgl_2 layer failed to download and process, while other layers did.
if prediction.rh_tgl_2 is None:
# This is unexpected, so we log it.
logger.warning(
'rh_tgl_2 is None for ModelRunGridSubsetPrediction.id == %s',
prediction.id)
station_prediction.rh_tgl_2 = None
else:
station_prediction.rh_tgl_2 = griddata(points,
prediction.rh_tgl_2,
coordinate,
method='linear')[0]
# Check that apcp_sfc_0 is None, since accumulated precipitation
# does not exist for 00 hour.
if prediction.apcp_sfc_0 is None:
station_prediction.apcp_sfc_0 = 0.0
else:
station_prediction.apcp_sfc_0 = griddata(points,
prediction.apcp_sfc_0,
coordinate,
method='linear')[0]
# Calculate the delta_precipitation based on station's previous prediction_timestamp
# for the same model run
# For some reason pylint doesn't think session has a flush!
self.session.flush() # pylint: disable=no-member
station_prediction.delta_precip = self._calculate_delta_precip(
station, model_run, prediction, station_prediction)
# Get the closest wind speed
if prediction.wind_tgl_10 is not None:
station_prediction.wind_tgl_10 = prediction.wind_tgl_10[
get_closest_index(coordinate, points)]
# Get the closest wind direcion
if prediction.wdir_tgl_10 is not None:
station_prediction.wdir_tgl_10 = prediction.wdir_tgl_10[
get_closest_index(coordinate, points)]
# Predict the temperature
station_prediction.bias_adjusted_temperature = machine.predict_temperature(
station_prediction.tmp_tgl_2,
station_prediction.prediction_timestamp)
# Predict the rh
station_prediction.bias_adjusted_rh = machine.predict_rh(
station_prediction.rh_tgl_2,
station_prediction.prediction_timestamp)
# Update the update time (this might be an update)
station_prediction.update_date = time_utils.get_utc_now()
# Add this prediction to the session (we'll commit it later.)
self.session.add(station_prediction)
