def save_forecasts(
new_entries: List[TimedBelief],
store: Union[Session, pd.DataFrame]) -> Union[Session, pd.DataFrame]:
"""Save forecasts which are deemed novel, to either file or database.
In the former case, the returned store object has been updated."""
if len(new_entries) > 0:
logging.info("Adding {} new entries.".format(len(new_entries)))
persistence_type = get_config("PERSISTENCE", "type")
if persistence_type == "file": # store is a DataFrame
new_entries_df = pd.DataFrame(new_entries,
columns=df_cols) # TODO: make work
store = store.append(new_entries_df, ignore_index=True, sort=True)
store.to_csv(
"%s/%s" % (path_to_data(), get_config("PERSISTENCE", "name")),
index=False,
columns=new_entries.columns,
)
elif persistence_type == "db": # store is a db session
for belief in new_entries:
store.add(belief)
store.commit()
else:
raise Exception("Unkown persistence type: %s" % persistence_type)
else:
logging.info("No new entries.")
return store
def create_forecasts(locations: pd.DataFrame,
sensor_names: List[str],
num_hours_to_save: int = 6) -> List[TimedBelief]:
logging.debug("Creating forecasts using these sensors: {} "
"and saving the next {} hours".format(
sensor_names, num_hours_to_save))
if get_config("PERSISTENCE", "type") == "file":
source = BeliefSource(name="DarkSky")
else:
# session = dbconfig.create_db_and_session()
from weatherforecast.utils.dbconfig import session
source = session.query(DBBeliefSource).filter_by(
name="DarkSky").first()
api_key = get_config("DARK_SKY", "API_KEY")
forecast_list = []
for location in locations.itertuples():
lat_long = (location[1], location[2])
location_name = location[3]
belief_time = datetime.utcnow().replace(tzinfo=pytz.utc)
logging.debug("Getting forecasts for {} at belief time {} ...".format(
location_name, belief_time))
hourly_forecast_48_list = call_darksky(api_key,
lat_long)["hourly"]["data"]
if get_config("PERSISTENCE", "type") == "file":
sensors = [
Sensor(name=sname) for sname in sensor_names
] # TODO: convey the special ID here (use a class with string ID?)
else:
sensors = get_or_create_sensors_for(session, lat_long[0],
lat_long[1], location_name)
for i in range(0, num_hours_to_save):
forecast = hourly_forecast_48_list[i]
event_start = datetime.fromtimestamp(int(forecast["time"]),
tz=pytz.utc)
add_forecast_for_sensors(
event_start,
belief_time,
forecast,
location_name,
source,
forecast_list,
sensors,
)
return forecast_list
def forecast_is_new(store: Union[Session, pd.DataFrame],
fc: TimedBelief) -> bool:
"""Find out if this specific forecast has already been added.
We take the belief horizon into account. We only want to compare to the latest relevant belief.
For instance, we believed x, later y and even later x again. That is all valuable knowledge.
Should be rare, as beliefs usually become more accurate over time, but still.
"""
persistence_type = get_config("PERSISTENCE", "type")
if persistence_type == "file": # store is a DataFrame
if store.empty:
return True
query = ( # TODO: only take latest belief into account, see db
"event_start == @fc.event_start & source == @fc.source.id &"
" sensor_id == @fc.sensor.id & event_value == @fc.event_value")
temp = store.query(query)
return temp.shape[0] == 0
elif persistence_type == "db": # store is a db session
latestRelevantBelief = (store.query(DBTimedBelief).filter(
DBTimedBelief.event_start == fc.event_start,
DBTimedBelief.source_id == fc.source.id,
DBTimedBelief.sensor_id == fc.sensor.id,
DBTimedBelief.belief_horizon != fc.belief_horizon,
).order_by(DBTimedBelief.belief_horizon.asc()).first())
if latestRelevantBelief is None:
return True
return latestRelevantBelief.event_value != fc.event_value
else:
raise Exception("Unkown persistence type: %s" % persistence_type)
def filter_out_known_forecast(
current_forecasts: List[TimedBelief],
store: Union[Session, pd.DataFrame]) -> List[TimedBelief]:
new_entries_list = []
for fc in current_forecasts:
if forecast_is_new(store, fc):
logging.debug("Adding new forecast: %s" % fc)
new_entries_list.append(fc)
else:
logging.debug("Did not add this new forecast: %s" % fc)
if get_config("PERSISTENCE", "type") == "db":
# is already in session because the linked sensor and source are
store.delete(fc)
return new_entries_list
def create_db_and_session():
db_connection_string = get_config("PERSISTENCE", "name")
print("[WeatherForecasting] Preparing database session for %s" %
db_connection_string)
global SessionClass, session
engine = create_engine(db_connection_string)
SessionClass.configure(bind=engine)
TBBase.metadata.create_all(engine)
if session is None:
session = SessionClass()
create_darksky_source(session)
return session
def append_forecast_entry(
event_start: str,
belief_time: str,
event_value: float,
sensor: Sensor,
source: BeliefSource,
forecast_list: List[TimedBelief],
):
"""Decide which TimedBelief class we need and make an instance."""
logging.debug(
"Adding a forecast to temporary list: {}, {}, {}, {}, {}".format(
event_start, belief_time, source, sensor, event_value))
if get_config("PERSISTENCE", "type") == "file":
tb_class = TimedBelief
else:
tb_class = DBTimedBelief
forecast_list.append(
tb_class(
event_start=event_start,
belief_time=belief_time,
source=source,
sensor=sensor,
value=event_value,
))
store.commit()
else:
raise Exception("Unkown persistence type: %s" % persistence_type)
else:
logging.info("No new entries.")
return store
if __name__ == "__main__":
"""
"""
logging.basicConfig(level=logging.INFO)
sensor_names = SensorName.ALL.value
store = None # this determines where we keep forecasts
if get_config("PERSISTENCE", "type") == "file":
# reading historical forecasts once, which we will need for file-based storage
logging.info("Reading in existing forecasts ...")
if os.path.exists("%s/forecasts.csv" % path_to_data()):
store = pd.read_csv("%s/forecasts.csv" % path_to_data())
else:
store = pd.DataFrame()
else:
store = create_db_and_session()
for forecast_location in get_config("LOCATIONS"):
city, country = [s.strip() for s in forecast_location.split(",")]
logging.info("Getting weather forecasts for %s (%s) ..." %
(city, country))
logging.info(