def post_price_data_response(
unit,
generic_asset_name_groups,
horizon,
rolling,
value_groups,
start,
duration,
resolution,
):
current_app.logger.info("POSTING PRICE DATA")
data_source = get_or_create_user_data_source(current_user)
prices = []
forecasting_jobs = []
for market_group, value_group in zip(generic_asset_name_groups,
value_groups):
for market in market_group:
# Parse the entity address
try:
ea = parse_entity_address(market, entity_type="market")
except EntityAddressException as eae:
return invalid_domain(str(eae))
market_name = ea["market_name"]
# Look for the Market object
market = Market.query.filter(
Market.name == market_name).one_or_none()
if market is None:
return unrecognized_market(market_name)
elif unit != market.unit:
return invalid_unit("%s prices" % market.display_name,
[market.unit])
# Create new Price objects
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - ((start + duration) -
(dt + duration / len(value_group)))
p = Price(
datetime=dt,
value=value,
horizon=h,
market_id=market.id,
data_source_id=data_source.id,
)
prices.append(p)
# Make forecasts, but not in play mode. Price forecasts (horizon>0) can still lead to other price forecasts,
# by the way, due to things like day-ahead markets.
if current_app.config.get("FLEXMEASURES_MODE", "") != "play":
# Forecast 24 and 48 hours ahead for at most the last 24 hours of posted price data
forecasting_jobs = create_forecasting_jobs(
"Price",
market.id,
max(start, start + duration - timedelta(hours=24)),
start + duration,
resolution=duration / len(value_group),
horizons=[timedelta(hours=24),
timedelta(hours=48)],
enqueue=
False, # will enqueue later, only if we successfully saved prices
)
# Put these into the database
current_app.logger.info("SAVING TO DB...")
try:
save_to_session(prices)
db.session.flush()
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
except IntegrityError as e:
current_app.logger.warning(e)
db.session.rollback()
# Allow price data to be replaced only in play mode
if current_app.config.get("FLEXMEASURES_MODE", "") == "play":
save_to_session(prices, overwrite=True)
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
else:
return already_received_and_successfully_processed()
def post_weather_data_response( # noqa: C901
unit,
generic_asset_name_groups,
horizon,
rolling,
value_groups,
start,
duration,
resolution,
):
current_app.logger.info("POSTING WEATHER DATA")
data_source = get_or_create_user_data_source(current_user)
weather_measurements = []
forecasting_jobs = []
for sensor_group, value_group in zip(generic_asset_name_groups,
value_groups):
for sensor in sensor_group:
# Parse the entity address
try:
ea = parse_entity_address(sensor, entity_type="sensor")
except EntityAddressException as eae:
return invalid_domain(str(eae))
weather_sensor_type_name = ea["weather_sensor_type_name"]
latitude = ea["latitude"]
longitude = ea["longitude"]
# Check whether the unit is valid for this sensor type (e.g. no m/s allowed for temperature data)
accepted_units = valid_sensor_units(weather_sensor_type_name)
if unit not in accepted_units:
return invalid_unit(weather_sensor_type_name, accepted_units)
weather_sensor = get_weather_sensor_by(weather_sensor_type_name,
latitude, longitude)
# Create new Weather objects
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - ((start + duration) -
(dt + duration / len(value_group)))
w = Weather(
datetime=dt,
value=value,
horizon=h,
sensor_id=weather_sensor.id,
data_source_id=data_source.id,
)
weather_measurements.append(w)
# make forecasts, but only if the sent-in values are not forecasts themselves (and also not in play)
if current_app.config.get(
"FLEXMEASURES_MODE", ""
) != "play" and horizon <= timedelta(
hours=0
): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this generic asset
forecasting_jobs.extend(
create_forecasting_jobs(
"Weather",
weather_sensor.id,
start,
start + duration,
resolution=duration / len(value_group),
horizons=[horizon],
enqueue=
False, # will enqueue later, only if we successfully saved weather measurements
))
# Put these into the database
current_app.logger.info("SAVING TO DB...")
try:
save_to_session(weather_measurements)
db.session.flush()
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
except IntegrityError as e:
current_app.logger.warning(e)
db.session.rollback()
# Allow meter data to be replaced only in play mode
if current_app.config.get("FLEXMEASURES_MODE", "") == "play":
save_to_session(weather_measurements, overwrite=True)
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
else:
return already_received_and_successfully_processed()
def create_connection_and_value_groups( # noqa: C901
unit, generic_asset_name_groups, value_groups, horizon, rolling, start,
duration):
"""
Code for POSTing Power values to the API.
Only lets users post to assets they own.
The sign of values is validated according to asset specs, but in USEF terms.
Then, we store the reverse sign for FlexMeasures specs (with positive production
and negative consumption).
If power values are not forecasts, forecasting jobs are created.
"""
from flask import current_app
current_app.logger.info("POSTING POWER DATA")
data_source = get_or_create_user_data_source(current_user)
user_assets = get_assets()
if not user_assets:
current_app.logger.info("User doesn't seem to have any assets")
user_asset_ids = [asset.id for asset in user_assets]
power_measurements = []
forecasting_jobs = []
for connection_group, value_group in zip(generic_asset_name_groups,
value_groups):
for connection in connection_group:
# TODO: get asset through util function after refactoring
# Parse the entity address
try:
connection = parse_entity_address(connection,
entity_type="connection")
except EntityAddressException as eae:
return invalid_domain(str(eae))
asset_id = connection["asset_id"]
# Look for the Asset object
if asset_id in user_asset_ids:
asset = Asset.query.filter(Asset.id == asset_id).one_or_none()
else:
current_app.logger.warning("Cannot identify connection %s" %
connection)
return unrecognized_connection_group()
# Validate the sign of the values (following USEF specs with positive consumption and negative production)
if asset.is_pure_consumer and any(v < 0 for v in value_group):
extra_info = (
"Connection %s is registered as a pure consumer and can only receive non-negative values."
% asset.entity_address)
return power_value_too_small(extra_info)
elif asset.is_pure_producer and any(v > 0 for v in value_group):
extra_info = (
"Connection %s is registered as a pure producer and can only receive non-positive values."
% asset.entity_address)
return power_value_too_big(extra_info)
# Create new Power objects
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - ((start + duration) -
(dt + duration / len(value_group)))
p = Power(
datetime=dt,
value=value *
-1, # Reverse sign for FlexMeasures specs with positive production and negative consumption
horizon=h,
asset_id=asset.id,
data_source_id=data_source.id,
)
power_measurements.append(p)
# make forecasts, but only if the sent-in values are not forecasts themselves
if horizon <= timedelta(
hours=0
): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this generic asset
forecasting_jobs.extend(
create_forecasting_jobs(
"Power",
asset_id,
start,
start + duration,
resolution=duration / len(value_group),
enqueue=False,
))
current_app.logger.info("SAVING TO DB AND QUEUEING...")
try:
save_to_session(power_measurements)
db.session.flush()
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
except IntegrityError as e:
current_app.logger.warning(e)
db.session.rollback()
# Allow meter data to be replaced only in play mode
if current_app.config.get("FLEXMEASURES_MODE", "") == "play":
save_to_session(power_measurements, overwrite=True)
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
else:
return already_received_and_successfully_processed()
def make_schedule(
asset_id: int,
start: datetime,
end: datetime,
belief_time: datetime,
resolution: timedelta,
soc_at_start: Optional[float] = None,
soc_targets: Optional[pd.Series] = None,
) -> bool:
"""Preferably, a starting soc is given.
Otherwise, we try to retrieve the current state of charge from the asset (if that is the valid one at the start).
Otherwise, we set the starting soc to 0 (some assets don't use the concept of a state of charge,
and without soc targets and limits the starting soc doesn't matter).
"""
# https://docs.sqlalchemy.org/en/13/faq/connections.html#how-do-i-use-engines-connections-sessions-with-python-multiprocessing-or-os-fork
db.engine.dispose()
rq_job = get_current_job()
# find asset
asset = Asset.query.filter_by(id=asset_id).one_or_none()
click.echo(
"Running Scheduling Job %s: %s, from %s to %s" % (rq_job.id, asset, start, end)
)
if soc_at_start is None:
if start == asset.soc_datetime and asset.soc_in_mwh is not None:
soc_at_start = asset.soc_in_mwh
else:
soc_at_start = 0
if soc_targets is None:
soc_targets = pd.Series(
np.nan, index=pd.date_range(start, end, freq=resolution, closed="right")
)
if asset.asset_type_name == "battery":
consumption_schedule = schedule_battery(
asset, asset.market, start, end, resolution, soc_at_start, soc_targets
)
elif asset.asset_type_name in (
"one-way_evse",
"two-way_evse",
):
consumption_schedule = schedule_charging_station(
asset, asset.market, start, end, resolution, soc_at_start, soc_targets
)
else:
raise ValueError(
"Scheduling is not supported for asset type %s." % asset.asset_type
)
data_source = get_data_source(
data_source_name="Seita",
data_source_type="scheduling script",
)
click.echo("Job %s made schedule." % rq_job.id)
ts_value_schedule = [
Power(
datetime=dt,
horizon=dt.astimezone(pytz.utc) - belief_time.astimezone(pytz.utc),
value=-value,
asset_id=asset_id,
data_source_id=data_source.id,
)
for dt, value in consumption_schedule.items()
] # For consumption schedules, positive values denote consumption. For the db, consumption is negative
try:
save_to_session(ts_value_schedule)
except IntegrityError as e:
current_app.logger.warning(e)
click.echo("Rolling back due to IntegrityError")
db.session.rollback()
if current_app.config.get("FLEXMEASURES_MODE", "") == "play":
click.echo("Saving again, with overwrite=True")
save_to_session(ts_value_schedule, overwrite=True)
db.session.commit()
return True
def save_to_db(
timed_values: Union[BeliefsDataFrame, List[Union[Power, Price, Weather]]],
forecasting_jobs: List[Job] = [],
save_changed_beliefs_only: bool = True,
) -> ResponseTuple:
"""Put the timed values into the database and enqueue forecasting jobs.
Data can only be replaced on servers in play mode.
TODO: remove this legacy function in its entirety (announced v0.8.0)
:param timed_values: BeliefsDataFrame or a list of Power, Price or Weather values to be saved
:param forecasting_jobs: list of forecasting Jobs for redis queues.
:param save_changed_beliefs_only: if True, beliefs that are already stored in the database with an earlier belief time are dropped.
:returns: ResponseTuple
"""
import warnings
warnings.warn(
"The method api.common.utils.api_utils.save_to_db is deprecated. Check out the following replacements:"
"- [recommended option] to store BeliefsDataFrames only, switch to data.utils.save_to_db"
"- to store BeliefsDataFrames and enqueue jobs, switch to api.common.utils.api_utils.save_and_enqueue"
)
if isinstance(timed_values, BeliefsDataFrame):
if save_changed_beliefs_only:
# Drop beliefs that haven't changed
timed_values = (
timed_values.convert_index_from_belief_horizon_to_time()
.groupby(level=["belief_time", "source"], as_index=False)
.apply(drop_unchanged_beliefs)
)
# Work around bug in which groupby still introduces an index level, even though we asked it not to
if None in timed_values.index.names:
timed_values.index = timed_values.index.droplevel(None)
if timed_values.empty:
current_app.logger.debug("Nothing new to save")
return already_received_and_successfully_processed()
current_app.logger.info("SAVING TO DB AND QUEUEING...")
try:
if isinstance(timed_values, BeliefsDataFrame):
TimedBelief.add_to_session(
session=db.session, beliefs_data_frame=timed_values
)
else:
save_to_session(timed_values)
db.session.flush()
[current_app.queues["forecasting"].enqueue_job(job) for job in forecasting_jobs]
db.session.commit()
return request_processed()
except IntegrityError as e:
current_app.logger.warning(e)
db.session.rollback()
# Possibly allow data to be replaced depending on config setting
if current_app.config.get("FLEXMEASURES_ALLOW_DATA_OVERWRITE", False):
if isinstance(timed_values, BeliefsDataFrame):
TimedBelief.add_to_session(
session=db.session,
beliefs_data_frame=timed_values,
allow_overwrite=True,
)
else:
save_to_session(timed_values, overwrite=True)
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
else:
return already_received_and_successfully_processed()
def make_forecasts(
asset_id: int,
timed_value_type: str,
horizon: timedelta,
start: datetime,
end: datetime,
custom_model_params: dict = None,
) -> int:
"""
Build forecasting model specs, make rolling forecasts, save the forecasts made.
Each individual forecast is a belief about an interval.
Returns the number of forecasts made.
Parameters
----------
:param asset_id: int
To identify which asset to forecast
:param timed_value_type: str
This should go away after a refactoring - we now use it to create the DB entry for the forecasts
:param horizon: timedelta
duration between the end of each interval and the time at which the belief about that interval is formed
:param start: datetime
start of forecast period, i.e. start time of the first interval to be forecast
:param end: datetime
end of forecast period, i.e end time of the last interval to be forecast
:param custom_model_params: dict
pass in params which will be passed to the model specs configurator,
e.g. outcome_var_transformation, only advisable to be used for testing.
"""
# https://docs.sqlalchemy.org/en/13/faq/connections.html#how-do-i-use-engines-connections-sessions-with-python-multiprocessing-or-os-fork
db.engine.dispose()
rq_job = get_current_job()
# find out which model to run, fall back to latest recommended
model_search_term = rq_job.meta.get("model_search_term", "linear-OLS")
# find asset
asset = get_asset(asset_id, timed_value_type)
click.echo(
"Running Forecasting Job %s: %s for %s on model '%s', from %s to %s"
% (rq_job.id, asset, horizon, model_search_term, start, end)
)
if hasattr(asset, "market_type"):
ex_post_horizon = None # Todo: until we sorted out the ex_post_horizon, use all available price data
else:
ex_post_horizon = timedelta(hours=0)
# Make model specs
model_configurator = lookup_model_specs_configurator(model_search_term)
model_specs, model_identifier, fallback_model_search_term = model_configurator(
generic_asset=asset,
forecast_start=as_server_time(start),
forecast_end=as_server_time(end),
forecast_horizon=horizon,
ex_post_horizon=ex_post_horizon,
custom_model_params=custom_model_params,
)
model_specs.creation_time = server_now()
rq_job.meta["model_identifier"] = model_identifier
rq_job.meta["fallback_model_search_term"] = fallback_model_search_term
rq_job.save()
# before we run the model, check if horizon is okay and enough data is available
if horizon not in supported_horizons():
raise InvalidHorizonException(
"Invalid horizon on job %s: %s" % (rq_job.id, horizon)
)
query_window = get_query_window(
model_specs.start_of_training,
end,
[lag * model_specs.frequency for lag in model_specs.lags],
)
check_data_availability(
asset,
determine_asset_value_class_by_asset(asset),
start,
end,
query_window,
horizon,
)
data_source = get_data_source(
data_source_name="Seita (%s)"
% rq_job.meta.get("model_identifier", "unknown model"),
data_source_type="forecasting script",
)
forecasts, model_state = make_rolling_forecasts(
start=as_server_time(start),
end=as_server_time(end),
model_specs=model_specs,
)
click.echo("Job %s made %d forecasts." % (rq_job.id, len(forecasts)))
ts_value_forecasts = [
make_timed_value(timed_value_type, asset_id, dt, value, horizon, data_source.id)
for dt, value in forecasts.items()
]
try:
save_to_session(ts_value_forecasts)
except IntegrityError as e:
current_app.logger.warning(e)
click.echo("Rolling back due to IntegrityError")
db.session.rollback()
if current_app.config.get("FLEXMEASURES_MODE", "") == "play":
click.echo("Saving again, with overwrite=True")
save_to_session(ts_value_forecasts, overwrite=True)
db.session.commit()
return len(forecasts)