def __init__(self, use_legacy_kwargs: bool = True, **kwargs):
# todo: deprecate the 'market_id' argument in favor of 'sensor_id' (announced v0.8.0)
if "market_id" in kwargs and "sensor_id" not in kwargs:
kwargs["sensor_id"] = tb_utils.replace_deprecated_argument(
"market_id",
kwargs["market_id"],
"sensor_id",
None,
)
kwargs.pop("market_id", None)
# todo: deprecate the 'Price' class in favor of 'TimedBelief' (announced v0.8.0)
if use_legacy_kwargs is False:
# Create corresponding TimedBelief
belief = TimedBelief(**kwargs)
db.session.add(belief)
# Convert key names for legacy model
kwargs["value"] = kwargs.pop("event_value")
kwargs["datetime"] = kwargs.pop("event_start")
kwargs["horizon"] = kwargs.pop("belief_horizon")
kwargs["sensor_id"] = kwargs.pop("sensor").id
kwargs["data_source_id"] = kwargs.pop("source").id
else:
import warnings
warnings.warn(
f"The {self.__class__} class is deprecated. Switch to using the TimedBelief class to suppress this warning.",
FutureWarning,
)
super(Price, self).__init__(**kwargs)
def add_test_weather_sensor_and_forecasts(db: SQLAlchemy, setup_generic_asset_types):
"""one day of test data (one complete sine curve) for two sensors"""
data_source = DataSource.query.filter_by(
name="Seita", type="demo script"
).one_or_none()
weather_station = GenericAsset(
name="Test weather station farther away",
generic_asset_type=setup_generic_asset_types["weather_station"],
latitude=100,
longitude=100,
)
for sensor_name, unit in (("irradiance", "kW/m²"), ("wind speed", "m/s")):
sensor = Sensor(name=sensor_name, generic_asset=weather_station, unit=unit)
db.session.add(sensor)
time_slots = pd.date_range(
datetime(2015, 1, 1), datetime(2015, 1, 2, 23, 45), freq="15T"
)
values = [random() * (1 + np.sin(x / 15)) for x in range(len(time_slots))]
if sensor_name == "temperature":
values = [value * 17 for value in values]
if sensor_name == "wind speed":
values = [value * 45 for value in values]
if sensor_name == "irradiance":
values = [value * 600 for value in values]
for dt, val in zip(time_slots, values):
db.session.add(
TimedBelief(
sensor=sensor,
event_start=as_server_time(dt),
event_value=val,
belief_horizon=timedelta(hours=6),
source=data_source,
)
)
def test_collect_power_resampled(db, app, query_start, query_end, resolution,
num_values, setup_test_data):
wind_device_1 = Sensor.query.filter_by(name="wind-asset-1").one_or_none()
bdf: tb.BeliefsDataFrame = TimedBelief.search(
wind_device_1.name,
event_starts_after=query_start,
event_ends_before=query_end,
resolution=resolution,
most_recent_beliefs_only=True,
)
print(bdf)
assert len(bdf) == num_values
def test_persist_beliefs(setup_beliefs, setup_test_data):
"""Check whether persisting beliefs works.
We load the already set up beliefs, and form new beliefs an hour later.
"""
sensor = Sensor.query.filter_by(name="epex_da").one_or_none()
source = DataSource.query.filter_by(name="ENTSO-E").one_or_none()
bdf: tb.BeliefsDataFrame = TimedBelief.search(
sensor, source=source, most_recent_beliefs_only=False)
# Form new beliefs
df = bdf.reset_index()
df["belief_time"] = df["belief_time"] + timedelta(hours=1)
df["event_value"] = df["event_value"] * 10
bdf = df.set_index(
["event_start", "belief_time", "source", "cumulative_probability"])
TimedBelief.add(bdf)
bdf: tb.BeliefsDataFrame = TimedBelief.search(
sensor, source=source, most_recent_beliefs_only=False)
assert len(bdf) == setup_beliefs * 2
def create_beliefs(db: SQLAlchemy, setup_markets, setup_sources) -> int:
"""
:returns: the number of beliefs set up
"""
sensor = Sensor.query.filter(Sensor.name == "epex_da").one_or_none()
beliefs = [
TimedBelief(
sensor=sensor,
source=setup_sources["ENTSO-E"],
event_value=21,
event_start="2021-03-28 16:00+01",
belief_horizon=timedelta(0),
),
TimedBelief(
sensor=sensor,
source=setup_sources["ENTSO-E"],
event_value=21,
event_start="2021-03-28 17:00+01",
belief_horizon=timedelta(0),
),
TimedBelief(
sensor=sensor,
source=setup_sources["ENTSO-E"],
event_value=20,
event_start="2021-03-28 17:00+01",
belief_horizon=timedelta(hours=2),
cp=0.2,
),
TimedBelief(
sensor=sensor,
source=setup_sources["ENTSO-E"],
event_value=21,
event_start="2021-03-28 17:00+01",
belief_horizon=timedelta(hours=2),
cp=0.5,
),
]
db.session.add_all(beliefs)
return len(beliefs)
def test_query_beliefs(setup_beliefs):
"""Check various ways of querying for beliefs."""
sensor = Sensor.query.filter_by(name="epex_da").one_or_none()
source = DataSource.query.filter_by(name="ENTSO-E").one_or_none()
bdfs = [
TimedBelief.search(sensor,
source=source,
most_recent_beliefs_only=False),
TimedBelief.search(sensor.id,
source=source,
most_recent_beliefs_only=False),
TimedBelief.search(sensor.name,
source=source,
most_recent_beliefs_only=False),
sensor.search_beliefs(source=source, most_recent_beliefs_only=False),
tb.BeliefsDataFrame(sensor.beliefs)[tb.BeliefsDataFrame(
sensor.beliefs).index.get_level_values("source") == source],
]
for bdf in bdfs:
assert sensor.event_resolution == timedelta(hours=1)
assert bdf.event_resolution == timedelta(hours=1)
assert len(bdf) == setup_beliefs
def add_market_prices(db: SQLAlchemy, setup_assets, setup_markets, setup_sources):
"""Add two days of market prices for the EPEX day-ahead market."""
# one day of test data (one complete sine curve)
time_slots = pd.date_range(
datetime(2015, 1, 1), datetime(2015, 1, 2), freq="1H", closed="left"
)
values = [
random() * (1 + np.sin(x * 2 * np.pi / 24)) for x in range(len(time_slots))
]
day1_beliefs = [
TimedBelief(
event_start=as_server_time(dt),
belief_horizon=timedelta(hours=0),
event_value=val,
source=setup_sources["Seita"],
sensor=setup_markets["epex_da"].corresponding_sensor,
)
for dt, val in zip(time_slots, values)
]
db.session.add_all(day1_beliefs)
# another day of test data (8 expensive hours, 8 cheap hours, and again 8 expensive hours)
time_slots = pd.date_range(
datetime(2015, 1, 2), datetime(2015, 1, 3), freq="1H", closed="left"
)
values = [100] * 8 + [90] * 8 + [100] * 8
day2_beliefs = [
TimedBelief(
event_start=as_server_time(dt),
belief_horizon=timedelta(hours=0),
event_value=val,
source=setup_sources["Seita"],
sensor=setup_markets["epex_da"].corresponding_sensor,
)
for dt, val in zip(time_slots, values)
]
db.session.add_all(day2_beliefs)
def test_simplify_index(setup_test_data, check_empty_frame):
"""Check whether simplify_index retains the event resolution."""
wind_device_1 = Sensor.query.filter_by(name="wind-asset-1").one_or_none()
bdf: tb.BeliefsDataFrame = TimedBelief.search(
wind_device_1.name,
event_starts_after=datetime(2015, 1, 1, tzinfo=pytz.utc),
event_ends_before=datetime(2015, 1, 2, tzinfo=pytz.utc),
resolution=timedelta(minutes=15),
).convert_index_from_belief_time_to_horizon()
if check_empty_frame:
# We empty the BeliefsDataFrame, which retains the metadata such as sensor and resolution
bdf = bdf.iloc[0:0, :]
df = simplify_index(bdf)
assert df.event_resolution == timedelta(minutes=15)
def setup_fresh_test_data(
fresh_db,
setup_markets_fresh_db,
setup_roles_users_fresh_db,
setup_generic_asset_types_fresh_db,
app,
fresh_remove_seasonality_for_power_forecasts,
):
db = fresh_db
setup_roles_users = setup_roles_users_fresh_db
setup_markets = setup_markets_fresh_db
data_source = DataSource(name="Seita", type="demo script")
db.session.add(data_source)
db.session.flush()
for asset_name in ["wind-asset-2", "solar-asset-1"]:
asset = Asset(
name=asset_name,
asset_type_name="wind" if "wind" in asset_name else "solar",
event_resolution=timedelta(minutes=15),
capacity_in_mw=1,
latitude=10,
longitude=100,
min_soc_in_mwh=0,
max_soc_in_mwh=0,
soc_in_mwh=0,
unit="MW",
market_id=setup_markets["epex_da"].id,
)
asset.owner = setup_roles_users["Test Prosumer User"]
db.session.add(asset)
time_slots = pd.date_range(
datetime(2015, 1, 1), datetime(2015, 1, 1, 23, 45), freq="15T"
)
values = [random() * (1 + np.sin(x / 15)) for x in range(len(time_slots))]
beliefs = [
TimedBelief(
event_start=as_server_time(dt),
belief_horizon=parse_duration("PT0M"),
event_value=val,
sensor=asset.corresponding_sensor,
source=data_source,
)
for dt, val in zip(time_slots, values)
]
db.session.add_all(beliefs)
add_test_weather_sensor_and_forecasts(fresh_db, setup_generic_asset_types_fresh_db)
def setup_assets(
db, setup_roles_users, setup_markets, setup_sources, setup_asset_types
) -> Dict[str, Asset]:
"""Add assets to known test users.
Deprecated. Remove with Asset model."""
assets = []
for asset_name in ["wind-asset-1", "wind-asset-2", "solar-asset-1"]:
asset = Asset(
name=asset_name,
owner_id=setup_roles_users["Test Prosumer User"].id,
asset_type_name="wind" if "wind" in asset_name else "solar",
event_resolution=timedelta(minutes=15),
capacity_in_mw=1,
latitude=10,
longitude=100,
min_soc_in_mwh=0,
max_soc_in_mwh=0,
soc_in_mwh=0,
unit="MW",
market_id=setup_markets["epex_da"].id,
)
db.session.add(asset)
assets.append(asset)
# one day of test data (one complete sine curve)
time_slots = pd.date_range(
datetime(2015, 1, 1), datetime(2015, 1, 1, 23, 45), freq="15T"
)
values = [
random() * (1 + np.sin(x * 2 * np.pi / (4 * 24)))
for x in range(len(time_slots))
]
beliefs = [
TimedBelief(
event_start=as_server_time(dt),
belief_horizon=parse_duration("PT0M"),
event_value=val,
sensor=asset.corresponding_sensor,
source=setup_sources["Seita"],
)
for dt, val in zip(time_slots, values)
]
db.session.add_all(beliefs)
return {asset.name: asset for asset in assets}
def test_collect_power(db, app, query_start, query_end, num_values,
setup_test_data):
wind_device_1 = Sensor.query.filter_by(name="wind-asset-1").one_or_none()
data = TimedBelief.query.filter(
TimedBelief.sensor_id == wind_device_1.id).all()
print(data)
bdf: tb.BeliefsDataFrame = TimedBelief.search(
wind_device_1.name,
event_starts_after=query_start,
event_ends_before=query_end,
)
print(bdf)
assert (
bdf.index.names[0] == "event_start"
) # first index level of collect function should be event_start, so that df.loc[] refers to event_start
assert pd.api.types.is_timedelta64_dtype(
bdf.convert_index_from_belief_time_to_horizon().index.get_level_values(
"belief_horizon")
) # dtype of belief_horizon is timedelta64[ns], so the minimum horizon on an empty BeliefsDataFrame is NaT instead of NaN
assert len(bdf) == num_values
for v1, v2 in zip(bdf["event_value"].tolist(), data):
assert abs(v1 - v2.event_value) < 10**-6
def __init__(self, use_legacy_kwargs: bool = True, **kwargs):
# todo: deprecate the 'Weather' class in favor of 'TimedBelief' (announced v0.8.0)
if use_legacy_kwargs is False:
# Create corresponding TimedBelief
belief = TimedBelief(**kwargs)
db.session.add(belief)
# Convert key names for legacy model
kwargs["value"] = kwargs.pop("event_value")
kwargs["datetime"] = kwargs.pop("event_start")
kwargs["horizon"] = kwargs.pop("belief_horizon")
kwargs["sensor_id"] = kwargs.pop("sensor").id
kwargs["data_source_id"] = kwargs.pop("source").id
else:
import warnings
warnings.warn(
f"The {self.__class__} class is deprecated. Switch to using the TimedBelief class to suppress this warning.",
FutureWarning,
)
super(Weather, self).__init__(**kwargs)
def save_to_db(
data: Union[BeliefsDataFrame, List[BeliefsDataFrame]],
bulk_save_objects: bool = False,
save_changed_beliefs_only: bool = True,
) -> str:
"""Save the timed beliefs to the database.
Note: This function does not commit. It does, however, flush the session. Best to keep transactions short.
We make the distinction between updating beliefs and replacing beliefs.
# Updating beliefs
An updated belief is a belief from the same source as some already saved belief, and about the same event,
but with a later belief time. If it has a different event value, then it represents a changed belief.
Note that it is possible to explicitly record unchanged beliefs (i.e. updated beliefs with a later belief time,
but with the same event value), by setting save_changed_beliefs_only to False.
# Replacing beliefs
A replaced belief is a belief from the same source as some already saved belief,
and about the same event and with the same belief time, but with a different event value.
Replacing beliefs is not allowed, because messing with the history corrupts data lineage.
Corrections should instead be recorded as updated beliefs.
Servers in 'play' mode are exempt from this rule, to facilitate replaying simulations.
:param data: BeliefsDataFrame (or a list thereof) to be saved
:param bulk_save_objects: if True, objects are bulk saved with session.bulk_save_objects(),
which is quite fast but has several caveats, see:
https://docs.sqlalchemy.org/orm/persistence_techniques.html#bulk-operations-caveats
:param save_changed_beliefs_only: if True, unchanged beliefs are skipped (updated beliefs are only stored if they represent changed beliefs)
if False, all updated beliefs are stored
:returns: status string, one of the following:
- 'success': all beliefs were saved
- 'success_with_unchanged_beliefs_skipped': not all beliefs represented a state change
- 'success_but_nothing_new': no beliefs represented a state change
"""
# Convert to list
if not isinstance(data, list):
timed_values_list = [data]
else:
timed_values_list = data
status = "success"
values_saved = 0
for timed_values in timed_values_list:
if timed_values.empty:
# Nothing to save
continue
len_before = len(timed_values)
if save_changed_beliefs_only:
# Drop beliefs that haven't changed
timed_values = (
timed_values.convert_index_from_belief_horizon_to_time(
).groupby(level=["belief_time", "source"],
as_index=False).apply(drop_unchanged_beliefs))
len_after = len(timed_values)
if len_after < len_before:
status = "success_with_unchanged_beliefs_skipped"
# Work around bug in which groupby still introduces an index level, even though we asked it not to
if None in timed_values.index.names:
timed_values.index = timed_values.index.droplevel(None)
if timed_values.empty:
# No state changes among the beliefs
continue
current_app.logger.info("SAVING TO DB...")
TimedBelief.add_to_session(
session=db.session,
beliefs_data_frame=timed_values,
bulk_save_objects=bulk_save_objects,
allow_overwrite=current_app.config.get(
"FLEXMEASURES_ALLOW_DATA_OVERWRITE", False),
)
values_saved += len(timed_values)
# Flush to bring up potential unique violations (due to attempting to replace beliefs)
db.session.flush()
if values_saved == 0:
status = "success_but_nothing_new"
return status
def setup_api_test_data(db, setup_accounts, setup_roles_users,
add_market_prices):
"""
Set up data for API v1 tests.
"""
print("Setting up data for API v1 tests on %s" % db.engine)
from flexmeasures.data.models.assets import Asset, AssetType
from flexmeasures.data.models.data_sources import DataSource
# Create an anonymous user TODO: used for demo purposes, maybe "demo-user" would be a better name
test_anonymous_user = create_user(
username="******",
email="*****@*****.**",
password="******",
account_name=setup_accounts["Dummy"].name,
user_roles=[
dict(name="anonymous", description="Anonymous test user"),
],
)
# Create 1 test asset for the anonymous user
test_asset_type = AssetType(name="test-type")
db.session.add(test_asset_type)
asset_names = ["CS 0"]
assets: List[Asset] = []
for asset_name in asset_names:
asset = Asset(
name=asset_name,
owner_id=test_anonymous_user.id,
asset_type_name="test-type",
event_resolution=timedelta(minutes=15),
capacity_in_mw=1,
latitude=100,
longitude=100,
unit="MW",
)
assets.append(asset)
db.session.add(asset)
# Create 5 test assets for the test user
test_user = setup_roles_users["Test Prosumer User"]
asset_names = ["CS 1", "CS 2", "CS 3", "CS 4", "CS 5"]
assets: List[Asset] = []
for asset_name in asset_names:
asset = Asset(
name=asset_name,
owner_id=test_user.id,
asset_type_name="test-type",
event_resolution=timedelta(minutes=15)
if not asset_name == "CS 4" else timedelta(hours=1),
capacity_in_mw=1,
latitude=100,
longitude=100,
unit="MW",
)
assets.append(asset)
db.session.add(asset)
# Add power forecasts to one of the assets, for two sources
cs_5 = Asset.query.filter(Asset.name == "CS 5").one_or_none()
user1_data_source = DataSource.query.filter(
DataSource.user == test_user).one_or_none()
test_user_2 = setup_roles_users["Test Prosumer User 2"]
user2_data_source = DataSource.query.filter(
DataSource.user == test_user_2).one_or_none()
user1_beliefs = [
TimedBelief(
event_start=isodate.parse_datetime("2015-01-01T00:00:00Z") +
timedelta(minutes=15 * i),
belief_horizon=timedelta(0),
event_value=(100.0 + i) * -1,
sensor=cs_5.corresponding_sensor,
source=user1_data_source,
) for i in range(6)
]
user2_beliefs = [
TimedBelief(
event_start=isodate.parse_datetime("2015-01-01T00:00:00Z") +
timedelta(minutes=15 * i),
belief_horizon=timedelta(hours=0),
event_value=(1000.0 - 10 * i) * -1,
sensor=cs_5.corresponding_sensor,
source=user2_data_source,
) for i in range(6)
]
db.session.add_all(user1_beliefs + user2_beliefs)
print("Done setting up data for API v1 tests")
def plot_beliefs(
sensors: List[Sensor],
start: datetime,
duration: timedelta,
belief_time_before: Optional[datetime],
source: Optional[DataSource],
):
"""
Show a simple plot of belief data directly in the terminal.
"""
sensors = list(sensors)
min_resolution = min([s.event_resolution for s in sensors])
# query data
beliefs_by_sensor = TimedBelief.search(
sensors=sensors,
event_starts_after=start,
event_ends_before=start + duration,
beliefs_before=belief_time_before,
source=source,
one_deterministic_belief_per_event=True,
resolution=min_resolution,
sum_multiple=False,
)
# only keep non-empty
for s in sensors:
if beliefs_by_sensor[s.name].empty:
click.echo(f"No data found for sensor '{s.name}' (Id: {s.id})")
beliefs_by_sensor.pop(s.name)
sensors.remove(s)
if len(beliefs_by_sensor.keys()) == 0:
click.echo("No data found!")
raise click.Abort()
first_df = beliefs_by_sensor[sensors[0].name]
# Build title
if len(sensors) == 1:
title = f"Beliefs for Sensor '{sensors[0].name}' (Id {sensors[0].id}).\n"
else:
title = f"Beliefs for Sensor(s) [{','.join([s.name for s in sensors])}], (Id(s): [{','.join([str(s.id) for s in sensors])}]).\n"
title += f"Data spans {naturaldelta(duration)} and starts at {start}."
if belief_time_before:
title += f"\nOnly beliefs made before: {belief_time_before}."
if source:
title += f"\nSource: {source.description}"
title += f"\nThe time resolution (x-axis) is {naturaldelta(min_resolution)}."
uniplot.plot(
[
beliefs.event_value
for beliefs in [beliefs_by_sensor[sn] for sn in [s.name for s in sensors]]
],
title=title,
color=True,
lines=True,
y_unit=first_df.sensor.unit
if len(beliefs_by_sensor) == 1
or all(sensor.unit == first_df.sensor.unit for sensor in sensors)
else "",
legend_labels=[s.name for s in sensors],
)
def post_price_data_response( # noqa C901
unit,
generic_asset_name_groups,
horizon,
prior,
value_groups,
start,
duration,
resolution,
) -> ResponseTuple:
# additional validation, todo: to be moved into Marshmallow
if horizon is None and prior is None:
extra_info = "Missing horizon or prior."
return invalid_horizon(extra_info)
current_app.logger.info("POSTING PRICE DATA")
data_source = get_or_create_source(current_user)
price_df_per_market = []
forecasting_jobs = []
for market_group, event_values in zip(generic_asset_name_groups, value_groups):
for market in market_group:
# Parse the entity address
try:
ea = parse_entity_address(market, entity_type="market")
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = ea["sensor_id"]
# Look for the Sensor object
sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
if sensor is None:
return unrecognized_market(sensor_id)
elif unit != sensor.unit:
return invalid_unit("%s prices" % sensor.name, [sensor.unit])
# Convert to timely-beliefs terminology
event_starts, belief_horizons = determine_belief_timing(
event_values, start, resolution, horizon, prior, sensor
)
# Create new Price objects
beliefs = [
TimedBelief(
event_start=event_start,
event_value=event_value,
belief_horizon=belief_horizon,
sensor=sensor,
source=data_source,
)
for event_start, event_value, belief_horizon in zip(
event_starts, event_values, belief_horizons
)
]
price_df_per_market.append(tb.BeliefsDataFrame(beliefs))
# Make forecasts, but not in play mode. Price forecasts (horizon>0) can still lead to other price forecasts,
# by the way, due to things like day-ahead markets.
if current_app.config.get("FLEXMEASURES_MODE", "") != "play":
# Forecast 24 and 48 hours ahead for at most the last 24 hours of posted price data
forecasting_jobs = create_forecasting_jobs(
sensor.id,
max(start, start + duration - timedelta(hours=24)),
start + duration,
resolution=duration / len(event_values),
horizons=[timedelta(hours=24), timedelta(hours=48)],
enqueue=False, # will enqueue later, after saving data
)
return save_and_enqueue(price_df_per_market, forecasting_jobs)
def post_power_data(
unit,
generic_asset_name_groups,
value_groups,
horizon,
prior,
start,
duration,
resolution,
create_forecasting_jobs_too,
):
# additional validation, todo: to be moved into Marshmallow
if horizon is None and prior is None:
extra_info = "Missing horizon or prior."
return invalid_horizon(extra_info)
current_app.logger.info("POSTING POWER DATA")
data_source = get_or_create_source(current_user)
user_sensors = get_sensors()
if not user_sensors:
current_app.logger.info("User doesn't seem to have any assets")
user_sensor_ids = [sensor.id for sensor in user_sensors]
power_df_per_connection = []
forecasting_jobs = []
for connection_group, event_values in zip(generic_asset_name_groups, value_groups):
for connection in connection_group:
# TODO: get asset through util function after refactoring
# Parse the entity address
try:
ea = parse_entity_address(connection, entity_type="connection")
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = ea["sensor_id"]
# Look for the Sensor object
if sensor_id in user_sensor_ids:
sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
else:
current_app.logger.warning("Cannot identify connection %s" % connection)
return unrecognized_connection_group()
# Validate the sign of the values (following USEF specs with positive consumption and negative production)
if sensor.get_attribute("is_strictly_non_positive") and any(
v < 0 for v in event_values
):
extra_info = (
"Connection %s is registered as a pure consumer and can only receive non-negative values."
% sensor.entity_address
)
return power_value_too_small(extra_info)
elif sensor.get_attribute("is_strictly_non_negative") and any(
v > 0 for v in event_values
):
extra_info = (
"Connection %s is registered as a pure producer and can only receive non-positive values."
% sensor.entity_address
)
return power_value_too_big(extra_info)
# Convert to timely-beliefs terminology
event_starts, belief_horizons = determine_belief_timing(
event_values, start, resolution, horizon, prior, sensor
)
# Create new Power objects
beliefs = [
TimedBelief(
event_start=event_start,
event_value=event_value
* -1, # Reverse sign for FlexMeasures specs with positive production and negative consumption
belief_horizon=belief_horizon,
sensor=sensor,
source=data_source,
)
for event_start, event_value, belief_horizon in zip(
event_starts, event_values, belief_horizons
)
]
power_df_per_connection.append(tb.BeliefsDataFrame(beliefs))
if create_forecasting_jobs_too:
forecasting_jobs.extend(
create_forecasting_jobs(
sensor_id,
start,
start + duration,
resolution=duration / len(event_values),
enqueue=False, # will enqueue later, after saving data
)
)
return save_and_enqueue(power_df_per_connection, forecasting_jobs)
def post_weather_data_response( # noqa: C901
unit,
generic_asset_name_groups,
horizon,
prior,
value_groups,
start,
duration,
resolution,
) -> ResponseTuple:
# additional validation, todo: to be moved into Marshmallow
if horizon is None and prior is None:
extra_info = "Missing horizon or prior."
return invalid_horizon(extra_info)
current_app.logger.info("POSTING WEATHER DATA")
data_source = get_or_create_source(current_user)
weather_df_per_sensor = []
forecasting_jobs = []
for sensor_group, event_values in zip(generic_asset_name_groups, value_groups):
for sensor in sensor_group:
# Parse the entity address
try:
ea = parse_entity_address(sensor, entity_type="weather_sensor")
except EntityAddressException as eae:
return invalid_domain(str(eae))
weather_sensor_type_name = ea["weather_sensor_type_name"]
latitude = ea["latitude"]
longitude = ea["longitude"]
# Check whether the unit is valid for this sensor type (e.g. no m/s allowed for temperature data)
accepted_units = valid_sensor_units(weather_sensor_type_name)
if unit not in accepted_units:
return invalid_unit(weather_sensor_type_name, accepted_units)
sensor: Sensor = get_sensor_by_generic_asset_type_and_location(
weather_sensor_type_name, latitude, longitude
)
# Convert to timely-beliefs terminology
event_starts, belief_horizons = determine_belief_timing(
event_values, start, resolution, horizon, prior, sensor
)
# Create new Weather objects
beliefs = [
TimedBelief(
event_start=event_start,
event_value=event_value,
belief_horizon=belief_horizon,
sensor=sensor,
source=data_source,
)
for event_start, event_value, belief_horizon in zip(
event_starts, event_values, belief_horizons
)
]
weather_df_per_sensor.append(tb.BeliefsDataFrame(beliefs))
# make forecasts, but only if the sent-in values are not forecasts themselves (and also not in play)
if current_app.config.get(
"FLEXMEASURES_MODE", ""
) != "play" and horizon <= timedelta(
hours=0
): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this generic asset
forecasting_jobs.extend(
create_forecasting_jobs(
sensor.id,
start,
start + duration,
resolution=duration / len(event_values),
horizons=[horizon],
enqueue=False, # will enqueue later, after saving data
)
)
return save_and_enqueue(weather_df_per_sensor, forecasting_jobs)
def setup_api_test_data(db, setup_accounts, setup_roles_users, add_market_prices):
"""
Set up data for API v1.1 tests.
"""
print("Setting up data for API v1.1 tests on %s" % db.engine)
from flexmeasures.data.models.user import User, Role
from flexmeasures.data.models.assets import Asset, AssetType
user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role)
# Create a user without proper registration as a data source
user_datastore.create_user(
username="******",
email="*****@*****.**",
password=hash_password("testtest"),
account_id=setup_accounts["Prosumer"].id,
)
# Create 3 test assets for the test_user
test_user = setup_roles_users["Test Prosumer User"]
test_asset_type = AssetType(name="test-type")
db.session.add(test_asset_type)
asset_names = ["CS 1", "CS 2", "CS 3"]
assets: List[Asset] = []
for asset_name in asset_names:
asset = Asset(
name=asset_name,
owner_id=test_user.id,
asset_type_name="test-type",
event_resolution=timedelta(minutes=15),
capacity_in_mw=1,
latitude=100,
longitude=100,
unit="MW",
)
assets.append(asset)
db.session.add(asset)
# Add power forecasts to the assets
cs_1 = Asset.query.filter(Asset.name == "CS 1").one_or_none()
cs_2 = Asset.query.filter(Asset.name == "CS 2").one_or_none()
cs_3 = Asset.query.filter(Asset.name == "CS 3").one_or_none()
data_source = DataSource.query.filter(DataSource.user == test_user).one_or_none()
cs1_beliefs = [
TimedBelief(
event_start=isodate.parse_datetime("2015-01-01T00:00:00Z")
+ timedelta(minutes=15 * i),
belief_horizon=timedelta(hours=6),
event_value=(300 + i) * -1,
sensor=cs_1.corresponding_sensor,
source=data_source,
)
for i in range(6)
]
cs2_beliefs = [
TimedBelief(
event_start=isodate.parse_datetime("2015-01-01T00:00:00Z")
+ timedelta(minutes=15 * i),
belief_horizon=timedelta(hours=6),
event_value=(300 - i) * -1,
sensor=cs_2.corresponding_sensor,
source=data_source,
)
for i in range(6)
]
cs3_beliefs = [
TimedBelief(
event_start=isodate.parse_datetime("2015-01-01T00:00:00Z")
+ timedelta(minutes=15 * i),
belief_horizon=timedelta(hours=6),
event_value=(0 + i) * -1,
sensor=cs_3.corresponding_sensor,
source=data_source,
)
for i in range(6)
]
db.session.add_all(cs1_beliefs + cs2_beliefs + cs3_beliefs)
add_legacy_weather_sensors(db)
print("Done setting up data for API v1.1 tests")
def create_connection_and_value_groups( # noqa: C901
unit, generic_asset_name_groups, value_groups, horizon, rolling, start, duration
):
"""
Code for POSTing Power values to the API.
Only lets users post to assets they own.
The sign of values is validated according to asset specs, but in USEF terms.
Then, we store the reverse sign for FlexMeasures specs (with positive production
and negative consumption).
If power values are not forecasts, forecasting jobs are created.
"""
current_app.logger.info("POSTING POWER DATA")
data_source = get_or_create_source(current_user)
user_sensors = get_sensors()
if not user_sensors:
current_app.logger.info("User doesn't seem to have any assets")
user_sensor_ids = [sensor.id for sensor in user_sensors]
power_df_per_connection = []
forecasting_jobs = []
for connection_group, value_group in zip(generic_asset_name_groups, value_groups):
for connection in connection_group:
# TODO: get asset through util function after refactoring
# Parse the entity address
try:
connection = parse_entity_address(
connection, entity_type="connection", fm_scheme="fm0"
)
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = connection["asset_id"]
# Look for the Sensor object
if sensor_id in user_sensor_ids:
sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
else:
current_app.logger.warning("Cannot identify connection %s" % connection)
return unrecognized_connection_group()
# Validate the sign of the values (following USEF specs with positive consumption and negative production)
if sensor.get_attribute("is_strictly_non_positive") and any(
v < 0 for v in value_group
):
extra_info = (
"Connection %s is registered as a pure consumer and can only receive non-negative values."
% sensor.entity_address
)
return power_value_too_small(extra_info)
elif sensor.get_attribute("is_strictly_non_negative") and any(
v > 0 for v in value_group
):
extra_info = (
"Connection %s is registered as a pure producer and can only receive non-positive values."
% sensor.entity_address
)
return power_value_too_big(extra_info)
# Create a new BeliefsDataFrame
beliefs = []
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - (
(start + duration) - (dt + duration / len(value_group))
)
p = TimedBelief(
event_start=dt,
event_value=value
* -1, # Reverse sign for FlexMeasures specs with positive production and negative consumption
belief_horizon=h,
sensor=sensor,
source=data_source,
)
assert p not in db.session
beliefs.append(p)
power_df_per_connection.append(tb.BeliefsDataFrame(beliefs))
# make forecasts, but only if the sent-in values are not forecasts themselves
if horizon <= timedelta(
hours=0
): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this sensor
forecasting_jobs.extend(
create_forecasting_jobs(
sensor_id,
start,
start + duration,
resolution=duration / len(value_group),
enqueue=False, # will enqueue later, after saving data
)
)
return save_and_enqueue(power_df_per_connection, forecasting_jobs)
def post_weather_data_response( # noqa: C901
unit,
generic_asset_name_groups,
horizon,
rolling,
value_groups,
start,
duration,
resolution,
):
current_app.logger.info("POSTING WEATHER DATA")
data_source = get_or_create_source(current_user)
weather_df_per_sensor = []
forecasting_jobs = []
for sensor_group, value_group in zip(generic_asset_name_groups, value_groups):
for sensor in sensor_group:
# Parse the entity address
try:
ea = parse_entity_address(
sensor, entity_type="weather_sensor", fm_scheme="fm0"
)
except EntityAddressException as eae:
return invalid_domain(str(eae))
weather_sensor_type_name = ea["weather_sensor_type_name"]
latitude = ea["latitude"]
longitude = ea["longitude"]
# Check whether the unit is valid for this sensor type (e.g. no m/s allowed for temperature data)
accepted_units = valid_sensor_units(weather_sensor_type_name)
if unit not in accepted_units:
return invalid_unit(weather_sensor_type_name, accepted_units)
sensor = get_sensor_by_generic_asset_type_and_location(
weather_sensor_type_name, latitude, longitude
)
if is_response_tuple(sensor):
# Error message telling the user about the nearest weather sensor they can post to
return sensor
# Create new Weather objects
beliefs = []
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - (
(start + duration) - (dt + duration / len(value_group))
)
w = TimedBelief(
event_start=dt,
event_value=value,
belief_horizon=h,
sensor=sensor,
source=data_source,
)
beliefs.append(w)
weather_df_per_sensor.append(tb.BeliefsDataFrame(beliefs))
# make forecasts, but only if the sent-in values are not forecasts themselves (and also not in play)
if current_app.config.get(
"FLEXMEASURES_MODE", ""
) != "play" and horizon <= timedelta(
hours=0
): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this sensor
forecasting_jobs.extend(
create_forecasting_jobs(
sensor.id,
start,
start + duration,
resolution=duration / len(value_group),
enqueue=False, # will enqueue later, after saving data
)
)
return save_and_enqueue(weather_df_per_sensor, forecasting_jobs)
def get_prices_data(
metrics: dict,
market_sensor: Sensor,
query_window: Tuple[datetime, datetime],
resolution: str,
forecast_horizon: timedelta,
) -> Tuple[pd.DataFrame, pd.DataFrame, dict]:
"""Get price data and metrics.
Return price observations, price forecasts (either might be an empty DataFrame)
and a dict with the following metrics:
- expected value
- mean absolute error
- mean absolute percentage error
- weighted absolute percentage error
"""
market_name = "" if market_sensor is None else market_sensor.name
# Get price data
price_bdf: tb.BeliefsDataFrame = TimedBelief.search(
[market_name],
event_starts_after=query_window[0],
event_ends_before=query_window[1],
resolution=resolution,
horizons_at_least=None,
horizons_at_most=timedelta(hours=0),
)
price_df: pd.DataFrame = simplify_index(
price_bdf, index_levels_to_columns=["belief_horizon", "source"])
if not price_bdf.empty:
metrics["realised_unit_price"] = price_df["event_value"].mean()
else:
metrics["realised_unit_price"] = np.NaN
# Get price forecast
price_forecast_bdf: tb.BeliefsDataFrame = TimedBelief.search(
[market_name],
event_starts_after=query_window[0],
event_ends_before=query_window[1],
resolution=resolution,
horizons_at_least=forecast_horizon,
horizons_at_most=None,
source_types=["user", "forecasting script", "script"],
)
price_forecast_df: pd.DataFrame = simplify_index(
price_forecast_bdf,
index_levels_to_columns=["belief_horizon", "source"])
# Calculate the price metrics
if not price_forecast_df.empty and price_forecast_df.size == price_df.size:
metrics["expected_unit_price"] = price_forecast_df["event_value"].mean(
)
metrics["mae_unit_price"] = calculations.mean_absolute_error(
price_df["event_value"], price_forecast_df["event_value"])
metrics[
"mape_unit_price"] = calculations.mean_absolute_percentage_error(
price_df["event_value"], price_forecast_df["event_value"])
metrics[
"wape_unit_price"] = calculations.weighted_absolute_percentage_error(
price_df["event_value"], price_forecast_df["event_value"])
else:
metrics["expected_unit_price"] = np.NaN
metrics["mae_unit_price"] = np.NaN
metrics["mape_unit_price"] = np.NaN
metrics["wape_unit_price"] = np.NaN
return price_df, price_forecast_df, metrics
def populate_time_series_forecasts( # noqa: C901
db: SQLAlchemy,
sensor_ids: List[int],
horizons: List[timedelta],
forecast_start: datetime,
forecast_end: datetime,
event_resolution: Optional[timedelta] = None,
):
training_and_testing_period = timedelta(days=30)
click.echo(
"Populating the database %s with time series forecasts of %s ahead ..."
% (db.engine,
infl_eng.join([naturaldelta(horizon) for horizon in horizons])))
# Set a data source for the forecasts
data_source = DataSource.query.filter_by(name="Seita",
type="demo script").one_or_none()
# List all sensors for which to forecast.
sensors = [Sensor.query.filter(Sensor.id.in_(sensor_ids)).one_or_none()]
if not sensors:
click.echo("No such sensors in db, so I will not add any forecasts.")
return
# Make a model for each sensor and horizon, make rolling forecasts and save to database.
# We cannot use (faster) bulk save, as forecasts might become regressors in other forecasts.
for sensor in sensors:
for horizon in horizons:
try:
default_model = lookup_model_specs_configurator()
model_specs, model_identifier, model_fallback = default_model(
sensor=sensor,
forecast_start=forecast_start,
forecast_end=forecast_end,
forecast_horizon=horizon,
custom_model_params=dict(
training_and_testing_period=training_and_testing_period,
event_resolution=event_resolution,
),
)
click.echo(
"Computing forecasts of %s ahead for sensor %s, "
"from %s to %s with a training and testing period of %s, using %s ..."
% (
naturaldelta(horizon),
sensor.id,
forecast_start,
forecast_end,
naturaldelta(training_and_testing_period),
model_identifier,
))
model_specs.creation_time = forecast_start
forecasts, model_state = make_rolling_forecasts(
start=forecast_start,
end=forecast_end,
model_specs=model_specs)
# Upsample to sensor resolution if needed
if forecasts.index.freq > pd.Timedelta(
sensor.event_resolution):
forecasts = model_specs.outcome_var.resample_data(
forecasts,
time_window=(forecasts.index.min(),
forecasts.index.max()),
expected_frequency=sensor.event_resolution,
)
except (NotEnoughDataException, MissingData, NaNData) as e:
click.echo("Skipping forecasts for sensor %s: %s" %
(sensor, str(e)))
continue
beliefs = [
TimedBelief(
event_start=ensure_local_timezone(dt,
tz_name=LOCAL_TIME_ZONE),
belief_horizon=horizon,
event_value=value,
sensor=sensor,
source=data_source,
) for dt, value in forecasts.items()
]
print("Saving %s %s-forecasts for %s..." %
(len(beliefs), naturaldelta(horizon), sensor.id))
for belief in beliefs:
db.session.add(belief)
click.echo("DB now has %d forecasts" %
db.session.query(TimedBelief).filter(
TimedBelief.belief_horizon > timedelta(hours=0)).count())
def make_rolling_viewpoint_forecasts(
sensor_id: int,
horizon: timedelta,
start: datetime,
end: datetime,
custom_model_params: dict = None,
) -> int:
"""Build forecasting model specs, make rolling-viewpoint forecasts, and save the forecasts made.
Each individual forecast is a belief about a time interval.
Rolling-viewpoint forecasts share the same belief horizon (the duration between belief time and knowledge time).
Model specs are also retrained in a rolling fashion, but with its own frequency set in custom_model_params.
See the timely-beliefs lib for relevant terminology.
Parameters
----------
:param sensor_id: int
To identify which sensor to forecast
:param horizon: timedelta
duration between the end of each interval and the time at which the belief about that interval is formed
:param start: datetime
start of forecast period, i.e. start time of the first interval to be forecast
:param end: datetime
end of forecast period, i.e end time of the last interval to be forecast
:param custom_model_params: dict
pass in params which will be passed to the model specs configurator,
e.g. outcome_var_transformation, only advisable to be used for testing.
:returns: int
the number of forecasts made
"""
# https://docs.sqlalchemy.org/en/13/faq/connections.html#how-do-i-use-engines-connections-sessions-with-python-multiprocessing-or-os-fork
db.engine.dispose()
rq_job = get_current_job()
# find out which model to run, fall back to latest recommended
model_search_term = rq_job.meta.get("model_search_term", "linear-OLS")
# find sensor
sensor = Sensor.query.filter_by(id=sensor_id).one_or_none()
click.echo(
"Running Forecasting Job %s: %s for %s on model '%s', from %s to %s" %
(rq_job.id, sensor, horizon, model_search_term, start, end))
if hasattr(sensor, "market_type"):
ex_post_horizon = None # Todo: until we sorted out the ex_post_horizon, use all available price data
else:
ex_post_horizon = timedelta(hours=0)
# Make model specs
model_configurator = lookup_model_specs_configurator(model_search_term)
model_specs, model_identifier, fallback_model_search_term = model_configurator(
sensor=sensor,
forecast_start=as_server_time(start),
forecast_end=as_server_time(end),
forecast_horizon=horizon,
ex_post_horizon=ex_post_horizon,
custom_model_params=custom_model_params,
)
model_specs.creation_time = server_now()
rq_job.meta["model_identifier"] = model_identifier
rq_job.meta["fallback_model_search_term"] = fallback_model_search_term
rq_job.save()
# before we run the model, check if horizon is okay and enough data is available
if horizon not in supported_horizons():
raise InvalidHorizonException("Invalid horizon on job %s: %s" %
(rq_job.id, horizon))
query_window = get_query_window(
model_specs.start_of_training,
end,
[lag * model_specs.frequency for lag in model_specs.lags],
)
check_data_availability(
sensor,
TimedBelief,
start,
end,
query_window,
horizon,
)
data_source = get_data_source(
data_source_name="Seita (%s)" %
rq_job.meta.get("model_identifier", "unknown model"),
data_source_type="forecasting script",
)
forecasts, model_state = make_rolling_forecasts(
start=as_server_time(start),
end=as_server_time(end),
model_specs=model_specs,
)
click.echo("Job %s made %d forecasts." % (rq_job.id, len(forecasts)))
ts_value_forecasts = [
TimedBelief(
event_start=dt,
belief_horizon=horizon,
event_value=value,
sensor=sensor,
source=data_source,
) for dt, value in forecasts.items()
]
bdf = tb.BeliefsDataFrame(ts_value_forecasts)
save_to_db(bdf)
db.session.commit()
return len(forecasts)
def make_schedule(
sensor_id: int,
start: datetime,
end: datetime,
belief_time: datetime,
resolution: timedelta,
soc_at_start: Optional[float] = None,
soc_targets: Optional[pd.Series] = None,
soc_min: Optional[float] = None,
soc_max: Optional[float] = None,
roundtrip_efficiency: Optional[float] = None,
price_sensor: Optional[Sensor] = None,
) -> bool:
"""Preferably, a starting soc is given.
Otherwise, we try to retrieve the current state of charge from the asset (if that is the valid one at the start).
Otherwise, we set the starting soc to 0 (some assets don't use the concept of a state of charge,
and without soc targets and limits the starting soc doesn't matter).
"""
# https://docs.sqlalchemy.org/en/13/faq/connections.html#how-do-i-use-engines-connections-sessions-with-python-multiprocessing-or-os-fork
db.engine.dispose()
rq_job = get_current_job()
# find sensor
sensor = Sensor.query.filter_by(id=sensor_id).one_or_none()
if rq_job:
click.echo("Running Scheduling Job %s: %s, from %s to %s" %
(rq_job.id, sensor, start, end))
if soc_at_start is None:
if (start == sensor.get_attribute("soc_datetime")
and sensor.get_attribute("soc_in_mwh") is not None):
soc_at_start = sensor.get_attribute("soc_in_mwh")
else:
soc_at_start = 0
if soc_targets is None:
soc_targets = pd.Series(np.nan,
index=pd.date_range(start,
end,
freq=resolution,
closed="right"))
if sensor.generic_asset.generic_asset_type.name == "battery":
consumption_schedule = schedule_battery(
sensor,
start,
end,
resolution,
soc_at_start,
soc_targets,
soc_min,
soc_max,
roundtrip_efficiency,
price_sensor=price_sensor,
)
elif sensor.generic_asset.generic_asset_type.name in (
"one-way_evse",
"two-way_evse",
):
consumption_schedule = schedule_charging_station(
sensor,
start,
end,
resolution,
soc_at_start,
soc_targets,
soc_min,
soc_max,
roundtrip_efficiency,
price_sensor=price_sensor,
)
else:
raise ValueError(
"Scheduling is not (yet) supported for asset type %s." %
sensor.generic_asset.generic_asset_type)
data_source = get_data_source(
data_source_name="Seita",
data_source_type="scheduling script",
)
if rq_job:
click.echo("Job %s made schedule." % rq_job.id)
ts_value_schedule = [
TimedBelief(
event_start=dt,
belief_time=belief_time,
event_value=-value,
sensor=sensor,
source=data_source,
) for dt, value in consumption_schedule.items()
] # For consumption schedules, positive values denote consumption. For the db, consumption is negative
bdf = tb.BeliefsDataFrame(ts_value_schedule)
save_to_db(bdf)
db.session.commit()
return True
def save_to_db(
timed_values: Union[BeliefsDataFrame, List[Union[Power, Price, Weather]]],
forecasting_jobs: List[Job] = [],
save_changed_beliefs_only: bool = True,
) -> ResponseTuple:
"""Put the timed values into the database and enqueue forecasting jobs.
Data can only be replaced on servers in play mode.
TODO: remove this legacy function in its entirety (announced v0.8.0)
:param timed_values: BeliefsDataFrame or a list of Power, Price or Weather values to be saved
:param forecasting_jobs: list of forecasting Jobs for redis queues.
:param save_changed_beliefs_only: if True, beliefs that are already stored in the database with an earlier belief time are dropped.
:returns: ResponseTuple
"""
import warnings
warnings.warn(
"The method api.common.utils.api_utils.save_to_db is deprecated. Check out the following replacements:"
"- [recommended option] to store BeliefsDataFrames only, switch to data.utils.save_to_db"
"- to store BeliefsDataFrames and enqueue jobs, switch to api.common.utils.api_utils.save_and_enqueue"
)
if isinstance(timed_values, BeliefsDataFrame):
if save_changed_beliefs_only:
# Drop beliefs that haven't changed
timed_values = (
timed_values.convert_index_from_belief_horizon_to_time()
.groupby(level=["belief_time", "source"], as_index=False)
.apply(drop_unchanged_beliefs)
)
# Work around bug in which groupby still introduces an index level, even though we asked it not to
if None in timed_values.index.names:
timed_values.index = timed_values.index.droplevel(None)
if timed_values.empty:
current_app.logger.debug("Nothing new to save")
return already_received_and_successfully_processed()
current_app.logger.info("SAVING TO DB AND QUEUEING...")
try:
if isinstance(timed_values, BeliefsDataFrame):
TimedBelief.add_to_session(
session=db.session, beliefs_data_frame=timed_values
)
else:
save_to_session(timed_values)
db.session.flush()
[current_app.queues["forecasting"].enqueue_job(job) for job in forecasting_jobs]
db.session.commit()
return request_processed()
except IntegrityError as e:
current_app.logger.warning(e)
db.session.rollback()
# Possibly allow data to be replaced depending on config setting
if current_app.config.get("FLEXMEASURES_ALLOW_DATA_OVERWRITE", False):
if isinstance(timed_values, BeliefsDataFrame):
TimedBelief.add_to_session(
session=db.session,
beliefs_data_frame=timed_values,
allow_overwrite=True,
)
else:
save_to_session(timed_values, overwrite=True)
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
else:
return already_received_and_successfully_processed()
def get_weather_data(
assets: List[Asset],
metrics: dict,
sensor_type: WeatherSensorType,
query_window: Tuple[datetime, datetime],
resolution: str,
forecast_horizon: timedelta,
) -> Tuple[pd.DataFrame, pd.DataFrame, str, Sensor, dict]:
"""Get most recent weather data and forecast weather data for the requested forecast horizon.
Return weather observations, weather forecasts (either might be an empty DataFrame),
the name of the sensor type, the weather sensor and a dict with the following metrics:
- expected value
- mean absolute error
- mean absolute percentage error
- weighted absolute percentage error"""
# Todo: for now we only collect weather data for a single asset
asset = assets[0]
weather_data = tb.BeliefsDataFrame(columns=["event_value"])
weather_forecast_data = tb.BeliefsDataFrame(columns=["event_value"])
sensor_type_name = ""
closest_sensor = None
if sensor_type:
# Find the 50 closest weather sensors
sensor_type_name = sensor_type.name
closest_sensors = Sensor.find_closest(
generic_asset_type_name=asset.generic_asset.generic_asset_type.
name,
sensor_name=sensor_type_name,
n=50,
object=asset,
)
if closest_sensors:
closest_sensor = closest_sensors[0]
# Collect the weather data for the requested time window
sensor_names = [sensor.name for sensor in closest_sensors]
# Get weather data
weather_bdf_dict: Dict[str,
tb.BeliefsDataFrame] = TimedBelief.search(
sensor_names,
event_starts_after=query_window[0],
event_ends_before=query_window[1],
resolution=resolution,
horizons_at_least=None,
horizons_at_most=timedelta(hours=0),
sum_multiple=False,
)
weather_df_dict: Dict[str, pd.DataFrame] = {}
for sensor_name in weather_bdf_dict:
weather_df_dict[sensor_name] = simplify_index(
weather_bdf_dict[sensor_name],
index_levels_to_columns=["belief_horizon", "source"],
)
# Get weather forecasts
weather_forecast_bdf_dict: Dict[
str, tb.BeliefsDataFrame] = TimedBelief.search(
sensor_names,
event_starts_after=query_window[0],
event_ends_before=query_window[1],
resolution=resolution,
horizons_at_least=forecast_horizon,
horizons_at_most=None,
source_types=["user", "forecasting script", "script"],
sum_multiple=False,
)
weather_forecast_df_dict: Dict[str, pd.DataFrame] = {}
for sensor_name in weather_forecast_bdf_dict:
weather_forecast_df_dict[sensor_name] = simplify_index(
weather_forecast_bdf_dict[sensor_name],
index_levels_to_columns=["belief_horizon", "source"],
)
# Take the closest weather sensor which contains some data for the selected time window
for sensor, sensor_name in zip(closest_sensors, sensor_names):
if (not weather_df_dict[sensor_name]
["event_value"].isnull().values.all()
or not weather_forecast_df_dict[sensor_name]
["event_value"].isnull().values.all()):
closest_sensor = sensor
break
weather_data = weather_df_dict[sensor_name]
weather_forecast_data = weather_forecast_df_dict[sensor_name]
# Calculate the weather metrics
if not weather_data.empty:
metrics["realised_weather"] = weather_data["event_value"].mean(
)
else:
metrics["realised_weather"] = np.NaN
if (not weather_forecast_data.empty
and weather_forecast_data.size == weather_data.size):
metrics["expected_weather"] = weather_forecast_data[
"event_value"].mean()
metrics["mae_weather"] = calculations.mean_absolute_error(
weather_data["event_value"],
weather_forecast_data["event_value"])
metrics[
"mape_weather"] = calculations.mean_absolute_percentage_error(
weather_data["event_value"],
weather_forecast_data["event_value"])
metrics[
"wape_weather"] = calculations.weighted_absolute_percentage_error(
weather_data["event_value"],
weather_forecast_data["event_value"])
else:
metrics["expected_weather"] = np.NaN
metrics["mae_weather"] = np.NaN
metrics["mape_weather"] = np.NaN
metrics["wape_weather"] = np.NaN
return (
weather_data,
weather_forecast_data,
sensor_type_name,
closest_sensor,
metrics,
)
def load_sensor_data(
self,
sensor_types: List[SensorType] = None,
start: datetime = None,
end: datetime = None,
resolution: str = None,
belief_horizon_window=(None, None),
belief_time_window=(None, None),
source_types: Optional[List[str]] = None,
exclude_source_types: Optional[List[str]] = None,
) -> Resource:
"""Load data for one or more assets and cache the results.
If the time range parameters are None, they will be gotten from the session.
The horizon window will default to the latest measurement (anything more in the future than the
end of the time interval.
To load data for a specific source, pass a source id.
:returns: self (to allow piping)
Usage
-----
>>> resource = Resource()
>>> resource.load_sensor_data([Power], start=datetime(2014, 3, 1), end=datetime(2014, 3, 1))
>>> resource.cached_power_data
>>> resource.load_sensor_data([Power, Price], start=datetime(2014, 3, 1), end=datetime(2014, 3, 1)).cached_price_data
"""
# Invalidate old caches
self.clear_cache()
# Look up all relevant sensor types for the given resource
if sensor_types is None:
# todo: after splitting Assets and Sensors, construct here a list of sensor types
sensor_types = [Power, Price, Weather]
# todo: after combining the Power, Price and Weather tables into one TimedBeliefs table,
# retrieve data from different sensor types in a single query,
# and cache the results grouped by sensor type (cached_price_data, cached_power_data, etc.)
for sensor_type in sensor_types:
if sensor_type == Power:
sensor_key_attribute = "name"
elif sensor_type == Price:
sensor_key_attribute = "market.name"
else:
raise NotImplementedError("Unsupported sensor type")
# Determine which sensors we need to query
names_of_resource_sensors = set(
coding_utils.rgetattr(asset, sensor_key_attribute)
for asset in self.assets)
# Query the sensors
resource_data: Dict[str, tb.BeliefsDataFrame] = TimedBelief.search(
list(names_of_resource_sensors),
event_starts_after=start,
event_ends_before=end,
horizons_at_least=belief_horizon_window[0],
horizons_at_most=belief_horizon_window[1],
beliefs_after=belief_time_window[0],
beliefs_before=belief_time_window[1],
source_types=source_types,
exclude_source_types=exclude_source_types,
resolution=resolution,
sum_multiple=False,
)
# Cache the data
setattr(
self, f"cached_{sensor_type.__name__.lower()}_data",
resource_data) # e.g. cached_price_data for sensor type Price
return self
def post_price_data_response(
unit,
generic_asset_name_groups,
horizon,
rolling,
value_groups,
start,
duration,
resolution,
):
current_app.logger.info("POSTING PRICE DATA")
data_source = get_or_create_source(current_user)
price_df_per_market = []
forecasting_jobs = []
for market_group, value_group in zip(generic_asset_name_groups, value_groups):
for market in market_group:
# Parse the entity address
try:
ea = parse_entity_address(market, entity_type="market", fm_scheme="fm0")
except EntityAddressException as eae:
return invalid_domain(str(eae))
market_name = ea["market_name"]
# Look for the Sensor object
sensor = get_sensor_by_unique_name(market_name, ["day_ahead", "tou_tariff"])
if is_response_tuple(sensor):
# Error message telling the user what to do
return sensor
if unit != sensor.unit:
return invalid_unit("%s prices" % sensor.name, [sensor.unit])
# Create new Price objects
beliefs = []
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - (
(start + duration) - (dt + duration / len(value_group))
)
p = TimedBelief(
event_start=dt,
event_value=value,
belief_horizon=h,
sensor=sensor,
source=data_source,
)
beliefs.append(p)
price_df_per_market.append(tb.BeliefsDataFrame(beliefs))
# Make forecasts, but not in play mode. Price forecasts (horizon>0) can still lead to other price forecasts,
# by the way, due to things like day-ahead markets.
if current_app.config.get("FLEXMEASURES_MODE", "") != "play":
# Forecast 24 and 48 hours ahead for at most the last 24 hours of posted price data
forecasting_jobs = create_forecasting_jobs(
sensor.id,
max(start, start + duration - timedelta(hours=24)),
start + duration,
resolution=duration / len(value_group),
horizons=[timedelta(hours=24), timedelta(hours=48)],
enqueue=False, # will enqueue later, after saving data
)
return save_and_enqueue(price_df_per_market, forecasting_jobs)
def collect_connection_and_value_groups(
unit: str,
resolution: str,
belief_horizon_window: Tuple[Union[None, timedelta], Union[None, timedelta]],
belief_time_window: Tuple[Optional[datetime_type], Optional[datetime_type]],
start: datetime_type,
duration: timedelta,
connection_groups: List[List[str]],
user_source_ids: Union[int, List[int]] = None, # None is interpreted as all sources
source_types: List[str] = None,
) -> Tuple[dict, int]:
"""
Code for GETting power values from the API.
Only allows to get values from assets owned by current user.
Returns value sign in accordance with USEF specs
(with negative production and positive consumption).
"""
current_app.logger.info("GETTING")
user_sensors = get_sensors()
if not user_sensors:
current_app.logger.info("User doesn't seem to have any assets")
user_sensor_ids = [sensor.id for sensor in user_sensors]
end = start + duration
value_groups = []
new_connection_groups = (
[]
) # Each connection in the old connection groups will be interpreted as a separate group
for connections in connection_groups:
# Get the sensor names
sensor_names: List[str] = []
for connection in connections:
# Parse the entity address
try:
connection_details = parse_entity_address(
connection, entity_type="connection", fm_scheme="fm0"
)
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = connection_details["asset_id"]
# Look for the Sensor object
if sensor_id in user_sensor_ids:
sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
else:
current_app.logger.warning("Cannot identify connection %s" % connection)
return unrecognized_connection_group()
sensor_names.append(sensor.name)
# Get the power values
# TODO: fill NaN for non-existing values
power_bdf_dict: Dict[str, tb.BeliefsDataFrame] = TimedBelief.search(
sensor_names,
event_starts_after=start,
event_ends_before=end,
resolution=resolution,
horizons_at_least=belief_horizon_window[0],
horizons_at_most=belief_horizon_window[1],
beliefs_after=belief_time_window[0],
beliefs_before=belief_time_window[1],
user_source_ids=user_source_ids,
source_types=source_types,
most_recent_beliefs_only=True,
one_deterministic_belief_per_event=True,
sum_multiple=False,
)
# Todo: parse time window of power_bdf_dict, which will be different for requests that are not of the form:
# - start is a timestamp on the hour or a multiple of 15 minutes thereafter
# - duration is a multiple of 15 minutes
for k, bdf in power_bdf_dict.items():
value_groups.append(
[x * -1 for x in bdf["event_value"].tolist()]
) # Reverse sign of values (from FlexMeasures specs to USEF specs)
new_connection_groups.append(k)
response = groups_to_dict(
new_connection_groups, value_groups, generic_asset_type_name="connection"
)
response["start"] = isodate.datetime_isoformat(start)
response["duration"] = isodate.duration_isoformat(duration)
response["unit"] = unit # TODO: convert to requested unit
d, s = request_processed()
return dict(**response, **d), s
