def setup_assets(db, setup_roles_users, setup_markets):
"""Make some asset types and add assets to known test users."""
data_source = DataSource(name="Seita", type="demo script")
db.session.add(data_source)
db.session.add(
AssetType(
name="solar",
is_producer=True,
can_curtail=True,
daily_seasonality=True,
yearly_seasonality=True,
)
)
db.session.add(
AssetType(
name="wind",
is_producer=True,
can_curtail=True,
daily_seasonality=True,
yearly_seasonality=True,
)
)
test_prosumer = find_user_by_email("*****@*****.**")
test_market = Market.query.filter_by(name="epex_da").one_or_none()
for asset_name in ["wind-asset-1", "wind-asset-2", "solar-asset-1"]:
asset = Asset(
name=asset_name,
asset_type_name="wind" if "wind" in asset_name else "solar",
event_resolution=timedelta(minutes=15),
capacity_in_mw=1,
latitude=10,
longitude=100,
min_soc_in_mwh=0,
max_soc_in_mwh=0,
soc_in_mwh=0,
unit="MW",
market_id=test_market.id,
)
asset.owner = test_prosumer
db.session.add(asset)
# one day of test data (one complete sine curve)
time_slots = pd.date_range(
datetime(2015, 1, 1), datetime(2015, 1, 1, 23, 45), freq="15T"
)
values = [random() * (1 + np.sin(x / 15)) for x in range(len(time_slots))]
for dt, val in zip(time_slots, values):
p = Power(
datetime=as_server_time(dt),
horizon=parse_duration("PT0M"),
value=val,
data_source_id=data_source.id,
)
p.asset = asset
db.session.add(p)
def test_collect_power_resampled(db, app, query_start, query_end, resolution,
num_values):
wind_device_1 = Asset.query.filter_by(name="wind-asset-1").one_or_none()
bdf: tb.BeliefsDataFrame = Power.collect(wind_device_1.name,
(query_start, query_end),
resolution=resolution)
print(bdf)
assert len(bdf) == num_values
def test_collect_power(db, app, query_start, query_end, num_values):
wind_device_1 = Asset.query.filter_by(name="wind-asset-1").one_or_none()
data = Power.query.filter(Power.asset_id == wind_device_1.id).all()
print(data)
bdf: tb.BeliefsDataFrame = Power.collect(wind_device_1.name,
(query_start, query_end))
print(bdf)
assert (
bdf.index.names[0] == "event_start"
) # first index level of collect function should be event_start, so that df.loc[] refers to event_start
assert pd.api.types.is_timedelta64_dtype(
bdf.index.get_level_values("belief_horizon")
) # dtype of belief_horizon is timedelta64[ns], so the minimum horizon on an empty BeliefsDataFrame is NaT instead of NaN
assert len(bdf) == num_values
for v1, v2 in zip(bdf.values, data):
assert abs(v1[0] - v2.value) < 10**-6
def make_timed_value(
timed_value_type: str,
asset_id: int,
dt: datetime,
value: float,
horizon: timedelta,
data_source_id: int,
) -> Union[Power, Price, Weather]:
if timed_value_type not in ("Power", "Price", "Weather"):
raise Exception("Cannot get asset for asset_type '%s'" % timed_value_type)
ts_value = None
if timed_value_type == "Power":
ts_value = Power(
datetime=dt,
horizon=horizon,
value=value,
asset_id=asset_id,
data_source_id=data_source_id,
)
elif timed_value_type == "Price":
ts_value = Price(
datetime=dt,
horizon=horizon,
value=value,
market_id=asset_id,
data_source_id=data_source_id,
)
elif timed_value_type == "Weather":
ts_value = Weather(
datetime=dt,
horizon=horizon,
value=value,
sensor_id=asset_id,
data_source_id=data_source_id,
)
if ts_value is None:
raise Exception(
"Cannot create asset of type %s with id %d" % (timed_value_type, asset_id)
)
return ts_value
def create_connection_and_value_groups( # noqa: C901
unit, generic_asset_name_groups, value_groups, horizon, rolling, start,
duration):
"""
Code for POSTing Power values to the API.
Only lets users post to assets they own.
The sign of values is validated according to asset specs, but in USEF terms.
Then, we store the reverse sign for FlexMeasures specs (with positive production
and negative consumption).
If power values are not forecasts, forecasting jobs are created.
"""
from flask import current_app
current_app.logger.info("POSTING POWER DATA")
data_source = get_or_create_user_data_source(current_user)
user_assets = get_assets()
if not user_assets:
current_app.logger.info("User doesn't seem to have any assets")
user_asset_ids = [asset.id for asset in user_assets]
power_measurements = []
forecasting_jobs = []
for connection_group, value_group in zip(generic_asset_name_groups,
value_groups):
for connection in connection_group:
# TODO: get asset through util function after refactoring
# Parse the entity address
try:
connection = parse_entity_address(connection,
entity_type="connection")
except EntityAddressException as eae:
return invalid_domain(str(eae))
asset_id = connection["asset_id"]
# Look for the Asset object
if asset_id in user_asset_ids:
asset = Asset.query.filter(Asset.id == asset_id).one_or_none()
else:
current_app.logger.warning("Cannot identify connection %s" %
connection)
return unrecognized_connection_group()
# Validate the sign of the values (following USEF specs with positive consumption and negative production)
if asset.is_pure_consumer and any(v < 0 for v in value_group):
extra_info = (
"Connection %s is registered as a pure consumer and can only receive non-negative values."
% asset.entity_address)
return power_value_too_small(extra_info)
elif asset.is_pure_producer and any(v > 0 for v in value_group):
extra_info = (
"Connection %s is registered as a pure producer and can only receive non-positive values."
% asset.entity_address)
return power_value_too_big(extra_info)
# Create new Power objects
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - ((start + duration) -
(dt + duration / len(value_group)))
p = Power(
datetime=dt,
value=value *
-1, # Reverse sign for FlexMeasures specs with positive production and negative consumption
horizon=h,
asset_id=asset.id,
data_source_id=data_source.id,
)
power_measurements.append(p)
# make forecasts, but only if the sent-in values are not forecasts themselves
if horizon <= timedelta(
hours=0
): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this generic asset
forecasting_jobs.extend(
create_forecasting_jobs(
"Power",
asset_id,
start,
start + duration,
resolution=duration / len(value_group),
enqueue=False,
))
current_app.logger.info("SAVING TO DB AND QUEUEING...")
try:
save_to_session(power_measurements)
db.session.flush()
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
except IntegrityError as e:
current_app.logger.warning(e)
db.session.rollback()
# Allow meter data to be replaced only in play mode
if current_app.config.get("FLEXMEASURES_MODE", "") == "play":
save_to_session(power_measurements, overwrite=True)
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
else:
return already_received_and_successfully_processed()
def collect_connection_and_value_groups(
unit: str,
resolution: str,
belief_horizon_window: Tuple[Union[None, timedelta], Union[None,
timedelta]],
belief_time_window: Tuple[Optional[datetime_type],
Optional[datetime_type]],
start: datetime_type,
duration: timedelta,
connection_groups: List[List[str]],
user_source_ids: Union[
int, List[int]] = None, # None is interpreted as all sources
source_types: List[str] = None,
) -> Tuple[dict, int]:
"""
Code for GETting power values from the API.
Only allows to get values from assets owned by current user.
Returns value sign in accordance with USEF specs
(with negative production and positive consumption).
"""
from flask import current_app
current_app.logger.info("GETTING")
user_assets = get_assets()
if not user_assets:
current_app.logger.info("User doesn't seem to have any assets")
user_asset_ids = [asset.id for asset in user_assets]
end = start + duration
value_groups = []
new_connection_groups = (
[]
) # Each connection in the old connection groups will be interpreted as a separate group
for connections in connection_groups:
# Get the asset names
asset_names: List[str] = []
for connection in connections:
# Parse the entity address
try:
connection_details = parse_entity_address(
connection, entity_type="connection")
except EntityAddressException as eae:
return invalid_domain(str(eae))
asset_id = connection_details["asset_id"]
# Look for the Asset object
if asset_id in user_asset_ids:
asset = Asset.query.filter(Asset.id == asset_id).one_or_none()
else:
current_app.logger.warning("Cannot identify connection %s" %
connection)
return unrecognized_connection_group()
asset_names.append(asset.name)
# Get the power values
# TODO: fill NaN for non-existing values
power_bdf_dict: Dict[str, tb.BeliefsDataFrame] = Power.collect(
generic_asset_names=asset_names,
query_window=(start, end),
resolution=resolution,
belief_horizon_window=belief_horizon_window,
belief_time_window=belief_time_window,
user_source_ids=user_source_ids,
source_types=source_types,
sum_multiple=False,
)
# Todo: parse time window of power_bdf_dict, which will be different for requests that are not of the form:
# - start is a timestamp on the hour or a multiple of 15 minutes thereafter
# - duration is a multiple of 15 minutes
for k, bdf in power_bdf_dict.items():
value_groups.append(
[x * -1 for x in bdf["event_value"].tolist()]
) # Reverse sign of values (from FlexMeasures specs to USEF specs)
new_connection_groups.append(k)
response = groups_to_dict(new_connection_groups,
value_groups,
generic_asset_type_name="connection")
response["start"] = isodate.datetime_isoformat(start)
response["duration"] = isodate.duration_isoformat(duration)
response["unit"] = unit # TODO: convert to requested unit
d, s = request_processed()
return dict(**response, **d), s
def setup_api_test_data(db):
"""
Set up data for API v1 tests.
"""
print("Setting up data for API v1 tests on %s" % db.engine)
from flexmeasures.data.models.user import User, Role
from flexmeasures.data.models.assets import Asset, AssetType, Power
from flexmeasures.data.models.data_sources import DataSource
user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role)
# Create an anonymous user
create_user(
username="******",
email="*****@*****.**",
password=hash_password("testtest"),
user_roles=[
"Prosumer",
dict(name="anonymous", description="Anonymous test user"),
],
)
# Create 1 test asset for the anonymous user
test_prosumer = user_datastore.find_user(email="*****@*****.**")
test_asset_type = AssetType(name="test-type")
db.session.add(test_asset_type)
asset_names = ["CS 0"]
assets: List[Asset] = []
for asset_name in asset_names:
asset = Asset(
name=asset_name,
asset_type_name="test-type",
event_resolution=timedelta(minutes=15),
capacity_in_mw=1,
latitude=100,
longitude=100,
unit="MW",
)
asset.owner = test_prosumer
assets.append(asset)
db.session.add(asset)
# Create a test user without a USEF role
create_user(
username="******",
email="*****@*****.**",
password=hash_password("testtest"),
)
# Create 5 test assets for the test_prosumer user
test_prosumer = user_datastore.find_user(email="*****@*****.**")
asset_names = ["CS 1", "CS 2", "CS 3", "CS 4", "CS 5"]
assets: List[Asset] = []
for asset_name in asset_names:
asset = Asset(
name=asset_name,
asset_type_name="test-type",
event_resolution=timedelta(minutes=15),
capacity_in_mw=1,
latitude=100,
longitude=100,
unit="MW",
)
asset.owner = test_prosumer
if asset_name == "CS 4":
asset.event_resolution = timedelta(hours=1)
assets.append(asset)
db.session.add(asset)
# Add power forecasts to one of the assets, for two sources
cs_5 = Asset.query.filter(Asset.name == "CS 5").one_or_none()
test_supplier = user_datastore.find_user(email="*****@*****.**")
prosumer_data_source = DataSource.query.filter(
DataSource.user == test_prosumer).one_or_none()
supplier_data_source = DataSource.query.filter(
DataSource.user == test_supplier).one_or_none()
meter_data = []
for i in range(6):
p_1 = Power(
datetime=isodate.parse_datetime("2015-01-01T00:00:00Z") +
timedelta(minutes=15 * i),
horizon=timedelta(0),
value=(100.0 + i) * -1,
asset_id=cs_5.id,
data_source_id=prosumer_data_source.id,
)
p_2 = Power(
datetime=isodate.parse_datetime("2015-01-01T00:00:00Z") +
timedelta(minutes=15 * i),
horizon=timedelta(hours=0),
value=(1000.0 - 10 * i) * -1,
asset_id=cs_5.id,
data_source_id=supplier_data_source.id,
)
meter_data.append(p_1)
meter_data.append(p_2)
db.session.bulk_save_objects(meter_data)
print("Done setting up data for API v1 tests")
def make_schedule(
asset_id: int,
start: datetime,
end: datetime,
belief_time: datetime,
resolution: timedelta,
soc_at_start: Optional[float] = None,
soc_targets: Optional[pd.Series] = None,
) -> bool:
"""Preferably, a starting soc is given.
Otherwise, we try to retrieve the current state of charge from the asset (if that is the valid one at the start).
Otherwise, we set the starting soc to 0 (some assets don't use the concept of a state of charge,
and without soc targets and limits the starting soc doesn't matter).
"""
# https://docs.sqlalchemy.org/en/13/faq/connections.html#how-do-i-use-engines-connections-sessions-with-python-multiprocessing-or-os-fork
db.engine.dispose()
rq_job = get_current_job()
# find asset
asset = Asset.query.filter_by(id=asset_id).one_or_none()
click.echo(
"Running Scheduling Job %s: %s, from %s to %s" % (rq_job.id, asset, start, end)
)
if soc_at_start is None:
if start == asset.soc_datetime and asset.soc_in_mwh is not None:
soc_at_start = asset.soc_in_mwh
else:
soc_at_start = 0
if soc_targets is None:
soc_targets = pd.Series(
np.nan, index=pd.date_range(start, end, freq=resolution, closed="right")
)
if asset.asset_type_name == "battery":
consumption_schedule = schedule_battery(
asset, asset.market, start, end, resolution, soc_at_start, soc_targets
)
elif asset.asset_type_name in (
"one-way_evse",
"two-way_evse",
):
consumption_schedule = schedule_charging_station(
asset, asset.market, start, end, resolution, soc_at_start, soc_targets
)
else:
raise ValueError(
"Scheduling is not supported for asset type %s." % asset.asset_type
)
data_source = get_data_source(
data_source_name="Seita",
data_source_type="scheduling script",
)
click.echo("Job %s made schedule." % rq_job.id)
ts_value_schedule = [
Power(
datetime=dt,
horizon=dt.astimezone(pytz.utc) - belief_time.astimezone(pytz.utc),
value=-value,
asset_id=asset_id,
data_source_id=data_source.id,
)
for dt, value in consumption_schedule.items()
] # For consumption schedules, positive values denote consumption. For the db, consumption is negative
try:
save_to_session(ts_value_schedule)
except IntegrityError as e:
current_app.logger.warning(e)
click.echo("Rolling back due to IntegrityError")
db.session.rollback()
if current_app.config.get("FLEXMEASURES_MODE", "") == "play":
click.echo("Saving again, with overwrite=True")
save_to_session(ts_value_schedule, overwrite=True)
db.session.commit()
return True
def setup_api_test_data(db):
"""
Set up data for API v1.1 tests.
"""
print("Setting up data for API v1.1 tests on %s" % db.engine)
from flexmeasures.data.models.user import User, Role
from flexmeasures.data.models.assets import Asset, AssetType
from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorType
user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role)
# Create a user without proper registration as a data source
user = user_datastore.create_user(
username="******",
email="*****@*****.**",
password=hash_password("testtest"),
)
role = user_datastore.find_role("Prosumer")
user_datastore.add_role_to_user(user, role)
# Create a test user without a USEF role
create_user(
username="******",
email="*****@*****.**",
password=hash_password("testtest"),
)
# Create 3 test assets for the test_prosumer user
test_prosumer = user_datastore.find_user(email="*****@*****.**")
test_asset_type = AssetType(name="test-type")
db.session.add(test_asset_type)
asset_names = ["CS 1", "CS 2", "CS 3"]
assets: List[Asset] = []
for asset_name in asset_names:
asset = Asset(
name=asset_name,
asset_type_name="test-type",
event_resolution=timedelta(minutes=15),
capacity_in_mw=1,
latitude=100,
longitude=100,
unit="MW",
)
asset.owner = test_prosumer
assets.append(asset)
db.session.add(asset)
# Add power forecasts to the assets
cs_1 = Asset.query.filter(Asset.name == "CS 1").one_or_none()
cs_2 = Asset.query.filter(Asset.name == "CS 2").one_or_none()
cs_3 = Asset.query.filter(Asset.name == "CS 3").one_or_none()
data_source = DataSource.query.filter(
DataSource.user == test_prosumer).one_or_none()
power_forecasts = []
for i in range(6):
p_1 = Power(
datetime=isodate.parse_datetime("2015-01-01T00:00:00Z") +
timedelta(minutes=15 * i),
horizon=timedelta(hours=6),
value=(300 + i) * -1,
asset_id=cs_1.id,
data_source_id=data_source.id,
)
p_2 = Power(
datetime=isodate.parse_datetime("2015-01-01T00:00:00Z") +
timedelta(minutes=15 * i),
horizon=timedelta(hours=6),
value=(300 - i) * -1,
asset_id=cs_2.id,
data_source_id=data_source.id,
)
p_3 = Power(
datetime=isodate.parse_datetime("2015-01-01T00:00:00Z") +
timedelta(minutes=15 * i),
horizon=timedelta(hours=6),
value=(0 + i) * -1,
asset_id=cs_3.id,
data_source_id=data_source.id,
)
power_forecasts.append(p_1)
power_forecasts.append(p_2)
power_forecasts.append(p_3)
db.session.bulk_save_objects(power_forecasts)
# Create 2 weather sensors
test_sensor_type = WeatherSensorType(name="wind_speed")
db.session.add(test_sensor_type)
sensor = WeatherSensor(
name="wind_speed_sensor",
weather_sensor_type_name="wind_speed",
event_resolution=timedelta(minutes=5),
latitude=33.4843866,
longitude=126,
unit="m/s",
)
db.session.add(sensor)
test_sensor_type = WeatherSensorType(name="temperature")
db.session.add(test_sensor_type)
sensor = WeatherSensor(
name="temperature_sensor",
weather_sensor_type_name="temperature",
event_resolution=timedelta(minutes=5),
latitude=33.4843866,
longitude=126,
unit="°C",
)
db.session.add(sensor)
print("Done setting up data for API v1.1 tests")
def populate_time_series_forecasts( # noqa: C901
db: SQLAlchemy,
generic_asset_type: str = None,
generic_asset_name: str = None,
from_date: str = "2015-02-08",
to_date: str = "2015-12-31",
):
start = ensure_local_timezone(datetime.strptime(from_date, "%Y-%m-%d"),
tz_name=LOCAL_TIME_ZONE)
end = ensure_local_timezone(
datetime.strptime(to_date, "%Y-%m-%d") + timedelta(days=1),
tz_name=LOCAL_TIME_ZONE,
)
training_and_testing_period = timedelta(days=30)
horizons = (
timedelta(hours=1),
timedelta(hours=6),
timedelta(hours=24),
timedelta(hours=48),
)
click.echo(
"Populating the database %s with time series forecasts of %s ahead ..."
% (db.engine,
infl_eng.join([naturaldelta(horizon) for horizon in horizons])))
# Set a data source for the forecasts
data_source = DataSource.query.filter_by(name="Seita",
type="demo script").one_or_none()
# List all generic assets for which to forecast.
# Look into asset type if no asset name is given. If an asset name is given,
generic_assets = []
if generic_asset_name is None:
if generic_asset_type is None or generic_asset_type == "WeatherSensor":
sensors = WeatherSensor.query.all()
generic_assets.extend(sensors)
if generic_asset_type is None or generic_asset_type == "Asset":
assets = Asset.query.all()
generic_assets.extend(assets)
if generic_asset_type is None or generic_asset_type == "Market":
markets = Market.query.all()
generic_assets.extend(markets)
else:
if generic_asset_type is None:
click.echo(
"If you specify --asset-name, please also specify --asset-type, so we can look it up."
)
return
if generic_asset_type == "WeatherSensor":
sensors = WeatherSensor.query.filter(
WeatherSensor.name == generic_asset_name).one_or_none()
if sensors is not None:
generic_assets.append(sensors)
if generic_asset_type == "Asset":
assets = Asset.query.filter(
Asset.name == generic_asset_name).one_or_none()
if assets is not None:
generic_assets.append(assets)
if generic_asset_type == "Market":
markets = Market.query.filter(
Market.name == generic_asset_name).one_or_none()
if markets is not None:
generic_assets.append(markets)
if not generic_assets:
click.echo("No such assets in db, so I will not add any forecasts.")
return
# Make a model for each asset and horizon, make rolling forecasts and save to database.
# We cannot use (faster) bulk save, as forecasts might become regressors in other forecasts.
for generic_asset in generic_assets:
for horizon in horizons:
try:
default_model = lookup_model_specs_configurator()
model_specs, model_identifier, model_fallback = default_model(
generic_asset=generic_asset,
forecast_start=start,
forecast_end=end,
forecast_horizon=horizon,
custom_model_params=dict(
training_and_testing_period=training_and_testing_period
),
)
click.echo(
"Computing forecasts of %s ahead for %s, "
"from %s to %s with a training and testing period of %s, using %s ..."
% (
naturaldelta(horizon),
generic_asset.name,
start,
end,
naturaldelta(training_and_testing_period),
model_identifier,
))
model_specs.creation_time = start
forecasts, model_state = make_rolling_forecasts(
start=start, end=end, model_specs=model_specs)
except (NotEnoughDataException, MissingData, NaNData) as e:
click.echo("Skipping forecasts for asset %s: %s" %
(generic_asset, str(e)))
continue
"""
import matplotlib.pyplot as plt
plt.plot(
model_state.specs.outcome_var.load_series().loc[
pd.date_range(start, end=end, freq="15T")
],
label="y",
)
plt.plot(forecasts, label="y^hat")
plt.legend()
plt.show()
"""
beliefs = []
if isinstance(generic_asset, Asset):
beliefs = [
Power(
datetime=ensure_local_timezone(
dt, tz_name=LOCAL_TIME_ZONE),
horizon=horizon,
value=value,
asset_id=generic_asset.id,
data_source_id=data_source.id,
) for dt, value in forecasts.items()
]
elif isinstance(generic_asset, Market):
beliefs = [
Price(
datetime=ensure_local_timezone(
dt, tz_name=LOCAL_TIME_ZONE),
horizon=horizon,
value=value,
market_id=generic_asset.id,
data_source_id=data_source.id,
) for dt, value in forecasts.items()
]
elif isinstance(generic_asset, WeatherSensor):
beliefs = [
Weather(
datetime=ensure_local_timezone(
dt, tz_name=LOCAL_TIME_ZONE),
horizon=horizon,
value=value,
sensor_id=generic_asset.id,
data_source_id=data_source.id,
) for dt, value in forecasts.items()
]
print("Saving %s %s-forecasts for %s..." %
(len(beliefs), naturaldelta(horizon), generic_asset.name))
for belief in beliefs:
db.session.add(belief)
click.echo("DB now has %d Power Forecasts" %
db.session.query(Power).filter(
Power.horizon > timedelta(hours=0)).count())
click.echo("DB now has %d Price Forecasts" %
db.session.query(Price).filter(
Price.horizon > timedelta(hours=0)).count())
click.echo("DB now has %d Weather Forecasts" %
db.session.query(Weather).filter(
Weather.horizon > timedelta(hours=0)).count())