def _deserialize(self, value, attr, obj, **kwargs) -> Sensor:
"""De-serialize to a Sensor."""
try:
ea = parse_entity_address(value, self.entity_type, self.fm_scheme)
if self.fm_scheme == "fm0":
if self.entity_type == "connection":
sensor = Sensor.query.filter(
Sensor.id == ea["asset_id"]).one_or_none()
elif self.entity_type == "market":
sensor = Sensor.query.filter(
Sensor.name == ea["market_name"]).one_or_none()
elif self.entity_type == "weather_sensor":
sensor = get_sensor_by_generic_asset_type_and_location(
ea["weather_sensor_type_name"], ea["latitude"],
ea["longitude"])
else:
return NotImplemented
else:
sensor = Sensor.query.filter(
Sensor.id == ea["sensor_id"]).one_or_none()
if sensor is not None:
return sensor
else:
raise EntityAddressValidationError(
f"{self.entity_type} with entity address {value} doesn't exist."
)
except EntityAddressException as eae:
raise EntityAddressValidationError(str(eae))
def get_market(post_message) -> Optional[Market]:
"""util method to get market from our post message"""
market_info = parse_entity_address(post_message["market"], "market")
if market_info is None:
return None
return Market.query.filter_by(
name=market_info["market_name"]).one_or_none()
def test_parse_entity_address(entity_type, entity_address, exp_result):
if isinstance(exp_result, str): # this means we expect an exception
with pytest.raises(EntityAddressException, match=exp_result):
parse_entity_address(entity_address, entity_type=entity_type)
else:
res = parse_entity_address(entity_address, entity_type=entity_type)
assert res["scheme"] == "ea1"
assert res["naming_authority"] == exp_result["naming_authority"]
if entity_type in ("connection", "event"):
for field in ("asset_id", "owner_id"):
assert res[field] == exp_result[field]
if entity_type == "market":
assert res["market_name"] == exp_result["market_name"]
if entity_type == "sensor":
for field in ("weather_sensor_type_name", "latitude", "longitude"):
assert res[field] == exp_result[field]
if entity_type == "event":
for field in ("event_type", "event_id"):
assert res[field] == exp_result[field]
def test_post_price_data_invalid_unit(client, post_message):
"""
Try to post price data with the wrong unit, which should fail.
"""
# post price data
auth_token = get_auth_token(client, "*****@*****.**", "testtest")
post_price_data_response = client.post(
url_for("flexmeasures_api_v1_1.post_price_data"),
json=post_message,
headers={"Authorization": auth_token},
)
print("Server responded with:\n%s" % post_price_data_response.json)
assert post_price_data_response.status_code == 400
assert post_price_data_response.json["type"] == "PostPriceDataResponse"
market = parse_entity_address(post_message["market"], "market")
market_name = market["market_name"]
market = Market.query.filter_by(name=market_name).one_or_none()
assert (post_price_data_response.json["message"] == invalid_unit(
"%s prices" % market.display_name, ["EUR/MWh"])[0]["message"])
def get_generic_asset(asset_descriptor, entity_type):
"""
Get a generic asset from form information
# TODO: After refactoring, unify 3 generic_asset cases -> 1 sensor case
"""
ea = parse_entity_address(asset_descriptor, entity_type=entity_type)
if ea is None:
return None
if entity_type == "connection":
return Asset.query.filter(Asset.id == ea["asset_id"]).one_or_none()
elif entity_type == "market":
return Market.query.filter(
Market.name == ea["market_name"]).one_or_none()
elif entity_type == "sensor":
return get_weather_sensor_by(
ea["weather_sensor_type_name"],
ea["latitude"],
ea["longitude"],
)
return None
def post_price_data_response(
unit,
generic_asset_name_groups,
horizon,
rolling,
value_groups,
start,
duration,
resolution,
):
current_app.logger.info("POSTING PRICE DATA")
data_source = get_or_create_user_data_source(current_user)
prices = []
forecasting_jobs = []
for market_group, value_group in zip(generic_asset_name_groups,
value_groups):
for market in market_group:
# Parse the entity address
try:
ea = parse_entity_address(market, entity_type="market")
except EntityAddressException as eae:
return invalid_domain(str(eae))
market_name = ea["market_name"]
# Look for the Market object
market = Market.query.filter(
Market.name == market_name).one_or_none()
if market is None:
return unrecognized_market(market_name)
elif unit != market.unit:
return invalid_unit("%s prices" % market.display_name,
[market.unit])
# Create new Price objects
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - ((start + duration) -
(dt + duration / len(value_group)))
p = Price(
datetime=dt,
value=value,
horizon=h,
market_id=market.id,
data_source_id=data_source.id,
)
prices.append(p)
# Make forecasts, but not in play mode. Price forecasts (horizon>0) can still lead to other price forecasts,
# by the way, due to things like day-ahead markets.
if current_app.config.get("FLEXMEASURES_MODE", "") != "play":
# Forecast 24 and 48 hours ahead for at most the last 24 hours of posted price data
forecasting_jobs = create_forecasting_jobs(
"Price",
market.id,
max(start, start + duration - timedelta(hours=24)),
start + duration,
resolution=duration / len(value_group),
horizons=[timedelta(hours=24),
timedelta(hours=48)],
enqueue=
False, # will enqueue later, only if we successfully saved prices
)
# Put these into the database
current_app.logger.info("SAVING TO DB...")
try:
save_to_session(prices)
db.session.flush()
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
except IntegrityError as e:
current_app.logger.warning(e)
db.session.rollback()
# Allow price data to be replaced only in play mode
if current_app.config.get("FLEXMEASURES_MODE", "") == "play":
save_to_session(prices, overwrite=True)
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
else:
return already_received_and_successfully_processed()
def post_weather_data_response( # noqa: C901
unit,
generic_asset_name_groups,
horizon,
rolling,
value_groups,
start,
duration,
resolution,
):
current_app.logger.info("POSTING WEATHER DATA")
data_source = get_or_create_user_data_source(current_user)
weather_measurements = []
forecasting_jobs = []
for sensor_group, value_group in zip(generic_asset_name_groups,
value_groups):
for sensor in sensor_group:
# Parse the entity address
try:
ea = parse_entity_address(sensor, entity_type="sensor")
except EntityAddressException as eae:
return invalid_domain(str(eae))
weather_sensor_type_name = ea["weather_sensor_type_name"]
latitude = ea["latitude"]
longitude = ea["longitude"]
# Check whether the unit is valid for this sensor type (e.g. no m/s allowed for temperature data)
accepted_units = valid_sensor_units(weather_sensor_type_name)
if unit not in accepted_units:
return invalid_unit(weather_sensor_type_name, accepted_units)
weather_sensor = get_weather_sensor_by(weather_sensor_type_name,
latitude, longitude)
# Create new Weather objects
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - ((start + duration) -
(dt + duration / len(value_group)))
w = Weather(
datetime=dt,
value=value,
horizon=h,
sensor_id=weather_sensor.id,
data_source_id=data_source.id,
)
weather_measurements.append(w)
# make forecasts, but only if the sent-in values are not forecasts themselves (and also not in play)
if current_app.config.get(
"FLEXMEASURES_MODE", ""
) != "play" and horizon <= timedelta(
hours=0
): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this generic asset
forecasting_jobs.extend(
create_forecasting_jobs(
"Weather",
weather_sensor.id,
start,
start + duration,
resolution=duration / len(value_group),
horizons=[horizon],
enqueue=
False, # will enqueue later, only if we successfully saved weather measurements
))
# Put these into the database
current_app.logger.info("SAVING TO DB...")
try:
save_to_session(weather_measurements)
db.session.flush()
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
except IntegrityError as e:
current_app.logger.warning(e)
db.session.rollback()
# Allow meter data to be replaced only in play mode
if current_app.config.get("FLEXMEASURES_MODE", "") == "play":
save_to_session(weather_measurements, overwrite=True)
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
else:
return already_received_and_successfully_processed()
def get_device_message_response(generic_asset_name_groups, duration):
unit = "MW"
min_planning_horizon = timedelta(
hours=24
) # user can request a shorter planning, but the scheduler takes into account at least this horizon
planning_horizon = min(
max(min_planning_horizon, duration),
current_app.config.get("FLEXMEASURES_PLANNING_HORIZON"),
)
if not has_assets():
current_app.logger.info("User doesn't seem to have any assets.")
value_groups = []
new_event_groups = []
for event_group in generic_asset_name_groups:
for event in event_group:
# Parse the entity address
try:
ea = parse_entity_address(event,
entity_type="event",
fm_scheme="fm0")
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = ea["asset_id"]
event_id = ea["event_id"]
event_type = ea["event_type"]
# Look for the Sensor object
sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
if sensor is None or not can_access_asset(sensor):
current_app.logger.warning(
"Cannot identify sensor given the event %s." % event)
return unrecognized_connection_group()
if sensor.generic_asset.generic_asset_type.name != "battery":
return invalid_domain(
"API version 1.2 only supports device messages for batteries. "
"Sensor ID:%s does not belong to a battery." % sensor_id)
if event_type != "soc" or event_id != sensor.generic_asset.get_attribute(
"soc_udi_event_id"):
return unrecognized_event(event_id, event_type)
start = datetime.fromisoformat(
sensor.generic_asset.get_attribute("soc_datetime"))
resolution = sensor.event_resolution
# Schedule the asset
try:
schedule = schedule_battery(
sensor,
start,
start + planning_horizon,
resolution,
soc_at_start=sensor.generic_asset.get_attribute(
"soc_in_mwh"),
prefer_charging_sooner=False,
)
except UnknownPricesException:
return unknown_prices()
except UnknownMarketException:
return invalid_market()
else:
# Update the planning window
start = schedule.index[0]
duration = min(duration,
schedule.index[-1] + resolution - start)
schedule = schedule[start:start + duration - resolution]
value_groups.append(schedule.tolist())
new_event_groups.append(event)
response = groups_to_dict(new_event_groups,
value_groups,
generic_asset_type_name="event")
response["start"] = isodate.datetime_isoformat(start)
response["duration"] = isodate.duration_isoformat(duration)
response["unit"] = unit
d, s = request_processed()
return dict(**response, **d), s
def post_udi_event_response(unit): # noqa: C901
if not has_assets():
current_app.logger.info("User doesn't seem to have any assets.")
form = get_form_from_request(request)
# check datetime, or use server_now
if "datetime" not in form:
return invalid_datetime("Missing datetime parameter.")
else:
datetime = parse_isodate_str(form.get("datetime"))
if datetime is None:
return invalid_datetime(
"Cannot parse datetime string %s as iso date" %
form.get("datetime"))
if datetime.tzinfo is None:
current_app.logger.warning(
"Cannot parse timezone of 'datetime' value %s" %
form.get("datetime"))
return invalid_timezone(
"Datetime should explicitly state a timezone.")
# parse event/address info
if "event" not in form:
return invalid_domain("No event identifier sent.")
try:
ea = parse_entity_address(form.get("event"),
entity_type="event",
fm_scheme="fm0")
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = ea["asset_id"]
event_id = ea["event_id"]
event_type = ea["event_type"]
if event_type != "soc":
return unrecognized_event(event_id, event_type)
# Look for the Sensor object
sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
if sensor is None or not can_access_asset(sensor):
current_app.logger.warning("Cannot identify sensor via %s." % ea)
return unrecognized_connection_group()
if sensor.generic_asset.generic_asset_type.name != "battery":
return invalid_domain(
"API version 1.2 only supports UDI events for batteries. "
"Sensor ID:%s does not belong to a battery." % sensor_id)
# unless on play, keep events ordered by entry date and ID
if current_app.config.get("FLEXMEASURES_MODE") != "play":
# do not allow new date to be after last date
if (isinstance(sensor.generic_asset.get_attribute("soc_datetime"), str)
and datetime.fromisoformat(
sensor.generic_asset.get_attribute("soc_datetime")) >=
datetime):
msg = "The date of the requested UDI event (%s) is earlier than the latest known date (%s)." % (
datetime,
datetime.fromisoformat(
sensor.generic_asset.get_attribute("soc_datetime")),
)
current_app.logger.warning(msg)
return invalid_datetime(msg)
# check if udi event id is higher than existing
soc_udi_event_id = sensor.generic_asset.get_attribute(
"soc_udi_event_id")
if soc_udi_event_id is not None and soc_udi_event_id >= event_id:
return outdated_event_id(event_id, soc_udi_event_id)
# get value
if "value" not in form:
return ptus_incomplete()
value = form.get("value")
if unit == "kWh":
value = value / 1000.0
# Store new soc info as GenericAsset attributes
sensor.generic_asset.set_attribute("soc_datetime", datetime.isoformat())
sensor.generic_asset.set_attribute("soc_udi_event_id", event_id)
sensor.generic_asset.set_attribute("soc_in_mwh", value)
db.session.commit()
return request_processed("Request has been processed.")
def post_udi_event_response(unit): # noqa: C901
if not has_assets():
current_app.logger.info("User doesn't seem to have any assets.")
form = get_form_from_request(request)
# check datetime, or use server_now
if "datetime" not in form:
return invalid_datetime("Missing datetime parameter.")
else:
datetime = parse_isodate_str(form.get("datetime"))
if datetime is None:
return invalid_datetime(
"Cannot parse datetime string %s as iso date" % form.get("datetime")
)
if datetime.tzinfo is None:
current_app.logger.warning(
"Cannot parse timezone of 'datetime' value %s" % form.get("datetime")
)
return invalid_timezone("Datetime should explicitly state a timezone.")
# parse event/address info
if "event" not in form:
return invalid_domain("No event identifier sent.")
try:
ea = parse_entity_address(form.get("event"), entity_type="event")
except EntityAddressException as eae:
return invalid_domain(str(eae))
asset_id = ea["asset_id"]
event_id = ea["event_id"]
event_type = ea["event_type"]
if event_type != "soc":
return unrecognized_event(event_id, event_type)
# get asset
asset: Asset = Asset.query.filter_by(id=asset_id).one_or_none()
if asset is None or not can_access_asset(asset):
current_app.logger.warning("Cannot identify asset via %s." % ea)
return unrecognized_connection_group()
if asset.asset_type_name != "battery":
return invalid_domain(
"API version 1.2 only supports UDI events for batteries. Asset ID:%s is not a battery."
% asset_id
)
# unless on play, keep events ordered by entry date and ID
if current_app.config.get("FLEXMEASURES_MODE") != "play":
# do not allow new date to be after last date
if asset.soc_datetime is not None:
if asset.soc_datetime >= datetime:
msg = (
"The date of the requested UDI event (%s) is earlier than the latest known date (%s)."
% (datetime, asset.soc_datetime)
)
current_app.logger.warning(msg)
return invalid_datetime(msg)
# check if udi event id is higher than existing
if asset.soc_udi_event_id is not None:
if asset.soc_udi_event_id >= event_id:
return outdated_event_id(event_id, asset.soc_udi_event_id)
# get value
if "value" not in form:
return ptus_incomplete()
value = form.get("value")
if unit == "kWh":
value = value / 1000.0
# store new soc in asset
asset.soc_datetime = datetime
asset.soc_udi_event_id = event_id
asset.soc_in_mwh = value
db.session.commit()
return request_processed("Request has been processed.")
def post_weather_data_response( # noqa: C901
unit,
generic_asset_name_groups,
horizon,
rolling,
value_groups,
start,
duration,
resolution,
):
current_app.logger.info("POSTING WEATHER DATA")
data_source = get_or_create_source(current_user)
weather_df_per_sensor = []
forecasting_jobs = []
for sensor_group, value_group in zip(generic_asset_name_groups, value_groups):
for sensor in sensor_group:
# Parse the entity address
try:
ea = parse_entity_address(
sensor, entity_type="weather_sensor", fm_scheme="fm0"
)
except EntityAddressException as eae:
return invalid_domain(str(eae))
weather_sensor_type_name = ea["weather_sensor_type_name"]
latitude = ea["latitude"]
longitude = ea["longitude"]
# Check whether the unit is valid for this sensor type (e.g. no m/s allowed for temperature data)
accepted_units = valid_sensor_units(weather_sensor_type_name)
if unit not in accepted_units:
return invalid_unit(weather_sensor_type_name, accepted_units)
sensor = get_sensor_by_generic_asset_type_and_location(
weather_sensor_type_name, latitude, longitude
)
if is_response_tuple(sensor):
# Error message telling the user about the nearest weather sensor they can post to
return sensor
# Create new Weather objects
beliefs = []
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - (
(start + duration) - (dt + duration / len(value_group))
)
w = TimedBelief(
event_start=dt,
event_value=value,
belief_horizon=h,
sensor=sensor,
source=data_source,
)
beliefs.append(w)
weather_df_per_sensor.append(tb.BeliefsDataFrame(beliefs))
# make forecasts, but only if the sent-in values are not forecasts themselves (and also not in play)
if current_app.config.get(
"FLEXMEASURES_MODE", ""
) != "play" and horizon <= timedelta(
hours=0
): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this sensor
forecasting_jobs.extend(
create_forecasting_jobs(
sensor.id,
start,
start + duration,
resolution=duration / len(value_group),
enqueue=False, # will enqueue later, after saving data
)
)
return save_and_enqueue(weather_df_per_sensor, forecasting_jobs)
def post_weather_data_response( # noqa: C901
unit,
generic_asset_name_groups,
horizon,
prior,
value_groups,
start,
duration,
resolution,
) -> ResponseTuple:
# additional validation, todo: to be moved into Marshmallow
if horizon is None and prior is None:
extra_info = "Missing horizon or prior."
return invalid_horizon(extra_info)
current_app.logger.info("POSTING WEATHER DATA")
data_source = get_or_create_source(current_user)
weather_df_per_sensor = []
forecasting_jobs = []
for sensor_group, event_values in zip(generic_asset_name_groups, value_groups):
for sensor in sensor_group:
# Parse the entity address
try:
ea = parse_entity_address(sensor, entity_type="weather_sensor")
except EntityAddressException as eae:
return invalid_domain(str(eae))
weather_sensor_type_name = ea["weather_sensor_type_name"]
latitude = ea["latitude"]
longitude = ea["longitude"]
# Check whether the unit is valid for this sensor type (e.g. no m/s allowed for temperature data)
accepted_units = valid_sensor_units(weather_sensor_type_name)
if unit not in accepted_units:
return invalid_unit(weather_sensor_type_name, accepted_units)
sensor: Sensor = get_sensor_by_generic_asset_type_and_location(
weather_sensor_type_name, latitude, longitude
)
# Convert to timely-beliefs terminology
event_starts, belief_horizons = determine_belief_timing(
event_values, start, resolution, horizon, prior, sensor
)
# Create new Weather objects
beliefs = [
TimedBelief(
event_start=event_start,
event_value=event_value,
belief_horizon=belief_horizon,
sensor=sensor,
source=data_source,
)
for event_start, event_value, belief_horizon in zip(
event_starts, event_values, belief_horizons
)
]
weather_df_per_sensor.append(tb.BeliefsDataFrame(beliefs))
# make forecasts, but only if the sent-in values are not forecasts themselves (and also not in play)
if current_app.config.get(
"FLEXMEASURES_MODE", ""
) != "play" and horizon <= timedelta(
hours=0
): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this generic asset
forecasting_jobs.extend(
create_forecasting_jobs(
sensor.id,
start,
start + duration,
resolution=duration / len(event_values),
horizons=[horizon],
enqueue=False, # will enqueue later, after saving data
)
)
return save_and_enqueue(weather_df_per_sensor, forecasting_jobs)
def create_connection_and_value_groups( # noqa: C901
unit, generic_asset_name_groups, value_groups, horizon, rolling, start,
duration):
"""
Code for POSTing Power values to the API.
Only lets users post to assets they own.
The sign of values is validated according to asset specs, but in USEF terms.
Then, we store the reverse sign for FlexMeasures specs (with positive production
and negative consumption).
If power values are not forecasts, forecasting jobs are created.
"""
from flask import current_app
current_app.logger.info("POSTING POWER DATA")
data_source = get_or_create_user_data_source(current_user)
user_assets = get_assets()
if not user_assets:
current_app.logger.info("User doesn't seem to have any assets")
user_asset_ids = [asset.id for asset in user_assets]
power_measurements = []
forecasting_jobs = []
for connection_group, value_group in zip(generic_asset_name_groups,
value_groups):
for connection in connection_group:
# TODO: get asset through util function after refactoring
# Parse the entity address
try:
connection = parse_entity_address(connection,
entity_type="connection")
except EntityAddressException as eae:
return invalid_domain(str(eae))
asset_id = connection["asset_id"]
# Look for the Asset object
if asset_id in user_asset_ids:
asset = Asset.query.filter(Asset.id == asset_id).one_or_none()
else:
current_app.logger.warning("Cannot identify connection %s" %
connection)
return unrecognized_connection_group()
# Validate the sign of the values (following USEF specs with positive consumption and negative production)
if asset.is_pure_consumer and any(v < 0 for v in value_group):
extra_info = (
"Connection %s is registered as a pure consumer and can only receive non-negative values."
% asset.entity_address)
return power_value_too_small(extra_info)
elif asset.is_pure_producer and any(v > 0 for v in value_group):
extra_info = (
"Connection %s is registered as a pure producer and can only receive non-positive values."
% asset.entity_address)
return power_value_too_big(extra_info)
# Create new Power objects
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - ((start + duration) -
(dt + duration / len(value_group)))
p = Power(
datetime=dt,
value=value *
-1, # Reverse sign for FlexMeasures specs with positive production and negative consumption
horizon=h,
asset_id=asset.id,
data_source_id=data_source.id,
)
power_measurements.append(p)
# make forecasts, but only if the sent-in values are not forecasts themselves
if horizon <= timedelta(
hours=0
): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this generic asset
forecasting_jobs.extend(
create_forecasting_jobs(
"Power",
asset_id,
start,
start + duration,
resolution=duration / len(value_group),
enqueue=False,
))
current_app.logger.info("SAVING TO DB AND QUEUEING...")
try:
save_to_session(power_measurements)
db.session.flush()
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
except IntegrityError as e:
current_app.logger.warning(e)
db.session.rollback()
# Allow meter data to be replaced only in play mode
if current_app.config.get("FLEXMEASURES_MODE", "") == "play":
save_to_session(power_measurements, overwrite=True)
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
else:
return already_received_and_successfully_processed()
def collect_connection_and_value_groups(
unit: str,
resolution: str,
belief_horizon_window: Tuple[Union[None, timedelta], Union[None,
timedelta]],
belief_time_window: Tuple[Optional[datetime_type],
Optional[datetime_type]],
start: datetime_type,
duration: timedelta,
connection_groups: List[List[str]],
user_source_ids: Union[
int, List[int]] = None, # None is interpreted as all sources
source_types: List[str] = None,
) -> Tuple[dict, int]:
"""
Code for GETting power values from the API.
Only allows to get values from assets owned by current user.
Returns value sign in accordance with USEF specs
(with negative production and positive consumption).
"""
from flask import current_app
current_app.logger.info("GETTING")
user_assets = get_assets()
if not user_assets:
current_app.logger.info("User doesn't seem to have any assets")
user_asset_ids = [asset.id for asset in user_assets]
end = start + duration
value_groups = []
new_connection_groups = (
[]
) # Each connection in the old connection groups will be interpreted as a separate group
for connections in connection_groups:
# Get the asset names
asset_names: List[str] = []
for connection in connections:
# Parse the entity address
try:
connection_details = parse_entity_address(
connection, entity_type="connection")
except EntityAddressException as eae:
return invalid_domain(str(eae))
asset_id = connection_details["asset_id"]
# Look for the Asset object
if asset_id in user_asset_ids:
asset = Asset.query.filter(Asset.id == asset_id).one_or_none()
else:
current_app.logger.warning("Cannot identify connection %s" %
connection)
return unrecognized_connection_group()
asset_names.append(asset.name)
# Get the power values
# TODO: fill NaN for non-existing values
power_bdf_dict: Dict[str, tb.BeliefsDataFrame] = Power.collect(
generic_asset_names=asset_names,
query_window=(start, end),
resolution=resolution,
belief_horizon_window=belief_horizon_window,
belief_time_window=belief_time_window,
user_source_ids=user_source_ids,
source_types=source_types,
sum_multiple=False,
)
# Todo: parse time window of power_bdf_dict, which will be different for requests that are not of the form:
# - start is a timestamp on the hour or a multiple of 15 minutes thereafter
# - duration is a multiple of 15 minutes
for k, bdf in power_bdf_dict.items():
value_groups.append(
[x * -1 for x in bdf["event_value"].tolist()]
) # Reverse sign of values (from FlexMeasures specs to USEF specs)
new_connection_groups.append(k)
response = groups_to_dict(new_connection_groups,
value_groups,
generic_asset_type_name="connection")
response["start"] = isodate.datetime_isoformat(start)
response["duration"] = isodate.duration_isoformat(duration)
response["unit"] = unit # TODO: convert to requested unit
d, s = request_processed()
return dict(**response, **d), s
def post_udi_event_response(unit: str, prior: datetime):
if not has_assets():
current_app.logger.info("User doesn't seem to have any assets.")
form = get_form_from_request(request)
if "datetime" not in form:
return invalid_datetime("Missing datetime parameter.")
else:
datetime = parse_isodate_str(form.get("datetime"))
if datetime is None:
return invalid_datetime(
"Cannot parse datetime string %s as iso date" %
form.get("datetime"))
if datetime.tzinfo is None:
current_app.logger.warning(
"Cannot parse timezone of 'datetime' value %s" %
form.get("datetime"))
return invalid_timezone(
"Datetime should explicitly state a timezone.")
# parse event/address info
if "event" not in form:
return invalid_domain("No event identifier sent.")
try:
ea = parse_entity_address(form.get("event"),
entity_type="event",
fm_scheme="fm0")
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = ea["asset_id"]
event_id = ea["event_id"]
event_type = ea["event_type"]
if event_type not in ("soc", "soc-with-targets"):
return unrecognized_event_type(event_type)
# Look for the Sensor object
sensor = Sensor.query.filter_by(id=sensor_id).one_or_none()
if sensor is None or not can_access_asset(sensor):
current_app.logger.warning("Cannot identify sensor via %s." % ea)
return unrecognized_connection_group()
if sensor.generic_asset.generic_asset_type.name not in (
"battery",
"one-way_evse",
"two-way_evse",
):
return invalid_domain(
f"API version 1.3 only supports UDI events for batteries and Electric Vehicle Supply Equipment (EVSE). "
f"Sensor ID:{sensor_id} does not belong to a battery or EVSE, but {p.a(sensor.generic_asset.generic_asset_type.description)}."
)
# unless on play, keep events ordered by entry date and ID
if current_app.config.get("FLEXMEASURES_MODE") != "play":
# do not allow new date to precede previous date
if isinstance(sensor.generic_asset.get_attribute("soc_datetime"),
str) and datetime < datetime.fromisoformat(
sensor.generic_asset.get_attribute("soc_datetime")):
msg = "The date of the requested UDI event (%s) is earlier than the latest known date (%s)." % (
datetime,
datetime.fromisoformat(
sensor.generic_asset.get_attribute("soc_datetime")),
)
current_app.logger.warning(msg)
return invalid_datetime(msg)
# check if udi event id is higher than existing
if sensor.generic_asset.get_attribute("soc_udi_event_id") is not None:
if sensor.generic_asset.get_attribute(
"soc_udi_event_id") >= event_id:
return outdated_event_id(
event_id,
sensor.generic_asset.get_attribute("soc_udi_event_id"))
# get value
if "value" not in form:
return ptus_incomplete()
try:
value = float(form.get("value"))
except ValueError:
extra_info = "Request includes empty or ill-formatted value(s)."
current_app.logger.warning(extra_info)
return ptus_incomplete(extra_info)
if unit == "kWh":
value = value / 1000.0
# get optional efficiency
roundtrip_efficiency = form.get("roundtrip_efficiency", None)
# get optional min and max SOC
soc_min = form.get("soc_min", None)
soc_max = form.get("soc_max", None)
if soc_min is not None and unit == "kWh":
soc_min = soc_min / 1000.0
if soc_max is not None and unit == "kWh":
soc_max = soc_max / 1000.0
# set soc targets
start_of_schedule = datetime
end_of_schedule = datetime + current_app.config.get(
"FLEXMEASURES_PLANNING_HORIZON")
resolution = sensor.event_resolution
soc_targets = pd.Series(
np.nan,
index=pd.date_range(
start_of_schedule,
end_of_schedule,
freq=resolution,
closed="right"
), # note that target values are indexed by their due date (i.e. closed="right")
)
if event_type == "soc-with-targets":
if "targets" not in form:
return incomplete_event(
event_id,
event_type,
"Cannot process event %s with missing targets." %
form.get("event"),
)
for target in form.get("targets"):
# get target value
if "value" not in target:
return ptus_incomplete("Target missing value parameter.")
try:
target_value = float(target["value"])
except ValueError:
extra_info = "Request includes empty or ill-formatted target value(s)."
current_app.logger.warning(extra_info)
return ptus_incomplete(extra_info)
if unit == "kWh":
target_value = target_value / 1000.0
# get target datetime
if "datetime" not in target:
return invalid_datetime("Target missing datetime parameter.")
else:
target_datetime = parse_isodate_str(target["datetime"])
if target_datetime is None:
return invalid_datetime(
"Cannot parse target datetime string %s as iso date" %
target["datetime"])
if target_datetime.tzinfo is None:
current_app.logger.warning(
"Cannot parse timezone of target 'datetime' value %s" %
target["datetime"])
return invalid_timezone(
"Target datetime should explicitly state a timezone.")
if target_datetime > end_of_schedule:
return invalid_datetime(
f'Target datetime exceeds {end_of_schedule}. Maximum scheduling horizon is {current_app.config.get("FLEXMEASURES_PLANNING_HORIZON")}.'
)
target_datetime = target_datetime.astimezone(
soc_targets.index.tzinfo
) # otherwise DST would be problematic
# set target
soc_targets.loc[target_datetime] = target_value
create_scheduling_job(
sensor_id,
start_of_schedule,
end_of_schedule,
resolution=resolution,
belief_time=prior, # server time if no prior time was sent
soc_at_start=value,
soc_targets=soc_targets,
soc_min=soc_min,
soc_max=soc_max,
roundtrip_efficiency=roundtrip_efficiency,
job_id=form.get("event"),
enqueue=True,
)
# Store new soc info as GenericAsset attributes
sensor.generic_asset.set_attribute("soc_datetime", datetime.isoformat())
sensor.generic_asset.set_attribute("soc_udi_event_id", event_id)
sensor.generic_asset.set_attribute("soc_in_mwh", value)
db.session.commit()
return request_processed()
def post_power_data(
unit,
generic_asset_name_groups,
value_groups,
horizon,
prior,
start,
duration,
resolution,
create_forecasting_jobs_too,
):
# additional validation, todo: to be moved into Marshmallow
if horizon is None and prior is None:
extra_info = "Missing horizon or prior."
return invalid_horizon(extra_info)
current_app.logger.info("POSTING POWER DATA")
data_source = get_or_create_source(current_user)
user_sensors = get_sensors()
if not user_sensors:
current_app.logger.info("User doesn't seem to have any assets")
user_sensor_ids = [sensor.id for sensor in user_sensors]
power_df_per_connection = []
forecasting_jobs = []
for connection_group, event_values in zip(generic_asset_name_groups, value_groups):
for connection in connection_group:
# TODO: get asset through util function after refactoring
# Parse the entity address
try:
ea = parse_entity_address(connection, entity_type="connection")
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = ea["sensor_id"]
# Look for the Sensor object
if sensor_id in user_sensor_ids:
sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
else:
current_app.logger.warning("Cannot identify connection %s" % connection)
return unrecognized_connection_group()
# Validate the sign of the values (following USEF specs with positive consumption and negative production)
if sensor.get_attribute("is_strictly_non_positive") and any(
v < 0 for v in event_values
):
extra_info = (
"Connection %s is registered as a pure consumer and can only receive non-negative values."
% sensor.entity_address
)
return power_value_too_small(extra_info)
elif sensor.get_attribute("is_strictly_non_negative") and any(
v > 0 for v in event_values
):
extra_info = (
"Connection %s is registered as a pure producer and can only receive non-positive values."
% sensor.entity_address
)
return power_value_too_big(extra_info)
# Convert to timely-beliefs terminology
event_starts, belief_horizons = determine_belief_timing(
event_values, start, resolution, horizon, prior, sensor
)
# Create new Power objects
beliefs = [
TimedBelief(
event_start=event_start,
event_value=event_value
* -1, # Reverse sign for FlexMeasures specs with positive production and negative consumption
belief_horizon=belief_horizon,
sensor=sensor,
source=data_source,
)
for event_start, event_value, belief_horizon in zip(
event_starts, event_values, belief_horizons
)
]
power_df_per_connection.append(tb.BeliefsDataFrame(beliefs))
if create_forecasting_jobs_too:
forecasting_jobs.extend(
create_forecasting_jobs(
sensor_id,
start,
start + duration,
resolution=duration / len(event_values),
enqueue=False, # will enqueue later, after saving data
)
)
return save_and_enqueue(power_df_per_connection, forecasting_jobs)
def get_device_message_response(generic_asset_name_groups, duration):
unit = "MW"
planning_horizon = min(
duration, current_app.config.get("FLEXMEASURES_PLANNING_HORIZON"))
if not has_assets():
current_app.logger.info("User doesn't seem to have any assets.")
value_groups = []
new_event_groups = []
for event_group in generic_asset_name_groups:
for event in event_group:
# Parse the entity address
try:
ea = parse_entity_address(event,
entity_type="event",
fm_scheme="fm0")
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = ea["asset_id"]
event_id = ea["event_id"]
event_type = ea["event_type"]
# Look for the Sensor object
sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
if sensor is None or not can_access_asset(sensor):
current_app.logger.warning(
"Cannot identify sensor given the event %s." % event)
return unrecognized_connection_group()
if sensor.generic_asset.generic_asset_type.name not in (
"battery",
"one-way_evse",
"two-way_evse",
):
return invalid_domain(
f"API version 1.3 only supports device messages for batteries and Electric Vehicle Supply Equipment (EVSE). "
f"Sensor ID:{sensor_id} does not belong to a battery or EVSE, but {p.a(sensor.generic_asset.generic_asset_type.description)}."
)
# Use the event_id to look up the schedule start
if event_type not in ("soc", "soc-with-targets"):
return unrecognized_event_type(event_type)
connection = current_app.queues["scheduling"].connection
try: # First try the scheduling queue
job = Job.fetch(event, connection=connection)
except NoSuchJobError: # Then try the most recent event_id (stored as a generic asset attribute)
if event_id == sensor.generic_asset.get_attribute(
"soc_udi_event_id"):
schedule_start = datetime.fromisoformat(
sensor.generic_asset.get_attribute("soc_datetime"))
message = (
"Your UDI event is the most recent event for this device, but "
)
else:
return unrecognized_event(event_id, event_type)
else:
if job.is_finished:
message = "A scheduling job has been processed based on your UDI event, but "
elif job.is_failed: # Try to inform the user on why the job failed
e = job.meta.get(
"exception",
Exception(
"The job does not state why it failed. "
"The worker may be missing an exception handler, "
"or its exception handler is not storing the exception as job meta data."
),
)
return unknown_schedule(
f"Scheduling job failed with {type(e).__name__}: {e}")
elif job.is_started:
return unknown_schedule("Scheduling job in progress.")
elif job.is_queued:
return unknown_schedule(
"Scheduling job waiting to be processed.")
elif job.is_deferred:
try:
preferred_job = job.dependency
except NoSuchJobError:
return unknown_schedule(
"Scheduling job waiting for unknown job to be processed."
)
return unknown_schedule(
f'Scheduling job waiting for {preferred_job.status} job "{preferred_job.id}" to be processed.'
)
else:
return unknown_schedule(
"Scheduling job has an unknown status.")
schedule_start = job.kwargs["start"]
schedule_data_source_name = "Seita"
scheduler_source = DataSource.query.filter_by(
name="Seita", type="scheduling script").one_or_none()
if scheduler_source is None:
return unknown_schedule(
message +
f'no data is known from "{schedule_data_source_name}".')
power_values = sensor.search_beliefs(
event_starts_after=schedule_start,
event_ends_before=schedule_start + planning_horizon,
source=scheduler_source,
most_recent_beliefs_only=True,
one_deterministic_belief_per_event=True,
)
# For consumption schedules, positive values denote consumption. For the db, consumption is negative
consumption_schedule = -simplify_index(power_values)["event_value"]
if consumption_schedule.empty:
return unknown_schedule(
message + "the schedule was not found in the database.")
# Update the planning window
resolution = sensor.event_resolution
start = consumption_schedule.index[0]
duration = min(duration,
consumption_schedule.index[-1] + resolution - start)
consumption_schedule = consumption_schedule[start:start +
duration - resolution]
value_groups.append(consumption_schedule.tolist())
new_event_groups.append(event)
response = groups_to_dict(new_event_groups,
value_groups,
generic_asset_type_name="event")
response["start"] = isodate.datetime_isoformat(start)
response["duration"] = isodate.duration_isoformat(duration)
response["unit"] = unit
d, s = request_processed()
return dict(**response, **d), s
def post_price_data_response( # noqa C901
unit,
generic_asset_name_groups,
horizon,
prior,
value_groups,
start,
duration,
resolution,
) -> ResponseTuple:
# additional validation, todo: to be moved into Marshmallow
if horizon is None and prior is None:
extra_info = "Missing horizon or prior."
return invalid_horizon(extra_info)
current_app.logger.info("POSTING PRICE DATA")
data_source = get_or_create_source(current_user)
price_df_per_market = []
forecasting_jobs = []
for market_group, event_values in zip(generic_asset_name_groups, value_groups):
for market in market_group:
# Parse the entity address
try:
ea = parse_entity_address(market, entity_type="market")
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = ea["sensor_id"]
# Look for the Sensor object
sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
if sensor is None:
return unrecognized_market(sensor_id)
elif unit != sensor.unit:
return invalid_unit("%s prices" % sensor.name, [sensor.unit])
# Convert to timely-beliefs terminology
event_starts, belief_horizons = determine_belief_timing(
event_values, start, resolution, horizon, prior, sensor
)
# Create new Price objects
beliefs = [
TimedBelief(
event_start=event_start,
event_value=event_value,
belief_horizon=belief_horizon,
sensor=sensor,
source=data_source,
)
for event_start, event_value, belief_horizon in zip(
event_starts, event_values, belief_horizons
)
]
price_df_per_market.append(tb.BeliefsDataFrame(beliefs))
# Make forecasts, but not in play mode. Price forecasts (horizon>0) can still lead to other price forecasts,
# by the way, due to things like day-ahead markets.
if current_app.config.get("FLEXMEASURES_MODE", "") != "play":
# Forecast 24 and 48 hours ahead for at most the last 24 hours of posted price data
forecasting_jobs = create_forecasting_jobs(
sensor.id,
max(start, start + duration - timedelta(hours=24)),
start + duration,
resolution=duration / len(event_values),
horizons=[timedelta(hours=24), timedelta(hours=48)],
enqueue=False, # will enqueue later, after saving data
)
return save_and_enqueue(price_df_per_market, forecasting_jobs)
def post_price_data_response(
unit,
generic_asset_name_groups,
horizon,
rolling,
value_groups,
start,
duration,
resolution,
):
current_app.logger.info("POSTING PRICE DATA")
data_source = get_or_create_source(current_user)
price_df_per_market = []
forecasting_jobs = []
for market_group, value_group in zip(generic_asset_name_groups, value_groups):
for market in market_group:
# Parse the entity address
try:
ea = parse_entity_address(market, entity_type="market", fm_scheme="fm0")
except EntityAddressException as eae:
return invalid_domain(str(eae))
market_name = ea["market_name"]
# Look for the Sensor object
sensor = get_sensor_by_unique_name(market_name, ["day_ahead", "tou_tariff"])
if is_response_tuple(sensor):
# Error message telling the user what to do
return sensor
if unit != sensor.unit:
return invalid_unit("%s prices" % sensor.name, [sensor.unit])
# Create new Price objects
beliefs = []
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - (
(start + duration) - (dt + duration / len(value_group))
)
p = TimedBelief(
event_start=dt,
event_value=value,
belief_horizon=h,
sensor=sensor,
source=data_source,
)
beliefs.append(p)
price_df_per_market.append(tb.BeliefsDataFrame(beliefs))
# Make forecasts, but not in play mode. Price forecasts (horizon>0) can still lead to other price forecasts,
# by the way, due to things like day-ahead markets.
if current_app.config.get("FLEXMEASURES_MODE", "") != "play":
# Forecast 24 and 48 hours ahead for at most the last 24 hours of posted price data
forecasting_jobs = create_forecasting_jobs(
sensor.id,
max(start, start + duration - timedelta(hours=24)),
start + duration,
resolution=duration / len(value_group),
horizons=[timedelta(hours=24), timedelta(hours=48)],
enqueue=False, # will enqueue later, after saving data
)
return save_and_enqueue(price_df_per_market, forecasting_jobs)
def create_connection_and_value_groups( # noqa: C901
unit, generic_asset_name_groups, value_groups, horizon, rolling, start, duration
):
"""
Code for POSTing Power values to the API.
Only lets users post to assets they own.
The sign of values is validated according to asset specs, but in USEF terms.
Then, we store the reverse sign for FlexMeasures specs (with positive production
and negative consumption).
If power values are not forecasts, forecasting jobs are created.
"""
current_app.logger.info("POSTING POWER DATA")
data_source = get_or_create_source(current_user)
user_sensors = get_sensors()
if not user_sensors:
current_app.logger.info("User doesn't seem to have any assets")
user_sensor_ids = [sensor.id for sensor in user_sensors]
power_df_per_connection = []
forecasting_jobs = []
for connection_group, value_group in zip(generic_asset_name_groups, value_groups):
for connection in connection_group:
# TODO: get asset through util function after refactoring
# Parse the entity address
try:
connection = parse_entity_address(
connection, entity_type="connection", fm_scheme="fm0"
)
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = connection["asset_id"]
# Look for the Sensor object
if sensor_id in user_sensor_ids:
sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
else:
current_app.logger.warning("Cannot identify connection %s" % connection)
return unrecognized_connection_group()
# Validate the sign of the values (following USEF specs with positive consumption and negative production)
if sensor.get_attribute("is_strictly_non_positive") and any(
v < 0 for v in value_group
):
extra_info = (
"Connection %s is registered as a pure consumer and can only receive non-negative values."
% sensor.entity_address
)
return power_value_too_small(extra_info)
elif sensor.get_attribute("is_strictly_non_negative") and any(
v > 0 for v in value_group
):
extra_info = (
"Connection %s is registered as a pure producer and can only receive non-positive values."
% sensor.entity_address
)
return power_value_too_big(extra_info)
# Create a new BeliefsDataFrame
beliefs = []
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - (
(start + duration) - (dt + duration / len(value_group))
)
p = TimedBelief(
event_start=dt,
event_value=value
* -1, # Reverse sign for FlexMeasures specs with positive production and negative consumption
belief_horizon=h,
sensor=sensor,
source=data_source,
)
assert p not in db.session
beliefs.append(p)
power_df_per_connection.append(tb.BeliefsDataFrame(beliefs))
# make forecasts, but only if the sent-in values are not forecasts themselves
if horizon <= timedelta(
hours=0
): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this sensor
forecasting_jobs.extend(
create_forecasting_jobs(
sensor_id,
start,
start + duration,
resolution=duration / len(value_group),
enqueue=False, # will enqueue later, after saving data
)
)
return save_and_enqueue(power_df_per_connection, forecasting_jobs)
