def save_and_enqueue(
data: Union[BeliefsDataFrame, List[BeliefsDataFrame]],
forecasting_jobs: List[Job] = None,
save_changed_beliefs_only: bool = True,
) -> ResponseTuple:
# Attempt to save
status = modern_save_to_db(
data, save_changed_beliefs_only=save_changed_beliefs_only
)
db.session.commit()
# Only enqueue forecasting jobs upon successfully saving new data
if status[:7] == "success" and status != "success_but_nothing_new":
enqueue_forecasting_jobs(forecasting_jobs)
# Pick a response
if status == "success":
return request_processed()
elif status in (
"success_with_unchanged_beliefs_skipped",
"success_but_nothing_new",
):
return already_received_and_successfully_processed()
return invalid_replacement()
def test_get_service(client, query):
get_service_response = client.get(
url_for("flexmeasures_api_v1.get_service"),
query_string=query,
headers={"content-type": "application/json"},
)
print("Server responded with:\n%s" % get_service_response.json)
assert get_service_response.status_code == 200
assert get_service_response.json["type"] == "GetServiceResponse"
assert get_service_response.json["status"] == request_processed()[0]["status"]
if "access" in query:
for service in get_service_response.json["services"]:
assert "Prosumer" in service["access"]
def restore_data_response():
delete_structure = True
delete_data = True
restore_structure = True
restore_data = True
try:
backup_name = request.args.get("backup", request.json["backup"])
except KeyError:
return no_backup()
# Make sure backup folder and files exist before restoring
backup_path = app.config.get("FLEXMEASURES_DB_BACKUP_PATH")
if (not Path("%s/%s" % (backup_path, backup_name)).exists()
or not Path("%s/%s" % (backup_path, backup_name)).is_dir()):
return unrecognized_backup()
affected_classes = get_affected_classes(structure=restore_structure,
data=restore_data)
for c in affected_classes:
file_path = "%s/%s/%s.obj" % (backup_path, backup_name,
c.__tablename__)
if not Path(file_path).exists():
return unrecognized_backup(
"Can't load table, because filename %s does not exist." %
c.__tablename__)
# Reset in play mode only (this endpoint should not have been registered otherwise)
assert app.config.get("FLEXMEASURES_MODE", "") == "play"
if delete_data:
depopulate_forecasts(db)
depopulate_data(db)
if delete_structure:
depopulate_structure(db)
# Load backup
load_tables(
db,
backup_name,
structure=restore_structure,
data=restore_data,
backup_path=backup_path,
)
return request_processed("Database restored to %s." % backup_name)
def get_service_response(service_listing) -> Union[dict, Tuple[dict, int]]:
"""
Lists the available services for the public endpoint version,
either all of them or only those that apply to the requested access role.
"""
requested_access_role = request.args.get("access")
response = {"version": service_listing["version"]}
if requested_access_role:
accessible_services = []
for service in service_listing["services"]:
if requested_access_role in service["access"]:
accessible_services.append(service)
response["services"] = accessible_services
if not accessible_services:
return invalid_role(requested_access_role)
else:
response["services"] = service_listing["services"]
d, s = request_processed()
return dict(**response, **d), s
def get_schedule(self, sensor: Sensor, job_id: str, duration: timedelta,
**kwargs):
"""Get a schedule from FlexMeasures.
.. :quickref: Schedule; Download schedule from the platform
**Optional fields**
- "duration" (6 hours by default; can be increased to plan further into the future)
**Example response**
This message contains a schedule indicating to consume at various power
rates from 10am UTC onwards for a duration of 45 minutes.
.. sourcecode:: json
{
"values": [
2.15,
3,
2
],
"start": "2015-06-02T10:00:00+00:00",
"duration": "PT45M",
"unit": "MW"
}
:reqheader Authorization: The authentication token
:reqheader Content-Type: application/json
:resheader Content-Type: application/json
:status 200: PROCESSED
:status 400: INVALID_TIMEZONE, INVALID_DOMAIN, INVALID_UNIT, UNKNOWN_SCHEDULE, UNRECOGNIZED_CONNECTION_GROUP
:status 401: UNAUTHORIZED
:status 403: INVALID_SENDER
:status 405: INVALID_METHOD
:status 422: UNPROCESSABLE_ENTITY
"""
planning_horizon = min( # type: ignore
duration, current_app.config.get("FLEXMEASURES_PLANNING_HORIZON"))
# Look up the scheduling job
connection = current_app.queues["scheduling"].connection
try: # First try the scheduling queue
job = Job.fetch(job_id, connection=connection)
except NoSuchJobError:
return unrecognized_event(job_id, "job")
if job.is_finished:
error_message = "A scheduling job has been processed with your job ID, but "
elif job.is_failed: # Try to inform the user on why the job failed
e = job.meta.get(
"exception",
Exception(
"The job does not state why it failed. "
"The worker may be missing an exception handler, "
"or its exception handler is not storing the exception as job meta data."
),
)
return unknown_schedule(
f"Scheduling job failed with {type(e).__name__}: {e}")
elif job.is_started:
return unknown_schedule("Scheduling job in progress.")
elif job.is_queued:
return unknown_schedule("Scheduling job waiting to be processed.")
elif job.is_deferred:
try:
preferred_job = job.dependency
except NoSuchJobError:
return unknown_schedule(
"Scheduling job waiting for unknown job to be processed.")
return unknown_schedule(
f'Scheduling job waiting for {preferred_job.status} job "{preferred_job.id}" to be processed.'
)
else:
return unknown_schedule("Scheduling job has an unknown status.")
schedule_start = job.kwargs["start"]
schedule_data_source_name = "Seita"
scheduler_source = DataSource.query.filter_by(
name="Seita", type="scheduling script").one_or_none()
if scheduler_source is None:
return unknown_schedule(
error_message +
f'no data is known from "{schedule_data_source_name}".')
power_values = sensor.search_beliefs(
event_starts_after=schedule_start,
event_ends_before=schedule_start + planning_horizon,
source=scheduler_source,
most_recent_beliefs_only=True,
one_deterministic_belief_per_event=True,
)
# For consumption schedules, positive values denote consumption. For the db, consumption is negative
consumption_schedule = -simplify_index(power_values)["event_value"]
if consumption_schedule.empty:
return unknown_schedule(
error_message + "the schedule was not found in the database.")
# Update the planning window
resolution = sensor.event_resolution
start = consumption_schedule.index[0]
duration = min(duration,
consumption_schedule.index[-1] + resolution - start)
consumption_schedule = consumption_schedule[start:start + duration -
resolution]
response = dict(
values=consumption_schedule.tolist(),
start=isodate.datetime_isoformat(start),
duration=isodate.duration_isoformat(duration),
unit=sensor.unit,
)
d, s = request_processed()
return dict(**response, **d), s
def trigger_schedule( # noqa: C901
self,
sensor: Sensor,
start_of_schedule: datetime,
unit: str,
prior: datetime,
roundtrip_efficiency: Optional[ur.Quantity] = None,
**kwargs,
):
"""
Trigger FlexMeasures to create a schedule.
.. :quickref: Schedule; Trigger scheduling job
The message should contain a flexibility model.
**Example request A**
This message triggers a schedule starting at 10.00am, at which the state of charge (soc) is 12.1 kWh.
.. code-block:: json
{
"start": "2015-06-02T10:00:00+00:00",
"soc-at-start": 12.1,
"soc-unit": "kWh"
}
**Example request B**
This message triggers a schedule starting at 10.00am, at which the state of charge (soc) is 12.1 kWh,
with a target state of charge of 25 kWh at 4.00pm.
The minimum and maximum soc are set to 10 and 25 kWh, respectively.
Roundtrip efficiency for use in scheduling is set to 98%.
.. code-block:: json
{
"start": "2015-06-02T10:00:00+00:00",
"soc-at-start": 12.1,
"soc-unit": "kWh",
"soc-targets": [
{
"value": 25,
"datetime": "2015-06-02T16:00:00+00:00"
}
],
"soc-min": 10,
"soc-max": 25,
"roundtrip-efficiency": 0.98
}
**Example response**
This message indicates that the scheduling request has been processed without any error.
A scheduling job has been created with some Universally Unique Identifier (UUID),
which will be picked up by a worker.
The given UUID may be used to obtain the resulting schedule: see /sensors/<id>/schedules/<uuid>.
.. sourcecode:: json
{
"status": "PROCESSED",
"schedule": "364bfd06-c1fa-430b-8d25-8f5a547651fb",
"message": "Request has been processed."
}
:reqheader Authorization: The authentication token
:reqheader Content-Type: application/json
:resheader Content-Type: application/json
:status 200: PROCESSED
:status 400: INVALID_TIMEZONE, INVALID_DATETIME, INVALID_DOMAIN, INVALID_UNIT, PTUS_INCOMPLETE
:status 401: UNAUTHORIZED
:status 403: INVALID_SENDER
:status 405: INVALID_METHOD
"""
# todo: if a soc-sensor entity address is passed, persist those values to the corresponding sensor
# (also update the note in posting_data.rst about flexibility states not being persisted).
# get value
if "value" not in kwargs:
return ptus_incomplete()
try:
value = float(kwargs.get("value")) # type: ignore
except ValueError:
extra_info = "Request includes empty or ill-formatted value(s)."
current_app.logger.warning(extra_info)
return ptus_incomplete(extra_info)
if unit == "kWh":
value = value / 1000.0
# Convert round-trip efficiency to dimensionless
if roundtrip_efficiency is not None:
roundtrip_efficiency = roundtrip_efficiency.to(
ur.Quantity("dimensionless")).magnitude
# get optional min and max SOC
soc_min = kwargs.get("soc_min", None)
soc_max = kwargs.get("soc_max", None)
if soc_min is not None and unit == "kWh":
soc_min = soc_min / 1000.0
if soc_max is not None and unit == "kWh":
soc_max = soc_max / 1000.0
# set soc targets
end_of_schedule = start_of_schedule + current_app.config.get( # type: ignore
"FLEXMEASURES_PLANNING_HORIZON")
resolution = sensor.event_resolution
soc_targets = pd.Series(
np.nan,
index=pd.date_range(
start_of_schedule,
end_of_schedule,
freq=resolution,
closed="right"
), # note that target values are indexed by their due date (i.e. closed="right")
)
# todo: move deserialization of targets into TargetSchema
for target in kwargs.get("targets", []):
# get target value
if "value" not in target:
return ptus_incomplete("Target missing 'value' parameter.")
try:
target_value = float(target["value"])
except ValueError:
extra_info = "Request includes empty or ill-formatted soc target(s)."
current_app.logger.warning(extra_info)
return ptus_incomplete(extra_info)
if unit == "kWh":
target_value = target_value / 1000.0
# get target datetime
if "datetime" not in target:
return invalid_datetime("Target missing datetime parameter.")
else:
target_datetime = target["datetime"]
if target_datetime is None:
return invalid_datetime(
"Cannot parse target datetime string %s as iso date" %
target["datetime"])
if target_datetime.tzinfo is None:
current_app.logger.warning(
"Cannot parse timezone of target 'datetime' value %s" %
target["datetime"])
return invalid_timezone(
"Target datetime should explicitly state a timezone.")
if target_datetime > end_of_schedule:
return invalid_datetime(
f'Target datetime exceeds {end_of_schedule}. Maximum scheduling horizon is {current_app.config.get("FLEXMEASURES_PLANNING_HORIZON")}.'
)
target_datetime = target_datetime.astimezone(
soc_targets.index.tzinfo
) # otherwise DST would be problematic
# set target
soc_targets.loc[target_datetime] = target_value
job = create_scheduling_job(
sensor.id,
start_of_schedule,
end_of_schedule,
resolution=resolution,
belief_time=prior, # server time if no prior time was sent
soc_at_start=value,
soc_targets=soc_targets,
soc_min=soc_min,
soc_max=soc_max,
roundtrip_efficiency=roundtrip_efficiency,
enqueue=True,
)
response = dict(schedule=job.id)
d, s = request_processed()
return dict(**response, **d), s
def get_device_message_response(generic_asset_name_groups, duration):
unit = "MW"
planning_horizon = min(
duration, current_app.config.get("FLEXMEASURES_PLANNING_HORIZON"))
if not has_assets():
current_app.logger.info("User doesn't seem to have any assets.")
value_groups = []
new_event_groups = []
for event_group in generic_asset_name_groups:
for event in event_group:
# Parse the entity address
try:
ea = parse_entity_address(event,
entity_type="event",
fm_scheme="fm0")
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = ea["asset_id"]
event_id = ea["event_id"]
event_type = ea["event_type"]
# Look for the Sensor object
sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
if sensor is None or not can_access_asset(sensor):
current_app.logger.warning(
"Cannot identify sensor given the event %s." % event)
return unrecognized_connection_group()
if sensor.generic_asset.generic_asset_type.name not in (
"battery",
"one-way_evse",
"two-way_evse",
):
return invalid_domain(
f"API version 1.3 only supports device messages for batteries and Electric Vehicle Supply Equipment (EVSE). "
f"Sensor ID:{sensor_id} does not belong to a battery or EVSE, but {p.a(sensor.generic_asset.generic_asset_type.description)}."
)
# Use the event_id to look up the schedule start
if event_type not in ("soc", "soc-with-targets"):
return unrecognized_event_type(event_type)
connection = current_app.queues["scheduling"].connection
try: # First try the scheduling queue
job = Job.fetch(event, connection=connection)
except NoSuchJobError: # Then try the most recent event_id (stored as a generic asset attribute)
if event_id == sensor.generic_asset.get_attribute(
"soc_udi_event_id"):
schedule_start = datetime.fromisoformat(
sensor.generic_asset.get_attribute("soc_datetime"))
message = (
"Your UDI event is the most recent event for this device, but "
)
else:
return unrecognized_event(event_id, event_type)
else:
if job.is_finished:
message = "A scheduling job has been processed based on your UDI event, but "
elif job.is_failed: # Try to inform the user on why the job failed
e = job.meta.get(
"exception",
Exception(
"The job does not state why it failed. "
"The worker may be missing an exception handler, "
"or its exception handler is not storing the exception as job meta data."
),
)
return unknown_schedule(
f"Scheduling job failed with {type(e).__name__}: {e}")
elif job.is_started:
return unknown_schedule("Scheduling job in progress.")
elif job.is_queued:
return unknown_schedule(
"Scheduling job waiting to be processed.")
elif job.is_deferred:
try:
preferred_job = job.dependency
except NoSuchJobError:
return unknown_schedule(
"Scheduling job waiting for unknown job to be processed."
)
return unknown_schedule(
f'Scheduling job waiting for {preferred_job.status} job "{preferred_job.id}" to be processed.'
)
else:
return unknown_schedule(
"Scheduling job has an unknown status.")
schedule_start = job.kwargs["start"]
schedule_data_source_name = "Seita"
scheduler_source = DataSource.query.filter_by(
name="Seita", type="scheduling script").one_or_none()
if scheduler_source is None:
return unknown_schedule(
message +
f'no data is known from "{schedule_data_source_name}".')
power_values = sensor.search_beliefs(
event_starts_after=schedule_start,
event_ends_before=schedule_start + planning_horizon,
source=scheduler_source,
most_recent_beliefs_only=True,
one_deterministic_belief_per_event=True,
)
# For consumption schedules, positive values denote consumption. For the db, consumption is negative
consumption_schedule = -simplify_index(power_values)["event_value"]
if consumption_schedule.empty:
return unknown_schedule(
message + "the schedule was not found in the database.")
# Update the planning window
resolution = sensor.event_resolution
start = consumption_schedule.index[0]
duration = min(duration,
consumption_schedule.index[-1] + resolution - start)
consumption_schedule = consumption_schedule[start:start +
duration - resolution]
value_groups.append(consumption_schedule.tolist())
new_event_groups.append(event)
response = groups_to_dict(new_event_groups,
value_groups,
generic_asset_type_name="event")
response["start"] = isodate.datetime_isoformat(start)
response["duration"] = isodate.duration_isoformat(duration)
response["unit"] = unit
d, s = request_processed()
return dict(**response, **d), s
def post_udi_event_response(unit: str, prior: datetime):
if not has_assets():
current_app.logger.info("User doesn't seem to have any assets.")
form = get_form_from_request(request)
if "datetime" not in form:
return invalid_datetime("Missing datetime parameter.")
else:
datetime = parse_isodate_str(form.get("datetime"))
if datetime is None:
return invalid_datetime(
"Cannot parse datetime string %s as iso date" %
form.get("datetime"))
if datetime.tzinfo is None:
current_app.logger.warning(
"Cannot parse timezone of 'datetime' value %s" %
form.get("datetime"))
return invalid_timezone(
"Datetime should explicitly state a timezone.")
# parse event/address info
if "event" not in form:
return invalid_domain("No event identifier sent.")
try:
ea = parse_entity_address(form.get("event"),
entity_type="event",
fm_scheme="fm0")
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = ea["asset_id"]
event_id = ea["event_id"]
event_type = ea["event_type"]
if event_type not in ("soc", "soc-with-targets"):
return unrecognized_event_type(event_type)
# Look for the Sensor object
sensor = Sensor.query.filter_by(id=sensor_id).one_or_none()
if sensor is None or not can_access_asset(sensor):
current_app.logger.warning("Cannot identify sensor via %s." % ea)
return unrecognized_connection_group()
if sensor.generic_asset.generic_asset_type.name not in (
"battery",
"one-way_evse",
"two-way_evse",
):
return invalid_domain(
f"API version 1.3 only supports UDI events for batteries and Electric Vehicle Supply Equipment (EVSE). "
f"Sensor ID:{sensor_id} does not belong to a battery or EVSE, but {p.a(sensor.generic_asset.generic_asset_type.description)}."
)
# unless on play, keep events ordered by entry date and ID
if current_app.config.get("FLEXMEASURES_MODE") != "play":
# do not allow new date to precede previous date
if isinstance(sensor.generic_asset.get_attribute("soc_datetime"),
str) and datetime < datetime.fromisoformat(
sensor.generic_asset.get_attribute("soc_datetime")):
msg = "The date of the requested UDI event (%s) is earlier than the latest known date (%s)." % (
datetime,
datetime.fromisoformat(
sensor.generic_asset.get_attribute("soc_datetime")),
)
current_app.logger.warning(msg)
return invalid_datetime(msg)
# check if udi event id is higher than existing
if sensor.generic_asset.get_attribute("soc_udi_event_id") is not None:
if sensor.generic_asset.get_attribute(
"soc_udi_event_id") >= event_id:
return outdated_event_id(
event_id,
sensor.generic_asset.get_attribute("soc_udi_event_id"))
# get value
if "value" not in form:
return ptus_incomplete()
try:
value = float(form.get("value"))
except ValueError:
extra_info = "Request includes empty or ill-formatted value(s)."
current_app.logger.warning(extra_info)
return ptus_incomplete(extra_info)
if unit == "kWh":
value = value / 1000.0
# get optional efficiency
roundtrip_efficiency = form.get("roundtrip_efficiency", None)
# get optional min and max SOC
soc_min = form.get("soc_min", None)
soc_max = form.get("soc_max", None)
if soc_min is not None and unit == "kWh":
soc_min = soc_min / 1000.0
if soc_max is not None and unit == "kWh":
soc_max = soc_max / 1000.0
# set soc targets
start_of_schedule = datetime
end_of_schedule = datetime + current_app.config.get(
"FLEXMEASURES_PLANNING_HORIZON")
resolution = sensor.event_resolution
soc_targets = pd.Series(
np.nan,
index=pd.date_range(
start_of_schedule,
end_of_schedule,
freq=resolution,
closed="right"
), # note that target values are indexed by their due date (i.e. closed="right")
)
if event_type == "soc-with-targets":
if "targets" not in form:
return incomplete_event(
event_id,
event_type,
"Cannot process event %s with missing targets." %
form.get("event"),
)
for target in form.get("targets"):
# get target value
if "value" not in target:
return ptus_incomplete("Target missing value parameter.")
try:
target_value = float(target["value"])
except ValueError:
extra_info = "Request includes empty or ill-formatted target value(s)."
current_app.logger.warning(extra_info)
return ptus_incomplete(extra_info)
if unit == "kWh":
target_value = target_value / 1000.0
# get target datetime
if "datetime" not in target:
return invalid_datetime("Target missing datetime parameter.")
else:
target_datetime = parse_isodate_str(target["datetime"])
if target_datetime is None:
return invalid_datetime(
"Cannot parse target datetime string %s as iso date" %
target["datetime"])
if target_datetime.tzinfo is None:
current_app.logger.warning(
"Cannot parse timezone of target 'datetime' value %s" %
target["datetime"])
return invalid_timezone(
"Target datetime should explicitly state a timezone.")
if target_datetime > end_of_schedule:
return invalid_datetime(
f'Target datetime exceeds {end_of_schedule}. Maximum scheduling horizon is {current_app.config.get("FLEXMEASURES_PLANNING_HORIZON")}.'
)
target_datetime = target_datetime.astimezone(
soc_targets.index.tzinfo
) # otherwise DST would be problematic
# set target
soc_targets.loc[target_datetime] = target_value
create_scheduling_job(
sensor_id,
start_of_schedule,
end_of_schedule,
resolution=resolution,
belief_time=prior, # server time if no prior time was sent
soc_at_start=value,
soc_targets=soc_targets,
soc_min=soc_min,
soc_max=soc_max,
roundtrip_efficiency=roundtrip_efficiency,
job_id=form.get("event"),
enqueue=True,
)
# Store new soc info as GenericAsset attributes
sensor.generic_asset.set_attribute("soc_datetime", datetime.isoformat())
sensor.generic_asset.set_attribute("soc_udi_event_id", event_id)
sensor.generic_asset.set_attribute("soc_in_mwh", value)
db.session.commit()
return request_processed()
def post_price_data_response(
unit,
generic_asset_name_groups,
horizon,
rolling,
value_groups,
start,
duration,
resolution,
):
current_app.logger.info("POSTING PRICE DATA")
data_source = get_or_create_user_data_source(current_user)
prices = []
forecasting_jobs = []
for market_group, value_group in zip(generic_asset_name_groups,
value_groups):
for market in market_group:
# Parse the entity address
try:
ea = parse_entity_address(market, entity_type="market")
except EntityAddressException as eae:
return invalid_domain(str(eae))
market_name = ea["market_name"]
# Look for the Market object
market = Market.query.filter(
Market.name == market_name).one_or_none()
if market is None:
return unrecognized_market(market_name)
elif unit != market.unit:
return invalid_unit("%s prices" % market.display_name,
[market.unit])
# Create new Price objects
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - ((start + duration) -
(dt + duration / len(value_group)))
p = Price(
datetime=dt,
value=value,
horizon=h,
market_id=market.id,
data_source_id=data_source.id,
)
prices.append(p)
# Make forecasts, but not in play mode. Price forecasts (horizon>0) can still lead to other price forecasts,
# by the way, due to things like day-ahead markets.
if current_app.config.get("FLEXMEASURES_MODE", "") != "play":
# Forecast 24 and 48 hours ahead for at most the last 24 hours of posted price data
forecasting_jobs = create_forecasting_jobs(
"Price",
market.id,
max(start, start + duration - timedelta(hours=24)),
start + duration,
resolution=duration / len(value_group),
horizons=[timedelta(hours=24),
timedelta(hours=48)],
enqueue=
False, # will enqueue later, only if we successfully saved prices
)
# Put these into the database
current_app.logger.info("SAVING TO DB...")
try:
save_to_session(prices)
db.session.flush()
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
except IntegrityError as e:
current_app.logger.warning(e)
db.session.rollback()
# Allow price data to be replaced only in play mode
if current_app.config.get("FLEXMEASURES_MODE", "") == "play":
save_to_session(prices, overwrite=True)
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
else:
return already_received_and_successfully_processed()
def post_weather_data_response( # noqa: C901
unit,
generic_asset_name_groups,
horizon,
rolling,
value_groups,
start,
duration,
resolution,
):
current_app.logger.info("POSTING WEATHER DATA")
data_source = get_or_create_user_data_source(current_user)
weather_measurements = []
forecasting_jobs = []
for sensor_group, value_group in zip(generic_asset_name_groups,
value_groups):
for sensor in sensor_group:
# Parse the entity address
try:
ea = parse_entity_address(sensor, entity_type="sensor")
except EntityAddressException as eae:
return invalid_domain(str(eae))
weather_sensor_type_name = ea["weather_sensor_type_name"]
latitude = ea["latitude"]
longitude = ea["longitude"]
# Check whether the unit is valid for this sensor type (e.g. no m/s allowed for temperature data)
accepted_units = valid_sensor_units(weather_sensor_type_name)
if unit not in accepted_units:
return invalid_unit(weather_sensor_type_name, accepted_units)
weather_sensor = get_weather_sensor_by(weather_sensor_type_name,
latitude, longitude)
# Create new Weather objects
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - ((start + duration) -
(dt + duration / len(value_group)))
w = Weather(
datetime=dt,
value=value,
horizon=h,
sensor_id=weather_sensor.id,
data_source_id=data_source.id,
)
weather_measurements.append(w)
# make forecasts, but only if the sent-in values are not forecasts themselves (and also not in play)
if current_app.config.get(
"FLEXMEASURES_MODE", ""
) != "play" and horizon <= timedelta(
hours=0
): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this generic asset
forecasting_jobs.extend(
create_forecasting_jobs(
"Weather",
weather_sensor.id,
start,
start + duration,
resolution=duration / len(value_group),
horizons=[horizon],
enqueue=
False, # will enqueue later, only if we successfully saved weather measurements
))
# Put these into the database
current_app.logger.info("SAVING TO DB...")
try:
save_to_session(weather_measurements)
db.session.flush()
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
except IntegrityError as e:
current_app.logger.warning(e)
db.session.rollback()
# Allow meter data to be replaced only in play mode
if current_app.config.get("FLEXMEASURES_MODE", "") == "play":
save_to_session(weather_measurements, overwrite=True)
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
else:
return already_received_and_successfully_processed()
def get_device_message_response(generic_asset_name_groups, duration):
unit = "MW"
min_planning_horizon = timedelta(
hours=24
) # user can request a shorter planning, but the scheduler takes into account at least this horizon
planning_horizon = min(
max(min_planning_horizon, duration),
current_app.config.get("FLEXMEASURES_PLANNING_HORIZON"),
)
if not has_assets():
current_app.logger.info("User doesn't seem to have any assets.")
value_groups = []
new_event_groups = []
for event_group in generic_asset_name_groups:
for event in event_group:
# Parse the entity address
try:
ea = parse_entity_address(event,
entity_type="event",
fm_scheme="fm0")
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = ea["asset_id"]
event_id = ea["event_id"]
event_type = ea["event_type"]
# Look for the Sensor object
sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
if sensor is None or not can_access_asset(sensor):
current_app.logger.warning(
"Cannot identify sensor given the event %s." % event)
return unrecognized_connection_group()
if sensor.generic_asset.generic_asset_type.name != "battery":
return invalid_domain(
"API version 1.2 only supports device messages for batteries. "
"Sensor ID:%s does not belong to a battery." % sensor_id)
if event_type != "soc" or event_id != sensor.generic_asset.get_attribute(
"soc_udi_event_id"):
return unrecognized_event(event_id, event_type)
start = datetime.fromisoformat(
sensor.generic_asset.get_attribute("soc_datetime"))
resolution = sensor.event_resolution
# Schedule the asset
try:
schedule = schedule_battery(
sensor,
start,
start + planning_horizon,
resolution,
soc_at_start=sensor.generic_asset.get_attribute(
"soc_in_mwh"),
prefer_charging_sooner=False,
)
except UnknownPricesException:
return unknown_prices()
except UnknownMarketException:
return invalid_market()
else:
# Update the planning window
start = schedule.index[0]
duration = min(duration,
schedule.index[-1] + resolution - start)
schedule = schedule[start:start + duration - resolution]
value_groups.append(schedule.tolist())
new_event_groups.append(event)
response = groups_to_dict(new_event_groups,
value_groups,
generic_asset_type_name="event")
response["start"] = isodate.datetime_isoformat(start)
response["duration"] = isodate.duration_isoformat(duration)
response["unit"] = unit
d, s = request_processed()
return dict(**response, **d), s
def post_udi_event_response(unit): # noqa: C901
if not has_assets():
current_app.logger.info("User doesn't seem to have any assets.")
form = get_form_from_request(request)
# check datetime, or use server_now
if "datetime" not in form:
return invalid_datetime("Missing datetime parameter.")
else:
datetime = parse_isodate_str(form.get("datetime"))
if datetime is None:
return invalid_datetime(
"Cannot parse datetime string %s as iso date" %
form.get("datetime"))
if datetime.tzinfo is None:
current_app.logger.warning(
"Cannot parse timezone of 'datetime' value %s" %
form.get("datetime"))
return invalid_timezone(
"Datetime should explicitly state a timezone.")
# parse event/address info
if "event" not in form:
return invalid_domain("No event identifier sent.")
try:
ea = parse_entity_address(form.get("event"),
entity_type="event",
fm_scheme="fm0")
except EntityAddressException as eae:
return invalid_domain(str(eae))
sensor_id = ea["asset_id"]
event_id = ea["event_id"]
event_type = ea["event_type"]
if event_type != "soc":
return unrecognized_event(event_id, event_type)
# Look for the Sensor object
sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
if sensor is None or not can_access_asset(sensor):
current_app.logger.warning("Cannot identify sensor via %s." % ea)
return unrecognized_connection_group()
if sensor.generic_asset.generic_asset_type.name != "battery":
return invalid_domain(
"API version 1.2 only supports UDI events for batteries. "
"Sensor ID:%s does not belong to a battery." % sensor_id)
# unless on play, keep events ordered by entry date and ID
if current_app.config.get("FLEXMEASURES_MODE") != "play":
# do not allow new date to be after last date
if (isinstance(sensor.generic_asset.get_attribute("soc_datetime"), str)
and datetime.fromisoformat(
sensor.generic_asset.get_attribute("soc_datetime")) >=
datetime):
msg = "The date of the requested UDI event (%s) is earlier than the latest known date (%s)." % (
datetime,
datetime.fromisoformat(
sensor.generic_asset.get_attribute("soc_datetime")),
)
current_app.logger.warning(msg)
return invalid_datetime(msg)
# check if udi event id is higher than existing
soc_udi_event_id = sensor.generic_asset.get_attribute(
"soc_udi_event_id")
if soc_udi_event_id is not None and soc_udi_event_id >= event_id:
return outdated_event_id(event_id, soc_udi_event_id)
# get value
if "value" not in form:
return ptus_incomplete()
value = form.get("value")
if unit == "kWh":
value = value / 1000.0
# Store new soc info as GenericAsset attributes
sensor.generic_asset.set_attribute("soc_datetime", datetime.isoformat())
sensor.generic_asset.set_attribute("soc_udi_event_id", event_id)
sensor.generic_asset.set_attribute("soc_in_mwh", value)
db.session.commit()
return request_processed("Request has been processed.")
def post_udi_event_response(unit): # noqa: C901
if not has_assets():
current_app.logger.info("User doesn't seem to have any assets.")
form = get_form_from_request(request)
# check datetime, or use server_now
if "datetime" not in form:
return invalid_datetime("Missing datetime parameter.")
else:
datetime = parse_isodate_str(form.get("datetime"))
if datetime is None:
return invalid_datetime(
"Cannot parse datetime string %s as iso date" % form.get("datetime")
)
if datetime.tzinfo is None:
current_app.logger.warning(
"Cannot parse timezone of 'datetime' value %s" % form.get("datetime")
)
return invalid_timezone("Datetime should explicitly state a timezone.")
# parse event/address info
if "event" not in form:
return invalid_domain("No event identifier sent.")
try:
ea = parse_entity_address(form.get("event"), entity_type="event")
except EntityAddressException as eae:
return invalid_domain(str(eae))
asset_id = ea["asset_id"]
event_id = ea["event_id"]
event_type = ea["event_type"]
if event_type != "soc":
return unrecognized_event(event_id, event_type)
# get asset
asset: Asset = Asset.query.filter_by(id=asset_id).one_or_none()
if asset is None or not can_access_asset(asset):
current_app.logger.warning("Cannot identify asset via %s." % ea)
return unrecognized_connection_group()
if asset.asset_type_name != "battery":
return invalid_domain(
"API version 1.2 only supports UDI events for batteries. Asset ID:%s is not a battery."
% asset_id
)
# unless on play, keep events ordered by entry date and ID
if current_app.config.get("FLEXMEASURES_MODE") != "play":
# do not allow new date to be after last date
if asset.soc_datetime is not None:
if asset.soc_datetime >= datetime:
msg = (
"The date of the requested UDI event (%s) is earlier than the latest known date (%s)."
% (datetime, asset.soc_datetime)
)
current_app.logger.warning(msg)
return invalid_datetime(msg)
# check if udi event id is higher than existing
if asset.soc_udi_event_id is not None:
if asset.soc_udi_event_id >= event_id:
return outdated_event_id(event_id, asset.soc_udi_event_id)
# get value
if "value" not in form:
return ptus_incomplete()
value = form.get("value")
if unit == "kWh":
value = value / 1000.0
# store new soc in asset
asset.soc_datetime = datetime
asset.soc_udi_event_id = event_id
asset.soc_in_mwh = value
db.session.commit()
return request_processed("Request has been processed.")
def create_connection_and_value_groups( # noqa: C901
unit, generic_asset_name_groups, value_groups, horizon, rolling, start,
duration):
"""
Code for POSTing Power values to the API.
Only lets users post to assets they own.
The sign of values is validated according to asset specs, but in USEF terms.
Then, we store the reverse sign for FlexMeasures specs (with positive production
and negative consumption).
If power values are not forecasts, forecasting jobs are created.
"""
from flask import current_app
current_app.logger.info("POSTING POWER DATA")
data_source = get_or_create_user_data_source(current_user)
user_assets = get_assets()
if not user_assets:
current_app.logger.info("User doesn't seem to have any assets")
user_asset_ids = [asset.id for asset in user_assets]
power_measurements = []
forecasting_jobs = []
for connection_group, value_group in zip(generic_asset_name_groups,
value_groups):
for connection in connection_group:
# TODO: get asset through util function after refactoring
# Parse the entity address
try:
connection = parse_entity_address(connection,
entity_type="connection")
except EntityAddressException as eae:
return invalid_domain(str(eae))
asset_id = connection["asset_id"]
# Look for the Asset object
if asset_id in user_asset_ids:
asset = Asset.query.filter(Asset.id == asset_id).one_or_none()
else:
current_app.logger.warning("Cannot identify connection %s" %
connection)
return unrecognized_connection_group()
# Validate the sign of the values (following USEF specs with positive consumption and negative production)
if asset.is_pure_consumer and any(v < 0 for v in value_group):
extra_info = (
"Connection %s is registered as a pure consumer and can only receive non-negative values."
% asset.entity_address)
return power_value_too_small(extra_info)
elif asset.is_pure_producer and any(v > 0 for v in value_group):
extra_info = (
"Connection %s is registered as a pure producer and can only receive non-positive values."
% asset.entity_address)
return power_value_too_big(extra_info)
# Create new Power objects
for j, value in enumerate(value_group):
dt = start + j * duration / len(value_group)
if rolling:
h = horizon
else: # Deduct the difference in end times of the individual timeslot and the timeseries duration
h = horizon - ((start + duration) -
(dt + duration / len(value_group)))
p = Power(
datetime=dt,
value=value *
-1, # Reverse sign for FlexMeasures specs with positive production and negative consumption
horizon=h,
asset_id=asset.id,
data_source_id=data_source.id,
)
power_measurements.append(p)
# make forecasts, but only if the sent-in values are not forecasts themselves
if horizon <= timedelta(
hours=0
): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this generic asset
forecasting_jobs.extend(
create_forecasting_jobs(
"Power",
asset_id,
start,
start + duration,
resolution=duration / len(value_group),
enqueue=False,
))
current_app.logger.info("SAVING TO DB AND QUEUEING...")
try:
save_to_session(power_measurements)
db.session.flush()
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
except IntegrityError as e:
current_app.logger.warning(e)
db.session.rollback()
# Allow meter data to be replaced only in play mode
if current_app.config.get("FLEXMEASURES_MODE", "") == "play":
save_to_session(power_measurements, overwrite=True)
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
else:
return already_received_and_successfully_processed()
def collect_connection_and_value_groups(
unit: str,
resolution: str,
belief_horizon_window: Tuple[Union[None, timedelta], Union[None,
timedelta]],
belief_time_window: Tuple[Optional[datetime_type],
Optional[datetime_type]],
start: datetime_type,
duration: timedelta,
connection_groups: List[List[str]],
user_source_ids: Union[
int, List[int]] = None, # None is interpreted as all sources
source_types: List[str] = None,
) -> Tuple[dict, int]:
"""
Code for GETting power values from the API.
Only allows to get values from assets owned by current user.
Returns value sign in accordance with USEF specs
(with negative production and positive consumption).
"""
from flask import current_app
current_app.logger.info("GETTING")
user_assets = get_assets()
if not user_assets:
current_app.logger.info("User doesn't seem to have any assets")
user_asset_ids = [asset.id for asset in user_assets]
end = start + duration
value_groups = []
new_connection_groups = (
[]
) # Each connection in the old connection groups will be interpreted as a separate group
for connections in connection_groups:
# Get the asset names
asset_names: List[str] = []
for connection in connections:
# Parse the entity address
try:
connection_details = parse_entity_address(
connection, entity_type="connection")
except EntityAddressException as eae:
return invalid_domain(str(eae))
asset_id = connection_details["asset_id"]
# Look for the Asset object
if asset_id in user_asset_ids:
asset = Asset.query.filter(Asset.id == asset_id).one_or_none()
else:
current_app.logger.warning("Cannot identify connection %s" %
connection)
return unrecognized_connection_group()
asset_names.append(asset.name)
# Get the power values
# TODO: fill NaN for non-existing values
power_bdf_dict: Dict[str, tb.BeliefsDataFrame] = Power.collect(
generic_asset_names=asset_names,
query_window=(start, end),
resolution=resolution,
belief_horizon_window=belief_horizon_window,
belief_time_window=belief_time_window,
user_source_ids=user_source_ids,
source_types=source_types,
sum_multiple=False,
)
# Todo: parse time window of power_bdf_dict, which will be different for requests that are not of the form:
# - start is a timestamp on the hour or a multiple of 15 minutes thereafter
# - duration is a multiple of 15 minutes
for k, bdf in power_bdf_dict.items():
value_groups.append(
[x * -1 for x in bdf["event_value"].tolist()]
) # Reverse sign of values (from FlexMeasures specs to USEF specs)
new_connection_groups.append(k)
response = groups_to_dict(new_connection_groups,
value_groups,
generic_asset_type_name="connection")
response["start"] = isodate.datetime_isoformat(start)
response["duration"] = isodate.duration_isoformat(duration)
response["unit"] = unit # TODO: convert to requested unit
d, s = request_processed()
return dict(**response, **d), s
def save_to_db(
timed_values: Union[BeliefsDataFrame, List[Union[Power, Price, Weather]]],
forecasting_jobs: List[Job] = [],
save_changed_beliefs_only: bool = True,
) -> ResponseTuple:
"""Put the timed values into the database and enqueue forecasting jobs.
Data can only be replaced on servers in play mode.
TODO: remove this legacy function in its entirety (announced v0.8.0)
:param timed_values: BeliefsDataFrame or a list of Power, Price or Weather values to be saved
:param forecasting_jobs: list of forecasting Jobs for redis queues.
:param save_changed_beliefs_only: if True, beliefs that are already stored in the database with an earlier belief time are dropped.
:returns: ResponseTuple
"""
import warnings
warnings.warn(
"The method api.common.utils.api_utils.save_to_db is deprecated. Check out the following replacements:"
"- [recommended option] to store BeliefsDataFrames only, switch to data.utils.save_to_db"
"- to store BeliefsDataFrames and enqueue jobs, switch to api.common.utils.api_utils.save_and_enqueue"
)
if isinstance(timed_values, BeliefsDataFrame):
if save_changed_beliefs_only:
# Drop beliefs that haven't changed
timed_values = (
timed_values.convert_index_from_belief_horizon_to_time()
.groupby(level=["belief_time", "source"], as_index=False)
.apply(drop_unchanged_beliefs)
)
# Work around bug in which groupby still introduces an index level, even though we asked it not to
if None in timed_values.index.names:
timed_values.index = timed_values.index.droplevel(None)
if timed_values.empty:
current_app.logger.debug("Nothing new to save")
return already_received_and_successfully_processed()
current_app.logger.info("SAVING TO DB AND QUEUEING...")
try:
if isinstance(timed_values, BeliefsDataFrame):
TimedBelief.add_to_session(
session=db.session, beliefs_data_frame=timed_values
)
else:
save_to_session(timed_values)
db.session.flush()
[current_app.queues["forecasting"].enqueue_job(job) for job in forecasting_jobs]
db.session.commit()
return request_processed()
except IntegrityError as e:
current_app.logger.warning(e)
db.session.rollback()
# Possibly allow data to be replaced depending on config setting
if current_app.config.get("FLEXMEASURES_ALLOW_DATA_OVERWRITE", False):
if isinstance(timed_values, BeliefsDataFrame):
TimedBelief.add_to_session(
session=db.session,
beliefs_data_frame=timed_values,
allow_overwrite=True,
)
else:
save_to_session(timed_values, overwrite=True)
[
current_app.queues["forecasting"].enqueue_job(job)
for job in forecasting_jobs
]
db.session.commit()
return request_processed()
else:
return already_received_and_successfully_processed()
