def save_and_enqueue( data: Union[BeliefsDataFrame, List[BeliefsDataFrame]], forecasting_jobs: List[Job] = None, save_changed_beliefs_only: bool = True, ) -> ResponseTuple: # Attempt to save status = modern_save_to_db( data, save_changed_beliefs_only=save_changed_beliefs_only ) db.session.commit() # Only enqueue forecasting jobs upon successfully saving new data if status[:7] == "success" and status != "success_but_nothing_new": enqueue_forecasting_jobs(forecasting_jobs) # Pick a response if status == "success": return request_processed() elif status in ( "success_with_unchanged_beliefs_skipped", "success_but_nothing_new", ): return already_received_and_successfully_processed() return invalid_replacement()
def test_get_service(client, query): get_service_response = client.get( url_for("flexmeasures_api_v1.get_service"), query_string=query, headers={"content-type": "application/json"}, ) print("Server responded with:\n%s" % get_service_response.json) assert get_service_response.status_code == 200 assert get_service_response.json["type"] == "GetServiceResponse" assert get_service_response.json["status"] == request_processed()[0]["status"] if "access" in query: for service in get_service_response.json["services"]: assert "Prosumer" in service["access"]
def restore_data_response(): delete_structure = True delete_data = True restore_structure = True restore_data = True try: backup_name = request.args.get("backup", request.json["backup"]) except KeyError: return no_backup() # Make sure backup folder and files exist before restoring backup_path = app.config.get("FLEXMEASURES_DB_BACKUP_PATH") if (not Path("%s/%s" % (backup_path, backup_name)).exists() or not Path("%s/%s" % (backup_path, backup_name)).is_dir()): return unrecognized_backup() affected_classes = get_affected_classes(structure=restore_structure, data=restore_data) for c in affected_classes: file_path = "%s/%s/%s.obj" % (backup_path, backup_name, c.__tablename__) if not Path(file_path).exists(): return unrecognized_backup( "Can't load table, because filename %s does not exist." % c.__tablename__) # Reset in play mode only (this endpoint should not have been registered otherwise) assert app.config.get("FLEXMEASURES_MODE", "") == "play" if delete_data: depopulate_forecasts(db) depopulate_data(db) if delete_structure: depopulate_structure(db) # Load backup load_tables( db, backup_name, structure=restore_structure, data=restore_data, backup_path=backup_path, ) return request_processed("Database restored to %s." % backup_name)
def get_service_response(service_listing) -> Union[dict, Tuple[dict, int]]: """ Lists the available services for the public endpoint version, either all of them or only those that apply to the requested access role. """ requested_access_role = request.args.get("access") response = {"version": service_listing["version"]} if requested_access_role: accessible_services = [] for service in service_listing["services"]: if requested_access_role in service["access"]: accessible_services.append(service) response["services"] = accessible_services if not accessible_services: return invalid_role(requested_access_role) else: response["services"] = service_listing["services"] d, s = request_processed() return dict(**response, **d), s
def get_schedule(self, sensor: Sensor, job_id: str, duration: timedelta, **kwargs): """Get a schedule from FlexMeasures. .. :quickref: Schedule; Download schedule from the platform **Optional fields** - "duration" (6 hours by default; can be increased to plan further into the future) **Example response** This message contains a schedule indicating to consume at various power rates from 10am UTC onwards for a duration of 45 minutes. .. sourcecode:: json { "values": [ 2.15, 3, 2 ], "start": "2015-06-02T10:00:00+00:00", "duration": "PT45M", "unit": "MW" } :reqheader Authorization: The authentication token :reqheader Content-Type: application/json :resheader Content-Type: application/json :status 200: PROCESSED :status 400: INVALID_TIMEZONE, INVALID_DOMAIN, INVALID_UNIT, UNKNOWN_SCHEDULE, UNRECOGNIZED_CONNECTION_GROUP :status 401: UNAUTHORIZED :status 403: INVALID_SENDER :status 405: INVALID_METHOD :status 422: UNPROCESSABLE_ENTITY """ planning_horizon = min( # type: ignore duration, current_app.config.get("FLEXMEASURES_PLANNING_HORIZON")) # Look up the scheduling job connection = current_app.queues["scheduling"].connection try: # First try the scheduling queue job = Job.fetch(job_id, connection=connection) except NoSuchJobError: return unrecognized_event(job_id, "job") if job.is_finished: error_message = "A scheduling job has been processed with your job ID, but " elif job.is_failed: # Try to inform the user on why the job failed e = job.meta.get( "exception", Exception( "The job does not state why it failed. " "The worker may be missing an exception handler, " "or its exception handler is not storing the exception as job meta data." ), ) return unknown_schedule( f"Scheduling job failed with {type(e).__name__}: {e}") elif job.is_started: return unknown_schedule("Scheduling job in progress.") elif job.is_queued: return unknown_schedule("Scheduling job waiting to be processed.") elif job.is_deferred: try: preferred_job = job.dependency except NoSuchJobError: return unknown_schedule( "Scheduling job waiting for unknown job to be processed.") return unknown_schedule( f'Scheduling job waiting for {preferred_job.status} job "{preferred_job.id}" to be processed.' ) else: return unknown_schedule("Scheduling job has an unknown status.") schedule_start = job.kwargs["start"] schedule_data_source_name = "Seita" scheduler_source = DataSource.query.filter_by( name="Seita", type="scheduling script").one_or_none() if scheduler_source is None: return unknown_schedule( error_message + f'no data is known from "{schedule_data_source_name}".') power_values = sensor.search_beliefs( event_starts_after=schedule_start, event_ends_before=schedule_start + planning_horizon, source=scheduler_source, most_recent_beliefs_only=True, one_deterministic_belief_per_event=True, ) # For consumption schedules, positive values denote consumption. For the db, consumption is negative consumption_schedule = -simplify_index(power_values)["event_value"] if consumption_schedule.empty: return unknown_schedule( error_message + "the schedule was not found in the database.") # Update the planning window resolution = sensor.event_resolution start = consumption_schedule.index[0] duration = min(duration, consumption_schedule.index[-1] + resolution - start) consumption_schedule = consumption_schedule[start:start + duration - resolution] response = dict( values=consumption_schedule.tolist(), start=isodate.datetime_isoformat(start), duration=isodate.duration_isoformat(duration), unit=sensor.unit, ) d, s = request_processed() return dict(**response, **d), s
def trigger_schedule( # noqa: C901 self, sensor: Sensor, start_of_schedule: datetime, unit: str, prior: datetime, roundtrip_efficiency: Optional[ur.Quantity] = None, **kwargs, ): """ Trigger FlexMeasures to create a schedule. .. :quickref: Schedule; Trigger scheduling job The message should contain a flexibility model. **Example request A** This message triggers a schedule starting at 10.00am, at which the state of charge (soc) is 12.1 kWh. .. code-block:: json { "start": "2015-06-02T10:00:00+00:00", "soc-at-start": 12.1, "soc-unit": "kWh" } **Example request B** This message triggers a schedule starting at 10.00am, at which the state of charge (soc) is 12.1 kWh, with a target state of charge of 25 kWh at 4.00pm. The minimum and maximum soc are set to 10 and 25 kWh, respectively. Roundtrip efficiency for use in scheduling is set to 98%. .. code-block:: json { "start": "2015-06-02T10:00:00+00:00", "soc-at-start": 12.1, "soc-unit": "kWh", "soc-targets": [ { "value": 25, "datetime": "2015-06-02T16:00:00+00:00" } ], "soc-min": 10, "soc-max": 25, "roundtrip-efficiency": 0.98 } **Example response** This message indicates that the scheduling request has been processed without any error. A scheduling job has been created with some Universally Unique Identifier (UUID), which will be picked up by a worker. The given UUID may be used to obtain the resulting schedule: see /sensors/<id>/schedules/<uuid>. .. sourcecode:: json { "status": "PROCESSED", "schedule": "364bfd06-c1fa-430b-8d25-8f5a547651fb", "message": "Request has been processed." } :reqheader Authorization: The authentication token :reqheader Content-Type: application/json :resheader Content-Type: application/json :status 200: PROCESSED :status 400: INVALID_TIMEZONE, INVALID_DATETIME, INVALID_DOMAIN, INVALID_UNIT, PTUS_INCOMPLETE :status 401: UNAUTHORIZED :status 403: INVALID_SENDER :status 405: INVALID_METHOD """ # todo: if a soc-sensor entity address is passed, persist those values to the corresponding sensor # (also update the note in posting_data.rst about flexibility states not being persisted). # get value if "value" not in kwargs: return ptus_incomplete() try: value = float(kwargs.get("value")) # type: ignore except ValueError: extra_info = "Request includes empty or ill-formatted value(s)." current_app.logger.warning(extra_info) return ptus_incomplete(extra_info) if unit == "kWh": value = value / 1000.0 # Convert round-trip efficiency to dimensionless if roundtrip_efficiency is not None: roundtrip_efficiency = roundtrip_efficiency.to( ur.Quantity("dimensionless")).magnitude # get optional min and max SOC soc_min = kwargs.get("soc_min", None) soc_max = kwargs.get("soc_max", None) if soc_min is not None and unit == "kWh": soc_min = soc_min / 1000.0 if soc_max is not None and unit == "kWh": soc_max = soc_max / 1000.0 # set soc targets end_of_schedule = start_of_schedule + current_app.config.get( # type: ignore "FLEXMEASURES_PLANNING_HORIZON") resolution = sensor.event_resolution soc_targets = pd.Series( np.nan, index=pd.date_range( start_of_schedule, end_of_schedule, freq=resolution, closed="right" ), # note that target values are indexed by their due date (i.e. closed="right") ) # todo: move deserialization of targets into TargetSchema for target in kwargs.get("targets", []): # get target value if "value" not in target: return ptus_incomplete("Target missing 'value' parameter.") try: target_value = float(target["value"]) except ValueError: extra_info = "Request includes empty or ill-formatted soc target(s)." current_app.logger.warning(extra_info) return ptus_incomplete(extra_info) if unit == "kWh": target_value = target_value / 1000.0 # get target datetime if "datetime" not in target: return invalid_datetime("Target missing datetime parameter.") else: target_datetime = target["datetime"] if target_datetime is None: return invalid_datetime( "Cannot parse target datetime string %s as iso date" % target["datetime"]) if target_datetime.tzinfo is None: current_app.logger.warning( "Cannot parse timezone of target 'datetime' value %s" % target["datetime"]) return invalid_timezone( "Target datetime should explicitly state a timezone.") if target_datetime > end_of_schedule: return invalid_datetime( f'Target datetime exceeds {end_of_schedule}. Maximum scheduling horizon is {current_app.config.get("FLEXMEASURES_PLANNING_HORIZON")}.' ) target_datetime = target_datetime.astimezone( soc_targets.index.tzinfo ) # otherwise DST would be problematic # set target soc_targets.loc[target_datetime] = target_value job = create_scheduling_job( sensor.id, start_of_schedule, end_of_schedule, resolution=resolution, belief_time=prior, # server time if no prior time was sent soc_at_start=value, soc_targets=soc_targets, soc_min=soc_min, soc_max=soc_max, roundtrip_efficiency=roundtrip_efficiency, enqueue=True, ) response = dict(schedule=job.id) d, s = request_processed() return dict(**response, **d), s
def get_device_message_response(generic_asset_name_groups, duration): unit = "MW" planning_horizon = min( duration, current_app.config.get("FLEXMEASURES_PLANNING_HORIZON")) if not has_assets(): current_app.logger.info("User doesn't seem to have any assets.") value_groups = [] new_event_groups = [] for event_group in generic_asset_name_groups: for event in event_group: # Parse the entity address try: ea = parse_entity_address(event, entity_type="event", fm_scheme="fm0") except EntityAddressException as eae: return invalid_domain(str(eae)) sensor_id = ea["asset_id"] event_id = ea["event_id"] event_type = ea["event_type"] # Look for the Sensor object sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none() if sensor is None or not can_access_asset(sensor): current_app.logger.warning( "Cannot identify sensor given the event %s." % event) return unrecognized_connection_group() if sensor.generic_asset.generic_asset_type.name not in ( "battery", "one-way_evse", "two-way_evse", ): return invalid_domain( f"API version 1.3 only supports device messages for batteries and Electric Vehicle Supply Equipment (EVSE). " f"Sensor ID:{sensor_id} does not belong to a battery or EVSE, but {p.a(sensor.generic_asset.generic_asset_type.description)}." ) # Use the event_id to look up the schedule start if event_type not in ("soc", "soc-with-targets"): return unrecognized_event_type(event_type) connection = current_app.queues["scheduling"].connection try: # First try the scheduling queue job = Job.fetch(event, connection=connection) except NoSuchJobError: # Then try the most recent event_id (stored as a generic asset attribute) if event_id == sensor.generic_asset.get_attribute( "soc_udi_event_id"): schedule_start = datetime.fromisoformat( sensor.generic_asset.get_attribute("soc_datetime")) message = ( "Your UDI event is the most recent event for this device, but " ) else: return unrecognized_event(event_id, event_type) else: if job.is_finished: message = "A scheduling job has been processed based on your UDI event, but " elif job.is_failed: # Try to inform the user on why the job failed e = job.meta.get( "exception", Exception( "The job does not state why it failed. " "The worker may be missing an exception handler, " "or its exception handler is not storing the exception as job meta data." ), ) return unknown_schedule( f"Scheduling job failed with {type(e).__name__}: {e}") elif job.is_started: return unknown_schedule("Scheduling job in progress.") elif job.is_queued: return unknown_schedule( "Scheduling job waiting to be processed.") elif job.is_deferred: try: preferred_job = job.dependency except NoSuchJobError: return unknown_schedule( "Scheduling job waiting for unknown job to be processed." ) return unknown_schedule( f'Scheduling job waiting for {preferred_job.status} job "{preferred_job.id}" to be processed.' ) else: return unknown_schedule( "Scheduling job has an unknown status.") schedule_start = job.kwargs["start"] schedule_data_source_name = "Seita" scheduler_source = DataSource.query.filter_by( name="Seita", type="scheduling script").one_or_none() if scheduler_source is None: return unknown_schedule( message + f'no data is known from "{schedule_data_source_name}".') power_values = sensor.search_beliefs( event_starts_after=schedule_start, event_ends_before=schedule_start + planning_horizon, source=scheduler_source, most_recent_beliefs_only=True, one_deterministic_belief_per_event=True, ) # For consumption schedules, positive values denote consumption. For the db, consumption is negative consumption_schedule = -simplify_index(power_values)["event_value"] if consumption_schedule.empty: return unknown_schedule( message + "the schedule was not found in the database.") # Update the planning window resolution = sensor.event_resolution start = consumption_schedule.index[0] duration = min(duration, consumption_schedule.index[-1] + resolution - start) consumption_schedule = consumption_schedule[start:start + duration - resolution] value_groups.append(consumption_schedule.tolist()) new_event_groups.append(event) response = groups_to_dict(new_event_groups, value_groups, generic_asset_type_name="event") response["start"] = isodate.datetime_isoformat(start) response["duration"] = isodate.duration_isoformat(duration) response["unit"] = unit d, s = request_processed() return dict(**response, **d), s
def post_udi_event_response(unit: str, prior: datetime): if not has_assets(): current_app.logger.info("User doesn't seem to have any assets.") form = get_form_from_request(request) if "datetime" not in form: return invalid_datetime("Missing datetime parameter.") else: datetime = parse_isodate_str(form.get("datetime")) if datetime is None: return invalid_datetime( "Cannot parse datetime string %s as iso date" % form.get("datetime")) if datetime.tzinfo is None: current_app.logger.warning( "Cannot parse timezone of 'datetime' value %s" % form.get("datetime")) return invalid_timezone( "Datetime should explicitly state a timezone.") # parse event/address info if "event" not in form: return invalid_domain("No event identifier sent.") try: ea = parse_entity_address(form.get("event"), entity_type="event", fm_scheme="fm0") except EntityAddressException as eae: return invalid_domain(str(eae)) sensor_id = ea["asset_id"] event_id = ea["event_id"] event_type = ea["event_type"] if event_type not in ("soc", "soc-with-targets"): return unrecognized_event_type(event_type) # Look for the Sensor object sensor = Sensor.query.filter_by(id=sensor_id).one_or_none() if sensor is None or not can_access_asset(sensor): current_app.logger.warning("Cannot identify sensor via %s." % ea) return unrecognized_connection_group() if sensor.generic_asset.generic_asset_type.name not in ( "battery", "one-way_evse", "two-way_evse", ): return invalid_domain( f"API version 1.3 only supports UDI events for batteries and Electric Vehicle Supply Equipment (EVSE). " f"Sensor ID:{sensor_id} does not belong to a battery or EVSE, but {p.a(sensor.generic_asset.generic_asset_type.description)}." ) # unless on play, keep events ordered by entry date and ID if current_app.config.get("FLEXMEASURES_MODE") != "play": # do not allow new date to precede previous date if isinstance(sensor.generic_asset.get_attribute("soc_datetime"), str) and datetime < datetime.fromisoformat( sensor.generic_asset.get_attribute("soc_datetime")): msg = "The date of the requested UDI event (%s) is earlier than the latest known date (%s)." % ( datetime, datetime.fromisoformat( sensor.generic_asset.get_attribute("soc_datetime")), ) current_app.logger.warning(msg) return invalid_datetime(msg) # check if udi event id is higher than existing if sensor.generic_asset.get_attribute("soc_udi_event_id") is not None: if sensor.generic_asset.get_attribute( "soc_udi_event_id") >= event_id: return outdated_event_id( event_id, sensor.generic_asset.get_attribute("soc_udi_event_id")) # get value if "value" not in form: return ptus_incomplete() try: value = float(form.get("value")) except ValueError: extra_info = "Request includes empty or ill-formatted value(s)." current_app.logger.warning(extra_info) return ptus_incomplete(extra_info) if unit == "kWh": value = value / 1000.0 # get optional efficiency roundtrip_efficiency = form.get("roundtrip_efficiency", None) # get optional min and max SOC soc_min = form.get("soc_min", None) soc_max = form.get("soc_max", None) if soc_min is not None and unit == "kWh": soc_min = soc_min / 1000.0 if soc_max is not None and unit == "kWh": soc_max = soc_max / 1000.0 # set soc targets start_of_schedule = datetime end_of_schedule = datetime + current_app.config.get( "FLEXMEASURES_PLANNING_HORIZON") resolution = sensor.event_resolution soc_targets = pd.Series( np.nan, index=pd.date_range( start_of_schedule, end_of_schedule, freq=resolution, closed="right" ), # note that target values are indexed by their due date (i.e. closed="right") ) if event_type == "soc-with-targets": if "targets" not in form: return incomplete_event( event_id, event_type, "Cannot process event %s with missing targets." % form.get("event"), ) for target in form.get("targets"): # get target value if "value" not in target: return ptus_incomplete("Target missing value parameter.") try: target_value = float(target["value"]) except ValueError: extra_info = "Request includes empty or ill-formatted target value(s)." current_app.logger.warning(extra_info) return ptus_incomplete(extra_info) if unit == "kWh": target_value = target_value / 1000.0 # get target datetime if "datetime" not in target: return invalid_datetime("Target missing datetime parameter.") else: target_datetime = parse_isodate_str(target["datetime"]) if target_datetime is None: return invalid_datetime( "Cannot parse target datetime string %s as iso date" % target["datetime"]) if target_datetime.tzinfo is None: current_app.logger.warning( "Cannot parse timezone of target 'datetime' value %s" % target["datetime"]) return invalid_timezone( "Target datetime should explicitly state a timezone.") if target_datetime > end_of_schedule: return invalid_datetime( f'Target datetime exceeds {end_of_schedule}. Maximum scheduling horizon is {current_app.config.get("FLEXMEASURES_PLANNING_HORIZON")}.' ) target_datetime = target_datetime.astimezone( soc_targets.index.tzinfo ) # otherwise DST would be problematic # set target soc_targets.loc[target_datetime] = target_value create_scheduling_job( sensor_id, start_of_schedule, end_of_schedule, resolution=resolution, belief_time=prior, # server time if no prior time was sent soc_at_start=value, soc_targets=soc_targets, soc_min=soc_min, soc_max=soc_max, roundtrip_efficiency=roundtrip_efficiency, job_id=form.get("event"), enqueue=True, ) # Store new soc info as GenericAsset attributes sensor.generic_asset.set_attribute("soc_datetime", datetime.isoformat()) sensor.generic_asset.set_attribute("soc_udi_event_id", event_id) sensor.generic_asset.set_attribute("soc_in_mwh", value) db.session.commit() return request_processed()
def post_price_data_response( unit, generic_asset_name_groups, horizon, rolling, value_groups, start, duration, resolution, ): current_app.logger.info("POSTING PRICE DATA") data_source = get_or_create_user_data_source(current_user) prices = [] forecasting_jobs = [] for market_group, value_group in zip(generic_asset_name_groups, value_groups): for market in market_group: # Parse the entity address try: ea = parse_entity_address(market, entity_type="market") except EntityAddressException as eae: return invalid_domain(str(eae)) market_name = ea["market_name"] # Look for the Market object market = Market.query.filter( Market.name == market_name).one_or_none() if market is None: return unrecognized_market(market_name) elif unit != market.unit: return invalid_unit("%s prices" % market.display_name, [market.unit]) # Create new Price objects for j, value in enumerate(value_group): dt = start + j * duration / len(value_group) if rolling: h = horizon else: # Deduct the difference in end times of the individual timeslot and the timeseries duration h = horizon - ((start + duration) - (dt + duration / len(value_group))) p = Price( datetime=dt, value=value, horizon=h, market_id=market.id, data_source_id=data_source.id, ) prices.append(p) # Make forecasts, but not in play mode. Price forecasts (horizon>0) can still lead to other price forecasts, # by the way, due to things like day-ahead markets. if current_app.config.get("FLEXMEASURES_MODE", "") != "play": # Forecast 24 and 48 hours ahead for at most the last 24 hours of posted price data forecasting_jobs = create_forecasting_jobs( "Price", market.id, max(start, start + duration - timedelta(hours=24)), start + duration, resolution=duration / len(value_group), horizons=[timedelta(hours=24), timedelta(hours=48)], enqueue= False, # will enqueue later, only if we successfully saved prices ) # Put these into the database current_app.logger.info("SAVING TO DB...") try: save_to_session(prices) db.session.flush() [ current_app.queues["forecasting"].enqueue_job(job) for job in forecasting_jobs ] db.session.commit() return request_processed() except IntegrityError as e: current_app.logger.warning(e) db.session.rollback() # Allow price data to be replaced only in play mode if current_app.config.get("FLEXMEASURES_MODE", "") == "play": save_to_session(prices, overwrite=True) [ current_app.queues["forecasting"].enqueue_job(job) for job in forecasting_jobs ] db.session.commit() return request_processed() else: return already_received_and_successfully_processed()
def post_weather_data_response( # noqa: C901 unit, generic_asset_name_groups, horizon, rolling, value_groups, start, duration, resolution, ): current_app.logger.info("POSTING WEATHER DATA") data_source = get_or_create_user_data_source(current_user) weather_measurements = [] forecasting_jobs = [] for sensor_group, value_group in zip(generic_asset_name_groups, value_groups): for sensor in sensor_group: # Parse the entity address try: ea = parse_entity_address(sensor, entity_type="sensor") except EntityAddressException as eae: return invalid_domain(str(eae)) weather_sensor_type_name = ea["weather_sensor_type_name"] latitude = ea["latitude"] longitude = ea["longitude"] # Check whether the unit is valid for this sensor type (e.g. no m/s allowed for temperature data) accepted_units = valid_sensor_units(weather_sensor_type_name) if unit not in accepted_units: return invalid_unit(weather_sensor_type_name, accepted_units) weather_sensor = get_weather_sensor_by(weather_sensor_type_name, latitude, longitude) # Create new Weather objects for j, value in enumerate(value_group): dt = start + j * duration / len(value_group) if rolling: h = horizon else: # Deduct the difference in end times of the individual timeslot and the timeseries duration h = horizon - ((start + duration) - (dt + duration / len(value_group))) w = Weather( datetime=dt, value=value, horizon=h, sensor_id=weather_sensor.id, data_source_id=data_source.id, ) weather_measurements.append(w) # make forecasts, but only if the sent-in values are not forecasts themselves (and also not in play) if current_app.config.get( "FLEXMEASURES_MODE", "" ) != "play" and horizon <= timedelta( hours=0 ): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this generic asset forecasting_jobs.extend( create_forecasting_jobs( "Weather", weather_sensor.id, start, start + duration, resolution=duration / len(value_group), horizons=[horizon], enqueue= False, # will enqueue later, only if we successfully saved weather measurements )) # Put these into the database current_app.logger.info("SAVING TO DB...") try: save_to_session(weather_measurements) db.session.flush() [ current_app.queues["forecasting"].enqueue_job(job) for job in forecasting_jobs ] db.session.commit() return request_processed() except IntegrityError as e: current_app.logger.warning(e) db.session.rollback() # Allow meter data to be replaced only in play mode if current_app.config.get("FLEXMEASURES_MODE", "") == "play": save_to_session(weather_measurements, overwrite=True) [ current_app.queues["forecasting"].enqueue_job(job) for job in forecasting_jobs ] db.session.commit() return request_processed() else: return already_received_and_successfully_processed()
def get_device_message_response(generic_asset_name_groups, duration): unit = "MW" min_planning_horizon = timedelta( hours=24 ) # user can request a shorter planning, but the scheduler takes into account at least this horizon planning_horizon = min( max(min_planning_horizon, duration), current_app.config.get("FLEXMEASURES_PLANNING_HORIZON"), ) if not has_assets(): current_app.logger.info("User doesn't seem to have any assets.") value_groups = [] new_event_groups = [] for event_group in generic_asset_name_groups: for event in event_group: # Parse the entity address try: ea = parse_entity_address(event, entity_type="event", fm_scheme="fm0") except EntityAddressException as eae: return invalid_domain(str(eae)) sensor_id = ea["asset_id"] event_id = ea["event_id"] event_type = ea["event_type"] # Look for the Sensor object sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none() if sensor is None or not can_access_asset(sensor): current_app.logger.warning( "Cannot identify sensor given the event %s." % event) return unrecognized_connection_group() if sensor.generic_asset.generic_asset_type.name != "battery": return invalid_domain( "API version 1.2 only supports device messages for batteries. " "Sensor ID:%s does not belong to a battery." % sensor_id) if event_type != "soc" or event_id != sensor.generic_asset.get_attribute( "soc_udi_event_id"): return unrecognized_event(event_id, event_type) start = datetime.fromisoformat( sensor.generic_asset.get_attribute("soc_datetime")) resolution = sensor.event_resolution # Schedule the asset try: schedule = schedule_battery( sensor, start, start + planning_horizon, resolution, soc_at_start=sensor.generic_asset.get_attribute( "soc_in_mwh"), prefer_charging_sooner=False, ) except UnknownPricesException: return unknown_prices() except UnknownMarketException: return invalid_market() else: # Update the planning window start = schedule.index[0] duration = min(duration, schedule.index[-1] + resolution - start) schedule = schedule[start:start + duration - resolution] value_groups.append(schedule.tolist()) new_event_groups.append(event) response = groups_to_dict(new_event_groups, value_groups, generic_asset_type_name="event") response["start"] = isodate.datetime_isoformat(start) response["duration"] = isodate.duration_isoformat(duration) response["unit"] = unit d, s = request_processed() return dict(**response, **d), s
def post_udi_event_response(unit): # noqa: C901 if not has_assets(): current_app.logger.info("User doesn't seem to have any assets.") form = get_form_from_request(request) # check datetime, or use server_now if "datetime" not in form: return invalid_datetime("Missing datetime parameter.") else: datetime = parse_isodate_str(form.get("datetime")) if datetime is None: return invalid_datetime( "Cannot parse datetime string %s as iso date" % form.get("datetime")) if datetime.tzinfo is None: current_app.logger.warning( "Cannot parse timezone of 'datetime' value %s" % form.get("datetime")) return invalid_timezone( "Datetime should explicitly state a timezone.") # parse event/address info if "event" not in form: return invalid_domain("No event identifier sent.") try: ea = parse_entity_address(form.get("event"), entity_type="event", fm_scheme="fm0") except EntityAddressException as eae: return invalid_domain(str(eae)) sensor_id = ea["asset_id"] event_id = ea["event_id"] event_type = ea["event_type"] if event_type != "soc": return unrecognized_event(event_id, event_type) # Look for the Sensor object sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none() if sensor is None or not can_access_asset(sensor): current_app.logger.warning("Cannot identify sensor via %s." % ea) return unrecognized_connection_group() if sensor.generic_asset.generic_asset_type.name != "battery": return invalid_domain( "API version 1.2 only supports UDI events for batteries. " "Sensor ID:%s does not belong to a battery." % sensor_id) # unless on play, keep events ordered by entry date and ID if current_app.config.get("FLEXMEASURES_MODE") != "play": # do not allow new date to be after last date if (isinstance(sensor.generic_asset.get_attribute("soc_datetime"), str) and datetime.fromisoformat( sensor.generic_asset.get_attribute("soc_datetime")) >= datetime): msg = "The date of the requested UDI event (%s) is earlier than the latest known date (%s)." % ( datetime, datetime.fromisoformat( sensor.generic_asset.get_attribute("soc_datetime")), ) current_app.logger.warning(msg) return invalid_datetime(msg) # check if udi event id is higher than existing soc_udi_event_id = sensor.generic_asset.get_attribute( "soc_udi_event_id") if soc_udi_event_id is not None and soc_udi_event_id >= event_id: return outdated_event_id(event_id, soc_udi_event_id) # get value if "value" not in form: return ptus_incomplete() value = form.get("value") if unit == "kWh": value = value / 1000.0 # Store new soc info as GenericAsset attributes sensor.generic_asset.set_attribute("soc_datetime", datetime.isoformat()) sensor.generic_asset.set_attribute("soc_udi_event_id", event_id) sensor.generic_asset.set_attribute("soc_in_mwh", value) db.session.commit() return request_processed("Request has been processed.")
def post_udi_event_response(unit): # noqa: C901 if not has_assets(): current_app.logger.info("User doesn't seem to have any assets.") form = get_form_from_request(request) # check datetime, or use server_now if "datetime" not in form: return invalid_datetime("Missing datetime parameter.") else: datetime = parse_isodate_str(form.get("datetime")) if datetime is None: return invalid_datetime( "Cannot parse datetime string %s as iso date" % form.get("datetime") ) if datetime.tzinfo is None: current_app.logger.warning( "Cannot parse timezone of 'datetime' value %s" % form.get("datetime") ) return invalid_timezone("Datetime should explicitly state a timezone.") # parse event/address info if "event" not in form: return invalid_domain("No event identifier sent.") try: ea = parse_entity_address(form.get("event"), entity_type="event") except EntityAddressException as eae: return invalid_domain(str(eae)) asset_id = ea["asset_id"] event_id = ea["event_id"] event_type = ea["event_type"] if event_type != "soc": return unrecognized_event(event_id, event_type) # get asset asset: Asset = Asset.query.filter_by(id=asset_id).one_or_none() if asset is None or not can_access_asset(asset): current_app.logger.warning("Cannot identify asset via %s." % ea) return unrecognized_connection_group() if asset.asset_type_name != "battery": return invalid_domain( "API version 1.2 only supports UDI events for batteries. Asset ID:%s is not a battery." % asset_id ) # unless on play, keep events ordered by entry date and ID if current_app.config.get("FLEXMEASURES_MODE") != "play": # do not allow new date to be after last date if asset.soc_datetime is not None: if asset.soc_datetime >= datetime: msg = ( "The date of the requested UDI event (%s) is earlier than the latest known date (%s)." % (datetime, asset.soc_datetime) ) current_app.logger.warning(msg) return invalid_datetime(msg) # check if udi event id is higher than existing if asset.soc_udi_event_id is not None: if asset.soc_udi_event_id >= event_id: return outdated_event_id(event_id, asset.soc_udi_event_id) # get value if "value" not in form: return ptus_incomplete() value = form.get("value") if unit == "kWh": value = value / 1000.0 # store new soc in asset asset.soc_datetime = datetime asset.soc_udi_event_id = event_id asset.soc_in_mwh = value db.session.commit() return request_processed("Request has been processed.")
def create_connection_and_value_groups( # noqa: C901 unit, generic_asset_name_groups, value_groups, horizon, rolling, start, duration): """ Code for POSTing Power values to the API. Only lets users post to assets they own. The sign of values is validated according to asset specs, but in USEF terms. Then, we store the reverse sign for FlexMeasures specs (with positive production and negative consumption). If power values are not forecasts, forecasting jobs are created. """ from flask import current_app current_app.logger.info("POSTING POWER DATA") data_source = get_or_create_user_data_source(current_user) user_assets = get_assets() if not user_assets: current_app.logger.info("User doesn't seem to have any assets") user_asset_ids = [asset.id for asset in user_assets] power_measurements = [] forecasting_jobs = [] for connection_group, value_group in zip(generic_asset_name_groups, value_groups): for connection in connection_group: # TODO: get asset through util function after refactoring # Parse the entity address try: connection = parse_entity_address(connection, entity_type="connection") except EntityAddressException as eae: return invalid_domain(str(eae)) asset_id = connection["asset_id"] # Look for the Asset object if asset_id in user_asset_ids: asset = Asset.query.filter(Asset.id == asset_id).one_or_none() else: current_app.logger.warning("Cannot identify connection %s" % connection) return unrecognized_connection_group() # Validate the sign of the values (following USEF specs with positive consumption and negative production) if asset.is_pure_consumer and any(v < 0 for v in value_group): extra_info = ( "Connection %s is registered as a pure consumer and can only receive non-negative values." % asset.entity_address) return power_value_too_small(extra_info) elif asset.is_pure_producer and any(v > 0 for v in value_group): extra_info = ( "Connection %s is registered as a pure producer and can only receive non-positive values." % asset.entity_address) return power_value_too_big(extra_info) # Create new Power objects for j, value in enumerate(value_group): dt = start + j * duration / len(value_group) if rolling: h = horizon else: # Deduct the difference in end times of the individual timeslot and the timeseries duration h = horizon - ((start + duration) - (dt + duration / len(value_group))) p = Power( datetime=dt, value=value * -1, # Reverse sign for FlexMeasures specs with positive production and negative consumption horizon=h, asset_id=asset.id, data_source_id=data_source.id, ) power_measurements.append(p) # make forecasts, but only if the sent-in values are not forecasts themselves if horizon <= timedelta( hours=0 ): # Todo: replace 0 hours with whatever the moment of switching from ex-ante to ex-post is for this generic asset forecasting_jobs.extend( create_forecasting_jobs( "Power", asset_id, start, start + duration, resolution=duration / len(value_group), enqueue=False, )) current_app.logger.info("SAVING TO DB AND QUEUEING...") try: save_to_session(power_measurements) db.session.flush() [ current_app.queues["forecasting"].enqueue_job(job) for job in forecasting_jobs ] db.session.commit() return request_processed() except IntegrityError as e: current_app.logger.warning(e) db.session.rollback() # Allow meter data to be replaced only in play mode if current_app.config.get("FLEXMEASURES_MODE", "") == "play": save_to_session(power_measurements, overwrite=True) [ current_app.queues["forecasting"].enqueue_job(job) for job in forecasting_jobs ] db.session.commit() return request_processed() else: return already_received_and_successfully_processed()
def collect_connection_and_value_groups( unit: str, resolution: str, belief_horizon_window: Tuple[Union[None, timedelta], Union[None, timedelta]], belief_time_window: Tuple[Optional[datetime_type], Optional[datetime_type]], start: datetime_type, duration: timedelta, connection_groups: List[List[str]], user_source_ids: Union[ int, List[int]] = None, # None is interpreted as all sources source_types: List[str] = None, ) -> Tuple[dict, int]: """ Code for GETting power values from the API. Only allows to get values from assets owned by current user. Returns value sign in accordance with USEF specs (with negative production and positive consumption). """ from flask import current_app current_app.logger.info("GETTING") user_assets = get_assets() if not user_assets: current_app.logger.info("User doesn't seem to have any assets") user_asset_ids = [asset.id for asset in user_assets] end = start + duration value_groups = [] new_connection_groups = ( [] ) # Each connection in the old connection groups will be interpreted as a separate group for connections in connection_groups: # Get the asset names asset_names: List[str] = [] for connection in connections: # Parse the entity address try: connection_details = parse_entity_address( connection, entity_type="connection") except EntityAddressException as eae: return invalid_domain(str(eae)) asset_id = connection_details["asset_id"] # Look for the Asset object if asset_id in user_asset_ids: asset = Asset.query.filter(Asset.id == asset_id).one_or_none() else: current_app.logger.warning("Cannot identify connection %s" % connection) return unrecognized_connection_group() asset_names.append(asset.name) # Get the power values # TODO: fill NaN for non-existing values power_bdf_dict: Dict[str, tb.BeliefsDataFrame] = Power.collect( generic_asset_names=asset_names, query_window=(start, end), resolution=resolution, belief_horizon_window=belief_horizon_window, belief_time_window=belief_time_window, user_source_ids=user_source_ids, source_types=source_types, sum_multiple=False, ) # Todo: parse time window of power_bdf_dict, which will be different for requests that are not of the form: # - start is a timestamp on the hour or a multiple of 15 minutes thereafter # - duration is a multiple of 15 minutes for k, bdf in power_bdf_dict.items(): value_groups.append( [x * -1 for x in bdf["event_value"].tolist()] ) # Reverse sign of values (from FlexMeasures specs to USEF specs) new_connection_groups.append(k) response = groups_to_dict(new_connection_groups, value_groups, generic_asset_type_name="connection") response["start"] = isodate.datetime_isoformat(start) response["duration"] = isodate.duration_isoformat(duration) response["unit"] = unit # TODO: convert to requested unit d, s = request_processed() return dict(**response, **d), s
def save_to_db( timed_values: Union[BeliefsDataFrame, List[Union[Power, Price, Weather]]], forecasting_jobs: List[Job] = [], save_changed_beliefs_only: bool = True, ) -> ResponseTuple: """Put the timed values into the database and enqueue forecasting jobs. Data can only be replaced on servers in play mode. TODO: remove this legacy function in its entirety (announced v0.8.0) :param timed_values: BeliefsDataFrame or a list of Power, Price or Weather values to be saved :param forecasting_jobs: list of forecasting Jobs for redis queues. :param save_changed_beliefs_only: if True, beliefs that are already stored in the database with an earlier belief time are dropped. :returns: ResponseTuple """ import warnings warnings.warn( "The method api.common.utils.api_utils.save_to_db is deprecated. Check out the following replacements:" "- [recommended option] to store BeliefsDataFrames only, switch to data.utils.save_to_db" "- to store BeliefsDataFrames and enqueue jobs, switch to api.common.utils.api_utils.save_and_enqueue" ) if isinstance(timed_values, BeliefsDataFrame): if save_changed_beliefs_only: # Drop beliefs that haven't changed timed_values = ( timed_values.convert_index_from_belief_horizon_to_time() .groupby(level=["belief_time", "source"], as_index=False) .apply(drop_unchanged_beliefs) ) # Work around bug in which groupby still introduces an index level, even though we asked it not to if None in timed_values.index.names: timed_values.index = timed_values.index.droplevel(None) if timed_values.empty: current_app.logger.debug("Nothing new to save") return already_received_and_successfully_processed() current_app.logger.info("SAVING TO DB AND QUEUEING...") try: if isinstance(timed_values, BeliefsDataFrame): TimedBelief.add_to_session( session=db.session, beliefs_data_frame=timed_values ) else: save_to_session(timed_values) db.session.flush() [current_app.queues["forecasting"].enqueue_job(job) for job in forecasting_jobs] db.session.commit() return request_processed() except IntegrityError as e: current_app.logger.warning(e) db.session.rollback() # Possibly allow data to be replaced depending on config setting if current_app.config.get("FLEXMEASURES_ALLOW_DATA_OVERWRITE", False): if isinstance(timed_values, BeliefsDataFrame): TimedBelief.add_to_session( session=db.session, beliefs_data_frame=timed_values, allow_overwrite=True, ) else: save_to_session(timed_values, overwrite=True) [ current_app.queues["forecasting"].enqueue_job(job) for job in forecasting_jobs ] db.session.commit() return request_processed() else: return already_received_and_successfully_processed()