class SensorAnnotationRelationship(db.Model):
"""Links annotations to sensors."""
__tablename__ = "annotations_sensors"
id = db.Column(db.Integer(), primary_key=True)
sensor_id = db.Column(db.Integer, db.ForeignKey("sensor.id"))
annotation_id = db.Column(db.Integer, db.ForeignKey("annotation.id"))
__table_args__ = (db.UniqueConstraint(
"annotation_id",
"sensor_id",
name="annotations_sensors_annotation_id_key",
), )
class GenericAssetAnnotationRelationship(db.Model):
"""Links annotations to generic assets."""
__tablename__ = "annotations_assets"
id = db.Column(db.Integer(), primary_key=True)
generic_asset_id = db.Column(db.Integer, db.ForeignKey("generic_asset.id"))
annotation_id = db.Column(db.Integer, db.ForeignKey("annotation.id"))
__table_args__ = (db.UniqueConstraint(
"annotation_id",
"generic_asset_id",
name="annotations_assets_annotation_id_key",
), )
class AccountAnnotationRelationship(db.Model):
"""Links annotations to accounts."""
__tablename__ = "annotations_accounts"
id = db.Column(db.Integer(), primary_key=True)
account_id = db.Column(db.Integer, db.ForeignKey("account.id"))
annotation_id = db.Column(db.Integer, db.ForeignKey("annotation.id"))
__table_args__ = (db.UniqueConstraint(
"annotation_id",
"account_id",
name="annotations_accounts_annotation_id_key",
), )
class Price(TimedValue, db.Model):
"""
All prices are stored in one slim table.
This model is now considered legacy. See TimedBelief.
"""
sensor_id = db.Column(db.Integer(),
db.ForeignKey("sensor.id"),
primary_key=True,
index=True)
sensor = db.relationship("Sensor", backref=db.backref("prices", lazy=True))
@classmethod
def make_query(cls, **kwargs) -> Query:
"""Construct the database query."""
return super().make_query(**kwargs)
def __init__(self, use_legacy_kwargs: bool = True, **kwargs):
# todo: deprecate the 'market_id' argument in favor of 'sensor_id' (announced v0.8.0)
if "market_id" in kwargs and "sensor_id" not in kwargs:
kwargs["sensor_id"] = tb_utils.replace_deprecated_argument(
"market_id",
kwargs["market_id"],
"sensor_id",
None,
)
kwargs.pop("market_id", None)
# todo: deprecate the 'Price' class in favor of 'TimedBelief' (announced v0.8.0)
if use_legacy_kwargs is False:
# Create corresponding TimedBelief
belief = TimedBelief(**kwargs)
db.session.add(belief)
# Convert key names for legacy model
kwargs["value"] = kwargs.pop("event_value")
kwargs["datetime"] = kwargs.pop("event_start")
kwargs["horizon"] = kwargs.pop("belief_horizon")
kwargs["sensor_id"] = kwargs.pop("sensor").id
kwargs["data_source_id"] = kwargs.pop("source").id
else:
import warnings
warnings.warn(
f"The {self.__class__} class is deprecated. Switch to using the TimedBelief class to suppress this warning.",
FutureWarning,
)
super(Price, self).__init__(**kwargs)
class Market(db.Model, tb.SensorDBMixin):
"""
Each market is a pricing service.
This model is now considered legacy. See GenericAsset and Sensor.
"""
id = db.Column(db.Integer,
db.ForeignKey("sensor.id"),
primary_key=True,
autoincrement=True)
name = db.Column(db.String(80), unique=True)
display_name = db.Column(db.String(80), default="", unique=True)
market_type_name = db.Column(db.String(80),
db.ForeignKey("market_type.name"),
nullable=False)
def __init__(self, **kwargs):
# Set default knowledge horizon function for an economic sensor
if "knowledge_horizon_fnc" not in kwargs:
kwargs[
"knowledge_horizon_fnc"] = knowledge_horizons.ex_ante.__name__
if "knowledge_horizon_par" not in kwargs:
kwargs["knowledge_horizon_par"] = {
knowledge_horizons.ex_ante.__code__.co_varnames[1]: "PT0H"
}
kwargs["name"] = kwargs["name"].replace(" ", "_").lower()
if "display_name" not in kwargs:
kwargs["display_name"] = humanize(kwargs["name"])
super(Market, self).__init__(**kwargs)
# Create a new Sensor with unique id across assets, markets and weather sensors
if "id" not in kwargs:
market_type = get_old_model_type(kwargs, MarketType,
"market_type_name", "market_type")
generic_asset_kwargs = {
**kwargs,
**copy_old_sensor_attributes(
self,
old_sensor_type_attributes=[],
old_sensor_attributes=[
"display_name",
],
old_sensor_type=market_type,
),
}
new_generic_asset = create_generic_asset("market",
**generic_asset_kwargs)
new_sensor = Sensor(
name=kwargs["name"],
generic_asset=new_generic_asset,
**copy_old_sensor_attributes(
self,
old_sensor_type_attributes=[
"daily_seasonality",
"weekly_seasonality",
"yearly_seasonality",
],
old_sensor_attributes=[
"display_name",
],
old_sensor_type=market_type,
),
)
db.session.add(new_sensor)
db.session.flush() # generates the pkey for new_sensor
new_sensor_id = new_sensor.id
else:
# The UI may initialize Market objects from API form data with a known id
new_sensor_id = kwargs["id"]
self.id = new_sensor_id
# Copy over additional columns from (newly created) Market to (newly created) Sensor
if "id" not in kwargs:
db.session.add(self)
db.session.flush(
) # make sure to generate each column for the old sensor
new_sensor.unit = self.unit
new_sensor.event_resolution = self.event_resolution
new_sensor.knowledge_horizon_fnc = self.knowledge_horizon_fnc
new_sensor.knowledge_horizon_par = self.knowledge_horizon_par
@property
def entity_address_fm0(self) -> str:
"""Entity address under the fm0 scheme for entity addresses."""
return build_entity_address(dict(market_name=self.name),
"market",
fm_scheme="fm0")
@property
def entity_address(self) -> str:
"""Entity address under the latest fm scheme for entity addresses."""
return build_entity_address(dict(sensor_id=self.id), "sensor")
@property
def corresponding_sensor(self) -> Sensor:
return db.session.query(Sensor).get(self.id)
@property
def generic_asset(self) -> GenericAsset:
return db.session.query(GenericAsset).get(self.corresponding_sensor.id)
def get_attribute(self, attribute: str):
"""Looks for the attribute on the corresponding Sensor.
This should be used by all code to read these attributes,
over accessing them directly on this class,
as this table is in the process to be replaced by the Sensor table.
"""
return self.corresponding_sensor.get_attribute(attribute)
@property
def price_unit(self) -> str:
"""Return the 'unit' property of the generic asset, just with a more insightful name."""
return self.unit
market_type = db.relationship("MarketType",
backref=db.backref("markets", lazy=True))
def __repr__(self):
return "<Market %s:%r (%r) res.: %s>" % (
self.id,
self.name,
self.market_type_name,
self.event_resolution,
)
def to_dict(self) -> Dict[str, str]:
return dict(name=self.name, market_type=self.market_type.name)
class DataSource(db.Model, tb.BeliefSourceDBMixin):
"""Each data source is a data-providing entity."""
__tablename__ = "data_source"
__table_args__ = (db.UniqueConstraint("name", "user_id", "model",
"version"), )
# The type of data source (e.g. user, forecasting script or scheduling script)
type = db.Column(db.String(80), default="")
# The id of the user source (can link e.g. to fm_user table)
user_id = db.Column(db.Integer,
db.ForeignKey("fm_user.id"),
nullable=True,
unique=True)
user = db.relationship("User",
backref=db.backref("data_source", lazy=True))
# The model and version of a script source
model = db.Column(db.String(80), nullable=True)
version = db.Column(
db.String(17), # length supports up to version 999.999.999dev999
nullable=True,
)
def __init__(
self,
name: Optional[str] = None,
type: Optional[str] = None,
user: Optional[User] = None,
**kwargs,
):
if user is not None:
name = user.username
type = "user"
self.user = user
elif user is None and type == "user":
raise TypeError(
"A data source cannot have type 'user' but no user set.")
self.type = type
tb.BeliefSourceDBMixin.__init__(self, name=name)
db.Model.__init__(self, **kwargs)
@property
def label(self):
"""Human-readable label (preferably not starting with a capital letter so it can be used in a sentence)."""
if self.type == "user":
return f"data entered by user {self.user.username}" # todo: give users a display name
elif self.type == "forecasting script":
return f"forecast by {self.name}" # todo: give DataSource an optional db column to persist versioned models separately to the name of the data source?
elif self.type == "scheduling script":
return f"schedule by {self.name}"
elif self.type == "crawling script":
return f"data retrieved from {self.name}"
elif self.type in ("demo script", "CLI script"):
return f"demo data entered by {self.name}"
else:
return f"data from {self.name}"
@property
def description(self):
"""Extended description
For example:
>>> DataSource("Seita", type="forecasting script", model="naive", version="1.2").description
<<< "Seita's naive model v1.2.0"
"""
descr = self.name
if self.model:
descr += f"'s {self.model} model"
if self.version:
descr += f" v{self.version}"
return descr
def __repr__(self):
return "<Data source %r (%s)>" % (self.id, self.description)
def __str__(self):
return self.description
def data_source_id(cls): # noqa: B902
return db.Column(db.Integer,
db.ForeignKey("data_source.id"),
primary_key=True)
class Sensor(db.Model, tb.SensorDBMixin, AuthModelMixin):
"""A sensor measures events."""
attributes = db.Column(MutableDict.as_mutable(db.JSON),
nullable=False,
default={})
generic_asset_id = db.Column(
db.Integer,
db.ForeignKey("generic_asset.id", ondelete="CASCADE"),
nullable=False,
)
generic_asset = db.relationship(
"GenericAsset",
foreign_keys=[generic_asset_id],
backref=db.backref("sensors",
lazy=True,
cascade="all, delete-orphan",
passive_deletes=True),
)
annotations = db.relationship(
"Annotation",
secondary="annotations_sensors",
backref=db.backref("sensors", lazy="dynamic"),
)
def __init__(
self,
name: str,
generic_asset: Optional[GenericAsset] = None,
generic_asset_id: Optional[int] = None,
attributes: Optional[dict] = None,
**kwargs,
):
assert (generic_asset is None) ^ (
generic_asset_id is
None), "Either generic_asset_id or generic_asset must be set."
tb.SensorDBMixin.__init__(self, name, **kwargs)
tb_utils.remove_class_init_kwargs(tb.SensorDBMixin, kwargs)
if generic_asset is not None:
kwargs["generic_asset"] = generic_asset
else:
kwargs["generic_asset_id"] = generic_asset_id
if attributes is not None:
kwargs["attributes"] = attributes
db.Model.__init__(self, **kwargs)
__table_args__ = (UniqueConstraint(
"name",
"generic_asset_id",
name="sensor_name_generic_asset_id_key",
), )
def __acl__(self):
"""
All logged-in users can read if the sensor belongs to a public asset.
Within same account, everyone can create, read and update.
Deletion is left to account admins.
"""
return {
"create-children":
f"account:{self.generic_asset.account_id}",
"read":
f"account:{self.generic_asset.account_id}"
if self.generic_asset.account_id is not None else
EVERY_LOGGED_IN_USER,
"update":
f"account:{self.generic_asset.account_id}",
"delete": (
f"account:{self.generic_asset.account_id}",
"role:account-admin",
),
}
@property
def entity_address(self) -> str:
return build_entity_address(dict(sensor_id=self.id), "sensor")
@property
def location(self) -> Optional[Tuple[float, float]]:
location = (self.get_attribute("latitude"),
self.get_attribute("longitude"))
if None not in location:
return location
@property
def measures_power(self) -> bool:
"""True if this sensor's unit is measuring power"""
return is_power_unit(self.unit)
@property
def measures_energy(self) -> bool:
"""True if this sensor's unit is measuring energy"""
return is_energy_unit(self.unit)
@property
def is_strictly_non_positive(self) -> bool:
"""Return True if this sensor strictly records non-positive values."""
return self.get_attribute(
"is_consumer",
False) and not self.get_attribute("is_producer", True)
@property
def is_strictly_non_negative(self) -> bool:
"""Return True if this sensor strictly records non-negative values."""
return self.get_attribute(
"is_producer",
False) and not self.get_attribute("is_consumer", True)
def get_attribute(self, attribute: str, default: Any = None) -> Any:
"""Looks for the attribute on the Sensor.
If not found, looks for the attribute on the Sensor's GenericAsset.
If not found, returns the default.
"""
if hasattr(self, attribute):
return getattr(self, attribute)
if attribute in self.attributes:
return self.attributes[attribute]
if hasattr(self.generic_asset, attribute):
return getattr(self.generic_asset, attribute)
if attribute in self.generic_asset.attributes:
return self.generic_asset.attributes[attribute]
return default
def has_attribute(self, attribute: str) -> bool:
return (attribute in self.attributes
or attribute in self.generic_asset.attributes)
def set_attribute(self, attribute: str, value):
if self.has_attribute(attribute):
self.attributes[attribute] = value
def check_required_attributes(
self,
attributes: List[Union[str, Tuple[str, Union[Type, Tuple[Type,
...]]]]],
):
"""Raises if any attribute in the list of attributes is missing, or has the wrong type.
:param attributes: List of either an attribute name or a tuple of an attribute name and its allowed type
(the allowed type may also be a tuple of several allowed types)
"""
check_required_attributes(self, attributes)
def latest_state(
self,
source: Optional[Union[DataSource, List[DataSource], int, List[int],
str, List[str]]] = None,
) -> tb.BeliefsDataFrame:
"""Search the most recent event for this sensor, and return the most recent ex-post belief.
:param source: search only beliefs by this source (pass the DataSource, or its name or id) or list of sources
"""
return self.search_beliefs(
horizons_at_most=timedelta(0),
source=source,
most_recent_beliefs_only=True,
most_recent_events_only=True,
one_deterministic_belief_per_event=True,
)
def search_annotations(
self,
annotation_starts_after: Optional[datetime_type] = None,
annotation_ends_before: Optional[datetime_type] = None,
source: Optional[Union[DataSource, List[DataSource], int, List[int],
str, List[str]]] = None,
include_asset_annotations: bool = False,
include_account_annotations: bool = False,
):
parsed_sources = parse_source_arg(source)
query = Annotation.query.join(SensorAnnotationRelationship).filter(
SensorAnnotationRelationship.sensor_id == self.id,
SensorAnnotationRelationship.annotation_id == Annotation.id,
)
if annotation_starts_after is not None:
query = query.filter(Annotation.start >= annotation_starts_after, )
if annotation_ends_before is not None:
query = query.filter(Annotation.end <= annotation_ends_before, )
if parsed_sources:
query = query.filter(Annotation.source.in_(parsed_sources), )
annotations = query.all()
if include_asset_annotations:
annotations += self.generic_asset.search_annotations(
annotation_starts_before=annotation_starts_after,
annotation_ends_before=annotation_ends_before,
source=source,
)
if include_account_annotations:
annotations += self.generic_asset.owner.search_annotations(
annotation_starts_before=annotation_starts_after,
annotation_ends_before=annotation_ends_before,
source=source,
)
return annotations
def search_beliefs(
self,
event_starts_after: Optional[datetime_type] = None,
event_ends_before: Optional[datetime_type] = None,
beliefs_after: Optional[datetime_type] = None,
beliefs_before: Optional[datetime_type] = None,
horizons_at_least: Optional[timedelta] = None,
horizons_at_most: Optional[timedelta] = None,
source: Optional[Union[DataSource, List[DataSource], int, List[int],
str, List[str]]] = None,
most_recent_beliefs_only: bool = True,
most_recent_events_only: bool = False,
most_recent_only: bool = None, # deprecated
one_deterministic_belief_per_event: bool = False,
as_json: bool = False,
) -> Union[tb.BeliefsDataFrame, str]:
"""Search all beliefs about events for this sensor.
If you don't set any filters, you get the most recent beliefs about all events.
:param event_starts_after: only return beliefs about events that start after this datetime (inclusive)
:param event_ends_before: only return beliefs about events that end before this datetime (inclusive)
:param beliefs_after: only return beliefs formed after this datetime (inclusive)
:param beliefs_before: only return beliefs formed before this datetime (inclusive)
:param horizons_at_least: only return beliefs with a belief horizon equal or greater than this timedelta (for example, use timedelta(0) to get ante knowledge time beliefs)
:param horizons_at_most: only return beliefs with a belief horizon equal or less than this timedelta (for example, use timedelta(0) to get post knowledge time beliefs)
:param source: search only beliefs by this source (pass the DataSource, or its name or id) or list of sources
:param most_recent_beliefs_only: only return the most recent beliefs for each event from each source (minimum belief horizon)
:param most_recent_events_only: only return (post knowledge time) beliefs for the most recent event (maximum event start)
:param one_deterministic_belief_per_event: only return a single value per event (no probabilistic distribution)
:param as_json: return beliefs in JSON format (e.g. for use in charts) rather than as BeliefsDataFrame
:returns: BeliefsDataFrame or JSON string (if as_json is True)
"""
# todo: deprecate the 'most_recent_only' argument in favor of 'most_recent_beliefs_only' (announced v0.8.0)
most_recent_beliefs_only = tb_utils.replace_deprecated_argument(
"most_recent_only",
most_recent_only,
"most_recent_beliefs_only",
most_recent_beliefs_only,
required_argument=False,
)
bdf = TimedBelief.search(
sensors=self,
event_starts_after=event_starts_after,
event_ends_before=event_ends_before,
beliefs_after=beliefs_after,
beliefs_before=beliefs_before,
horizons_at_least=horizons_at_least,
horizons_at_most=horizons_at_most,
source=source,
most_recent_beliefs_only=most_recent_beliefs_only,
most_recent_events_only=most_recent_events_only,
one_deterministic_belief_per_event=
one_deterministic_belief_per_event,
)
if as_json:
df = bdf.reset_index()
df["source"] = df["source"].astype(str)
return df.to_json(orient="records")
return bdf
def chart(
self,
chart_type: str = "bar_chart",
event_starts_after: Optional[datetime_type] = None,
event_ends_before: Optional[datetime_type] = None,
beliefs_after: Optional[datetime_type] = None,
beliefs_before: Optional[datetime_type] = None,
source: Optional[Union[DataSource, List[DataSource], int, List[int],
str, List[str]]] = None,
most_recent_beliefs_only: bool = True,
include_data: bool = False,
dataset_name: Optional[str] = None,
**kwargs,
) -> dict:
"""Create a chart showing sensor data.
:param chart_type: currently only "bar_chart" # todo: where can we properly list the available chart types?
:param event_starts_after: only return beliefs about events that start after this datetime (inclusive)
:param event_ends_before: only return beliefs about events that end before this datetime (inclusive)
:param beliefs_after: only return beliefs formed after this datetime (inclusive)
:param beliefs_before: only return beliefs formed before this datetime (inclusive)
:param source: search only beliefs by this source (pass the DataSource, or its name or id) or list of sources
:param most_recent_beliefs_only: only return the most recent beliefs for each event from each source (minimum belief horizon)
:param include_data: if True, include data in the chart, or if False, exclude data
:param dataset_name: optionally name the dataset used in the chart (the default name is sensor_<id>)
"""
# Set up chart specification
if dataset_name is None:
dataset_name = "sensor_" + str(self.id)
self.sensor_type = (
self.name
) # todo remove this placeholder when sensor types are modelled
chart_specs = chart_type_to_chart_specs(
chart_type,
sensor=self,
dataset_name=dataset_name,
**kwargs,
)
if include_data:
# Set up data
data = self.search_beliefs(
as_json=True,
event_starts_after=event_starts_after,
event_ends_before=event_ends_before,
beliefs_after=beliefs_after,
beliefs_before=beliefs_before,
most_recent_beliefs_only=most_recent_beliefs_only,
source=source,
)
# Combine chart specs and data
chart_specs["datasets"] = {dataset_name: json.loads(data)}
return chart_specs
@property
def timerange(self) -> Dict[str, datetime_type]:
"""Time range for which sensor data exists.
:returns: dictionary with start and end, for example:
{
'start': datetime.datetime(2020, 12, 3, 14, 0, tzinfo=pytz.utc),
'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc)
}
"""
least_recent_query = (TimedBelief.query.filter(
TimedBelief.sensor == self).order_by(
TimedBelief.event_start.asc()).limit(1))
most_recent_query = (TimedBelief.query.filter(
TimedBelief.sensor == self).order_by(
TimedBelief.event_start.desc()).limit(1))
results = least_recent_query.union_all(most_recent_query).all()
if not results:
# return now in case there is no data for the sensor
now = server_now()
return dict(start=now, end=now)
least_recent, most_recent = results
return dict(start=least_recent.event_start, end=most_recent.event_end)
def __repr__(self) -> str:
return f"<Sensor {self.id}: {self.name}, unit: {self.unit} res.: {self.event_resolution}>"
@classmethod
def find_closest(cls,
generic_asset_type_name: str,
sensor_name: str,
n: int = 1,
**kwargs) -> Union["Sensor", List["Sensor"], None]:
"""Returns the closest n sensors within a given asset type (as a list if n > 1).
Parses latitude and longitude values stated in kwargs.
Can be called with an object that has latitude and longitude properties, for example:
sensor = Sensor.find_closest("weather_station", "wind speed", object=generic_asset)
Can also be called with latitude and longitude parameters, for example:
sensor = Sensor.find_closest("weather_station", "temperature", latitude=32, longitude=54)
sensor = Sensor.find_closest("weather_station", "temperature", lat=32, lng=54)
Finally, pass in an account_id parameter if you want to query an account other than your own. This only works for admins. Public assets are always queried.
"""
latitude, longitude = parse_lat_lng(kwargs)
account_id_filter = kwargs[
"account_id"] if "account_id" in kwargs else None
query = query_sensors_by_proximity(
latitude=latitude,
longitude=longitude,
generic_asset_type_name=generic_asset_type_name,
sensor_name=sensor_name,
account_id=account_id_filter,
)
if n == 1:
return query.first()
else:
return query.limit(n).all()
class Annotation(db.Model):
"""An annotation is a nominal value that applies to a specific time or time span.
Examples of annotation types:
- user annotation: annotation.type == "label" and annotation.source.type == "user"
- unresolved alert: annotation.type == "alert"
- resolved alert: annotation.type == "label" and annotation.source.type == "alerting script"
- organisation holiday: annotation.type == "holiday" and annotation.source.type == "user"
- public holiday: annotation.type == "holiday" and annotation.source.name == "workalendar"
"""
id = db.Column(db.Integer,
nullable=False,
autoincrement=True,
primary_key=True)
start = db.Column(db.DateTime(timezone=True), nullable=False)
end = db.Column(db.DateTime(timezone=True), nullable=False)
belief_time = db.Column(db.DateTime(timezone=True), nullable=True)
source_id = db.Column(db.Integer, db.ForeignKey("data_source.id"))
source = db.relationship(
"DataSource",
foreign_keys=[source_id],
backref=db.backref("annotations", lazy=True),
)
type = db.Column(
db.Enum("alert",
"holiday",
"label",
"feedback",
name="annotation_type"))
content = db.Column(db.String(1024), nullable=False)
__table_args__ = (db.UniqueConstraint(
"content",
"start",
"belief_time",
"source_id",
"type",
name="annotation_content_key",
), )
@property
def duration(self) -> timedelta:
return self.end - self.start
@classmethod
def add(
cls,
df: pd.DataFrame,
annotation_type: str,
expunge_session: bool = False,
allow_overwrite: bool = False,
bulk_save_objects: bool = False,
commit_transaction: bool = False,
) -> List["Annotation"]:
"""Add a data frame describing annotations to the database and return the Annotation objects.
:param df: Data frame describing annotations.
Expects the following columns (or multi-index levels):
- start
- end or duration
- content
- belief_time
- source
:param annotation_type: One of the possible Enum values for annotation.type
:param expunge_session: if True, all non-flushed instances are removed from the session before adding annotations.
Expunging can resolve problems you might encounter with states of objects in your session.
When using this option, you might want to flush newly-created objects which are not annotations
(e.g. a sensor or data source object).
:param allow_overwrite: if True, new objects are merged
if False, objects are added to the session or bulk saved
:param bulk_save_objects: if True, objects are bulk saved with session.bulk_save_objects(),
which is quite fast but has several caveats, see:
https://docs.sqlalchemy.org/orm/persistence_techniques.html#bulk-operations-caveats
if False, objects are added to the session with session.add_all()
:param commit_transaction: if True, the session is committed
if False, you can still add other data to the session
and commit it all within an atomic transaction
"""
df = df.reset_index()
starts = df["start"]
if "end" in df.columns:
ends = df["end"]
elif "start" in df.columns and "duration" in df.columns:
ends = df["start"] + df["duration"]
else:
raise ValueError(
"Missing 'end' column cannot be derived from columns 'start' and 'duration'."
)
values = df["content"]
belief_times = df["belief_time"]
sources = df["source"]
annotations = [
cls(
content=row[0],
start=row[1],
end=row[2],
belief_time=row[3],
source=row[4],
type=annotation_type,
) for row in zip(values, starts, ends, belief_times, sources)
]
# Deal with the database session
if expunge_session:
db.session.expunge_all()
if not allow_overwrite:
if bulk_save_objects:
db.session.bulk_save_objects(annotations)
else:
db.session.add_all(annotations)
else:
for annotation in annotations:
db.session.merge(annotation)
if commit_transaction:
db.session.commit()
return annotations
def __repr__(self) -> str:
return f"<Annotation {self.id}: {self.content} ({self.type}), start: {self.start} end: {self.end}, source: {self.source}>"
class Asset(db.Model, tb.SensorDBMixin):
"""
Each asset is an energy- consuming or producing hardware.
This model is now considered legacy. See GenericAsset and Sensor.
"""
id = db.Column(
db.Integer,
db.ForeignKey("sensor.id", ondelete="CASCADE"),
primary_key=True,
autoincrement=True,
)
# The name
name = db.Column(db.String(80), default="", unique=True)
# The name we want to see (don't unnecessarily capitalize, so it can be used in a sentence)
display_name = db.Column(db.String(80), default="", unique=True)
# The name of the assorted AssetType
asset_type_name = db.Column(
db.String(80), db.ForeignKey("asset_type.name"), nullable=False
)
# How many MW at peak usage
capacity_in_mw = db.Column(db.Float, nullable=False)
# State of charge in MWh and its datetime and udi event
min_soc_in_mwh = db.Column(db.Float, nullable=True)
max_soc_in_mwh = db.Column(db.Float, nullable=True)
soc_in_mwh = db.Column(db.Float, nullable=True)
soc_datetime = db.Column(db.DateTime(timezone=True), nullable=True)
soc_udi_event_id = db.Column(db.Integer, nullable=True)
# latitude is the North/South coordinate
latitude = db.Column(db.Float, nullable=False)
# longitude is the East/West coordinate
longitude = db.Column(db.Float, nullable=False)
# owner
owner_id = db.Column(db.Integer, db.ForeignKey("fm_user.id", ondelete="CASCADE"))
# market
market_id = db.Column(db.Integer, db.ForeignKey("market.id"), nullable=True)
def __init__(self, **kwargs):
if "unit" not in kwargs:
kwargs["unit"] = "MW" # current default
super(Asset, self).__init__(**kwargs)
# Create a new Sensor with unique id across assets, markets and weather sensors
# Also keep track of ownership by creating a GenericAsset and assigning the new Sensor to it.
if "id" not in kwargs:
asset_type = get_old_model_type(
kwargs, AssetType, "asset_type_name", "asset_type"
)
# Set up generic asset
generic_asset_kwargs = {
**kwargs,
**copy_old_sensor_attributes(
self,
old_sensor_type_attributes=[
"can_curtail",
"can_shift",
],
old_sensor_attributes=[
"display_name",
"min_soc_in_mwh",
"max_soc_in_mwh",
"soc_in_mwh",
"soc_datetime",
"soc_udi_event_id",
],
old_sensor_type=asset_type,
),
}
if "owner_id" in kwargs:
owner = User.query.get(kwargs["owner_id"])
if owner:
generic_asset_kwargs.update(account_id=owner.account_id)
new_generic_asset = create_generic_asset("asset", **generic_asset_kwargs)
# Set up sensor
new_sensor = Sensor(
name=kwargs["name"],
generic_asset=new_generic_asset,
**copy_old_sensor_attributes(
self,
old_sensor_type_attributes=[
"is_consumer",
"is_producer",
"daily_seasonality",
"weekly_seasonality",
"yearly_seasonality",
"weather_correlations",
],
old_sensor_attributes=[
"display_name",
"capacity_in_mw",
"market_id",
],
old_sensor_type=asset_type,
),
)
db.session.add(new_sensor)
db.session.flush() # generates the pkey for new_sensor
sensor_id = new_sensor.id
else:
# The UI may initialize Asset objects from API form data with a known id
sensor_id = kwargs["id"]
self.id = sensor_id
if self.unit != "MW":
raise Exception("FlexMeasures only supports MW as unit for now.")
self.name = self.name.replace(" (MW)", "")
if "display_name" not in kwargs:
self.display_name = humanize(self.name)
# Copy over additional columns from (newly created) Asset to (newly created) Sensor
if "id" not in kwargs:
db.session.add(self)
db.session.flush() # make sure to generate each column for the old sensor
new_sensor.unit = self.unit
new_sensor.event_resolution = self.event_resolution
new_sensor.knowledge_horizon_fnc = self.knowledge_horizon_fnc
new_sensor.knowledge_horizon_par = self.knowledge_horizon_par
asset_type = db.relationship("AssetType", backref=db.backref("assets", lazy=True))
owner = db.relationship(
"User",
backref=db.backref(
"assets", lazy=True, cascade="all, delete-orphan", passive_deletes=True
),
)
market = db.relationship("Market", backref=db.backref("assets", lazy=True))
def latest_state(self, event_ends_before: Optional[datetime] = None) -> "Power":
"""Search the most recent event for this sensor, optionally before some datetime."""
# todo: replace with Sensor.latest_state
power_query = (
Power.query.filter(Power.sensor_id == self.id)
.filter(Power.horizon <= timedelta(hours=0))
.order_by(Power.datetime.desc())
)
if event_ends_before is not None:
power_query = power_query.filter(
Power.datetime + self.event_resolution <= event_ends_before
)
return power_query.first()
@property
def corresponding_sensor(self) -> Sensor:
return db.session.query(Sensor).get(self.id)
@property
def generic_asset(self) -> GenericAsset:
return db.session.query(GenericAsset).get(self.corresponding_sensor.id)
def get_attribute(self, attribute: str):
"""Looks for the attribute on the corresponding Sensor.
This should be used by all code to read these attributes,
over accessing them directly on this class,
as this table is in the process to be replaced by the Sensor table.
"""
return self.corresponding_sensor.get_attribute(attribute)
@property
def power_unit(self) -> float:
"""Return the 'unit' property of the generic asset, just with a more insightful name."""
return self.unit
@property
def entity_address_fm0(self) -> str:
"""Entity address under the fm0 scheme for entity addresses."""
return build_entity_address(
dict(owner_id=self.owner_id, asset_id=self.id),
"connection",
fm_scheme="fm0",
)
@property
def entity_address(self) -> str:
"""Entity address under the latest fm scheme for entity addresses."""
return build_entity_address(dict(sensor_id=self.id), "sensor")
@property
def location(self) -> Tuple[float, float]:
return self.latitude, self.longitude
def capacity_factor_in_percent_for(self, load_in_mw) -> int:
if self.capacity_in_mw == 0:
return 0
return min(round((load_in_mw / self.capacity_in_mw) * 100, 2), 100)
@property
def is_pure_consumer(self) -> bool:
"""Return True if this asset is consuming but not producing."""
return self.asset_type.is_consumer and not self.asset_type.is_producer
@property
def is_pure_producer(self) -> bool:
"""Return True if this asset is producing but not consuming."""
return self.asset_type.is_producer and not self.asset_type.is_consumer
def to_dict(self) -> Dict[str, Union[str, float]]:
return dict(
name=self.name,
display_name=self.display_name,
asset_type_name=self.asset_type_name,
latitude=self.latitude,
longitude=self.longitude,
capacity_in_mw=self.capacity_in_mw,
)
def __repr__(self):
return "<Asset %s:%r (%s), res.: %s on market %s>" % (
self.id,
self.name,
self.asset_type_name,
self.event_resolution,
self.market,
)
class Power(TimedValue, db.Model):
"""
All measurements of power data are stored in one slim table.
Negative values indicate consumption.
This model is now considered legacy. See TimedBelief.
"""
sensor_id = db.Column(
db.Integer(),
db.ForeignKey("sensor.id", ondelete="CASCADE"),
primary_key=True,
index=True,
)
sensor = db.relationship(
"Sensor",
backref=db.backref(
"measurements",
lazy=True,
cascade="all, delete-orphan",
passive_deletes=True,
),
)
@classmethod
def make_query(
cls,
**kwargs,
) -> Query:
"""Construct the database query."""
return super().make_query(**kwargs)
def to_dict(self):
return {
"datetime": isodate.datetime_isoformat(self.datetime),
"sensor_id": self.sensor_id,
"value": self.value,
"horizon": self.horizon,
}
def __init__(self, use_legacy_kwargs: bool = True, **kwargs):
# todo: deprecate the 'asset_id' argument in favor of 'sensor_id' (announced v0.8.0)
if "asset_id" in kwargs and "sensor_id" not in kwargs:
kwargs["sensor_id"] = tb_utils.replace_deprecated_argument(
"asset_id",
kwargs["asset_id"],
"sensor_id",
None,
)
kwargs.pop("asset_id", None)
# todo: deprecate the 'Power' class in favor of 'TimedBelief' (announced v0.8.0)
if use_legacy_kwargs is False:
# Create corresponding TimedBelief
belief = TimedBelief(**kwargs)
db.session.add(belief)
# Convert key names for legacy model
kwargs["value"] = kwargs.pop("event_value")
kwargs["datetime"] = kwargs.pop("event_start")
kwargs["horizon"] = kwargs.pop("belief_horizon")
kwargs["sensor_id"] = kwargs.pop("sensor").id
kwargs["data_source_id"] = kwargs.pop("source").id
else:
import warnings
warnings.warn(
f"The {self.__class__} class is deprecated. Switch to using the TimedBelief class to suppress this warning.",
FutureWarning,
)
super(Power, self).__init__(**kwargs)
def __repr__(self):
return "<Power %.5f on Sensor %s at %s by DataSource %s, horizon %s>" % (
self.value,
self.sensor_id,
self.datetime,
self.data_source_id,
self.horizon,
)
