def test_rename_entity(hass_recorder):
"""Test statistics is migrated when entity_id is changed."""
hass = hass_recorder()
recorder = hass.data[DATA_INSTANCE]
setup_component(hass, "sensor", {})
entity_reg = mock_registry(hass)
reg_entry = entity_reg.async_get_or_create(
"sensor",
"test",
"unique_0000",
suggested_object_id="test1",
)
assert reg_entry.entity_id == "sensor.test1"
zero, four, states = record_states(hass)
hist = history.get_significant_states(hass, zero, four)
assert dict(states) == dict(hist)
for kwargs in ({}, {"statistic_ids": ["sensor.test1"]}):
stats = statistics_during_period(hass, zero, period="5minute", **kwargs)
assert stats == {}
stats = get_last_statistics(hass, 0, "sensor.test1", True)
assert stats == {}
recorder.do_adhoc_statistics(start=zero)
wait_recording_done(hass)
expected_1 = {
"statistic_id": "sensor.test1",
"start": process_timestamp_to_utc_isoformat(zero),
"end": process_timestamp_to_utc_isoformat(zero + timedelta(minutes=5)),
"mean": approx(14.915254237288135),
"min": approx(10.0),
"max": approx(20.0),
"last_reset": None,
"state": None,
"sum": None,
}
expected_stats1 = [
{**expected_1, "statistic_id": "sensor.test1"},
]
expected_stats2 = [
{**expected_1, "statistic_id": "sensor.test2"},
]
expected_stats99 = [
{**expected_1, "statistic_id": "sensor.test99"},
]
stats = statistics_during_period(hass, zero, period="5minute")
assert stats == {"sensor.test1": expected_stats1, "sensor.test2": expected_stats2}
entity_reg.async_update_entity(reg_entry.entity_id, new_entity_id="sensor.test99")
hass.block_till_done()
stats = statistics_during_period(hass, zero, period="5minute")
assert stats == {"sensor.test99": expected_stats99, "sensor.test2": expected_stats2}
def test_compile_hourly_statistics(hass_recorder):
"""Test compiling hourly statistics."""
hass = hass_recorder()
recorder = hass.data[DATA_INSTANCE]
setup_component(hass, "sensor", {})
zero, four, states = record_states(hass)
hist = history.get_significant_states(hass, zero, four)
assert dict(states) == dict(hist)
for kwargs in ({}, {"statistic_ids": ["sensor.test1"]}):
stats = statistics_during_period(hass, zero, **kwargs)
assert stats == {}
stats = get_last_statistics(hass, 0, "sensor.test1")
assert stats == {}
recorder.do_adhoc_statistics(period="hourly", start=zero)
recorder.do_adhoc_statistics(period="hourly", start=four)
wait_recording_done(hass)
expected_1 = {
"statistic_id": "sensor.test1",
"start": process_timestamp_to_utc_isoformat(zero),
"mean": approx(14.915254237288135),
"min": approx(10.0),
"max": approx(20.0),
"last_reset": None,
"state": None,
"sum": None,
}
expected_2 = {
"statistic_id": "sensor.test1",
"start": process_timestamp_to_utc_isoformat(four),
"mean": approx(20.0),
"min": approx(20.0),
"max": approx(20.0),
"last_reset": None,
"state": None,
"sum": None,
}
expected_stats1 = [
{
**expected_1, "statistic_id": "sensor.test1"
},
{
**expected_2, "statistic_id": "sensor.test1"
},
]
expected_stats2 = [
{
**expected_1, "statistic_id": "sensor.test2"
},
{
**expected_2, "statistic_id": "sensor.test2"
},
]
# Test statistics_during_period
stats = statistics_during_period(hass, zero)
assert stats == {
"sensor.test1": expected_stats1,
"sensor.test2": expected_stats2
}
stats = statistics_during_period(hass,
zero,
statistic_ids=["sensor.test2"])
assert stats == {"sensor.test2": expected_stats2}
stats = statistics_during_period(hass,
zero,
statistic_ids=["sensor.test3"])
assert stats == {}
# Test get_last_statistics
stats = get_last_statistics(hass, 0, "sensor.test1")
assert stats == {}
stats = get_last_statistics(hass, 1, "sensor.test1")
assert stats == {
"sensor.test1": [{
**expected_2, "statistic_id": "sensor.test1"
}]
}
stats = get_last_statistics(hass, 2, "sensor.test1")
assert stats == {"sensor.test1": expected_stats1[::-1]}
stats = get_last_statistics(hass, 3, "sensor.test1")
assert stats == {"sensor.test1": expected_stats1[::-1]}
stats = get_last_statistics(hass, 1, "sensor.test3")
assert stats == {}
def _compile_statistics( # noqa: C901
hass: HomeAssistant,
session: Session,
start: datetime.datetime,
end: datetime.datetime,
) -> list[StatisticResult]:
"""Compile statistics for all entities during start-end."""
result: list[StatisticResult] = []
sensor_states = _get_sensor_states(hass)
wanted_statistics = _wanted_statistics(sensor_states)
old_metadatas = statistics.get_metadata_with_session(
hass, session, statistic_ids=[i.entity_id for i in sensor_states])
# Get history between start and end
entities_full_history = [
i.entity_id for i in sensor_states
if "sum" in wanted_statistics[i.entity_id]
]
history_list = {}
if entities_full_history:
history_list = history.get_significant_states_with_session( # type: ignore
hass,
session,
start - datetime.timedelta.resolution,
end,
entity_ids=entities_full_history,
significant_changes_only=False,
)
entities_significant_history = [
i.entity_id for i in sensor_states
if "sum" not in wanted_statistics[i.entity_id]
]
if entities_significant_history:
_history_list = history.get_significant_states_with_session( # type: ignore
hass,
session,
start - datetime.timedelta.resolution,
end,
entity_ids=entities_significant_history,
)
history_list = {**history_list, **_history_list}
# If there are no recent state changes, the sensor's state may already be pruned
# from the recorder. Get the state from the state machine instead.
for _state in sensor_states:
if _state.entity_id not in history_list:
history_list[_state.entity_id] = (_state, )
for _state in sensor_states: # pylint: disable=too-many-nested-blocks
entity_id = _state.entity_id
if entity_id not in history_list:
continue
state_class = _state.attributes[ATTR_STATE_CLASS]
device_class = _state.attributes.get(ATTR_DEVICE_CLASS)
entity_history = history_list[entity_id]
unit, fstates = _normalize_states(hass, session, old_metadatas,
entity_history, device_class,
entity_id)
if not fstates:
continue
# Check metadata
if old_metadata := old_metadatas.get(entity_id):
if old_metadata[1]["unit_of_measurement"] != unit:
if WARN_UNSTABLE_UNIT not in hass.data:
hass.data[WARN_UNSTABLE_UNIT] = set()
if entity_id not in hass.data[WARN_UNSTABLE_UNIT]:
hass.data[WARN_UNSTABLE_UNIT].add(entity_id)
_LOGGER.warning(
"The %sunit of %s (%s) does not match the unit of already "
"compiled statistics (%s). Generation of long term statistics "
"will be suppressed unless the unit changes back to %s",
"normalized "
if device_class in DEVICE_CLASS_UNITS else "",
entity_id,
unit,
old_metadata[1]["unit_of_measurement"],
old_metadata[1]["unit_of_measurement"],
)
continue
# Set meta data
meta: StatisticMetaData = {
"has_mean": "mean" in wanted_statistics[entity_id],
"has_sum": "sum" in wanted_statistics[entity_id],
"name": None,
"source": RECORDER_DOMAIN,
"statistic_id": entity_id,
"unit_of_measurement": unit,
}
# Make calculations
stat: StatisticData = {"start": start}
if "max" in wanted_statistics[entity_id]:
stat["max"] = max(*itertools.islice(
zip(*fstates), 1)) # type: ignore[typeddict-item]
if "min" in wanted_statistics[entity_id]:
stat["min"] = min(*itertools.islice(
zip(*fstates), 1)) # type: ignore[typeddict-item]
if "mean" in wanted_statistics[entity_id]:
stat["mean"] = _time_weighted_average(fstates, start, end)
if "sum" in wanted_statistics[entity_id]:
last_reset = old_last_reset = None
new_state = old_state = None
_sum = 0.0
last_stats = statistics.get_last_statistics(
hass, 1, entity_id, False)
if entity_id in last_stats:
# We have compiled history for this sensor before, use that as a starting point
last_reset = old_last_reset = last_stats[entity_id][0][
"last_reset"]
new_state = old_state = last_stats[entity_id][0]["state"]
_sum = last_stats[entity_id][0]["sum"] or 0.0
for fstate, state in fstates:
# Deprecated, will be removed in Home Assistant 2021.11
if ("last_reset" not in state.attributes
and state_class == STATE_CLASS_MEASUREMENT):
continue
reset = False
if (state_class != STATE_CLASS_TOTAL_INCREASING
and (last_reset := _last_reset_as_utc_isoformat(
state.attributes.get("last_reset"), entity_id)) !=
old_last_reset and last_reset is not None):
if old_state is None:
_LOGGER.info(
"Compiling initial sum statistics for %s, zero point set to %s",
entity_id,
fstate,
)
else:
_LOGGER.info(
"Detected new cycle for %s, last_reset set to %s (old last_reset %s)",
entity_id,
last_reset,
old_last_reset,
)
reset = True
elif old_state is None and last_reset is None:
reset = True
_LOGGER.info(
"Compiling initial sum statistics for %s, zero point set to %s",
entity_id,
fstate,
)
elif state_class == STATE_CLASS_TOTAL_INCREASING:
try:
if old_state is None or reset_detected(
hass, entity_id, fstate, new_state, state):
reset = True
_LOGGER.info(
"Detected new cycle for %s, value dropped from %s to %s, "
"triggered by state with last_updated set to %s",
entity_id,
new_state,
state.last_updated.isoformat(),
fstate,
)
except HomeAssistantError:
continue
if reset:
# The sensor has been reset, update the sum
if old_state is not None:
_sum += new_state - old_state
# ..and update the starting point
new_state = fstate
old_last_reset = last_reset
# Force a new cycle for an existing sensor to start at 0
if old_state is not None:
old_state = 0.0
else:
old_state = new_state
else:
new_state = fstate
async def test_external_statistics(hass, hass_ws_client, caplog):
"""Test inserting external statistics."""
client = await hass_ws_client()
await async_init_recorder_component(hass)
assert "Compiling statistics for" not in caplog.text
assert "Statistics already compiled" not in caplog.text
zero = dt_util.utcnow()
period1 = zero.replace(minute=0, second=0,
microsecond=0) + timedelta(hours=1)
period2 = zero.replace(minute=0, second=0,
microsecond=0) + timedelta(hours=2)
external_statistics1 = {
"start": period1,
"last_reset": None,
"state": 0,
"sum": 2,
}
external_statistics2 = {
"start": period2,
"last_reset": None,
"state": 1,
"sum": 3,
}
external_metadata = {
"has_mean": False,
"has_sum": True,
"name": "Total imported energy",
"source": "test",
"statistic_id": "test:total_energy_import",
"unit_of_measurement": "kWh",
}
async_add_external_statistics(hass, external_metadata,
(external_statistics1, external_statistics2))
await async_wait_recording_done_without_instance(hass)
stats = statistics_during_period(hass, zero, period="hour")
assert stats == {
"test:total_energy_import": [
{
"statistic_id": "test:total_energy_import",
"start": period1.isoformat(),
"end": (period1 + timedelta(hours=1)).isoformat(),
"max": None,
"mean": None,
"min": None,
"last_reset": None,
"state": approx(0.0),
"sum": approx(2.0),
},
{
"statistic_id": "test:total_energy_import",
"start": period2.isoformat(),
"end": (period2 + timedelta(hours=1)).isoformat(),
"max": None,
"mean": None,
"min": None,
"last_reset": None,
"state": approx(1.0),
"sum": approx(3.0),
},
]
}
statistic_ids = list_statistic_ids(hass)
assert statistic_ids == [{
"has_mean": False,
"has_sum": True,
"statistic_id": "test:total_energy_import",
"name": "Total imported energy",
"source": "test",
"unit_of_measurement": "kWh",
}]
metadata = get_metadata(hass, statistic_ids=("test:total_energy_import", ))
assert metadata == {
"test:total_energy_import": (
1,
{
"has_mean": False,
"has_sum": True,
"name": "Total imported energy",
"source": "test",
"statistic_id": "test:total_energy_import",
"unit_of_measurement": "kWh",
},
)
}
last_stats = get_last_statistics(hass, 1, "test:total_energy_import", True)
assert last_stats == {
"test:total_energy_import": [
{
"statistic_id": "test:total_energy_import",
"start": period2.isoformat(),
"end": (period2 + timedelta(hours=1)).isoformat(),
"max": None,
"mean": None,
"min": None,
"last_reset": None,
"state": approx(1.0),
"sum": approx(3.0),
},
]
}
# Update the previously inserted statistics
external_statistics = {
"start": period1,
"last_reset": None,
"state": 5,
"sum": 6,
}
async_add_external_statistics(hass, external_metadata,
(external_statistics, ))
await async_wait_recording_done_without_instance(hass)
stats = statistics_during_period(hass, zero, period="hour")
assert stats == {
"test:total_energy_import": [
{
"statistic_id": "test:total_energy_import",
"start": period1.isoformat(),
"end": (period1 + timedelta(hours=1)).isoformat(),
"max": None,
"mean": None,
"min": None,
"last_reset": None,
"state": approx(5.0),
"sum": approx(6.0),
},
{
"statistic_id": "test:total_energy_import",
"start": period2.isoformat(),
"end": (period2 + timedelta(hours=1)).isoformat(),
"max": None,
"mean": None,
"min": None,
"last_reset": None,
"state": approx(1.0),
"sum": approx(3.0),
},
]
}
# Update the previously inserted statistics
external_statistics = {
"start": period1,
"max": 1,
"mean": 2,
"min": 3,
"last_reset": None,
"state": 4,
"sum": 5,
}
async_add_external_statistics(hass, external_metadata,
(external_statistics, ))
await async_wait_recording_done_without_instance(hass)
stats = statistics_during_period(hass, zero, period="hour")
assert stats == {
"test:total_energy_import": [
{
"statistic_id": "test:total_energy_import",
"start": period1.isoformat(),
"end": (period1 + timedelta(hours=1)).isoformat(),
"max": approx(1.0),
"mean": approx(2.0),
"min": approx(3.0),
"last_reset": None,
"state": approx(4.0),
"sum": approx(5.0),
},
{
"statistic_id": "test:total_energy_import",
"start": period2.isoformat(),
"end": (period2 + timedelta(hours=1)).isoformat(),
"max": None,
"mean": None,
"min": None,
"last_reset": None,
"state": approx(1.0),
"sum": approx(3.0),
},
]
}
await client.send_json({
"id": 1,
"type": "recorder/adjust_sum_statistics",
"statistic_id": "test:total_energy_import",
"start_time": period2.isoformat(),
"adjustment": 1000.0,
})
response = await client.receive_json()
assert response["success"]
await async_wait_recording_done_without_instance(hass)
stats = statistics_during_period(hass, zero, period="hour")
assert stats == {
"test:total_energy_import": [
{
"statistic_id": "test:total_energy_import",
"start": period1.isoformat(),
"end": (period1 + timedelta(hours=1)).isoformat(),
"max": approx(1.0),
"mean": approx(2.0),
"min": approx(3.0),
"last_reset": None,
"state": approx(4.0),
"sum": approx(5.0),
},
{
"statistic_id": "test:total_energy_import",
"start": period2.isoformat(),
"end": (period2 + timedelta(hours=1)).isoformat(),
"max": None,
"mean": None,
"min": None,
"last_reset": None,
"state": approx(1.0),
"sum": approx(1003.0),
},
]
}
def compile_statistics( # noqa: C901
hass: HomeAssistant, start: datetime.datetime,
end: datetime.datetime) -> dict:
"""Compile statistics for all entities during start-end.
Note: This will query the database and must not be run in the event loop
"""
result: dict = {}
entities = _get_entities(hass)
wanted_statistics = _wanted_statistics(entities)
# Get history between start and end
entities_full_history = [
i[0] for i in entities if "sum" in wanted_statistics[i[0]]
]
history_list = {}
if entities_full_history:
history_list = history.get_significant_states( # type: ignore
hass,
start - datetime.timedelta.resolution,
end,
entity_ids=entities_full_history,
significant_changes_only=False,
)
entities_significant_history = [
i[0] for i in entities if "sum" not in wanted_statistics[i[0]]
]
if entities_significant_history:
_history_list = history.get_significant_states( # type: ignore
hass,
start - datetime.timedelta.resolution,
end,
entity_ids=entities_significant_history,
)
history_list = {**history_list, **_history_list}
for ( # pylint: disable=too-many-nested-blocks
entity_id,
state_class,
device_class,
) in entities:
if entity_id not in history_list:
continue
entity_history = history_list[entity_id]
unit, fstates = _normalize_states(hass, entity_history, device_class,
entity_id)
if not fstates:
continue
# Check metadata
if old_metadata := statistics.get_metadata(hass, entity_id):
if old_metadata["unit_of_measurement"] != unit:
if WARN_UNSTABLE_UNIT not in hass.data:
hass.data[WARN_UNSTABLE_UNIT] = set()
if entity_id not in hass.data[WARN_UNSTABLE_UNIT]:
hass.data[WARN_UNSTABLE_UNIT].add(entity_id)
_LOGGER.warning(
"The unit of %s (%s) does not match the unit of already "
"compiled statistics (%s). Generation of long term statistics "
"will be suppressed unless the unit changes back to %s",
entity_id,
unit,
old_metadata["unit_of_measurement"],
old_metadata["unit_of_measurement"],
)
continue
result[entity_id] = {}
# Set meta data
result[entity_id]["meta"] = {
"unit_of_measurement": unit,
"has_mean": "mean" in wanted_statistics[entity_id],
"has_sum": "sum" in wanted_statistics[entity_id],
}
# Make calculations
stat: dict = {}
if "max" in wanted_statistics[entity_id]:
stat["max"] = max(*itertools.islice(zip(*fstates), 1))
if "min" in wanted_statistics[entity_id]:
stat["min"] = min(*itertools.islice(zip(*fstates), 1))
if "mean" in wanted_statistics[entity_id]:
stat["mean"] = _time_weighted_average(fstates, start, end)
if "sum" in wanted_statistics[entity_id]:
last_reset = old_last_reset = None
new_state = old_state = None
_sum = 0.0
sum_increase = 0.0
sum_increase_tmp = 0.0
last_stats = statistics.get_last_statistics(
hass, 1, entity_id, False)
if entity_id in last_stats:
# We have compiled history for this sensor before, use that as a starting point
last_reset = old_last_reset = last_stats[entity_id][0][
"last_reset"]
new_state = old_state = last_stats[entity_id][0]["state"]
_sum = last_stats[entity_id][0]["sum"] or 0.0
sum_increase = last_stats[entity_id][0]["sum_increase"] or 0.0
for fstate, state in fstates:
# Deprecated, will be removed in Home Assistant 2021.11
if ("last_reset" not in state.attributes
and state_class == STATE_CLASS_MEASUREMENT):
continue
reset = False
if (state_class != STATE_CLASS_TOTAL_INCREASING
and (last_reset := state.attributes.get("last_reset"))
!= old_last_reset):
if old_state is None:
_LOGGER.info(
"Compiling initial sum statistics for %s, zero point set to %s",
entity_id,
fstate,
)
else:
_LOGGER.info(
"Detected new cycle for %s, last_reset set to %s (old last_reset %s)",
entity_id,
last_reset,
old_last_reset,
)
reset = True
elif old_state is None and last_reset is None:
reset = True
_LOGGER.info(
"Compiling initial sum statistics for %s, zero point set to %s",
entity_id,
fstate,
)
elif state_class == STATE_CLASS_TOTAL_INCREASING and (
old_state is None
or reset_detected(hass, entity_id, fstate, new_state)):
reset = True
_LOGGER.info(
"Detected new cycle for %s, value dropped from %s to %s",
entity_id,
new_state,
fstate,
)
if reset:
# The sensor has been reset, update the sum
if old_state is not None:
_sum += new_state - old_state
sum_increase += sum_increase_tmp
sum_increase_tmp = 0.0
if fstate > 0:
sum_increase_tmp += fstate
# ..and update the starting point
new_state = fstate
old_last_reset = last_reset
# Force a new cycle for an existing sensor to start at 0
if old_state is not None:
old_state = 0.0
else:
old_state = new_state
else:
if new_state is not None and fstate > new_state:
sum_increase_tmp += fstate - new_state
new_state = fstate
def compile_statistics(hass: HomeAssistant, start: datetime.datetime,
end: datetime.datetime) -> dict:
"""Compile statistics for all entities during start-end.
Note: This will query the database and must not be run in the event loop
"""
result: dict = {}
entities = _get_entities(hass)
# Get history between start and end
history_list = history.get_significant_states( # type: ignore
hass, start - datetime.timedelta.resolution, end,
[i[0] for i in entities])
for entity_id, key in entities:
wanted_statistics = DEVICE_CLASS_OR_UNIT_STATISTICS[key]
if entity_id not in history_list:
continue
entity_history = history_list[entity_id]
unit, fstates = _normalize_states(entity_history, key, entity_id)
if not fstates:
continue
result[entity_id] = {}
# Set meta data
result[entity_id]["meta"] = {
"unit_of_measurement": unit,
"has_mean": "mean" in wanted_statistics,
"has_sum": "sum" in wanted_statistics,
}
# Make calculations
stat: dict = {}
if "max" in wanted_statistics:
stat["max"] = max(*itertools.islice(zip(*fstates), 1))
if "min" in wanted_statistics:
stat["min"] = min(*itertools.islice(zip(*fstates), 1))
if "mean" in wanted_statistics:
stat["mean"] = _time_weighted_average(fstates, start, end)
if "sum" in wanted_statistics:
last_reset = old_last_reset = None
new_state = old_state = None
_sum = 0
last_stats = statistics.get_last_statistics(hass, 1, entity_id)
if entity_id in last_stats:
# We have compiled history for this sensor before, use that as a starting point
last_reset = old_last_reset = last_stats[entity_id][0][
"last_reset"]
new_state = old_state = last_stats[entity_id][0]["state"]
_sum = last_stats[entity_id][0]["sum"]
for fstate, state in fstates:
if "last_reset" not in state.attributes:
continue
if (last_reset :=
state.attributes["last_reset"]) != old_last_reset:
# The sensor has been reset, update the sum
if old_state is not None:
_sum += new_state - old_state
# ..and update the starting point
new_state = fstate
old_last_reset = last_reset
old_state = new_state
else:
new_state = fstate
if last_reset is None or new_state is None or old_state is None:
# No valid updates
result.pop(entity_id)
continue
# Update the sum with the last state
_sum += new_state - old_state
stat["last_reset"] = dt_util.parse_datetime(last_reset)
stat["sum"] = _sum
stat["state"] = new_state
result[entity_id]["stat"] = stat
