def test_compile_hourly_sum_statistics(hass_recorder, caplog, device_class,
unit, native_unit, factor):
"""Test compiling hourly statistics."""
zero = dt_util.utcnow()
hass = hass_recorder()
recorder = hass.data[DATA_INSTANCE]
setup_component(hass, "sensor", {})
attributes = {
"device_class": device_class,
"state_class": "measurement",
"unit_of_measurement": unit,
"last_reset": None,
}
seq = [10, 15, 20, 10, 30, 40, 50, 60, 70]
four, eight, states = record_meter_states(hass, zero, "sensor.test1",
attributes, seq)
hist = history.get_significant_states(hass, zero - timedelta.resolution,
eight + timedelta.resolution)
assert dict(states)["sensor.test1"] == dict(hist)["sensor.test1"]
recorder.do_adhoc_statistics(period="hourly", start=zero)
wait_recording_done(hass)
recorder.do_adhoc_statistics(period="hourly",
start=zero + timedelta(hours=1))
wait_recording_done(hass)
recorder.do_adhoc_statistics(period="hourly",
start=zero + timedelta(hours=2))
wait_recording_done(hass)
statistic_ids = list_statistic_ids(hass)
assert statistic_ids == [{
"statistic_id": "sensor.test1",
"unit_of_measurement": native_unit
}]
stats = statistics_during_period(hass, zero)
assert stats == {
"sensor.test1": [
{
"statistic_id": "sensor.test1",
"start": process_timestamp_to_utc_isoformat(zero),
"max": None,
"mean": None,
"min": None,
"last_reset": process_timestamp_to_utc_isoformat(zero),
"state": approx(factor * seq[2]),
"sum": approx(factor * 10.0),
},
{
"statistic_id":
"sensor.test1",
"start":
process_timestamp_to_utc_isoformat(zero + timedelta(hours=1)),
"max":
None,
"mean":
None,
"min":
None,
"last_reset":
process_timestamp_to_utc_isoformat(four),
"state":
approx(factor * seq[5]),
"sum":
approx(factor * 10.0),
},
{
"statistic_id":
"sensor.test1",
"start":
process_timestamp_to_utc_isoformat(zero + timedelta(hours=2)),
"max":
None,
"mean":
None,
"min":
None,
"last_reset":
process_timestamp_to_utc_isoformat(four),
"state":
approx(factor * seq[8]),
"sum":
approx(factor * 40.0),
},
]
}
assert "Error while processing event StatisticsTask" not in caplog.text
def test_compile_hourly_statistics(hass_recorder):
"""Test compiling hourly statistics."""
hass = hass_recorder()
recorder = hass.data[DATA_INSTANCE]
setup_component(hass, "sensor", {})
zero, four, states = record_states(hass)
hist = history.get_significant_states(hass, zero, four)
assert dict(states) == dict(hist)
for kwargs in ({}, {"statistic_ids": ["sensor.test1"]}):
stats = statistics_during_period(hass,
zero,
period="5minute",
**kwargs)
assert stats == {}
stats = get_last_statistics(hass, 0, "sensor.test1", True)
assert stats == {}
recorder.do_adhoc_statistics(start=zero)
recorder.do_adhoc_statistics(start=four)
wait_recording_done(hass)
expected_1 = {
"statistic_id": "sensor.test1",
"start": process_timestamp_to_utc_isoformat(zero),
"end": process_timestamp_to_utc_isoformat(zero + timedelta(minutes=5)),
"mean": approx(14.915254237288135),
"min": approx(10.0),
"max": approx(20.0),
"last_reset": None,
"state": None,
"sum": None,
}
expected_2 = {
"statistic_id": "sensor.test1",
"start": process_timestamp_to_utc_isoformat(four),
"end": process_timestamp_to_utc_isoformat(four + timedelta(minutes=5)),
"mean": approx(20.0),
"min": approx(20.0),
"max": approx(20.0),
"last_reset": None,
"state": None,
"sum": None,
}
expected_stats1 = [
{
**expected_1, "statistic_id": "sensor.test1"
},
{
**expected_2, "statistic_id": "sensor.test1"
},
]
expected_stats2 = [
{
**expected_1, "statistic_id": "sensor.test2"
},
{
**expected_2, "statistic_id": "sensor.test2"
},
]
# Test statistics_during_period
stats = statistics_during_period(hass, zero, period="5minute")
assert stats == {
"sensor.test1": expected_stats1,
"sensor.test2": expected_stats2
}
stats = statistics_during_period(hass,
zero,
statistic_ids=["sensor.test2"],
period="5minute")
assert stats == {"sensor.test2": expected_stats2}
stats = statistics_during_period(hass,
zero,
statistic_ids=["sensor.test3"],
period="5minute")
assert stats == {}
# Test get_last_statistics
stats = get_last_statistics(hass, 0, "sensor.test1", True)
assert stats == {}
stats = get_last_statistics(hass, 1, "sensor.test1", True)
assert stats == {
"sensor.test1": [{
**expected_2, "statistic_id": "sensor.test1"
}]
}
stats = get_last_statistics(hass, 2, "sensor.test1", True)
assert stats == {"sensor.test1": expected_stats1[::-1]}
stats = get_last_statistics(hass, 3, "sensor.test1", True)
assert stats == {"sensor.test1": expected_stats1[::-1]}
stats = get_last_statistics(hass, 1, "sensor.test3", True)
assert stats == {}
def compile_statistics( # noqa: C901
hass: HomeAssistant, start: datetime.datetime,
end: datetime.datetime) -> dict:
"""Compile statistics for all entities during start-end.
Note: This will query the database and must not be run in the event loop
"""
result: dict = {}
entities = _get_entities(hass)
wanted_statistics = _wanted_statistics(entities)
# Get history between start and end
entities_full_history = [
i[0] for i in entities if "sum" in wanted_statistics[i[0]]
]
history_list = {}
if entities_full_history:
history_list = history.get_significant_states( # type: ignore
hass,
start - datetime.timedelta.resolution,
end,
entity_ids=entities_full_history,
significant_changes_only=False,
)
entities_significant_history = [
i[0] for i in entities if "sum" not in wanted_statistics[i[0]]
]
if entities_significant_history:
_history_list = history.get_significant_states( # type: ignore
hass,
start - datetime.timedelta.resolution,
end,
entity_ids=entities_significant_history,
)
history_list = {**history_list, **_history_list}
for entity_id, state_class, device_class in entities:
if entity_id not in history_list:
continue
entity_history = history_list[entity_id]
unit, fstates = _normalize_states(hass, entity_history, device_class,
entity_id)
if not fstates:
continue
# Check metadata
if old_metadata := statistics.get_metadata(hass, entity_id):
if old_metadata["unit_of_measurement"] != unit:
if WARN_UNSTABLE_UNIT not in hass.data:
hass.data[WARN_UNSTABLE_UNIT] = set()
if entity_id not in hass.data[WARN_UNSTABLE_UNIT]:
hass.data[WARN_UNSTABLE_UNIT].add(entity_id)
_LOGGER.warning(
"The unit of %s (%s) does not match the unit of already "
"compiled statistics (%s). Generation of long term statistics "
"will be suppressed unless the unit changes back to %s",
entity_id,
unit,
old_metadata["unit_of_measurement"],
old_metadata["unit_of_measurement"],
)
continue
result[entity_id] = {}
# Set meta data
result[entity_id]["meta"] = {
"unit_of_measurement": unit,
"has_mean": "mean" in wanted_statistics[entity_id],
"has_sum": "sum" in wanted_statistics[entity_id],
}
# Make calculations
stat: dict = {}
if "max" in wanted_statistics[entity_id]:
stat["max"] = max(*itertools.islice(zip(*fstates), 1))
if "min" in wanted_statistics[entity_id]:
stat["min"] = min(*itertools.islice(zip(*fstates), 1))
if "mean" in wanted_statistics[entity_id]:
stat["mean"] = _time_weighted_average(fstates, start, end)
if "sum" in wanted_statistics[entity_id]:
last_reset = old_last_reset = None
new_state = old_state = None
_sum = 0
last_stats = statistics.get_last_statistics(hass, 1, entity_id)
if entity_id in last_stats:
# We have compiled history for this sensor before, use that as a starting point
last_reset = old_last_reset = last_stats[entity_id][0][
"last_reset"]
new_state = old_state = last_stats[entity_id][0]["state"]
_sum = last_stats[entity_id][0]["sum"] or 0
for fstate, state in fstates:
# Deprecated, will be removed in Home Assistant 2021.10
if ("last_reset" not in state.attributes
and state_class == STATE_CLASS_MEASUREMENT):
continue
reset = False
if (state_class != STATE_CLASS_TOTAL_INCREASING
and (last_reset := state.attributes.get("last_reset"))
!= old_last_reset):
if old_state is None:
_LOGGER.info(
"Compiling initial sum statistics for %s, zero point set to %s",
entity_id,
fstate,
)
else:
_LOGGER.info(
"Detected new cycle for %s, last_reset set to %s (old last_reset %s)",
entity_id,
last_reset,
old_last_reset,
)
reset = True
elif old_state is None and last_reset is None:
reset = True
_LOGGER.info(
"Compiling initial sum statistics for %s, zero point set to %s",
entity_id,
fstate,
)
elif state_class == STATE_CLASS_TOTAL_INCREASING and (
old_state is None
or reset_detected(hass, entity_id, fstate, new_state)):
reset = True
_LOGGER.info(
"Detected new cycle for %s, value dropped from %s to %s",
entity_id,
fstate,
new_state,
)
if reset:
# The sensor has been reset, update the sum
if old_state is not None:
_sum += new_state - old_state
# ..and update the starting point
new_state = fstate
old_last_reset = last_reset
# Force a new cycle for an existing sensor to start at 0
if old_state is not None:
old_state = 0.0
else:
old_state = new_state
else:
new_state = fstate
def test_rename_entity(hass_recorder):
"""Test statistics is migrated when entity_id is changed."""
hass = hass_recorder()
recorder = hass.data[DATA_INSTANCE]
setup_component(hass, "sensor", {})
entity_reg = mock_registry(hass)
reg_entry = entity_reg.async_get_or_create(
"sensor",
"test",
"unique_0000",
suggested_object_id="test1",
)
assert reg_entry.entity_id == "sensor.test1"
zero, four, states = record_states(hass)
hist = history.get_significant_states(hass, zero, four)
assert dict(states) == dict(hist)
for kwargs in ({}, {"statistic_ids": ["sensor.test1"]}):
stats = statistics_during_period(hass,
zero,
period="5minute",
**kwargs)
assert stats == {}
stats = get_last_statistics(hass, 0, "sensor.test1", True)
assert stats == {}
recorder.do_adhoc_statistics(start=zero)
wait_recording_done(hass)
expected_1 = {
"statistic_id": "sensor.test1",
"start": process_timestamp_to_utc_isoformat(zero),
"end": process_timestamp_to_utc_isoformat(zero + timedelta(minutes=5)),
"mean": approx(14.915254237288135),
"min": approx(10.0),
"max": approx(20.0),
"last_reset": None,
"state": None,
"sum": None,
}
expected_stats1 = [
{
**expected_1, "statistic_id": "sensor.test1"
},
]
expected_stats2 = [
{
**expected_1, "statistic_id": "sensor.test2"
},
]
expected_stats99 = [
{
**expected_1, "statistic_id": "sensor.test99"
},
]
stats = statistics_during_period(hass, zero, period="5minute")
assert stats == {
"sensor.test1": expected_stats1,
"sensor.test2": expected_stats2
}
entity_reg.async_update_entity(reg_entry.entity_id,
new_entity_id="sensor.test99")
hass.block_till_done()
stats = statistics_during_period(hass, zero, period="5minute")
assert stats == {
"sensor.test99": expected_stats99,
"sensor.test2": expected_stats2
}
async def handle_presence_simulation(call,
restart=False,
entities_after_restart=None,
delta_after_restart=None):
"""Start the presence simulation"""
if call is not None: #if we are here, it is a call of the service, or a restart at the end of a cycle
if isinstance(call.data.get("entity_id", entities), list):
overridden_entities = call.data.get("entity_id", entities)
else:
overridden_entities = [call.data.get("entity_id", entities)]
overridden_delta = call.data.get("delta", delta)
overridden_restore = call.data.get("restore_states",
restoreAfterStop)
else: #if we are it is a call from the toggle service or from the turn_on action of the switch entity
# or this is a restart and the simulation was launched after a restart of HA
if entities_after_restart is not None:
overridden_entities = entities_after_restart
else:
overridden_entities = entities
if delta_after_restart is not None:
overridden_delta = delta_after_restart
else:
overridden_delta = delta
overridden_restore = restoreAfterStop
#get the switch entity
entity = hass.data[DOMAIN][SWITCH_PLATFORM][SWITCH]
_LOGGER.debug("Is already running ? %s", entity.state)
if is_running():
_LOGGER.warning(
"Presence simulation already running. Doing nothing")
return
running = True
#turn on the switch. Not calling turn_on() to avoid calling the start service again
entity.internal_turn_on()
_LOGGER.debug("setting restore states %s", overridden_restore)
await entity.set_restore_states(overridden_restore)
_LOGGER.debug("Presence simulation started")
current_date = datetime.now(timezone.utc)
#compute the start date that will be used in the query to get the historic of the entities
minus_delta = current_date + timedelta(-overridden_delta)
#expand the entitiies, meaning replace the groups with the entities in it
try:
expanded_entities = await async_expand_entities(overridden_entities
)
except Exception as e:
_LOGGER.error("Error during identifing entities")
running = False
entity.internal_turn_off()
return
if not restart:
#set attribute on the switch
try:
await entity.set_start_datetime(
datetime.now(hass.config.time_zone))
except Exception as e:
try:
await entity.set_start_datetime(
datetime.now(pytz.timezone(hass.config.time_zone)))
except Exception as e:
_LOGGER.warning(
"Start datetime could not be set to HA timezone: ", e)
await entity.set_start_datetime(datetime.now())
if overridden_restore:
service_data = {}
service_data["scene_id"] = RESTORE_SCENE
service_data["snapshot_entities"] = expanded_entities
_LOGGER.debug("Saving scene before launching the simulation")
try:
await hass.services.async_call("scene",
"create",
service_data,
blocking=True)
except Exception as e:
_LOGGER.error(
"Scene could not be created, continue without the restore functionality: %s",
e)
await entity.set_entities(expanded_entities)
await entity.set_delta(overridden_delta)
_LOGGER.debug("Getting the historic from %s for %s", minus_delta,
expanded_entities)
dic = get_significant_states(hass=hass,
start_time=minus_delta,
entity_ids=expanded_entities,
significant_changes_only=False)
_LOGGER.debug("history: %s", dic)
for entity_id in dic:
_LOGGER.debug('Entity %s', entity_id)
#launch an async task by entity_id
hass.async_create_task(
simulate_single_entity(entity_id, dic[entity_id]))
#launch an async task that will restart the simulation after the delay has passed
hass.async_create_task(
restart_presence_simulation(
call,
entities_after_restart=entities_after_restart,
delta_after_restart=delta_after_restart))
_LOGGER.debug("All async tasks launched")
def compile_statistics(hass: HomeAssistant, start: datetime.datetime,
end: datetime.datetime) -> dict:
"""Compile statistics for all entities during start-end.
Note: This will query the database and must not be run in the event loop
"""
result: dict = {}
entities = _get_entities(hass)
# Get history between start and end
history_list = history.get_significant_states( # type: ignore
hass, start - datetime.timedelta.resolution, end,
[i[0] for i in entities])
for entity_id, device_class in entities:
wanted_statistics = DEVICE_CLASS_STATISTICS[device_class]
if entity_id not in history_list:
continue
entity_history = history_list[entity_id]
unit, fstates = _normalize_states(entity_history, device_class,
entity_id)
if not fstates:
continue
result[entity_id] = {}
# Set meta data
result[entity_id]["meta"] = {
"unit_of_measurement": unit,
"has_mean": "mean" in wanted_statistics,
"has_sum": "sum" in wanted_statistics,
}
# Make calculations
stat: dict = {}
if "max" in wanted_statistics:
stat["max"] = max(*itertools.islice(zip(*fstates), 1))
if "min" in wanted_statistics:
stat["min"] = min(*itertools.islice(zip(*fstates), 1))
if "mean" in wanted_statistics:
stat["mean"] = _time_weighted_average(fstates, start, end)
if "sum" in wanted_statistics:
last_reset = old_last_reset = None
new_state = old_state = None
_sum = 0
last_stats = statistics.get_last_statistics(
hass, 1, entity_id) # type: ignore
if entity_id in last_stats:
# We have compiled history for this sensor before, use that as a starting point
last_reset = old_last_reset = last_stats[entity_id][0][
"last_reset"]
new_state = old_state = last_stats[entity_id][0]["state"]
_sum = last_stats[entity_id][0]["sum"]
for fstate, state in fstates:
if "last_reset" not in state.attributes:
continue
if (last_reset :=
state.attributes["last_reset"]) != old_last_reset:
# The sensor has been reset, update the sum
if old_state is not None:
_sum += new_state - old_state
# ..and update the starting point
new_state = fstate
old_last_reset = last_reset
old_state = new_state
else:
new_state = fstate
if last_reset is None or new_state is None or old_state is None:
# No valid updates
result.pop(entity_id)
continue
# Update the sum with the last state
_sum += new_state - old_state
stat["last_reset"] = dt_util.parse_datetime(last_reset)
stat["sum"] = _sum
stat["state"] = new_state
result[entity_id]["stat"] = stat
def test_compile_hourly_energy_statistics(hass_recorder):
"""Test compiling hourly statistics."""
hass = hass_recorder()
recorder = hass.data[DATA_INSTANCE]
setup_component(hass, "sensor", {})
sns1_attr = {"device_class": "energy", "state_class": "measurement"}
sns2_attr = {"device_class": "energy"}
sns3_attr = {}
zero, four, eight, states = record_energy_states(hass, sns1_attr,
sns2_attr, sns3_attr)
hist = history.get_significant_states(hass, zero - timedelta.resolution,
eight + timedelta.resolution)
assert dict(states)["sensor.test1"] == dict(hist)["sensor.test1"]
recorder.do_adhoc_statistics(period="hourly", start=zero)
wait_recording_done(hass)
recorder.do_adhoc_statistics(period="hourly",
start=zero + timedelta(hours=1))
wait_recording_done(hass)
recorder.do_adhoc_statistics(period="hourly",
start=zero + timedelta(hours=2))
wait_recording_done(hass)
stats = statistics_during_period(hass, zero)
assert stats == {
"sensor.test1": [
{
"statistic_id": "sensor.test1",
"start": process_timestamp_to_utc_isoformat(zero),
"max": None,
"mean": None,
"min": None,
"last_reset": process_timestamp_to_utc_isoformat(zero),
"state": approx(20.0),
"sum": approx(10.0),
},
{
"statistic_id":
"sensor.test1",
"start":
process_timestamp_to_utc_isoformat(zero + timedelta(hours=1)),
"max":
None,
"mean":
None,
"min":
None,
"last_reset":
process_timestamp_to_utc_isoformat(four),
"state":
approx(40.0),
"sum":
approx(10.0),
},
{
"statistic_id":
"sensor.test1",
"start":
process_timestamp_to_utc_isoformat(zero + timedelta(hours=2)),
"max":
None,
"mean":
None,
"min":
None,
"last_reset":
process_timestamp_to_utc_isoformat(four),
"state":
approx(70.0),
"sum":
approx(40.0),
},
]
}
def test_compile_hourly_energy_statistics_multiple(hass_recorder, caplog):
"""Test compiling multiple hourly statistics."""
zero = dt_util.utcnow()
hass = hass_recorder()
recorder = hass.data[DATA_INSTANCE]
setup_component(hass, "sensor", {})
sns1_attr = {**ENERGY_SENSOR_ATTRIBUTES, "last_reset": None}
sns2_attr = {**ENERGY_SENSOR_ATTRIBUTES, "last_reset": None}
sns3_attr = {
**ENERGY_SENSOR_ATTRIBUTES,
"unit_of_measurement": "Wh",
"last_reset": None,
}
sns4_attr = {**ENERGY_SENSOR_ATTRIBUTES}
seq1 = [10, 15, 20, 10, 30, 40, 50, 60, 70]
seq2 = [110, 120, 130, 0, 30, 45, 55, 65, 75]
seq3 = [0, 0, 5, 10, 30, 50, 60, 80, 90]
seq4 = [0, 0, 5, 10, 30, 50, 60, 80, 90]
four, eight, states = record_energy_states(hass, zero, "sensor.test1",
sns1_attr, seq1)
_, _, _states = record_energy_states(hass, zero, "sensor.test2", sns2_attr,
seq2)
states = {**states, **_states}
_, _, _states = record_energy_states(hass, zero, "sensor.test3", sns3_attr,
seq3)
states = {**states, **_states}
_, _, _states = record_energy_states(hass, zero, "sensor.test4", sns4_attr,
seq4)
states = {**states, **_states}
hist = history.get_significant_states(hass, zero - timedelta.resolution,
eight + timedelta.resolution)
assert dict(states)["sensor.test1"] == dict(hist)["sensor.test1"]
recorder.do_adhoc_statistics(period="hourly", start=zero)
wait_recording_done(hass)
recorder.do_adhoc_statistics(period="hourly",
start=zero + timedelta(hours=1))
wait_recording_done(hass)
recorder.do_adhoc_statistics(period="hourly",
start=zero + timedelta(hours=2))
wait_recording_done(hass)
statistic_ids = list_statistic_ids(hass)
assert statistic_ids == [
{
"statistic_id": "sensor.test1",
"unit_of_measurement": "kWh"
},
{
"statistic_id": "sensor.test2",
"unit_of_measurement": "kWh"
},
{
"statistic_id": "sensor.test3",
"unit_of_measurement": "kWh"
},
]
stats = statistics_during_period(hass, zero)
assert stats == {
"sensor.test1": [
{
"statistic_id": "sensor.test1",
"start": process_timestamp_to_utc_isoformat(zero),
"max": None,
"mean": None,
"min": None,
"last_reset": process_timestamp_to_utc_isoformat(zero),
"state": approx(20.0),
"sum": approx(10.0),
},
{
"statistic_id":
"sensor.test1",
"start":
process_timestamp_to_utc_isoformat(zero + timedelta(hours=1)),
"max":
None,
"mean":
None,
"min":
None,
"last_reset":
process_timestamp_to_utc_isoformat(four),
"state":
approx(40.0),
"sum":
approx(10.0),
},
{
"statistic_id":
"sensor.test1",
"start":
process_timestamp_to_utc_isoformat(zero + timedelta(hours=2)),
"max":
None,
"mean":
None,
"min":
None,
"last_reset":
process_timestamp_to_utc_isoformat(four),
"state":
approx(70.0),
"sum":
approx(40.0),
},
],
"sensor.test2": [
{
"statistic_id": "sensor.test2",
"start": process_timestamp_to_utc_isoformat(zero),
"max": None,
"mean": None,
"min": None,
"last_reset": process_timestamp_to_utc_isoformat(zero),
"state": approx(130.0),
"sum": approx(20.0),
},
{
"statistic_id":
"sensor.test2",
"start":
process_timestamp_to_utc_isoformat(zero + timedelta(hours=1)),
"max":
None,
"mean":
None,
"min":
None,
"last_reset":
process_timestamp_to_utc_isoformat(four),
"state":
approx(45.0),
"sum":
approx(-95.0),
},
{
"statistic_id":
"sensor.test2",
"start":
process_timestamp_to_utc_isoformat(zero + timedelta(hours=2)),
"max":
None,
"mean":
None,
"min":
None,
"last_reset":
process_timestamp_to_utc_isoformat(four),
"state":
approx(75.0),
"sum":
approx(-65.0),
},
],
"sensor.test3": [
{
"statistic_id": "sensor.test3",
"start": process_timestamp_to_utc_isoformat(zero),
"max": None,
"mean": None,
"min": None,
"last_reset": process_timestamp_to_utc_isoformat(zero),
"state": approx(5.0 / 1000),
"sum": approx(5.0 / 1000),
},
{
"statistic_id":
"sensor.test3",
"start":
process_timestamp_to_utc_isoformat(zero + timedelta(hours=1)),
"max":
None,
"mean":
None,
"min":
None,
"last_reset":
process_timestamp_to_utc_isoformat(four),
"state":
approx(50.0 / 1000),
"sum":
approx(30.0 / 1000),
},
{
"statistic_id":
"sensor.test3",
"start":
process_timestamp_to_utc_isoformat(zero + timedelta(hours=2)),
"max":
None,
"mean":
None,
"min":
None,
"last_reset":
process_timestamp_to_utc_isoformat(four),
"state":
approx(90.0 / 1000),
"sum":
approx(70.0 / 1000),
},
],
}
assert "Error while processing event StatisticsTask" not in caplog.text