def setup_recorder(config=None):
"""Setup with params."""
init_recorder_component(hass, config)
hass.start()
hass.block_till_done()
recorder.get_instance().block_till_done()
return hass
def init_recorder(self):
"""Initialize the recorder."""
init_recorder_component(self.hass)
self.hass.start()
recorder.get_instance().block_till_db_ready()
self.hass.block_till_done()
recorder.get_instance().block_till_done()
def wait_recording_done(hass: HomeAssistant) -> None:
"""Block till recording is done."""
hass.block_till_done()
trigger_db_commit(hass)
hass.block_till_done()
recorder.get_instance(hass).block_till_done()
hass.block_till_done()
def init_recorder(self):
"""Initialize the recorder."""
init_recorder_component(self.hass)
self.hass.start()
recorder.get_instance().block_till_db_ready()
self.hass.block_till_done()
recorder.get_instance().block_till_done()
def setup_recorder(config=None):
"""Setup with params."""
init_recorder_component(hass, config)
hass.start()
hass.block_till_done()
recorder.get_instance().block_till_done()
return hass
def init_recorder_component(hass, add_config=None, db_ready_callback=None):
"""Initialize the recorder."""
config = dict(add_config) if add_config else {}
config[recorder.CONF_DB_URL] = 'sqlite://' # In memory DB
assert setup_component(hass, recorder.DOMAIN,
{recorder.DOMAIN: config})
assert recorder.DOMAIN in hass.config.components
recorder.get_instance().block_till_db_ready()
_LOGGER.info("In-memory recorder successfully started")
async def async_block_recorder(hass: HomeAssistant, seconds: float) -> None:
"""Block the recorders event loop for testing.
Returns as soon as the recorder has started the block.
Does not wait for the block to finish.
"""
event = asyncio.Event()
get_instance(hass).queue_task(BlockRecorderTask(event, seconds))
await event.wait()
def last_recorder_run():
"""Retireve the last closed recorder run from the DB."""
recorder.get_instance()
rec_runs = recorder.get_model('RecorderRuns')
with recorder.session_scope() as session:
res = recorder.query(rec_runs).order_by(rec_runs.end.desc()).first()
if res is None:
return None
session.expunge(res)
return res
def last_recorder_run():
"""Retireve the last closed recorder run from the DB."""
recorder.get_instance()
rec_runs = recorder.get_model('RecorderRuns')
with recorder.session_scope() as session:
res = recorder.query(rec_runs).order_by(rec_runs.end.desc()).first()
if res is None:
return None
session.expunge(res)
return res
async def test_run_history(hass, recorder_mock):
"""Test the run history gives the correct run."""
instance = recorder.get_instance(hass)
now = dt_util.utcnow()
three_days_ago = now - timedelta(days=3)
two_days_ago = now - timedelta(days=2)
one_day_ago = now - timedelta(days=1)
with instance.get_session() as session:
session.add(RecorderRuns(start=three_days_ago, created=three_days_ago))
session.add(RecorderRuns(start=two_days_ago, created=two_days_ago))
session.add(RecorderRuns(start=one_day_ago, created=one_day_ago))
session.commit()
instance.run_history.load_from_db(session)
assert (process_timestamp(
instance.run_history.get(three_days_ago + timedelta(
microseconds=1)).start) == three_days_ago)
assert (process_timestamp(
instance.run_history.get(two_days_ago + timedelta(
microseconds=1)).start) == two_days_ago)
assert (process_timestamp(
instance.run_history.get(one_day_ago + timedelta(
microseconds=1)).start) == one_day_ago)
assert (process_timestamp(instance.run_history.get(now).start) ==
instance.run_history.recording_start)
async def test_recorder_system_health_db_url_missing_host(
hass, recorder_mock, dialect_name):
"""Test recorder system health with a db_url without a hostname."""
assert await async_setup_component(hass, "system_health", {})
await async_wait_recording_done(hass)
instance = get_instance(hass)
with patch(
"homeassistant.components.recorder.core.Recorder.dialect_name",
dialect_name
), patch.object(
instance,
"db_url",
"postgresql://*****:*****@/home_assistant?host=/config/socket",
), patch(
"sqlalchemy.orm.session.Session.execute",
return_value=Mock(first=Mock(return_value=("1048576", ))),
):
info = await get_system_health_info(hass, "recorder")
assert info == {
"current_recorder_run": instance.run_history.current.start,
"oldest_recorder_run": instance.run_history.first.start,
"estimated_db_size": "1.00 MiB",
"database_engine": dialect_name.value,
"database_version": ANY,
}
def setup(hass, config):
"""Setup the history hooks."""
filters = Filters()
exclude = config[DOMAIN].get(CONF_EXCLUDE)
if exclude:
filters.excluded_entities = exclude[CONF_ENTITIES]
filters.excluded_domains = exclude[CONF_DOMAINS]
include = config[DOMAIN].get(CONF_INCLUDE)
if include:
filters.included_entities = include[CONF_ENTITIES]
filters.included_domains = include[CONF_DOMAINS]
recorder.get_instance()
hass.http.register_view(HistoryPeriodView(filters))
register_built_in_panel(hass, 'history', 'History', 'mdi:poll-box')
return True
def setup(hass, config):
"""Setup the history hooks."""
filters = Filters()
exclude = config[DOMAIN].get(CONF_EXCLUDE)
if exclude:
filters.excluded_entities = exclude[CONF_ENTITIES]
filters.excluded_domains = exclude[CONF_DOMAINS]
include = config[DOMAIN].get(CONF_INCLUDE)
if include:
filters.included_entities = include[CONF_ENTITIES]
filters.included_domains = include[CONF_DOMAINS]
recorder.get_instance()
hass.http.register_view(HistoryPeriodView(filters))
register_built_in_panel(hass, 'history', 'History', 'mdi:poll-box')
return True
async def _async_wait_for_recorder_sync(hass: HomeAssistant) -> None:
"""Wait for the recorder to sync."""
try:
await asyncio.wait_for(
get_instance(hass).async_block_till_done(), MAX_RECORDER_WAIT)
except asyncio.TimeoutError:
_LOGGER.debug(
"Recorder is behind more than %s seconds, starting live stream; Some results may be missing"
)
async def test_recorder_system_health(hass, recorder_mock):
"""Test recorder system health."""
assert await async_setup_component(hass, "system_health", {})
await async_wait_recording_done(hass)
info = await get_system_health_info(hass, "recorder")
instance = get_instance(hass)
assert info == {
"current_recorder_run": instance.run_history.current.start,
"oldest_recorder_run": instance.run_history.first.start,
"estimated_db_size": ANY,
}
def test_combined_checks(hass_recorder, caplog):
"""Run Checks on the open database."""
hass = hass_recorder()
instance = recorder.get_instance(hass)
instance.db_retry_wait = 0
cursor = instance.engine.raw_connection().cursor()
assert util.run_checks_on_open_db("fake_db_path", cursor) is None
assert "could not validate that the sqlite3 database" in caplog.text
caplog.clear()
# We are patching recorder.util here in order
# to avoid creating the full database on disk
with patch("homeassistant.components.recorder.util.basic_sanity_check",
return_value=False):
caplog.clear()
assert util.run_checks_on_open_db("fake_db_path", cursor) is None
assert "could not validate that the sqlite3 database" in caplog.text
# We are patching recorder.util here in order
# to avoid creating the full database on disk
with patch(
"homeassistant.components.recorder.util.last_run_was_recently_clean"
):
caplog.clear()
assert util.run_checks_on_open_db("fake_db_path", cursor) is None
assert "restarted cleanly and passed the basic sanity check" in caplog.text
caplog.clear()
with patch(
"homeassistant.components.recorder.util.last_run_was_recently_clean",
side_effect=sqlite3.DatabaseError,
), pytest.raises(sqlite3.DatabaseError):
util.run_checks_on_open_db("fake_db_path", cursor)
caplog.clear()
with patch(
"homeassistant.components.recorder.util.last_run_was_recently_clean",
side_effect=sqlite3.DatabaseError,
), pytest.raises(sqlite3.DatabaseError):
util.run_checks_on_open_db("fake_db_path", cursor)
cursor.execute("DROP TABLE events;")
caplog.clear()
with pytest.raises(sqlite3.DatabaseError):
util.run_checks_on_open_db("fake_db_path", cursor)
caplog.clear()
with pytest.raises(sqlite3.DatabaseError):
util.run_checks_on_open_db("fake_db_path", cursor)
def test_recorder_errors_exceptions(hass_recorder): \
# pylint: disable=redefined-outer-name
"""Test session_scope and get_model errors."""
# Model cannot be resolved
assert recorder.get_model('dont-exist') is None
# Verify the instance fails before setup
with pytest.raises(RuntimeError):
recorder.get_instance()
# Setup the recorder
hass_recorder()
recorder.get_instance()
# Verify session scope raises (and prints) an exception
with patch('homeassistant.components.recorder._LOGGER.error') as e_mock, \
pytest.raises(Exception) as err:
with recorder.session_scope() as session:
session.execute('select * from notthere')
assert e_mock.call_count == 1
assert recorder.ERROR_QUERY[:-4] in e_mock.call_args[0][0]
assert 'no such table' in str(err.value)
def test_recorder_errors_exceptions(hass_recorder): \
# pylint: disable=redefined-outer-name
"""Test session_scope and get_model errors."""
# Model cannot be resolved
assert recorder.get_model('dont-exist') is None
# Verify the instance fails before setup
with pytest.raises(RuntimeError):
recorder.get_instance()
# Setup the recorder
hass_recorder()
recorder.get_instance()
# Verify session scope raises (and prints) an exception
with patch('homeassistant.components.recorder._LOGGER.error') as e_mock, \
pytest.raises(Exception) as err:
with recorder.session_scope() as session:
session.execute('select * from notthere')
assert e_mock.call_count == 1
assert recorder.ERROR_QUERY[:-4] in e_mock.call_args[0][0]
assert 'no such table' in str(err.value)
def test_execute_stmt(hass_recorder):
"""Test executing with execute_stmt."""
hass = hass_recorder()
instance = recorder.get_instance(hass)
hass.states.set("sensor.on", "on")
new_state = hass.states.get("sensor.on")
wait_recording_done(hass)
now = dt_util.utcnow()
tomorrow = now + timedelta(days=1)
one_week_from_now = now + timedelta(days=7)
class MockExecutor:
_calls = 0
def __init__(self, stmt):
"""Init the mock."""
def all(self):
MockExecutor._calls += 1
if MockExecutor._calls == 2:
return ["mock_row"]
raise SQLAlchemyError
with session_scope(hass=hass) as session:
# No time window, we always get a list
stmt = history._get_single_entity_states_stmt(instance.schema_version,
dt_util.utcnow(),
"sensor.on", False)
rows = util.execute_stmt(session, stmt)
assert isinstance(rows, list)
assert rows[0].state == new_state.state
assert rows[0].entity_id == new_state.entity_id
# Time window >= 2 days, we get a ChunkedIteratorResult
rows = util.execute_stmt(session, stmt, now, one_week_from_now)
assert isinstance(rows, ChunkedIteratorResult)
row = next(rows)
assert row.state == new_state.state
assert row.entity_id == new_state.entity_id
# Time window < 2 days, we get a list
rows = util.execute_stmt(session, stmt, now, tomorrow)
assert isinstance(rows, list)
assert rows[0].state == new_state.state
assert rows[0].entity_id == new_state.entity_id
with patch.object(session, "execute", MockExecutor):
rows = util.execute_stmt(session, stmt, now, tomorrow)
assert rows == ["mock_row"]
async def _async_history_from_db(
self,
current_period_start: datetime.datetime,
current_period_end: datetime.datetime,
) -> None:
"""Update history data for the current period from the database."""
instance = get_instance(self.hass)
states = await instance.async_add_executor_job(
self._state_changes_during_period,
current_period_start,
current_period_end,
)
self._history_current_period = [
HistoryState(state.state, state.last_changed.timestamp())
for state in states
]
async def test_recorder_system_health_alternate_dbms(hass, recorder_mock, dialect_name):
"""Test recorder system health."""
assert await async_setup_component(hass, "system_health", {})
await async_wait_recording_done(hass)
with patch(
"homeassistant.components.recorder.core.Recorder.dialect_name", dialect_name
), patch(
"sqlalchemy.orm.session.Session.execute",
return_value=Mock(first=Mock(return_value=("1048576",))),
):
info = await get_system_health_info(hass, "recorder")
instance = get_instance(hass)
assert info == {
"current_recorder_run": instance.run_history.current.start,
"oldest_recorder_run": instance.run_history.first.start,
"estimated_db_size": "1.00 MiB",
}
def query_and_join_attributes(
hass: HomeAssistant, no_attributes: bool) -> tuple[list[Column], bool]:
"""Return the query keys and if StateAttributes should be joined."""
# If no_attributes was requested we do the query
# without the attributes fields and do not join the
# state_attributes table
if no_attributes:
return QUERY_STATE_NO_ATTR, False
# If we in the process of migrating schema we do
# not want to join the state_attributes table as we
# do not know if it will be there yet
if recorder.get_instance(hass).migration_in_progress:
return QUERY_STATES_PRE_SCHEMA_25, False
# Finally if no migration is in progress and no_attributes
# was not requested, we query both attributes columns and
# join state_attributes
return QUERY_STATES, True
def bake_query_and_join_attributes(
hass: HomeAssistant,
no_attributes: bool,
include_last_updated: bool = True) -> tuple[Any, bool]:
"""Return the initial backed query and if StateAttributes should be joined.
Because these are baked queries the values inside the lambdas need
to be explicitly written out to avoid caching the wrong values.
"""
bakery: baked.bakery = hass.data[HISTORY_BAKERY]
# If no_attributes was requested we do the query
# without the attributes fields and do not join the
# state_attributes table
if no_attributes:
if include_last_updated:
return bakery(
lambda session: session.query(*QUERY_STATE_NO_ATTR)), False
return (
bakery(lambda session: session.query(
*QUERY_STATE_NO_ATTR_NO_LAST_UPDATED)),
False,
)
# If we in the process of migrating schema we do
# not want to join the state_attributes table as we
# do not know if it will be there yet
if recorder.get_instance(hass).schema_version < 25:
if include_last_updated:
return (
bakery(lambda session: session.query(
*QUERY_STATES_PRE_SCHEMA_25)),
False,
)
return (
bakery(lambda session: session.query(
*QUERY_STATES_PRE_SCHEMA_25_NO_LAST_UPDATED)),
False,
)
# Finally if no migration is in progress and no_attributes
# was not requested, we query both attributes columns and
# join state_attributes
if include_last_updated:
return bakery(lambda session: session.query(*QUERY_STATES)), True
return bakery(
lambda session: session.query(*QUERY_STATES_NO_LAST_UPDATED)), True
async def test_shutdown_closes_connections(hass, recorder_mock):
"""Test shutdown closes connections."""
hass.state = CoreState.not_running
instance = get_instance(hass)
await instance.async_db_ready
await hass.async_block_till_done()
pool = instance.engine.pool
pool.shutdown = Mock()
def _ensure_connected():
with session_scope(hass=hass) as session:
list(session.query(States))
await instance.async_add_executor_job(_ensure_connected)
hass.bus.async_fire(EVENT_HOMEASSISTANT_STOP)
await hass.async_block_till_done()
assert len(pool.shutdown.mock_calls) == 1
def _get_states_with_session(
hass: HomeAssistant,
session: Session,
utc_point_in_time: datetime,
entity_ids: list[str] | None = None,
run: RecorderRuns | None = None,
filters: Any | None = None,
no_attributes: bool = False,
) -> list[State]:
"""Return the states at a specific point in time."""
if entity_ids and len(entity_ids) == 1:
return _get_single_entity_states_with_session(
hass, session, utc_point_in_time, entity_ids[0], no_attributes
)
if run is None:
run = recorder.get_instance(hass).run_history.get(utc_point_in_time)
if run is None or process_timestamp(run.start) > utc_point_in_time:
# History did not run before utc_point_in_time
return []
# We have more than one entity to look at so we need to do a query on states
# since the last recorder run started.
if entity_ids:
baked_query = _get_states_baked_query_for_entites(hass, no_attributes)
else:
baked_query = _get_states_baked_query_for_all(hass, filters, no_attributes)
attr_cache: dict[str, dict[str, Any]] = {}
return [
LazyState(row, attr_cache)
for row in execute(
baked_query(session).params(
run_start=run.start,
utc_point_in_time=utc_point_in_time,
entity_ids=entity_ids,
)
)
]
async def test_recorder_info_bad_recorder_config(hass, hass_ws_client):
"""Test getting recorder status when recorder is not started."""
config = {recorder.CONF_DB_URL: "sqlite://no_file", recorder.CONF_DB_RETRY_WAIT: 0}
client = await hass_ws_client()
with patch("homeassistant.components.recorder.migration.migrate_schema"):
recorder_helper.async_initialize_recorder(hass)
assert not await async_setup_component(
hass, recorder.DOMAIN, {recorder.DOMAIN: config}
)
assert recorder.DOMAIN not in hass.config.components
await hass.async_block_till_done()
# Wait for recorder to shut down
await hass.async_add_executor_job(recorder.get_instance(hass).join)
await client.send_json({"id": 1, "type": "recorder/info"})
response = await client.receive_json()
assert response["success"]
assert response["result"]["recording"] is False
assert response["result"]["thread_running"] is False
def _get_rows_with_session(
hass: HomeAssistant,
session: Session,
utc_point_in_time: datetime,
entity_ids: list[str] | None = None,
run: RecorderRuns | None = None,
filters: Filters | None = None,
no_attributes: bool = False,
) -> Iterable[Row]:
"""Return the states at a specific point in time."""
schema_version = _schema_version(hass)
if entity_ids and len(entity_ids) == 1:
return execute_stmt_lambda_element(
session,
_get_single_entity_states_stmt(
schema_version, utc_point_in_time, entity_ids[0], no_attributes
),
)
if run is None:
run = recorder.get_instance(hass).run_history.get(utc_point_in_time)
if run is None or process_timestamp(run.start) > utc_point_in_time:
# History did not run before utc_point_in_time
return []
# We have more than one entity to look at so we need to do a query on states
# since the last recorder run started.
if entity_ids:
stmt = _get_states_for_entites_stmt(
schema_version, run.start, utc_point_in_time, entity_ids, no_attributes
)
else:
stmt = _get_states_for_all_stmt(
schema_version, run.start, utc_point_in_time, filters, no_attributes
)
return execute_stmt_lambda_element(session, stmt)
def _schema_version(hass: HomeAssistant) -> int:
return recorder.get_instance(hass).schema_version
async def async_recorder_block_till_done(hass: HomeAssistant) -> None:
"""Non blocking version of recorder.block_till_done()."""
await hass.async_add_executor_job(recorder.get_instance(hass).block_till_done)
"""Block the recorders event loop for testing.
Returns as soon as the recorder has started the block.
Does not wait for the block to finish.
"""
event = asyncio.Event()
get_instance(hass).queue_task(BlockRecorderTask(event, seconds))
await event.wait()
def do_adhoc_statistics(hass: HomeAssistant, **kwargs: Any) -> None:
"""Trigger an adhoc statistics run."""
if not (start := kwargs.get("start")):
start = statistics.get_start_time()
get_instance(hass).queue_task(StatisticsTask(start))
def wait_recording_done(hass: HomeAssistant) -> None:
"""Block till recording is done."""
hass.block_till_done()
trigger_db_commit(hass)
hass.block_till_done()
recorder.get_instance(hass).block_till_done()
hass.block_till_done()
def trigger_db_commit(hass: HomeAssistant) -> None:
"""Force the recorder to commit."""
for _ in range(recorder.DEFAULT_COMMIT_INTERVAL):
# We only commit on time change
message, last_event_time = await _async_get_ws_formatted_events(
hass,
msg["id"],
start_time,
subscriptions_setup_complete_time,
messages.event_message,
event_processor,
)
# If there is no last_time there are no historical
# results, but we still send an empty message so
# consumers of the api know their request was
# answered but there were no results
connection.send_message(message)
try:
await asyncio.wait_for(
get_instance(hass).async_block_till_done(), MAX_RECORDER_WAIT)
except asyncio.TimeoutError:
_LOGGER.debug(
"Recorder is behind more than %s seconds, starting live stream; Some results may be missing"
)
if setup_complete_future.cancelled():
# Unsubscribe happened while waiting for recorder
return
#
# Fetch any events from the database that have
# not been committed since the original fetch
# so we can switch over to using the subscriptions
#
# We only want events that happened after the last event
def wait_recording_done(self):
"""Block till recording is done."""
self.hass.block_till_done()
recorder.get_instance().block_till_done()
def setUp(self): # pylint: disable=invalid-name
"""Setup things to be run when tests are started."""
self.hass = get_test_home_assistant()
init_recorder_component(self.hass)
self.hass.start()
recorder.get_instance().block_till_done()
def tearDown(self): # pylint: disable=invalid-name
"""Stop everything that was started."""
self.hass.stop()
with self.assertRaises(RuntimeError):
recorder.get_instance()
def _get_states_with_session(
hass: HomeAssistant,
session: Session,
utc_point_in_time: datetime,
entity_ids: list[str] | None = None,
run: RecorderRuns | None = None,
filters: Any | None = None,
no_attributes: bool = False,
) -> list[State]:
"""Return the states at a specific point in time."""
if entity_ids and len(entity_ids) == 1:
return _get_single_entity_states_with_session(hass, session,
utc_point_in_time,
entity_ids[0],
no_attributes)
if run is None:
run = recorder.get_instance(hass).run_history.get(utc_point_in_time)
if run is None or process_timestamp(run.start) > utc_point_in_time:
# History did not run before utc_point_in_time
return []
# We have more than one entity to look at so we need to do a query on states
# since the last recorder run started.
query_keys, join_attributes = query_and_join_attributes(
hass, no_attributes)
query = session.query(*query_keys)
if entity_ids:
# We got an include-list of entities, accelerate the query by filtering already
# in the inner query.
most_recent_state_ids = (session.query(
func.max(States.state_id).label("max_state_id"), ).filter(
(States.last_updated >= run.start)
& (States.last_updated < utc_point_in_time)).filter(
States.entity_id.in_(entity_ids)))
most_recent_state_ids = most_recent_state_ids.group_by(
States.entity_id)
most_recent_state_ids = most_recent_state_ids.subquery()
query = query.join(
most_recent_state_ids,
States.state_id == most_recent_state_ids.c.max_state_id,
)
if join_attributes:
query = query.outerjoin(
StateAttributes,
(States.attributes_id == StateAttributes.attributes_id))
else:
# We did not get an include-list of entities, query all states in the inner
# query, then filter out unwanted domains as well as applying the custom filter.
# This filtering can't be done in the inner query because the domain column is
# not indexed and we can't control what's in the custom filter.
most_recent_states_by_date = (session.query(
States.entity_id.label("max_entity_id"),
func.max(States.last_updated).label("max_last_updated"),
).filter((States.last_updated >= run.start)
& (States.last_updated < utc_point_in_time)).group_by(
States.entity_id).subquery())
most_recent_state_ids = (session.query(
func.max(States.state_id).label("max_state_id")).join(
most_recent_states_by_date,
and_(
States.entity_id ==
most_recent_states_by_date.c.max_entity_id,
States.last_updated ==
most_recent_states_by_date.c.max_last_updated,
),
).group_by(States.entity_id).subquery())
query = query.join(
most_recent_state_ids,
States.state_id == most_recent_state_ids.c.max_state_id,
)
for entity_domain in IGNORE_DOMAINS_ENTITY_ID_LIKE:
query = query.filter(~States.entity_id.like(entity_domain))
if filters:
query = filters.apply(query)
if join_attributes:
query = query.outerjoin(
StateAttributes,
(States.attributes_id == StateAttributes.attributes_id))
attr_cache: dict[str, dict[str, Any]] = {}
return [LazyState(row, attr_cache) for row in execute(query)]
def setUp(self): # pylint: disable=invalid-name
"""Setup things to be run when tests are started."""
self.hass = get_test_home_assistant()
init_recorder_component(self.hass)
self.hass.start()
recorder.get_instance().block_till_done()
def tearDown(self): # pylint: disable=invalid-name
"""Stop everything that was started."""
self.hass.stop()
with self.assertRaises(RuntimeError):
recorder.get_instance()
