def test_service_disable_states_not_recording(opp, opp_recorder): """Test that state changes are not recorded when recorder is disabled using service.""" opp = opp_recorder() assert opp.services.call( DOMAIN, SERVICE_DISABLE, {}, blocking=True, ) opp.states.set("test.one", "on", {}) wait_recording_done(opp) with session_scope(opp=opp) as session: assert len(list(session.query(States))) == 0 assert opp.services.call( DOMAIN, SERVICE_ENABLE, {}, blocking=True, ) opp.states.set("test.two", "off", {}) wait_recording_done(opp) with session_scope(opp=opp) as session: db_states = list(session.query(States)) assert len(db_states) == 1 assert db_states[0].event_id > 0 assert db_states[0].to_native() == _state_empty_context( opp, "test.two")
def _add_events(opp, events): with session_scope(opp=opp) as session: session.query(Events).delete(synchronize_session=False) for event_type in events: opp.bus.fire(event_type) wait_recording_done(opp) with session_scope(opp=opp) as session: return [ev.to_native() for ev in session.query(Events)]
def _add_events(opp, events): with session_scope(opp=opp) as session: session.query(Events).delete(synchronize_session=False) for event_type in events: opp.bus.fire(event_type) opp.block_till_done() opp.data[DATA_INSTANCE].block_till_done() with session_scope(opp=opp) as session: return [ev.to_native() for ev in session.query(Events)]
def test_entity_id_filter(opp_recorder): """Test that entity ID filtering filters string and list.""" opp = opp_recorder({ "include": { "domains": "hello" }, "exclude": { "domains": "hidden_domain" } }) for idx, data in enumerate(( {}, { "entity_id": "hello.world" }, { "entity_id": ["hello.world"] }, { "entity_id": ["hello.world", "hidden_domain.person"] }, { "entity_id": { "unexpected": "data" } }, )): opp.bus.fire("hello", data) wait_recording_done(opp) with session_scope(opp=opp) as session: db_events = list( session.query(Events).filter_by(event_type="hello")) assert len(db_events) == idx + 1, data for data in ( { "entity_id": "hidden_domain.person" }, { "entity_id": ["hidden_domain.person"] }, ): opp.bus.fire("hello", data) wait_recording_done(opp) with session_scope(opp=opp) as session: db_events = list( session.query(Events).filter_by(event_type="hello")) # Keep referring idx + 1, as no new events are being added assert len(db_events) == idx + 1, data
def test_run_information(opp_recorder): """Ensure run_information returns expected data.""" before_start_recording = dt_util.utcnow() opp = opp_recorder() run_info = run_information_from_instance(opp) assert isinstance(run_info, RecorderRuns) assert run_info.closed_incorrect is False with session_scope(opp=opp) as session: run_info = run_information_with_session(session) assert isinstance(run_info, RecorderRuns) assert run_info.closed_incorrect is False run_info = run_information(opp) assert isinstance(run_info, RecorderRuns) assert run_info.closed_incorrect is False opp.states.set("test.two", "on", {}) wait_recording_done(opp) run_info = run_information(opp) assert isinstance(run_info, RecorderRuns) assert run_info.closed_incorrect is False run_info = run_information(opp, before_start_recording) assert run_info is None run_info = run_information(opp, dt_util.utcnow()) assert isinstance(run_info, RecorderRuns) assert run_info.closed_incorrect is False
async def test_purge_old_states( opp: OpenPeerPower, async_setup_recorder_instance: SetupRecorderInstanceT ): """Test deleting old states.""" instance = await async_setup_recorder_instance(opp) await _add_test_states(opp, instance) # make sure we start with 6 states with session_scope(opp=opp) as session: states = session.query(States) assert states.count() == 6 assert states[0].old_state_id is None assert states[-1].old_state_id == states[-2].state_id events = session.query(Events).filter(Events.event_type == "state_changed") assert events.count() == 6 # run purge_old_data() finished = purge_old_data(instance, 4, repack=False) assert not finished assert states.count() == 2 states_after_purge = session.query(States) assert states_after_purge[1].old_state_id == states_after_purge[0].state_id assert states_after_purge[0].old_state_id is None finished = purge_old_data(instance, 4, repack=False) assert finished assert states.count() == 2
def state_changes_during_period(opp, start_time, end_time=None, entity_id=None): """Return states changes during UTC period start_time - end_time.""" with session_scope(opp=opp) as session: baked_query = opp.data[HISTORY_BAKERY]( lambda session: session.query(*QUERY_STATES)) baked_query += lambda q: q.filter( (States.last_changed == States.last_updated) & (States.last_updated > bindparam("start_time"))) if end_time is not None: baked_query += lambda q: q.filter(States.last_updated < bindparam( "end_time")) if entity_id is not None: baked_query += lambda q: q.filter_by(entity_id=bindparam( "entity_id")) entity_id = entity_id.lower() baked_query += lambda q: q.order_by(States.entity_id, States. last_updated) states = execute( baked_query(session).params(start_time=start_time, end_time=end_time, entity_id=entity_id)) entity_ids = [entity_id] if entity_id is not None else None return _sorted_states_to_dict(opp, session, states, start_time, entity_ids)
def _load_history_from_db(self): """Load the history of the brightness values from the database. This only needs to be done once during startup. """ start_date = datetime.now() - timedelta(days=self._conf_check_days) entity_id = self._readingmap.get(READING_BRIGHTNESS) if entity_id is None: _LOGGER.debug("Not reading the history from the database as " "there is no brightness sensor configured") return _LOGGER.debug("Initializing values for %s from the database", self._name) with session_scope(opp=self.opp) as session: query = (session.query(States).filter( (States.entity_id == entity_id.lower()) and (States.last_updated > start_date)).order_by( States.last_updated.asc())) states = execute(query, to_native=True, validate_entity_ids=False) for state in states: # filter out all None, NaN and "unknown" states # only keep real values with suppress(ValueError): self._brightness_history.add_measurement( int(state.state), state.last_updated) _LOGGER.debug("Initializing from database completed")
def _get_events(opp, config, start_day, end_day, entity_id=None): """Get events for a period of time.""" entities_filter = _generate_filter_from_config(config) def yield_events(query): """Yield Events that are not filtered away.""" for row in query.yield_per(500): event = row.to_native() if _keep_event(event, entities_filter): yield event with session_scope(opp=opp) as session: if entity_id is not None: entity_ids = [entity_id.lower()] else: entity_ids = _get_related_entity_ids(session, entities_filter) query = (session.query(Events).order_by(Events.time_fired).outerjoin( States, (Events.event_id == States.event_id)).filter( Events.event_type.in_(ALL_EVENT_TYPES) ).filter((Events.time_fired > start_day) & (Events.time_fired < end_day) ).filter(((States.last_updated == States.last_changed) & States.entity_id.in_(entity_ids)) | (States.state_id.is_(None)))) return list(humanify(opp, yield_events(query)))
def test_saving_state_with_sqlalchemy_exception(opp, opp_recorder, caplog): """Test saving state when there is an SQLAlchemyError.""" opp = opp_recorder() entity_id = "test.recorder" state = "restoring_from_db" attributes = {"test_attr": 5, "test_attr_10": "nice"} def _throw_if_state_in_session(*args, **kwargs): for obj in opp.data[DATA_INSTANCE].event_session: if isinstance(obj, States): raise SQLAlchemyError("insert the state", "fake params", "forced to fail") with patch("time.sleep"), patch.object( opp.data[DATA_INSTANCE].event_session, "flush", side_effect=_throw_if_state_in_session, ): opp.states.set(entity_id, "fail", attributes) wait_recording_done(opp) assert "SQLAlchemyError error processing event" in caplog.text caplog.clear() opp.states.set(entity_id, state, attributes) wait_recording_done(opp) with session_scope(opp=opp) as session: db_states = list(session.query(States)) assert len(db_states) >= 1 assert "Error executing query" not in caplog.text assert "Error saving events" not in caplog.text assert "SQLAlchemyError error processing event" not in caplog.text
async def test_purge_old_states_encouters_database_corruption( opp: OpenPeerPower, async_setup_recorder_instance: SetupRecorderInstanceT ): """Test database image image is malformed while deleting old states.""" instance = await async_setup_recorder_instance(opp) await _add_test_states(opp, instance) await async_wait_recording_done_without_instance(opp) sqlite3_exception = DatabaseError("statement", {}, []) sqlite3_exception.__cause__ = sqlite3.DatabaseError() with patch( "openpeerpower.components.recorder.move_away_broken_database" ) as move_away, patch( "openpeerpower.components.recorder.purge.purge_old_data", side_effect=sqlite3_exception, ): await opp.services.async_call( recorder.DOMAIN, recorder.SERVICE_PURGE, {"keep_days": 0} ) await opp.async_block_till_done() await async_wait_recording_done_without_instance(opp) assert move_away.called # Ensure the whole database was reset due to the database error with session_scope(opp=opp) as session: states_after_purge = session.query(States) assert states_after_purge.count() == 0
def get_last_state_changes(opp, number_of_states, entity_id): """Return the last number_of_states.""" start_time = dt_util.utcnow() with session_scope(opp=opp) as session: baked_query = opp.data[HISTORY_BAKERY]( lambda session: session.query(*QUERY_STATES)) baked_query += lambda q: q.filter(States.last_changed == States. last_updated) if entity_id is not None: baked_query += lambda q: q.filter_by(entity_id=bindparam( "entity_id")) entity_id = entity_id.lower() baked_query += lambda q: q.order_by(States.entity_id, States.last_updated.desc()) baked_query += lambda q: q.limit(bindparam("number_of_states")) states = execute( baked_query(session).params(number_of_states=number_of_states, entity_id=entity_id)) entity_ids = [entity_id] if entity_id is not None else None return _sorted_states_to_dict( opp, session, reversed(states), start_time, entity_ids, include_start_time_state=False, )
def test_session_scope_not_setup(opp_recorder): """Try to create a session scope when not setup.""" opp = opp_recorder() with patch.object(opp.data[DATA_INSTANCE], "get_session", return_value=None), pytest.raises(RuntimeError): with util.session_scope(opp=opp): pass
def test_purge_method(self): """Test purge method.""" service_data = {"keep_days": 4} self._add_test_events() self._add_test_states() # make sure we start with 6 states with session_scope(opp=self.opp) as session: states = session.query(States) assert states.count() == 6 events = session.query(Events).filter( Events.event_type.like("EVENT_TEST%")) assert events.count() == 6 self.opp.data[DATA_INSTANCE].block_till_done() # run purge method - no service data, use defaults self.opp.services.call("recorder", "purge") self.opp.block_till_done() # Small wait for recorder thread self.opp.data[DATA_INSTANCE].block_till_done() # only purged old events assert states.count() == 4 assert events.count() == 4 # run purge method - correct service data self.opp.services.call("recorder", "purge", service_data=service_data) self.opp.block_till_done() # Small wait for recorder thread self.opp.data[DATA_INSTANCE].block_till_done() # we should only have 2 states left after purging assert states.count() == 2 # now we should only have 2 events left assert events.count() == 2 assert not ("EVENT_TEST_PURGE" in (event.event_type for event in events.all())) # run purge method - correct service data, with repack with patch("openpeerpower.components.recorder.purge._LOGGER" ) as mock_logger: service_data["repack"] = True self.opp.services.call("recorder", "purge", service_data=service_data) self.opp.block_till_done() self.opp.data[DATA_INSTANCE].block_till_done() assert (mock_logger.debug.mock_calls[3][1][0] == "Vacuuming SQL DB to free space")
def _add_entities(opp, entity_ids): """Add entities.""" attributes = {"test_attr": 5, "test_attr_10": "nice"} for idx, entity_id in enumerate(entity_ids): opp.states.set(entity_id, f"state{idx}", attributes) wait_recording_done(opp) with session_scope(opp=opp) as session: return [st.to_native() for st in session.query(States)]
def _add_entities(opp, entity_ids): """Add entities.""" attributes = {"test_attr": 5, "test_attr_10": "nice"} for idx, entity_id in enumerate(entity_ids): opp.states.set(entity_id, "state{}".format(idx), attributes) opp.block_till_done() opp.data[DATA_INSTANCE].block_till_done() with session_scope(opp=opp) as session: return [st.to_native() for st in session.query(States)]
async def test_purge_edge_case( opp: OpenPeerPower, async_setup_recorder_instance: SetupRecorderInstanceT, ): """Test states and events are purged even if they occurred shortly before purge_before.""" async def _add_db_entries(opp: OpenPeerPower, timestamp: datetime) -> None: with recorder.session_scope(opp=opp) as session: session.add( Events( event_id=1001, event_type="EVENT_TEST_PURGE", event_data="{}", origin="LOCAL", created=timestamp, time_fired=timestamp, ) ) session.add( States( entity_id="test.recorder2", domain="sensor", state="purgeme", attributes="{}", last_changed=timestamp, last_updated=timestamp, created=timestamp, event_id=1001, ) ) instance = await async_setup_recorder_instance(opp, None) await async_wait_purge_done(opp, instance) service_data = {"keep_days": 2} timestamp = dt_util.utcnow() - timedelta(days=2, minutes=1) await _add_db_entries(opp, timestamp) with session_scope(opp=opp) as session: states = session.query(States) assert states.count() == 1 events = session.query(Events).filter(Events.event_type == "EVENT_TEST_PURGE") assert events.count() == 1 await opp.services.async_call( recorder.DOMAIN, recorder.SERVICE_PURGE, service_data ) await opp.async_block_till_done() await async_recorder_block_till_done(opp, instance) await async_wait_purge_done(opp, instance) assert states.count() == 0 assert events.count() == 0
def test_service_disable_run_information_recorded(tmpdir): """Test that runs are still recorded when recorder is disabled.""" test_db_file = tmpdir.mkdir("sqlite").join("test_run_info.db") dburl = f"{SQLITE_URL_PREFIX}//{test_db_file}" opp = get_test_open_peer_power() setup_component(opp, DOMAIN, {DOMAIN: {CONF_DB_URL: dburl}}) opp.start() wait_recording_done(opp) with session_scope(opp=opp) as session: db_run_info = list(session.query(RecorderRuns)) assert len(db_run_info) == 1 assert db_run_info[0].start is not None assert db_run_info[0].end is None assert opp.services.call( DOMAIN, SERVICE_DISABLE, {}, blocking=True, ) wait_recording_done(opp) opp.stop() opp = get_test_open_peer_power() setup_component(opp, DOMAIN, {DOMAIN: {CONF_DB_URL: dburl}}) opp.start() wait_recording_done(opp) with session_scope(opp=opp) as session: db_run_info = list(session.query(RecorderRuns)) assert len(db_run_info) == 2 assert db_run_info[0].start is not None assert db_run_info[0].end is not None assert db_run_info[1].start is not None assert db_run_info[1].end is None opp.stop()
def test_purge_old_states(self): """Test deleting old states.""" self._add_test_states() # make sure we start with 6 states with session_scope(opp=self.opp) as session: states = session.query(States) assert states.count() == 6 # run purge_old_data() purge_old_data(self.opp.data[DATA_INSTANCE], 4, repack=False) # we should only have 2 states left after purging assert states.count() == 2
def test_recorder_bad_commit(opp_recorder): """Bad _commit should retry 3 times.""" opp = opp_recorder() def work(session): """Bad work.""" session.execute("select * from notthere") with patch("openpeerpower.components.recorder.time.sleep" ) as e_mock, util.session_scope(opp=opp) as session: res = util.commit(session, work) assert res is False assert e_mock.call_count == 3
def test_purge_old_events(self): """Test deleting old events.""" self._add_test_events() with session_scope(opp=self.opp) as session: events = session.query(Events).filter( Events.event_type.like("EVENT_TEST%")) assert events.count() == 6 # run purge_old_data() purge_old_data(self.opp.data[DATA_INSTANCE], 4, repack=False) # we should only have 2 events left assert events.count() == 2
def get_states(opp, utc_point_in_time, entity_ids=None, run=None, filters=None): """Return the states at a specific point in time.""" if run is None: run = recorder.run_information_from_instance(opp, utc_point_in_time) # History did not run before utc_point_in_time if run is None: return [] with session_scope(opp=opp) as session: return _get_states_with_session(opp, session, utc_point_in_time, entity_ids, run, filters)
def _sorted_significant_states_json( self, opp, start_time, end_time, entity_ids, include_start_time_state, significant_changes_only, minimal_response, ): """Fetch significant stats from the database as json.""" timer_start = time.perf_counter() with session_scope(opp=opp) as session: result = ( history._get_significant_states( # pylint: disable=protected-access opp, session, start_time, end_time, entity_ids, self.filters, include_start_time_state, significant_changes_only, minimal_response, ) ) result = list(result.values()) if _LOGGER.isEnabledFor(logging.DEBUG): elapsed = time.perf_counter() - timer_start _LOGGER.debug("Extracted %d states in %fs", sum(map(len, result)), elapsed) # Optionally reorder the result to respect the ordering given # by any entities explicitly included in the configuration. if self.filters and self.use_include_order: sorted_result = [] for order_entity in self.filters.included_entities: for state_list in result: if state_list[0].entity_id == order_entity: sorted_result.append(state_list) result.remove(state_list) break sorted_result.extend(result) result = sorted_result return self.json(result)
def test_saving_state_with_commit_interval_zero(opp_recorder): """Test saving a state with a commit interval of zero.""" opp = opp_recorder({"commit_interval": 0}) assert opp.data[DATA_INSTANCE].commit_interval == 0 entity_id = "test.recorder" state = "restoring_from_db" attributes = {"test_attr": 5, "test_attr_10": "nice"} opp.states.set(entity_id, state, attributes) wait_recording_done(opp) with session_scope(opp=opp) as session: db_states = list(session.query(States)) assert len(db_states) == 1 assert db_states[0].event_id > 0
def test_saving_state(self): """Test saving and restoring a state.""" entity_id = "test.recorder" state = "restoring_from_db" attributes = {"test_attr": 5, "test_attr_10": "nice"} self.opp.states.set(entity_id, state, attributes) self.opp.block_till_done() self.opp.data[DATA_INSTANCE].block_till_done() with session_scope(opp=self.opp) as session: db_states = list(session.query(States)) assert len(db_states) == 1 assert db_states[0].event_id > 0 state = db_states[0].to_native() assert state == self.opp.states.get(entity_id)
def test_saving_state_and_removing_entity(opp, opp_recorder): """Test saving the state of a removed entity.""" opp = opp_recorder() entity_id = "lock.mine" opp.states.set(entity_id, STATE_LOCKED) opp.states.set(entity_id, STATE_UNLOCKED) opp.states.async_remove(entity_id) wait_recording_done(opp) with session_scope(opp=opp) as session: states = list(session.query(States)) assert len(states) == 3 assert states[0].entity_id == entity_id assert states[0].state == STATE_LOCKED assert states[1].entity_id == entity_id assert states[1].state == STATE_UNLOCKED assert states[2].entity_id == entity_id assert states[2].state is None
async def test_purge_old_recorder_runs( opp: OpenPeerPower, async_setup_recorder_instance: SetupRecorderInstanceT ): """Test deleting old recorder runs keeps current run.""" instance = await async_setup_recorder_instance(opp) await _add_test_recorder_runs(opp, instance) # make sure we start with 7 recorder runs with session_scope(opp=opp) as session: recorder_runs = session.query(RecorderRuns) assert recorder_runs.count() == 7 # run purge_old_data() finished = purge_old_data(instance, 0, repack=False) assert not finished finished = purge_old_data(instance, 0, repack=False) assert finished assert recorder_runs.count() == 1
def get_significant_states( opp, start_time, end_time=None, entity_ids=None, filters=None, include_start_time_state=True, ): """ Return states changes during UTC period start_time - end_time. Significant states are all states where there is a state change, as well as all states from certain domains (for instance thermostat so that we get current temperature in our graphs). """ timer_start = time.perf_counter() with session_scope(opp=opp) as session: query = session.query(States).filter( (States.domain.in_(SIGNIFICANT_DOMAINS) | (States.last_changed == States.last_updated)) & (States.last_updated > start_time)) if filters: query = filters.apply(query, entity_ids) if end_time is not None: query = query.filter(States.last_updated < end_time) query = query.order_by(States.last_updated) states = (state for state in execute(query) if (_is_significant(state) and not state.attributes.get(ATTR_HIDDEN, False))) if _LOGGER.isEnabledFor(logging.DEBUG): elapsed = time.perf_counter() - timer_start _LOGGER.debug("get_significant_states took %fs", elapsed) return states_to_json(opp, states, start_time, entity_ids, filters, include_start_time_state)
async def test_purge_old_events( opp: OpenPeerPower, async_setup_recorder_instance: SetupRecorderInstanceT ): """Test deleting old events.""" instance = await async_setup_recorder_instance(opp) await _add_test_events(opp, instance) with session_scope(opp=opp) as session: events = session.query(Events).filter(Events.event_type.like("EVENT_TEST%")) assert events.count() == 6 # run purge_old_data() finished = purge_old_data(instance, 4, repack=False) assert not finished assert events.count() == 2 # we should only have 2 events left finished = purge_old_data(instance, 4, repack=False) assert finished assert events.count() == 2
async def test_saving_state( opp: OpenPeerPower, async_setup_recorder_instance: SetupRecorderInstanceT): """Test saving and restoring a state.""" instance = await async_setup_recorder_instance(opp) entity_id = "test.recorder" state = "restoring_from_db" attributes = {"test_attr": 5, "test_attr_10": "nice"} opp.states.async_set(entity_id, state, attributes) await async_wait_recording_done(opp, instance) with session_scope(opp=opp) as session: db_states = list(session.query(States)) assert len(db_states) == 1 assert db_states[0].event_id > 0 state = db_states[0].to_native() assert state == _state_empty_context(opp, entity_id)