def run_alert_query(alert_id: int, database_id: int, sql: str, label: str) -> Optional[bool]: """ Execute alert.sql and return value if any rows are returned """ logger.info("Processing alert ID: %i", alert_id) database = db.session.query(Database).get(database_id) if not database: logger.error("Alert database not preset") return None if not sql: logger.error("Alert SQL not preset") return None parsed_query = ParsedQuery(sql) sql = parsed_query.stripped() state = None dttm_start = datetime.utcnow() df = pd.DataFrame() try: logger.info("Evaluating SQL for alert <%s:%s>", alert_id, label) df = database.get_df(sql) except Exception as exc: # pylint: disable=broad-except state = AlertState.ERROR logging.exception(exc) logging.error("Failed at evaluating alert: %s (%s)", label, alert_id) dttm_end = datetime.utcnow() last_eval_dttm = datetime.utcnow() if state != AlertState.ERROR: if not df.empty: # Looking for truthy cells for row in df.to_records(): if any(row): state = AlertState.TRIGGER deliver_alert(alert_id) break if not state: state = AlertState.PASS db.session.commit() alert = db.session.query(Alert).get(alert_id) if state != AlertState.ERROR: alert.last_eval_dttm = last_eval_dttm alert.last_state = state alert.logs.append( AlertLog( scheduled_dttm=dttm_start, dttm_start=dttm_start, dttm_end=dttm_end, state=state, )) db.session.commit() return None
def evaluate_alert( alert_id: int, label: str, session: Session, recipients: Optional[str] = None, slack_channel: Optional[str] = None, ) -> None: """Processes an alert to see if it should be triggered""" logger.info("Processing alert ID: %i", alert_id) state = None dttm_start = datetime.utcnow() try: logger.info("Querying observers for alert <%s:%s>", alert_id, label) error_msg = observe(alert_id, session) if error_msg: state = AlertState.ERROR logging.error(error_msg) except Exception as exc: # pylint: disable=broad-except state = AlertState.ERROR logging.exception(exc) logging.error("Failed at query observers for alert: %s (%s)", label, alert_id) dttm_end = datetime.utcnow() if state != AlertState.ERROR: # Don't validate alert on test runs since it may not be triggered if recipients or slack_channel: deliver_alert(alert_id, session, recipients, slack_channel) state = AlertState.TRIGGER # Validate during regular workflow and deliver only if triggered elif validate_observations(alert_id, label, session): deliver_alert(alert_id, session, recipients, slack_channel) state = AlertState.TRIGGER else: state = AlertState.PASS session.commit() alert = session.query(Alert).get(alert_id) if state != AlertState.ERROR: alert.last_eval_dttm = dttm_end alert.last_state = state alert.logs.append( AlertLog( scheduled_dttm=dttm_start, dttm_start=dttm_start, dttm_end=dttm_end, state=state, )) session.commit()