def format_alert(self, alert_json): alert = self.parse_alert_metrics(alert_json) try: alert["alert_id"] = None alert["node_id"] = utils.find_node_id( alert['tags']['integration_id'], alert['tags']['fqdn']) alert["time_stamp"] = tendrl_now().isoformat() alert["resource"] = self.representive_name alert['alert_type'] = constants.ALERT_TYPE alert['significance'] = constants.SIGNIFICANCE_HIGH alert['pid'] = utils.find_grafana_pid() alert['source'] = constants.ALERT_SOURCE alert['tags']['fqdn'] = alert['tags']['fqdn'] if alert_json['State'] == constants.GRAFANA_ALERT: if "critical" in alert_json['Name'].lower(): alert['severity'] = \ constants.TENDRL_SEVERITY_MAP['critical'] else: alert['severity'] = \ constants.TENDRL_SEVERITY_MAP['warning'] alert['tags']['message'] = ( "Cpu utilization on node %s in %s" " at %s %% and running out of cpu" % (alert['tags']['fqdn'], alert['tags']['cluster_short_name'], alert['current_value'])) elif alert_json['State'] == constants.GRAFANA_CLEAR_ALERT: # Identifying clear alert from which panel critical/warning if "critical" in alert_json['Name'].lower(): alert['tags']['clear_alert'] = \ constants.TENDRL_SEVERITY_MAP['critical'] elif "warning" in alert_json['Name'].lower(): alert['tags']['clear_alert'] = \ constants.TENDRL_SEVERITY_MAP['warning'] alert['severity'] = constants.TENDRL_SEVERITY_MAP['info'] alert['tags']['message'] = \ ("Cpu utilization on node %s in" " %s back to normal" % ( alert['tags']['fqdn'], alert['tags']['cluster_short_name'])) else: logger.log( "error", NS.publisher_id, { "message": "Unsupported alert %s " "severity" % alert_json }) raise InvalidAlertSeverity return alert except (KeyError, CalledProcessError, EtcdKeyNotFound, NodeNotFound, InvalidAlertSeverity) as ex: Event( ExceptionMessage( "debug", NS.publisher_id, { "message": "Error in converting grafana" "alert into tendrl alert %s" % alert_json, "exception": ex }))
def format_alert(self, alert_json): alert = self.parse_alert_metrics(alert_json) try: alert["alert_id"] = None alert["node_id"] = utils.find_node_id( alert['tags']['integration_id'], alert['tags']['fqdn'] ) alert["time_stamp"] = alert_json['NewStateDate'] alert["resource"] = self.representive_name alert['alert_type'] = constants.ALERT_TYPE alert['severity'] = constants.TENDRL_GRAFANA_SEVERITY_MAP[ alert_json['State']] alert['significance'] = constants.SIGNIFICANCE_HIGH alert['pid'] = utils.find_grafana_pid() alert['source'] = constants.ALERT_SOURCE alert['tags']['fqdn'] = alert['tags']['fqdn'] alert['classification'] = alert_json["classification"] if alert['severity'] == "WARNING": alert['tags']['message'] = ("Swap utilization of node %s is" " %s which is above the %s " "threshold (%s)." % ( alert['tags']['fqdn'], alert['current_value'], alert['severity'], alert['tags']['warning_max'] )) elif alert['severity'] == "INFO": alert['tags']['message'] = ("Swap utilization of node %s is" " back to normal" % ( alert['tags']['fqdn'])) else: logger.log( "error", NS.publisher_id, { "message": "Alert %s have unsupported alert" "severity" % alert_json } ) raise InvalidAlertSeverity return alert except (KeyError, CalledProcessError, EtcdKeyNotFound, NodeNotFound, InvalidAlertSeverity) as ex: Event( ExceptionMessage( "error", NS.publisher_id, { "message": "Error in converting grafana" "alert into tendrl alert %s" % alert_json, "exception": ex } ) )
def format_alert(self, alert_json): alert = self.parse_alert_metrics(alert_json) try: alert["alert_id"] = None alert["node_id"] = utils.find_node_id( alert['tags']['integration_id'], alert['tags']['fqdn']) alert["time_stamp"] = tendrl_now().isoformat() alert["resource"] = self.representive_name alert['alert_type'] = constants.ALERT_TYPE alert['significance'] = constants.SIGNIFICANCE_HIGH alert['pid'] = utils.find_grafana_pid() alert['source'] = constants.ALERT_SOURCE alert['tags']['cluster_name'] = utils.find_cluster_name( alert['tags']['integration_id']) alert["tags"]["volume_name"] = utils.find_volume_name( alert['tags']['integration_id'], alert['tags']['fqdn'].replace('_', '.'), alert['tags']['brick_path'].strip(":").replace( grafana_constants.BRICK_PATH_SEPARATOR, '_')) if alert_json['State'] == constants.GRAFANA_ALERT: if "critical" in alert_json['Name'].lower(): alert['severity'] = \ constants.TENDRL_SEVERITY_MAP['critical'] else: alert['severity'] = \ constants.TENDRL_SEVERITY_MAP['warning'] # Modify brick path symbol to slash(/) in alert message alert['tags']['message'] = ( "Brick utilization on %s:%s in %s " "at %s %% and nearing full capacity" % (alert['tags']['fqdn'], alert['tags']['brick_path'].replace( grafana_constants.BRICK_PATH_SEPARATOR, "/"), alert["tags"]["volume_name"], alert['current_value'])) elif alert_json['State'] == constants.GRAFANA_CLEAR_ALERT: # Identifying clear alert from which panel critical/warning if "critical" in alert_json['Name'].lower(): alert['tags']['clear_alert'] = \ constants.TENDRL_SEVERITY_MAP['critical'] elif "warning" in alert_json['Name'].lower(): alert['tags']['clear_alert'] = \ constants.TENDRL_SEVERITY_MAP['warning'] alert['severity'] = constants.TENDRL_SEVERITY_MAP['info'] # Modify brick path symbol to slash(/) in alert message alert['tags']['message'] = ( "Brick utilization of %s:%s in %s " "back to normal" % (alert['tags']['fqdn'], alert['tags']['brick_path'].replace( grafana_constants.BRICK_PATH_SEPARATOR, "/"), alert["tags"]["volume_name"])) else: logger.log( "error", NS.publisher_id, { "message": "Unsupported alert %s " "severity" % alert_json }) raise InvalidAlertSeverity return alert except (KeyError, CalledProcessError, EtcdKeyNotFound, NodeNotFound, InvalidAlertSeverity) as ex: Event( ExceptionMessage( "debug", NS.publisher_id, { "message": "Error in converting grafana" "alert into tendrl alert %s" % alert_json, "exception": ex }))
def format_alert(self, alert_json): alert = self.parse_alert_metrics(alert_json) try: alert["alert_id"] = None alert["node_id"] = utils.find_node_id( alert['tags']['integration_id'], alert['tags']['fqdn'] ) alert["time_stamp"] = alert_json['NewStateDate'] alert["resource"] = self.representive_name alert['alert_type'] = constants.ALERT_TYPE alert['significance'] = constants.SIGNIFICANCE_HIGH alert['pid'] = utils.find_grafana_pid() alert['source'] = constants.ALERT_SOURCE alert['tags']['cluster_name'] = utils.find_cluster_name( alert['tags']['integration_id']) alert["tags"]["volume_name"] = utils.find_volume_name( alert['tags']['integration_id'], alert['tags']['fqdn'].replace('_', '.'), alert['tags']['brick_path'].strip("|").replace('|', '_') ) if alert_json['State'] == constants.GRAFANA_ALERT: if "critical" in alert_json['Name'].lower(): alert['severity'] = \ constants.TENDRL_SEVERITY_MAP['critical'] else: alert['severity'] = \ constants.TENDRL_SEVERITY_MAP['warning'] alert['tags']['message'] = ( "Brick utilization of %s:%s under volume %s in " "cluster %s is %s %% which is above %s" " threshold (%s %%)" % ( alert['tags']['fqdn'], alert['tags']['brick_path'], alert["tags"]["volume_name"], alert['tags']['integration_id'], alert['current_value'], alert['severity'], alert['tags']['warning_max'] ) ) elif alert_json['State'] == constants.GRAFANA_CLEAR_ALERT: # Identifying clear alert from which panel critical/warning if "critical" in alert_json['Name'].lower(): alert['tags']['clear_alert'] = \ constants.TENDRL_SEVERITY_MAP['critical'] elif "warning" in alert_json['Name'].lower(): alert['tags']['clear_alert'] = \ constants.TENDRL_SEVERITY_MAP['warning'] alert['severity'] = constants.TENDRL_SEVERITY_MAP['info'] alert['tags']['message'] = ( "Brick utilization of %s:%s under volume %s in " "cluster %s is back normal" % ( alert['tags']['fqdn'], alert['tags']['brick_path'], alert["tags"]["volume_name"], alert['tags']['integration_id'] ) ) else: logger.log( "error", NS.publisher_id, { "message": "Alert %s have unsupported alert" "severity" % alert_json } ) raise InvalidAlertSeverity return alert except (KeyError, CalledProcessError, EtcdKeyNotFound, NodeNotFound, InvalidAlertSeverity) as ex: Event( ExceptionMessage( "debug", NS.publisher_id, { "message": "Error in converting grafana" "alert into tendrl alert %s" % alert_json, "exception": ex } ) )