def put(self, alert_rules_id):
json_data = request.get_json(force=True)
action = json_data['action']
if action['method'] == 'modify':
statistical_period = action['param']['statistical_period']
# statistical_approach = action['param']['statistical_approach']
compute_mode = action['param']['compute_mode']
threshold_value = action['param']['threshold_value']
contact_groups = action['param']['contact_groups']
notify_type = action['param']['notify_type']
try:
data_res = AlertManager.get_alert_rules(alert_rules_id)
host_id = data_res['host_id']
port = data_res['port']
service = data_res['service']
monitor_items = data_res['monitor_items']
Ansible.check(MONITOR_SERVER)
inv_file = Ansible.init_target_yaml(MONITOR_SERVER)
instance = host_id + ":" + str(port)
yml_file = Ansible.init_metrics_yaml(service, monitor_items,
host_id, instance,
threshold_value,
statistical_period,
compute_mode)
Ansible.execute(yml_file, inv_file, alert_rules_id)
data_res = AlertManager.update_alert_rules(
alert_rules_id, statistical_period, compute_mode,
threshold_value, contact_groups, notify_type)
except Exception as e:
raise Error(str(e))
elif action['method'] == 'disable':
try:
silence_time = action['param']['silence_time']
silence_time_hour = str(silence_time) + 'h'
data_res = AlertManager.get_alert_rules(alert_rules_id)
alert_name = data_res['service'] + '_' + data_res[
'monitor_items'] + '_' + data_res['host_id'] + ':' + str(
data_res['port'])
AlertManager.disable_alert(alert_name, silence_time_hour)
AlertRulesData.update_silence_time(alert_rules_id,
int(silence_time))
except Exception as e:
raise Error(str(e))
elif action['method'] == 'enable':
data_res = AlertManager.get_alert_rules(alert_rules_id)
alert_name = data_res['service'] + '_' + data_res[
'monitor_items'] + '_' + data_res['host_id'] + ':' + str(
data_res['port'])
AlertManager.enable_alert(alert_name)
AlertRulesData.update_silence_time(alert_rules_id, 0)
return {'status': 'ok'}, 201
def post(self):
args = CreateAlertRules.alert_rules_parser.parse_args()
host_id = args['host_id']
port = args['port']
service = args['service']
monitor_items = args['monitor_items']
statistical_period = args['statistical_period']
statistical_approach = args['statistical_approach']
compute_mode = args['compute_mode']
threshold_value = args['threshold_value']
# silence_time = args['silence_time']
contact_groups = args['contact_groups']
notify_type = args['notify_type']
try:
# running
alert_rules_id = str(uuid.uuid1())
# insert mysql
data_res = AlertManager.create_alert_rules(
alert_rules_id, host_id, port, service, monitor_items,
statistical_period, statistical_approach, compute_mode,
threshold_value, contact_groups, notify_type, 0)
Ansible.check(MONITOR_SERVER)
inv_file = Ansible.init_target_yaml(MONITOR_SERVER)
instance = host_id + ":" + str(port)
yml_file = Ansible.init_metrics_yaml(service, monitor_items,
host_id, instance,
threshold_value,
statistical_period,
compute_mode)
Ansible.execute(yml_file, inv_file, alert_rules_id)
except Exception as e:
raise Error(e)
return {'status': 'ok'}, 201
def write_to_file(file, str_config):
try:
with open(file, "w") as fobj:
fobj.write(str_config)
except Exception, err:
errmsg = "Write to file error: %s" % err
raise Error(str(errmsg))
def post(self):
try:
result = request.get_json(force=True)
json_string = json.dumps(result)
json_data = json.loads(json_string)
alerts_info = json_data['alerts'][0]
state = alerts_info['status']
exported_instance = alerts_info['labels']['exported_instance']
host_id = exported_instance.split(':')[0]
port = int(exported_instance.split(':')[1])
alertname = alerts_info['labels']['alertname']
monitor_iterm = alertname.split('_')[1] + '_' + alertname.split('_')[2]
service = alerts_info['labels']['service']
# endsAt = alerts_info['endsAt'] #exec last time
startsAt = alerts_info['startsAt']
start_time = datetime.datetime.strptime(startsAt.split('.')[0], '%Y-%m-%dT%H:%M:%S')
description = alerts_info['annotations']['description']
current_value = description.split('=')[1]
summary = alerts_info['annotations']['summary']
last_time = 0
alert_rules = AlertManager.get_alert_rules_by_name(host_id, port, monitor_iterm)
alert_rules_id = alert_rules[1]
contact_groups = alert_rules[11]
status = AlertManager.send_alert(monitor_iterm, summary, description, contact_groups)
data_res = AlertManager.create_alert(alert_rules_id, host_id, port, service, monitor_iterm,
start_time, current_value, last_time, state, contact_groups, status)
except Exception as e:
raise Error(e)
return {'status': 'ok'}, 201
def init_target_yaml(cls, host_ip):
file_name = host_ip + ".inv"
inv_file = YML_LOCATION + file_name
fobj = None
try:
fobj = open(inv_file, 'w')
except Exception, e:
msg = "Can't open file {0}: {1}".format(inv_file, e)
raise Error(msg)
def execute(cls, yml_file, inv_file, alert_rules_id):
if os.path.exists(yml_file):
cmd = "ansible-playbook {0} -i {1}".format(yml_file, inv_file)
# async_cmd_task.delay(cmd)
# async_cmd_run(cmd)
push_alert.delay(cmd, alert_rules_id)
else:
msg = "Can't found {0}".format(yml_file)
raise Error(msg)
def disable_alert(cls, alert_name, silence_time_hour):
silence_add = "{0}/amtool --alertmanager.url={1} silence add alertname={2} --expires={3}".format(
ALERT_MANAGER_PATH, ALERT_MANAGER_URL, alert_name,
silence_time_hour)
status, output, err = execute_command(silence_add)
if status != 0 or output == None:
errmsg = "Execute silence add command error: %s" % err
raise Error(str(errmsg))
def put(self):
json_data = request.get_json(force=True)
host_id = json_data['id']
# need get from db
port = 9100
service = json_data['service']
temp_name = json_data['name']
rules = json_data['rules']
contact_groups = json_data['contact_groups']
notify_type = json_data['notify_type']
for rule in rules:
monitor_items = rule['monitor_items']
alert_rules = AlertManager.get_alert_rules_by_name(host_id, port, monitor_items)
alert_rules_id = alert_rules[1]
#default
statistical_period = '5s'
# default
statistical_approach = 'max'
compute_mode = rule['compute_mode']
threshold_value = rule['threshold_value']
silence_tage = rule['silence']
try:
if silence_tage == 0:
# running
# update mysql
data_res = AlertManager.update_alert_rules(alert_rules_id, statistical_period, compute_mode,
threshold_value, ','.join(contact_groups), ','.join(notify_type), 0)
Ansible.check(MONITOR_SERVER)
inv_file = Ansible.init_target_yaml(MONITOR_SERVER)
instance = host_id + ":" + str(port)
yml_file = Ansible.init_metrics_yaml(service, monitor_items, host_id, instance, threshold_value,
statistical_period, compute_mode)
Ansible.execute(yml_file, inv_file, alert_rules_id)
elif silence_tage == 1:
silence_time = 2400
silence_time_hour = str(silence_time) + 'h'
data_res = AlertManager.get_alert_rules(alert_rules_id)
alert_name = data_res['service'] + '_' + data_res['monitor_items'] + '_' + data_res[
'host_id'] + ':' + str(data_res['port'])
AlertManager.disable_alert(alert_name, silence_time_hour)
AlertRulesData.update_silence_time(alert_rules_id, int(silence_time))
elif silence_tage == 2:
data_res = AlertManager.get_alert_rules(alert_rules_id)
alert_name = data_res['service'] + '_' + data_res['monitor_items'] + '_' + data_res[
'host_id'] + ':' + str(data_res['port'])
AlertManager.enable_alert(alert_name)
AlertRulesData.update_silence_time(alert_rules_id, 0)
except Exception as e:
raise Error(e)
return {'status': 'ok'}, 201
def init_metrics_yaml(cls, service, metrics, host_ip, instance,
threshold_value, statistical_period, compute_mode):
file_name = host_ip + ".yml"
yml_file = YML_LOCATION + file_name
fobj = None
try:
fobj = open(yml_file, 'w')
except Exception, e:
msg = "Can't open file {0}: {1}".format(yml_file, e)
raise Error(msg)
def delete(self, alert_rules_id):
try:
data_res = AlertManager.get_alert_rules(alert_rules_id)
file_name = data_res['host_id'] + '_' + str(data_res['port']) + '_' + data_res['service'] + '_' + data_res[
'monitor_items'] + '.yml'
AlertManager.delete_alert_rules(file_name)
AlertRulesData.delete_alert_rule(alert_rules_id)
except Exception as e:
raise Error(str(e))
return {'status': 'ok'}, 201
def enable_alert(cls, alert_name):
silence_query = "{0}/amtool --alertmanager.url={1} silence query alertname={2}".format(
ALERT_MANAGER_PATH, ALERT_MANAGER_URL, alert_name)
status, output, err = execute_command(silence_query)
if status != 0 or output == None:
errmsg = "Execute silence query command error: %s" % err
raise Error(str(errmsg))
for line in output.split("\n"):
if line == None or line == "":
continue
if not line.find('Matchers') == -1 or not line.find(
'Comment') == -1:
continue
else:
id = line.split(' ')[0]
silence_expire = "{0}/amtool --alertmanager.url={1} silence expire {2}".format(
ALERT_MANAGER_PATH, ALERT_MANAGER_URL, id)
status, output, err = execute_command(silence_expire)
if status != 0 or output == None:
errmsg = "Execute silence expire command error: %s" % err
raise Error(str(errmsg))
def put(self, alert_rules_id):
args = AlertRules.alert_rules_parser.parse_args()
statistical_period = args['statistical_period']
statistical_approach = args['statistical_approach']
compute_mode = args['compute_mode']
threshold_value = args['threshold_value']
try:
data_res = AlertManager.update_alert_rules(alert_rules_id,
statistical_period,
statistical_approach,
compute_mode,
threshold_value)
except Exception as e:
raise Error(str(e))
return {'status': 'ok'}, 201
def post(self):
json_data = request.get_json(force=True)
host_id = json_data['id']
# need get from db
port = 9100
service = json_data['service']
temp_name = json_data['name']
rules = json_data['rules']
contact_groups = json_data['contact_groups']
notify_type = json_data['notify_type']
for rule in rules:
monitor_items = rule['monitor_items']
#default
statistical_period = '5s'
# default
statistical_approach = 'max'
compute_mode = rule['compute_mode']
threshold_value = int(rule['threshold_value'])
# silence_time = args['silence_time']
try:
# running
alert_rules_id = str(uuid.uuid1())
# insert mysql
data_res = AlertManager.create_alert_rules(alert_rules_id, host_id, port, service, temp_name, monitor_items,
statistical_period, statistical_approach, compute_mode,
threshold_value, ','.join(contact_groups), ','.join(notify_type), 0)
Ansible.check(MONITOR_SERVER)
inv_file = Ansible.init_target_yaml(MONITOR_SERVER)
instance = host_id + ":" + str(port)
yml_file = Ansible.init_metrics_yaml(service, monitor_items, host_id, instance, threshold_value,
statistical_period, compute_mode)
Ansible.execute(yml_file, inv_file, alert_rules_id)
except Exception as e:
raise Error(e)
return {'status': 'ok'}, 201
