def check_alarm():
alarm_time = now()
checklog.logger.info('初始化告警信息表')
mysql_exec('insert into alarm_info_his select * from alarm_info')
mysql_exec('delete from alarm_info')
check_list = mysql_query(
"select name,judge_value,judge_sql,judge_table,conf_table,conf_column from alarm_conf where judge_sql is not null and judge_table is not null"
)
for each_check in check_list:
alarm_name, judge_value, judge_sql, judge_table, conf_table, conf_column = each_check
checklog.logger.info("开始告警检查：{}".format(alarm_name))
select_sql = "select count(*) from {}".format(judge_table)
select_res = mysql_query(select_sql)
if select_res[0][0] == 0:
checklog.logger.info("%s未采集到数据" % alarm_name)
else:
is_judge_sql = ' tags in (select tags from {} where {} =1)'.format(
conf_table, conf_column)
judge_sql = judge_sql % (
judge_value,
is_judge_sql) if judge_value else judge_sql % is_judge_sql
check_res = mysql_query(judge_sql)
if check_res == 0:
checklog.logger.info("{}:告警检查正常" % alarm_name)
else:
for each in check_res:
tags, url, alarm_content = each
alarm_title = tags + ':' + alarm_name
insert_sql = "insert into alarm_info (tags,url,alarm_type,alarm_header,alarm_content,alarm_time) values('{}','{}','{}','{}','{}','{}') ".format(
tags, url, alarm_name, alarm_title, alarm_content,
alarm_time)
checklog.logger.warning(alarm_content)
mysql_exec(insert_sql)
def get_oracle_alert(tags, db_conn, oracle_params, linux_params):
db_version = oracle_params['db_version']
host = oracle_params['host']
# 取alert日志
sql = "select alert_log,alert_log_seek from oracle_list where tags='{}' ".format(
tags)
alert_log, alert_log_seek = mysql_query(sql)[0]
if not alert_log:
sql = "select value from v$diag_info where name = 'Diag Trace'"
alert_dir = query_one(db_conn, sql)
# 取实例名
sql = "select instance_name from v$instance"
instance_name = query_one(db_conn, sql)
alert_log = '{}/alert_{}.log'.format(alert_dir[0], instance_name[0])
alert_log_seek = 0
sql = "delete from alert_log where tags='{}' and type=1 ".format(tags)
mysql_exec(sql)
# ssh获取日志内容
linux_oper = LinuxBase(linux_params)
# 日志解析
alert_content = linux_oper.readfile(alert_log, seek=alert_log_seek)
alert_log_seek = parse_oracle_alert_logs(tags, host, alert_content,
db_version)
# 更新配置表中日志路径,日志偏移量
sql = "update oracle_list set alert_log='{}',alert_log_seek={} where tags='{}' ".format(
alert_log, alert_log_seek, tags)
mysql_exec(sql)
def get_mysql_slowquery(tags,mysql_params,linux_params):
host = mysql_params['host']
# slow query log location
sql = "select slowquery_log,slowquery_log_seek from mysql_list where tags='{}' ".format(tags)
slowquery_log,slowquery_log_seek = mysql_query(sql)[0]
if not slowquery_log:
slowquery_log = Mysql_Do(mysql_params).get_para('slow_query_log_file')
slowquery_log_seek = 0
# get slowquery log content
linux_oper = LinuxBase(linux_params)
slowquery_content = linux_oper.readfile(slowquery_log,seek=slowquery_log_seek)
# parse log
slowquery_log_seek = parse_mysql_slowquery_logs(tags,host,slowquery_content)
# update alert log info to mysqlinfo
sql = "update mysql_list set slowquery_log='{}',slowquery_log_seek={} where tags='{}' " .format(slowquery_log,slowquery_log_seek,tags)
mysql_exec(sql)
def get_mysql_alert(tags,mysql_params,linux_params):
host = mysql_params['host']
# logfile loglocation
sql = "select alert_log,alert_log_seek from mysql_list where tags='{}' ".format(tags)
alert_log,alert_log_seek = mysql_query(sql)[0]
if not alert_log:
alert_log = Mysql_Do(mysql_params).get_para('log_error')
alert_log_seek = 0
sql = "delete from alert_log where tags='{}' and type=2 ".format(tags)
mysql_exec(sql)
# get alert log content
linux_oper = LinuxBase(linux_params)
alert_content = linux_oper.readfile(alert_log,seek=alert_log_seek)
# parse log
alert_log_seek = parse_mysql_alert_logs(tags,host,alert_content)
# update alert log info to mysqlinfo
sql = "update mysql_list set alert_log='{}',alert_log_seek={} where tags='{}' " .format(alert_log,alert_log_seek,tags)
mysql_exec(sql)
def get_redis_log(tags,redis_params,linux_params):
host = redis_params['host']
# logfile loglocation
sql = "select log,log_seek from redis_list where tags='{}' ".format(tags)
log,log_seek = mysql_query(sql)[0]
if not log:
redis_conn = RedisBase(redis_params).connection()
log_dir = redis_conn.config_get('dir')['dir']
log = redis_conn.config_get('logfile')['logfile']
log = os.path.join(log_dir,log)
log_seek = 0
sql = "delete from alert_log where tags='{}' and type=3 ".format(tags)
mysql_exec(sql)
# get log content
linux_oper = LinuxBase(linux_params)
alert_content = linux_oper.readfile(log,seek=log_seek)
# parse log
log_seek = parse_redis_logs(tags,host,alert_content)
# update alert log info to mysqlinfo
sql = "update redis_list set log='{}',log_seek={} where tags='{}' " .format(log,log_seek,tags)
mysql_exec(sql)
# encoding:utf-8
from check.check_linux import check_linux
from check.alarm import check_alarm
from utils.tools import clear_table,archive_table,mysql_query
from multiprocessing import Process
if __name__ == '__main__':
# all minitoring servers
linux_list = mysql_query('select adname from linux_list')
# windows_list = mysql_query('select tags,host,sshport,user,password from linux_list where system ="windows"')
# check_linux
check_pool = []
if linux_list:
for each in linux_list:
adname = each[0]
# linux_params = {
# 'hostname': each[1],
# 'port': each[2],
# 'username': each[3],
# 'password': each[4]
# }
linux_check = Process(target=check_linux, args=(adname))
linux_check.start()
check_pool.append(linux_check)
for each in check_pool:
each.join()
def checkall():
# all minitoring servers
linux_list = mysql_query('select tags,host,sshport,user,password from linux_list')
oracle_list = mysql_query(
'select t1.tags,t1.host,t1.port,t1.service_name,t1.db_user,t1.db_password,t1.db_user_cdb,t1.db_password_cdb,t1.service_name_cdb,'
't2.user,t2.password,t2.sshport,t1.db_version from oracle_list t1 left join linux_list t2 on t1.linux_tags=t2.tags ')
mysql_list = mysql_query(
'select t1.tags,t1.host,t1.port,t1.db_user,t1.db_password,t2.user,t2.password,t2.sshport,t1.db_version'
' from mysql_list t1 left join linux_list t2 on t1.linux_tags=t2.tags')
redis_list = mysql_query(
'select t1.tags,t1.host,t1.port,t1.redis_version,t1.password,t2.user,t2.password,t2.sshport'
' from redis_list t1 left join linux_list t2 on t1.linux_tags=t2.tags')
# check_linux
check_pool = []
if linux_list:
for each in linux_list:
tags = each[0]
linux_params = {
'hostname': each[1],
'port': each[2],
'username': each[3],
'password': each[4]
}
linux_check = Process(target=check_linux,args=(tags,linux_params))
linux_check.start()
check_pool.append(linux_check)
if oracle_list:
for each in oracle_list:
tags = each[0]
oracle_params = {
'host': each[1],
'port': each[2],
'service_name': each[3],
'user': each[4],
'password': each[5],
'user_cdb': each[6],
'password_cdb': each[7],
'service_name_cdb': each[8],
'user_os': each[9],
'password_os': each[10],
'sshport_os': each[11],
'db_version': each[12]
}
oracle_check = Process(target=check_oracle, args=(tags, oracle_params))
oracle_check.start()
check_pool.append(oracle_check)
if mysql_list:
for each in mysql_list:
tags = each[0]
mysql_params = {
'host': each[1],
'port': each[2],
'user': each[3],
'password': each[4],
'user_os': each[5],
'password_os': each[6],
'sshport_os':each[7]
}
mysql_check = Process(target=check_mysql, args=(tags, mysql_params))
mysql_check.start()
check_pool.append(mysql_check)
if redis_list:
for each in redis_list:
tags = each[0]
redis_params = {
'host': each[1],
'port': each[2],
'version': each[3],
'password': each[4],
'user_os': each[5],
'password_os': each[6],
'sshport_os':each[7] }
redis_check = Process(target=check_redis, args=(tags, redis_params))
redis_check.start()
check_pool.append(redis_check)
for each in check_pool:
each.join()
# 告警
check_alarm()
# encoding:utf-8
from check.check_oracle import check_oracle
from check.check_linux import check_linux
from check.check_mysql import check_mysql
from check.check_redis import check_redis
from check.alarm import check_alarm
from utils.tools import clear_table,archive_table,mysql_query
from multiprocessing import Process
if __name__ == '__main__':
# all minitoring servers
linux_list = mysql_query('select tags,host,sshport,user,password from linux_list')
oracle_list = mysql_query(
'select t1.tags,t1.host,t1.port,t1.service_name,t1.db_user,t1.db_password,t1.db_user_cdb,t1.db_password_cdb,t1.service_name_cdb,'
't2.user,t2.password,t2.sshport,t1.db_version from oracle_list t1 left join linux_list t2 on t1.linux_tags=t2.tags ')
mysql_list = mysql_query(
'select t1.tags,t1.host,t1.port,t1.db_user,t1.db_password,t2.user,t2.password,t2.sshport,t1.db_version'
' from mysql_list t1 left join linux_list t2 on t1.linux_tags=t2.tags')
redis_list = mysql_query(
'select t1.tags,t1.host,t1.port,t1.redis_version,t1.password,t2.user,t2.password,t2.sshport'
' from redis_list t1 left join linux_list t2 on t1.linux_tags=t2.tags')
# check_linux
check_pool = []
if linux_list:
for each in linux_list: