def send_alarm_mail(): sql = "select alarm.application_id,app.display_name application,alarm.server_id,servers.host,servers.port,alarm.create_time,db_type,alarm_type,alarm_value,level,message,alarm.send_mail from alarm left join servers on alarm.server_id=servers.id left join application app on servers.application_id=app.id;" result = func.mysql_query(sql) if result <> 0: send_alarm_mail = func.get_option('send_alarm_mail') mail_to_list = func.get_option('mail_to_list') mailto_list = mail_to_list.split(';') for line in result: application_id = line[0] application = line[1] server_id = [2] host = line[3] port = line[4] create_time = line[5] db_type = line[6] alarm_type = line[7] alarm_value = line[8] level = line[9] message = line[10] send_mail = line[11] if send_alarm_mail == "1": if send_mail == 1: mail_subject = message + ' 当前值:' + alarm_value + ' 服务器:' + application + '-' + host + ':' + port + ' 时间:' + create_time.strftime( '%Y-%m-%d %H:%M:%S') mail_content = "please check!" result = func.send_mail(mailto_list, mail_subject, mail_content) if result: send_mail_status = 1 else: send_mail_status = 0 else: send_mail_status = 0 else: send_mail_status = 0 if send_mail_status == 1: func.mysql_exec( "insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1 from alarm;", '') elif send_mail_status == 0: func.mysql_exec( "insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0 from alarm;", '') func.mysql_exec("delete from alarm", '') else: pass
def clear_history_data(): history_save_days =int(func.get_option('history_save_days')) if history_save_days >= 1: old_date=str(func_time.get_day_of_day(history_save_days*-1)) sql="delete from redis_coldback_info where date <'"+old_date+"'" log.debug("sql:%s"%(sql)) func.mysql_exec(sql,'') sql="delete from alarm_history where left(create_time,10)<'"+old_date+"'" log.debug("sql:%s"%(sql)) func.mysql_exec(sql,'') sql="delete from redis_keyspace_history where left(create_time,10)<'"+old_date+"'" log.debug("sql:%s"%(sql)) func.mysql_exec(sql,'') sql="delete from redis_persistence_status_history where left(create_time,10)<'"+old_date+"'" log.debug("sql:%s"%(sql)) func.mysql_exec(sql,'') sql="delete from redis_replication_history where left(create_time,10)<'"+old_date+"'" log.debug("sql:%s"%(sql)) func.mysql_exec(sql,'') sql="delete from redis_resource_status_history where left(create_time,10)<'"+old_date+"'" log.debug("sql:%s"%(sql)) func.mysql_exec(sql,'') sql="delete from redis_run_status_history where left(create_time,10)<'"+old_date+"'" log.debug("sql:%s"%(sql)) func.mysql_exec(sql,'') sql="delete from redis_server_status_history where left(create_time,10)<'"+old_date+"'" log.debug("sql:%s"%(sql)) func.mysql_exec(sql,'') tCmd="rm -f ./log/*"+old_date log.debug("tCmd:%s"%(tCmd)) os.system(tCmd) else: log.error("get history_save_days from options error!")
def check_mysql_widget(host,port,user,passwd,server_id): try: connect=MySQLdb.connect(host=host,user=user,passwd=passwd,port=int(port),connect_timeout=2,charset='utf8') cur=connect.cursor() connect.select_db('information_schema') try: bigtable=cur.execute("SELECT table_schema as 'DB',table_name as 'TABLE',CONCAT(ROUND(( data_length + index_length ) / ( 1024 * 1024 * 1024 ), 2), '') 'TOTAL' , table_comment as COMMENT FROM information_schema.TABLES ORDER BY data_length + index_length DESC ;"); big_table_size = func.get_option('big_table_size') if bigtable: for row in cur.fetchall(): datalist=[] for r in row: datalist.append(r) result=datalist if result: table_size = float(string.atof(result[2])) if table_size >= int(big_table_size): sql="insert into mysql_widget_bigtable(server_id,db_name,table_name,table_size,table_comment) values(%s,%s,%s,%s,%s);" param=(server_id,result[0],result[1],result[2],result[3]) func.mysql_exec(sql,param) except : pass cur.close() connect.close() exit except MySQLdb.Error,e: pass print "Mysql Error %d: %s" %(e.args[0],e.args[1])
def clear_history_data(): history_save_days = int(func.get_option('history_save_days')) if history_save_days >= 1: old_date = str(func_time.get_day_of_day(history_save_days * -1)) sql = "delete from redis_coldback_info where date <'" + old_date + "'" log.debug("sql:%s" % (sql)) func.mysql_exec(sql, '') sql = "delete from alarm_history where left(create_time,10)<'" + old_date + "'" log.debug("sql:%s" % (sql)) func.mysql_exec(sql, '') sql = "delete from redis_keyspace_history where left(create_time,10)<'" + old_date + "'" log.debug("sql:%s" % (sql)) func.mysql_exec(sql, '') sql = "delete from redis_persistence_status_history where left(create_time,10)<'" + old_date + "'" log.debug("sql:%s" % (sql)) func.mysql_exec(sql, '') sql = "delete from redis_replication_history where left(create_time,10)<'" + old_date + "'" log.debug("sql:%s" % (sql)) func.mysql_exec(sql, '') sql = "delete from redis_resource_status_history where left(create_time,10)<'" + old_date + "'" log.debug("sql:%s" % (sql)) func.mysql_exec(sql, '') sql = "delete from redis_run_status_history where left(create_time,10)<'" + old_date + "'" log.debug("sql:%s" % (sql)) func.mysql_exec(sql, '') sql = "delete from redis_server_status_history where left(create_time,10)<'" + old_date + "'" log.debug("sql:%s" % (sql)) func.mysql_exec(sql, '') tCmd = "rm -f ./log/*" + old_date log.debug("tCmd:%s" % (tCmd)) os.system(tCmd) else: log.error("get history_save_days from options error!")
def check_mysql_widget(host,port,user,passwd,server_id): try: connect=MySQLdb.connect(host=host,user=user,passwd=passwd,port=int(port),connect_timeout=2,charset='utf8') cur=connect.cursor() connect.select_db('information_schema') try: bigtable=cur.execute("SELECT TABLE_SCHEMA as 'DB',TABLE_NAME as 'TABLE',CONCAT(ROUND(( DATA_LENGTH + INDEX_LENGTH ) / ( 1024 * 1024 * 1024 ), 2), '') 'TOTAL' , TABLE_COMMENT as COMMENT FROM information_schema.TABLES ORDER BY DATA_LENGTH + INDEX_LENGTH DESC ;"); big_table_size = func.get_option('big_table_size') if bigtable: for row in cur.fetchall(): datalist=[] for r in row: datalist.append(r) result=datalist if result: table_size = float(string.atof(result[2])) if table_size >= int(big_table_size): sql="insert into mysql_widget_bigtable(server_id,db_name,table_name,table_size,table_comment) values(%s,%s,%s,%s,%s);" param=(server_id,result[0],result[1],result[2],result[3]) func.mysql_exec(sql,param) except : pass cur.close() connect.close() exit except MySQLdb.Error,e: pass print "Mysql Error %d: %s" %(e.args[0],e.args[1])
def start_message_or_mail(): choice_message_or_mail = int(func.get_option('choice_message_or_mail')) try: if choice_message_or_mail == 1: #message send_alarm_message() elif choice_message_or_mail == 0: send_alarm_mail() except Exception as e: print(e)
def send_alarm_mail(): sql="select alarm.application_id,app.display_name application,alarm.server_id,servers.host,servers.port,alarm.create_time,db_type,alarm_type,alarm_value,level,message,alarm.send_mail from alarm left join servers on alarm.server_id=servers.id left join application app on servers.application_id=app.id;" result=func.mysql_query(sql) if result <> 0: send_alarm_mail = func.get_option('send_alarm_mail') mail_to_list = func.get_option('mail_to_list') mailto_list=mail_to_list.split(';') for line in result: application_id=line[0] application=line[1] server_id=[2] host=line[3] port=line[4] create_time=line[5] db_type=line[6] alarm_type=line[7] alarm_value=line[8] level=line[9] message=line[10] send_mail=line[11] if send_alarm_mail=="1": if send_mail==1: mail_subject=message+' 当前值:'+alarm_value+' 服务器:'+application+'-'+host+':'+port+' 时间:'+create_time.strftime('%Y-%m-%d %H:%M:%S') mail_content="please check!" result = func.send_mail(mailto_list,mail_subject,mail_content) if result: send_mail_status=1 else: send_mail_status=0 else: send_mail_status=0 else: send_mail_status=0 if send_mail_status==1: func.mysql_exec("insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1 from alarm;",'') elif send_mail_status==0: func.mysql_exec("insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0 from alarm;",'') func.mysql_exec("delete from alarm",'') else: pass
def main(): print("%s: controller started." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()), )) monitor = func.get_option('monitor') monitor_status = func.get_option('monitor_status') monitor_replication = func.get_option('monitor_replication') monitor_process = func.get_option('monitor_process') alarm = func.get_option('alarm') frequency_monitor = func.get_option('frequency_monitor') frequency_alarm = func.get_option('frequency_alarm') kill_process = func.get_option('kill_process') joblist = [] if monitor == "1": if monitor_status == "1": job = Process(target=job_run, args=('check_mysql_status_ext.py', frequency_monitor)) joblist.append(job) job.start() job = Process(target=job_run, args=('check_mysql_status.py', frequency_monitor)) joblist.append(job) job.start() if monitor_replication == "1": job = Process(target=job_run, args=('check_mysql_replication.py', frequency_monitor)) joblist.append(job) job.start() if monitor_process == "1": job = Process(target=job_run, args=('check_mysql_process.py', 4)) joblist.append(job) job.start() if alarm == "1": job = Process(target=job_run, args=('alarm_mysql.py', frequency_alarm)) joblist.append(job) job.start() if kill_process == "1": job = Process(target=job_run, args=('admin_mysql_kill_process.py', 3)) joblist.append(job) job.start() for job in joblist: job.join() print("%s: controller finished." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()), ))
def get_alarm_last_status(): sql = "delete from alarm_last_status where server_id not in (select distinct server_id from alarm);" func.mysql_exec(sql, '') sql = "update alarm_last_status set connect='',status_connections='',status_used_memory='',status_replication_relay='';" func.mysql_exec(sql, '') #insert new warn sql = "insert into alarm_last_status(application_id,server_id) select distinct application_id,server_id from alarm where server_id not in (select distinct server_id from alarm_last_status);" func.mysql_exec(sql, '') #old warn, update sql = "select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id in (select distinct server_id from alarm_last_status);" result = func.mysql_query(sql) if result <> 0: for line in result: update_sql = "update alarm_last_status set " application_id = line[0] server_id = line[1] alarm_type = line[2] alarm_value = line[3] level = line[4] message = line[5] send_mail = line[6] if alarm_type == "connect": update_sql = update_sql + "connect=\"" + message + "\"," elif alarm_type == "connections": update_sql = update_sql + "connections='" + alarm_value + "',status_connections='" + message + "'," elif alarm_type == "memory": update_sql = update_sql + "used_memory='" + alarm_value + "',status_used_memory='" + message + "'," elif alarm_type == "delay": update_sql = update_sql + "replication_relay='" + alarm_value + "',status_replication_relay='" + message + "'," elif alarm_type == "replication": update_sql = update_sql + "replication_relay='-1',status_replication_relay='" + message + "'," update_sql = update_sql + "send_alarm='" + str( send_mail) + "' where application_id='" + str( application_id) + "' and server_id='" + str( server_id) + "';" log.debug("update_sql:%s" % (update_sql)) func.mysql_exec(update_sql, '') #should not appear, send alarm to me sql = "select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id not in (select distinct server_id from alarm_last_status);" result = func.mysql_query(sql) if result <> 0: mailto_list = func.get_option('mail_to_list') mail_subject = "redis monitor error" mail_content = "this should not appear:" + sql func.send_alarm(0, mailto_list, mail_subject, mail_content.encode('utf-8'))
def main(): print("%s: controller started." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()), )) ##全局配置--全局监控 monitor = func.get_option('monitor') ##全局配置--状态监控 monitor_status = func.get_option('monitor_status') ##全局配置--状态监控频率 frequency_monitor = func.get_option('frequency_monitor') ##全局配置--开启告警 alarm = func.get_option('alarm') ###全局配置--报警检测频率 frequency_alarm = func.get_option('frequency_alarm') ##全局配置--冷备管理 data_back_status = func.get_option('data_back_status') ##全局配置--冷备管理频率 frequency_data_back = func.get_option('frequency_data_back') ##全局配置--历史数据自动清理 history_clear_status = func.get_option('history_clear_status') joblist = [] if monitor == "1": if monitor_status == "1": job = Process(target=job_run, args=('check_redis_status.py', frequency_monitor)) joblist.append(job) job.start() if alarm == "1": job = Process(target=job_run, args=('alarm_redis.py', frequency_alarm)) joblist.append(job) job.start() if data_back_status == "1": job = Process(target=job_run, args=('redis_data_back.py', frequency_data_back)) joblist.append(job) job.start() if history_clear_status == "1": job = Process(target=job_run, args=('redis_history_clear.py', 36000)) joblist.append(job) job.start() for job in joblist: job.join() print("%s: controller finished." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()), ))
def get_alarm_last_status(): sql="delete from alarm_last_status where server_id not in (select distinct server_id from alarm);" func.mysql_exec(sql,'') sql="update alarm_last_status set connect='',status_connections='',status_used_memory='',status_replication_relay='';" func.mysql_exec(sql,'') #insert new warn sql="insert into alarm_last_status(application_id,server_id) select distinct application_id,server_id from alarm where server_id not in (select distinct server_id from alarm_last_status);" func.mysql_exec(sql,'') #old warn, update sql="select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id in (select distinct server_id from alarm_last_status);" result=func.mysql_query(sql) if result <> 0: for line in result: update_sql="update alarm_last_status set " application_id=line[0] server_id=line[1] alarm_type=line[2] alarm_value=line[3] level=line[4] message=line[5] send_mail=line[6] if alarm_type=="connect": update_sql=update_sql + "connect=\"" + message + "\"," elif alarm_type=="connections": update_sql=update_sql + "connections='" + alarm_value + "',status_connections='" + message + "'," elif alarm_type=="memory": update_sql=update_sql + "used_memory='" + alarm_value + "',status_used_memory='" + message + "'," elif alarm_type=="delay": update_sql=update_sql + "replication_relay='" + alarm_value + "',status_replication_relay='" + message + "'," elif alarm_type=="replication": update_sql=update_sql + "replication_relay='-1',status_replication_relay='" + message + "'," update_sql=update_sql + "send_alarm='" + str(send_mail) + "' where application_id='" + str(application_id) + "' and server_id='" + str(server_id) + "';" log.debug("update_sql:%s"%(update_sql)) func.mysql_exec(update_sql,'') #should not appear, send alarm to me sql="select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id not in (select distinct server_id from alarm_last_status);" result=func.mysql_query(sql) if result <> 0: mailto_list = func.get_option('mail_to_list') mail_subject="redis monitor error" mail_content="this should not appear:"+sql func.send_alarm(0,mailto_list,mail_subject,mail_content.encode('utf-8'))
def main(): print("%s: controller started." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()),)) ##全局配置--全局监控 monitor = func.get_option('monitor') ##全局配置--状态监控 monitor_status = func.get_option('monitor_status') ##全局配置--状态监控频率 frequency_monitor = func.get_option('frequency_monitor') ##全局配置--开启告警 alarm = func.get_option('alarm') ###全局配置--报警检测频率 frequency_alarm = func.get_option('frequency_alarm') ##全局配置--冷备管理 data_back_status = func.get_option('data_back_status') ##全局配置--冷备管理频率 frequency_data_back = func.get_option('frequency_data_back') ##全局配置--历史数据自动清理 history_clear_status = func.get_option('history_clear_status') joblist = [] if monitor=="1": if monitor_status=="1": job = Process(target = job_run, args = ('check_redis_status.py',frequency_monitor)) joblist.append(job) job.start() if alarm=="1": job = Process(target = job_run, args = ('alarm_redis.py',frequency_alarm)) joblist.append(job) job.start() if data_back_status=="1": job = Process(target = job_run, args = ('redis_data_back.py',frequency_data_back)) joblist.append(job) job.start() if history_clear_status=="1": job = Process(target = job_run, args = ('redis_history_clear.py',36000)) joblist.append(job) job.start() for job in joblist: job.join(); print("%s: controller finished." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()),))
def main(): print("%s: controller started." % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),)) monitor = func.get_option("monitor") monitor_status = func.get_option("monitor_status") monitor_replication = func.get_option("monitor_replication") monitor_process = func.get_option("monitor_process") alarm = func.get_option("alarm") frequency_monitor = func.get_option("frequency_monitor") frequency_alarm = func.get_option("frequency_alarm") kill_process = func.get_option("kill_process") joblist = [] if monitor == "1": if monitor_status == "1": job = Process(target=job_run, args=("check_mysql_status_ext.py", frequency_monitor)) joblist.append(job) job.start() job = Process(target=job_run, args=("check_mysql_status.py", frequency_monitor)) joblist.append(job) job.start() if monitor_replication == "1": job = Process(target=job_run, args=("check_mysql_replication.py", frequency_monitor)) joblist.append(job) job.start() if monitor_process == "1": job = Process(target=job_run, args=("check_mysql_process.py", 4)) joblist.append(job) job.start() if alarm == "1": job = Process(target=job_run, args=("alarm_mysql.py", frequency_alarm)) joblist.append(job) job.start() if kill_process == "1": job = Process(target=job_run, args=("admin_mysql_kill_process.py", 3)) joblist.append(job) job.start() for job in joblist: job.join() print("%s: controller finished." % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),))
def main(): print("%s: controller started." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()),)) monitor = func.get_option('monitor') monitor_status = func.get_option('monitor_status') monitor_replication = func.get_option('monitor_replication') monitor_process = func.get_option('monitor_process') alarm = func.get_option('alarm') frequency_monitor = func.get_option('frequency_monitor') frequency_alarm = func.get_option('frequency_alarm') kill_process = func.get_option('kill_process') joblist = [] if monitor=="1": if monitor_status=="1": job = Process(target = job_run, args = ('check_mysql_status_ext.py',frequency_monitor)) joblist.append(job) job.start() job = Process(target = job_run, args = ('check_mysql_status.py',frequency_monitor)) joblist.append(job) job.start() if monitor_replication=="1": job = Process(target = job_run, args = ('check_mysql_replication.py',frequency_monitor)) joblist.append(job) job.start() if monitor_process=="1": job = Process(target = job_run, args = ('check_mysql_process.py',4)) joblist.append(job) job.start() if alarm=="1": job = Process(target = job_run, args = ('alarm_mysql.py',frequency_alarm)) joblist.append(job) job.start() if kill_process=="1": job = Process(target = job_run, args = ('admin_mysql_kill_process.py',3)) joblist.append(job) job.start() for job in joblist: job.join(); print("%s: controller finished." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()),))
def main(): log_level = "info" log_config = global_logs.CLogConfig(level=log_level) log_name = "./log/check_redis_status.log" log_config._path = "%s.%s" % (log_name, func_time.today()) log = global_logs.initLog(log_config, logmodule="status") if log == None: print "init log error!" servers = func.mysql_query( "select id,host,port,passwd,application_id,status from servers where is_delete=0;" ) frequency_monitor = func.get_option('frequency_monitor') if servers: #print("servers %s" %(servers,)); log.info("check_mysql_status controller started...") init_table() plist = [] for row in servers: server_id = row[0] host = row[1] port = row[2] passwd = row[3] application_id = row[4] status = row[5] if status <> 0: p = Process(target=check_redis_status, args=(host, port, passwd, server_id, application_id, frequency_monitor, log)) plist.append(p) p.start() for p in plist: p.join() #set replication delay sql = "delete from redis_replication_last;" func.mysql_exec(sql, '') sql = "insert into redis_replication_last select * from redis_replication;" func.mysql_exec(sql, '') sql = "update redis_replication a,redis_replication_last b,servers c set a.delay=b.master_repl_offset-a.slave_repl_offset where a.master_host=c.host and a.master_port=c.port and b.server_id=c.id and a.is_slave=1 and b.is_master=1;" func.mysql_exec(sql, '') log.info("check_mysql_status controller finished...")
def main(): log_level="info" log_config=global_logs.CLogConfig(level=log_level) log_name="./log/check_redis_status.log" log_config._path="%s.%s"%(log_name,func_time.today()) log=global_logs.initLog(log_config,logmodule="status") if log == None: print "init log error!" servers=func.mysql_query("select id,host,port,passwd,application_id,status from servers where is_delete=0;") frequency_monitor = func.get_option('frequency_monitor') if servers: #print("servers %s" %(servers,)); log.info("check_mysql_status controller started..." ); init_table() plist = [] for row in servers: server_id=row[0] host=row[1] port=row[2] passwd=row[3] application_id=row[4] status=row[5] if status <> 0: p = Process(target = check_redis_status, args = (host,port,passwd,server_id,application_id,frequency_monitor,log)) plist.append(p) p.start() for p in plist: p.join() #set replication delay sql="delete from redis_replication_last;" func.mysql_exec(sql,'') sql="insert into redis_replication_last select * from redis_replication;" func.mysql_exec(sql,'') sql="update redis_replication a,redis_replication_last b,servers c set a.delay=b.master_repl_offset-a.slave_repl_offset where a.master_host=c.host and a.master_port=c.port and b.server_id=c.id and a.is_slave=1 and b.is_master=1;" func.mysql_exec(sql,'') log.info("check_mysql_status controller finished..." )
#!/usr/bin/env python # -*- coding: utf8 -*- from datetime import * import global_functions as func mail_to_list = func.get_option('mail_to_list') mailto_list = mail_to_list.split(';') result = func.send_mail(mailto_list, " I hope you can learn", "Beautiful Day") print result if result: send_mail_status = "success" else: send_mail_status = "fail" print "send_mail_status:" + send_mail_status
def send_alarm_mail(idc_type='ts'): sql = "select alarm.application_id,app.display_name application,alarm.server_id,servers.host,servers.port,alarm.create_time,db_type,alarm_type,alarm_value,level,message,alarm.send_mail,servers.alarm_time,servers.threshold_starttime,servers.threshold_endtime from alarm left join servers on alarm.server_id=servers.id left join application app on servers.application_id=app.id where servers.idc_type='" + str( idc_type) + "';" result = func.mysql_query(sql) if result <> 0: send_alarm_mail = func.get_option('send_alarm_mail') mail_to_list = func.get_option('mail_to_list') mailto_list = mail_to_list.split(';') global_alarm_type = func.get_config('global_alarm_type', 'alarm_type') global_alarm_type_list = global_alarm_type.split(',') for line in result: application_id = line[0] application = line[1] server_id = line[2] host = line[3] port = line[4] create_time = line[5] db_type = line[6] alarm_type = line[7] alarm_value = line[8] level = line[9] message = line[10] send_mail = line[11] alarm_time = line[12] threshold_starttime = line[13] threshold_endtime = line[14] app_email = func.get_application_alarm_email(str(application_id)) app_email_list = app_email.split(',') for email_app in app_email_list: if email_app not in mailto_list: mailto_list.append(email_app) else: pass if send_alarm_mail == "1": if send_mail == 1: mail_subject = message + ' 当前值:' + alarm_value + ' 服务器:' + application + '-' + host + ':' + port + ' 时间:' + create_time.strftime( '%Y-%m-%d %H:%M:%S') mail_content = "请检查下!" now_hour = ("0" + str(datetime.now().hour) if datetime.now().hour < 10 else str( datetime.now().hour)) + ":" + ( "0" + str(datetime.now().minute) if datetime.now().minute < 10 else str( datetime.now().minute)) if alarm_type not in global_alarm_type_list: if (int(alarm_time) == 1) and ( threshold_starttime.strip() != "") and (threshold_endtime.strip() != "") and ( threshold_endtime > threshold_starttime ) and (now_hour <= threshold_endtime) and ( now_hour >= threshold_starttime): result_mail = func.send_mail( mailto_list, mail_subject, mail_content) elif int(alarm_time) != 1: result_mail = func.send_mail( mailto_list, mail_subject, mail_content) else: result_mail = False else: result_mail = func.send_mail(mailto_list, mail_subject, mail_content) if result_mail: send_mail_status = 1 else: send_mail_status = 0 else: send_mail_status = 0 else: send_mail_status = 0 if send_mail_status == 1: func.mysql_exec( "insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status,idc_type) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1,'" + str(idc_type) + "' from alarm;", '') elif send_mail_status == 0: func.mysql_exec( "insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status,idc_type) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0,'" + str(idc_type) + "' from alarm;", '') func.mysql_exec( "delete from alarm where idc_type='" + str(idc_type) + "'", '') else: pass
def send_warn(): #server_id not delete??? sql="select alarm_last_status.application_id,app.display_name application,alarm_last_status.server_id,servers.host,servers.port,alarm_last_status.modify_time,servers.alarm_interval,servers.alarm_type,servers.converge_type,servers.alarm_person,servers.send_mail,alarm_last_status.connect,alarm_last_status.status_connections,alarm_last_status.status_used_memory,alarm_last_status.status_binlog_count,alarm_last_status.status_innodb_space_free,alarm_last_status.status_replication_relay,alarm_last_status.alarm_num,alarm_last_status.time_error_continue,alarm_last_status.slow_querys,alarm_last_status.status_slow_querys,alarm_last_status.error_log,alarm_last_status.status_error_log from alarm_last_status left join servers on alarm_last_status.server_id=servers.id left join application app on servers.application_id=app.id;" result=func.mysql_query(sql) if result <> 0: domain="www.oa.com" if(pingServerCall(domain)==False): #系统告警接收人 mailto_list = func.get_option('mail_to_list') mail_subject="redis monitor error" mail_content="ping failed:"+domain +",do not send warn..." func.send_alarm(0,mailto_list,mail_subject,mail_content.encode('utf-8')) return None #全局配置--发送告警 send_alarm_mail = func.get_option('send_alarm_mail') #全局配置--报警检测 frequency_alarm = func.get_option('frequency_alarm') #mail_to_list = func.get_option('mail_to_list') #mailto_list=mail_to_list.split(';') for line in result: application_id=line[0] application=line[1] server_id=line[2] host=line[3] port=line[4] create_time=line[5] alarm_interval=line[6] alarm_type=line[7] converge_type=line[8] alarm_person=line[9] send_alarm=line[10] connect=line[11] status_connections=line[12] status_used_memory=line[13] status_binlog_count=line[14] status_innodb_space_free=line[15] status_replication_relay=line[16] alarm_num=line[17] time_error_continue=line[18] slow_querys=line[19] status_slow_querys=line[20] error_log=line[21] status_error_log=line[22] warn_content=application+'-'+host+':'+port eStr="ERROR:" wStr="WARN:" if (eStr in connect) or (wStr in connect): warn_content=warn_content+' '+connect else: if (eStr in status_connections) or (wStr in status_connections): warn_content=warn_content+' '+status_connections if (eStr in status_used_memory) or (wStr in status_used_memory): warn_content=warn_content+' '+status_used_memory if (eStr in status_binlog_count) or (wStr in status_binlog_count): warn_content=warn_content+' '+status_binlog_count if (eStr in status_innodb_space_free) or (wStr in status_innodb_space_free): warn_content=warn_content+' '+status_innodb_space_free if (eStr in status_replication_relay) or (wStr in status_replication_relay): warn_content=warn_content+' '+status_replication_relay if (eStr in status_slow_querys) or (wStr in status_slow_querys): warn_content=warn_content+' '+status_slow_querys if (eStr in status_error_log) or (wStr in status_error_log): warn_content=warn_content+' '+status_error_log warn_content=warn_content+' ['+create_time.strftime('%Y-%m-%d %H:%M:%S')+']' #warn_content=warn_content.encode('utf-8') #print send_alarm_mail #print send_alarm #print warn_content #print("WARN:%s send alarm:%s alarm:%s" %(warn_content,send_alarm_mail,send_alarm)) mailto_list=alarm_person mail_subject="REDIS WARNING" send_mail_status=0 if send_alarm_mail=='1': if send_alarm==1: next_alarm_time=getTimeInterVal(converge_type,alarm_interval,alarm_num,0); if(func.test_flag): log.debug("time_error_continue:%s next_alarm_time:%s converge_type:%s alarm_interval:%s alarm_num:%s"%(str(time_error_continue),str(next_alarm_time),converge_type,str(alarm_interval),str(alarm_num))) if time_error_continue >= next_alarm_time: result=func.send_alarm(alarm_type,mailto_list,mail_subject,warn_content.encode('utf-8')) if result: send_mail_status=1 else: send_mail_status=0 sql="update alarm_last_status set alarm_num=alarm_num+1 where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"'" func.mysql_exec(sql,'') log.warning("send alarm:alarm_type:%s mailto_list:%s mail_subject:%s warn_content:%s"%(str(alarm_type),mailto_list,mail_subject,warn_content.encode('utf-8'))) else: sql="delete from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"'" func.mysql_exec(sql,'') else: send_mail_status=0 else: send_mail_status=0 sql="update alarm_last_status set time_error_continue=time_error_continue+"+str(frequency_alarm)+" where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"'" func.mysql_exec(sql,'') #param=(str(application_id),str(server_id)) if send_mail_status==1: sql="insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1 from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"';" func.mysql_exec(sql,'') elif send_mail_status==0: sql="insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0 from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"';" func.mysql_exec(sql,'') sql="delete from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"';" func.mysql_exec(sql,'') else: pass
def send_warn(): #server_id not delete??? sql = "select alarm_last_status.application_id,app.display_name application,alarm_last_status.server_id,servers.host,servers.port,alarm_last_status.modify_time,servers.alarm_interval,servers.alarm_type,servers.converge_type,servers.alarm_person,servers.send_mail,alarm_last_status.connect,alarm_last_status.status_connections,alarm_last_status.status_used_memory,alarm_last_status.status_binlog_count,alarm_last_status.status_innodb_space_free,alarm_last_status.status_replication_relay,alarm_last_status.alarm_num,alarm_last_status.time_error_continue,alarm_last_status.slow_querys,alarm_last_status.status_slow_querys,alarm_last_status.error_log,alarm_last_status.status_error_log from alarm_last_status left join servers on alarm_last_status.server_id=servers.id left join application app on servers.application_id=app.id;" result = func.mysql_query(sql) if result <> 0: domain = "www.oa.com" if (pingServerCall(domain) == False): #系统告警接收人 mailto_list = func.get_option('mail_to_list') mail_subject = "redis monitor error" mail_content = "ping failed:" + domain + ",do not send warn..." func.send_alarm(0, mailto_list, mail_subject, mail_content.encode('utf-8')) return None #全局配置--发送告警 send_alarm_mail = func.get_option('send_alarm_mail') #全局配置--报警检测 frequency_alarm = func.get_option('frequency_alarm') #mail_to_list = func.get_option('mail_to_list') #mailto_list=mail_to_list.split(';') for line in result: application_id = line[0] application = line[1] server_id = line[2] host = line[3] port = line[4] create_time = line[5] alarm_interval = line[6] alarm_type = line[7] converge_type = line[8] alarm_person = line[9] send_alarm = line[10] connect = line[11] status_connections = line[12] status_used_memory = line[13] status_binlog_count = line[14] status_innodb_space_free = line[15] status_replication_relay = line[16] alarm_num = line[17] time_error_continue = line[18] slow_querys = line[19] status_slow_querys = line[20] error_log = line[21] status_error_log = line[22] warn_content = application + '-' + host + ':' + port eStr = "ERROR:" wStr = "WARN:" if (eStr in connect) or (wStr in connect): warn_content = warn_content + ' ' + connect else: if (eStr in status_connections) or (wStr in status_connections): warn_content = warn_content + ' ' + status_connections if (eStr in status_used_memory) or (wStr in status_used_memory): warn_content = warn_content + ' ' + status_used_memory if (eStr in status_binlog_count) or (wStr in status_binlog_count): warn_content = warn_content + ' ' + status_binlog_count if (eStr in status_innodb_space_free) or ( wStr in status_innodb_space_free): warn_content = warn_content + ' ' + status_innodb_space_free if (eStr in status_replication_relay) or ( wStr in status_replication_relay): warn_content = warn_content + ' ' + status_replication_relay if (eStr in status_slow_querys) or (wStr in status_slow_querys): warn_content = warn_content + ' ' + status_slow_querys if (eStr in status_error_log) or (wStr in status_error_log): warn_content = warn_content + ' ' + status_error_log warn_content = warn_content + ' [' + create_time.strftime( '%Y-%m-%d %H:%M:%S') + ']' #warn_content=warn_content.encode('utf-8') #print send_alarm_mail #print send_alarm #print warn_content #print("WARN:%s send alarm:%s alarm:%s" %(warn_content,send_alarm_mail,send_alarm)) mailto_list = alarm_person mail_subject = "REDIS WARNING" send_mail_status = 0 if send_alarm_mail == '1': if send_alarm == 1: next_alarm_time = getTimeInterVal(converge_type, alarm_interval, alarm_num, 0) if (func.test_flag): log.debug( "time_error_continue:%s next_alarm_time:%s converge_type:%s alarm_interval:%s alarm_num:%s" % (str(time_error_continue), str(next_alarm_time), converge_type, str(alarm_interval), str(alarm_num))) if time_error_continue >= next_alarm_time: result = func.send_alarm(alarm_type, mailto_list, mail_subject, warn_content.encode('utf-8')) if result: send_mail_status = 1 else: send_mail_status = 0 sql = "update alarm_last_status set alarm_num=alarm_num+1 where application_id='" + str( application_id) + "' and server_id='" + str( server_id) + "'" func.mysql_exec(sql, '') log.warning( "send alarm:alarm_type:%s mailto_list:%s mail_subject:%s warn_content:%s" % (str(alarm_type), mailto_list, mail_subject, warn_content.encode('utf-8'))) else: sql = "delete from alarm where application_id='" + str( application_id) + "' and server_id='" + str( server_id) + "'" func.mysql_exec(sql, '') else: send_mail_status = 0 else: send_mail_status = 0 sql = "update alarm_last_status set time_error_continue=time_error_continue+" + str( frequency_alarm) + " where application_id='" + str( application_id) + "' and server_id='" + str( server_id) + "'" func.mysql_exec(sql, '') #param=(str(application_id),str(server_id)) if send_mail_status == 1: sql = "insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1 from alarm where application_id='" + str( application_id) + "' and server_id='" + str( server_id) + "';" func.mysql_exec(sql, '') elif send_mail_status == 0: sql = "insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0 from alarm where application_id='" + str( application_id) + "' and server_id='" + str( server_id) + "';" func.mysql_exec(sql, '') sql = "delete from alarm where application_id='" + str( application_id) + "' and server_id='" + str(server_id) + "';" func.mysql_exec(sql, '') else: pass
def send_alarm_message(): sql = "select alarm.application_id,app.display_name application,alarm.server_id,servers.host,servers.port,alarm.create_time,db_type,alarm_type,alarm_value,level,message,alarm.send_message from alarm left join servers on alarm.server_id=servers.id left join application app on servers.application_id=app.id;" result = func.mysql_query(sql) if result <> 0: send_alarm_message = func.get_option('send_alarm_message') phone = func.get_option('message_to_list') for line in result: application_id = line[0] application = line[1] server_id = [2] host = line[3] port = line[4] create_time = line[5] db_type = line[6] alarm_type = line[7] alarm_value = line[8] level = line[9] message = line[10] send_message = line[11] send_message_status = 0 if send_alarm_message == "1": if send_message == 1: phone_list = phone.split(';') all_send_result = [] for i in phone_list: values = {} values['phone'] = i values['type'] = 7 values['signname'] = '小店监控' values['server'] = host values['errmsg'] = message result = func.send_message(values) all_send_result.append(result) all_flag = True # 有一人收到 成功发送 for index, res in enumerate(all_send_result): if res == True: all_flag = True if all_flag: send_message_status = 1 else: send_message_status = 0 else: send_message_status = 0 else: send_message_status = 0 if send_message_status == 1: func.mysql_exec( "insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_message,send_message_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_message,1 from alarm;", '') elif send_message_status == 0: func.mysql_exec( "insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_message,send_message_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_message,0 from alarm;", '') func.mysql_exec("delete from alarm", '') else: pass
#!/usr/bin/env python # -*- coding: utf8 -*- from datetime import * import global_functions as func mail_to_list = func.get_option('mail_to_list') mailto_list=mail_to_list.split(';') result = func.send_mail(mailto_list," I hope you can learn","Beautiful Day") print result if result: send_mail_status = "success" else: send_mail_status = "fail" print "send_mail_status:"+send_mail_status
#!/usr/bin/env python # -*- coding: utf8 -*- from datetime import * import global_functions as func phone = func.get_option('message_to_list') values = {} values['phone'] = phone values['type'] = 7 values['signname'] = '小店监控' values['server'] = '10.18.1.1' values['errmsg'] = '测试报警功能' result = func.send_message(values) print "发送结果:" print result print type(result)
def send_alarm_mail(idc_type='ts'): sql="select alarm.application_id,app.display_name application,alarm.server_id,servers.host,servers.port,alarm.create_time,db_type,alarm_type,alarm_value,level,message,alarm.send_mail,servers.alarm_time,servers.threshold_starttime,servers.threshold_endtime from alarm left join servers on alarm.server_id=servers.id left join application app on servers.application_id=app.id where servers.idc_type='"+str(idc_type)+"';" result=func.mysql_query(sql) if result <> 0: send_alarm_mail = func.get_option('send_alarm_mail') mail_to_list = func.get_option('mail_to_list') mailto_list=mail_to_list.split(';') global_alarm_type=func.get_config('global_alarm_type','alarm_type') global_alarm_type_list=global_alarm_type.split(',') for line in result: application_id=line[0] application=line[1] server_id=line[2] host=line[3] port=line[4] create_time=line[5] db_type=line[6] alarm_type=line[7] alarm_value=line[8] level=line[9] message=line[10] send_mail=line[11] alarm_time=line[12] threshold_starttime=line[13] threshold_endtime=line[14] app_email=func.get_application_alarm_email(str(application_id)) app_email_list=app_email.split(',') for email_app in app_email_list: if email_app not in mailto_list: mailto_list.append(email_app) else: pass if send_alarm_mail=="1": if send_mail==1: mail_subject=message+' 当前值:'+alarm_value+' 服务器:'+application+'-'+host+':'+port+' 时间:'+create_time.strftime('%Y-%m-%d %H:%M:%S') mail_content="请检查下!" now_hour = ("0"+str(datetime.now().hour) if datetime.now().hour<10 else str(datetime.now().hour))+":"+( "0"+str(datetime.now().minute) if datetime.now().minute<10 else str(datetime.now().minute)) if alarm_type not in global_alarm_type_list: if (int(alarm_time) == 1) and (threshold_starttime.strip() != "") and (threshold_endtime.strip() != "") and (threshold_endtime > threshold_starttime) and (now_hour <= threshold_endtime) and (now_hour >= threshold_starttime): result_mail = func.send_mail(mailto_list,mail_subject,mail_content) elif int(alarm_time) != 1: result_mail = func.send_mail(mailto_list,mail_subject,mail_content) else: result_mail = False else: result_mail = func.send_mail(mailto_list,mail_subject,mail_content) if result_mail: send_mail_status=1 else: send_mail_status=0 else: send_mail_status=0 else: send_mail_status=0 if send_mail_status==1: func.mysql_exec("insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status,idc_type) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1,'"+str(idc_type)+"' from alarm;",'') elif send_mail_status==0: func.mysql_exec("insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status,idc_type) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0,'"+str(idc_type)+"' from alarm;",'') func.mysql_exec("delete from alarm where idc_type='"+str(idc_type)+"'",'') else: pass