def get_alarm_last_status(): sql = "delete from alarm_last_status where server_id not in (select distinct server_id from alarm);" func.mysql_exec(sql, '') sql = "update alarm_last_status set connect='',status_connections='',status_used_memory='',status_replication_relay='';" func.mysql_exec(sql, '') #insert new warn sql = "insert into alarm_last_status(application_id,server_id) select distinct application_id,server_id from alarm where server_id not in (select distinct server_id from alarm_last_status);" func.mysql_exec(sql, '') #old warn, update sql = "select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id in (select distinct server_id from alarm_last_status);" result = func.mysql_query(sql) if result <> 0: for line in result: update_sql = "update alarm_last_status set " application_id = line[0] server_id = line[1] alarm_type = line[2] alarm_value = line[3] level = line[4] message = line[5] send_mail = line[6] if alarm_type == "connect": update_sql = update_sql + "connect=\"" + message + "\"," elif alarm_type == "connections": update_sql = update_sql + "connections='" + alarm_value + "',status_connections='" + message + "'," elif alarm_type == "memory": update_sql = update_sql + "used_memory='" + alarm_value + "',status_used_memory='" + message + "'," elif alarm_type == "delay": update_sql = update_sql + "replication_relay='" + alarm_value + "',status_replication_relay='" + message + "'," elif alarm_type == "replication": update_sql = update_sql + "replication_relay='-1',status_replication_relay='" + message + "'," update_sql = update_sql + "send_alarm='" + str( send_mail) + "' where application_id='" + str( application_id) + "' and server_id='" + str( server_id) + "';" log.debug("update_sql:%s" % (update_sql)) func.mysql_exec(update_sql, '') #should not appear, send alarm to me sql = "select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id not in (select distinct server_id from alarm_last_status);" result = func.mysql_query(sql) if result <> 0: mailto_list = func.get_option('mail_to_list') mail_subject = "redis monitor error" mail_content = "this should not appear:" + sql func.send_alarm(0, mailto_list, mail_subject, mail_content.encode('utf-8'))
def get_alarm_last_status(): sql="delete from alarm_last_status where server_id not in (select distinct server_id from alarm);" func.mysql_exec(sql,'') sql="update alarm_last_status set connect='',status_connections='',status_used_memory='',status_replication_relay='';" func.mysql_exec(sql,'') #insert new warn sql="insert into alarm_last_status(application_id,server_id) select distinct application_id,server_id from alarm where server_id not in (select distinct server_id from alarm_last_status);" func.mysql_exec(sql,'') #old warn, update sql="select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id in (select distinct server_id from alarm_last_status);" result=func.mysql_query(sql) if result <> 0: for line in result: update_sql="update alarm_last_status set " application_id=line[0] server_id=line[1] alarm_type=line[2] alarm_value=line[3] level=line[4] message=line[5] send_mail=line[6] if alarm_type=="connect": update_sql=update_sql + "connect=\"" + message + "\"," elif alarm_type=="connections": update_sql=update_sql + "connections='" + alarm_value + "',status_connections='" + message + "'," elif alarm_type=="memory": update_sql=update_sql + "used_memory='" + alarm_value + "',status_used_memory='" + message + "'," elif alarm_type=="delay": update_sql=update_sql + "replication_relay='" + alarm_value + "',status_replication_relay='" + message + "'," elif alarm_type=="replication": update_sql=update_sql + "replication_relay='-1',status_replication_relay='" + message + "'," update_sql=update_sql + "send_alarm='" + str(send_mail) + "' where application_id='" + str(application_id) + "' and server_id='" + str(server_id) + "';" log.debug("update_sql:%s"%(update_sql)) func.mysql_exec(update_sql,'') #should not appear, send alarm to me sql="select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id not in (select distinct server_id from alarm_last_status);" result=func.mysql_query(sql) if result <> 0: mailto_list = func.get_option('mail_to_list') mail_subject="redis monitor error" mail_content="this should not appear:"+sql func.send_alarm(0,mailto_list,mail_subject,mail_content.encode('utf-8'))
def send_backup_alarm(alarm_flag,mailto_list,warn_content,application_id,server_id,db_name,current_date,log): alarm_type=0 mail_subject="REDIS BACKUP WARN" create_time=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) db_type="redis" alarm_type="backup" alarm_value="fail" level="warning" if alarm_flag== "y": result=func.send_alarm(alarm_type,mailto_list,mail_subject,warn_content.encode('utf-8')) if result: send_mail_status=1 else: send_mail_status=0 log.error("send alarm:alarm_type:%s mailto_list:%s mail_subject:%s warn_content:%s"%(str(alarm_type),mailto_list,mail_subject,warn_content.encode('utf-8'))) #print("send alarm:alarm_type:%s mailto_list:%s mail_subject:%s warn_content:%s"%(str(alarm_type),mailto_list,mail_subject,warn_content.encode('utf-8'))) sql="insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" param=(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,warn_content,'1',send_mail_status) func.mysql_exec(sql,param) sql="replace into redis_coldback_info(date,server_id,db_name,file_name,suc_flag) values(%s,%s,%s,%s,%s)" param=(current_date,server_id,db_name,'---','n') func.mysql_exec(sql,param)
def send_warn(): #server_id not delete??? sql="select alarm_last_status.application_id,app.display_name application,alarm_last_status.server_id,servers.host,servers.port,alarm_last_status.modify_time,servers.alarm_interval,servers.alarm_type,servers.converge_type,servers.alarm_person,servers.send_mail,alarm_last_status.connect,alarm_last_status.status_connections,alarm_last_status.status_used_memory,alarm_last_status.status_binlog_count,alarm_last_status.status_innodb_space_free,alarm_last_status.status_replication_relay,alarm_last_status.alarm_num,alarm_last_status.time_error_continue,alarm_last_status.slow_querys,alarm_last_status.status_slow_querys,alarm_last_status.error_log,alarm_last_status.status_error_log from alarm_last_status left join servers on alarm_last_status.server_id=servers.id left join application app on servers.application_id=app.id;" result=func.mysql_query(sql) if result <> 0: domain="www.oa.com" if(pingServerCall(domain)==False): #系统告警接收人 mailto_list = func.get_option('mail_to_list') mail_subject="redis monitor error" mail_content="ping failed:"+domain +",do not send warn..." func.send_alarm(0,mailto_list,mail_subject,mail_content.encode('utf-8')) return None #全局配置--发送告警 send_alarm_mail = func.get_option('send_alarm_mail') #全局配置--报警检测 frequency_alarm = func.get_option('frequency_alarm') #mail_to_list = func.get_option('mail_to_list') #mailto_list=mail_to_list.split(';') for line in result: application_id=line[0] application=line[1] server_id=line[2] host=line[3] port=line[4] create_time=line[5] alarm_interval=line[6] alarm_type=line[7] converge_type=line[8] alarm_person=line[9] send_alarm=line[10] connect=line[11] status_connections=line[12] status_used_memory=line[13] status_binlog_count=line[14] status_innodb_space_free=line[15] status_replication_relay=line[16] alarm_num=line[17] time_error_continue=line[18] slow_querys=line[19] status_slow_querys=line[20] error_log=line[21] status_error_log=line[22] warn_content=application+'-'+host+':'+port eStr="ERROR:" wStr="WARN:" if (eStr in connect) or (wStr in connect): warn_content=warn_content+' '+connect else: if (eStr in status_connections) or (wStr in status_connections): warn_content=warn_content+' '+status_connections if (eStr in status_used_memory) or (wStr in status_used_memory): warn_content=warn_content+' '+status_used_memory if (eStr in status_binlog_count) or (wStr in status_binlog_count): warn_content=warn_content+' '+status_binlog_count if (eStr in status_innodb_space_free) or (wStr in status_innodb_space_free): warn_content=warn_content+' '+status_innodb_space_free if (eStr in status_replication_relay) or (wStr in status_replication_relay): warn_content=warn_content+' '+status_replication_relay if (eStr in status_slow_querys) or (wStr in status_slow_querys): warn_content=warn_content+' '+status_slow_querys if (eStr in status_error_log) or (wStr in status_error_log): warn_content=warn_content+' '+status_error_log warn_content=warn_content+' ['+create_time.strftime('%Y-%m-%d %H:%M:%S')+']' #warn_content=warn_content.encode('utf-8') #print send_alarm_mail #print send_alarm #print warn_content #print("WARN:%s send alarm:%s alarm:%s" %(warn_content,send_alarm_mail,send_alarm)) mailto_list=alarm_person mail_subject="REDIS WARNING" send_mail_status=0 if send_alarm_mail=='1': if send_alarm==1: next_alarm_time=getTimeInterVal(converge_type,alarm_interval,alarm_num,0); if(func.test_flag): log.debug("time_error_continue:%s next_alarm_time:%s converge_type:%s alarm_interval:%s alarm_num:%s"%(str(time_error_continue),str(next_alarm_time),converge_type,str(alarm_interval),str(alarm_num))) if time_error_continue >= next_alarm_time: result=func.send_alarm(alarm_type,mailto_list,mail_subject,warn_content.encode('utf-8')) if result: send_mail_status=1 else: send_mail_status=0 sql="update alarm_last_status set alarm_num=alarm_num+1 where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"'" func.mysql_exec(sql,'') log.warning("send alarm:alarm_type:%s mailto_list:%s mail_subject:%s warn_content:%s"%(str(alarm_type),mailto_list,mail_subject,warn_content.encode('utf-8'))) else: sql="delete from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"'" func.mysql_exec(sql,'') else: send_mail_status=0 else: send_mail_status=0 sql="update alarm_last_status set time_error_continue=time_error_continue+"+str(frequency_alarm)+" where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"'" func.mysql_exec(sql,'') #param=(str(application_id),str(server_id)) if send_mail_status==1: sql="insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1 from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"';" func.mysql_exec(sql,'') elif send_mail_status==0: sql="insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0 from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"';" func.mysql_exec(sql,'') sql="delete from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"';" func.mysql_exec(sql,'') else: pass
def send_warn(): #server_id not delete??? sql = "select alarm_last_status.application_id,app.display_name application,alarm_last_status.server_id,servers.host,servers.port,alarm_last_status.modify_time,servers.alarm_interval,servers.alarm_type,servers.converge_type,servers.alarm_person,servers.send_mail,alarm_last_status.connect,alarm_last_status.status_connections,alarm_last_status.status_used_memory,alarm_last_status.status_binlog_count,alarm_last_status.status_innodb_space_free,alarm_last_status.status_replication_relay,alarm_last_status.alarm_num,alarm_last_status.time_error_continue,alarm_last_status.slow_querys,alarm_last_status.status_slow_querys,alarm_last_status.error_log,alarm_last_status.status_error_log from alarm_last_status left join servers on alarm_last_status.server_id=servers.id left join application app on servers.application_id=app.id;" result = func.mysql_query(sql) if result <> 0: domain = "www.oa.com" if (pingServerCall(domain) == False): #系统告警接收人 mailto_list = func.get_option('mail_to_list') mail_subject = "redis monitor error" mail_content = "ping failed:" + domain + ",do not send warn..." func.send_alarm(0, mailto_list, mail_subject, mail_content.encode('utf-8')) return None #全局配置--发送告警 send_alarm_mail = func.get_option('send_alarm_mail') #全局配置--报警检测 frequency_alarm = func.get_option('frequency_alarm') #mail_to_list = func.get_option('mail_to_list') #mailto_list=mail_to_list.split(';') for line in result: application_id = line[0] application = line[1] server_id = line[2] host = line[3] port = line[4] create_time = line[5] alarm_interval = line[6] alarm_type = line[7] converge_type = line[8] alarm_person = line[9] send_alarm = line[10] connect = line[11] status_connections = line[12] status_used_memory = line[13] status_binlog_count = line[14] status_innodb_space_free = line[15] status_replication_relay = line[16] alarm_num = line[17] time_error_continue = line[18] slow_querys = line[19] status_slow_querys = line[20] error_log = line[21] status_error_log = line[22] warn_content = application + '-' + host + ':' + port eStr = "ERROR:" wStr = "WARN:" if (eStr in connect) or (wStr in connect): warn_content = warn_content + ' ' + connect else: if (eStr in status_connections) or (wStr in status_connections): warn_content = warn_content + ' ' + status_connections if (eStr in status_used_memory) or (wStr in status_used_memory): warn_content = warn_content + ' ' + status_used_memory if (eStr in status_binlog_count) or (wStr in status_binlog_count): warn_content = warn_content + ' ' + status_binlog_count if (eStr in status_innodb_space_free) or ( wStr in status_innodb_space_free): warn_content = warn_content + ' ' + status_innodb_space_free if (eStr in status_replication_relay) or ( wStr in status_replication_relay): warn_content = warn_content + ' ' + status_replication_relay if (eStr in status_slow_querys) or (wStr in status_slow_querys): warn_content = warn_content + ' ' + status_slow_querys if (eStr in status_error_log) or (wStr in status_error_log): warn_content = warn_content + ' ' + status_error_log warn_content = warn_content + ' [' + create_time.strftime( '%Y-%m-%d %H:%M:%S') + ']' #warn_content=warn_content.encode('utf-8') #print send_alarm_mail #print send_alarm #print warn_content #print("WARN:%s send alarm:%s alarm:%s" %(warn_content,send_alarm_mail,send_alarm)) mailto_list = alarm_person mail_subject = "REDIS WARNING" send_mail_status = 0 if send_alarm_mail == '1': if send_alarm == 1: next_alarm_time = getTimeInterVal(converge_type, alarm_interval, alarm_num, 0) if (func.test_flag): log.debug( "time_error_continue:%s next_alarm_time:%s converge_type:%s alarm_interval:%s alarm_num:%s" % (str(time_error_continue), str(next_alarm_time), converge_type, str(alarm_interval), str(alarm_num))) if time_error_continue >= next_alarm_time: result = func.send_alarm(alarm_type, mailto_list, mail_subject, warn_content.encode('utf-8')) if result: send_mail_status = 1 else: send_mail_status = 0 sql = "update alarm_last_status set alarm_num=alarm_num+1 where application_id='" + str( application_id) + "' and server_id='" + str( server_id) + "'" func.mysql_exec(sql, '') log.warning( "send alarm:alarm_type:%s mailto_list:%s mail_subject:%s warn_content:%s" % (str(alarm_type), mailto_list, mail_subject, warn_content.encode('utf-8'))) else: sql = "delete from alarm where application_id='" + str( application_id) + "' and server_id='" + str( server_id) + "'" func.mysql_exec(sql, '') else: send_mail_status = 0 else: send_mail_status = 0 sql = "update alarm_last_status set time_error_continue=time_error_continue+" + str( frequency_alarm) + " where application_id='" + str( application_id) + "' and server_id='" + str( server_id) + "'" func.mysql_exec(sql, '') #param=(str(application_id),str(server_id)) if send_mail_status == 1: sql = "insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1 from alarm where application_id='" + str( application_id) + "' and server_id='" + str( server_id) + "';" func.mysql_exec(sql, '') elif send_mail_status == 0: sql = "insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0 from alarm where application_id='" + str( application_id) + "' and server_id='" + str( server_id) + "';" func.mysql_exec(sql, '') sql = "delete from alarm where application_id='" + str( application_id) + "' and server_id='" + str(server_id) + "';" func.mysql_exec(sql, '') else: pass