def get_alarm_last_status():
sql = "delete from alarm_last_status where server_id not in (select distinct server_id from alarm);"
func.mysql_exec(sql, '')
sql = "update alarm_last_status set connect='',status_connections='',status_used_memory='',status_replication_relay='';"
func.mysql_exec(sql, '')
#insert new warn
sql = "insert into alarm_last_status(application_id,server_id) select distinct application_id,server_id from alarm where server_id not in (select distinct server_id from alarm_last_status);"
func.mysql_exec(sql, '')
#old warn, update
sql = "select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id in (select distinct server_id from alarm_last_status);"
result = func.mysql_query(sql)
if result <> 0:
for line in result:
update_sql = "update alarm_last_status set "
application_id = line[0]
server_id = line[1]
alarm_type = line[2]
alarm_value = line[3]
level = line[4]
message = line[5]
send_mail = line[6]
if alarm_type == "connect":
update_sql = update_sql + "connect=\"" + message + "\","
elif alarm_type == "connections":
update_sql = update_sql + "connections='" + alarm_value + "',status_connections='" + message + "',"
elif alarm_type == "memory":
update_sql = update_sql + "used_memory='" + alarm_value + "',status_used_memory='" + message + "',"
elif alarm_type == "delay":
update_sql = update_sql + "replication_relay='" + alarm_value + "',status_replication_relay='" + message + "',"
elif alarm_type == "replication":
update_sql = update_sql + "replication_relay='-1',status_replication_relay='" + message + "',"
update_sql = update_sql + "send_alarm='" + str(
send_mail) + "' where application_id='" + str(
application_id) + "' and server_id='" + str(
server_id) + "';"
log.debug("update_sql:%s" % (update_sql))
func.mysql_exec(update_sql, '')
#should not appear, send alarm to me
sql = "select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id not in (select distinct server_id from alarm_last_status);"
result = func.mysql_query(sql)
if result <> 0:
mailto_list = func.get_option('mail_to_list')
mail_subject = "redis monitor error"
mail_content = "this should not appear:" + sql
func.send_alarm(0, mailto_list, mail_subject,
mail_content.encode('utf-8'))
def get_alarm_last_status():
sql="delete from alarm_last_status where server_id not in (select distinct server_id from alarm);"
func.mysql_exec(sql,'')
sql="update alarm_last_status set connect='',status_connections='',status_used_memory='',status_replication_relay='';"
func.mysql_exec(sql,'')
#insert new warn
sql="insert into alarm_last_status(application_id,server_id) select distinct application_id,server_id from alarm where server_id not in (select distinct server_id from alarm_last_status);"
func.mysql_exec(sql,'')
#old warn, update
sql="select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id in (select distinct server_id from alarm_last_status);"
result=func.mysql_query(sql)
if result <> 0:
for line in result:
update_sql="update alarm_last_status set "
application_id=line[0]
server_id=line[1]
alarm_type=line[2]
alarm_value=line[3]
level=line[4]
message=line[5]
send_mail=line[6]
if alarm_type=="connect":
update_sql=update_sql + "connect=\"" + message + "\","
elif alarm_type=="connections":
update_sql=update_sql + "connections='" + alarm_value + "',status_connections='" + message + "',"
elif alarm_type=="memory":
update_sql=update_sql + "used_memory='" + alarm_value + "',status_used_memory='" + message + "',"
elif alarm_type=="delay":
update_sql=update_sql + "replication_relay='" + alarm_value + "',status_replication_relay='" + message + "',"
elif alarm_type=="replication":
update_sql=update_sql + "replication_relay='-1',status_replication_relay='" + message + "',"
update_sql=update_sql + "send_alarm='" + str(send_mail) + "' where application_id='" + str(application_id) + "' and server_id='" + str(server_id) + "';"
log.debug("update_sql:%s"%(update_sql))
func.mysql_exec(update_sql,'')
#should not appear, send alarm to me
sql="select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id not in (select distinct server_id from alarm_last_status);"
result=func.mysql_query(sql)
if result <> 0:
mailto_list = func.get_option('mail_to_list')
mail_subject="redis monitor error"
mail_content="this should not appear:"+sql
func.send_alarm(0,mailto_list,mail_subject,mail_content.encode('utf-8'))
def send_backup_alarm(alarm_flag,mailto_list,warn_content,application_id,server_id,db_name,current_date,log):
alarm_type=0
mail_subject="REDIS BACKUP WARN"
create_time=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
db_type="redis"
alarm_type="backup"
alarm_value="fail"
level="warning"
if alarm_flag== "y":
result=func.send_alarm(alarm_type,mailto_list,mail_subject,warn_content.encode('utf-8'))
if result:
send_mail_status=1
else:
send_mail_status=0
log.error("send alarm:alarm_type:%s mailto_list:%s mail_subject:%s warn_content:%s"%(str(alarm_type),mailto_list,mail_subject,warn_content.encode('utf-8')))
#print("send alarm:alarm_type:%s mailto_list:%s mail_subject:%s warn_content:%s"%(str(alarm_type),mailto_list,mail_subject,warn_content.encode('utf-8')))
sql="insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
param=(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,warn_content,'1',send_mail_status)
func.mysql_exec(sql,param)
sql="replace into redis_coldback_info(date,server_id,db_name,file_name,suc_flag) values(%s,%s,%s,%s,%s)"
param=(current_date,server_id,db_name,'---','n')
func.mysql_exec(sql,param)
def send_warn():
#server_id not delete???
sql="select alarm_last_status.application_id,app.display_name application,alarm_last_status.server_id,servers.host,servers.port,alarm_last_status.modify_time,servers.alarm_interval,servers.alarm_type,servers.converge_type,servers.alarm_person,servers.send_mail,alarm_last_status.connect,alarm_last_status.status_connections,alarm_last_status.status_used_memory,alarm_last_status.status_binlog_count,alarm_last_status.status_innodb_space_free,alarm_last_status.status_replication_relay,alarm_last_status.alarm_num,alarm_last_status.time_error_continue,alarm_last_status.slow_querys,alarm_last_status.status_slow_querys,alarm_last_status.error_log,alarm_last_status.status_error_log from alarm_last_status left join servers on alarm_last_status.server_id=servers.id left join application app on servers.application_id=app.id;"
result=func.mysql_query(sql)
if result <> 0:
domain="www.oa.com"
if(pingServerCall(domain)==False):
#系统告警接收人
mailto_list = func.get_option('mail_to_list')
mail_subject="redis monitor error"
mail_content="ping failed:"+domain +",do not send warn..."
func.send_alarm(0,mailto_list,mail_subject,mail_content.encode('utf-8'))
return None
#全局配置--发送告警
send_alarm_mail = func.get_option('send_alarm_mail')
#全局配置--报警检测
frequency_alarm = func.get_option('frequency_alarm')
#mail_to_list = func.get_option('mail_to_list')
#mailto_list=mail_to_list.split(';')
for line in result:
application_id=line[0]
application=line[1]
server_id=line[2]
host=line[3]
port=line[4]
create_time=line[5]
alarm_interval=line[6]
alarm_type=line[7]
converge_type=line[8]
alarm_person=line[9]
send_alarm=line[10]
connect=line[11]
status_connections=line[12]
status_used_memory=line[13]
status_binlog_count=line[14]
status_innodb_space_free=line[15]
status_replication_relay=line[16]
alarm_num=line[17]
time_error_continue=line[18]
slow_querys=line[19]
status_slow_querys=line[20]
error_log=line[21]
status_error_log=line[22]
warn_content=application+'-'+host+':'+port
eStr="ERROR:"
wStr="WARN:"
if (eStr in connect) or (wStr in connect):
warn_content=warn_content+' '+connect
else:
if (eStr in status_connections) or (wStr in status_connections):
warn_content=warn_content+' '+status_connections
if (eStr in status_used_memory) or (wStr in status_used_memory):
warn_content=warn_content+' '+status_used_memory
if (eStr in status_binlog_count) or (wStr in status_binlog_count):
warn_content=warn_content+' '+status_binlog_count
if (eStr in status_innodb_space_free) or (wStr in status_innodb_space_free):
warn_content=warn_content+' '+status_innodb_space_free
if (eStr in status_replication_relay) or (wStr in status_replication_relay):
warn_content=warn_content+' '+status_replication_relay
if (eStr in status_slow_querys) or (wStr in status_slow_querys):
warn_content=warn_content+' '+status_slow_querys
if (eStr in status_error_log) or (wStr in status_error_log):
warn_content=warn_content+' '+status_error_log
warn_content=warn_content+' ['+create_time.strftime('%Y-%m-%d %H:%M:%S')+']'
#warn_content=warn_content.encode('utf-8')
#print send_alarm_mail
#print send_alarm
#print warn_content
#print("WARN:%s send alarm:%s alarm:%s" %(warn_content,send_alarm_mail,send_alarm))
mailto_list=alarm_person
mail_subject="REDIS WARNING"
send_mail_status=0
if send_alarm_mail=='1':
if send_alarm==1:
next_alarm_time=getTimeInterVal(converge_type,alarm_interval,alarm_num,0);
if(func.test_flag):
log.debug("time_error_continue:%s next_alarm_time:%s converge_type:%s alarm_interval:%s alarm_num:%s"%(str(time_error_continue),str(next_alarm_time),converge_type,str(alarm_interval),str(alarm_num)))
if time_error_continue >= next_alarm_time:
result=func.send_alarm(alarm_type,mailto_list,mail_subject,warn_content.encode('utf-8'))
if result:
send_mail_status=1
else:
send_mail_status=0
sql="update alarm_last_status set alarm_num=alarm_num+1 where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"'"
func.mysql_exec(sql,'')
log.warning("send alarm:alarm_type:%s mailto_list:%s mail_subject:%s warn_content:%s"%(str(alarm_type),mailto_list,mail_subject,warn_content.encode('utf-8')))
else:
sql="delete from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"'"
func.mysql_exec(sql,'')
else:
send_mail_status=0
else:
send_mail_status=0
sql="update alarm_last_status set time_error_continue=time_error_continue+"+str(frequency_alarm)+" where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"'"
func.mysql_exec(sql,'')
#param=(str(application_id),str(server_id))
if send_mail_status==1:
sql="insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1 from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"';"
func.mysql_exec(sql,'')
elif send_mail_status==0:
sql="insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0 from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"';"
func.mysql_exec(sql,'')
sql="delete from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"';"
func.mysql_exec(sql,'')
else:
pass
def send_warn():
#server_id not delete???
sql = "select alarm_last_status.application_id,app.display_name application,alarm_last_status.server_id,servers.host,servers.port,alarm_last_status.modify_time,servers.alarm_interval,servers.alarm_type,servers.converge_type,servers.alarm_person,servers.send_mail,alarm_last_status.connect,alarm_last_status.status_connections,alarm_last_status.status_used_memory,alarm_last_status.status_binlog_count,alarm_last_status.status_innodb_space_free,alarm_last_status.status_replication_relay,alarm_last_status.alarm_num,alarm_last_status.time_error_continue,alarm_last_status.slow_querys,alarm_last_status.status_slow_querys,alarm_last_status.error_log,alarm_last_status.status_error_log from alarm_last_status left join servers on alarm_last_status.server_id=servers.id left join application app on servers.application_id=app.id;"
result = func.mysql_query(sql)
if result <> 0:
domain = "www.oa.com"
if (pingServerCall(domain) == False):
#系统告警接收人
mailto_list = func.get_option('mail_to_list')
mail_subject = "redis monitor error"
mail_content = "ping failed:" + domain + ",do not send warn..."
func.send_alarm(0, mailto_list, mail_subject,
mail_content.encode('utf-8'))
return None
#全局配置--发送告警
send_alarm_mail = func.get_option('send_alarm_mail')
#全局配置--报警检测
frequency_alarm = func.get_option('frequency_alarm')
#mail_to_list = func.get_option('mail_to_list')
#mailto_list=mail_to_list.split(';')
for line in result:
application_id = line[0]
application = line[1]
server_id = line[2]
host = line[3]
port = line[4]
create_time = line[5]
alarm_interval = line[6]
alarm_type = line[7]
converge_type = line[8]
alarm_person = line[9]
send_alarm = line[10]
connect = line[11]
status_connections = line[12]
status_used_memory = line[13]
status_binlog_count = line[14]
status_innodb_space_free = line[15]
status_replication_relay = line[16]
alarm_num = line[17]
time_error_continue = line[18]
slow_querys = line[19]
status_slow_querys = line[20]
error_log = line[21]
status_error_log = line[22]
warn_content = application + '-' + host + ':' + port
eStr = "ERROR:"
wStr = "WARN:"
if (eStr in connect) or (wStr in connect):
warn_content = warn_content + ' ' + connect
else:
if (eStr in status_connections) or (wStr
in status_connections):
warn_content = warn_content + ' ' + status_connections
if (eStr in status_used_memory) or (wStr
in status_used_memory):
warn_content = warn_content + ' ' + status_used_memory
if (eStr in status_binlog_count) or (wStr
in status_binlog_count):
warn_content = warn_content + ' ' + status_binlog_count
if (eStr in status_innodb_space_free) or (
wStr in status_innodb_space_free):
warn_content = warn_content + ' ' + status_innodb_space_free
if (eStr in status_replication_relay) or (
wStr in status_replication_relay):
warn_content = warn_content + ' ' + status_replication_relay
if (eStr in status_slow_querys) or (wStr
in status_slow_querys):
warn_content = warn_content + ' ' + status_slow_querys
if (eStr in status_error_log) or (wStr in status_error_log):
warn_content = warn_content + ' ' + status_error_log
warn_content = warn_content + ' [' + create_time.strftime(
'%Y-%m-%d %H:%M:%S') + ']'
#warn_content=warn_content.encode('utf-8')
#print send_alarm_mail
#print send_alarm
#print warn_content
#print("WARN:%s send alarm:%s alarm:%s" %(warn_content,send_alarm_mail,send_alarm))
mailto_list = alarm_person
mail_subject = "REDIS WARNING"
send_mail_status = 0
if send_alarm_mail == '1':
if send_alarm == 1:
next_alarm_time = getTimeInterVal(converge_type,
alarm_interval,
alarm_num, 0)
if (func.test_flag):
log.debug(
"time_error_continue:%s next_alarm_time:%s converge_type:%s alarm_interval:%s alarm_num:%s"
% (str(time_error_continue), str(next_alarm_time),
converge_type, str(alarm_interval),
str(alarm_num)))
if time_error_continue >= next_alarm_time:
result = func.send_alarm(alarm_type, mailto_list,
mail_subject,
warn_content.encode('utf-8'))
if result:
send_mail_status = 1
else:
send_mail_status = 0
sql = "update alarm_last_status set alarm_num=alarm_num+1 where application_id='" + str(
application_id) + "' and server_id='" + str(
server_id) + "'"
func.mysql_exec(sql, '')
log.warning(
"send alarm:alarm_type:%s mailto_list:%s mail_subject:%s warn_content:%s"
% (str(alarm_type), mailto_list, mail_subject,
warn_content.encode('utf-8')))
else:
sql = "delete from alarm where application_id='" + str(
application_id) + "' and server_id='" + str(
server_id) + "'"
func.mysql_exec(sql, '')
else:
send_mail_status = 0
else:
send_mail_status = 0
sql = "update alarm_last_status set time_error_continue=time_error_continue+" + str(
frequency_alarm) + " where application_id='" + str(
application_id) + "' and server_id='" + str(
server_id) + "'"
func.mysql_exec(sql, '')
#param=(str(application_id),str(server_id))
if send_mail_status == 1:
sql = "insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1 from alarm where application_id='" + str(
application_id) + "' and server_id='" + str(
server_id) + "';"
func.mysql_exec(sql, '')
elif send_mail_status == 0:
sql = "insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0 from alarm where application_id='" + str(
application_id) + "' and server_id='" + str(
server_id) + "';"
func.mysql_exec(sql, '')
sql = "delete from alarm where application_id='" + str(
application_id) + "' and server_id='" + str(server_id) + "';"
func.mysql_exec(sql, '')
else:
pass
