def send_alarm_mail():
sql = "select alarm.application_id,app.display_name application,alarm.server_id,servers.host,servers.port,alarm.create_time,db_type,alarm_type,alarm_value,level,message,alarm.send_mail from alarm left join servers on alarm.server_id=servers.id left join application app on servers.application_id=app.id;"
result = func.mysql_query(sql)
if result <> 0:
send_alarm_mail = func.get_option('send_alarm_mail')
mail_to_list = func.get_option('mail_to_list')
mailto_list = mail_to_list.split(';')
for line in result:
application_id = line[0]
application = line[1]
server_id = [2]
host = line[3]
port = line[4]
create_time = line[5]
db_type = line[6]
alarm_type = line[7]
alarm_value = line[8]
level = line[9]
message = line[10]
send_mail = line[11]
if send_alarm_mail == "1":
if send_mail == 1:
mail_subject = message + ' 当前值:' + alarm_value + ' 服务器:' + application + '-' + host + ':' + port + ' 时间:' + create_time.strftime(
'%Y-%m-%d %H:%M:%S')
mail_content = "please check!"
result = func.send_mail(mailto_list, mail_subject,
mail_content)
if result:
send_mail_status = 1
else:
send_mail_status = 0
else:
send_mail_status = 0
else:
send_mail_status = 0
if send_mail_status == 1:
func.mysql_exec(
"insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1 from alarm;",
'')
elif send_mail_status == 0:
func.mysql_exec(
"insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0 from alarm;",
'')
func.mysql_exec("delete from alarm", '')
else:
pass
def clear_history_data():
history_save_days =int(func.get_option('history_save_days'))
if history_save_days >= 1:
old_date=str(func_time.get_day_of_day(history_save_days*-1))
sql="delete from redis_coldback_info where date <'"+old_date+"'"
log.debug("sql:%s"%(sql))
func.mysql_exec(sql,'')
sql="delete from alarm_history where left(create_time,10)<'"+old_date+"'"
log.debug("sql:%s"%(sql))
func.mysql_exec(sql,'')
sql="delete from redis_keyspace_history where left(create_time,10)<'"+old_date+"'"
log.debug("sql:%s"%(sql))
func.mysql_exec(sql,'')
sql="delete from redis_persistence_status_history where left(create_time,10)<'"+old_date+"'"
log.debug("sql:%s"%(sql))
func.mysql_exec(sql,'')
sql="delete from redis_replication_history where left(create_time,10)<'"+old_date+"'"
log.debug("sql:%s"%(sql))
func.mysql_exec(sql,'')
sql="delete from redis_resource_status_history where left(create_time,10)<'"+old_date+"'"
log.debug("sql:%s"%(sql))
func.mysql_exec(sql,'')
sql="delete from redis_run_status_history where left(create_time,10)<'"+old_date+"'"
log.debug("sql:%s"%(sql))
func.mysql_exec(sql,'')
sql="delete from redis_server_status_history where left(create_time,10)<'"+old_date+"'"
log.debug("sql:%s"%(sql))
func.mysql_exec(sql,'')
tCmd="rm -f ./log/*"+old_date
log.debug("tCmd:%s"%(tCmd))
os.system(tCmd)
else:
log.error("get history_save_days from options error!")
def check_mysql_widget(host,port,user,passwd,server_id):
try:
connect=MySQLdb.connect(host=host,user=user,passwd=passwd,port=int(port),connect_timeout=2,charset='utf8')
cur=connect.cursor()
connect.select_db('information_schema')
try:
bigtable=cur.execute("SELECT table_schema as 'DB',table_name as 'TABLE',CONCAT(ROUND(( data_length + index_length ) / ( 1024 * 1024 * 1024 ), 2), '') 'TOTAL' , table_comment as COMMENT FROM information_schema.TABLES ORDER BY data_length + index_length DESC ;");
big_table_size = func.get_option('big_table_size')
if bigtable:
for row in cur.fetchall():
datalist=[]
for r in row:
datalist.append(r)
result=datalist
if result:
table_size = float(string.atof(result[2]))
if table_size >= int(big_table_size):
sql="insert into mysql_widget_bigtable(server_id,db_name,table_name,table_size,table_comment) values(%s,%s,%s,%s,%s);"
param=(server_id,result[0],result[1],result[2],result[3])
func.mysql_exec(sql,param)
except :
pass
cur.close()
connect.close()
exit
except MySQLdb.Error,e:
pass
print "Mysql Error %d: %s" %(e.args[0],e.args[1])
def clear_history_data():
history_save_days = int(func.get_option('history_save_days'))
if history_save_days >= 1:
old_date = str(func_time.get_day_of_day(history_save_days * -1))
sql = "delete from redis_coldback_info where date <'" + old_date + "'"
log.debug("sql:%s" % (sql))
func.mysql_exec(sql, '')
sql = "delete from alarm_history where left(create_time,10)<'" + old_date + "'"
log.debug("sql:%s" % (sql))
func.mysql_exec(sql, '')
sql = "delete from redis_keyspace_history where left(create_time,10)<'" + old_date + "'"
log.debug("sql:%s" % (sql))
func.mysql_exec(sql, '')
sql = "delete from redis_persistence_status_history where left(create_time,10)<'" + old_date + "'"
log.debug("sql:%s" % (sql))
func.mysql_exec(sql, '')
sql = "delete from redis_replication_history where left(create_time,10)<'" + old_date + "'"
log.debug("sql:%s" % (sql))
func.mysql_exec(sql, '')
sql = "delete from redis_resource_status_history where left(create_time,10)<'" + old_date + "'"
log.debug("sql:%s" % (sql))
func.mysql_exec(sql, '')
sql = "delete from redis_run_status_history where left(create_time,10)<'" + old_date + "'"
log.debug("sql:%s" % (sql))
func.mysql_exec(sql, '')
sql = "delete from redis_server_status_history where left(create_time,10)<'" + old_date + "'"
log.debug("sql:%s" % (sql))
func.mysql_exec(sql, '')
tCmd = "rm -f ./log/*" + old_date
log.debug("tCmd:%s" % (tCmd))
os.system(tCmd)
else:
log.error("get history_save_days from options error!")
def check_mysql_widget(host,port,user,passwd,server_id):
try:
connect=MySQLdb.connect(host=host,user=user,passwd=passwd,port=int(port),connect_timeout=2,charset='utf8')
cur=connect.cursor()
connect.select_db('information_schema')
try:
bigtable=cur.execute("SELECT TABLE_SCHEMA as 'DB',TABLE_NAME as 'TABLE',CONCAT(ROUND(( DATA_LENGTH + INDEX_LENGTH ) / ( 1024 * 1024 * 1024 ), 2), '') 'TOTAL' , TABLE_COMMENT as COMMENT FROM information_schema.TABLES ORDER BY DATA_LENGTH + INDEX_LENGTH DESC ;");
big_table_size = func.get_option('big_table_size')
if bigtable:
for row in cur.fetchall():
datalist=[]
for r in row:
datalist.append(r)
result=datalist
if result:
table_size = float(string.atof(result[2]))
if table_size >= int(big_table_size):
sql="insert into mysql_widget_bigtable(server_id,db_name,table_name,table_size,table_comment) values(%s,%s,%s,%s,%s);"
param=(server_id,result[0],result[1],result[2],result[3])
func.mysql_exec(sql,param)
except :
pass
cur.close()
connect.close()
exit
except MySQLdb.Error,e:
pass
print "Mysql Error %d: %s" %(e.args[0],e.args[1])
def start_message_or_mail():
choice_message_or_mail = int(func.get_option('choice_message_or_mail'))
try:
if choice_message_or_mail == 1:
#message
send_alarm_message()
elif choice_message_or_mail == 0:
send_alarm_mail()
except Exception as e:
print(e)
def send_alarm_mail():
sql="select alarm.application_id,app.display_name application,alarm.server_id,servers.host,servers.port,alarm.create_time,db_type,alarm_type,alarm_value,level,message,alarm.send_mail from alarm left join servers on alarm.server_id=servers.id left join application app on servers.application_id=app.id;"
result=func.mysql_query(sql)
if result <> 0:
send_alarm_mail = func.get_option('send_alarm_mail')
mail_to_list = func.get_option('mail_to_list')
mailto_list=mail_to_list.split(';')
for line in result:
application_id=line[0]
application=line[1]
server_id=[2]
host=line[3]
port=line[4]
create_time=line[5]
db_type=line[6]
alarm_type=line[7]
alarm_value=line[8]
level=line[9]
message=line[10]
send_mail=line[11]
if send_alarm_mail=="1":
if send_mail==1:
mail_subject=message+' 当前值:'+alarm_value+' 服务器:'+application+'-'+host+':'+port+' 时间:'+create_time.strftime('%Y-%m-%d %H:%M:%S')
mail_content="please check!"
result = func.send_mail(mailto_list,mail_subject,mail_content)
if result:
send_mail_status=1
else:
send_mail_status=0
else:
send_mail_status=0
else:
send_mail_status=0
if send_mail_status==1:
func.mysql_exec("insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1 from alarm;",'')
elif send_mail_status==0:
func.mysql_exec("insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0 from alarm;",'')
func.mysql_exec("delete from alarm",'')
else:
pass
def main():
print("%s: controller started." %
(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()), ))
monitor = func.get_option('monitor')
monitor_status = func.get_option('monitor_status')
monitor_replication = func.get_option('monitor_replication')
monitor_process = func.get_option('monitor_process')
alarm = func.get_option('alarm')
frequency_monitor = func.get_option('frequency_monitor')
frequency_alarm = func.get_option('frequency_alarm')
kill_process = func.get_option('kill_process')
joblist = []
if monitor == "1":
if monitor_status == "1":
job = Process(target=job_run,
args=('check_mysql_status_ext.py',
frequency_monitor))
joblist.append(job)
job.start()
job = Process(target=job_run,
args=('check_mysql_status.py', frequency_monitor))
joblist.append(job)
job.start()
if monitor_replication == "1":
job = Process(target=job_run,
args=('check_mysql_replication.py',
frequency_monitor))
joblist.append(job)
job.start()
if monitor_process == "1":
job = Process(target=job_run, args=('check_mysql_process.py', 4))
joblist.append(job)
job.start()
if alarm == "1":
job = Process(target=job_run,
args=('alarm_mysql.py', frequency_alarm))
joblist.append(job)
job.start()
if kill_process == "1":
job = Process(target=job_run,
args=('admin_mysql_kill_process.py', 3))
joblist.append(job)
job.start()
for job in joblist:
job.join()
print("%s: controller finished." %
(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()), ))
def get_alarm_last_status():
sql = "delete from alarm_last_status where server_id not in (select distinct server_id from alarm);"
func.mysql_exec(sql, '')
sql = "update alarm_last_status set connect='',status_connections='',status_used_memory='',status_replication_relay='';"
func.mysql_exec(sql, '')
#insert new warn
sql = "insert into alarm_last_status(application_id,server_id) select distinct application_id,server_id from alarm where server_id not in (select distinct server_id from alarm_last_status);"
func.mysql_exec(sql, '')
#old warn, update
sql = "select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id in (select distinct server_id from alarm_last_status);"
result = func.mysql_query(sql)
if result <> 0:
for line in result:
update_sql = "update alarm_last_status set "
application_id = line[0]
server_id = line[1]
alarm_type = line[2]
alarm_value = line[3]
level = line[4]
message = line[5]
send_mail = line[6]
if alarm_type == "connect":
update_sql = update_sql + "connect=\"" + message + "\","
elif alarm_type == "connections":
update_sql = update_sql + "connections='" + alarm_value + "',status_connections='" + message + "',"
elif alarm_type == "memory":
update_sql = update_sql + "used_memory='" + alarm_value + "',status_used_memory='" + message + "',"
elif alarm_type == "delay":
update_sql = update_sql + "replication_relay='" + alarm_value + "',status_replication_relay='" + message + "',"
elif alarm_type == "replication":
update_sql = update_sql + "replication_relay='-1',status_replication_relay='" + message + "',"
update_sql = update_sql + "send_alarm='" + str(
send_mail) + "' where application_id='" + str(
application_id) + "' and server_id='" + str(
server_id) + "';"
log.debug("update_sql:%s" % (update_sql))
func.mysql_exec(update_sql, '')
#should not appear, send alarm to me
sql = "select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id not in (select distinct server_id from alarm_last_status);"
result = func.mysql_query(sql)
if result <> 0:
mailto_list = func.get_option('mail_to_list')
mail_subject = "redis monitor error"
mail_content = "this should not appear:" + sql
func.send_alarm(0, mailto_list, mail_subject,
mail_content.encode('utf-8'))
def main():
print("%s: controller started." %
(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()), ))
##全局配置--全局监控
monitor = func.get_option('monitor')
##全局配置--状态监控
monitor_status = func.get_option('monitor_status')
##全局配置--状态监控频率
frequency_monitor = func.get_option('frequency_monitor')
##全局配置--开启告警
alarm = func.get_option('alarm')
###全局配置--报警检测频率
frequency_alarm = func.get_option('frequency_alarm')
##全局配置--冷备管理
data_back_status = func.get_option('data_back_status')
##全局配置--冷备管理频率
frequency_data_back = func.get_option('frequency_data_back')
##全局配置--历史数据自动清理
history_clear_status = func.get_option('history_clear_status')
joblist = []
if monitor == "1":
if monitor_status == "1":
job = Process(target=job_run,
args=('check_redis_status.py', frequency_monitor))
joblist.append(job)
job.start()
if alarm == "1":
job = Process(target=job_run,
args=('alarm_redis.py', frequency_alarm))
joblist.append(job)
job.start()
if data_back_status == "1":
job = Process(target=job_run,
args=('redis_data_back.py', frequency_data_back))
joblist.append(job)
job.start()
if history_clear_status == "1":
job = Process(target=job_run,
args=('redis_history_clear.py', 36000))
joblist.append(job)
job.start()
for job in joblist:
job.join()
print("%s: controller finished." %
(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()), ))
def get_alarm_last_status():
sql="delete from alarm_last_status where server_id not in (select distinct server_id from alarm);"
func.mysql_exec(sql,'')
sql="update alarm_last_status set connect='',status_connections='',status_used_memory='',status_replication_relay='';"
func.mysql_exec(sql,'')
#insert new warn
sql="insert into alarm_last_status(application_id,server_id) select distinct application_id,server_id from alarm where server_id not in (select distinct server_id from alarm_last_status);"
func.mysql_exec(sql,'')
#old warn, update
sql="select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id in (select distinct server_id from alarm_last_status);"
result=func.mysql_query(sql)
if result <> 0:
for line in result:
update_sql="update alarm_last_status set "
application_id=line[0]
server_id=line[1]
alarm_type=line[2]
alarm_value=line[3]
level=line[4]
message=line[5]
send_mail=line[6]
if alarm_type=="connect":
update_sql=update_sql + "connect=\"" + message + "\","
elif alarm_type=="connections":
update_sql=update_sql + "connections='" + alarm_value + "',status_connections='" + message + "',"
elif alarm_type=="memory":
update_sql=update_sql + "used_memory='" + alarm_value + "',status_used_memory='" + message + "',"
elif alarm_type=="delay":
update_sql=update_sql + "replication_relay='" + alarm_value + "',status_replication_relay='" + message + "',"
elif alarm_type=="replication":
update_sql=update_sql + "replication_relay='-1',status_replication_relay='" + message + "',"
update_sql=update_sql + "send_alarm='" + str(send_mail) + "' where application_id='" + str(application_id) + "' and server_id='" + str(server_id) + "';"
log.debug("update_sql:%s"%(update_sql))
func.mysql_exec(update_sql,'')
#should not appear, send alarm to me
sql="select application_id,server_id,alarm_type,alarm_value,level,message,send_mail from alarm where server_id not in (select distinct server_id from alarm_last_status);"
result=func.mysql_query(sql)
if result <> 0:
mailto_list = func.get_option('mail_to_list')
mail_subject="redis monitor error"
mail_content="this should not appear:"+sql
func.send_alarm(0,mailto_list,mail_subject,mail_content.encode('utf-8'))
def main():
print("%s: controller started." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()),))
##全局配置--全局监控
monitor = func.get_option('monitor')
##全局配置--状态监控
monitor_status = func.get_option('monitor_status')
##全局配置--状态监控频率
frequency_monitor = func.get_option('frequency_monitor')
##全局配置--开启告警
alarm = func.get_option('alarm')
###全局配置--报警检测频率
frequency_alarm = func.get_option('frequency_alarm')
##全局配置--冷备管理
data_back_status = func.get_option('data_back_status')
##全局配置--冷备管理频率
frequency_data_back = func.get_option('frequency_data_back')
##全局配置--历史数据自动清理
history_clear_status = func.get_option('history_clear_status')
joblist = []
if monitor=="1":
if monitor_status=="1":
job = Process(target = job_run, args = ('check_redis_status.py',frequency_monitor))
joblist.append(job)
job.start()
if alarm=="1":
job = Process(target = job_run, args = ('alarm_redis.py',frequency_alarm))
joblist.append(job)
job.start()
if data_back_status=="1":
job = Process(target = job_run, args = ('redis_data_back.py',frequency_data_back))
joblist.append(job)
job.start()
if history_clear_status=="1":
job = Process(target = job_run, args = ('redis_history_clear.py',36000))
joblist.append(job)
job.start()
for job in joblist:
job.join();
print("%s: controller finished." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()),))
def main():
print("%s: controller started." % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),))
monitor = func.get_option("monitor")
monitor_status = func.get_option("monitor_status")
monitor_replication = func.get_option("monitor_replication")
monitor_process = func.get_option("monitor_process")
alarm = func.get_option("alarm")
frequency_monitor = func.get_option("frequency_monitor")
frequency_alarm = func.get_option("frequency_alarm")
kill_process = func.get_option("kill_process")
joblist = []
if monitor == "1":
if monitor_status == "1":
job = Process(target=job_run, args=("check_mysql_status_ext.py", frequency_monitor))
joblist.append(job)
job.start()
job = Process(target=job_run, args=("check_mysql_status.py", frequency_monitor))
joblist.append(job)
job.start()
if monitor_replication == "1":
job = Process(target=job_run, args=("check_mysql_replication.py", frequency_monitor))
joblist.append(job)
job.start()
if monitor_process == "1":
job = Process(target=job_run, args=("check_mysql_process.py", 4))
joblist.append(job)
job.start()
if alarm == "1":
job = Process(target=job_run, args=("alarm_mysql.py", frequency_alarm))
joblist.append(job)
job.start()
if kill_process == "1":
job = Process(target=job_run, args=("admin_mysql_kill_process.py", 3))
joblist.append(job)
job.start()
for job in joblist:
job.join()
print("%s: controller finished." % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),))
def main():
print("%s: controller started." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()),))
monitor = func.get_option('monitor')
monitor_status = func.get_option('monitor_status')
monitor_replication = func.get_option('monitor_replication')
monitor_process = func.get_option('monitor_process')
alarm = func.get_option('alarm')
frequency_monitor = func.get_option('frequency_monitor')
frequency_alarm = func.get_option('frequency_alarm')
kill_process = func.get_option('kill_process')
joblist = []
if monitor=="1":
if monitor_status=="1":
job = Process(target = job_run, args = ('check_mysql_status_ext.py',frequency_monitor))
joblist.append(job)
job.start()
job = Process(target = job_run, args = ('check_mysql_status.py',frequency_monitor))
joblist.append(job)
job.start()
if monitor_replication=="1":
job = Process(target = job_run, args = ('check_mysql_replication.py',frequency_monitor))
joblist.append(job)
job.start()
if monitor_process=="1":
job = Process(target = job_run, args = ('check_mysql_process.py',4))
joblist.append(job)
job.start()
if alarm=="1":
job = Process(target = job_run, args = ('alarm_mysql.py',frequency_alarm))
joblist.append(job)
job.start()
if kill_process=="1":
job = Process(target = job_run, args = ('admin_mysql_kill_process.py',3))
joblist.append(job)
job.start()
for job in joblist:
job.join();
print("%s: controller finished." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()),))
def main():
log_level = "info"
log_config = global_logs.CLogConfig(level=log_level)
log_name = "./log/check_redis_status.log"
log_config._path = "%s.%s" % (log_name, func_time.today())
log = global_logs.initLog(log_config, logmodule="status")
if log == None:
print "init log error!"
servers = func.mysql_query(
"select id,host,port,passwd,application_id,status from servers where is_delete=0;"
)
frequency_monitor = func.get_option('frequency_monitor')
if servers:
#print("servers %s" %(servers,));
log.info("check_mysql_status controller started...")
init_table()
plist = []
for row in servers:
server_id = row[0]
host = row[1]
port = row[2]
passwd = row[3]
application_id = row[4]
status = row[5]
if status <> 0:
p = Process(target=check_redis_status,
args=(host, port, passwd, server_id,
application_id, frequency_monitor, log))
plist.append(p)
p.start()
for p in plist:
p.join()
#set replication delay
sql = "delete from redis_replication_last;"
func.mysql_exec(sql, '')
sql = "insert into redis_replication_last select * from redis_replication;"
func.mysql_exec(sql, '')
sql = "update redis_replication a,redis_replication_last b,servers c set a.delay=b.master_repl_offset-a.slave_repl_offset where a.master_host=c.host and a.master_port=c.port and b.server_id=c.id and a.is_slave=1 and b.is_master=1;"
func.mysql_exec(sql, '')
log.info("check_mysql_status controller finished...")
def main():
log_level="info"
log_config=global_logs.CLogConfig(level=log_level)
log_name="./log/check_redis_status.log"
log_config._path="%s.%s"%(log_name,func_time.today())
log=global_logs.initLog(log_config,logmodule="status")
if log == None:
print "init log error!"
servers=func.mysql_query("select id,host,port,passwd,application_id,status from servers where is_delete=0;")
frequency_monitor = func.get_option('frequency_monitor')
if servers:
#print("servers %s" %(servers,));
log.info("check_mysql_status controller started..." );
init_table()
plist = []
for row in servers:
server_id=row[0]
host=row[1]
port=row[2]
passwd=row[3]
application_id=row[4]
status=row[5]
if status <> 0:
p = Process(target = check_redis_status, args = (host,port,passwd,server_id,application_id,frequency_monitor,log))
plist.append(p)
p.start()
for p in plist:
p.join()
#set replication delay
sql="delete from redis_replication_last;"
func.mysql_exec(sql,'')
sql="insert into redis_replication_last select * from redis_replication;"
func.mysql_exec(sql,'')
sql="update redis_replication a,redis_replication_last b,servers c set a.delay=b.master_repl_offset-a.slave_repl_offset where a.master_host=c.host and a.master_port=c.port and b.server_id=c.id and a.is_slave=1 and b.is_master=1;"
func.mysql_exec(sql,'')
log.info("check_mysql_status controller finished..." )
#!/usr/bin/env python
# -*- coding: utf8 -*-
from datetime import *
import global_functions as func
mail_to_list = func.get_option('mail_to_list')
mailto_list = mail_to_list.split(';')
result = func.send_mail(mailto_list, " I hope you can learn", "Beautiful Day")
print result
if result:
send_mail_status = "success"
else:
send_mail_status = "fail"
print "send_mail_status:" + send_mail_status
def send_alarm_mail(idc_type='ts'):
sql = "select alarm.application_id,app.display_name application,alarm.server_id,servers.host,servers.port,alarm.create_time,db_type,alarm_type,alarm_value,level,message,alarm.send_mail,servers.alarm_time,servers.threshold_starttime,servers.threshold_endtime from alarm left join servers on alarm.server_id=servers.id left join application app on servers.application_id=app.id where servers.idc_type='" + str(
idc_type) + "';"
result = func.mysql_query(sql)
if result <> 0:
send_alarm_mail = func.get_option('send_alarm_mail')
mail_to_list = func.get_option('mail_to_list')
mailto_list = mail_to_list.split(';')
global_alarm_type = func.get_config('global_alarm_type', 'alarm_type')
global_alarm_type_list = global_alarm_type.split(',')
for line in result:
application_id = line[0]
application = line[1]
server_id = line[2]
host = line[3]
port = line[4]
create_time = line[5]
db_type = line[6]
alarm_type = line[7]
alarm_value = line[8]
level = line[9]
message = line[10]
send_mail = line[11]
alarm_time = line[12]
threshold_starttime = line[13]
threshold_endtime = line[14]
app_email = func.get_application_alarm_email(str(application_id))
app_email_list = app_email.split(',')
for email_app in app_email_list:
if email_app not in mailto_list:
mailto_list.append(email_app)
else:
pass
if send_alarm_mail == "1":
if send_mail == 1:
mail_subject = message + ' 当前值:' + alarm_value + ' 服务器:' + application + '-' + host + ':' + port + ' 时间:' + create_time.strftime(
'%Y-%m-%d %H:%M:%S')
mail_content = "请检查下!"
now_hour = ("0" + str(datetime.now().hour)
if datetime.now().hour < 10 else str(
datetime.now().hour)) + ":" + (
"0" + str(datetime.now().minute)
if datetime.now().minute < 10 else str(
datetime.now().minute))
if alarm_type not in global_alarm_type_list:
if (int(alarm_time) == 1) and (
threshold_starttime.strip() !=
"") and (threshold_endtime.strip() != "") and (
threshold_endtime > threshold_starttime
) and (now_hour <= threshold_endtime) and (
now_hour >= threshold_starttime):
result_mail = func.send_mail(
mailto_list, mail_subject, mail_content)
elif int(alarm_time) != 1:
result_mail = func.send_mail(
mailto_list, mail_subject, mail_content)
else:
result_mail = False
else:
result_mail = func.send_mail(mailto_list, mail_subject,
mail_content)
if result_mail:
send_mail_status = 1
else:
send_mail_status = 0
else:
send_mail_status = 0
else:
send_mail_status = 0
if send_mail_status == 1:
func.mysql_exec(
"insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status,idc_type) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1,'"
+ str(idc_type) + "' from alarm;", '')
elif send_mail_status == 0:
func.mysql_exec(
"insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status,idc_type) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0,'"
+ str(idc_type) + "' from alarm;", '')
func.mysql_exec(
"delete from alarm where idc_type='" + str(idc_type) + "'", '')
else:
pass
def send_warn():
#server_id not delete???
sql="select alarm_last_status.application_id,app.display_name application,alarm_last_status.server_id,servers.host,servers.port,alarm_last_status.modify_time,servers.alarm_interval,servers.alarm_type,servers.converge_type,servers.alarm_person,servers.send_mail,alarm_last_status.connect,alarm_last_status.status_connections,alarm_last_status.status_used_memory,alarm_last_status.status_binlog_count,alarm_last_status.status_innodb_space_free,alarm_last_status.status_replication_relay,alarm_last_status.alarm_num,alarm_last_status.time_error_continue,alarm_last_status.slow_querys,alarm_last_status.status_slow_querys,alarm_last_status.error_log,alarm_last_status.status_error_log from alarm_last_status left join servers on alarm_last_status.server_id=servers.id left join application app on servers.application_id=app.id;"
result=func.mysql_query(sql)
if result <> 0:
domain="www.oa.com"
if(pingServerCall(domain)==False):
#系统告警接收人
mailto_list = func.get_option('mail_to_list')
mail_subject="redis monitor error"
mail_content="ping failed:"+domain +",do not send warn..."
func.send_alarm(0,mailto_list,mail_subject,mail_content.encode('utf-8'))
return None
#全局配置--发送告警
send_alarm_mail = func.get_option('send_alarm_mail')
#全局配置--报警检测
frequency_alarm = func.get_option('frequency_alarm')
#mail_to_list = func.get_option('mail_to_list')
#mailto_list=mail_to_list.split(';')
for line in result:
application_id=line[0]
application=line[1]
server_id=line[2]
host=line[3]
port=line[4]
create_time=line[5]
alarm_interval=line[6]
alarm_type=line[7]
converge_type=line[8]
alarm_person=line[9]
send_alarm=line[10]
connect=line[11]
status_connections=line[12]
status_used_memory=line[13]
status_binlog_count=line[14]
status_innodb_space_free=line[15]
status_replication_relay=line[16]
alarm_num=line[17]
time_error_continue=line[18]
slow_querys=line[19]
status_slow_querys=line[20]
error_log=line[21]
status_error_log=line[22]
warn_content=application+'-'+host+':'+port
eStr="ERROR:"
wStr="WARN:"
if (eStr in connect) or (wStr in connect):
warn_content=warn_content+' '+connect
else:
if (eStr in status_connections) or (wStr in status_connections):
warn_content=warn_content+' '+status_connections
if (eStr in status_used_memory) or (wStr in status_used_memory):
warn_content=warn_content+' '+status_used_memory
if (eStr in status_binlog_count) or (wStr in status_binlog_count):
warn_content=warn_content+' '+status_binlog_count
if (eStr in status_innodb_space_free) or (wStr in status_innodb_space_free):
warn_content=warn_content+' '+status_innodb_space_free
if (eStr in status_replication_relay) or (wStr in status_replication_relay):
warn_content=warn_content+' '+status_replication_relay
if (eStr in status_slow_querys) or (wStr in status_slow_querys):
warn_content=warn_content+' '+status_slow_querys
if (eStr in status_error_log) or (wStr in status_error_log):
warn_content=warn_content+' '+status_error_log
warn_content=warn_content+' ['+create_time.strftime('%Y-%m-%d %H:%M:%S')+']'
#warn_content=warn_content.encode('utf-8')
#print send_alarm_mail
#print send_alarm
#print warn_content
#print("WARN:%s send alarm:%s alarm:%s" %(warn_content,send_alarm_mail,send_alarm))
mailto_list=alarm_person
mail_subject="REDIS WARNING"
send_mail_status=0
if send_alarm_mail=='1':
if send_alarm==1:
next_alarm_time=getTimeInterVal(converge_type,alarm_interval,alarm_num,0);
if(func.test_flag):
log.debug("time_error_continue:%s next_alarm_time:%s converge_type:%s alarm_interval:%s alarm_num:%s"%(str(time_error_continue),str(next_alarm_time),converge_type,str(alarm_interval),str(alarm_num)))
if time_error_continue >= next_alarm_time:
result=func.send_alarm(alarm_type,mailto_list,mail_subject,warn_content.encode('utf-8'))
if result:
send_mail_status=1
else:
send_mail_status=0
sql="update alarm_last_status set alarm_num=alarm_num+1 where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"'"
func.mysql_exec(sql,'')
log.warning("send alarm:alarm_type:%s mailto_list:%s mail_subject:%s warn_content:%s"%(str(alarm_type),mailto_list,mail_subject,warn_content.encode('utf-8')))
else:
sql="delete from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"'"
func.mysql_exec(sql,'')
else:
send_mail_status=0
else:
send_mail_status=0
sql="update alarm_last_status set time_error_continue=time_error_continue+"+str(frequency_alarm)+" where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"'"
func.mysql_exec(sql,'')
#param=(str(application_id),str(server_id))
if send_mail_status==1:
sql="insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1 from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"';"
func.mysql_exec(sql,'')
elif send_mail_status==0:
sql="insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0 from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"';"
func.mysql_exec(sql,'')
sql="delete from alarm where application_id='"+str(application_id)+"' and server_id='"+str(server_id)+"';"
func.mysql_exec(sql,'')
else:
pass
def send_warn():
#server_id not delete???
sql = "select alarm_last_status.application_id,app.display_name application,alarm_last_status.server_id,servers.host,servers.port,alarm_last_status.modify_time,servers.alarm_interval,servers.alarm_type,servers.converge_type,servers.alarm_person,servers.send_mail,alarm_last_status.connect,alarm_last_status.status_connections,alarm_last_status.status_used_memory,alarm_last_status.status_binlog_count,alarm_last_status.status_innodb_space_free,alarm_last_status.status_replication_relay,alarm_last_status.alarm_num,alarm_last_status.time_error_continue,alarm_last_status.slow_querys,alarm_last_status.status_slow_querys,alarm_last_status.error_log,alarm_last_status.status_error_log from alarm_last_status left join servers on alarm_last_status.server_id=servers.id left join application app on servers.application_id=app.id;"
result = func.mysql_query(sql)
if result <> 0:
domain = "www.oa.com"
if (pingServerCall(domain) == False):
#系统告警接收人
mailto_list = func.get_option('mail_to_list')
mail_subject = "redis monitor error"
mail_content = "ping failed:" + domain + ",do not send warn..."
func.send_alarm(0, mailto_list, mail_subject,
mail_content.encode('utf-8'))
return None
#全局配置--发送告警
send_alarm_mail = func.get_option('send_alarm_mail')
#全局配置--报警检测
frequency_alarm = func.get_option('frequency_alarm')
#mail_to_list = func.get_option('mail_to_list')
#mailto_list=mail_to_list.split(';')
for line in result:
application_id = line[0]
application = line[1]
server_id = line[2]
host = line[3]
port = line[4]
create_time = line[5]
alarm_interval = line[6]
alarm_type = line[7]
converge_type = line[8]
alarm_person = line[9]
send_alarm = line[10]
connect = line[11]
status_connections = line[12]
status_used_memory = line[13]
status_binlog_count = line[14]
status_innodb_space_free = line[15]
status_replication_relay = line[16]
alarm_num = line[17]
time_error_continue = line[18]
slow_querys = line[19]
status_slow_querys = line[20]
error_log = line[21]
status_error_log = line[22]
warn_content = application + '-' + host + ':' + port
eStr = "ERROR:"
wStr = "WARN:"
if (eStr in connect) or (wStr in connect):
warn_content = warn_content + ' ' + connect
else:
if (eStr in status_connections) or (wStr
in status_connections):
warn_content = warn_content + ' ' + status_connections
if (eStr in status_used_memory) or (wStr
in status_used_memory):
warn_content = warn_content + ' ' + status_used_memory
if (eStr in status_binlog_count) or (wStr
in status_binlog_count):
warn_content = warn_content + ' ' + status_binlog_count
if (eStr in status_innodb_space_free) or (
wStr in status_innodb_space_free):
warn_content = warn_content + ' ' + status_innodb_space_free
if (eStr in status_replication_relay) or (
wStr in status_replication_relay):
warn_content = warn_content + ' ' + status_replication_relay
if (eStr in status_slow_querys) or (wStr
in status_slow_querys):
warn_content = warn_content + ' ' + status_slow_querys
if (eStr in status_error_log) or (wStr in status_error_log):
warn_content = warn_content + ' ' + status_error_log
warn_content = warn_content + ' [' + create_time.strftime(
'%Y-%m-%d %H:%M:%S') + ']'
#warn_content=warn_content.encode('utf-8')
#print send_alarm_mail
#print send_alarm
#print warn_content
#print("WARN:%s send alarm:%s alarm:%s" %(warn_content,send_alarm_mail,send_alarm))
mailto_list = alarm_person
mail_subject = "REDIS WARNING"
send_mail_status = 0
if send_alarm_mail == '1':
if send_alarm == 1:
next_alarm_time = getTimeInterVal(converge_type,
alarm_interval,
alarm_num, 0)
if (func.test_flag):
log.debug(
"time_error_continue:%s next_alarm_time:%s converge_type:%s alarm_interval:%s alarm_num:%s"
% (str(time_error_continue), str(next_alarm_time),
converge_type, str(alarm_interval),
str(alarm_num)))
if time_error_continue >= next_alarm_time:
result = func.send_alarm(alarm_type, mailto_list,
mail_subject,
warn_content.encode('utf-8'))
if result:
send_mail_status = 1
else:
send_mail_status = 0
sql = "update alarm_last_status set alarm_num=alarm_num+1 where application_id='" + str(
application_id) + "' and server_id='" + str(
server_id) + "'"
func.mysql_exec(sql, '')
log.warning(
"send alarm:alarm_type:%s mailto_list:%s mail_subject:%s warn_content:%s"
% (str(alarm_type), mailto_list, mail_subject,
warn_content.encode('utf-8')))
else:
sql = "delete from alarm where application_id='" + str(
application_id) + "' and server_id='" + str(
server_id) + "'"
func.mysql_exec(sql, '')
else:
send_mail_status = 0
else:
send_mail_status = 0
sql = "update alarm_last_status set time_error_continue=time_error_continue+" + str(
frequency_alarm) + " where application_id='" + str(
application_id) + "' and server_id='" + str(
server_id) + "'"
func.mysql_exec(sql, '')
#param=(str(application_id),str(server_id))
if send_mail_status == 1:
sql = "insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1 from alarm where application_id='" + str(
application_id) + "' and server_id='" + str(
server_id) + "';"
func.mysql_exec(sql, '')
elif send_mail_status == 0:
sql = "insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0 from alarm where application_id='" + str(
application_id) + "' and server_id='" + str(
server_id) + "';"
func.mysql_exec(sql, '')
sql = "delete from alarm where application_id='" + str(
application_id) + "' and server_id='" + str(server_id) + "';"
func.mysql_exec(sql, '')
else:
pass
def send_alarm_message():
sql = "select alarm.application_id,app.display_name application,alarm.server_id,servers.host,servers.port,alarm.create_time,db_type,alarm_type,alarm_value,level,message,alarm.send_message from alarm left join servers on alarm.server_id=servers.id left join application app on servers.application_id=app.id;"
result = func.mysql_query(sql)
if result <> 0:
send_alarm_message = func.get_option('send_alarm_message')
phone = func.get_option('message_to_list')
for line in result:
application_id = line[0]
application = line[1]
server_id = [2]
host = line[3]
port = line[4]
create_time = line[5]
db_type = line[6]
alarm_type = line[7]
alarm_value = line[8]
level = line[9]
message = line[10]
send_message = line[11]
send_message_status = 0
if send_alarm_message == "1":
if send_message == 1:
phone_list = phone.split(';')
all_send_result = []
for i in phone_list:
values = {}
values['phone'] = i
values['type'] = 7
values['signname'] = '小店监控'
values['server'] = host
values['errmsg'] = message
result = func.send_message(values)
all_send_result.append(result)
all_flag = True
# 有一人收到 成功发送
for index, res in enumerate(all_send_result):
if res == True:
all_flag = True
if all_flag:
send_message_status = 1
else:
send_message_status = 0
else:
send_message_status = 0
else:
send_message_status = 0
if send_message_status == 1:
func.mysql_exec(
"insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_message,send_message_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_message,1 from alarm;",
'')
elif send_message_status == 0:
func.mysql_exec(
"insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_message,send_message_status) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_message,0 from alarm;",
'')
func.mysql_exec("delete from alarm", '')
else:
pass
#!/usr/bin/env python
# -*- coding: utf8 -*-
from datetime import *
import global_functions as func
mail_to_list = func.get_option('mail_to_list')
mailto_list=mail_to_list.split(';')
result = func.send_mail(mailto_list," I hope you can learn","Beautiful Day")
print result
if result:
send_mail_status = "success"
else:
send_mail_status = "fail"
print "send_mail_status:"+send_mail_status
#!/usr/bin/env python
# -*- coding: utf8 -*-
from datetime import *
import global_functions as func
phone = func.get_option('message_to_list')
values = {}
values['phone'] = phone
values['type'] = 7
values['signname'] = '小店监控'
values['server'] = '10.18.1.1'
values['errmsg'] = '测试报警功能'
result = func.send_message(values)
print "发送结果:"
print result
print type(result)
def send_alarm_mail(idc_type='ts'):
sql="select alarm.application_id,app.display_name application,alarm.server_id,servers.host,servers.port,alarm.create_time,db_type,alarm_type,alarm_value,level,message,alarm.send_mail,servers.alarm_time,servers.threshold_starttime,servers.threshold_endtime from alarm left join servers on alarm.server_id=servers.id left join application app on servers.application_id=app.id where servers.idc_type='"+str(idc_type)+"';"
result=func.mysql_query(sql)
if result <> 0:
send_alarm_mail = func.get_option('send_alarm_mail')
mail_to_list = func.get_option('mail_to_list')
mailto_list=mail_to_list.split(';')
global_alarm_type=func.get_config('global_alarm_type','alarm_type')
global_alarm_type_list=global_alarm_type.split(',')
for line in result:
application_id=line[0]
application=line[1]
server_id=line[2]
host=line[3]
port=line[4]
create_time=line[5]
db_type=line[6]
alarm_type=line[7]
alarm_value=line[8]
level=line[9]
message=line[10]
send_mail=line[11]
alarm_time=line[12]
threshold_starttime=line[13]
threshold_endtime=line[14]
app_email=func.get_application_alarm_email(str(application_id))
app_email_list=app_email.split(',')
for email_app in app_email_list:
if email_app not in mailto_list:
mailto_list.append(email_app)
else:
pass
if send_alarm_mail=="1":
if send_mail==1:
mail_subject=message+' 当前值:'+alarm_value+' 服务器:'+application+'-'+host+':'+port+' 时间:'+create_time.strftime('%Y-%m-%d %H:%M:%S')
mail_content="请检查下!"
now_hour = ("0"+str(datetime.now().hour) if datetime.now().hour<10 else str(datetime.now().hour))+":"+( "0"+str(datetime.now().minute) if datetime.now().minute<10 else str(datetime.now().minute))
if alarm_type not in global_alarm_type_list:
if (int(alarm_time) == 1) and (threshold_starttime.strip() != "") and (threshold_endtime.strip() != "") and (threshold_endtime > threshold_starttime) and (now_hour <= threshold_endtime) and (now_hour >= threshold_starttime):
result_mail = func.send_mail(mailto_list,mail_subject,mail_content)
elif int(alarm_time) != 1:
result_mail = func.send_mail(mailto_list,mail_subject,mail_content)
else:
result_mail = False
else:
result_mail = func.send_mail(mailto_list,mail_subject,mail_content)
if result_mail:
send_mail_status=1
else:
send_mail_status=0
else:
send_mail_status=0
else:
send_mail_status=0
if send_mail_status==1:
func.mysql_exec("insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status,idc_type) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,1,'"+str(idc_type)+"' from alarm;",'')
elif send_mail_status==0:
func.mysql_exec("insert into alarm_history(application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,send_mail_status,idc_type) select application_id,server_id,create_time,db_type,alarm_type,alarm_value,level,message,send_mail,0,'"+str(idc_type)+"' from alarm;",'')
func.mysql_exec("delete from alarm where idc_type='"+str(idc_type)+"'",'')
else:
pass
