def fpga_task():
linuxInfo = ct.get_server_config('./config/fpga_config.txt')
# linuxInfo = [['192.168.238.7', 22, 'trade', 'trade', 'tradeserver',/home/trade/FPGA']]
try:
ms = MonitorServer(linuxInfo)
ms.fpga_monitor_run()
check_result_list = ms.fpga_Check_flag_list
logger.debug("check_result_list:")
logger.debug(check_result_list)
if len(check_result_list) == 0:
check_result = False
else:
check_result = (sum(check_result_list) == len(check_result_list))
except Exception:
check_result = False
logger.error('Faild to check fpga!', exc_info=True)
logger.info(
"************************The FPGA File Monitor Result: ************************"
)
if check_result:
logger.info("OK,FPGA Server is OK")
ct.send_sms_control("fpga", "OK:盘前FPGA文件检查正常")
else:
msg = "error:FPGA文件检查异常，请查看详细日志"
logger.error(msg)
# send_sms_control("fpga", "error:奇点服务器FPGA文件检查异常，请查看详细日志")
sysstr = platform.system()
if sysstr == "Windows":
ct.readTexts("VIP Server is Worning")
def before_cleanup_db_monitor_task():
with open('./config/table_check.json', 'r') as f:
Jsonlist = json.load(f)
logger.debug(Jsonlist)
thrlist = range(len(Jsonlist))
threads=[]
for (i,info) in zip(thrlist, Jsonlist):
#print("alltask.__name__:", alltask.__name__)
t = dbm.MyThread(dbm.before_cleanup_db_monitor,(info,),dbm.before_cleanup_db_monitor.__name__ + str(i))
threads.append(t)
for i in thrlist:
threads[i].start()
for i in thrlist:
threads[i].join()
threadResult = threads[i].get_result()
sysstr = platform.system()
if (not threadResult) :
logger.error("error:数据库盘后清库盘前检查失败，请检查详细错误信息")
ct.send_sms_control("NoLimit", "error:数据库盘后清库盘前检查失败，请检查详细错误信息")
if (sysstr == "Windows"):
ct.readTexts("Database cleanup Worning")
else:
logger.info("OK:数据库盘后清库盘前检查正常")
def ping_monitor_task():
linuxInfo = ct.get_server_config('./config/server_ping_config.txt')
try:
ms = MonitorServer(linuxInfo)
ms.ping_server_monitor()
check_result_list = ms.ping_Check_flag_list
if len(check_result_list)==0:
check_result =False
else:
check_result = (sum(check_result_list)==len(check_result_list))
# print "check_result: ", check_result
except Exception as e:
check_result = False
msg = str(e)
logger.warning(msg)
if check_result:
logger.info("All Server is OK")
else:
msg = "服务器Ping值异常，请检查详细日志内容！"
logger.warning(msg)
# ct.send_sms_control("ping", msg)
sysstr = platform.system()
if sysstr == "Windows":
ct.readTexts("Ping Monitor is Worning")
def mem_monitor_task():
linuxInfo = ct.get_server_config('./config/server_status_config.txt')
try:
ms = MonitorServer(linuxInfo)
ms.mem_monitor()
check_result_list = ms.mem_Check_flag_list
if len(check_result_list) == 0:
check_result = False
else:
check_result = (sum(check_result_list) == len(check_result_list))
# print "check_result: ", check_result
except Exception as e:
check_result = False
msg = str(e)
logger.error(msg)
logger.info(
"************************The Monitor Result: ************************")
if check_result:
logger.info("All Server is OK")
else:
msg = "内存监控报警，请查看服务器详细信息"
logger.error(msg)
sysstr = platform.system()
if sysstr == "Windows":
ct.readTexts("Memory Monitor is Worning")
ct.generate_file(check_result, 'Memory_Monitor')
def common_monitor_task(task, single_handle, linuxInfo):
#linuxInfo = ct.get_server_config('./config/server_status_config.txt')
try:
#single_handle = "core_file_info"
ms = MonitorServer(linuxInfo, single_handle)
task_monitor = ms.single_common_monitor
task_monitor()
check_result_list = ms.common_Check_flag_list
if len(check_result_list)==0:
check_result =False
else:
check_result = (sum(check_result_list)==len(check_result_list))
# print "check_result: ", check_result
except Exception as e:
check_result = False
msg = str(e)
logger.warning(msg)
if check_result:
msg = "OK,监控任务[%s]验证结果成功,All Server is OK！" % single_handle
logger.info(msg)
else:
msg = task + "::" + single_handle + " 任务监控报警，请检查详细日志内容！"
logger.warning(msg)
# ct.send_sms_control("disk", msg)
sysstr = platform.system()
if sysstr == "Windows":
ct.readTexts("Monitor task is Worning")
def db_init_monitor_task():
with open('./config/table_check.json', 'r') as f:
Jsonlist = json.load(f)
logger.debug(Jsonlist)
logger.info("Start to excute the before trade monitor")
thrlist = range(len(Jsonlist))
threads = []
for (i, info) in zip(thrlist, Jsonlist):
#print("alltask.__name__:", alltask.__name__)
t = dbm.MyThread(dbm.before_trade_monitor, (info, ),
dbm.before_trade_monitor.__name__ + str(i))
threads.append(t)
for i in thrlist:
threads[i].start()
for i in thrlist:
threads[i].join()
threadResult = threads[i].get_result()
sysstr = platform.system()
if (not threadResult):
logger.error("error:奇点服务器盘前数据库初始化数据错误，请检查详细错误信息")
# ct.send_sms_control("db_init", "error:奇点服务器盘前数据库初始化数据错误，请检查详细错误信息")
if (sysstr == "Windows"):
ct.readTexts("Database init Worning")
else:
logger.info("OK:奇点服务器数据库init检查正常")
ct.send_sms_control("db_init", "OK:奇点服务器盘前数据库init检查正常")
def port_process_task():
linuxInfo = ct.get_server_config('./config/server_status_config.txt')
try:
ms = MonitorServer(linuxInfo)
ms.socket_ps_monitor()
check_result_list = ms.SocPs_Check_flag_list
if len(check_result_list) == 0:
check_result = False
else:
check_result = (sum(check_result_list) == len(check_result_list))
# print "check_result: ", check_result
except Exception as e:
check_result = False
msg = str(e)
logger.warning(msg)
logger.info(
"************************The Monitor Result: ************************")
if check_result:
logger.info("All Server is OK")
else:
msg = "端口和进程监控报警，请查看服务器详细信息!"
logger.error(msg)
sysstr = platform.system()
if sysstr == "Windows":
ct.readTexts("Port Process Monitor is Worning")
ct.generate_file(check_result, 'Port_PS_Monitor')
def db_trade_monitor_task():
with open('./config/table_check.json', 'r') as f:
Jsonlist = json.load(f)
logger.debug(Jsonlist)
# if os.path.isdir("./tempdata"):
# for filename in os.listdir('./tempdata'):
# os.remove('./tempdata/' + filename)
# else:
# os.mkdir("./tempdata")
logger.info("Start to excute the trading_monitor")
thrlist = range(len(Jsonlist))
threads = []
for (i, info) in zip(thrlist, Jsonlist):
t = dbm.MyThread(dbm.trading_monitor, (info, ),
dbm.trading_monitor.__name__ + str(i))
threads.append(t)
# print "thrcouat3:", threading.active_count()
for i in thrlist:
threads[i].start()
for i in thrlist:
threads[i].join()
trading_check = threads[i].get_result()
sysstr = platform.system()
if (not trading_check):
logger.error("数据库交易中监控报警，请查看服务器详细信息")
if (sysstr == "Windows"):
ct.readTexts("Database trading Worning")
def alltask():
linuxInfo = ct.get_server_config('./config/server_status_config.txt')
try:
ms = MonitorServer(linuxInfo)
ms.monitor_run()
check_result_list = ms.Check_flag_list
# print "check_result_list:", check_result_list
if len(check_result_list) == 0:
check_result = False
else:
check_result = (sum(check_result_list) == len(check_result_list))
# print "check_result: ", check_result
except Exception as e:
check_result = False
msg = str(e)
logger.error(msg)
logger.info(
"************************The Monitor Result: ************************")
if check_result:
logger.info("All Server is OK")
else:
msg = "error: Basic Monitor is Worning, Please Check it!"
logger.error(msg)
# ct.fortunesms(msg)
sysstr = platform.system()
if sysstr == "Windows":
ct.readTexts("Basic Monitor is Worning")
# else:
# ct.send_mail(msg,"Failed_Basic_Monitor")
ct.generate_file(check_result, 'Basic_Monitor')
def errorLog_monitor_task():
linuxInfo = ct.get_server_config('./config/server_logDir_config.txt')
grep_lists = mel.get_errorLog(linuxInfo)
fileNlist = mel.get_result_file_list()
check_flag = mel.errorLog_check(fileNlist, grep_lists)
sysstr = platform.system()
if (not check_flag):
logger.error("错误日志检查报警，请查看服务器详细信息")
# send_sms_control("errorLog", "error:Server log error warning")
if (sysstr == "Windows"):
ct.readTexts("Server log warning")
def monitor_errorLog_run(argv):
#备份上一次的grep_result_file文件到back_file
if (os.path.exists(grep_result_file)):
os.rename(grep_result_file, back_file)
yaml_path = './config/monitor_errorLog_logger.yaml'
ct.setup_logging(yaml_path)
#init interval
inc = 60
try:
opts, args = getopt.getopt(argv, "hl:", ["loopsecends="])
except getopt.GetoptError:
print('monitor_errorLog.py -l <loopsecends>')
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('monitor_errorLog.py -l <loopsecends> \n \
loopsecends=0 means no loop and just run once.\n \
loopsecends=N means loop interval is N second. \n \
(default:python monitor_errorLog.py) means loop interval is 60 seconds'
)
sys.exit()
elif opt in ("-l", "--loopsecends"):
inc = int(arg)
print('interval is: ', inc)
if inc == 0:
grep_lists = get_errorLog(linuxInfo)
fileNlist = get_result_file_list()
check_flag = errorLog_check(fileNlist, grep_lists)
sysstr = platform.system()
if (not check_flag) and (sysstr == "Windows"):
ct.readTexts("Server log warning")
else:
while True:
# 执行方法，函数
start_time = '05:45'
end_time = '19:25'
if (ct.time_check(start_time, end_time)):
grep_lists = get_errorLog(linuxInfo)
fileNlist = get_result_file_list()
check_flag = errorLog_check(fileNlist, grep_lists)
sysstr = platform.system()
if (not check_flag) and (sysstr == "Windows"):
ct.readTexts("Server log warning")
time.sleep(inc)
def errorLog_monitor_task():
linuxInfo = ct.get_server_config('./config/server_logDir_config.txt')
ntimes = dt.datetime.now().strftime("%Y%m%d%H%M%S")
ndates = dt.datetime.now().strftime("%Y%m%d")
#cur_dir_i = os.getcwd()
cur_dir = os.getcwd().replace("\\", "/") + "/"
log_dir = cur_dir + "mylog/"
grep_result_file = log_dir + "errorLog_result_" + ndates + '.txt'
back_file = log_dir + "errorLog_result_" + ntimes + '.txt'
#备份上一次的grep_result_file文件到back_file
if (os.path.exists(grep_result_file)):
os.rename(grep_result_file, back_file)
grep_lists = mel.get_errorLog(linuxInfo)
fileNlist = mel.get_result_file_list()
check_flag = mel.errorLog_check(fileNlist, grep_lists)
sysstr = platform.system()
if (not check_flag):
logger.error("错误日志检查报警，请查看服务器详细信息")
# send_sms_control("errorLog", "error:Server log error warning")
if (sysstr == "Windows"):
ct.readTexts("Server log warning")
def fpgatask():
linuxInfo = ct.get_server_config('./config/fpga_config.txt')
# linuxInfo = [['192.168.238.7', 22, 'trade', 'trade', 'tradeserver',/home/trade/FPGA']]
try:
ms = MonitorServer(linuxInfo)
ms.fpga_monitor_run()
check_result_list = ms.fpga_Check_flag_list
logger.debug("check_result_list:")
logger.debug(check_result_list)
if len(check_result_list) == 0:
check_result = False
else:
check_result = (sum(check_result_list) == len(check_result_list))
except Exception as e:
check_result = False
logger.error('Faild to check fpga!', exc_info=True)
logger.info(
"************************The FPGA File Monitor Result: ************************"
)
if check_result:
logger.info("FPGA Server is OK")
else:
msg = "error:FPGA is Exception, Please Check it!"
logger.error(msg)
# ct.fortunesms(msg)
sysstr = platform.system()
if sysstr == "Windows":
ct.readTexts("VIP Server is Worning")
# else:
# ct.send_mail(msg,"Failed_FPGA_Monitor")
ct.generate_file(check_result, 'FPAG_File_Monitor')
def main(argv):
try:
yaml_path = './config/db_check_logger.yaml'
ct.setup_logging(yaml_path)
with open('./config/table_check.json', 'r') as f:
Jsonlist = json.load(f)
logger.debug(Jsonlist)
#init interval
inc = 600
modul = ''
try:
opts, args = getopt.getopt(argv, "hl:e:",
["loopsecends=", "excute="])
except getopt.GetoptError:
print('db_monitor.py -l <loopsecends> -e <excute>')
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('db_monitor.py -l <loopsecends> -e <excute>\n \
loopsecends=0 means no loop and just run once.\n \
loopsecends=N means loop interval is N second. \n \
(default:python db_monitor.py) means loop interval is 600 seconds. \n \
excute=before means excute the before trade db monitor. \n \
excute=trading means excute the trading db monitor. \n \
excute is Null means excute before and trading db monitor.'
)
sys.exit()
elif opt in ("-l", "--loopsecends"):
inc = int(arg)
elif opt in ("-e", "--excute"):
modul = arg
logger.info('interval is: %d' % inc)
logger.info('modull is:s: %s' % modul)
if inc == 0:
if modul == 'before' or modul == '':
#before trade montior
logger.info("Start to excute the before trade monitor")
thrlist = range(len(Jsonlist))
threads = []
for (i, info) in zip(thrlist, Jsonlist):
#print("alltask.__name__:", alltask.__name__)
t = MyThread(before_trade_monitor, (info, ),
before_trade_monitor.__name__ + str(i))
threads.append(t)
for i in thrlist:
threads[i].start()
for i in thrlist:
threads[i].join()
threadResult = threads[i].get_result()
sysstr = platform.system()
if (not threadResult) and (sysstr == "Windows"):
ct.readTexts("Database trade before Worning")
if modul == 'trading' or modul == '':
#trading monitor
#delete all tempdata/*.json
if os.path.isdir("./tempdata"):
for filename in os.listdir('./tempdata'):
os.remove('./tempdata/' + filename)
else:
os.mkdir("./tempdata")
logger.info("Start to excute the trading_monitor")
thrlist = range(len(Jsonlist))
threads = []
for (i, info) in zip(thrlist, Jsonlist):
t = MyThread(trading_monitor, (info, ),
trading_monitor.__name__ + str(i))
threads.append(t)
# print "thrcouat3:", threading.active_count()
for i in thrlist:
threads[i].start()
for i in thrlist:
threads[i].join()
trading_check = threads[i].get_result()
if (not trading_check) and (sysstr == "Windows"):
ct.readTexts("Database trading Worning")
elif inc > 20:
#before trade montior
logger.info("Start to excute the before trade monitor")
thrlist = range(len(Jsonlist))
threads = []
for (i, info) in zip(thrlist, Jsonlist):
#print("alltask.__name__:", alltask.__name__)
t = MyThread(before_trade_monitor, (info, ),
before_trade_monitor.__name__ + str(i))
threads.append(t)
for i in thrlist:
threads[i].start()
for i in thrlist:
threads[i].join()
threadResult = threads[i].get_result()
sysstr = platform.system()
if (not threadResult) and (sysstr == "Windows"):
ct.readTexts("Database trade before Worning")
# print "thrcount:", threading.active_count()
#trading monitor
#delete all tempdata/*.json
if os.path.isdir("./tempdata"):
for filename in os.listdir('./tempdata'):
os.remove('./tempdata/' + filename)
else:
os.mkdir("./tempdata")
while True:
# while False:
if (ct.trade_check()):
logger.info("Start to excute the trading_monitor")
thrlist = range(len(Jsonlist))
threads = []
for (i, info) in zip(thrlist, Jsonlist):
t = MyThread(trading_monitor, (info, ),
trading_monitor.__name__ + str(i))
threads.append(t)
for i in thrlist:
threads[i].start()
for i in thrlist:
threads[i].join()
trading_check = threads[i].get_result()
if (not trading_check) and (sysstr == "Windows"):
ct.readTexts("Database trading Worning")
# print "thrcouat3:", threading.active_count()
time.sleep(inc - 20)
if (ct.time_check('15:00', '15:12')):
logger.info("exit to monitor")
break
else:
logger.info("It's not time to trading monitor")
time.sleep(20)
else:
logger.error(
"Input parameter error: The interval must greater than 20!")
except Exception:
logger.error('Faild to run monitor db!', exc_info=True)
finally:
for handler in logger.handlers:
logger.removeHandler(handler)