def follow_monitor_task():
linuxInfo = ct.get_server_config('./config/check_follow_config.txt')
check_flag = 0
for info in linuxInfo:
hostip = info[0]
try:
file_checker = rfc.remote_file_check(info)
res_file = file_checker.check_follow()
if res_file[:5] != 'Error':
msg = "ok:系统 %s 跟投费率优惠文件检查成功，文件名[%s]" % (hostip,res_file)
logger.info(msg)
check_flag += 1
#20200526正常状态不再发短信
#ct.send_sms_control('NoLimit', msg)
else:
msg = "Error:系统 %s 跟投费率优惠文件检查失败 失败的原因：%s" % (hostip, res_file[6:])
logger.error(msg)
ct.send_sms_control('NoLimit', msg)
except Exception:
msg = "Error:系统 %s 跟投费率优惠文件检查失败，出现异常，请查看服务器日志信息！" % hostip
logger.error(msg, exc_info=True)
ct.send_sms_control('NoLimit', msg)
if check_flag != 0 and check_flag == len(linuxInfo):
logger.info(u"OK:跟投费率优惠文件数据成功!")
else:
logger.warning(u"Error:跟投费率优惠文件检查数据失败!")
def alltask():
linuxInfo = ct.get_server_config('./config/server_status_config.txt')
try:
ms = MonitorServer(linuxInfo)
ms.monitor_run()
check_result_list = ms.Check_flag_list
# print "check_result_list:", check_result_list
if len(check_result_list) == 0:
check_result = False
else:
check_result = (sum(check_result_list) == len(check_result_list))
# print "check_result: ", check_result
except Exception as e:
check_result = False
msg = str(e)
logger.error(msg)
logger.info(
"************************The Monitor Result: ************************")
if check_result:
logger.info("All Server is OK")
else:
msg = "error: Basic Monitor is Worning, Please Check it!"
logger.error(msg)
# ct.fortunesms(msg)
sysstr = platform.system()
if sysstr == "Windows":
ct.readTexts("Basic Monitor is Worning")
# else:
# ct.send_mail(msg,"Failed_Basic_Monitor")
ct.generate_file(check_result, 'Basic_Monitor')
def sjdr_monitor_task():
linuxInfo = ct.get_server_config('./config/check_sjdr_config.txt')
check_flag = 0
for info in linuxInfo:
try:
hostip = info[0]
file_checker = rfc.remote_file_check(info)
error_list = file_checker.check_sdjr()
if len(error_list) == 0:
msg = "ok:系统 %s 盘后当天的节点委托回传数据检查成功" % hostip
logger.info(msg)
check_flag += 1
else:
msg = "系统 %s 盘后当天的节点委托回传数据检查失败 失败的文件列表：%s " % (hostip, ';'.join(error_list))
logger.error(msg)
ct.send_sms_control('NoLimit', msg)
except Exception:
logger.error("查询席位代码失败，出现异常，请查看服务器日志信息！", exc_info=True)
ct.send_sms_control('xwdm', "查询席位代码失败，出现异常，请查看服务器日志信息！")
if check_flag != 0 and check_flag == len(linuxInfo):
logger.info(u"OK:检查盘后当天的节点委托回传数据成功!")
else:
logger.warning(u"Error:检查盘后当天的节点委托回传数据失败!")
def ping_monitor_task():
linuxInfo = ct.get_server_config('./config/server_ping_config.txt')
try:
ms = MonitorServer(linuxInfo)
ms.ping_server_monitor()
check_result_list = ms.ping_Check_flag_list
if len(check_result_list)==0:
check_result =False
else:
check_result = (sum(check_result_list)==len(check_result_list))
# print "check_result: ", check_result
except Exception as e:
check_result = False
msg = str(e)
logger.warning(msg)
if check_result:
logger.info("All Server is OK")
else:
msg = "服务器Ping值异常，请检查详细日志内容！"
logger.warning(msg)
# ct.send_sms_control("ping", msg)
sysstr = platform.system()
if sysstr == "Windows":
ct.readTexts("Ping Monitor is Worning")
def exchange_file_monitor_task():
linuxInfo = ct.get_server_config('./config/exchange_file_config.txt')
check_flag = 0
for info in linuxInfo:
hostip = info[0]
try:
file_checker = rfc.remote_file_check(info)
error_file_list = file_checker.check_exchange_file()
if len(error_file_list) == 0:
msg = "ok:所有系统交易所基础文件检查成功"
logger.info(msg)
check_flag += 1
ct.send_sms_control('NoLimit', msg)
else:
list_str = ';'.join(error_file_list)
msg = "Error:有系统交易所基础文件检查失败,失败的文件：%s" % list_str
logger.error(msg)
ct.send_sms_control('NoLimit', msg)
except Exception:
msg = "Error:系统交易所基础文件检查失败，出现异常，请查看服务器日志信息！"
logger.error(msg, exc_info=True)
ct.send_sms_control('NoLimit', msg)
def port_process_task():
linuxInfo = ct.get_server_config('./config/server_status_config.txt')
try:
ms = MonitorServer(linuxInfo)
ms.socket_ps_monitor()
check_result_list = ms.SocPs_Check_flag_list
if len(check_result_list) == 0:
check_result = False
else:
check_result = (sum(check_result_list) == len(check_result_list))
# print "check_result: ", check_result
except Exception as e:
check_result = False
msg = str(e)
logger.warning(msg)
logger.info(
"************************The Monitor Result: ************************")
if check_result:
logger.info("All Server is OK")
else:
msg = "端口和进程监控报警，请查看服务器详细信息!"
logger.error(msg)
sysstr = platform.system()
if sysstr == "Windows":
ct.readTexts("Port Process Monitor is Worning")
ct.generate_file(check_result, 'Port_PS_Monitor')
def mem_monitor_task():
linuxInfo = ct.get_server_config('./config/server_status_config.txt')
try:
ms = MonitorServer(linuxInfo)
ms.mem_monitor()
check_result_list = ms.mem_Check_flag_list
if len(check_result_list) == 0:
check_result = False
else:
check_result = (sum(check_result_list) == len(check_result_list))
# print "check_result: ", check_result
except Exception as e:
check_result = False
msg = str(e)
logger.error(msg)
logger.info(
"************************The Monitor Result: ************************")
if check_result:
logger.info("All Server is OK")
else:
msg = "内存监控报警，请查看服务器详细信息"
logger.error(msg)
sysstr = platform.system()
if sysstr == "Windows":
ct.readTexts("Memory Monitor is Worning")
ct.generate_file(check_result, 'Memory_Monitor')
def main(argv):
#默认是check_sjdr的监控检查
try:
yaml_path = './config/check_sjdr_logger.yaml'
ct.setup_logging(yaml_path)
linuxInfo = ct.get_server_config('./config/check_sjdr_config.txt')
check_flag = 0
for info in linuxInfo:
hostip = info[0]
rfc = remote_file_check(info)
error_list = rfc.check_sdjr()
if len(error_list) == 0:
msg = "ok:系统 %s 盘后当天的节点委托回传数据检查成功" % hostip
logger.info(msg)
check_flag += 1
else:
msg = "系统 %s 盘后当天的节点委托回传数据检查失败 失败的文件列表：%s " % (hostip, ';'.join(error_list))
logger.error(msg)
ct.send_sms_control('NoLimit', msg)
if check_flag == len(linuxInfo):
logger.info(u"OK:检查盘后当天的节点委托回传数据成功!")
else:
logger.error(u"Error:检查盘后当天的节点委托回传数据失败!")
except Exception:
logger.error("盘后当天的节点委托回传数据出现异常，请参看错误日志！", exc_info=True)
finally:
for handler in logger.handlers:
logger.removeHandler(handler)
def fpga_task():
linuxInfo = ct.get_server_config('./config/fpga_config.txt')
# linuxInfo = [['192.168.238.7', 22, 'trade', 'trade', 'tradeserver',/home/trade/FPGA']]
try:
ms = MonitorServer(linuxInfo)
ms.fpga_monitor_run()
check_result_list = ms.fpga_Check_flag_list
logger.debug("check_result_list:")
logger.debug(check_result_list)
if len(check_result_list) == 0:
check_result = False
else:
check_result = (sum(check_result_list) == len(check_result_list))
except Exception:
check_result = False
logger.error('Faild to check fpga!', exc_info=True)
logger.info(
"************************The FPGA File Monitor Result: ************************"
)
if check_result:
logger.info("OK,FPGA Server is OK")
ct.send_sms_control("fpga", "OK:盘前FPGA文件检查正常")
else:
msg = "error:FPGA文件检查异常，请查看详细日志"
logger.error(msg)
# send_sms_control("fpga", "error:奇点服务器FPGA文件检查异常，请查看详细日志")
sysstr = platform.system()
if sysstr == "Windows":
ct.readTexts("VIP Server is Worning")
def errorLog_monitor_task():
linuxInfo = ct.get_server_config('./config/server_logDir_config.txt')
grep_lists = mel.get_errorLog(linuxInfo)
fileNlist = mel.get_result_file_list()
check_flag = mel.errorLog_check(fileNlist, grep_lists)
sysstr = platform.system()
if (not check_flag):
logger.error("错误日志检查报警，请查看服务器详细信息")
# send_sms_control("errorLog", "error:Server log error warning")
if (sysstr == "Windows"):
ct.readTexts("Server log warning")
def xwdm_monitor_task():
linuxInfo = ct.get_server_config('./config/check_xwdm_config.txt')
for info in linuxInfo:
try:
check_flag = 0
hostip = info[0]
c_x = cx.check_csv_file(info)
error_list = c_x.check_xwdm()
check_flag = (sum(error_list)==len(error_list))
if check_flag:
logger.info(u"ok:系统 %s 席位代码文件检查成功！" % hostip)
else:
logger.error(u"系统 %s 检查席位代码文件失败!" % hostip)
except Exception:
logger.error("查询席位代码失败，出现异常，请查看服务器日志信息！", exc_info=True)
ct.send_sms_control('xwdm', "Error:查询席位代码失败，出现异常，请查看服务器日志信息！")
def ssh_remote_command_task():
error_list = []
linuxInfo = ct.get_server_config('./config/ssh_remote_command_config.txt')
for info in linuxInfo:
hostip = info[0]
port = int(info[1])
username = info[2]
password = info[3]
command = info[5]
sshClient = ct.sshConnect(hostip, port, username, password)
if sshClient == 999:
msg = "Failed:服务器[%s],连接失败，请检查密码是否正确" % hostip
logger.error(msg)
#ct.send_sms_control("NoLimit", msg)
error_list.append(msg)
else:
logger.info("Ok: 服务器[%s]连接正常" % hostip)
logger.info(hostip + "::" + command)
#sshRes = ct.sshExecCmd(sshClient, command)
stdin, stdout, stderr = sshClient.exec_command(command)
stdoutstr = stdout.read().decode('utf-8')
ssherr = stderr.read().decode('utf-8')
if ssherr:
msg = "服务器[%s]ssh执行命令返回错误：[%s]" % (hostip, ssherr)
logger.warning(msg)
error_list.append(msg)
sshRes = []
sshRes = stdoutstr.strip().split('\n')
if sshRes == ['']:
sshRes = []
logger.info("sshRes:")
logger.info(sshRes)
sshClient.close()
if len(error_list) != 0:
temstr = ';'.join(error_list)
msg = "Failed:服务器ssh执行命令失败列表：[%s]" % temstr
logger.error(msg)
ct.send_sms_control("NoLimit", msg)
else:
logger.info("所有服务器ssh执行命令正常！")
def newserver_employ_task():
error_list = []
linuxInfo = ct.get_server_config('./config/newserver_employ_config.txt')
for info in linuxInfo:
hostip = info[0]
port = int(info[1])
username = info[2]
password = info[3]
#command = info[5]
command = "/tmp/employ_server_package/employment_script.sh"
#上传部署服务器./employ_server_package到新服务器的tmp目录
ft.upload("./employ_server_package", "/tmp/employ_server_package", hostip, port, username, password, '')
#执行shell脚本部署环境
sshClient = ct.sshConnect(hostip, port, username, password)
if sshClient == 999:
msg = "Failed:服务器[%s],连接失败，请检查密码是否正确" % hostip
logger.error(msg)
#ct.send_sms_control("NoLimit", msg)
error_list.append(msg)
else:
logger.info("Ok: 服务器[%s]连接正常" % hostip)
#执行shell脚本部署环境
logger.info(hostip + "::" + command)
# sshRes = ct.sshExecCmd(sshClient, command)
stdin, stdout, stderr = sshClient.exec_command(command)
stdoutstr = stdout.read().decode('utf-8')
ssherr = stderr.read().decode('utf-8')
if ssherr:
msg = "服务器[%s]ssh执行命令返回错误：[%s]" % (hostip, ssherr)
logger.debug(msg)
#error_list.append(msg)
sshRes = []
sshRes = stdoutstr.strip().split('\n')
if sshRes == ['']:
sshRes = []
logger.info("sshRes:")
logger.info(sshRes)
sshClient.close()
def main(argv):
yaml_path = './config/non_trade_monitor_logger.yaml'
ct.setup_logging(yaml_path)
linuxInfo = ct.get_server_config('./config/offer_connect_config.txt')
check_result = []
for info in linuxInfo:
try:
check_flag = 0
hostip = info[0]
tgw_names = info[5]
scc = ssh_command_check(info)
error_list = scc.offer_connect_check()
check_flag = (sum(error_list) == len(error_list))
if check_flag:
logger.info(u"ok:系统 %s 交易前置 %s 连接检查成功！" % (hostip, tgw_names))
check_result.append(1)
else:
logger.error(u"系统 %s 交易前置 %s 连接检查失败：" % (hostip, tgw_names))
check_result.append(0)
except Exception:
logger.error('Faild to check offer_connect.', exc_info=True)
check_result.append(0)
# finally:
# for handler in logger.handlers:
# logger.removeHandler(handler)
final_check_flag = (sum(check_result) == len(check_result))
logger.info(check_result)
if final_check_flag:
msg = "所有报盘前置连接地址检查成功"
ct.send_sms_control("NoLimit", msg)
logger.info(msg)
else:
logger.error("报盘连接前置检查有错误！")
for handler in logger.handlers:
logger.removeHandler(handler)
def check_ssh_connect_task():
error_list = []
linuxInfo = ct.get_server_config('./config/check_root_passwd_config.txt')
for info in linuxInfo:
hostip = info[0]
port = int(info[1])
username = info[2]
password = info[3]
os_flag = info[4]
if os_flag == 'l':
sshClient = ct.sshConnect(hostip, port, username, password)
if sshClient == 999:
msg = "Failed:服务器[%s],连接失败，请检查密码是否正确" % hostip
logger.error(msg)
#ct.send_sms_control("NoLimit", msg)
error_list.append(hostip + ":::" + username + ":::" + password)
else:
logger.info("Ok: 服务器[%s]连接正常" % hostip)
sshClient.close()
else:
os_info = ct.get_remote_windows_os_info(username, password, hostip)
if os_info == 'Null':
msg = "Failed:服务器[%s],连接失败，请检查密码是否正确" % hostip
logger.error(msg)
#ct.send_sms_control("NoLimit", msg)
error_list.append(hostip + ":::" + username + ":::" + password)
else:
logger.info("Ok: 服务器[%s]连接正常,操作系统是[%s]" % (hostip,os_info))
if len(error_list) != 0:
temstr = ';'.join(error_list)
msg = "Failed:服务器连接失败列表：[%s]，请检查密码是否正确" % temstr
logger.error(msg)
ct.send_sms_control("NoLimit", msg)
else:
logger.info("所有服务器连接正常！")
def main(argv):
try:
yaml_path = './config/check_xwdm_logger.yaml'
ct.setup_logging(yaml_path)
linuxInfo = ct.get_server_config('./config/check_xwdm_config.txt')
for info in linuxInfo:
check_flag = 0
hostip = info[0]
c_x = check_csv_file(info)
error_list = c_x.check_xwdm()
check_flag = (sum(error_list)==len(error_list))
if check_flag:
logger.info(u"ok:系统 %s 席位代码文件检查成功！" % hostip)
else:
logger.error(u"系统 %s 检查席位代码文件失败：" % hostip)
except Exception:
logger.error('Faild to check xwdm!', exc_info=True)
finally:
for handler in logger.handlers:
logger.removeHandler(handler)
def main(argv):
try:
yaml_path = './config/backup_file_check_logger.yaml'
ct.setup_logging(yaml_path)
linuxInfo = ct.get_server_config(
'./config/backup_file_check_config.txt')
for info in linuxInfo:
bfc = backup_file_check(info)
origin_check_flag = bfc.check_origin_file()
back_check = False
if origin_check_flag:
back_check = bfc.check_backupserver_lastday_file()
if back_check:
bfc.del_before_2days_file()
except Exception:
logger.error('Faild to check backup file!', exc_info=True)
finally:
for handler in logger.handlers:
logger.removeHandler(handler)
def errorLog_monitor_task():
linuxInfo = ct.get_server_config('./config/server_logDir_config.txt')
ntimes = dt.datetime.now().strftime("%Y%m%d%H%M%S")
ndates = dt.datetime.now().strftime("%Y%m%d")
#cur_dir_i = os.getcwd()
cur_dir = os.getcwd().replace("\\", "/") + "/"
log_dir = cur_dir + "mylog/"
grep_result_file = log_dir + "errorLog_result_" + ndates + '.txt'
back_file = log_dir + "errorLog_result_" + ntimes + '.txt'
#备份上一次的grep_result_file文件到back_file
if (os.path.exists(grep_result_file)):
os.rename(grep_result_file, back_file)
grep_lists = mel.get_errorLog(linuxInfo)
fileNlist = mel.get_result_file_list()
check_flag = mel.errorLog_check(fileNlist, grep_lists)
sysstr = platform.system()
if (not check_flag):
logger.error("错误日志检查报警，请查看服务器详细信息")
# send_sms_control("errorLog", "error:Server log error warning")
if (sysstr == "Windows"):
ct.readTexts("Server log warning")
def fpgatask():
linuxInfo = ct.get_server_config('./config/fpga_config.txt')
# linuxInfo = [['192.168.238.7', 22, 'trade', 'trade', 'tradeserver',/home/trade/FPGA']]
try:
ms = MonitorServer(linuxInfo)
ms.fpga_monitor_run()
check_result_list = ms.fpga_Check_flag_list
logger.debug("check_result_list:")
logger.debug(check_result_list)
if len(check_result_list) == 0:
check_result = False
else:
check_result = (sum(check_result_list) == len(check_result_list))
except Exception as e:
check_result = False
logger.error('Faild to check fpga!', exc_info=True)
logger.info(
"************************The FPGA File Monitor Result: ************************"
)
if check_result:
logger.info("FPGA Server is OK")
else:
msg = "error:FPGA is Exception, Please Check it!"
logger.error(msg)
# ct.fortunesms(msg)
sysstr = platform.system()
if sysstr == "Windows":
ct.readTexts("VIP Server is Worning")
# else:
# ct.send_mail(msg,"Failed_FPGA_Monitor")
ct.generate_file(check_result, 'FPAG_File_Monitor')
@Desc : None
'''
# here put the import lib
import common_tools as ct
import time
import datetime as dt
import os
import logging
import subprocess
#import platform
#import sys
#reload(sys)
#sys.setdefaultencoding('utf-8')
linuxInfo = ct.get_server_config('./config/ssh_keygen_config_test.csv')
def copy_ssh_rsa(linuxInfo):
logger = logging.getLogger()
yaml_path = './config/non_trade_monitor_logger.yaml'
ct.setup_logging(yaml_path)
error_list = []
for info in linuxInfo:
hostip = info[3]
username = '******'
password = '******'
# servername = info[4]
command = './copy_ssh_rsa.sh %s %s %s' % (hostip, username, password)
@author: zhangwei
@comment: 通过ssh连接Linux服务器，egrep指令搜索错误日志信息，并写入文件，并报警，报警后，ctrl+c退出程序，再次启动的话将不会报警。
"""
import paramiko
import common_tools as ct
import time
import datetime as dt
import os
import re
import platform
import sys
import getopt
import logging
linuxInfo = ct.get_server_config('./config/server_logDir_config.txt')
#print "linuxInfo:", linuxInfo
ntimes = dt.datetime.now().strftime("%Y%m%d%H%M%S")
ndates = dt.datetime.now().strftime("%Y%m%d")
#cur_dir_i = os.getcwd()
cur_dir = os.getcwd().replace("\\", "/") + "/"
log_dir = cur_dir + "mylog/"
grep_result_file = log_dir + "errorLog_result_" + ndates + '.txt'
back_file = log_dir + "errorLog_result_" + ntimes + '.txt'
#log_file = log_dir + "montitor_errorLog_run_" + ndates + '.log'
logger = logging.getLogger()
#将监控到的信息写入grep_result_file文件。
def get_errorLog(linuxInfo):
def tcp_connect_monitor_task():
linuxInfo = ct.get_server_config('./config/tcp_connect_config.txt')
ntm.common_monitor_task("single_common_monitor", "tcp_connect_info",
linuxInfo)
def main(argv):
try:
yaml_path = './config/file_transfer_logger.yaml'
ct.setup_logging(yaml_path)
linuxInfo = ct.get_server_config('./config/file_transfer_config.txt')
#读取输入参数
ftmethod = ''
windir = ''
lindir = ''
singlefile = ''
try:
opts, args = getopt.getopt(
argv, "hm:w:l:f:",
["method=", "windir=", "lindir=", "filename="])
except getopt.GetoptError:
print(
'file_transfer.py -m <method> -w <windir> -l <lindir> -f <filename>'
)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print(
'file_transfer.py -m <method> -w <windir> -l <lindir> -f <filename>\n \
method=upload, means upload file from windows to linux.\n \
method=download, means download file from linux to windows. \n \
windir=dir, Window dir, The format like this: "D:\\my_project\\python". \n \
lindir=dir, Linux dir, The format like this: "/home/trade/temp". \n \
filename=filename, options- The single file to upload or download.'
)
sys.exit()
elif opt in ("-m", "--method"):
ftmethod = arg
elif opt in ("-w", "--windir"):
windir = arg
elif opt in ("-l", "--lindir"):
lindir = arg
elif opt in ("-f", "--filename"):
singlefile = arg
#多线程处理
thrlist = range(len(linuxInfo))
threads = []
# singlefile = 'monitor_log_2019-05-30.txt'
# ftmethod = 'download'
if ftmethod == 'upload':
# thrlist = range(len(linuxInfo))
# threads=[]
for (i, info) in zip(thrlist, linuxInfo):
hostip = info[0]
port = int(info[1])
username = info[2]
password = info[3]
servername = info[4]
local_upload = info[5]
local_download = info[6]
remote_dir = info[7]
# hostip = '192.168.238.7'
# username = '******'
# password = '******'
# port = 22
# local_upload = 'D:/my_project/python/auto_monitor/upload'
# download_dir = 'D:/my_project/python/auto_monitor/download'
# remote_dir = '/home/trade/temp'
if windir != '':
local_upload = windir
if lindir != '':
remote_dir = lindir
t = MyThread(upload, (
local_upload,
remote_dir,
hostip,
port,
username,
password,
singlefile,
), upload.__name__ + str(i))
threads.append(t)
elif ftmethod == 'download':
# thrlist = range(len(linuxInfo))
# threads=[]
for (i, info) in zip(thrlist, linuxInfo):
hostip = info[0]
port = int(info[1])
username = info[2]
password = info[3]
servername = info[4]
local_upload = info[5]
local_download = info[6]
remote_dir = info[7]
if windir != '':
local_download = windir
if lindir != '':
remote_dir = lindir
t = MyThread(download, (
local_download,
remote_dir,
hostip,
port,
username,
password,
singlefile,
), download.__name__ + str(i))
threads.append(t)
else:
logger.error("method input is invalid! -m upload or -m download?")
return
# print "thrcouat3:", threading.active_count()
for i in thrlist:
threads[i].start()
for i in thrlist:
threads[i].join()
except Exception:
logger.error('Faild to transfer file!', exc_info=True)
finally:
for handler in logger.handlers:
logger.removeHandler(handler)
@author: zhangwei
@comment: 通过ssh连接Linux服务器，实现一些自定义的指令查询，并将结果保存到文件。
"""
import common_tools as ct
import time
import datetime as dt
import os
import logging
#import platform
#import sys
#reload(sys)
#sys.setdefaultencoding('utf-8')
linuxInfo = ct.get_server_config('./config/normal_query_config.txt')
ntimes = dt.datetime.now().strftime("%Y%m%d%H%M%S")
ndates = dt.datetime.now().strftime("%Y%m%d")
cur_dir = os.getcwd().replace("\\", "/") + "/"
if os.path.exists("./normal_query"):
pass
else:
os.mkdir('./normal_query')
log_dir = cur_dir + "normal_query/"
query_result_file = log_dir + "normal_query_result_" + ntimes + '.txt'
log_file = log_dir + "normal_query_run_log_" + ndates + '.txt'
def get_query_data(linuxInfo):
logger = logging.getLogger()
szdbf_date_dir = 'D-COM/File/' + todays[:4] + '-' + m + '-' + d
return szdbf_date_dir
#红利征税数据表文件名
dbf_dir = '/home/trade/monitor_server/SSCC/'
#dbf_dir = './DBdata/InvestorExchFee/dbf_file/'
todays = dt.datetime.now().strftime("%Y%m%d")
sh_date_str = shdbf_date_str(todays)
szdbf_date_dir = szdbf_date_dir(todays)
#print("sh_date_str:",sh_date_str)
abcsj_dbf = dbf_dir + 'PROP/' + todays + '/abcsjjs588.' + sh_date_str
zsmx_dbf = dbf_dir + szdbf_date_dir +'/ZSMX' + str(todays)[-4:] + '.DBF'
csv_file_name = './DBdata/InvestorExchFee/InvestorExchFee_' + ndates + '_dbdata.csv'
null_investorID = []
info = ct.get_server_config('./config/mysql_config_sf.txt')
mysql_db_ip = info[0][0]
mysql_user = info[0][1]
mysql_passwd = info[0][2]
mysql_dbname = info[0][3]
#mysql_dbname = 'singular_field'
table_name = 't_CustShareholderInfo'
mysql_port = int(info[0][4])
mysqldb_info = [mysql_db_ip, mysql_user, mysql_passwd,mysql_dbname, mysql_port]
#print("mysqldb_info:",mysqldb_info)
#mysql_obj = myc.mysql_tools(mysqldb_info)
#数据库查询匹配inverstorID,匹配不到的话用'999999'替代
def get_investorID(shareholderID,mysql_obj):
query_sql = "SELECT investorID FROM " + mysql_dbname + "." + table_name + " WHERE shareholderID = " + "'" + shareholderID + "';"
def process_boardOrder(info):
try:
mssql_db_ip = info["mssql_db_ip"]
mssql_user = info["mssql_user"]
mssql_passwd = info["mssql_passwd"]
mssql_dl_db = info["mssql_dl_db"]
mssql_up_db = info["mssql_up_db"]
mssql_dbname = info["mssql_dbname"]
# mysql_db_ip = info["mysql_db_ip"]
# mysql_user = info["mysql_user"]
# mysql_passwd = info["mysql_passwd"]
# mysql_dbname = info["mysql_dbname"]
# mysql_port = info["mysql_port"]
csv_file_name = './DBdata/BoardOrder/' + ndates + '_BoardOrder_df.csv'
#先连接upload库查询修改成功的订单
upDB_info = [mssql_db_ip, mssql_user, mssql_passwd, mssql_up_db]
downDB_info = [mssql_db_ip, mssql_user, mssql_passwd, mssql_dl_db]
#(cursor, conn) = mst.connect_mssql(uploaddb_info)
EQ_sql = "SELECT Content FROM wanping_upload.dbo.t_EQCommand WHERE MdbErrCode = 0"
content_res, title = mst.get_db_data(EQ_sql, upDB_info)
logger.info(content_res)
print("title:", title)
if content_res != []:
result_df = pd.DataFrame(columns=['TradingDay','OrderLocalID','OrderSysID','Direction','Price',\
'VolumeTotalOriginal','VolumeTraded','SecurityID','ExchangeID','InvestorID','SInfo',\
'InvestorName','SecurityName','ShortSecurityName','ExchangeName','DBname'])
for item in content_res:
logger.info(item[0])
cont_list = item[0].split('|')
print(cont_list)
#SSEOrder
if int(cont_list[0]) == 1:
ExchangeName = "上海证券交易所"
order_sql = "SELECT a.TradingDay,a.OrderLocalID,a.OrderSysID,a.Direction,a.Price,\
a.VolumeTotalOriginal,a.VolumeTraded,a.SecurityID,a.ExchangeID,\
a.InvestorID,a.SInfo,b.InvestorName,c.SecurityName,c.ShortSecurityName FROM %s.dbo.t_SSEOrder as a \
LEFT JOIN %s.dbo.t_Investor as b ON a.InvestorID = b.InvestorID\
LEFT JOIN %s.dbo.t_SSESecurity as c ON a.SecurityID = c.SecurityID\
WHERE a.OrderSysID = '%s' " % (
mssql_dl_db, mssql_dl_db, mssql_dl_db, cont_list[1])
#是否为打板
#SELECT * FROM t1 LEFT JOIN t2 on t1.cid=t2.id AND t1.name=‘su’
#sql3 = "SELECT BoardFlag FROM wanping_download.dbo.t_SSEMarketData WHERE SecurityID = " + order_df[0][2]
#SZSEOrder
elif int(cont_list[0]) == 2:
print("深圳订单")
ExchangeName = "深圳证券交易所"
order_sql = "SELECT a.TradingDay,a.OrderLocalID,a.OrderSysID,a.Direction,a.Price,\
a.VolumeTotalOriginal,a.VolumeTraded,a.SecurityID,a.ExchangeID,\
a.InvestorID,a.SInfo,b.InvestorName,c.SecurityName,c.ShortSecurityName FROM %s.dbo.t_SZSEOrder as a \
LEFT JOIN %s.dbo.t_Investor as b ON a.InvestorID = b.InvestorID\
LEFT JOIN %s.dbo.t_SZSESecurity as c ON a.SecurityID = c.SecurityID\
WHERE a.OrderSysID = '%s' " % (
mssql_dl_db, mssql_dl_db, mssql_dl_db, cont_list[1])
logger.info(order_sql)
res_df = mst.get_db_df(order_sql, downDB_info)
res_df['ExchangeName'] = ExchangeName
res_df['DBname'] = mssql_dbname
result_df = pd.concat([result_df, res_df], ignore_index=True)
#result_df.to_csv(csv_file_name, encoding='utf-8',index=False)
result_df.to_csv(csv_file_name, encoding='utf-8')
else:
logger.info("t_EQCommand没有MdbErrCode等于0的数据!")
#导入到mysql数据库
if not os.path.isfile(csv_file_name):
logger.error("当天没有BoardOrder订单文件生成")
else:
logger.info("导入mysql....")
info = ct.get_server_config('./config/mysql_config.txt')
mysql_db_ip = info[0][0]
mysql_user = info[0][1]
mysql_passwd = info[0][2]
mysql_dbname = info[0][3]
mysql_port = int(info[0][4])
mysqldb_info = [
mysql_db_ip, mysql_user, mysql_passwd, mysql_dbname, mysql_port
]
mysql_obj = myc.mysql_tools(mysqldb_info)
local_infile_value = mysql_obj.get_local_infile_value()
#判断mysql参数是否打开允许导入文件
if local_infile_value == 'ON':
file_sql = mysql_obj.load_table_commend_gen(
csv_file_name, 'fireball_board_order')
logger.info(file_sql)
mysql_obj.execute_sql(file_sql)
logger.info("导入mysql完成")
else:
local_file_msg = "Error，mysql导入csv失败，local_infile 的值为： %s" % local_infile_value
ct.send_sms_control('NoLimit', local_file_msg, '13681919346')
except Exception:
msg = '处理boardOrder数据出现异常'
logger.error(msg, exc_info=True)
ct.send_sms_control('NoLimit', msg, '13681919346')
finally:
pass
def main(argv):
try:
yaml_path = './config/file_transfer_logger.yaml'
ct.setup_logging(yaml_path)
linuxInfo = ct.get_server_config('./config/file_transfer_config.txt')
#读取输入参数
ftmethod = ''
windir = ''
lindir = ''
singlefile = ''
try:
opts, args = getopt.getopt(argv,"hm:l:r:f:",["method=", "local_dir=", "remote_dir=" ,"filename="])
except getopt.GetoptError:
print('file_transfer.py -m <method> -l <local_dir> -r <remote_dir> -f <filename>')
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('file_transfer.py -m <method> -w <windir> -l <lindir> -f <filename>\n \
method=upload, means upload file from windows to linux.\n \
method=download, means download file from linux to windows. \n \
windir=dir, Window dir, The format like this: "D:\\my_project\\python". \n \
lindir=dir, Linux dir, The format like this: "/home/trade/temp". \n \
filename=filename, options- The single file to upload or download.' )
sys.exit()
elif opt in ("-m", "--method"):
ftmethod = arg
elif opt in ("-l", "--local_dir"):
windir = arg
elif opt in ("-r", "--remote_dir"):
lindir = arg
elif opt in ("-f", "--filename"):
singlefile = arg
# hostip = '192.168.238.7'
# username = '******'
# password = '******'
# port = 22
# local_upload = 'D:/my_project/python/auto_monitor/upload'
# download_dir = 'D:/my_project/python/auto_monitor/download'
# remote_dir = '/home/trade/temp'
# singlefile = 'monitor_log_2019-05-30.txt'
# ftmethod = 'download'
if ftmethod == 'upload':
for info in linuxInfo:
hostip = info[0]
port = int(info[1])
username = info[2]
password = info[3]
local_upload = info[5]
local_download = info[6]
remote_dir = info[7]
if windir !='':
local_upload = windir
if lindir !='':
remote_dir = lindir
upload(local_upload, remote_dir, hostip, port, username, password, singlefile)
elif ftmethod == 'download':
for info in linuxInfo:
hostip = info[0]
port = int(info[1])
username = info[2]
password = info[3]
local_upload = info[5]
local_download = info[6]
remote_dir = info[7]
if windir !='':
local_download = windir
if lindir !='':
remote_dir = lindir
download(local_download, remote_dir, hostip, port, username, password, singlefile)
else:
logger.error("method input is invalid! -m upload or -m download?")
except Exception:
logger.error('Faild to transfer file!', exc_info=True)
finally:
for handler in logger.handlers:
logger.removeHandler(handler)
def core_file_monitor_task():
linuxInfo = ct.get_server_config('./config/server_status_config.txt')
common_monitor_task("single_common_monitor", "core_file_info", linuxInfo)
