def save_to_disk(self):
timestr = time.strftime('%Y%m%d%H%M%S',time.localtime())
logger.debug(self.ip + ' start to save config to disk')
fname = self.name + '-' + timestr
city = hashlib.md5((self.city).encode('UTF-8')).hexdigest()
fpath = base_dir + r'/down/conf/%s/%s' % (city, self.name)
if not os.path.exists(fpath):
try:
os.makedirs(fpath)
except Exception as e:
logger.error(str(e))
return
os.chdir(fpath)
#save config
try:
file = open(fname, 'w')
file.writelines(self.txt_conf)
except Exception as e:
logger.error(str(e))
else:
logger.info(self.ip + ' finish saving config!')
finally:
file.close()
def pull_from_cmm(self):
ip = envi['cmm_sftp_ip']
port = envi['cmm_sftp_port']
usr = envi['cmm_sftp_user']
pwd = envi['cmm_sftp_pass']
src = envi['data_src_name']
p = envi['cmm_data_pattern']
try:
target = paramiko.Transport((ip, port))
target.connect(username=usr, password=pwd)
sftp = paramiko.SFTPClient.from_transport(target)
logger.debug("connect to cmm's sftp server")
# to find files for transfaring
for rdir in envi['cmm_data_dir']:
for f in sftp.listdir(rdir):
if f not in self.loc_files and re.match(p, f) != None:
self.trans_files.append(f)
# transfar via sftp get
sftp.get(rdir + f, self.ldir + f)
logger.debug('file: <' + str(f) + '> transfared')
if self.trans_files == []:
logger.warn('no data to pull')
send_mail(src + u'未更新', u'请咨询数据提供技术人员')
else:
logger.info('pulling finished: ' + str(self.trans_files))
except Exception as e:
logger.error(str(e))
send_mail(src + u'自动同步失败', str(e))
return False
target.close()
return True
def get_html_content(http_url):
"""
获取网页对象
:param http_url:地址
:return: BeautifulSoup对象，日期字符串 as 2017/07/01
"""
try:
# 获取上月的日期字符串 as 2017/07/01 2017/07/31
last_month_start, last_month_end = get_last_month()
post_data = [
("curr_id", "8849"),
("smlID", "300060"),
("st_date", last_month_start),
("end_date", last_month_end),
("interval_sec", "Daily"),
("sort_col", "date"),
("sort_ord", "DESC"),
("action", "historical_data")
]
headers = {"X-Requested-With": "XMLHttpRequest"}
status, html = util_urllib.post_content_utf8(http_url, post_data, headers)
# print(status, html)
if 200 == status:
return BeautifulSoup(html, "html.parser"), last_month_start
else:
logger.warn("status:" + status)
except Exception as e:
logger.error(e)
return None, ""
logger.error("status:" + status + " url:" + http_url)
return None, ""
def push_to_prd(self):
if self.trans_files == []:
logger.warn('no data to push')
return False
ip = envi['prd_sftp_ip']
port = envi['prd_sftp_port']
usr = envi['prd_sftp_user']
pwd = envi['prd_sftp_pass']
src = envi['data_src_name']
rdir = envi['prd_data_dir']
try:
target = paramiko.Transport((ip, port))
target.connect(username=usr, password=pwd)
sftp = paramiko.SFTPClient.from_transport(target)
# transfar via sftp put
for f in self.trans_files:
sftp.put(self.ldir + f, rdir + f)
logger.info('pushing to prd finished')
except Exception as e:
logger.error(str(e))
send_mail(src + u'自动同步失败', str(e))
return False
target.close()
send_mail(src + u'自动同步成功', u'请使用md5文件校验数据文件*.tar.gz')
return True
def parse_data(http_url, conn):
logger.info("请求url:" + http_url)
soup, date_str = get_html_content(http_url)
if soup is not None:
get_data(conn=conn, date_str=date_str, soup4=soup)
else:
logger.error("获取数据失败，请检查网络连接")
def parse_data(http_url, conn, year='',month=''):
logger.info("请求url:" + http_url)
soup, date_str = get_html_content(http_url,year,month)
time.sleep(random.randint(2, 4))
if soup is not None:
get_data(conn=conn, date_str=date_str, soup4=soup)
else:
logger.error("获取数据失败，请检查网络连接")
def xgetone(sql):
ret = ()
try:
db = MySQLdb.connect(host=_HOST,
user=_USER,
passwd=_PWD,
db=_DB,
charset=_CHAR)
except MySQLdb.Error, e:
logger.error(str(e))
def mupdate(list1, list2):
ret = False
try:
db = MySQLdb.connect(host=_HOST,
user=_USER,
passwd=_PWD,
db=_DB,
charset=_CHAR)
except MySQLdb.Error, e:
logger.error(str(e))
def db_encode():
ret = False
list1 = xgetall('select rip, pass1, pass2, pass3 from router;')
try:
db = MySQLdb.connect(host=_HOST,
user=_USER,
passwd=_PWD,
db=_DB,
charset=_CHAR)
except MySQLdb.Error, e:
logger.error(str(e))
def __init__(self):
ldir = envi['cwd'] + 'data/'
if not os.path.exists(ldir):
try:
os.makedirs(ldir)
except Exception as e:
logger.error(str(e))
return False
self.ldir = ldir
self.loc_files = os.listdir(ldir)
self.trans_files = []
def insert(conn, fid, month, value):
# date
hx_range = '国内航线'
if fid == 6:
hx_range = '国际航线'
param = {"FID": fid, "MONTH": month, "RANGE": hx_range, "VALUE": value}
try:
my_database.insert_model(conn=conn, param=param)
except Exception as e:
if "PK" not in e:
logger.exception("入库出错" + param)
logger.error(e)
def send_mail_int_nologin(sub,content,receiver):
sender = "netmon<%s>" % envi['smtp_usr']
msg = MIMEText(content,'plain','gbk')
msg['Subject'] = Header(sub, 'gbk')
msg['From'] = sender
try:
smtp = smtplib.SMTP(envi['smtp_server'])
smtp.sendmail(sender, receiver, msg.as_string())
smtp.quit()
except Exception, e:
logger.error(str(e))
return False
def h3c_tel_login2(self):
try:
logger.debug(self.ip + " Connecting...")
tel = pexpect.spawn('telnet %s' % self.ip)
i = tel.expect(
['word:', 'refused', 'fail', 'time', pexpect.TIMEOUT],
timeout=15)
if i == 1:
logger.error(self.ip + " Can not reach the remote router!")
tel.close()
return None
tel.sendline(self.pass1)
i = tel.expect([self.wait1, 'word:', pexpect.TIMEOUT], timeout=5)
if i >= 1:
logger.error(self.ip + " Invald password!")
tel.close()
return None
tel.sendline('sup')
tel.expect('ssword:', timeout=1)
tel.sendline(self.pass2)
i = tel.expect([self.wait1, 'ssword:', pexpect.TIMEOUT], timeout=3)
if i >= 1:
logger.error(self.ip + " Error super password!")
tel.close()
return None
except Exception as e:
logger.error(self.ip + ' ' + str(e))
tel.close()
return None
else:
logger.info(self.ip + " Logged in!")
return tel
def insert(conn, fid, month, value):
# date
param = {
"FID": fid,
"MONTH": month,
"RANGE":"全部航线",
"VALUE": value
}
try:
my_database.insert_model(conn=conn, param=param)
except Exception as e:
if "PK" not in e:
logger.exception("入库出错" + param)
logger.error(e)
def urlopen(url, data=""):
# 出错重连，默认3次
for i in range(reconnection_count):
try:
if data == "":
resp = request.urlopen(url=url, timeout=time_out)
else:
resp = request.urlopen(url=url, data=data, timeout=time_out)
logger.info("获取" + url.full_url + "内容成功")
return resp
except Exception as e:
logger.error(e)
logger.error("第" + str(i + 1) + "次请求失败：" + url.full_url)
time.sleep(3)
return ""
def linux_ssh_login2(self):
try:
logger.debug(self.ip + " Connecting...")
ssh = pexpect.spawn('ssh %s@%s' % (self.pass1, self.ip))
i = ssh.expect(['$', pexpect.TIMEOUT], timeout=8)
if i >= 1:
logger.error(self.ip + " Error login using ssh key!")
ssh.close()
return None
except Exception as e:
logger.error(self.ip + ' ' + str(e))
ssh.close()
return None
else:
logger.info(self.ip + " Logged in!")
return ssh
def ftp_alert(msg, phone):
#新建临时文件
fpath = r'/var/www/html/netmon/log'
os.chdir(fpath)
timestr = time.strftime('%H%M%S', time.localtime())
fname = '220000000_000000000000_' + timestr + '_NETMON.txt'
text = ''
#print type(msg)
#将中文编码转换成ftp服务器上的编码
msg = msg.encode('gbk')
for num in phone:
text = text + '|' + num + '||' + msg + "|0|\n"
#print type(text)
try:
file = open(fname, 'w')
file.writelines(text)
file.close()
except Exception as e:
logger.error(str(e))
file.close()
return False
else:
logger.debug('temp file [' + fname + '] created')
#临时文件上传ftp服务器
try:
ftp = FTP()
ftp.set_debuglevel(1)
ftp.connect(ftp_server, ftp_port)
ftp.login(ftp_usr, ftp_pwd)
logger.debug(" Ftp Server Logged in!")
ftp.cwd(ftp_dir)
ftp.storlines('STOR ' + fname, open(fname, 'r'))
ftp.set_debuglevel(0)
ftp.close()
except Exception as e:
logger.error(str(e))
ftp.close()
return False
else:
logger.info(" file uploaded to FTP server!")
#清理临时文件
os.remove(fname)
return True
def linux_ssh_login1(self):
try:
logger.debug(self.ip + " Connecting...")
ssh = pexpect.spawn('ssh -p 22 %s@%s' % (self.pass1, self.ip))
i = ssh.expect([
'word:', 'continue connecting (yes/no)?', 'refused', 'fail',
'time', pexpect.TIMEOUT
],
timeout=8)
if i == 1:
ssh.sendline('yes')
ssh.expect('word:', timeout=3)
if i >= 2:
logger.error(self.ip + " Can not reach the remote router!")
ssh.close()
return None
ssh.sendline(self.pass2)
i = ssh.expect(['[#$]', 'word:', pexpect.TIMEOUT], timeout=3)
if i >= 1:
logger.error(self.ip + " Error username or password!")
ssh.close()
return None
except Exception as e:
logger.error(self.ip + ' ' + str(e))
ssh.close()
return None
else:
logger.info(self.ip + " Logged in!")
return ssh
def dell_tel_login1(self):
try:
logger.debug(self.ip + " Connecting...")
tel = pexpect.spawn('telnet %s' % self.ip)
i = tel.expect(
['ogin:', 'refused', 'fail', 'time', pexpect.TIMEOUT],
timeout=5)
if i >= 1:
logger.error(self.ip + " Can not reach the remote router!")
tel.close()
return None
tel.sendline(self.pass1)
i = tel.expect('word:', timeout=2)
tel.sendline(self.pass2)
i = tel.expect([self.wait2, 'ogin:', pexpect.TIMEOUT], timeout=5)
if i >= 1:
logger.error(self.ip + " Error username or password!")
tel.close()
return None
#
except Exception as e:
logger.error(self.ip + ' ' + str(e))
tel.close()
return None
else:
logger.info(self.ip + " Logged in!")
return tel
def login(self):
if self.name == '':
logger.warn(self.ip + ' No device with this IP!')
return None
if self.login_mode == 22010:
return self.cisco_ssh_login1()
elif self.login_mode == 22012:
return self.cisco_ssh_login2()
elif self.login_mode == 22020:
return self.h3c_ssh_login1()
elif self.login_mode == 22022:
return self.h3c_ssh_login2()
elif self.login_mode == 22030:
return self.huawei_ssh_login1()
elif self.login_mode == 22032:
return self.huawei_ssh_login2()
elif self.login_mode == 23010 or self.login_mode == 23040:
return self.cisco_tel_login1()
elif self.login_mode == 23011 or self.login_mode == 23041:
return self.cisco_tel_login2()
elif self.login_mode == 23012 or self.login_mode == 23042:
return self.cisco_tel_login3()
elif self.login_mode == 23020 or self.login_mode == 23030:
return self.h3c_tel_login1()
elif self.login_mode == 23021 or self.login_mode == 23031:
return self.h3c_tel_login2()
elif self.login_mode == 23022 or self.login_mode == 23032:
return self.h3c_tel_login3()
elif self.login_mode == 22052:
return self.junos_ssh_login1()
elif self.login_mode == 22062:
return self.dell_ssh_login1()
elif self.login_mode == 23062:
return self.dell_tel_login1()
elif self.login_mode == 22072:
return self.linux_ssh_login1()
else:
logger.error(self.ip + ' Error login_mode!')
return None
def send_mail_ext(sub,content,receiver):
if receiver == ['']:
logger.warn('mail[' + sub + '] fail, empty receive list!')
return False
if(envi['smtp_server2'] == ''):
return send_mail(sub,content,receiver)
sender = "netmon<%s>" % envi['smtp_usr2']
msg = MIMEText(content,'plain','gbk')
msg['Subject'] = Header(sub, 'gbk')
msg['From'] = sender
try:
smtp = smtplib.SMTP()
smtp.connect(envi['smtp_server2'])
smtp.login(envi['smtp_usr2'], envi['smtp_pwd2'])
smtp.sendmail(sender, receiver, msg.as_string())
smtp.quit()
except Exception, e:
logger.error(str(e))
return False
def send_mail(sub, content):
receiver = envi['mail_receiver']
if receiver == ['']:
logger.warn('mail[' + sub + '] fail, empty receive list!')
return False
sender = "netmon<%s>" % envi['smtp_usr']
msg = MIMEText(content, 'plain', 'gbk')
msg['Subject'] = Header(sub, 'gbk')
msg['From'] = sender
try:
smtp = smtplib.SMTP()
smtp.connect(envi['smtp_server'])
smtp.login(envi['smtp_usr'], envi['smtp_pwd'])
smtp.sendmail(sender, receiver, msg.as_string())
smtp.quit()
except Exception as e:
logger.error(str(e))
return False
else:
logger.info('success sending mail [' + sub + '] to ' + str(receiver))
return True
def h3c_ssh_login1(self):
try:
logger.debug(self.ip + " Connecting...")
ssh = pexpect.spawn('ssh -p 22 %s@%s' % (self.pass1, self.ip))
#openssh-6.6.1p1-25 登陆h3c的bug，必须用下行的ssh连接方法
#ssh=pexpect.spawn('ssh -1 %s@%s' %(self.pass1, self.ip))
i = ssh.expect([
'word:', 'continue connecting (yes/no)?', 'fail', 'refused',
'time', pexpect.TIMEOUT
],
timeout=8)
if i == 1:
ssh.sendline('yes')
ssh.expect('word:', timeout=3)
if i >= 2:
logger.error(self.ip + " Can not reach the remote router!")
ssh.close()
return None
ssh.sendline(self.pass2)
i = ssh.expect([self.wait1, 'word:', pexpect.TIMEOUT], timeout=5)
if i >= 1:
logger.error(self.ip + " Error username or password!")
ssh.close()
return None
ssh.sendline('sup')
ssh.expect('ssword:', timeout=1)
ssh.sendline(self.pass3)
i = ssh.expect([self.wait1, 'ssword:', pexpect.TIMEOUT], timeout=3)
if i >= 1:
logger.error(self.ip + " Error super password!")
ssh.close()
return None
except Exception as e:
logger.error(self.ip + ' ' + str(e))
ssh.close()
return None
else:
logger.info(self.ip + " Logged in!")
return ssh
def save(self):
if device_netcap.get(self.corp) == None:
logger.error(self.ip + ' Error : unsupported device!')
return
obj = self.login()
if obj == None:
logger.error(self.ip + ' Error : saving config failed due to login error!')
return
prompt = self.name + device_netcap.get(self.corp).get('prompt')
pageCmd = device_netcap.get(self.corp).get('page')
saveCmd = device_netcap.get(self.corp).get('conf')[0]
try:
obj.sendline(pageCmd)
obj.expect([prompt, pexpect.TIMEOUT], timeout=3)
#capture running-configration
logger.debug(self.ip + ' executing: ' + saveCmd)
obj.sendline(saveCmd)
i = obj.expect([prompt, pexpect.TIMEOUT], timeout=20)
if i == 1:
logger.error(self.ip + ' error exec ' + saveCmd)
obj.close()
return
self.txt_conf = obj.before
logger.debug(self.ip + ' logged out!')
obj.close()
except Exception as e:
logger.error(self.ip + ' ' + str(e))
obj.close()
return
else:
self.save_to_disk()
def parse_json(data, data_type, conn):
"""
解析json对象，并入库
:param conn: mysql连接对象
:param data_type: 数据类型
:param data:json对象
:return: 200 OK,其他 ERROR
"""
# 校验请求是否正常
status = data["returncode"]
if status != 200:
# return null, "error"
logger.error("error status:" + status)
return status
returndata = data["returndata"]
# 为记录中文，遍历并初始化names {code:name}
wdnodes = returndata["wdnodes"]
names = {}
units = {}
for wdnode in wdnodes:
nodes = wdnode["nodes"]
if wdnode["wdcode"] == "zb":
for node in nodes:
code = node["code"]
names[code] = node["cname"]
units[code] = node["unit"]
# 遍历并插入数据库
datanodes = returndata["datanodes"]
index = 0
index_code = ""
for i, datanode in enumerate(datanodes):
data_val = datanode["data"]["data"]
zb_code = ""
sj_code = ""
for wds in datanode["wds"]:
if wds["wdcode"] == "zb":
zb_code = wds["valuecode"]
if wds["wdcode"] == "sj":
sj_code = wds["valuecode"]
name = names[zb_code]
unit = units[zb_code]
date = time.strptime(sj_code, "%Y%m")
data_date = datetime.datetime.strptime(sj_code, "%Y%m").date()
if index_code != zb_code:
index = index + 1
index_code = zb_code
yearmonth = sj_code[0:4] + "-" + sj_code[4:6]
if '社会消费品零售总额_当期值' in name:
insert(conn, 1, yearmonth, int(data_val))
if '进口总值_当期值' == name:
insert(conn, 2, yearmonth, int(data_val))
if '出口总值_当期值' == name:
insert(conn, 3, yearmonth, int(data_val))
if '民航货运量_当期值' in name:
insert(conn, 4, yearmonth, int(data_val))
return 200
def parse_data(url, date_type, conn):
logger.info("请求地址：" + url)
json_data = util_urllib.get_json_utf8(url)
status = parse_json(json_data, date_type, conn)
if status != 200:
logger.error("请求地址：" + url)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import class_login, sys
from my_log import logger
if len(sys.argv) == 2:
class_login.NetLogin(sys.argv[1]).test()
else:
logger.error(sys.argv[0] + " Error参数错误")
def capture(self):
if device_netcap.get(self.corp) == None:
logger.error(self.ip + ' Error : unsupported device to capture!')
return
obj = self.login()
if obj == None:
logger.error(self.ip +
' Error : capture failure due to login error!')
return
prompt = self.name + device_netcap.get(self.corp).get('prompt')
pageCmd = device_netcap.get(self.corp).get('page')
cmdSet = {
'conf': device_netcap.get(self.corp).get('conf'),
'routesum': device_netcap.get(self.corp).get('routesum'),
'route': device_netcap.get(self.corp).get('route'),
'mod': device_netcap.get(self.corp).get('mod'),
'int': device_netcap.get(self.corp).get('int'),
'stp': device_netcap.get(self.corp).get('stp')
}
try:
obj.sendline(pageCmd)
obj.expect([prompt, pexpect.TIMEOUT], timeout=3)
#exec CMDs on the target device
for (cmd_key, cmd_value) in cmdSet.items():
for cmd_str in cmd_value:
logger.debug(self.ip + ' executing ' + cmd_str)
obj.sendline(cmd_str)
i = obj.expect([prompt, pexpect.TIMEOUT], timeout=20)
if i == 1:
logger.error(self.ip + ' error exec ' + cmd_str)
obj.close()
return
temp = self.__getattribute__('re_%s' % cmd_key)
temp += obj.before
self.__setattr__('re_%s' % cmd_key, temp)
logger.debug(self.ip + ' logged out!')
obj.close()
except Exception as e:
logger.error(self.ip + ' ' + str(e))
obj.close()
return
else:
#save things to disk
logger.debug(self.ip + ' start to save info to disk')
city = hashlib.md5((self.city).encode('UTF-8')).hexdigest()
fpath = base_dir + r'/down/cap/%s/%s' % (city, self.name)
if not os.path.exists(fpath):
try:
os.makedirs(fpath)
except Exception as e:
logger.error(str(e))
return
os.chdir(fpath)
for cmd_key in cmdSet.keys():
try:
file = open(cmd_key, 'w')
file.writelines(self.__getattribute__('re_%s' % cmd_key))
finally:
file.close()
#trim routing table
cmd = 'bash %s/script/trim_%s.sh' % (base_dir, self.corp)
val = os.system(cmd)
if (val != 0):
logger.error(self.ip + ' Error exec trim_route script')
def mon(self):
if self.target == ():
logger.warn(self.ip + ' No target to monitor!')
return
if device_netmon.get(self.corp) == None:
logger.error(self.ip + ' Error : unsupported device to monitor!')
return
obj = self.login()
if obj == None:
logger.error(self.ip +
' Error : monitor failed due to login error!')
return
success = device_netmon.get(self.corp).get('succ')
fail = device_netmon.get(self.corp).get('fail')
pingCmd = device_netmon.get(self.corp).get('ping')
list_tdes = []
list_rtts = []
try:
for line in self.target:
cmd = pingCmd % line[0]
logger.debug(self.ip + ' ' + cmd)
obj.sendline(cmd)
rtt = 0
i = obj.expect([success, fail, pexpect.TIMEOUT], timeout=10)
if i == 2:
logger.error(self.ip + " Command runs abnormal!")
obj.close()
return
if i == 1:
if line[2] > 0:
msg = self.name + line[1] + u':线路中断'
my_alert(msg, line[1])
if i == 0:
rtt = int(obj.after.split(' ')[2]) + 1
if line[2] == 0:
msg = self.name + line[1] + u':线路恢复' + str(rtt) + 'ms'
my_alert(msg, line[1])
elif line[2] > RTT_MAX:
if rtt < RTT_MAX:
msg = self.name + line[1] + u': 线路延时恢复' + str(
rtt) + 'ms'
my_alert(msg, line[1])
else:
if rtt > RTT_MAX:
msg = self.name + line[1] + u': 线路延时过高' + str(
rtt) + 'ms'
my_alert(msg, line[1])
list_tdes.append(line[1])
list_rtts.append(rtt)
obj.close()
#批量更新线路延迟数据rtt,减少数据库访问
mupdate(list_tdes, list_rtts)
except Exception as e:
logger.error(self.ip + ' ' + str(e))
obj.close()
return
else:
logger.info(self.ip + " monitoring finished!")
return
ret = ()
try:
db = MySQLdb.connect(host=_HOST,
user=_USER,
passwd=_PWD,
db=_DB,
charset=_CHAR)
except MySQLdb.Error, e:
logger.error(str(e))
else:
try:
cur = db.cursor()
cur.execute(sql)
ret = cur.fetchall()
except MySQLdb.Error, e:
logger.error(str(e))
cur.close()
db.close()
return ret
### get data from DB
def xgetone(sql):
ret = ()
try:
db = MySQLdb.connect(host=_HOST,
user=_USER,
passwd=_PWD,
db=_DB,
charset=_CHAR)
except MySQLdb.Error, e:
