def save_to_disk(self): timestr = time.strftime('%Y%m%d%H%M%S',time.localtime()) logger.debug(self.ip + ' start to save config to disk') fname = self.name + '-' + timestr city = hashlib.md5((self.city).encode('UTF-8')).hexdigest() fpath = base_dir + r'/down/conf/%s/%s' % (city, self.name) if not os.path.exists(fpath): try: os.makedirs(fpath) except Exception as e: logger.error(str(e)) return os.chdir(fpath) #save config try: file = open(fname, 'w') file.writelines(self.txt_conf) except Exception as e: logger.error(str(e)) else: logger.info(self.ip + ' finish saving config!') finally: file.close()
def pull_from_cmm(self): ip = envi['cmm_sftp_ip'] port = envi['cmm_sftp_port'] usr = envi['cmm_sftp_user'] pwd = envi['cmm_sftp_pass'] src = envi['data_src_name'] p = envi['cmm_data_pattern'] try: target = paramiko.Transport((ip, port)) target.connect(username=usr, password=pwd) sftp = paramiko.SFTPClient.from_transport(target) logger.debug("connect to cmm's sftp server") # to find files for transfaring for rdir in envi['cmm_data_dir']: for f in sftp.listdir(rdir): if f not in self.loc_files and re.match(p, f) != None: self.trans_files.append(f) # transfar via sftp get sftp.get(rdir + f, self.ldir + f) logger.debug('file: <' + str(f) + '> transfared') if self.trans_files == []: logger.warn('no data to pull') send_mail(src + u'未更新', u'请咨询数据提供技术人员') else: logger.info('pulling finished: ' + str(self.trans_files)) except Exception as e: logger.error(str(e)) send_mail(src + u'自动同步失败', str(e)) return False target.close() return True
def get_html_content(http_url): """ 获取网页对象 :param http_url:地址 :return: BeautifulSoup对象,日期字符串 as 2017/07/01 """ try: # 获取上月的日期字符串 as 2017/07/01 2017/07/31 last_month_start, last_month_end = get_last_month() post_data = [ ("curr_id", "8849"), ("smlID", "300060"), ("st_date", last_month_start), ("end_date", last_month_end), ("interval_sec", "Daily"), ("sort_col", "date"), ("sort_ord", "DESC"), ("action", "historical_data") ] headers = {"X-Requested-With": "XMLHttpRequest"} status, html = util_urllib.post_content_utf8(http_url, post_data, headers) # print(status, html) if 200 == status: return BeautifulSoup(html, "html.parser"), last_month_start else: logger.warn("status:" + status) except Exception as e: logger.error(e) return None, "" logger.error("status:" + status + " url:" + http_url) return None, ""
def push_to_prd(self): if self.trans_files == []: logger.warn('no data to push') return False ip = envi['prd_sftp_ip'] port = envi['prd_sftp_port'] usr = envi['prd_sftp_user'] pwd = envi['prd_sftp_pass'] src = envi['data_src_name'] rdir = envi['prd_data_dir'] try: target = paramiko.Transport((ip, port)) target.connect(username=usr, password=pwd) sftp = paramiko.SFTPClient.from_transport(target) # transfar via sftp put for f in self.trans_files: sftp.put(self.ldir + f, rdir + f) logger.info('pushing to prd finished') except Exception as e: logger.error(str(e)) send_mail(src + u'自动同步失败', str(e)) return False target.close() send_mail(src + u'自动同步成功', u'请使用md5文件校验数据文件*.tar.gz') return True
def parse_data(http_url, conn): logger.info("请求url:" + http_url) soup, date_str = get_html_content(http_url) if soup is not None: get_data(conn=conn, date_str=date_str, soup4=soup) else: logger.error("获取数据失败,请检查网络连接")
def parse_data(http_url, conn, year='',month=''): logger.info("请求url:" + http_url) soup, date_str = get_html_content(http_url,year,month) time.sleep(random.randint(2, 4)) if soup is not None: get_data(conn=conn, date_str=date_str, soup4=soup) else: logger.error("获取数据失败,请检查网络连接")
def xgetone(sql): ret = () try: db = MySQLdb.connect(host=_HOST, user=_USER, passwd=_PWD, db=_DB, charset=_CHAR) except MySQLdb.Error, e: logger.error(str(e))
def mupdate(list1, list2): ret = False try: db = MySQLdb.connect(host=_HOST, user=_USER, passwd=_PWD, db=_DB, charset=_CHAR) except MySQLdb.Error, e: logger.error(str(e))
def db_encode(): ret = False list1 = xgetall('select rip, pass1, pass2, pass3 from router;') try: db = MySQLdb.connect(host=_HOST, user=_USER, passwd=_PWD, db=_DB, charset=_CHAR) except MySQLdb.Error, e: logger.error(str(e))
def __init__(self): ldir = envi['cwd'] + 'data/' if not os.path.exists(ldir): try: os.makedirs(ldir) except Exception as e: logger.error(str(e)) return False self.ldir = ldir self.loc_files = os.listdir(ldir) self.trans_files = []
def insert(conn, fid, month, value): # date hx_range = '国内航线' if fid == 6: hx_range = '国际航线' param = {"FID": fid, "MONTH": month, "RANGE": hx_range, "VALUE": value} try: my_database.insert_model(conn=conn, param=param) except Exception as e: if "PK" not in e: logger.exception("入库出错" + param) logger.error(e)
def send_mail_int_nologin(sub,content,receiver): sender = "netmon<%s>" % envi['smtp_usr'] msg = MIMEText(content,'plain','gbk') msg['Subject'] = Header(sub, 'gbk') msg['From'] = sender try: smtp = smtplib.SMTP(envi['smtp_server']) smtp.sendmail(sender, receiver, msg.as_string()) smtp.quit() except Exception, e: logger.error(str(e)) return False
def h3c_tel_login2(self): try: logger.debug(self.ip + " Connecting...") tel = pexpect.spawn('telnet %s' % self.ip) i = tel.expect( ['word:', 'refused', 'fail', 'time', pexpect.TIMEOUT], timeout=15) if i == 1: logger.error(self.ip + " Can not reach the remote router!") tel.close() return None tel.sendline(self.pass1) i = tel.expect([self.wait1, 'word:', pexpect.TIMEOUT], timeout=5) if i >= 1: logger.error(self.ip + " Invald password!") tel.close() return None tel.sendline('sup') tel.expect('ssword:', timeout=1) tel.sendline(self.pass2) i = tel.expect([self.wait1, 'ssword:', pexpect.TIMEOUT], timeout=3) if i >= 1: logger.error(self.ip + " Error super password!") tel.close() return None except Exception as e: logger.error(self.ip + ' ' + str(e)) tel.close() return None else: logger.info(self.ip + " Logged in!") return tel
def insert(conn, fid, month, value): # date param = { "FID": fid, "MONTH": month, "RANGE":"全部航线", "VALUE": value } try: my_database.insert_model(conn=conn, param=param) except Exception as e: if "PK" not in e: logger.exception("入库出错" + param) logger.error(e)
def urlopen(url, data=""): # 出错重连,默认3次 for i in range(reconnection_count): try: if data == "": resp = request.urlopen(url=url, timeout=time_out) else: resp = request.urlopen(url=url, data=data, timeout=time_out) logger.info("获取" + url.full_url + "内容成功") return resp except Exception as e: logger.error(e) logger.error("第" + str(i + 1) + "次请求失败:" + url.full_url) time.sleep(3) return ""
def linux_ssh_login2(self): try: logger.debug(self.ip + " Connecting...") ssh = pexpect.spawn('ssh %s@%s' % (self.pass1, self.ip)) i = ssh.expect(['$', pexpect.TIMEOUT], timeout=8) if i >= 1: logger.error(self.ip + " Error login using ssh key!") ssh.close() return None except Exception as e: logger.error(self.ip + ' ' + str(e)) ssh.close() return None else: logger.info(self.ip + " Logged in!") return ssh
def ftp_alert(msg, phone): #新建临时文件 fpath = r'/var/www/html/netmon/log' os.chdir(fpath) timestr = time.strftime('%H%M%S', time.localtime()) fname = '220000000_000000000000_' + timestr + '_NETMON.txt' text = '' #print type(msg) #将中文编码转换成ftp服务器上的编码 msg = msg.encode('gbk') for num in phone: text = text + '|' + num + '||' + msg + "|0|\n" #print type(text) try: file = open(fname, 'w') file.writelines(text) file.close() except Exception as e: logger.error(str(e)) file.close() return False else: logger.debug('temp file [' + fname + '] created') #临时文件上传ftp服务器 try: ftp = FTP() ftp.set_debuglevel(1) ftp.connect(ftp_server, ftp_port) ftp.login(ftp_usr, ftp_pwd) logger.debug(" Ftp Server Logged in!") ftp.cwd(ftp_dir) ftp.storlines('STOR ' + fname, open(fname, 'r')) ftp.set_debuglevel(0) ftp.close() except Exception as e: logger.error(str(e)) ftp.close() return False else: logger.info(" file uploaded to FTP server!") #清理临时文件 os.remove(fname) return True
def linux_ssh_login1(self): try: logger.debug(self.ip + " Connecting...") ssh = pexpect.spawn('ssh -p 22 %s@%s' % (self.pass1, self.ip)) i = ssh.expect([ 'word:', 'continue connecting (yes/no)?', 'refused', 'fail', 'time', pexpect.TIMEOUT ], timeout=8) if i == 1: ssh.sendline('yes') ssh.expect('word:', timeout=3) if i >= 2: logger.error(self.ip + " Can not reach the remote router!") ssh.close() return None ssh.sendline(self.pass2) i = ssh.expect(['[#$]', 'word:', pexpect.TIMEOUT], timeout=3) if i >= 1: logger.error(self.ip + " Error username or password!") ssh.close() return None except Exception as e: logger.error(self.ip + ' ' + str(e)) ssh.close() return None else: logger.info(self.ip + " Logged in!") return ssh
def dell_tel_login1(self): try: logger.debug(self.ip + " Connecting...") tel = pexpect.spawn('telnet %s' % self.ip) i = tel.expect( ['ogin:', 'refused', 'fail', 'time', pexpect.TIMEOUT], timeout=5) if i >= 1: logger.error(self.ip + " Can not reach the remote router!") tel.close() return None tel.sendline(self.pass1) i = tel.expect('word:', timeout=2) tel.sendline(self.pass2) i = tel.expect([self.wait2, 'ogin:', pexpect.TIMEOUT], timeout=5) if i >= 1: logger.error(self.ip + " Error username or password!") tel.close() return None # except Exception as e: logger.error(self.ip + ' ' + str(e)) tel.close() return None else: logger.info(self.ip + " Logged in!") return tel
def login(self): if self.name == '': logger.warn(self.ip + ' No device with this IP!') return None if self.login_mode == 22010: return self.cisco_ssh_login1() elif self.login_mode == 22012: return self.cisco_ssh_login2() elif self.login_mode == 22020: return self.h3c_ssh_login1() elif self.login_mode == 22022: return self.h3c_ssh_login2() elif self.login_mode == 22030: return self.huawei_ssh_login1() elif self.login_mode == 22032: return self.huawei_ssh_login2() elif self.login_mode == 23010 or self.login_mode == 23040: return self.cisco_tel_login1() elif self.login_mode == 23011 or self.login_mode == 23041: return self.cisco_tel_login2() elif self.login_mode == 23012 or self.login_mode == 23042: return self.cisco_tel_login3() elif self.login_mode == 23020 or self.login_mode == 23030: return self.h3c_tel_login1() elif self.login_mode == 23021 or self.login_mode == 23031: return self.h3c_tel_login2() elif self.login_mode == 23022 or self.login_mode == 23032: return self.h3c_tel_login3() elif self.login_mode == 22052: return self.junos_ssh_login1() elif self.login_mode == 22062: return self.dell_ssh_login1() elif self.login_mode == 23062: return self.dell_tel_login1() elif self.login_mode == 22072: return self.linux_ssh_login1() else: logger.error(self.ip + ' Error login_mode!') return None
def send_mail_ext(sub,content,receiver): if receiver == ['']: logger.warn('mail[' + sub + '] fail, empty receive list!') return False if(envi['smtp_server2'] == ''): return send_mail(sub,content,receiver) sender = "netmon<%s>" % envi['smtp_usr2'] msg = MIMEText(content,'plain','gbk') msg['Subject'] = Header(sub, 'gbk') msg['From'] = sender try: smtp = smtplib.SMTP() smtp.connect(envi['smtp_server2']) smtp.login(envi['smtp_usr2'], envi['smtp_pwd2']) smtp.sendmail(sender, receiver, msg.as_string()) smtp.quit() except Exception, e: logger.error(str(e)) return False
def send_mail(sub, content): receiver = envi['mail_receiver'] if receiver == ['']: logger.warn('mail[' + sub + '] fail, empty receive list!') return False sender = "netmon<%s>" % envi['smtp_usr'] msg = MIMEText(content, 'plain', 'gbk') msg['Subject'] = Header(sub, 'gbk') msg['From'] = sender try: smtp = smtplib.SMTP() smtp.connect(envi['smtp_server']) smtp.login(envi['smtp_usr'], envi['smtp_pwd']) smtp.sendmail(sender, receiver, msg.as_string()) smtp.quit() except Exception as e: logger.error(str(e)) return False else: logger.info('success sending mail [' + sub + '] to ' + str(receiver)) return True
def h3c_ssh_login1(self): try: logger.debug(self.ip + " Connecting...") ssh = pexpect.spawn('ssh -p 22 %s@%s' % (self.pass1, self.ip)) #openssh-6.6.1p1-25 登陆h3c的bug,必须用下行的ssh连接方法 #ssh=pexpect.spawn('ssh -1 %s@%s' %(self.pass1, self.ip)) i = ssh.expect([ 'word:', 'continue connecting (yes/no)?', 'fail', 'refused', 'time', pexpect.TIMEOUT ], timeout=8) if i == 1: ssh.sendline('yes') ssh.expect('word:', timeout=3) if i >= 2: logger.error(self.ip + " Can not reach the remote router!") ssh.close() return None ssh.sendline(self.pass2) i = ssh.expect([self.wait1, 'word:', pexpect.TIMEOUT], timeout=5) if i >= 1: logger.error(self.ip + " Error username or password!") ssh.close() return None ssh.sendline('sup') ssh.expect('ssword:', timeout=1) ssh.sendline(self.pass3) i = ssh.expect([self.wait1, 'ssword:', pexpect.TIMEOUT], timeout=3) if i >= 1: logger.error(self.ip + " Error super password!") ssh.close() return None except Exception as e: logger.error(self.ip + ' ' + str(e)) ssh.close() return None else: logger.info(self.ip + " Logged in!") return ssh
def save(self): if device_netcap.get(self.corp) == None: logger.error(self.ip + ' Error : unsupported device!') return obj = self.login() if obj == None: logger.error(self.ip + ' Error : saving config failed due to login error!') return prompt = self.name + device_netcap.get(self.corp).get('prompt') pageCmd = device_netcap.get(self.corp).get('page') saveCmd = device_netcap.get(self.corp).get('conf')[0] try: obj.sendline(pageCmd) obj.expect([prompt, pexpect.TIMEOUT], timeout=3) #capture running-configration logger.debug(self.ip + ' executing: ' + saveCmd) obj.sendline(saveCmd) i = obj.expect([prompt, pexpect.TIMEOUT], timeout=20) if i == 1: logger.error(self.ip + ' error exec ' + saveCmd) obj.close() return self.txt_conf = obj.before logger.debug(self.ip + ' logged out!') obj.close() except Exception as e: logger.error(self.ip + ' ' + str(e)) obj.close() return else: self.save_to_disk()
def parse_json(data, data_type, conn): """ 解析json对象,并入库 :param conn: mysql连接对象 :param data_type: 数据类型 :param data:json对象 :return: 200 OK,其他 ERROR """ # 校验请求是否正常 status = data["returncode"] if status != 200: # return null, "error" logger.error("error status:" + status) return status returndata = data["returndata"] # 为记录中文,遍历并初始化names {code:name} wdnodes = returndata["wdnodes"] names = {} units = {} for wdnode in wdnodes: nodes = wdnode["nodes"] if wdnode["wdcode"] == "zb": for node in nodes: code = node["code"] names[code] = node["cname"] units[code] = node["unit"] # 遍历并插入数据库 datanodes = returndata["datanodes"] index = 0 index_code = "" for i, datanode in enumerate(datanodes): data_val = datanode["data"]["data"] zb_code = "" sj_code = "" for wds in datanode["wds"]: if wds["wdcode"] == "zb": zb_code = wds["valuecode"] if wds["wdcode"] == "sj": sj_code = wds["valuecode"] name = names[zb_code] unit = units[zb_code] date = time.strptime(sj_code, "%Y%m") data_date = datetime.datetime.strptime(sj_code, "%Y%m").date() if index_code != zb_code: index = index + 1 index_code = zb_code yearmonth = sj_code[0:4] + "-" + sj_code[4:6] if '社会消费品零售总额_当期值' in name: insert(conn, 1, yearmonth, int(data_val)) if '进口总值_当期值' == name: insert(conn, 2, yearmonth, int(data_val)) if '出口总值_当期值' == name: insert(conn, 3, yearmonth, int(data_val)) if '民航货运量_当期值' in name: insert(conn, 4, yearmonth, int(data_val)) return 200
def parse_data(url, date_type, conn): logger.info("请求地址:" + url) json_data = util_urllib.get_json_utf8(url) status = parse_json(json_data, date_type, conn) if status != 200: logger.error("请求地址:" + url)
#!/usr/bin/env python # -*- coding: utf-8 -*- import class_login, sys from my_log import logger if len(sys.argv) == 2: class_login.NetLogin(sys.argv[1]).test() else: logger.error(sys.argv[0] + " Error参数错误")
def capture(self): if device_netcap.get(self.corp) == None: logger.error(self.ip + ' Error : unsupported device to capture!') return obj = self.login() if obj == None: logger.error(self.ip + ' Error : capture failure due to login error!') return prompt = self.name + device_netcap.get(self.corp).get('prompt') pageCmd = device_netcap.get(self.corp).get('page') cmdSet = { 'conf': device_netcap.get(self.corp).get('conf'), 'routesum': device_netcap.get(self.corp).get('routesum'), 'route': device_netcap.get(self.corp).get('route'), 'mod': device_netcap.get(self.corp).get('mod'), 'int': device_netcap.get(self.corp).get('int'), 'stp': device_netcap.get(self.corp).get('stp') } try: obj.sendline(pageCmd) obj.expect([prompt, pexpect.TIMEOUT], timeout=3) #exec CMDs on the target device for (cmd_key, cmd_value) in cmdSet.items(): for cmd_str in cmd_value: logger.debug(self.ip + ' executing ' + cmd_str) obj.sendline(cmd_str) i = obj.expect([prompt, pexpect.TIMEOUT], timeout=20) if i == 1: logger.error(self.ip + ' error exec ' + cmd_str) obj.close() return temp = self.__getattribute__('re_%s' % cmd_key) temp += obj.before self.__setattr__('re_%s' % cmd_key, temp) logger.debug(self.ip + ' logged out!') obj.close() except Exception as e: logger.error(self.ip + ' ' + str(e)) obj.close() return else: #save things to disk logger.debug(self.ip + ' start to save info to disk') city = hashlib.md5((self.city).encode('UTF-8')).hexdigest() fpath = base_dir + r'/down/cap/%s/%s' % (city, self.name) if not os.path.exists(fpath): try: os.makedirs(fpath) except Exception as e: logger.error(str(e)) return os.chdir(fpath) for cmd_key in cmdSet.keys(): try: file = open(cmd_key, 'w') file.writelines(self.__getattribute__('re_%s' % cmd_key)) finally: file.close() #trim routing table cmd = 'bash %s/script/trim_%s.sh' % (base_dir, self.corp) val = os.system(cmd) if (val != 0): logger.error(self.ip + ' Error exec trim_route script')
def mon(self): if self.target == (): logger.warn(self.ip + ' No target to monitor!') return if device_netmon.get(self.corp) == None: logger.error(self.ip + ' Error : unsupported device to monitor!') return obj = self.login() if obj == None: logger.error(self.ip + ' Error : monitor failed due to login error!') return success = device_netmon.get(self.corp).get('succ') fail = device_netmon.get(self.corp).get('fail') pingCmd = device_netmon.get(self.corp).get('ping') list_tdes = [] list_rtts = [] try: for line in self.target: cmd = pingCmd % line[0] logger.debug(self.ip + ' ' + cmd) obj.sendline(cmd) rtt = 0 i = obj.expect([success, fail, pexpect.TIMEOUT], timeout=10) if i == 2: logger.error(self.ip + " Command runs abnormal!") obj.close() return if i == 1: if line[2] > 0: msg = self.name + line[1] + u':线路中断' my_alert(msg, line[1]) if i == 0: rtt = int(obj.after.split(' ')[2]) + 1 if line[2] == 0: msg = self.name + line[1] + u':线路恢复' + str(rtt) + 'ms' my_alert(msg, line[1]) elif line[2] > RTT_MAX: if rtt < RTT_MAX: msg = self.name + line[1] + u': 线路延时恢复' + str( rtt) + 'ms' my_alert(msg, line[1]) else: if rtt > RTT_MAX: msg = self.name + line[1] + u': 线路延时过高' + str( rtt) + 'ms' my_alert(msg, line[1]) list_tdes.append(line[1]) list_rtts.append(rtt) obj.close() #批量更新线路延迟数据rtt,减少数据库访问 mupdate(list_tdes, list_rtts) except Exception as e: logger.error(self.ip + ' ' + str(e)) obj.close() return else: logger.info(self.ip + " monitoring finished!") return
ret = () try: db = MySQLdb.connect(host=_HOST, user=_USER, passwd=_PWD, db=_DB, charset=_CHAR) except MySQLdb.Error, e: logger.error(str(e)) else: try: cur = db.cursor() cur.execute(sql) ret = cur.fetchall() except MySQLdb.Error, e: logger.error(str(e)) cur.close() db.close() return ret ### get data from DB def xgetone(sql): ret = () try: db = MySQLdb.connect(host=_HOST, user=_USER, passwd=_PWD, db=_DB, charset=_CHAR) except MySQLdb.Error, e: