def dell_tel_login1(self):
try:
logger.debug(self.ip + " Connecting...")
tel = pexpect.spawn('telnet %s' % self.ip)
i = tel.expect(
['ogin:', 'refused', 'fail', 'time', pexpect.TIMEOUT],
timeout=5)
if i >= 1:
logger.error(self.ip + " Can not reach the remote router!")
tel.close()
return None
tel.sendline(self.pass1)
i = tel.expect('word:', timeout=2)
tel.sendline(self.pass2)
i = tel.expect([self.wait2, 'ogin:', pexpect.TIMEOUT], timeout=5)
if i >= 1:
logger.error(self.ip + " Error username or password!")
tel.close()
return None
#
except Exception as e:
logger.error(self.ip + ' ' + str(e))
tel.close()
return None
else:
logger.info(self.ip + " Logged in!")
return tel
def linux_ssh_login1(self):
try:
logger.debug(self.ip + " Connecting...")
ssh = pexpect.spawn('ssh -p 22 %s@%s' % (self.pass1, self.ip))
i = ssh.expect([
'word:', 'continue connecting (yes/no)?', 'refused', 'fail',
'time', pexpect.TIMEOUT
],
timeout=8)
if i == 1:
ssh.sendline('yes')
ssh.expect('word:', timeout=3)
if i >= 2:
logger.error(self.ip + " Can not reach the remote router!")
ssh.close()
return None
ssh.sendline(self.pass2)
i = ssh.expect(['[#$]', 'word:', pexpect.TIMEOUT], timeout=3)
if i >= 1:
logger.error(self.ip + " Error username or password!")
ssh.close()
return None
except Exception as e:
logger.error(self.ip + ' ' + str(e))
ssh.close()
return None
else:
logger.info(self.ip + " Logged in!")
return ssh
def save_to_disk(self):
timestr = time.strftime('%Y%m%d%H%M%S',time.localtime())
logger.debug(self.ip + ' start to save config to disk')
fname = self.name + '-' + timestr
city = hashlib.md5((self.city).encode('UTF-8')).hexdigest()
fpath = base_dir + r'/down/conf/%s/%s' % (city, self.name)
if not os.path.exists(fpath):
try:
os.makedirs(fpath)
except Exception as e:
logger.error(str(e))
return
os.chdir(fpath)
#save config
try:
file = open(fname, 'w')
file.writelines(self.txt_conf)
except Exception as e:
logger.error(str(e))
else:
logger.info(self.ip + ' finish saving config!')
finally:
file.close()
def h3c_tel_login2(self):
try:
logger.debug(self.ip + " Connecting...")
tel = pexpect.spawn('telnet %s' % self.ip)
i = tel.expect(
['word:', 'refused', 'fail', 'time', pexpect.TIMEOUT],
timeout=15)
if i == 1:
logger.error(self.ip + " Can not reach the remote router!")
tel.close()
return None
tel.sendline(self.pass1)
i = tel.expect([self.wait1, 'word:', pexpect.TIMEOUT], timeout=5)
if i >= 1:
logger.error(self.ip + " Invald password!")
tel.close()
return None
tel.sendline('sup')
tel.expect('ssword:', timeout=1)
tel.sendline(self.pass2)
i = tel.expect([self.wait1, 'ssword:', pexpect.TIMEOUT], timeout=3)
if i >= 1:
logger.error(self.ip + " Error super password!")
tel.close()
return None
except Exception as e:
logger.error(self.ip + ' ' + str(e))
tel.close()
return None
else:
logger.info(self.ip + " Logged in!")
return tel
def pull_from_cmm(self):
ip = envi['cmm_sftp_ip']
port = envi['cmm_sftp_port']
usr = envi['cmm_sftp_user']
pwd = envi['cmm_sftp_pass']
src = envi['data_src_name']
p = envi['cmm_data_pattern']
try:
target = paramiko.Transport((ip, port))
target.connect(username=usr, password=pwd)
sftp = paramiko.SFTPClient.from_transport(target)
logger.debug("connect to cmm's sftp server")
# to find files for transfaring
for rdir in envi['cmm_data_dir']:
for f in sftp.listdir(rdir):
if f not in self.loc_files and re.match(p, f) != None:
self.trans_files.append(f)
# transfar via sftp get
sftp.get(rdir + f, self.ldir + f)
logger.debug('file: <' + str(f) + '> transfared')
if self.trans_files == []:
logger.warn('no data to pull')
send_mail(src + u'未更新', u'请咨询数据提供技术人员')
else:
logger.info('pulling finished: ' + str(self.trans_files))
except Exception as e:
logger.error(str(e))
send_mail(src + u'自动同步失败', str(e))
return False
target.close()
return True
def linux_ssh_login2(self):
try:
logger.debug(self.ip + " Connecting...")
ssh = pexpect.spawn('ssh %s@%s' % (self.pass1, self.ip))
i = ssh.expect(['$', pexpect.TIMEOUT], timeout=8)
if i >= 1:
logger.error(self.ip + " Error login using ssh key!")
ssh.close()
return None
except Exception as e:
logger.error(self.ip + ' ' + str(e))
ssh.close()
return None
else:
logger.info(self.ip + " Logged in!")
return ssh
def ftp_alert(msg, phone):
#新建临时文件
fpath = r'/var/www/html/netmon/log'
os.chdir(fpath)
timestr = time.strftime('%H%M%S', time.localtime())
fname = '220000000_000000000000_' + timestr + '_NETMON.txt'
text = ''
#print type(msg)
#将中文编码转换成ftp服务器上的编码
msg = msg.encode('gbk')
for num in phone:
text = text + '|' + num + '||' + msg + "|0|\n"
#print type(text)
try:
file = open(fname, 'w')
file.writelines(text)
file.close()
except Exception as e:
logger.error(str(e))
file.close()
return False
else:
logger.debug('temp file [' + fname + '] created')
#临时文件上传ftp服务器
try:
ftp = FTP()
ftp.set_debuglevel(1)
ftp.connect(ftp_server, ftp_port)
ftp.login(ftp_usr, ftp_pwd)
logger.debug(" Ftp Server Logged in!")
ftp.cwd(ftp_dir)
ftp.storlines('STOR ' + fname, open(fname, 'r'))
ftp.set_debuglevel(0)
ftp.close()
except Exception as e:
logger.error(str(e))
ftp.close()
return False
else:
logger.info(" file uploaded to FTP server!")
#清理临时文件
os.remove(fname)
return True
def get_weibo(mid):
url = "http://10.75.57.27/getdata/querydata2.php?condition=%s&mode=weibo&format=json" % (
mid.strip())
#try:
if 1:
start_time = clock()
text = requests.get(url, timeout=4).text
logger.info(str(text))
logger.debug(url)
end_time = clock()
testLogger()
#lock.acquire()
#print text
#with open('request.log', 'a+') as logs:
# logs.write('request %s cost: %s\n' % (url, end_time - start_time))
#print url
#lock.release()
return text
def h3c_ssh_login1(self):
try:
logger.debug(self.ip + " Connecting...")
ssh = pexpect.spawn('ssh -p 22 %s@%s' % (self.pass1, self.ip))
#openssh-6.6.1p1-25 登陆h3c的bug，必须用下行的ssh连接方法
#ssh=pexpect.spawn('ssh -1 %s@%s' %(self.pass1, self.ip))
i = ssh.expect([
'word:', 'continue connecting (yes/no)?', 'fail', 'refused',
'time', pexpect.TIMEOUT
],
timeout=8)
if i == 1:
ssh.sendline('yes')
ssh.expect('word:', timeout=3)
if i >= 2:
logger.error(self.ip + " Can not reach the remote router!")
ssh.close()
return None
ssh.sendline(self.pass2)
i = ssh.expect([self.wait1, 'word:', pexpect.TIMEOUT], timeout=5)
if i >= 1:
logger.error(self.ip + " Error username or password!")
ssh.close()
return None
ssh.sendline('sup')
ssh.expect('ssword:', timeout=1)
ssh.sendline(self.pass3)
i = ssh.expect([self.wait1, 'ssword:', pexpect.TIMEOUT], timeout=3)
if i >= 1:
logger.error(self.ip + " Error super password!")
ssh.close()
return None
except Exception as e:
logger.error(self.ip + ' ' + str(e))
ssh.close()
return None
else:
logger.info(self.ip + " Logged in!")
return ssh
def save(self):
if device_netcap.get(self.corp) == None:
logger.error(self.ip + ' Error : unsupported device!')
return
obj = self.login()
if obj == None:
logger.error(self.ip + ' Error : saving config failed due to login error!')
return
prompt = self.name + device_netcap.get(self.corp).get('prompt')
pageCmd = device_netcap.get(self.corp).get('page')
saveCmd = device_netcap.get(self.corp).get('conf')[0]
try:
obj.sendline(pageCmd)
obj.expect([prompt, pexpect.TIMEOUT], timeout=3)
#capture running-configration
logger.debug(self.ip + ' executing: ' + saveCmd)
obj.sendline(saveCmd)
i = obj.expect([prompt, pexpect.TIMEOUT], timeout=20)
if i == 1:
logger.error(self.ip + ' error exec ' + saveCmd)
obj.close()
return
self.txt_conf = obj.before
logger.debug(self.ip + ' logged out!')
obj.close()
except Exception as e:
logger.error(self.ip + ' ' + str(e))
obj.close()
return
else:
self.save_to_disk()
def testLogger():
logger.debug('this is mod.testLogger...')
submod.tst()
def mon(self):
if self.target == ():
logger.warn(self.ip + ' No target to monitor!')
return
if device_netmon.get(self.corp) == None:
logger.error(self.ip + ' Error : unsupported device to monitor!')
return
obj = self.login()
if obj == None:
logger.error(self.ip +
' Error : monitor failed due to login error!')
return
success = device_netmon.get(self.corp).get('succ')
fail = device_netmon.get(self.corp).get('fail')
pingCmd = device_netmon.get(self.corp).get('ping')
list_tdes = []
list_rtts = []
try:
for line in self.target:
cmd = pingCmd % line[0]
logger.debug(self.ip + ' ' + cmd)
obj.sendline(cmd)
rtt = 0
i = obj.expect([success, fail, pexpect.TIMEOUT], timeout=10)
if i == 2:
logger.error(self.ip + " Command runs abnormal!")
obj.close()
return
if i == 1:
if line[2] > 0:
msg = self.name + line[1] + u':线路中断'
my_alert(msg, line[1])
if i == 0:
rtt = int(obj.after.split(' ')[2]) + 1
if line[2] == 0:
msg = self.name + line[1] + u':线路恢复' + str(rtt) + 'ms'
my_alert(msg, line[1])
elif line[2] > RTT_MAX:
if rtt < RTT_MAX:
msg = self.name + line[1] + u': 线路延时恢复' + str(
rtt) + 'ms'
my_alert(msg, line[1])
else:
if rtt > RTT_MAX:
msg = self.name + line[1] + u': 线路延时过高' + str(
rtt) + 'ms'
my_alert(msg, line[1])
list_tdes.append(line[1])
list_rtts.append(rtt)
obj.close()
#批量更新线路延迟数据rtt,减少数据库访问
mupdate(list_tdes, list_rtts)
except Exception as e:
logger.error(self.ip + ' ' + str(e))
obj.close()
return
else:
logger.info(self.ip + " monitoring finished!")
return
def capture(self):
if device_netcap.get(self.corp) == None:
logger.error(self.ip + ' Error : unsupported device to capture!')
return
obj = self.login()
if obj == None:
logger.error(self.ip +
' Error : capture failure due to login error!')
return
prompt = self.name + device_netcap.get(self.corp).get('prompt')
pageCmd = device_netcap.get(self.corp).get('page')
cmdSet = {
'conf': device_netcap.get(self.corp).get('conf'),
'routesum': device_netcap.get(self.corp).get('routesum'),
'route': device_netcap.get(self.corp).get('route'),
'mod': device_netcap.get(self.corp).get('mod'),
'int': device_netcap.get(self.corp).get('int'),
'stp': device_netcap.get(self.corp).get('stp')
}
try:
obj.sendline(pageCmd)
obj.expect([prompt, pexpect.TIMEOUT], timeout=3)
#exec CMDs on the target device
for (cmd_key, cmd_value) in cmdSet.items():
for cmd_str in cmd_value:
logger.debug(self.ip + ' executing ' + cmd_str)
obj.sendline(cmd_str)
i = obj.expect([prompt, pexpect.TIMEOUT], timeout=20)
if i == 1:
logger.error(self.ip + ' error exec ' + cmd_str)
obj.close()
return
temp = self.__getattribute__('re_%s' % cmd_key)
temp += obj.before
self.__setattr__('re_%s' % cmd_key, temp)
logger.debug(self.ip + ' logged out!')
obj.close()
except Exception as e:
logger.error(self.ip + ' ' + str(e))
obj.close()
return
else:
#save things to disk
logger.debug(self.ip + ' start to save info to disk')
city = hashlib.md5((self.city).encode('UTF-8')).hexdigest()
fpath = base_dir + r'/down/cap/%s/%s' % (city, self.name)
if not os.path.exists(fpath):
try:
os.makedirs(fpath)
except Exception as e:
logger.error(str(e))
return
os.chdir(fpath)
for cmd_key in cmdSet.keys():
try:
file = open(cmd_key, 'w')
file.writelines(self.__getattribute__('re_%s' % cmd_key))
finally:
file.close()
#trim routing table
cmd = 'bash %s/script/trim_%s.sh' % (base_dir, self.corp)
val = os.system(cmd)
if (val != 0):
logger.error(self.ip + ' Error exec trim_route script')
def start():
logger.debug('Program begin: auto sync files via sftp')
logger.debug('step 0: start a vpn tunnel')
vpn_log = envi['cwd'] + 'log/vpn.log'
vpn_conf = envi['cwd'] + 'conf/client.ovpn'
cmd = 'openvpn --config %s > %s 2>&1 &' % (vpn_conf, vpn_log)
try:
os.system(cmd)
time.sleep(30)
sync = sftp_sync()
if sync is not None:
logger.debug('step 1: pull data from cmm')
if sync.pull_from_cmm():
logger.debug('step 2: push data to prd')
sync.push_to_prd()
finally:
logger.debug('step 3: stop the vpn tunnel')
cmd = "kill `ps -ef|grep openvpn|grep -v 'grep'|awk '{print $2}'`"
os.system(cmd)
time.sleep(10)
logger.debug('Program end')
"predict_file": "data/{task_name}/predict".format(task_name=task_name),
"predict_result": "data/{task_name}/predict_result".format(task_name=task_name),
"tf_serving_save_dir": "result/{task_name}/saved_model/{model_version}".format(task_name=task_name,model_version = time.strftime('%Y%m%d')),
"parameter_information": "result/{task_name}/saved_model/parameter_information.json".format(task_name=task_name),
"save_dir": "result/{task_name}/saved_model/".format(task_name=task_name),
"tag_to_id": "result/{task_name}/saved_model/tag_to_id.json".format(task_name=task_name),
"id_to_tag": "result/{task_name}/saved_model/id_to_tag.json".format(task_name=task_name)
}
bert_config = helper.obj_load(base_config["bert_config"])
base_config = helper.merge_two_dicts(base_config, para)
config = {
"base_config": base_config,
"bert_config": bert_config
}
helper.obj_save(config, base_config["parameter_information"])
if os.path.exists(os.path.join(base_config["save_dir"], "vocab.txt")):
logger.debug(["model_result vocab_file existed!!"])
else:
shutil.copy(base_config["vocab_dir"], base_config["save_dir"])
logger.info(base_config)
else:
base_config = helper.obj_load("result/{task_name}/saved_model/parameter_information.json".format(task_name=task_name))["base_config"]
model = Model(base_config)
if ARGS.entry == "train":
model.train()
elif ARGS.entry == "predict":
model.predict()
