def __enter_app_shell(self):
self.set_prompt('% ')
self.send_command('attach where='+self.str_app_type, 5)
self.clean_buff()
self.send_command('exit')
self.set_prompt('WinG% ')
log_info('enter_app_shell type='+self.str_app_type)
def main_process():
int_slot_num=0
try:
int_slot_num = eval(sys.argv[1])
except:
print_help_info()
if int_slot_num>14 or int_slot_num<3:
print_help_info()
try:
type(sys.argv[2])
except:
print_help_info()
try:
type(sys.argv[3])
if 'unstable' != sys.argv[3]:
print_help_info()
except:
pass
base_name='diagnosis_slot%d_%s' %(int_slot_num,time.strftime('%Y%m%d_%H%M%S',time.localtime(time.time())))
repo_path='/tmp/'+base_name
if sys.argv[2]=='hsgw':
collect_info_on_hsgw(int_slot_num,base_name,repo_path)
collect_info_on_sysmgr(int_slot_num,base_name,repo_path)
else:
log_info('Unkown application type.')
def __init__(self, slot_num, str_app_type,base_name,repo_path):
self.str_app_type = str_app_type
self.int_slot=slot_num
self.base_name=base_name
self.repo_path=repo_path
if 0 != os.system('mkdir '+self.repo_path):
log_info('Failed! mkdir '+self.repo_path)
self.log_file_name='%s/slot_cmd_output.log' %self.repo_path
super(ssh_on_slotx,self).__init__('slot-%d' %slot_num,'root','pdg.net')
def __write_log_file(self,str_content):
try:
file_object = open(self.log_file_name, 'a')
except:
log_info("Failed! open %s 'a'" %self.log_file_name)
return
try:
file_object.write(time.strftime('\n================== %Y%m%d-%H:%M:%S =================\n',time.localtime(time.time())))
file_object.write(str_content)
except:
log_info("Failed! write %s" %self.log_file_name)
file_object.close()
def login_CLI(self):
self.set_prompt('system# ')
self.send_command('su - sysadmin', timeout=10)
#self.send_command('cli', timeout=10)
self.clean_buff()
if self.str_app_type == 'system':
log_info('login CLI type=system')
return
self.set_prompt(self.str_app_type+'# ')
self.send_command('set context=%s' %self.str_app_type,timeout=5)
self.clean_buff()
self.send_command('cls',timeout=10)
log_info('login CLI type='+self.str_app_type)
def __proc_multi_line_cmd(self,command_block):
cmd_list = re.split('[\r\n]+\s*', command_block)
for command in cmd_list:
match_sleep = re.search('sleep\s+(\d+)', command)
if match_sleep:
try:
duration = eval(match_sleep.group(1))
time.sleep(duration)
except:
log_info("Failed! exec %s" %command)
elif re.search('[\w\d]+', command):
log_buff=self.send_command(command, timeout=30)
self.__write_log_file(log_buff)
self.clean_buff()
def flush_gcov(ip_addr, username, password, debug_flag):
global g_debug
try:
testbed=ssh_connection(ip_addr, username, password, debug_on=debug_flag)
except:
log_info('Connect failed.')
return
try:
testbed.set_prompt('% ')
testbed.send_command('dtach -a /tmp/dtach-ha -e ^C', timeout=30)
testbed.set_prompt('WinG% ')
testbed.send_command('exit', timeout=1)
testbed.clean_buff()
log_buff = testbed.send_command('FlushGCover ', timeout=30)
print log_buff
except:
testbed.disconnect()
log_info('flush gcov failed.')
def config_seagull(ip_addr, username, password, debug_flag):
try:
testbed=ssh_connection(ip_addr, username, password, debug_on=debug_flag)
except:
log_info('Connect failed.')
return
try:
log_buff=testbed.send_command('ps -ef | grep seagull', timeout=30)
#root 15378 1 0 10:58 ? 00:00:15 seagull -conf
for seagull_pid in re.findall('root\s+(\d+)[^\r\n]+seagull\s+\-conf',log_buff):
try:
type(seagull_pid)
testbed.send_command('kill -9 '+seagull_pid, timeout=30)
except:
pass
testbed.set_prompt('run# ')
testbed.send_command('cd /opt/seagull/diameter-env/run', timeout=30)
testbed.send_command('./start_server_eap.ksh', timeout=30)
testbed.send_command('./start_server_rf.ksh', timeout=30)
testbed.send_command('./start_server_gxx_extern.ksh', timeout=30)
log_buff=testbed.send_command('ps -ef | grep seagull', timeout=30)
print log_buff
except:
testbed.disconnect()
log_info('config_seagull failed.')
log_info('config_seagull OK.')
def config_checkone(ip_addr, username, password, debug_flag):
global g_debug
try:
testbed=ssh_connection(ip_addr, username, password, debug_on=debug_flag)
except:
log_info('Connect failed.')
return
try:
testbed.set_prompt('ue_simulator# ')
testbed.send_command('cd /home/ue_simulator', timeout=30)
log_buff=testbed.send_command('./checkone', timeout=30)
print log_buff
except:
testbed.disconnect()
log_info('config_ue_simulator failed.')
log_info('config_ue_simulator OK.')
def config_ui_one_py(ip_addr, username, password, debug_flag):
global g_debug
try:
testbed=ssh_connection(ip_addr, username, password, debug_on=debug_flag)
except:
log_info('Connect failed.')
return
try:
testbed.set_prompt('binary# ')
testbed.send_command(r'cd /home/ue_simulator/binary', timeout=30)
log_buff=testbed.send_command(r'nohup ./ui_one.py /home/ue_simulator/binary >log_ui_one.txt 2>&1 &', timeout=30)
log_buff=testbed.send_command(r'cat /home/ue_simulator/binary/log_ui_one.txt', timeout=30)
print log_buff
except:
testbed.disconnect()
log_info('config_ue_simulator failed.')
log_info('config_ue_simulator OK.')
def config_host_route(ip_addr, username, password, debug_flag):
try:
testbed=ssh_connection(ip_addr, username, password, debug_on=debug_flag)
except:
log_info('Connect failed.')
return
try:
log_buff=testbed.send_command(r'nohup route -A inet6 add 3680:95::/60 gw 3600:105::95 &', timeout=30)
log_buff=testbed.send_command(r'route -A inet6 -n', timeout=30)
print log_buff
log_buff=testbed.send_command(r'mount -t nfs 10.180.24.89:/home/ue_simulator /home/ue_simulator', timeout=30)
log_buff=testbed.send_command(r'ls /home/ue_simulator/binary', timeout=30)
print log_buff
except:
testbed.disconnect()
log_info('config_host_route failed.')
log_info('config_host_route OK.')
def config_dns_server(ip_addr, username, password, debug_flag):
try:
testbed=ssh_connection(ip_addr, username, password, debug_on=debug_flag)
except:
log_info('Connect failed.')
return
try:
log_buff=testbed.send_command('ps -ef | grep named.conf', timeout=30)
#root 6699 1 0 10:21 ? 00:00:00 /usr/local/named/sbin/named
re_pid=re.search('root\s+(\d+)[^\r\n]+named\s+\-c',log_buff)
if re_pid:
testbed.send_command('kill -9 '+re_pid.group(1), timeout=30)
testbed.send_command('nohup /usr/local/named/sbin/named -c /usr/local/named/etc/named.conf -g &', timeout=30)
log_buff=testbed.send_command('ps -ef | grep named.conf', timeout=30)
print log_buff
except:
testbed.disconnect()
log_info('config_dns_server failed.')
log_info('config_dns_server OK.')
def config_testbed_pgw(ip_addr, username, password, debug_flag):
global g_debug
try:
testbed=ssh_connection(ip_addr, username, password, debug_on=debug_flag)
except:
log_info('Connect failed.')
return
try:
testbed.set_prompt('% ')
testbed.send_command('dtach -a /tmp/dtach-ha -e ^C', timeout=30)
testbed.set_prompt('WinG% ')
testbed.send_command('exit', timeout=1)
testbed.clean_buff()
testbed.send_command('LMA DisableAuth', timeout=30)
log_buff=testbed.send_command('LMA ShowConfiguration', timeout=30)
print log_buff
except:
testbed.disconnect()
log_info('config_testbed_pgw failed.')
log_info('config_testbed_pgw OK.')
def config_testbed_hsgw(ip_addr, username, password, debug_flag):
global g_debug
try:
testbed=ssh_connection(ip_addr, username, password, debug_on=debug_flag)
except:
log_info('Connect failed.')
return
try:
testbed.send_command('killall hsgw', timeout=10)
testbed.send_command('/etc/init.d/monit restart', timeout=10)
testbed.send_command('export HW_PLATFORM=tc3k', timeout=5)
testbed.set_prompt('system# ')
testbed.send_command('su sysadmin', timeout=5)
testbed.clean_buff()
testbed.set_prompt('hsgw# ')
testbed.send_command('set context=hsgw', timeout=30)
testbed.send_command('set tm use-gxa=disabled', timeout=30)
log_buff=testbed.send_command('show tm use-gxa', timeout=30)
print log_buff
except:
testbed.disconnect()
log_info('config_testbed_hsgw failed.')
log_info('config_testbed_hsgw OK.')
global g_debug
try:
testbed=ssh_connection(ip_addr, username, password, debug_on=debug_flag)
except:
log_info('Connect failed.')
return
try:
testbed.set_prompt('% ')
testbed.send_command('dtach -a /tmp/dtach-ha -e ^C', timeout=30)
testbed.set_prompt('WinG% ')
testbed.send_command('exit', timeout=1)
testbed.clean_buff()
log_buff = testbed.send_command('FlushGCover ', timeout=30)
print log_buff
except:
testbed.disconnect()
log_info('flush gcov failed.')
def main():
debug_flag = 0
flush_gcov('10.180.24.19', 'root', 'root', debug_flag)
if __name__ == '__main__':
log_info('============= START ==============')
main()
log_info('=============THE END==============')
def __leave_x_shell(self):
self.set_prompt(self.str_app_type+'# ')
self.send_command('\3\n', timeout=2)
self.clean_buff()
log_info('leave_x_shell type='+self.str_app_type)
def __enter_ma_shell(self):
self.set_prompt('MA\$')
self.send_command('attach where=ma\n', timeout=5)
self.clean_buff()
self.send_command('exit')
log_info('enter_ma_shell type='+self.str_app_type)
def logout_CLI(self):
self.set_prompt('tmp# ')
self.send_command('exit')
self.clean_buff()
log_info('logout CLI type='+self.str_app_type)
def collect_info_on_sysmgr(slot_num,base_name,repo_path):
'''if 0 != os.system ('scp slot-%d:/opt/utsnt/run/*.dmp %s' %(slot_num,repo_path)):
log_info('Failed! scp slot-%d:/opt/utsnt/run/*.dmp %s' %(slot_num,repo_path))
if 0 != os.system ('scp slot-%d:/opt/utsnt/run/*.crashdump.* %s' %(slot_num,repo_path)):
log_info('Failed! scp slot-%d:/opt/utsnt/run/*.crashdump.* %s' %(slot_num,repo_path))
if 0 != os.system ('scp slot-%d:/tmp/*.dmp %s' %(slot_num,repo_path)):
log_info('Failed! scp slot-%d:/tmp/*.dmp %s' %(slot_num,repo_path))'''
if 0 != os.system ('cp /home/ftp/neconfig/slot-%d/*.dmp %s' %(slot_num,repo_path)):
log_info('Failed! cp /home/ftp/neconfig/slot-%d/*.dmp %s' %(slot_num,repo_path))
if 0 != os.system ('cp /var/log/tmp/slot%d.app.log* %s' %(slot_num,repo_path)):
log_info('Failed! cp /var/log/tmp/slot%d.app.log* %s' %(slot_num,repo_path))
if 0 != os.system ('cp /var/log/tmp/slot%d.ma.log* %s' %(slot_num,repo_path)):
log_info('Failed! cp /var/log/tmp/slot%d.ma.log* %s' %(slot_num,repo_path))
if 0 != os.system ('cp /var/log/tmp/slot%d.monit.log* %s' %(slot_num,repo_path)):
log_info('Failed! cp /var/log/tmp/slot%d.monit.log* %s' %(slot_num,repo_path))
if 0 != os.system ('cp /var/log/tmp/slot%d.sys.log* %s' %(slot_num,repo_path)):
log_info('Failed! cp /var/log/tmp/slot%d.sys.log* %s' %(slot_num,repo_path))
if 0 != os.system('tar -czf /home/%s.tar.gz %s' %(base_name,repo_path)):
log_info('Failed! tar -cvf /home/%s.tar %s' %(base_name,repo_path))
def collect_info_on_hsgw(int_slot_num,base_name,repo_path):
try:
ssh2slot=ssh_on_slotx(int_slot_num, 'hsgw',base_name,repo_path)
except:
log_info("ssh slot-%s failed." %int_slot_num)
return
ma_commands='''
print_mp
print_sa'''
app_commands='''
CpuUsage
dr
PI Listpi
sleep 5
PI Listpi
sleep 5
PI Listpi
sleep 5
PI Listpi
sleep 5
PI Listpi
sleep 5
PI Listpi
sleep 5
PI Listpi
sleep 5
PI Listpi
sleep 5
PI Listpi
sleep 5
PI Listpi
listpools
dp
lt
lm
Sa ListSaInfo
ui dq
rp ShowLastErr 10000
rp scs
rp ShowMsgStats
rp ShowNcepsStat
ppp ss
ppp ShowMessgeStats
ppp ShowPrepaidStats
sc ShowStats'''
cli_commands='''
show active-session summary
show vr throughput
show ppp statistics
show rp reg-req
show call-statistics statistics
show aaa di-server'''
linux_commands='''
top -b -n 8
mpstat -P ALL 2 8'''
minidump_cmds='''
genminidump.sh hsgw
sleep 2
genminidump.sh hsgw
sleep 2
genminidump.sh hsgw
sleep 2
genminidump.sh hsgw
sleep 2
genminidump.sh hsgw
sleep 2
genminidump.sh hsgw
sleep 2
genminidump.sh hsgw
sleep 2
genminidump.sh hsgw
sleep 2
genminidump.sh hsgw
sleep 2
genminidump.sh hsgw
sleep 2
scp /tmp/*.dmp 197.1.20.3:%s
scp /opt/utsnt/run/*.dmp 197.1.20.3:%s
scp /opt/utsnt/run/*.crashdump 197.1.20.3:%s
''' %(repo_path, repo_path, repo_path)
ssh2slot.exec_ma_commands(ma_commands)
ssh2slot.exec_app_commands(app_commands)
ssh2slot.exec_cli_commands(cli_commands)
ssh2slot.exec_linux_commands(linux_commands)
try:
type(sys.argv[3])
if 'unstable' == sys.argv[3]:
ssh2slot.exec_linux_commands(minidump_cmds)
except:
pass
ssh2slot.parse_dmp_files()
try:
type(sys.argv[3])
except:
log_info('collect_info_on_hsgw finished.')
return
if 'unstable' == sys.argv[3]:
start_app_cmds='''
/etc/init.d/monit stop
/etc/init.d/hsgw stop
/opt/utsnt/sbin/hsgw'''
ssh2slot.exec_linux_commands(start_app_cmds)
ssh2slot.ssh_handle.send('\3')
ssh2slot.send_command('y', timeout=2)
ssh2slot.clean_buff()
start_app_cmds='''
/etc/init.d/monit start
/etc/init.d/monit start'''
ssh2slot.exec_linux_commands(start_app_cmds)
log_info('restart app finished.')