def set_search_list(cls):
try:
for cmd in cls.command_list:
cls.cli_search_list.append(cmd)
cls.cli_validate_list.append(cmd)
if (CONFIG.get_config_instance().has_section(cmd)):
opt_list = CONFIG.cli_get_value(cmd, CONFIG.get_cmd_opt_key_name())
tmp = []
for opt in opt_list.split(','):
tmp.append(opt.strip())
cls.cli_validate_list.append(cmd + ' ' + opt.strip())
cls.cli_search_list_sub[cmd] = tmp
cls.cli_search_list.append('menu')
cls.cli_search_list.append('quit')
cls.cli_search_list.append('exit')
cls.cli_search_list.append('sys')
cls.cli_search_list.append('dis-system')
cls.cli_search_list.append('help')
tmp_sys = []
tmp_sys.append('all')
cls.cli_validate_list.append('sys all')
for onos_name in SYS.get_sys_list():
tmp_sys.append(onos_name)
cls.cli_validate_list.append('sys ' + onos_name)
cls.cli_search_list_sub['sys'] = tmp_sys
except:
LOG.exception_err_write()
def draw_select(menu_list, selected_menu_no):
box_type = curses.newwin(
len(menu_list) + 2, MAIN_WIDTH,
SYS.get_sys_line_count() + 3 + 3, 1)
box_type.box()
try:
highlightText = curses.color_pair(1)
normalText = curses.A_NORMAL
box_type.addstr(0, MAIN_CENTER - 3, ' MENU ', normalText)
for i in range(1, len(menu_list) + 1):
if i is selected_menu_no:
box_type.addstr(i, 2,
str(i) + "." + menu_list[i - 1],
highlightText)
else:
box_type.addstr(i, 2,
str(i) + "." + menu_list[i - 1],
normalText)
except:
LOG.exception_err_write()
return box_type
def draw_event(cls, type='default'):
try:
warn_color = curses.color_pair(3)
box_event = curses.newwin(3, MAIN_WIDTH, SYS.get_sys_line_count() + 3, 1)
box_event.box()
normalText = curses.A_NORMAL
box_event.addstr(0, 22, ' EVENT ', normalText)
if type == 'disconnect':
box_event.addstr(1, 2, '[Server shutdown] check server and restart', warn_color)
elif type == 'rest_warn':
box_event.addstr(1, 2, '[Rest failure] check client and restart', warn_color)
else:
# if occur event
if SYS.abnormal_flag:
str = '[Event occurred]'
box_event.addstr(1, 2, str, warn_color)
box_event.addstr(1, 2 + len(str), ' Check the event history.', normalText)
else:
str = '[Event] normal'
box_event.addstr(1, 2, str, normalText)
box_event.refresh()
except:
LOG.exception_err_write()
def listen_evt(evt):
while SYS.get_sys_thr_flag():
evt.wait(3)
if evt.is_set():
evt.clear()
# system check
check_system()
def listen_disconnect_evt(evt):
while SYS.get_sys_thr_flag():
evt.wait(3)
if evt.is_set():
SYS.disconnect_flag = True
SCREEN.draw_event(SYS.disconnect_flag)
evt.clear()
def draw_refresh_time(cls, menu_list):
try:
time_color = curses.color_pair(3)
str_time = 'Last Check Time [' + SYS.last_check_time.split('.')[0] + ']'
cls.set_main_str(SYS.get_sys_line_count() + 2 + 3 + len(menu_list) + 2 + 1, MAIN_WIDTH - len(str_time),
str_time, time_color)
except:
LOG.exception_err_write()
def listen_disconnect_evt(evt, rest_evt):
global evt_thread
while SYS.get_sys_thr_flag():
try:
evt.wait(1)
if evt.is_set():
LOG.debug_log('Get disconnect event')
evt.clear()
SYS.disconnect_type = 'disconnect'
if SYS.get_sys_redraw_flag():
SCREEN.draw_event(SYS.disconnect_type)
SCREEN.get_screen().clear()
SCREEN.screen_exit()
print '\nCheck monitoring server.'
os.killpg(os.getpid(), signal.SIGKILL)
#os.kill(os.getpid(), signal.SIGKILL)
time.sleep(1)
rest_evt.wait(1)
if rest_evt.is_set():
LOG.debug_log('Get rest error event')
rest_evt.clear()
SYS.disconnect_type = 'rest_warn'
if SYS.get_sys_redraw_flag():
SCREEN.draw_event(SYS.disconnect_type)
SCREEN.get_screen().clear()
SCREEN.screen_exit()
print '\nCheck client rest server.'
os.killpg(os.getpid(), signal.SIGKILL)
#os.kill(os.getpid(), signal.SIGKILL)
time.sleep(1)
except:
LOG.exception_err_write()
def check_system():
try:
# inquiry onos info
res_code, sys_info = CLI.req_sys_info()
if res_code != 200:
SYS.disconnect_type = 'disconnect'
SCREEN.draw_event(SYS.disconnect_type)
LOG.debug_log(
'[SYSTEM_CHECK_THREAD] Rest server does not respond to the request. RES_CODE = ' + str(res_code))
return
ret = SYS.changed_sys_info(sys_info)
if SYS.get_sys_redraw_flag():
if ret is True:
SCREEN.draw_system(menu_list)
SCREEN.draw_event(SYS.disconnect_type)
else:
SCREEN.draw_refresh_time(menu_list)
except:
LOG.exception_err_write()
def listen_evt(evt):
while SYS.get_sys_thr_flag():
try:
evt.wait(1)
if evt.is_set():
evt.clear()
# system check
check_system()
time.sleep(1)
except:
LOG.exception_err_write()
def display_sys_info(cls):
box_sys = curses.newwin(SYS.get_sys_line_count() + 2, MAIN_WIDTH, 1, 1)
box_sys.box()
try:
status_text_OK = curses.color_pair(2)
status_text_NOK = curses.color_pair(3)
normal_text = curses.A_NORMAL
box_sys.addstr(0, 18, ' CONTROL PLAN ', normal_text)
i = 1
sorted_list = sorted(SYS.sys_list.keys())
alarm_flag = False
for sys in sorted_list:
str_info = sys.ljust(6) + ' ['
box_sys.addstr(i, 2, str_info)
str_status = 'OK'
status_list = (dict)(SYS.sys_list[sys]).values()
for status in status_list:
if (status == 'none'):
str_status = 'loading'
break
elif not (status == 'ok' or status == 'normal'):
str_status = 'NOK'
alarm_flag = True
break
if str_status is 'OK':
box_sys.addstr(i, 2 + len(str_info), str_status,
status_text_OK)
else:
box_sys.addstr(i, 2 + len(str_info), str_status,
status_text_NOK)
box_sys.addstr(i, 2 + len(str_info) + len(str_status), ']')
i += 1
SYS.abnormal_flag = alarm_flag
except:
LOG.exception_err_write()
return box_sys
def set_search_list(cls):
try:
for cmd in cls.command_list:
cls.cli_search_list.append(cmd)
cls.cli_validate_list.append(cmd)
if (CONFIG.get_config_instance().has_section(cmd)):
opt_list = CONFIG.cli_get_value(
cmd, CONFIG.get_cmd_opt_key_name())
tmp = []
for opt in opt_list.split(','):
tmp.append(opt.strip())
cls.cli_validate_list.append(cmd + ' ' + opt.strip())
cls.cli_search_list_sub[cmd] = tmp
cls.cli_search_list.append('menu')
cls.cli_search_list.append('exit')
cls.cli_search_list.append('sys')
cls.cli_search_list.append('onos-shell')
cls.cli_search_list.append('os-shell')
cls.cli_search_list.append('monitoring-details')
cls.cli_search_list.append('event-history')
cls.cli_search_list.append('flow-trace')
cls.cli_search_list.append('traffic-test')
cls.cli_search_list.append('help')
onos_list = []
shell_list = []
tmp_sys = []
tmp_sys.append('all')
cls.cli_validate_list.append('sys all')
for sys_name in SYS.get_sys_list():
tmp_sys.append(sys_name)
cls.cli_validate_list.append('sys ' + sys_name)
cls.cli_validate_list.append('os-shell ' + sys_name)
if dict(SYS.sys_list[sys_name])['TYPE'] == 'ONOS':
onos_list.append(sys_name)
shell_list.append(sys_name)
cls.cli_search_list_sub['sys'] = tmp_sys
cls.cli_search_list_sub['onos-shell'] = onos_list
cls.cli_search_list_sub['os-shell'] = shell_list
except:
LOG.exception_err_write()
def display_sys_info(cls):
box_sys = curses.newwin(SYS.get_sys_line_count() + 2, MAIN_WIDTH, 1, 1)
box_sys.box()
try:
status_text_OK = curses.color_pair(2)
status_text_NOK = curses.color_pair(3)
normal_text = curses.A_NORMAL
box_sys.addstr(0, 16, ' MONITORING STATUS ', normal_text)
i = 1
sorted_list = sorted(SYS.sys_list.keys())
for sys in sorted_list:
str_info = sys.ljust(6) + ' ['
box_sys.addstr(i, 2, str_info)
str_status = 'OK'
status_list = (dict)(SYS.sys_list[sys]).keys()
for key in status_list:
if str(key).upper() == 'TYPE' or str(key).upper() == 'IP':
continue
value = (dict)(SYS.sys_list[sys])[key]
if (value == 'none'):
str_status = 'loading'
break
elif not (value == 'ok' or value == 'normal' or value == '-'):
str_status = 'NOK'
SYS.abnormal_flag = True
break
if str_status is 'OK':
box_sys.addstr(i, 2 + len(str_info), str_status, status_text_OK)
else:
box_sys.addstr(i, 2 + len(str_info), str_status, status_text_NOK)
box_sys.addstr(i, 2 + len(str_info) + len(str_status), ']')
i += 1
except:
LOG.exception_err_write()
return box_sys
def display_sys_info(cls):
box_sys = curses.newwin(SYS.get_sys_line_count() + 2, MAIN_WIDTH, 1, 1)
box_sys.box()
try:
status_text_OK = curses.color_pair(2)
status_text_NOK = curses.color_pair(3)
normal_text = curses.A_NORMAL
box_sys.addstr(0, MAIN_CENTER - 10, ' MONITORING STATUS ',
normal_text)
i = 1
sorted_list = sorted(SYS.sys_list.keys())
for sys in sorted_list:
# show node name
col = 2
str_info = sys.ljust(6)
box_sys.addstr(i, col, str_info)
col += len(str_info) + 1
# show status
box_sys.addstr(i, col, "[")
col += 1
str_status = 'OK'
status_list = (dict)(SYS.sys_list[sys]).keys()
for key in status_list:
if str(key).upper() == 'TYPE' or str(key).upper() == 'IP':
continue
value = (dict)(SYS.sys_list[sys])[key]
if (value == 'none'):
str_status = 'LOADING'
break
elif not (value == 'ok' or value == 'normal'
or value == '-'):
str_status = 'NOK'
SYS.abnormal_flag = True
break
if str_status is 'OK':
box_sys.addstr(i, col, str_status, status_text_OK)
else:
box_sys.addstr(i, col, str_status, status_text_NOK)
box_sys.addstr(i, col + len(str_status), ']')
col += 8
# show base info
#type = 'TYPE=' + SYS.sys_list[sys]['TYPE']
#box_sys.addstr(i, col, type)
#col += len(type) + 1
#ip = '(IP=' + SYS.sys_list[sys]['IP'] + ')'
#box_sys.addstr(i, col, ip)
#col += len(ip) + 1
#i += 1
#col = 9;
# show monitoring details
if (SYS.sys_list[sys]["TYPE"] == 'ONOS'):
box_sys.addstr(i, col, '(')
col += 1
for key in ONOS_STATUS_ITEMS:
value = (dict)(SYS.sys_list[sys])[key]
if str(key).startswith('ONOS_'):
key = key[len('ONOS_'):]
box_sys.addstr(i, col, key + '=')
col += len(key + '=')
str_status = 'ok'
if not (value == 'ok' or value == 'normal'
or value == '-'):
str_status = 'nok'
if str_status is 'ok':
box_sys.addstr(i, col, str_status, status_text_OK)
else:
box_sys.addstr(i, col, str_status, status_text_NOK)
col += len(str_status) + 1
box_sys.addstr(i, col - 1, ')')
i += 1
except:
LOG.exception_err_write()
return box_sys
def main():
global evt_thread
global conn_evt_thread
try:
from PIL import Image
except:
print "This program requires the pillow package."
print 'Please run \'sudo pip install pillow\' and try again.'
return
try:
from asciimatics.scene import Scene
except:
print "This program requires the asciimatics package."
print 'Please run \'sudo pip install asciimatics\' and try again.'
return
# add for readline bug fix
os.unsetenv('LINES')
os.unsetenv('COLUMNS')
# read config
if not CONFIG.init_config(LOG):
print 'read config fail...'
return
# set log
LOG.set_default_log('sonawatched_err.log')
# set history log
history_log = USER_LOG()
history_log.set_log('evt_history.log', CONFIG.get_cli_log_rotate(), int(CONFIG.get_cli_log_backup()))
# set cli log
cli_log = USER_LOG()
cli_log.set_log('sonawatched_cli.log', CONFIG.get_cli_log_rotate(), int(CONFIG.get_cli_log_backup()))
CLI.set_cli_log(cli_log, history_log)
# set trace log
trace_log = USER_LOG()
trace_log.set_log('sonawatched_trace.log', CONFIG.get_trace_log_rotate(), int(CONFIG.get_trace_log_backup()))
TRACE.set_trace_log(trace_log)
# read log option
LOG.set_log_config()
# Start RESTful server for event
evt = multiprocessing.Event()
disconnect_evt = multiprocessing.Event()
rest_evt = multiprocessing.Event()
# create listen event thread
evt_thread = threading.Thread(target=listen_evt, args=(evt,))
evt_thread.daemon = False
evt_thread.start()
conn_evt_thread = threading.Thread(target=listen_disconnect_evt, args=(disconnect_evt, rest_evt))
conn_evt_thread.daemon = False
conn_evt_thread.start()
try:
# create rest server process
child_pid = cli_rest.rest_server_start(evt, disconnect_evt, rest_evt, history_log)
except:
print 'Rest Server failed to start'
print 'Processing shutdown...'
LOG.exception_err_write()
SYS.set_sys_thr_flag(False)
conn_evt_thread.join()
evt_thread.join()
return
# regi event
if CLI.send_regi() == False:
print 'Event registration failed'
print 'Processing shutdown...'
LOG.exception_err_write()
SYS.set_sys_thr_flag(False)
conn_evt_thread.join()
evt_thread.join()
return
# get event list
if CLI.get_event_list() == False:
print 'Get Event List failed'
print 'Processing shutdown...'
LOG.exception_err_write()
SYS.set_sys_thr_flag(False)
conn_evt_thread.join()
evt_thread.join()
return
# inquiry controller info
try:
res_code, sys_info = CLI.req_sys_info()
except:
print "Cannot connect rest server."
print 'Processing shutdown...'
SYS.set_sys_thr_flag(False)
conn_evt_thread.join()
evt_thread.join()
return
if res_code != 200:
print "Rest server does not respond to the request."
print "code = " + str(res_code)
print 'Processing shutdown...'
if not SYS.disconnect_type == 'disconnect':
CLI.send_regi('unregi')
SYS.set_sys_thr_flag(False)
conn_evt_thread.join()
evt_thread.join()
return
SYS.set_sys_info(sys_info)
# set command list
CLI.set_cmd_list()
# set trace cond list
TRACE.set_cnd_list()
# set search list
CLI.set_search_list()
set_readline_opt()
# system check
check_system()
# select input menu
select_menu()
# exit
print 'Processing shutdown...'
if not SYS.disconnect_type == 'disconnect':
CLI.send_regi('unregi')
SYS.set_sys_thr_flag(False)
conn_evt_thread.join()
evt_thread.join()
return
def select_menu():
selected_menu_no = 1
try:
SCREEN.set_screen()
SCREEN.draw_system(menu_list)
SCREEN.draw_event(SYS.disconnect_type)
SCREEN.draw_menu(menu_list, selected_menu_no)
SYS.set_sys_redraw_flag(True)
x = SCREEN.get_ch()
# 27 = ESC
while x != 27:
if x == curses.KEY_DOWN:
if selected_menu_no != len(menu_list):
selected_menu_no += 1
elif x == curses.KEY_UP:
if selected_menu_no != 1:
selected_menu_no -= 1
elif x == ord("\n"):
# stop timer
SYS.set_sys_redraw_flag(False)
SCREEN.refresh_screen()
SCREEN.screen_exit()
menu = menu_list[selected_menu_no - 1]
if menu in ['CLI', 'Monitoring Details', 'Event History']:
if menu == 'CLI':
SCREEN.display_header(menu_list[selected_menu_no - 1])
SCREEN.display_sys(True)
elif menu == 'Monitoring Details':
SCREEN.display_status()
elif menu == 'Event History':
SCREEN.display_event()
readline.set_completer(CLI.pre_complete_cli)
while True:
# mac OS
if 'libedit' in readline.__doc__:
CLI.modify_flag = True
CLI.save_buffer = readline.get_line_buffer()
# select_command (handling tab event)
cmd = CLI.input_cmd()
cmd = cmd.strip()
if CLI.is_menu(cmd):
break
elif CLI.is_exit(cmd):
return
elif cmd == 'help':
SCREEN.display_help()
elif cmd == 'monitoring-details':
SCREEN.display_status()
elif cmd.startswith('event-history'):
if len(cmd.split(' ')) == 2:
try:
num = int(cmd.split(' ')[1])
SCREEN.display_event(num)
except:
print 'log count param is wrong. ex) event-history 15'
else:
SCREEN.display_event()
elif cmd == 'flow-trace':
source_ip = raw_input('CLI(flow-trace)> source ip : ')
dest_ip = raw_input('CLI(flow-trace)> destination ip : ')
TRACE.process_trace_rest(source_ip, dest_ip)
elif cmd.startswith('traffic-test'):
if cmd.startswith('traffic-test -f '):
try:
filename = cmd.split(' ')[2]
except:
filename = ''
if not os.path.exists(filename):
print 'file does not exist [' + filename + ']'
continue
result_file = open(filename, 'r')
test_list = []
for line in result_file.read().splitlines():
if line.startswith('#'):
continue
tmp = line.split(';')
test_dict = dict()
test_dict['node'] = tmp[0]
test_dict['instance_id'] = tmp[1]
test_dict['vm_user_id'] = tmp[2]
test_dict['vm_user_password'] = tmp[3]
test_dict['traffic_test_command'] = tmp[4]
try:
test_dict['next_command_interval'] = int(tmp[5])
except:
test_dict['next_command_interval'] = 0
test_list.append(test_dict)
TRACE.process_traffic_test(test_list)
else:
yes_no = 'y'
test_list = []
while yes_no.upper() == 'Y':
test_dict = dict()
test_dict['node'] = raw_input('CLI(traffic-test)> node : ')
test_dict['instance_id'] = raw_input('CLI(traffic-test)> instance id : ')
test_dict['vm_user_id'] = raw_input('CLI(traffic-test)> vm user id : ')
test_dict['vm_user_password'] = raw_input('CLI(traffic-test)> vm user pw : ')
test_dict['traffic_test_command'] = raw_input('CLI(traffic-test)> traffic test command : ')
try:
test_dict['next_command_interval'] = int(raw_input('CLI(traffic-test)> next command interval : '))
except:
test_dict['next_command_interval']= 0
yes_no = raw_input('CLI(traffic-test)> Do you want to add test conditions?(y/n) ')
test_list.append(test_dict)
TRACE.process_traffic_test(test_list)
else:
# send command
CLI.process_cmd(cmd)
while not CLI.get_cli_ret_flag():
time.sleep(1)
elif menu == 'Flow Trace':
from asciimatics.screen import Screen
from asciimatics.exceptions import ResizeScreenError
# clear screen
SCREEN.get_screen().clear()
SCREEN.screen_exit()
last_scene = None
while True:
try:
Screen.wrapper(SCREEN.start_screen, catch_interrupt=True, arguments=[last_scene])
if SCREEN.quit_flag:
return
elif SCREEN.menu_flag:
SCREEN.set_screen()
SCREEN.main_scr.timeout(-1)
break
elif SCREEN.resize_err_flag:
SCREEN.draw_trace_warning()
SCREEN.screen_exit()
SCREEN.menu_flag = True
return
except ResizeScreenError as e:
last_scene = e.scene
elif menu == 'Quit':
break
SCREEN.draw_system(menu_list)
SCREEN.draw_event(SYS.disconnect_type)
SCREEN.draw_menu(menu_list, selected_menu_no)
SCREEN.refresh_screen()
SYS.set_sys_redraw_flag(True)
x = SCREEN.get_ch()
SCREEN.screen_exit()
except:
LOG.exception_err_write()
def select_menu():
selected_menu_no = 1
try:
SCREEN.set_screen()
SCREEN.draw_system(menu_list)
SCREEN.draw_event(SYS.disconnect_flag)
SCREEN.draw_menu(menu_list, selected_menu_no)
SYS.set_sys_redraw_flag(True)
x = SCREEN.get_ch()
# 27 = ESC
while x != 27:
if x == curses.KEY_DOWN:
if selected_menu_no != len(menu_list):
selected_menu_no += 1
elif x == curses.KEY_UP:
if selected_menu_no != 1:
selected_menu_no -= 1
elif x == ord("\n"):
# stop timer
SYS.set_sys_redraw_flag(False)
# ?? is it necessary?
SCREEN.refresh_screen()
SCREEN.screen_exit()
menu = menu_list[selected_menu_no - 1]
if menu == 'CLI' or menu == 'Event List':
if menu == 'CLI':
SCREEN.display_header(menu_list[selected_menu_no - 1])
SCREEN.display_sys(True)
elif menu == 'Event List':
SCREEN.display_event()
readline.set_completer(CLI.pre_complete_cli)
while True:
# mac OS
if 'libedit' in readline.__doc__:
CLI.modify_flag = True
CLI.save_buffer = readline.get_line_buffer()
# select_command (handling tab event)
cmd = CLI.input_cmd()
cmd = cmd.strip()
if CLI.is_menu(cmd):
break
elif CLI.is_exit(cmd):
return
elif cmd == 'dis-system':
SCREEN.display_sys()
elif cmd == 'help':
SCREEN.display_help()
else:
# send command
CLI.process_cmd(cmd)
while not CLI.get_cli_ret_flag():
time.sleep(1)
elif menu == 'Flow Trace':
from asciimatics.screen import Screen
from asciimatics.exceptions import ResizeScreenError
# clear screen
SCREEN.get_screen().clear()
SCREEN.screen_exit()
last_scene = None
while True:
try:
Screen.wrapper(SCREEN.start_screen,
catch_interrupt=True,
arguments=[last_scene])
if SCREEN.quit_flag:
return
elif SCREEN.menu_flag:
return
elif SCREEN.resize_err_flag:
SCREEN.draw_trace_warning()
SCREEN.screen_exit()
SCREEN.menu_flag = True
return
except ResizeScreenError as e:
last_scene = e.scene
SCREEN.draw_system(menu_list)
SCREEN.draw_event(SYS.disconnect_flag)
SCREEN.draw_menu(menu_list, selected_menu_no)
SCREEN.refresh_screen()
SYS.set_sys_redraw_flag(True)
x = SCREEN.get_ch()
SCREEN.screen_exit()
except:
LOG.exception_err_write()