def stop(self):
try:
pid = self.get_pid()
except IOError:
pid = None
if not pid:
message = "pidfile %s does not exist. Daemon not running?\n"
sys.stderr.write(message % self.pidfile)
return # not an error in a restart
# Try killing the daemon process
try:
LOG.info("--- Daemon STOP ---")
while 1:
for cpid in self.get_child_pid(pid):
os.kill(cpid, SIGTERM)
os.kill(pid, SIGTERM)
time.sleep(0.1)
except OSError, err:
err = str(err)
if err.find("No such process") > 0:
if os.path.exists(self.pidfile):
os.remove(self.pidfile)
else:
print str(err)
print "Stopping Fail ..."
sys.exit(1)
def onsway_trace(sona_topology, trace_conditions):
retry_flag = True
up_down_result = []
is_success = False
while retry_flag:
ssh_result = SshCommand.ssh_exec(
CONF.openstack()['account'].split(':')[0],
trace_conditions.cur_target_ip, make_command(trace_conditions))
LOG.info('target_node = ' + trace_conditions.cur_target_ip)
LOG.info('TRACE RESULT = ' + str(ssh_result))
node_trace = dict()
node_trace['trace_node_name'] = trace_conditions.cur_target_hostname
process_result, retry_flag, is_success = process_trace(
ssh_result, sona_topology, trace_conditions)
node_trace['flow_rules'] = process_result
trace_conditions.cond_dict['in_port'] = ''
trace_conditions.cond_dict['dl_src'] = ''
trace_conditions.cond_dict['dl_dst'] = ''
trace_conditions.cond_dict['eth_dst'] = ''
trace_conditions.cond_dict['eth_src'] = ''
up_down_result.append(node_trace)
return up_down_result, is_success
def connection():
try:
conn = sqlite3.connect(CONF.base()['db_file'])
conn.isolation_level = None
return conn
except:
LOG.exception()
sys.exit(1)
def get_floatingip(nova_credentials):
for network in nova_credentials.networks.list():
try:
new_floatingip = nova_credentials.floating_ips.create(pool=network.label)
if new_floatingip:
LOG.info("[Create Floating_IP] Success %s", new_floatingip.ip)
return new_floatingip
except client.exceptions.NotFound:
LOG.info("[Create Floating_IP] Fail from %s network", network.label)
def __init__(self, openstack_only=False):
if not openstack_only:
self.subnets = self.get_subnets(self.get_onos_ip())
self.floatingips = self.get_floatingips(self.get_onos_ip())
self.routers = self.get_routers(self.get_onos_ip())
LOG.info('router = ' + str(self.routers))
self.openstack = self.get_openstacknodes(self.get_onos_ip())
def sql_insert_nodes(cls,
db_log,
node_list,
username,
type,
sub_type='none'):
try:
for node in node_list:
name, ip = str(node).split(':')
db_log.write_log('Insert node [%s %s %s %s]', name, ip,
username, type)
sql = 'INSERT INTO ' + cls.NODE_INFO_TBL + \
' VALUES (\'' + name + '\', \'' + ip + '\', \'' + username + '\', \'' + type.upper() + '\', \'' + sub_type.upper() + '\')'
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log(
" [NODE TABLE] Node data insert fail \n%s", sql_rt)
sys.exit(1)
# set status tbl
sql = 'INSERT INTO ' + cls.STATUS_TBL + \
' VALUES (\'' + name + '\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\')'
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log(
" [STATUS TABLE] Status data insert fail \n%s", sql_rt)
sys.exit(1)
# add Alarm Items
evt_list = DB.get_event_list(type)
for item in evt_list:
db_log.write_log('Insert item [%s %s]', name, item)
sql = 'INSERT INTO ' + cls.EVENT_TBL + \
' VALUES (\'' + name + '\',\'' + item + '\', \'none\', \'none\', \'none\', \'none\')'
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log(
" [EVENT TABLE] Event data insert fail \n%s",
sql_rt)
sys.exit(1)
if type.upper() == 'ONOS':
# set app tbl
sql = 'INSERT INTO ' + cls.ONOS_TBL + ' VALUES (\'' + name + '\', \'none\', \'none\', \'none\', \'none\')'
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log(
" [ONOS TABLE] Onos data insert fail \n%s", sql_rt)
sys.exit(1)
except:
LOG.exception()
def make_command(trace_conditions):
cond_list = ''
for key in dict(trace_conditions.cond_dict).keys():
value = trace_conditions.cond_dict[key]
if value != '':
cond_list = cond_list + key + '=' + str(value) + ','
command = 'sudo ovs-appctl ofproto/trace br-int \'' + cond_list.rstrip(
',') + '\''
LOG.info('trace command = ' + command)
return command
def server_vm_check(server_ip, client_floatingip):
for i in range(20):
check_result = SshCommand.ssh_tperf_exec(CONF.openstack()['key_file'],
CONF.openstack()['tperf_vm_username'],
client_floatingip,
'ping -c 1 ' + server_ip + ' | grep transmitted',
timeout=2)
if ' 0% packet loss' in check_result.split(','):
return True
else:
LOG.error('[Server Network Check Fail and Retry %d', i)
time.sleep(1)
return False
def ssh_exec(cls, username, node, command):
cmd = 'ssh %s %s@%s %s' % (cls.ssh_options(), username, node, command)
try:
result = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
output, error = result.communicate()
if result.returncode != 0:
LOG.error("\'%s\' SSH_Cmd Fail, cause => %s", node, error)
return
else:
# LOG.info("ssh command execute successful \n%s", output)
return output
except:
LOG.exception()
def db_initiation(cls):
LOG.info("--- Initiating SONA DB ---")
init_sql = [
'CREATE TABLE ' + cls.NODE_INFO_TBL +
'(nodename text primary key, ip_addr, username)',
'CREATE TABLE ' + cls.STATUS_TBL +
'(nodename text primary key, ping, app, cpu, memory, disk, time)',
'CREATE TABLE ' + cls.RESOURCE_TBL +
'(nodename text primary key, cpu real, memory real, disk real)',
'CREATE TABLE ' + cls.REGI_SYS_TBL +
'(url text primary key, auth)', 'CREATE TABLE ' + cls.EVENT_TBL +
'(nodename, item, grade, desc, time, PRIMARY KEY (nodename, item))'
]
for sql in init_sql:
sql_rt = cls.sql_execute(sql)
if "already exist" in sql_rt:
table_name = sql_rt.split()[1]
LOG.info(
"\'%s\' table already exist. Delete all tuple of this table...",
table_name)
sql = 'DELETE FROM ' + table_name
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
LOG.info("DB %s table initiation fail\n%s", table_name,
sql_rt)
sys.exit(1)
elif sql_rt != 'SUCCESS':
LOG.info("DB initiation fail\n%s", sql_rt)
sys.exit(1)
LOG.info('Insert nodes information ...')
for node in CONF.watchdog()['check_system']:
if str(node).lower() == 'onos':
cls.sql_insert_nodes(CONF.onos()['list'],
str(CONF.onos()['account']).split(':')[0])
elif str(node).lower() == 'xos':
cls.sql_insert_nodes(CONF.xos()['list'],
str(CONF.xos()['account']).split(':')[0])
elif str(node).lower() == 'swarm':
cls.sql_insert_nodes(
CONF.swarm()['list'],
str(CONF.swarm()['account']).split(':')[0])
elif str(node).lower() == 'openstack':
cls.sql_insert_nodes(
CONF.openstack()['list'],
str(CONF.openstack()['account']).split(':')[0])
def sql_execute(cls, sql, conn=None):
try:
if conn == None:
with cls.connection() as conn:
conn.cursor().execute(sql)
conn.commit()
conn.close()
else:
conn.cursor().execute(sql)
conn.commit()
return 'SUCCESS'
except sqlite3.OperationalError, err:
LOG.error(err.message)
return err.message
def ssh_pexpect(cls, username, node, onos_ip, command):
cmd = 'ssh %s %s@%s' % (cls.ssh_options(), username, node)
try:
LOG.info('ssh_pexpect cmd = ' + cmd)
ssh_conn = pexpect.spawn(cmd)
rt1 = ssh_conn.expect(['#', '\$', pexpect.EOF],
timeout=CONF.ssh_conn()['ssh_req_timeout'])
if rt1 == 0:
cmd = 'ssh -p 8101 karaf@' + onos_ip + ' ' + command
LOG.info('ssh_pexpect cmd = ' + cmd)
ssh_conn.sendline(cmd)
rt2 = ssh_conn.expect(
['Password:'******'ssh_req_timeout'])
if rt2 == 0:
ssh_conn.sendline('karaf')
ssh_conn.expect(['#', '\$', pexpect.EOF],
timeout=CONF.ssh_conn()['ssh_req_timeout'])
str_output = str(ssh_conn.before)
ret = ''
for line in str_output.splitlines():
if (line.strip() == '') or ('#' in line) or (
'$' in line) or ('~' in line) or ('@' in line):
continue
ret = ret + line + '\n'
return ret
else:
return "fail"
elif rt1 == 1:
LOG.error('%s', ssh_conn.before)
elif rt1 == 2:
LOG.error("[ssh_pexpect] connection timeout")
return "fail"
except:
LOG.exception()
return "fail"
def onos_ssh_exec(cls, node_ip, command):
local_ssh_options = cls.ssh_options() + " -p 8101"
cmd = 'ssh %s %s %s' % (local_ssh_options, node_ip, command)
try:
result = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
output, error = result.communicate()
if result.returncode != 0:
LOG.error("ONOS(%s) SSH_Cmd Fail, cause => %s", node_ip, error)
return
else:
# LOG.info("ONOS ssh command execute successful \n%s", output)
return output
except:
LOG.exception()
def flush_pending_alarm():
global alarm_count, alarm_subject, alarm_body
if alarm_count <= 0:
return; # no alarm pending
conf = CONF.alarm()
# copy to local variables and clear global variables
count = alarm_count
subject = '[%s] %s' % (conf['site_name'], alarm_subject)
if (count > 1):
subject += ' (+ %d events)' % (count - 1)
body = alarm_body
alarm_count = 0
alarm_subject = ''
alarm_body = ''
if conf['mail_alarm']:
mail_from = conf['mail_user'] + '@' + conf['mail_server'].split(':')[0]
# send to each mail_list entry for gmail smtp seems not handling mutiple To: addresses
for mail_to in conf['mail_list']:
msg = MIMEText(body)
msg['Subject'] = subject
msg['From'] = mail_from
msg['To'] = mail_to
LOG.info('Send Email Alarm: subject=%s to=%s body=%s', subject, mail_to, body)
try:
ms = smtplib.SMTP(conf['mail_server'])
if conf['mail_tls']:
ms.starttls()
ms.login(conf['mail_user'], conf['mail_password'])
ms.sendmail(mail_from, mail_to, msg.as_string())
ms.quit()
except:
LOG.exception()
if conf['slack_alarm']:
ch = conf['slack_channel'].strip()
if ch[0] != '#':
ch = '#' + ch
LOG.info('Send Slack Alarm: channel=%s text=%s', ch, body)
sc = SlackClient(conf['slack_token'])
try:
sc.api_call("chat.postMessage", channel=ch, text=body)
except:
LOG.exception()
def ssh_tperf_exec(cls, keypair, username, node_ip, command, timeout):
ssh_options = '-o StrictHostKeyChecking=no ' \
'-o ConnectTimeout=' + str(timeout)
if not os.path.exists(keypair):
LOG.error('[SSH Fail] keypaire file not exist. ---')
return 'fail'
cmd = 'ssh %s -i %s %s@%s %s' % (ssh_options, keypair, username, node_ip, command)
LOG.info("[SB SSH CMD] cmd = %s", cmd)
try:
result = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
(output, error) = result.communicate()
if result.returncode != 0:
LOG.error("\'%s\' SSH_Cmd Fail, cause(%d) => %s", node_ip, result.returncode, str(error))
return 'fail'
else:
LOG.info("ssh command execute successful \n >> [%s]", output)
return output
except:
LOG.exception()
pass
def db_initiation(cls, db_log):
try:
db_path = CONF.base()['db_file']
if os.path.isfile(db_path):
os.remove(db_path)
db_log.write_log("--- Initiating SONA DB ---")
init_sql = [
'CREATE TABLE ' + cls.NODE_INFO_TBL +
'(nodename text primary key, ip_addr, username, type, sub_type)',
'CREATE TABLE ' + cls.STATUS_TBL +
'(nodename text primary key, ' + cls.item_list + ', time)',
'CREATE TABLE ' + cls.REGI_SYS_TBL +
'(url text primary key, auth)',
'CREATE TABLE ' + cls.EVENT_TBL +
'(nodename, item, grade, pre_grade, reason, time, PRIMARY KEY (nodename, item))',
'CREATE TABLE ' + cls.ONOS_TBL +
'(nodename text primary key, cluster, device, link, app)'
]
for sql in init_sql:
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log("DB initiation fail\n%s", sql_rt)
sys.exit(1)
db_log.write_log('Insert nodes information ...')
for node_type in CONF.watchdog()['check_system']:
cls.sql_insert_nodes(
db_log, (CONF_MAP[node_type.upper()]())['list'],
str((CONF_MAP[node_type.upper()]()
)['account']).split(':')[0], node_type)
except:
LOG.exception()
def find_target(trace_condition, sona_topology):
# find switch info
LOG.info('source ip = ' + trace_condition.cond_dict['nw_src'])
result = sona_topology.get_hosts(
sona_topology.get_onos_ip(),
'\"\[' + trace_condition.cond_dict['nw_src'] + '\]\"')
# source ip = internal vm
if result != None:
switch_info = (result.split(',')[2].split('=')[1])[1:-1]
switch_id = switch_info.split('/')[0]
vm_port = switch_info.split('/')[1]
LOG.info('swtich id = ' + switch_id)
# find target node
if trace_condition.cond_dict['in_port'] == '':
trace_condition.cond_dict['in_port'] = vm_port
trace_condition.cur_target_ip = sona_topology.get_openstack_info(
' ' + switch_id + ' ', 'ip')
trace_condition.cur_target_hostname = sona_topology.get_openstack_info(
' ' + switch_id + ' ', 'hostname')
if trace_condition.cur_target_ip == '' or trace_condition.cur_target_hostname == '':
return False
# source ip = external
else:
if not trace_condition.cond_dict['nw_src'] in sona_topology.routers:
for net in sona_topology.subnets:
if IPAddress(
trace_condition.cond_dict['nw_src']) in IPNetwork(net):
return False
if trace_condition.cond_dict[
'nw_dst'] in sona_topology.floatingips.values():
LOG.info('floating ip : ' + str(sona_topology.floatingips))
for key in sona_topology.floatingips.keys():
value = sona_topology.floatingips[key]
if trace_condition.cond_dict['nw_dst'] == value:
trace_condition.cond_dict['nw_dst'] = key
break
trace_condition.cur_target_ip = sona_topology.get_gateway_ip()
# find hostname
trace_condition.cur_target_hostname = sona_topology.get_openstack_info(
' ' + trace_condition.cur_target_ip + ' ', 'hostname')
return True
def delete_test_instance(server_vm, client_vm, client_floatingip):
try:
nova_credentials = client.Client(CONF.openstack()['version'],
CONF.openstack()['username'],
CONF.openstack()['api_key'],
CONF.openstack()['project_id'],
CONF.openstack()['auth_url'])
nova_credentials.floating_ips.delete(client_floatingip)
LOG.info('[Tperf Test] Client floatingip Deleted --- ')
for vm in [server_vm, client_vm]:
if vm:
nova_credentials.servers.delete(vm)
LOG.info('[Tperf Test] Server and Client instance Deleted] --- ')
except:
LOG.exception()
def traffic_test(condition_json):
seconds = 0
trace_result = []
sona_topology = Topology(True)
LOG.info('COND_JSON = ' + str(condition_json['traffic_test_list']))
timeout_arr = []
for test in condition_json['traffic_test_list']:
timeout_json = {
'command_result': 'timeout',
'node': test['node'],
'instance-id': test['instance_id']
}
timeout_arr.append(timeout_json)
timeout = 10
if dict(condition_json).has_key('timeout'):
timeout = condition_json['timeout']
i = 0
for test in condition_json['traffic_test_list']:
LOG.info('test = ' + str(test))
run_thread = threading.Thread(target=run_test,
args=(sona_topology, test, timeout_arr,
i, timeout))
run_thread.daemon = False
run_thread.start()
interval = 0
if dict(test).has_key('next_command_interval'):
interval = test['next_command_interval']
while interval > 0:
time.sleep(1)
seconds = seconds + 1
interval = interval - 1
i = i + 1
if timeout > seconds:
wait_time = timeout - seconds
while wait_time > 0:
find_timeout = False
i = 0
while i < len(timeout_arr):
if timeout_arr[i]['command_result'] == 'timeout':
find_timeout = True
i = i + 1
if find_timeout:
time.sleep(1)
else:
break
wait_time = wait_time - 1
i = 0
while i < len(timeout_arr):
trace_result.append(timeout_arr[i])
i = i + 1
return True, trace_result
def process_trace(output, sona_topology, trace_conditions):
try:
retry_flag = False
is_success = False
result_flow = []
lines = output.splitlines()
for line in lines:
line = line.strip()
if line.startswith('Rule:'):
rule_dict = dict()
tmp = line.split(' ')
rule_dict['table'] = int(tmp[1].split('=')[1])
rule_dict['cookie'] = tmp[2].split('=')[1]
selector_dict = {}
for col in tmp[3].split(','):
tmp = col.split('=')
if len(tmp) == 2:
if tmp[0] in ['priority']:
rule_dict[tmp[0]] = int(tmp[1])
elif tmp[0] in ['in_port']:
selector_dict[tmp[0]] = int(tmp[1])
else:
selector_dict[tmp[0]] = tmp[1]
if len(selector_dict.keys()) > 0:
rule_dict['selector'] = selector_dict
elif line.startswith('OpenFlow actions='):
action_dict = dict()
action_list = line.split('=')[1].split(',')
for action in action_list:
if action.startswith('set_field'):
type = action.split('->')[1]
value = action[action.find(':') + 1:action.find('-')]
action_dict[type] = value
if type == 'tun_dst':
# find next target
trace_conditions.cur_target_ip = sona_topology.get_openstack_info(
' ' + value + ' ', 'ip')
trace_conditions.cur_target_hostname = sona_topology.get_openstack_info(
' ' + value + ' ', 'hostname')
else:
trace_conditions.cond_dict[type] = value
if type == 'ip_dst':
trace_conditions.cond_dict['nw_dst'] = ''
else:
if action.startswith('group:'):
trace_conditions.cur_target_ip = sona_topology.get_gateway_ip(
)
trace_conditions.cur_target_hostname = sona_topology.get_openstack_info(
' ' + trace_conditions.cur_target_ip + ' ',
'hostname')
if len(line.split('=')) == 3:
action = action + '=' + line.split('=')[2]
LOG.info('action = ' + action)
tmp = action.split(':')
if action.startswith('group:') or action.startswith(
'goto_table:'):
action_dict[tmp[0]] = int(tmp[1])
elif len(tmp) < 2:
action_dict[tmp[0]] = tmp[0]
else:
action_dict[tmp[0]] = tmp[1]
rule_dict['actions'] = action_dict
result_flow.append(rule_dict)
if 'tun_dst' in line or 'group' in line:
retry_flag = True
if 'output' in line or 'CONTROLLER' in line:
is_success = True
break
return result_flow, retry_flag, is_success
except:
LOG.exception()
return 'parsing error\n' + output
if pid > 0:
# exit from second parent
sys.exit(0)
except OSError, e:
sys.stderr.write("fork #2 failed: %d (%s)\n" %
(e.errno, e.strerror))
sys.exit(1)
# redirect standard file descriptors
si = file(self.stdin, 'r')
so = file(self.stdout, 'a+')
se = file(self.stderr, 'a+', 0)
pid = str(os.getpid())
LOG.info("--- Daemon START ---")
sys.stderr.write("Started with pid %s\n" % pid)
sys.stderr.flush()
if self.pidfile:
file(self.pidfile, 'w+').write("%s\n" % pid)
atexit.register(self.delpid)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
# delete pid file when parent process kill
def delpid(self):
try:
os.remove(self.pidfile)
def sql_insert_nodes(cls, node_list, username):
for node in node_list:
name, ip = str(node).split(':')
LOG.info('Insert node [%s %s %s]', name, ip, username)
sql = 'INSERT INTO ' + cls.NODE_INFO_TBL + \
' VALUES (\'' + name + '\',\'' + ip + '\',\'' + username + '\')'
LOG.info('%s', sql)
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
LOG.info(" [NODE TABLE] Node data insert fail \n%s", sql_rt)
sys.exit(1)
# set status tbl
sql = 'INSERT INTO ' + cls.STATUS_TBL + \
' VALUES (\'' + name + '\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\')'
LOG.info('%s', sql)
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
LOG.info(" [STATUS TABLE] Node data insert fail \n%s", sql_rt)
sys.exit(1)
# set resource tbl
sql = 'INSERT INTO ' + cls.RESOURCE_TBL + ' VALUES (\'' + name + '\', -1, -1, -1)'
LOG.info('%s', sql)
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
LOG.info(" [RESOURCE TABLE] Node data insert fail \n%s",
sql_rt)
sys.exit(1)
# add Alarm Items
for item in CONF.alarm()['item_list']:
LOG.info('Insert item [%s %s]', name, item)
sql = 'INSERT INTO ' + cls.EVENT_TBL + \
' VALUES (\'' + name + '\',\'' + item + '\',\'none\', \'none\', \'none\')'
LOG.info('%s', sql)
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
LOG.info(" [ITEM TABLE] Item data insert fail \n%s",
sql_rt)
sys.exit(1)
def create_instance(server_options, client_options):
server_instance = client_instance = None
image_name = CONF.openstack()['image']
flavor_name = CONF.openstack()['flavor']
securitygroups = CONF.openstack()['security_groups']
keypair = CONF.openstack()['keypair_name']
# TODO add exception for connection
nova_credentials = client.Client(CONF.openstack()['version'],
CONF.openstack()['username'],
CONF.openstack()['api_key'],
CONF.openstack()['project_id'],
CONF.openstack()['auth_url'])
image = nova_credentials.images.find(name=image_name)
flavor = nova_credentials.flavors.find(name=flavor_name)
hypervisors = nova_credentials.hypervisors.list()
onos_ip = CONF.onos()['list'].pop().split(':')[-1]
dpid2ip = {c[2]: c[3] for c in [(" ".join(l.split()).split(" "))
for l in SshCommand.onos_ssh_exec(onos_ip, 'openstack-nodes | grep COMPUTE').splitlines()]}
def get_zone(dpid):
if dpid:
for h in hypervisors:
if h.host_ip == dpid2ip[dpid]:
return 'nova:' + h.service['host']
else:
return "nova"
# TODO: when no network_id, choice a network excepted external netwrok
# network_id = server_options['network_id'] if server_options['network_id'] \
# else random.choice(nova.networks.list()).id
# network_list = nova_credentials.networks.list()
# target_network.append(str(network_list[-1]).split(':')[1][:-1].strip())
# Create server VM info
vm_name = 'tperf_server_vm_' + str(random.randrange(10000, 20000))
LOG.info('[server] - vmname = %s', vm_name)
LOG.info(' | image = %s', image)
LOG.info(' | flavor = %s', flavor)
LOG.info(' | availability_zone = %s', get_zone(server_options['vm_location']))
LOG.info(' | nics = %s', server_options['network_id'])
LOG.info(' | security_groups = %s', securitygroups)
LOG.info(' | key_pair = %s', keypair)
nova_credentials.servers.create(name=vm_name,
image=image,
flavor=flavor,
availability_zone=get_zone(server_options['vm_location']),
nics=[{'net-id': server_options['network_id']}],
security_groups=securitygroups,
key_name=keypair)
for i in range(20):
time.sleep(1)
server_instance = nova_credentials.servers.list(search_opts={'name': vm_name})[0]
# server_instance = nova_credentials.servers.list(search_opts={'name': 'tperf_server_vm_17693'})[0]
if server_instance.__dict__['addresses']:
LOG.info("[Server VM created and ACTIVE] - %s", server_instance)
break
# Create client VM info
vm_name = 'tperf_client_vm_' + str(random.randrange(10000, 20000))
LOG.info('[client] - vmname = %s', vm_name)
LOG.info(' | image = %s', image)
LOG.info(' | flavor = %s', flavor)
LOG.info(' | availability_zone = %s', get_zone(client_options['vm_location']))
LOG.info(' | nics = %s', client_options['network_id'])
LOG.info(' | security_groups = %s', securitygroups)
LOG.info(' | key_pair = %s', keypair)
nova_credentials.servers.create(name=vm_name,
image=image,
flavor=flavor,
availability_zone=get_zone(client_options['vm_location']),
nics=[{'net-id': client_options['network_id']}],
security_groups=securitygroups,
key_name=keypair)
client_floatingip = get_floatingip(nova_credentials)
# client_floatingip = '172.27.0.179'
for i in range(20):
time.sleep(1)
client_instance = nova_credentials.servers.list(search_opts={'name': vm_name})[0]
# client_instance = nova_credentials.servers.list(search_opts={'name': 'tperf_client_vm_15442'})[0]
if client_instance.__dict__['addresses']:
LOG.info("[Client VM created and ACTIVE] - %s", client_instance)
nova_credentials.servers.add_floating_ip(client_instance, client_floatingip.ip)
LOG.info("[Floating_IP Assignment] to Client ---")
break
return server_instance, client_instance, client_floatingip
def db_initiation(cls, db_log):
try:
db_path = CONF.base()['db_file']
if os.path.isfile(db_path):
os.remove(db_path)
db_log.write_log("--- Initiating SONA DB ---")
init_sql = [
'CREATE TABLE ' + cls.NODE_INFO_TBL +
'(nodename text primary key, ip_addr, username, type, sub_type)',
'CREATE TABLE ' + cls.STATUS_TBL +
'(nodename text primary key, ' + cls.item_list + ', time)',
'CREATE TABLE ' + cls.RESOURCE_TBL +
'(nodename text primary key, cpu real, memory real, disk real)',
'CREATE TABLE ' + cls.REGI_SYS_TBL +
'(url text primary key, auth)',
'CREATE TABLE ' + cls.ONOS_TBL +
'(nodename text primary key, applist, weblist, nodelist, port, openflow, cluster, traffic_stat)',
'CREATE TABLE ' + cls.SWARM_TBL +
'(nodename text primary key, node, service, ps)',
'CREATE TABLE ' + cls.XOS_TBL +
'(nodename text primary key, xos_status, synchronizer)',
'CREATE TABLE ' + cls.OPENSTACK_TBL +
'(nodename text primary key, sub_type, data_ip, of_id, hostname, docker, onosApp, routingTable, gw_ratio, vxlan_traffic, internal_traffic)',
'CREATE TABLE ' + cls.HA_TBL +
'(ha_key text primary key, stats)',
'CREATE TABLE ' + cls.EVENT_TBL +
'(nodename, item, grade, pre_grade, reason, time, PRIMARY KEY (nodename, item))'
]
for sql in init_sql:
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log("DB initiation fail\n%s", sql_rt)
sys.exit(1)
db_log.write_log('Insert nodes information ...')
for node_type in CONF.watchdog()['check_system']:
if node_type == 'OPENSTACK':
cls.sql_insert_nodes(
db_log, CONF_MAP[node_type.upper()]()['compute_list'],
str((CONF_MAP[node_type.upper()]()
)['account']).split(':')[0], node_type, 'COMPUTE')
cls.sql_insert_nodes(
db_log, CONF_MAP[node_type.upper()]()['gateway_list'],
str((CONF_MAP[node_type.upper()]()
)['account']).split(':')[0], node_type, 'GATEWAY')
else:
cls.sql_insert_nodes(
db_log, CONF_MAP[node_type.upper()]()['list'],
str((CONF_MAP[node_type.upper()]()
)['account']).split(':')[0], node_type)
# set ha proxy tbl
sql = 'INSERT INTO ' + cls.HA_TBL + ' VALUES (\'' + 'HA' + '\', \'none\')'
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log(
" [HA PROXY TABLE] Node data insert fail \n%s", sql_rt)
sys.exit(1)
except:
LOG.exception()
def run_test(sona_topology, test_json, timeout_arr, index, total_timeout):
try:
node = test_json['node']
ins_id = test_json['instance_id']
user = test_json['vm_user_id']
pw = test_json['vm_user_password']
command = test_json['traffic_test_command']
ip = sona_topology.get_openstack_info(node, 'ip')
if ip == '':
str_output = node + ' node does not exist'
else:
node_id = CONF.openstack()['account'].split(':')[0]
ssh_options = '-o StrictHostKeyChecking=no ' \
'-o ConnectTimeout=' + str(CONF.ssh_conn()['ssh_req_timeout'])
cmd = 'ssh %s %s@%s' % (ssh_options, node_id, ip)
try:
LOG.info('ssh_pexpect cmd = ' + cmd)
ssh_conn = pexpect.spawn(cmd)
rt1 = ssh_conn.expect(
PROMPT, timeout=CONF.ssh_conn()['ssh_req_timeout'])
if rt1 == 0:
cmd = 'virsh console ' + ins_id
LOG.info('ssh_pexpect cmd = ' + cmd)
ssh_conn.sendline(cmd)
rt2 = ssh_conn.expect(
[
pexpect.TIMEOUT, 'Escape character is', 'error:',
pexpect.EOF
],
timeout=CONF.ssh_conn()['ssh_req_timeout'])
if rt2 == 0:
str_output = cmd + ' timeout'
elif rt2 == 1:
ssh_conn.sendline('\n')
try:
rt3 = ssh_conn.expect(
['login: '******'ssh_req_timeout'])
LOG.info('rt3 = ' + str(rt3))
if rt3 == 2:
str_output = 'Permission denied'
else:
ssh_conn.sendline(user)
rt_pw = ssh_conn.expect(
[
pexpect.TIMEOUT, '[P|p]assword:',
pexpect.EOF
],
timeout=CONF.ssh_conn()['ssh_req_timeout'])
if rt_pw == 1:
ssh_conn.sendline(pw)
rt4 = ssh_conn.expect(
[
pexpect.TIMEOUT, 'Login incorrect',
'~# ', 'onos> ', '\$ ', '\# ',
':~$ '
],
timeout=CONF.ssh_conn()
['ssh_req_timeout'])
LOG.info('rt4 = ' + str(rt4))
if rt4 == 0 or rt4 == 1:
str_output = 'auth fail'
else:
ssh_conn.sendline(command)
rt5 = ssh_conn.expect(
[
pexpect.TIMEOUT, '~# ',
'onos> ', '\$ ', '\# ', ':~$ '
],
timeout=total_timeout)
if rt5 == 0:
str_output = 'timeout'
ssh_conn.sendline('exit')
ssh_conn.close()
else:
str_output = ssh_conn.before
ssh_conn.sendline('exit')
ssh_conn.close()
else:
str_output = 'auth fail'
except:
str_output = 'exception'
ssh_conn.sendline('exit')
ssh_conn.close()
elif rt2 == 2:
result = {'command_result': 'virsh console error'}
timeout_arr[index] = result
return
else:
str_output = 'connection fail'
except:
LOG.exception()
str_output = 'exception 1'
except:
LOG.exception()
str_output = 'exception 2'
result = {
'command_result': str_output.replace('\r\n', '\n'),
'node': node,
'instance_id': ins_id
}
timeout_arr[index] = result
def sql_insert_nodes(cls,
db_log,
node_list,
username,
type,
sub_type='none'):
try:
for node in node_list:
name, ip = str(node).split(':')
db_log.write_log('Insert node [%s %s %s %s]', name, ip,
username, type)
sql = 'INSERT INTO ' + cls.NODE_INFO_TBL + \
' VALUES (\'' + name + '\', \'' + ip + '\', \'' + username + '\', \'' + type.upper() + '\', \'' + sub_type.upper() + '\')'
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log(
" [NODE TABLE] Node data insert fail \n%s", sql_rt)
sys.exit(1)
# set status tbl
sql = 'INSERT INTO ' + cls.STATUS_TBL + \
' VALUES (\'' + name + '\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\', ' \
'\'none\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\')'
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log(
" [STATUS TABLE] Node data insert fail \n%s", sql_rt)
sys.exit(1)
# add Alarm Items
evt_list = DB.get_event_list(type)
for item in evt_list:
db_log.write_log('Insert item [%s %s]', name, item)
sql = 'INSERT INTO ' + cls.EVENT_TBL + \
' VALUES (\'' + name + '\',\'' + item + '\', \'none\', \'none\', \'none\', \'none\')'
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log(
" [ITEM TABLE] Item data insert fail \n%s", sql_rt)
sys.exit(1)
# set resource tbl
sql = 'INSERT INTO ' + cls.RESOURCE_TBL + ' VALUES (\'' + name + '\', -1, -1, -1)'
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log(
" [RESOURCE TABLE] Node data insert fail \n%s", sql_rt)
sys.exit(1)
if type.upper() == 'ONOS':
# set app tbl
sql = 'INSERT INTO ' + cls.ONOS_TBL + ' VALUES (\'' + name + '\', \'none\', \'none\', \'none\', ' \
'\'none\', \'none\', \'none\', \'none\')'
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log(
" [APP TABLE] Node data insert fail \n%s", sql_rt)
sys.exit(1)
elif type.upper() == 'XOS':
# set xos tbl
sql = 'INSERT INTO ' + cls.XOS_TBL + ' VALUES (\'' + name + '\', \'none\', \'none\')'
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log(
" [XOS TABLE] Node data insert fail \n%s", sql_rt)
sys.exit(1)
# set swarm tbl
sql = 'INSERT INTO ' + cls.SWARM_TBL + ' VALUES (\'' + name + '\', \'none\', \'none\', \'none\')'
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log(
" [SWARM TABLE] Node data insert fail \n%s",
sql_rt)
sys.exit(1)
elif type.upper() == 'OPENSTACK':
# set vrouter tbl
sql = 'INSERT INTO ' + cls.OPENSTACK_TBL + \
' VALUES (\'' + name + '\', \'' + sub_type + '\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\', \'none\')'
sql_rt = cls.sql_execute(sql)
if sql_rt != 'SUCCESS':
db_log.write_log(
" [VROUTER TABLE] Node data insert fail \n%s",
sql_rt)
sys.exit(1)
except:
LOG.exception()
i = i + 1
try:
if conn == None:
with cls.connection() as conn:
conn.cursor().execute(sql)
conn.commit()
conn.close()
else:
conn.cursor().execute(sql)
conn.commit()
return 'SUCCESS'
except sqlite3.OperationalError, err:
LOG.error(err.message)
except:
LOG.exception()
return 'FAIL'
DB_CONN = DB().connection()
CONF_MAP = {
'ONOS': CONF.onos,
'XOS': CONF.xos,
'SWARM': CONF.swarm,
'OPENSTACK': CONF.openstack,
'HA': CONF.ha
}
def get_hosts(self, onos_ip, find_cond):
LOG.info('onos command = ' + self.ONOS_HOST + '| grep ' + find_cond)
onos_ssh_result = SshCommand.onos_ssh_exec(
onos_ip, self.ONOS_HOST + '| grep ' + find_cond)
return onos_ssh_result
def tperf3_output_parse(tperf_out_list):
parse_result = {'test_options': {}, 'test_result': {}, 'cpu_utilization_percent': {}}
tout_protocol_l = list()
tout_num_streams_l = list()
tout_blksize_l = list()
tout_duration_l = list()
tout_start_l = list()
tout_end_l = list()
tout_seconds_l = list()
tout_bytes_l = list()
tout_bps_l = list()
tout_jitter_l = list()
tout_lost_packet_l = list()
tout_packet_l = list()
tout_lost_percent_l = list()
tout_client_cpu_l = list()
tout_server_cpu_l = list()
for i in range(len(tperf_out_list)):
if tperf_out_list[i] == 'fail':
return {'result': 'FAIL', 'fail_reason': 'Tperf command run fail on Client VM'}
try:
tperf_out_list[i] = json.loads(' '.join(str(tperf_out_list[i]).split()))
tout_protocol_l.append(tperf_out_list[i]['start']['test_start']['protocol'])
tout_num_streams_l.append(tperf_out_list[i]['start']['test_start']['num_streams'])
tout_blksize_l.append(tperf_out_list[i]['start']['test_start']['blksize'])
tout_duration_l.append(tperf_out_list[i]['start']['test_start']['duration'])
tout_start_l.append(float(tperf_out_list[i]['end']['sum']['start']))
tout_end_l.append(float(tperf_out_list[i]['end']['sum']['end']))
tout_seconds_l.append(float(tperf_out_list[i]['end']['sum']['seconds']))
tout_bytes_l.append(float(tperf_out_list[i]['end']['sum']['bytes']))
tout_bps_l.append(float(tperf_out_list[i]['end']['sum']['bits_per_second']))
tout_jitter_l.append(float(tperf_out_list[i]['end']['sum']['jitter_ms']))
tout_lost_packet_l.append(float(tperf_out_list[i]['end']['sum']['lost_packets']))
tout_packet_l.append(float(tperf_out_list[i]['end']['sum']['packets']))
tout_lost_percent_l.append(float(tperf_out_list[i]['end']['sum']['lost_percent']))
tout_client_cpu_l.append(float(tperf_out_list[i]['end']['cpu_utilization_percent']['host_total']))
tout_server_cpu_l.append(float(tperf_out_list[i]['end']['cpu_utilization_percent']['remote_total']))
except:
LOG.exception()
return {'result': 'FAIL', 'fail_reason': ''}
parse_result['test_options'].update({'protocol': tout_protocol_l[0]})
parse_result['test_options'].update({'parallel': sum(tout_num_streams_l)})
parse_result['test_options'].update({'blksize': tout_blksize_l[0]})
parse_result['test_options'].update({'duration': tout_duration_l[0]})
parse_result['test_options'].update({'reverse': tperf_out_list[0]['start']['test_start']['reverse']})
parse_result['test_result'].update({'start': sum(tout_start_l)/len(tperf_out_list)})
parse_result['test_result'].update({'end': sum(tout_end_l)/len(tperf_out_list)})
parse_result['test_result'].update({'seconds': sum(tout_seconds_l)/len(tperf_out_list)})
parse_result['test_result'].update({'bytes': sum(tout_bytes_l)})
parse_result['test_result'].update({'bits_per_second': sum(tout_bps_l)})
parse_result['test_result'].update({'jitter_ms': sum(tout_jitter_l)/len(tperf_out_list)})
parse_result['test_result'].update({'lost_packets': sum(tout_lost_packet_l)})
parse_result['test_result'].update({'packets': sum(tout_packet_l)})
parse_result['test_result'].update({'lost_percent': sum(tout_lost_percent_l)/len(tperf_out_list)})
parse_result['cpu_utilization_percent'].update({'client_total': sum(tout_client_cpu_l)/len(tperf_out_list)})
parse_result['cpu_utilization_percent'].update({'server_total': sum(tout_server_cpu_l)/len(tperf_out_list)})
parse_result['result'] = 'SUCCESS'
return parse_result