def run_remote_cmd(cmd):
debug('Running: ' + cmd)
return subprocess.check_output(cmd,
shell=True,
stderr=subprocess.STDOUT,
universal_newlines=True).replace(
'\t', ' ')
def run_ping_command(cmd, comment=''):
debug('Running ' + comment + ': ' + cmd)
return subprocess.check_output(cmd,
shell=True,
stderr=subprocess.STDOUT,
universal_newlines=True).replace(
'\t', ' ')
def analyze(json_filename, params):
settings['debug'] = True
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
CUR_DIR = os.getcwd()
os.chdir(BASE_DIR + '/ovs')
# NEW_DIR = os.getcwd()
# return BASE_DIR + ':' + CUR_DIR + ':' + NEW_DIR
debug('This is what I am going to analyze')
my_info = load_json(json_filename)
for test in test_suite.keys():
flag = 'test:' + test
if params[flag] or params['test:all']:
(result, cmds) = test_suite[test]['func'](my_info)
if result:
test_suite[test]['result'] = 'PASS'
else:
test_suite[test]['result'] = 'FAIL'
lines = test_suite[test]['formatter'](cmds, result, my_info,
test_suite[test]['help'])
test_suite[test]['html'] = lines
debug(params['test:report_file'])
report_file_write(params['test:report_file'])
for test in test_suite.keys():
if test_suite[test]['html']:
for line in test_suite[test]['html']:
with open(params['test:report_file'], 'a') as f:
f.write(line)
report_file_close(params['test:report_file'])
os.chdir(CUR_DIR)
def main():
deployment_type = load_config()
my_env = os.environ.copy()
my_env.update(get_env('admin-openrc.sh'))
check_args()
iteration = 0
# Parser of any specific command might add more commands to be executed.
# Hence continue in a loop.
while True:
if (all_commands_executed(commands) or iteration >= 10):
break
iteration += 1
common.status_update('Iteration: ' + str(iteration))
sorted_keys = sorted(commands.items(), key=lambda (k, v): v['order'])
for (cmd, dontcare) in sorted_keys:
# Only collect stuff for which we have written a parser
if commands[cmd]['parser']:
if commands[cmd].get('done', False):
continue
if commands[cmd].has_key('help'):
common.status_update(commands[cmd]['help'])
shell = commands[cmd].get('shell', False)
env = None
if commands[cmd].get('env', False):
env = my_env
sudo = commands[cmd].get('sudo', False)
if deployment_type == 'multinode':
# handling for network node
if cmd.startswith('netns_'):
commands[cmd]['output'] = exec_on_remote(
commands[cmd]['cmd'])
if cmd == 'libvirt_instance':
commands[cmd]['output'] = get_vm_info_from_compute(
commands[cmd]['cmd'], my_env)
print(commands[cmd]['output'])
else:
commands[cmd]['output'] = common.execute_cmd(
commands[cmd]['cmd'],
sudo=sudo,
shell=shell,
env=env).split('\n')
else:
commands[cmd]['output'] = common.execute_cmd(
commands[cmd]['cmd'], sudo=sudo, shell=shell,
env=env).split('\n')
commands[cmd]['parser'](commands[cmd]['output'])
commands[cmd]['done'] = True
common.debug('============= COMMANDS =============')
common.status_update('Writing collected info into ' +
common.settings['info_file'])
common.dump_json(info, common.settings['info_file'])
def path(params):
global src_info
global dst_info
global net_info
src_info = None
dst_info = None
net_info = None
settings['debug'] = True
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
CUR_DIR = os.getcwd()
if not re.search('/openstack_dashboard/don/', CUR_DIR):
os.chdir(BASE_DIR + '/ovs')
NEW_DIR = os.getcwd()
debug(BASE_DIR + ':' + CUR_DIR + ':' + NEW_DIR)
src_ip = params['src_ip']
dst_ip = params['dst_ip']
json_file = params['json_file']
router = params['router']
debug('Json_file: ' + json_file)
info = load_json(json_file)
qrouter = router_to_namespace(info, router)
params['qrouter'] = qrouter
src_info = get_port_info(info, src_ip)
dst_info = get_port_info(info, dst_ip)
if src_info is None:
return "Source ip not found on the network"
if dst_info is None:
return "Destination ip not found on the network"
if qrouter is None:
return "No such router information found on the network"
# src and dst are in the same network
if src_info['tag'] == dst_info['tag']:
path_same_network(params)
else:
status_update('The source and destination are in different networks')
next_hop_list = get_next_hop(src_info, dst_info, qrouter, params)
if len(next_hop_list) == 0:
error('Could not find next hop list from %s to %s' %
(src_ip, dst_ip))
path_same_network(params, next_hop_list)
def did_ping_test_pass(cmds):
qrouter_result = True
for qrouter in sorted(cmds.keys()):
debug('Checking ping status in qrouter %s' % qrouter)
qrouter_result = True
for key in cmds[qrouter].keys():
(src_vm, dst_vm) = key
for key2 in sorted(cmds[qrouter][key].keys()):
(src_ip, dst_ip) = key2
result = cmds[qrouter][key][key2]['pass']
if not result:
qrouter_result = False
break # check the next namsepace, this one failed
# if all ping passed via this qrouter, return true
if qrouter_result:
return qrouter_result
# There was no qrouter via which all pings passed!
return qrouter_result
def test_ovs(info):
debug('Running OVS test')
ovs_test_passed = True
cmds = {}
# first construct a dictionary of tag to br-int ports
br_int_ports = info['bridges']['br-int']['ports']
tag_to_port = {}
for port in br_int_ports.keys():
if not re.search('^qvo', port):
continue
tag = br_int_ports[port]['tag']
port_id = br_int_ports[port]['id']
if not tag_to_port.has_key(tag):
tag_to_port[tag] = []
tag_to_port[tag].append((port_id, port))
debug(pprint.pformat(tag_to_port))
for tag in sorted(tag_to_port.keys(), key=lambda x: int(x)):
cmds[tag] = {}
port_count = len(tag_to_port[tag])
if port_count < 2:
debug('tag %s is used by single port %s. Skipping test!' %
(tag, tag_to_port[tag][0]))
continue
port_list = list(map(lambda (x, y): x, tag_to_port[tag]))
sorted_port_list = sorted(port_list, key=lambda x: int(x))
port_pairs = list(combinations(sorted_port_list, 2))
for (src_port, dst_port) in port_pairs:
cmds[tag][(src_port, dst_port)] = {}
cmd = ''
cmd += 'python ovs.py --src_port_id %s --dst_port_id %s --tag %s --ovs_bridge br-int' % \
(src_port, dst_port, tag)
comment = 'ovs [tag: %s port %s => %s' % (tag, src_port, dst_port)
output = run_ovs_command(cmd, comment=comment)
success = process_ovs_output(output)
if not success:
ovs_test_passed = False
cmds[tag][(src_port, dst_port)] = {
'cmd': cmd,
'output': output,
'pass': success
}
return (ovs_test_passed, cmds)
def test_ping(info):
debug('Running ping test')
vms = info['vms']
vm_credentials = get_vm_credentials()
for vm in sorted(vms.keys()):
vms[vm]['qrouter'] = get_vm_qrouters(info, vm)
vm_pairs = list(combinations(sorted(vms.keys()), 2))
pprint.pprint(vm_pairs)
cmds = {}
for (src_vm, dst_vm) in vm_pairs:
for qrouter in vms[src_vm]['qrouter']:
debug(qrouter)
if not cmds.has_key(qrouter):
cmds[qrouter] = {}
cmds[qrouter][(src_vm, dst_vm)] = {}
for src_ip in vms[src_vm]['interfaces'].keys():
if is_network_public(src_ip, src_vm, info):
continue
for dst_ip in vms[dst_vm]['interfaces'].keys():
if is_network_public(dst_ip, dst_vm, info):
continue
username = vm_credentials.get(
src_vm, vm_credentials['default'])['username']
passwd = vm_credentials.get(
src_vm, vm_credentials['default'])['password']
cmd = 'sudo ip netns exec ' + qrouter
cmd += ' python ping.py --src_ip %s --dst_ip %s --username "%s" --passwd "%s" --count %d --timeout %d' % \
(src_ip, dst_ip, username, passwd, 1, 2)
comment = 'Ping [%s (%s) => %s (%s)]' % (src_vm, src_ip,
dst_vm, dst_ip)
output = run_ping_command(cmd, comment=comment)
a = json.loads(output)
success = a['pass']
cmds[qrouter][(src_vm, dst_vm)][(src_ip, dst_ip)] = {
'cmd': cmd,
'output': output,
'pass': success
}
debug(pprint.pformat(cmds))
ping_test_passed = did_ping_test_pass(cmds)
return (ping_test_passed, cmds)
def path_same_network(params, nms_hops=None):
src_ip = params['src_ip']
dst_ip = params['dst_ip']
json_file = params['json_file']
username = params['username']
passwd = params['passwd']
count = params['count']
timeout = params['timeout']
qrouter = params['qrouter']
router = params['router']
if qrouter_usable(qrouter, src_ip, dst_ip, username, passwd):
outfile = 'path.ping.txt'
ping_process = launch_ping(src_ip, dst_ip, username, passwd, count,
timeout, qrouter, outfile)
debug('Ping started with pid: %d' % ping_process.pid)
capture_packets(params, 'src')
capture_packets(params, 'dst', src_tag=src_info['tag'])
if src_info['tag'] != dst_info['tag']:
capture_network_packets(params, nms_hops)
status_update(
'Waiting %s sec for tcpdump and ping processes to complete' %
(params['count'] + 2))
time.sleep(params['count'] + 4)
status_update('if processes have not stopped, lets kill them')
cleanup_processes([ping_process.pid] + src_info['pids'] +
dst_info['pids'])
if net_info:
cleanup_processes(net_info['pids'])
process_captures('src')
process_captures('dst')
if src_info['tag'] != dst_info['tag']:
process_network_captures()
ping_pass = process_ping(outfile)
debug(pprint.pformat(src_info))
debug(pprint.pformat(dst_info))
debug(pprint.pformat(net_info))
info = {
'src': src_ip,
'dst': dst_ip,
'src_info': src_info,
'dst_info': dst_info,
'net_info': net_info,
'ping_pass': ping_pass,
'error': '',
}
status_update('Dumping results into %s in JSON format' %
params['path_file'])
dump_json(info, params['path_file'])
if params['plot']:
cmd = 'python plot.py --info_file %s --highlight_file %s --combined_file static/ping' % (
json_file, params['path_file'])
status_update('Running ' + cmd)
output = execute_cmd(cmd, shell=True).split('\n')
debug(pprint.pformat(output))
status_update('Done')
else:
err_msg = 'Cannot reach %s via router %s' % (src_ip, router)
info = {
'src': src_ip,
'dst': dst_ip,
'src_info': src_info,
'dst_info': dst_info,
'ping_pass': False,
'error': err_msg
}
error(err_msg)
status_update('Dumping results into %s in JSON format' %
params['path_file'])
dump_json(info, params['path_file'])
status_update('Done')
def ovs_test(src_port_id, dst_port_id, tag, ovs_bridge):
smac = 'AA:BB:CC:DD:EE:11'
dmac = 'AA:BB:CC:DD:EE:22'
cmd_dict = {}
# Step 0. Flush the fdb
cmd = ''
cmd += 'sudo ovs-appctl fdb/flush br-int'
output = execute_cmd(cmd, shell=True).split('\n')
cmd_dict['cmd'] = cmd
cmd_dict['output'] = output
output_dict['command_list'].append(cmd_dict)
cmd_dict = {}
# Step 1. run command that will learn smac
cmd = ''
cmd += 'sudo ovs-appctl ofproto/trace %s in_port=%s' % (ovs_bridge,
src_port_id)
cmd += ',dl_src=' + smac + ',dl_dst=' + dmac + ' -generate'
output = execute_cmd(cmd, shell=True).split('\n')
cmd_dict['cmd'] = cmd
cmd_dict['output'] = output
output_dict['command_list'].append(cmd_dict)
cmd_dict = {}
# Step 2. verify that the mac has been learnt
cmd = ''
cmd += 'sudo ovs-appctl fdb/show br-int'
output = execute_cmd(cmd, shell=True).split('\n')
cmd_dict['cmd'] = cmd
cmd_dict['output'] = output
output_dict['command_list'].append(cmd_dict)
cmd_dict = {}
port = None
for line in output:
m = re.search('(\d)\s+(\d+)\s+(\S+)\s+\d+', line)
if m:
mac = m.group(3)
if mac.lower() == smac.lower():
port = m.group(1)
vlan = m.group(2)
debug(line)
break
if not port:
output_dict['errors'].append('%s not learnt on port %s' %
(smac, src_port_id))
output_dict['pass'] = False
return False
if vlan != tag:
output_dict['errors'].append(
'%s learnt on vlan %s but should have been learnt on vlan %s on port %s'
% (smac, vlan, tag, port))
output_dict['pass'] = False
return False
output_dict['debugs'].append('%s learnt on expected vlan %s on port %s' %
(smac, vlan, port))
# Step 3. now do a lookup using the dst port id and dmac as the smac of
# step 1.
cmd = ''
cmd += 'sudo ovs-appctl ofproto/trace %s in_port=%s' % (ovs_bridge,
dst_port_id)
cmd += ',dl_src=' + dmac + ',dl_dst=' + smac + ' -generate'
output = execute_cmd(cmd, shell=True).split('\n')
cmd_dict['cmd'] = cmd
cmd_dict['output'] = output
output_dict['command_list'].append(cmd_dict)
cmd_dict = {}
forwarded = False
egress_port = None
for line in output:
if re.search('forwarding to learned port', line):
forwarded = True
continue
m = re.search('Datapath actions: (.*)', line)
if m:
egress_port = m.group(1)
continue
result = True
if not forwarded:
output_dict['errors'].append('Packet for learnt mac not forwarded!')
result = False
else:
output_dict['debugs'].append(
'Packet for learnt mac forwarded properly')
if egress_port:
if egress_port == src_port_id:
output_dict['debugs'].append(
'Packet forwarded to correct port %s' % egress_port)
else:
output_dict['errors'].append(
'Packet forwarded to incorrect port %s, expected %s' %
(egress_port, src_port_id))
result = False
else:
output_dict['errors'].append(
'No egress port assigned to packet! Expected %s' % src_port_id)
result = False
output_dict['pass'] = result
return result
def dummy_parser(parse_this):
common.debug('Dummy Parser :-)')
pass