def test_timeout_blocking(self):
tcpd = TCPDump()
tcps = TCPSender()
out = LinuxCLI().cmd('ip l | grep "LOOPBACK" | cut -f 2 -d " "| cut -f 1 -d ":"')
lo_iface = out.split()[0].rstrip()
try:
tcpd.start_capture(
interface=lo_iface,
count=1,
pcap_filter=PCAP_And(
[
PCAP_Host("localhost", proto="ip", source=True, dest=True),
PCAP_Port(6015, proto="tcp", source=True),
PCAP_Port(6055, proto="tcp", dest=True),
]
),
blocking=True,
timeout=3,
)
except SubprocessTimeoutException:
pass
else:
self.assertTrue(False, "Blocking tcpdump call should have timed out since no packets were sent")
finally:
tcpd.stop_capture()
def rollover_logs_by_date(self, backup_dir=None,
date_pattern='%Y%m%d%H%M%S', zip_file=True):
"""
If the filename exists, roll it over to a new file based on the parameters. Return
the name of the new file.
:type backup_dir: str
:type date_pattern: str
:type zip_file: bool
:return: str
"""
for file_loc, logger_list in self.open_log_files.iteritems():
cli = LinuxCLI(priv=False)
if cli.exists(file_loc.full_path()) and os.path.getsize(file_loc.full_path()) > 0:
# Close previous, now-stale file handlers
for l, h, df, dp in logger_list:
h.close()
l.removeHandler(h)
self._rollover_file(file_loc.full_path(), backup_dir, date_pattern, zip_file)
# Pop off old logger/handler pairs and re-populate with new handler objects
# which point to the original file location
for i in range(0, len(logger_list)):
l, h, df, dp = logger_list.pop()
new_handler = logging.FileHandler(filename=file_loc.full_path(), mode='w')
l.addHandler(new_handler)
logger_list.append((l, new_handler, df, dp))
def test_read_packet_buffered(self):
tcpd = TCPDump()
out = LinuxCLI().cmd('ip l | grep "LOOPBACK" | cut -f 2 -d " "| cut -f 1 -d ":"')
lo_iface = out.split()[0].rstrip()
p = multiprocessing.Process(target=send_packet)
p.start()
ret = tcpd._read_packet(
interface=lo_iface,
count=3,
pcap_filter=PCAP_And(
[
PCAP_Host("localhost", proto="ip", source=True, dest=True),
PCAP_Port(6015, proto="tcp", source=True),
PCAP_Port(6055, proto="tcp", dest=True),
]
),
save_dump_file=True,
save_dump_filename="tcp.out",
)
p.join()
p1 = ret.get(block=False)
p2 = ret.get(block=False)
p3 = ret.get(block=False)
""" :type: PCAPPacket"""
self.assertTrue(p3 is not None)
self.assertTrue("ethernet" in p3)
def test_sniff_host_packet(self):
tcpd = TCPDump()
tcps = TCPSender()
try:
out = LinuxCLI().cmd('ip l | grep "LOOPBACK" | cut -f 2 -d " "| cut -f 1 -d ":"')
lo_iface = out.split()[0].rstrip()
tcpd.start_capture(interface=lo_iface, count=1)
tcps.start_send(
interface=lo_iface,
packet_type="tcp",
count=1,
source_ip="127.0.0.1",
dest_ip="127.0.0.1",
dest_port=6055,
source_port=6015,
)
ret = tcpd.wait_for_packets(count=1, timeout=3)
self.assertEquals(1, len(ret))
self.assertTrue("ethernet" in ret[0])
finally:
tcpd.stop_capture()
def install_packages(self, package, exact_version=None):
"""
:type packages: list[str]
:type exact_version: str
:return:
"""
cli = LinuxCLI()
cli.cmd('yum install -y ' + package + ('-' + exact_version if exact_version is not None else ''))
def install_packages(self, packages, exact_version=None):
"""
:type packages: list[str]
:type exact_version: str
:return:
"""
cli = LinuxCLI(log_cmd=True)
pkg_list = ' '.join(map(lambda v: v + (('=' + exact_version) if exact_version is not None else ''),
packages))
print 'Installing Debian packages: ' + pkg_list
print cli.cmd('apt-get install --allow-unauthenticated -y ' + pkg_list)
def uninstall_packages(self, packages, exact_version=None):
"""
:type packages: list[str]
:type exact_version: str
:return:
"""
cli = LinuxCLI(log_cmd=True)
pkg_list = ' '.join(map(lambda v: v + (('=' + exact_version) if exact_version is not None else ''),
packages))
print 'Uninstalling Debian packages: ' + pkg_list
ret = cli.cmd('apt-get remove -y ' + pkg_list)
def send_packet():
time.sleep(5)
tcps = TCPSender()
out = LinuxCLI().cmd('ip l | grep "LOOPBACK" | cut -f 2 -d " "| cut -f 1 -d ":"')
lo_iface = out.split()[0].rstrip()
tcps.start_send(
interface=lo_iface,
packet_type="tcp",
count=3,
source_ip="127.0.0.1",
dest_ip="127.0.0.1",
dest_port=6055,
source_port=6015,
)
def __init__(self):
super(RootServer, self).__init__('root', LinuxCLI(), lambda name: None,
lambda name: None, self)
self.zookeeper_hosts = []
""" :type: list[ZookeeperHost]"""
self.cassandra_hosts = []
""" :type: list[CassandraHost]"""
self.compute_hosts = []
""" :type: list[ComputeHost]"""
self.network_hosts = []
""" :type: list[NetworkHost]"""
self.hosted_vms = []
""" :type: list[VM]"""
self.vlans = []
""" :type: list[VLAN]"""
self.routers = []
""" :type: list[Router]"""
self.hosts = {}
""" :type: dict[str, Host]"""
self.zookeeper_ips = []
""" :type: list[IPDef]"""
self.cassandra_ips = []
""" :type: list[IPDef]"""
self.generic_hosts = []
""" :type: list[IPDef]"""
def test_callback(self):
tcpd = TCPDump()
tcps = TCPSender()
out = LinuxCLI().cmd('ip l | grep "LOOPBACK" | cut -f 2 -d " "| cut -f 1 -d ":"')
lo_iface = out.split()[0].rstrip()
LinuxCLI().rm("tmp.file")
tcpd.start_capture(
interface=lo_iface,
count=3,
pcap_filter=PCAP_And(
[
PCAP_Host("localhost", proto="ip", source=True, dest=True),
PCAP_Port(6015, proto="tcp", source=True),
PCAP_Port(6055, proto="tcp", dest=True),
]
),
callback=packet_callback,
callback_args=("tmp.file",),
save_dump_file=True,
save_dump_filename="tcp.callback.out",
)
tcps.start_send(
interface=lo_iface,
packet_type="tcp",
count=3,
source_ip="127.0.0.1",
dest_ip="127.0.0.1",
dest_port=6055,
source_port=6015,
)
ret = tcpd.wait_for_packets(count=3)
tcpd.stop_capture()
time.sleep(2)
self.assertEquals(3, len(ret))
self.assertTrue(LinuxCLI().exists("tmp.file"))
file_str = LinuxCLI().read_from_file("tmp.file")
LinuxCLI().rm("tmp.file")
self.assertEquals(3, file_str.count("END PACKET time["))
def test_sniff_specific_host_packet(self):
tcpd = TCPDump()
tcps = TCPSender()
out = LinuxCLI().cmd('ip l | grep "LOOPBACK" | cut -f 2 -d " "| cut -f 1 -d ":"')
iface = out.split()[0].rstrip()
tcpd.start_capture(
timeout=10,
interface=iface,
count=1,
pcap_filter=PCAP_Or(
[
PCAP_And(
[
PCAP_Host("localhost", proto="ip", source=True, dest=True),
PCAP_Port(6015, proto="tcp", source=True),
PCAP_Port(6055, proto="tcp", dest=True),
PCAP_LessThanEqual("len", 1500),
]
),
PCAP_And(
[
PCAP_Port(80, proto="tcp", dest=True),
PCAP_PortRange(8000, 8500, proto="tcp", source=True),
PCAP_LessThanEqual("len", 1500),
]
),
]
),
)
tcps.start_send(
interface=iface,
packet_type="tcp",
count=1,
source_ip="127.0.0.1",
dest_ip="127.0.0.1",
dest_port=6055,
source_port=6015,
)
ret = tcpd.wait_for_packets(count=1, timeout=3)
tcpd.stop_capture()
self.assertEquals(1, len(ret))
def get_installed_midolman_version():
cli = LinuxCLI()
if get_linux_dist() == LINUX_UBUNTU:
full_ver_str = cli.cmd('dpkg -l | grep -w midolman')
if full_ver_str == "":
raise ArgMismatchException('Midolman package not found. Zephyr cannot run without Midolman.')
full_ver = full_ver_str.split()[2].split('~')
ver = full_ver[0]
tag_ver = '' if len(full_ver) == 1 else full_ver[1]
elif get_linux_dist() == LINUX_CENTOS:
ver = cli.cmd('yum info midolman | grep Version').split()[2]
tag_ver = cli.cmd('yum info midolman | grep Release').split()[2]
else:
raise ArgMismatchException('Must run on Ubuntu or CentOS')
return parse_midolman_version(ver, tag_ver)
def test_early_stop(self):
tcpd = TCPDump()
tcps = TCPSender()
out = LinuxCLI().cmd('ip l | grep "LOOPBACK" | cut -f 2 -d " "| cut -f 1 -d ":"')
lo_iface = out.split()[0].rstrip()
tcpd.start_capture(
interface=lo_iface,
count=0,
pcap_filter=PCAP_And(
[
PCAP_Host("localhost", proto="ip", source=True, dest=True),
PCAP_Port(6015, proto="tcp", source=True),
PCAP_Port(6055, proto="tcp", dest=True),
]
),
)
tcps.start_send(
interface=lo_iface,
packet_type="tcp",
count=5,
source_ip="127.0.0.1",
dest_ip="127.0.0.1",
dest_port=6055,
source_port=6015,
)
ret = tcpd.wait_for_packets(count=3, timeout=3)
self.assertEqual(3, len(ret))
ret = tcpd.wait_for_packets(count=1, timeout=3)
self.assertEqual(1, len(ret))
proc = tcpd.stop_capture()
ret = tcpd.wait_for_packets(count=1, timeout=3)
self.assertEqual(1, len(ret))
self.assertTrue(not proc.is_alive())
def _rollover_file(self, file_path, backup_dir=None,
date_pattern='%Y%m%d%H%M%S', zip_file=True):
cli = LinuxCLI(priv=False)
suff_str = '.' + datetime.datetime.now().strftime(date_pattern)
dest_dir = backup_dir if backup_dir is not None else (self.root_dir + '/log_bak')
if not cli.exists(dest_dir):
cli.mkdir(dest_dir)
dest_filename = dest_dir + '/' + os.path.basename(file_path) + suff_str
# Move the file, zip if requested
cli.move(file_path, dest_filename)
if zip_file:
cli.cmd('gzip -9 ' + dest_filename)
def test_use_active_iface(self):
cli = LinuxCLI(log_cmd=True)
h = Host('test', None, cli, lambda name: None, lambda name: None)
i = Interface('testi', h, ip_addr=['192.168.0.2'])
cli.cmd('ip l add dev testi type dummy')
self.assertTrue(cli.grep_cmd('ip l | grep testi', 'state DOWN'))
i.up()
time.sleep(1)
self.assertTrue(cli.grep_cmd('ip l | grep testi', 'UP'))
i.down()
self.assertFalse(cli.grep_cmd('ip l | grep testi', 'UP'))
cli.cmd('ip l del testi')
def init(self):
for i in self.zookeeper_hosts:
i.zookeeper_ips = self.zookeeper_ips
for i in self.cassandra_hosts:
i.cassandra_ips = self.cassandra_ips
for h in self.compute_hosts:
h.zookeeper_ips = self.zookeeper_ips
h.cassandra_ips = self.cassandra_ips
net_host = NetworkHost('net-node', LinuxCLI(), lambda name: None,
lambda name: None, self)
net_host.zookeeper_ips = self.zookeeper_ips
self.network_hosts.append(net_host)
def create_results(self, results_dir='./results', leeway=5):
cli = LinuxCLI(priv=False)
cli.rm(results_dir)
for scen, res in self.result_map.iteritems():
log_out_dir = results_dir + '/' + scen.__name__
cli.write_to_file(wfile=log_out_dir + '/results.xml',
data=res.to_junit_xml())
self.log_manager.collate_logs(log_out_dir + '/full-logs')
for tc in res.all_tests():
tcname = tc.id().split('.')[-1]
cli.mkdir(log_out_dir + '/' + tcname)
self.log_manager.slice_log_files_by_time(log_out_dir + '/' + tcname,
start_time=tc.start_time,
stop_time=tc.stop_time,
leeway=leeway,
collated_only=True)
def create_repo_file(self, component, repo_scheme, repo_dir,
repo_user=None, repo_pass=None,
repo_distro='nightly', repo_component='main'):
cli = LinuxCLI(log_cmd=True)
url_line = repo_scheme + "://"
if repo_scheme == 'https':
if repo_user is not None:
url_line += repo_user
if repo_pass is not None:
url_line += ':' + repo_pass
url_line += '@'
url_line += self.repo_server + '/' + repo_dir
cli.write_to_file('/etc/apt/sources.list.d/' + component + '.list',
'deb ' + url_line + ' ' + repo_distro + ' ' + repo_component + '\n',
append=False)
cli.cmd('curl -k ' + self.curl_server + ' | apt-key add -')
cli.cmd('apt-get update')
def test_get_file(self):
try:
cli = LinuxCLI(priv=False)
cli.write_to_file('test', 'teststr')
cli.write_to_file('testdir/test2', 'test2str')
fl = FileLocation('test')
self.assertEqual('.', fl.path)
fl.get_file(FileAccessor(), near_filename='test_copy')
self.assertTrue(cli.exists('test_copy'))
fl2 = FileLocation('testdir/test2')
self.assertEqual('testdir', fl2.path)
#fl2.get_file(SSHFileAccessor('localhost', LinuxCLI().whoami()), near_filename='test2_copy')
#self.assertTrue(cli.exists('test2_copy'))
finally:
LinuxCLI().rm('test')
LinuxCLI().rm('testdir')
LinuxCLI().rm('test_copy')
LinuxCLI().rm('test2_copy')
def install_packages(self, repo, exact_version=None):
if repo.get_type() == "rpm":
raise ArgMismatchException("Not yet supported on Redhat!")
# neutron_dep_packages = ['mysql', 'rabbitmq-server']
# neutron_packages = ['openstack-neutron', 'python-neutronclient', 'openstack-neutron-ml2']
cli = LinuxCLI(log_cmd=True)
cli.cmd("sudo debconf-set-selections <<< 'mysql-server-5.1 mysql-server/root_password password 'cat''")
cli.cmd("sudo debconf-set-selections <<< 'mysql-server-5.1 mysql-server/root_password_again password 'cat''")
if not repo.is_installed("mysql-server-5.5"):
repo.install_packages(["mysql-server-5.5", "mysql-client-5.5", "python-mysqldb"])
cli.cmd("mysqladmin -u root password cat")
cli.regex_file("/etc/mysql/my.cnf", "s/.*bind-address.*/bind-address = 127.0.0.1/")
cli.regex_file("/etc/mysql/my.cnf", "s/.*max_connections.*/max_connections = 1024/")
cli.cmd("service mysql start")
if not repo.is_installed("rabbitmq-server"):
repo.install_packages(["rabbitmq-server"])
cli.cmd("rabbitmqctl change_password guest cat")
cli.cmd("service rabbitmq-server restart")
if not repo.is_installed("neutron-server"):
repo.install_packages(
[
"neutron-server",
"neutron-dhcp-agent",
"python-neutronclient",
"python-neutron-lbaas",
"python-mysql.connector",
]
)
cli.cmd('mysql --user=root --password=cat -e "CREATE DATABASE IF NOT EXISTS neutron"')
cli.cmd("mysql --user=root --password=cat -e \"CREATE USER neutron@localhost IDENTIFIED BY 'cat'\"")
cli.cmd("mysql --user=root --password=cat -e \"CREATE USER neutron@'%' IDENTIFIED BY 'cat'\"")
cli.cmd('mysql --user=root --password=cat -e "GRANT ALL PRIVILEGES ON neutron.* TO neutron@localhost"')
cli.cmd("mysql --user=root --password=cat -e \"GRANT ALL PRIVILEGES ON neutron.* TO neutron@'%'\"")
version_config.get_installed_midolman_version()
mn_api_url = version_config.ConfigMap.get_configured_parameter("param_midonet_api_url")
cfg_file_str = (
"[DEFAULT]\n"
"core_plugin = midonet.neutron.plugin_v2.MidonetPluginV2\n"
"auth_strategy = noauth\n"
"rpc_backend = neutron.openstack.common.rpc.impl_kombu\n"
"rabbit_host = localhost\n"
"rabbit_userid = guest\n"
"rabbit_password = cat\n"
"service_plugins = lbaas\n"
"allow_overlapping_ips = True\n"
"router_scheduler_driver =\n"
"\n"
"[agent]\n"
"root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf\n"
"\n"
"[database]\n"
"connection = mysql://neutron:cat@localhost/neutron\n"
"\n"
"[oslo_concurrency]\n"
"lock_path = $state_path/lock\n"
)
cli.write_to_file("/etc/neutron/neutron.conf", cfg_file_str)
dhcp_ini_str = (
"[DEFAULT]\n"
"interface_driver = neutron.agent.linux.interface.MidonetInterfaceDriver\n"
"dhcp_driver = midonet.neutron.agent.midonet_driver.DhcpNoOpDriver\n"
"use_namespaces = True\n"
"dnsmasq_config_file = /etc/neutron/dnsmasq-neutron.conf\n"
"enable_isolated_metadata = True\n"
"\n"
"[MIDONET]\n"
"midonet_uri = " + mn_api_url + "\n"
"username = admin\n"
"password = cat\n"
"project_id = admin\n"
"auth_url = http://localhost:5000/v2.0\n"
)
cli.write_to_file("/etc/neutron/dhcp_agent.ini", dhcp_ini_str)
lbaas_cfg_str = (
"[service_providers]\n"
"service_provider = LOADBALANCER:Midonet:"
"midonet.neutron.services.loadbalancer.driver.MidonetLoadbalancerDriver:default\n"
)
cli.write_to_file("/etc/neutron/neutron_lbaas.conf", lbaas_cfg_str)
mn_plugin_str = (
"[DATABASE]\n"
"sql_connection = mysql+mysqlconnector://neutron:cat@localhost/neutron\n"
"sql_max_retries = 100\n"
"[MIDONET]\n"
"midonet_uri = " + mn_api_url + "\n"
"username = admin\n"
"password = cat\n"
"project_id = admin\n"
"auth_url = http://localhost:5000/v2.0\n"
"provider_router_id =\n"
)
cli.mkdir("/etc/neutron/plugins/midonet")
cli.write_to_file("/etc/neutron/plugins/midonet/midonet_plugin.ini", mn_plugin_str)
cli.rm("/etc/neutron/plugin.ini")
cli.cmd("ln -s /etc/neutron/plugins/midonet/midonet_plugin.ini /etc/neutron/plugin.ini")
if exact_version == "kilo" or exact_version is None:
cli.cmd(
"neutron-db-manage --config-file /etc/neutron/neutron.conf "
"--config-file /etc/neutron/plugins/ml2/ml2_conf.ini upgrade head"
)
cli.cmd("midonet-db-manage upgrade head")
else:
cli.cmd(
"neutron-db-manage --config-file /etc/neutron/neutron.conf "
"--config-file /etc/neutron/plugins/ml2/ml2_conf.ini upgrade " + exact_version
)
cli.cmd("midonet-db-manage upgrade " + exact_version)
cli.write_to_file("/etc/default/neutron-server", 'NEUTRON_PLUGIN_CONFIG="/etc/neutron/plugin.ini"\n')
cli.cmd("service neutron-server restart")
cli.cmd("service neutron-dhcp-agent restart")
repo.install_packages(
[
"neutron-server",
"neutron-dhcp-agent",
"python-neutronclient",
"python-neutron-lbaas",
"python-mysql.connector",
]
)
def clean_neutron(api, log=None):
"""
Deletes the network named 'main' in the 'admin' tenant along with the subnet. Deletes the
default secutiry group.
:type api: neutron_client.Client
:type log: logging.Logger
:return:
"""
log.debug('Clearing neutron database')
cli = LinuxCLI(log_cmd=True)
current_neutron_db = cli.cmd(r"neutron-db-manage current 2>&1 | grep '(.*)' | "
r"awk '{ print $2 }' | sed 's/(\(.*\))/\1/g'").strip()
current_midonet_db = cli.cmd(r"midonet-db-manage current 2>&1 | grep '(.*)' | "
r"awk '{ print $2 }' | sed 's/(\(.*\))/\1/g'").strip()
cli.cmd('mysql -u root --password=cat neutron -e "DROP DATABASE neutron"')
log.debug('Re-creating clean neutron database')
cli.cmd('mysql --user=root --password=cat -e "CREATE DATABASE IF NOT EXISTS neutron"')
log.debug('Re-populating neutron and midonet tables for version (neutron/mn): (' +
current_neutron_db + '/' + current_midonet_db + ')')
cli.cmd("neutron-db-manage upgrade " + current_neutron_db)
cli.cmd("midonet-db-manage upgrade " + current_midonet_db)
log.debug('Restarting neutron')
cli.cmd("service neutron-server restart")
LinuxCLI(priv=False).cmd('for i in `ip netns | grep qdhcp`; do sudo ip netns del $i; done')
def is_installed(self, package):
cli = LinuxCLI()
if not cli.grep_cmd('dpkg -l', package):
return False
return True
def install_neutron_client():
cli_safe = LinuxCLI(priv=False)
if LinuxCLI().exists('devstack') is True:
pass #cli_safe.cmd('cd devstack ; git pull; cd ..')
else:
pass #cli_safe.cmd('git clone http://github.com/openstack-dev/devstack',)
cli = LinuxCLI(priv=False)
cli.add_environment_variable(
'HOME',
cli_safe.cmd('echo $HOME', return_output=True).strip('\n'))
cli.add_environment_variable(
'PATH',
cli_safe.cmd('echo $PATH', return_output=True).strip('\n'))
cli.add_environment_variable(
'USER',
cli_safe.cmd('echo $USER', return_output=True).strip('\n'))
cli.add_environment_variable(
'WORKSPACE',
cli_safe.cmd('pwd', return_output=True).strip('\n'))
cli.add_environment_variable('MIDONET_ENABLE_Q_SVC_ONLY', 'True')
cli.add_environment_variable('LOG_COLOR', 'False')
cli.add_environment_variable('LOGDIR', cli_safe.cmd('echo `pwd`/logs'))
cli.add_environment_variable('SCREEN_LOGDIR',
cli_safe.cmd('echo `pwd`/logs'))
cli.add_environment_variable(
'LOGFILE', cli_safe.cmd('echo `pwd`/logs/stack.sh.log'))
cli.add_environment_variable('MIDONET_ENABLE_Q_SVC_ONLY', 'True')
cli.add_environment_variable('NEUTRON_REPO',
'http://github.com/tomoe/neutron')
cli.add_environment_variable('NEUTRON_REPO',
'http://github.com/tomoe/neutron')
cli.add_environment_variable('NEUTRON_BRANCH', 'midonet1')
cf_str = '#!/usr/bin/env bash\n' \
'[[local|localrc]]\n' \
'\n' \
'enable_plugin networking-midonet http://github.com/tomoe/networking-midonet.git midonet1'
LinuxCLI().write_to_file('devstack/local.conf', cf_str, False)
if cli.cmd('cd devstack ; ./stack.sh') is not 0:
raise SubprocessFailedException('devstack/stack.sh')
LinuxCLI().regex_file('/etc/midolman/midolman.conf',
's/\(enabled = \)true/\1false/')