def test_08_stop_all_containers_started_using_docker_compose(self):
print "Test_08_stop_all_containers_started_using_docker_compose_Start:" \
"------------------ "
t0 = time.time()
try:
# commands to stop and clear the docker images
cmds = [command_defs['docker_compose_stop'],
command_defs['docker_compose_rm_f']]
print "Stopping all containers ..."
for cmd in cmds:
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# Verify that no docker process is running
print "Verify no containers is running..."
cmd = command_defs['docker_compose_services_running']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
finally:
print "Test_08_stop_all_containers_started_using_docker_compose_:" \
"------------------ took {} secs \n\n".format(
time.time() - t0)
def test_08_stop_all_containers_started_using_docker_compose(self):
print "Test_08_stop_all_containers_started_using_docker_compose_Start:" \
"------------------ "
t0 = time.time()
try:
# commands to stop and clear the docker images
cmds = [
command_defs['docker_compose_stop'],
command_defs['docker_compose_rm_f']
]
print "Stopping all containers ..."
for cmd in cmds:
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# Verify that no docker process is running
print "Verify no containers is running..."
cmd = command_defs['docker_compose_services_running']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
finally:
print "Test_08_stop_all_containers_started_using_docker_compose_:" \
"------------------ took {} secs \n\n".format(
time.time() - t0)
def test_10_scale_voltha(self):
print "Test_10_scale_voltha_Start:------------------"
t0 = time.time()
try:
# start all containers
print "Start all containers..."
self._start_all_containers()
print "Waiting for all containers to be ready ..."
time.sleep(10)
rc, not_found_list = self._wait_for_all_containers_to_ready()
if rc:
print "Not found patterns:{}".format(not_found_list)
self.assertEqual(rc, 0)
# Scale voltha to 10 instances
print "Scale voltha to 10 instances ..."
cmd = command_defs['docker_compose_scale_voltha_to_10']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# Verify that 10 instances are running
print "Verify 10 instances of voltha are running ..."
cmd = command_defs['docker_compose_scaled_voltha_ps']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertEqual(out.split(), ['10'])
finally:
print "Stopping all containers ..."
# clean up all created containers for this test
self._stop_and_remove_all_containers()
print "Test_10_scale_voltha_End:------------------took {} secs " \
"\n\n".format(time.time() - t0)
def _stop_docker_container_by_id(self, instance_id):
# stop
cmd = command_defs['docker_stop'] + " {}".format(instance_id)
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# remove
cmd = command_defs['docker_rm'] + " {}".format(instance_id)
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
def _stop_docker_container_by_id(self, instance_id):
# stop
cmd = command_defs['docker_stop'] + " {}".format(instance_id)
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# remove
cmd = command_defs['docker_rm'] + " {}".format(instance_id)
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
def _run_consul(self):
# run consul
cmd = command_defs['docker_compose_start_consul']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# verify consul is up
cmd = command_defs['docker_compose_is_consul_up']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertIn('compose_consul_1', out)
def _run_consul(self):
# run consul
cmd = command_defs['docker_compose_start_consul']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# verify consul is up
cmd = command_defs['docker_compose_is_consul_up']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertIn('compose_consul_1', out)
def test_05_run_consul_only(self):
print "Test_05_run_consul_only_Start:------------------ "
t0 = time.time()
try:
# run consul
print "Start consul ..."
self._run_consul()
print "Waiting for consul to be ready ..."
rc = self._wait_for_consul_to_be_ready()
self.assertEqual(rc, 0)
# Get the docker IP address and port number of the consul instance
print "Get consul leader IP ..."
cmd = command_defs['consul_get_leader_ip_port']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# validate that the returned ip:port is valid and open
print "Verify consul IP and port is reachable ..."
self.assertTrue(is_open(out))
finally:
# clean up all created containers for this test
print "Stop consul ..."
self._stop_and_remove_all_containers()
print "Test_05_run_consul_only_End:------------------ took {} secs" \
"\n\n".format(time.time() - t0)
def test_05_run_consul_only(self):
print "Test_05_run_consul_only_Start:------------------ "
t0 = time.time()
try:
# run consul
print "Start consul ..."
self._run_consul()
print "Waiting for consul to be ready ..."
rc = self._wait_for_consul_to_be_ready()
self.assertEqual(rc, 0)
# Get the docker IP address and port number of the consul instance
print "Get consul leader IP ..."
cmd = command_defs['consul_get_leader_ip_port']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# validate that the returned ip:port is valid and open
print "Verify consul IP and port is reachable ..."
self.assertTrue(is_open(out))
finally:
# clean up all created containers for this test
print "Stop consul ..."
self._stop_and_remove_all_containers()
print "Test_05_run_consul_only_End:------------------ took {} secs" \
"\n\n".format(time.time() - t0)
def test_09_dig_consul_command(self):
print "Test_09_dig_consul_command_Start:------------------"
t0 = time.time()
try:
# start all containers
print "Start all containers..."
self._start_all_containers()
print "Waiting for all containers to be ready ..."
time.sleep(10)
rc = verify_all_services_healthy(LOCAL_CONSUL)
if not rc:
print "Not all services are up"
self.assertEqual(rc, True)
# Get the IP address(es) for voltha's REST interface
print "Get IP of Voltha REST interface..."
cmd = command_defs['consul_get_voltha_rest_a_record']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertGreaterEqual(out.find("voltha-health.service.consul"),
0)
# Get only the ip address
cmd = command_defs['consul_get_voltha_rest_ip']
ip, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertTrue(is_valid_ip(ip))
# Get the exposed service port
print "Get Voltha exposed service port..."
cmd = command_defs['consul_get_voltha_service_port']
port, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# Verify that we can connect to the port using the previously
# acquired ip
print "Verify connectivity with voltha ip and port..."
self.assertTrue(is_open('{}:{}'.format(ip, port)))
finally:
print "Stopping all containers ..."
# clean up all created containers for this test
self._stop_and_remove_all_containers()
print "Test_09_dig_consul_command_Start_End:------------------" \
"took {} secs \n\n".format(time.time() - t0)
def test_09_dig_consul_command(self):
print "Test_09_dig_consul_command_Start:------------------"
t0 = time.time()
try:
# start all containers
print "Start all containers..."
self._start_all_containers()
print "Waiting for all containers to be ready ..."
time.sleep(10)
rc = verify_all_services_healthy(LOCAL_CONSUL)
if not rc:
print "Not all services are up"
self.assertEqual(rc, True)
# Get the IP address(es) for voltha's REST interface
print "Get IP of Voltha REST interface..."
cmd = command_defs['consul_get_voltha_rest_a_record']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertGreaterEqual(out.find("voltha-health.service.consul"),
0)
# Get only the ip address
cmd = command_defs['consul_get_voltha_rest_ip']
ip, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertTrue(is_valid_ip(ip))
# Get the exposed service port
print "Get Voltha exposed service port..."
cmd = command_defs['consul_get_voltha_service_port']
port, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# Verify that we can connect to the port using the previously
# acquired ip
print "Verify connectivity with voltha ip and port..."
self.assertTrue(is_open('{}:{}'.format(ip, port)))
finally:
print "Stopping all containers ..."
# clean up all created containers for this test
self._stop_and_remove_all_containers()
print "Test_09_dig_consul_command_Start_End:------------------" \
"took {} secs \n\n".format(time.time() - t0)
def _stop_and_remove_all_containers(self):
# check if there are any running containers first
cmd = command_defs['docker_ps']
out, err, rc = run_command_to_completion_with_stdout_in_list(cmd)
self.assertEqual(rc, 0)
if len(out) > 1: # not counting docker ps header
cmd = command_defs['docker_stop_and_remove_all_containers']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
def _stop_and_remove_all_containers(self):
# check if there are any running containers first
cmd = command_defs['docker_ps']
out, err, rc = run_command_to_completion_with_stdout_in_list(cmd)
self.assertEqual(rc, 0)
if len(out) > 1: # not counting docker ps header
cmd = command_defs['docker_stop_and_remove_all_containers']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
def test_03_make(self):
print "Test_03_make_build_Start:------------------"
t0 = time.time()
try:
cmd = command_defs['make_clean_build']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
finally:
print "Test_03_make_build_Start:------------------ took {} secs \n\n" \
.format(time.time() - t0)
def test_03_make(self):
print "Test_03_make_build_Start:------------------"
t0 = time.time()
try:
cmd = command_defs['make_clean_build']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
finally:
print "Test_03_make_build_Start:------------------ took {} secs \n\n" \
.format(time.time() - t0)
def test_10_scale_voltha(self):
print "Test_10_scale_voltha_Start:------------------"
t0 = time.time()
try:
# start all containers
print "Start all containers..."
self._start_all_containers()
# Instead of using only a fixed timeout:
# 1) wait until the services are ready (polling per second)
# 2) bail out after a longer timeout.
print "Waiting for all containers to be ready ..."
self.wait_till('Not all services are up',
self._is_voltha_ensemble_ready,
interval=1,
timeout=30)
# Scale voltha to 10 instances
print "Scale voltha to 10 instances ..."
cmd = command_defs['docker_compose_scale_voltha_to_10']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# Verify that 10 instances are running
print "Verify 10 instances of voltha are running ..."
cmd = command_defs['docker_compose_scaled_voltha_ps']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertEqual(out.split(), ['10'])
finally:
print "Stopping all containers ..."
# clean up all created containers for this test
self._stop_and_remove_all_containers()
print "Test_10_scale_voltha_End:------------------took {} secs " \
"\n\n".format(time.time() - t0)
def test_10_scale_voltha(self):
print "Test_10_scale_voltha_Start:------------------"
t0 = time.time()
try:
# start all containers
print "Start all containers..."
self._start_all_containers()
# Instead of using only a fixed timeout:
# 1) wait until the services are ready (polling per second)
# 2) bail out after a longer timeout.
print "Waiting for all containers to be ready ..."
self.wait_till('Not all services are up',
self._is_voltha_ensemble_ready,
interval=1,
timeout=30)
# Scale voltha to 10 instances
print "Scale voltha to 10 instances ..."
cmd = command_defs['docker_compose_scale_voltha_to_10']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# Verify that 10 instances are running
print "Verify 10 instances of voltha are running ..."
cmd = command_defs['docker_compose_scaled_voltha_ps']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertEqual(out.split(), ['10'])
finally:
print "Stopping all containers ..."
# clean up all created containers for this test
self._stop_and_remove_all_containers()
print "Test_10_scale_voltha_End:------------------took {} secs " \
"\n\n".format(time.time() - t0)
def test_01_setup(self):
print "Test_01_setup_Start:------------------"
t0 = time.time()
try:
# remove the venv-linux directory
print "Remove venv-linux ..."
cmd = command_defs['remove_env_directory']
rm_venv, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# make clean
print "Make clean ..."
cmd = command_defs['make_clean']
mk_clean, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# source the env
print "Source environment ..."
self._source_env()
finally:
print "Test_01_setup_End:------------------ took {} " \
"secs\n\n".format(time.time() - t0)
def test_01_setup(self):
print "Test_01_setup_Start:------------------"
t0 = time.time()
try:
# remove the venv-linux directory
print "Remove venv-linux ..."
cmd = command_defs['remove_env_directory']
rm_venv, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# make clean
print "Make clean ..."
cmd = command_defs['make_clean']
mk_clean, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# source the env
print "Source environment ..."
self._source_env()
finally:
print "Test_01_setup_End:------------------ took {} " \
"secs\n\n".format(time.time() - t0)
def test_02_make_fetch(self):
print "Test_02_make_fetch_Start:------------------"
t0 = time.time()
try:
# Get list of images to fetch from the Makefile
print "Get list of images to fetch ..."
cmd = command_defs['makefile_fetch_images']
makefile_images_to_fetch, err, rc \
= run_command_to_completion_with_stdout_in_list(cmd)
self.assertEqual(rc, 0)
images_to_fetch = []
for image in makefile_images_to_fetch:
tmp = ''.join(image.split())
images_to_fetch.append(tmp[len('dockerpull'):])
# make fetch
print "Fetching images {} ...".format(images_to_fetch)
cmd = command_defs['make_fetch']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# verify that the images have been downloaded
print "Verify images downloaded and present locally ..."
cmd = command_defs['docker_images']
local_images, err, rc = \
run_command_to_completion_with_stdout_in_list(cmd)
self.assertEqual(rc, 0)
local_images_list = []
for local_image in local_images:
words = local_image.split()
local_images_list.append('{}:{}'.format(words[0], words[1]))
intersection_list = [
i for i in images_to_fetch if i in local_images_list
]
assert len(intersection_list) == len(images_to_fetch)
finally:
print "Test_02_make_fetch_End:------------------ took {} " \
"secs \n\n".format(time.time() - t0)
def test_02_make_fetch(self):
print "Test_02_make_fetch_Start:------------------"
t0 = time.time()
try:
# Get list of images to fetch from the Makefile
print "Get list of images to fetch ..."
cmd = command_defs['makefile_fetch_images']
makefile_images_to_fetch, err, rc \
= run_command_to_completion_with_stdout_in_list(cmd)
self.assertEqual(rc, 0)
images_to_fetch = []
for image in makefile_images_to_fetch:
tmp = ''.join(image.split())
images_to_fetch.append(tmp[len('dockerpull'):])
# make fetch
print "Fetching images {} ...".format(images_to_fetch)
cmd = command_defs['make_fetch']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# verify that the images have been downloaded
print "Verify images downloaded and present locally ..."
cmd = command_defs['docker_images']
local_images, err, rc = \
run_command_to_completion_with_stdout_in_list(cmd)
self.assertEqual(rc, 0)
local_images_list = []
for local_image in local_images:
words = local_image.split()
local_images_list.append('{}:{}'.format(words[0], words[1]))
intersection_list = [i for i in images_to_fetch if
i in local_images_list]
assert len(intersection_list) == len(images_to_fetch)
finally:
print "Test_02_make_fetch_End:------------------ took {} " \
"secs \n\n".format(time.time() - t0)
def _wait_for_consul_to_be_ready(self):
# Consul is ready when it's leader ip and port is set. The maximum
# time to wait of 60 secs as consul should be ready by then
max_wait_time = 60
t0 = time.time()
while True:
# Get the docker IP address and port number of the consul instance
print "waiting for consul to be ready ..."
cmd = command_defs['consul_get_leader_ip_port']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
out = out.strip()
if rc != 0:
# Something is wrong, return
return -1 # error
elif out is not None and out != '':
return 0 # found something
elif time.time() - t0 > max_wait_time:
return -1 # consul should have come up by this time
else:
time.sleep(2) # constant sleep for testing
def _wait_for_consul_to_be_ready(self):
# Consul is ready when it's leader ip and port is set. The maximum
# time to wait of 60 secs as consul should be ready by then
max_wait_time = 60
t0 = time.time()
while True:
# Get the docker IP address and port number of the consul instance
print "waiting for consul to be ready ..."
cmd = command_defs['consul_get_leader_ip_port']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
out = out.strip()
if rc != 0:
# Something is wrong, return
return -1 # error
elif out is not None and out != '':
return 0 # found something
elif time.time() - t0 > max_wait_time:
return -1 # consul should have come up by this time
else:
time.sleep(2) # constant sleep for testing
def test_06_run_voltha_standalone_with_consul_only(self):
print "Test_06_run_voltha_standalone_with_consul_only_Start:----------" \
"-------- "
t0 = time.time()
try:
# run consul first
print "Start consul ..."
self._run_consul()
# get consul ip
print "Get consul IP ..."
cmd = command_defs['docker_get_consul_ip']
consul_ip, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertIsNotNone(consul_ip)
# start voltha now for 15 secs and verify it can now connect to
# consul - following message in the output
print "Start voltha with consul IP ..."
expected_pattern = ['coordinator', 'event: created-consul-session']
cmd = command_defs['docker_start_voltha_with_consul_ip'] + \
'{}:8500'.format(consul_ip.strip())
command_output = run_long_running_command_with_timeout(cmd, 10)
# Verify the output of voltha and get the container instance id
print "Verify voltha is registered with consul ..."
instance_id = None
for out in command_output:
if all(ep for ep in expected_pattern if ep in out):
self.assertTrue(True)
instance_id = re.findall(r'[0-9a-f]+', out)[-1]
break
self.assertIsNotNone(instance_id)
# Verify Voltha's self-registration with consul
expected_output = ['ModifyIndex', 'CreateIndex', 'Session',
'Value',
'Flags', 'Key', 'LockIndex']
cmd = command_defs['consul_verify_voltha_registration']
registration_info, err, rc = \
run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
try:
jr_info = json.loads(registration_info)
intersect_elems = [e for e in jr_info[0] if
e in expected_output]
self.assertEqual(len(expected_output), len(intersect_elems))
except Exception as e:
self.assertRaises(e)
# stop voltha
print "Stop voltha ..."
self._stop_docker_container_by_id(instance_id)
# check the service has deregistered
print "Verify voltha is no longer registered in consul..."
cmd = command_defs['consul_verify_voltha_registration']
registration_info, err, rc = \
run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertEqual(registration_info, '')
finally:
# clean up all created containers for this test
print "Stop consul ..."
self._stop_and_remove_all_containers()
print "Test_06_run_voltha_standalone_with_consul_only_End:--------" \
"---------- took {} " \
"secs \n\n".format(time.time() - t0)
def test_07_start_all_containers(self):
print "Test_07_start_all_containers_Start:------------------ "
t0 = time.time()
try:
# Pre-test - clean up all running docker containers
print "Pre-test: Removing all running containers ..."
cmd = command_defs['docker_compose_stop']
_, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
cmd = command_defs['docker_compose_rm_f']
_, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# get a list of services in the docker-compose file
print "Getting list of services in docker compose file ..."
cmd = command_defs['docker_compose_services']
services, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
docker_service_list = services.split()
self.assertGreaterEqual(len(docker_service_list),
DOCKER_COMPOSE_FILE_SERVICES_COUNT)
# start all the containers
print "Starting all containers ..."
cmd = command_defs['docker_compose_start_all']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# Instead of using only a fixed timeout:
# 1) wait until the services are ready (polling per second)
# 2) bail out after a longer timeout.
print "Waiting for all containers to be ready ..."
self.wait_till('Not all services are up',
self._is_voltha_ensemble_ready,
interval=1,
timeout=30)
# verify that all containers are running
print "Verify all services are running using docker command ..."
for service in docker_service_list:
cmd = command_defs['docker_compose_ps'] + ' {} | wc -l'.format(
service)
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertGreaterEqual(out, 3) # 2 are for headers
# Verify that 'docker ps' return the same number of running process
cmd = command_defs['docker_ps_count']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertGreaterEqual(out.split(), [str(len(
docker_service_list))])
# Retrieve the list of services from consul and validate against
# the list obtained from docker composed
print "Verify all services are registered in consul ..."
expected_services = ['consul-rest', 'fluentd-intake',
'voltha-grpc',
'voltha-health',
'consul-8600', 'zookeeper', 'consul',
'kafka']
cmd = command_defs['consul_get_services']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
try:
consul_services = json.loads(out)
intersected_services = [s for s in expected_services if
s in consul_services]
self.assertEqual(len(intersected_services),
len(expected_services))
# services_match = 0
# for d_service in docker_service_list:
# for c_service in consul_services:
# if c_service.find(d_service) != -1:
# services_match += 1
# print d_service, c_service
# break
# self.assertEqual(services_match, len(docker_service_list))
except Exception as e:
self.assertRaises(e)
# Verify the service record of the voltha service
print "Verify the service record of voltha in consul ..."
expected_srv_elements = ['ModifyIndex', 'CreateIndex',
'ServiceEnableTagOverride', 'Node',
'Address', 'TaggedAddresses', 'ServiceID',
'ServiceName', 'ServiceTags',
'ServiceAddress', 'ServicePort']
cmd = command_defs['consul_get_srv_voltha_health']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
try:
srv = json.loads(out)
intersect_elems = [e for e in srv[0] if
e in expected_srv_elements]
self.assertEqual(len(expected_srv_elements),
len(intersect_elems))
except Exception as e:
self.assertRaises(e)
# Verify kafka client is receiving the messages
print "Verify kafka client has heartbeat topic ..."
expected_pattern = ['voltha.heartbeat']
kafka_endpoint = get_endpoint_from_consul(LOCAL_CONSUL,'kafka')
cmd = command_defs['kafka_client_run'].format(kafka_endpoint)
kafka_client_output = run_long_running_command_with_timeout(cmd, 20)
# Verify the kafka client output
# instance id
found = False
for out in kafka_client_output:
if all(ep for ep in expected_pattern if ep in out):
found = True
break
self.assertTrue(found)
# Commented the heartbeat messages from voltha as on Jenkins this
# test fails more often than not. On local or cluster environment
# the kafka event bus works well.
# verify docker-compose logs are being produced - just get the
# first work of each line
print "Verify docker compose logs has output from all the services " \
"..."
expected_output = ['voltha_1', 'fluentd_1', 'consul_1',
'registrator_1', 'kafka_1', 'zookeeper_1',
'ofagent_1', 'netconf_1']
cmd = command_defs['docker_compose_logs']
docker_compose_logs = run_long_running_command_with_timeout(cmd, 5, 0)
intersected_logs = [l for l in expected_output if
l in docker_compose_logs]
self.assertEqual(len(intersected_logs), len(expected_output))
# verify docker voltha logs are being produced - we will just verify
# some
# key messages in the logs
print "Verify docker voltha logs are produced ..."
expected_output = ['coordinator._renew_session', 'main.heartbeat']
cmd = command_defs['docker_voltha_logs']
docker_voltha_logs = run_long_running_command_with_timeout(cmd,
0.5, 5)
intersected_logs = [l for l in expected_output if
l in docker_voltha_logs]
self.assertEqual(len(intersected_logs), len(expected_output))
finally:
print "Stopping all containers ..."
# clean up all created containers for this test
#self._stop_and_remove_all_containers()
cmd = command_defs['docker_compose_down']
_, err, rc = run_command_to_completion_with_raw_stdout(cmd)
print "Test_07_start_all_containers_End:------------------ took {}" \
" secs \n\n".format(time.time() - t0)
def _start_all_containers(self):
cmd = command_defs['docker_compose_start_all']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
def _start_all_containers(self):
cmd = command_defs['docker_compose_start_all']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
def test_07_start_all_containers(self):
print "Test_07_start_all_containers_Start:------------------ "
t0 = time.time()
try:
# Pre-test - clean up all running docker containers
print "Pre-test: Removing all running containers ..."
self._stop_and_remove_all_containers()
# get a list of services in the docker-compose file
print "Getting list of services in docker compose file ..."
cmd = command_defs['docker_compose_services']
services, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
docker_service_list = services.split()
self.assertGreaterEqual(len(docker_service_list),
DOCKER_COMPOSE_FILE_SERVICES_COUNT)
# start all the containers
print "Starting all containers ..."
cmd = command_defs['docker_compose_start_all']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
print "Waiting for all containers to be ready ..."
time.sleep(10)
rc = verify_all_services_healthy(LOCAL_CONSUL)
if not rc:
print "Not all services are up"
self.assertEqual(rc, True)
# verify that all containers are running
print "Verify all services are running using docker command ..."
for service in docker_service_list:
cmd = command_defs['docker_compose_ps'] + ' {} | wc -l'.format(
service)
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertGreaterEqual(out, 3) # 2 are for headers
# Verify that 'docker ps' return the same number of running process
cmd = command_defs['docker_ps_count']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertGreaterEqual(out.split(),
[str(len(docker_service_list))])
# Retrieve the list of services from consul and validate against
# the list obtained from docker composed
print "Verify all services are registered in consul ..."
expected_services = [
'consul-rest', 'fluentd-intake', 'chameleon-rest',
'voltha-grpc', 'voltha-health', 'consul-8600', 'zookeeper',
'consul', 'kafka'
]
cmd = command_defs['consul_get_services']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
try:
consul_services = json.loads(out)
intersected_services = [
s for s in expected_services if s in consul_services
]
self.assertEqual(len(intersected_services),
len(expected_services))
# services_match = 0
# for d_service in docker_service_list:
# for c_service in consul_services:
# if c_service.find(d_service) != -1:
# services_match += 1
# print d_service, c_service
# break
# self.assertEqual(services_match, len(docker_service_list))
except Exception as e:
self.assertRaises(e)
# Verify the service record of the voltha service
print "Verify the service record of voltha in consul ..."
expected_srv_elements = [
'ModifyIndex', 'CreateIndex', 'ServiceEnableTagOverride',
'Node', 'Address', 'TaggedAddresses', 'ServiceID',
'ServiceName', 'ServiceTags', 'ServiceAddress', 'ServicePort'
]
cmd = command_defs['consul_get_srv_voltha_health']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
try:
srv = json.loads(out)
intersect_elems = [
e for e in srv[0] if e in expected_srv_elements
]
self.assertEqual(len(expected_srv_elements),
len(intersect_elems))
except Exception as e:
self.assertRaises(e)
# Verify kafka client is receiving the messages
print "Verify kafka client has heartbeat topic ..."
expected_pattern = ['voltha.heartbeat']
kafka_endpoint = get_endpoint_from_consul(LOCAL_CONSUL, 'kafka')
cmd = command_defs['kafka_client_run'].format(kafka_endpoint)
kafka_client_output = run_long_running_command_with_timeout(
cmd, 20)
# TODO check that there are heartbeats
# Verify the kafka client output
# instance id
found = False
for out in kafka_client_output:
if all(ep for ep in expected_pattern if ep in out):
found = True
break
self.assertTrue(found)
print "Verify kafka client is receiving the heartbeat messages from voltha..."
expected_pattern = ['heartbeat', 'compose_voltha_1']
cmd = command_defs['kafka_client_heart_check'].format(
kafka_endpoint)
kafka_client_output = run_long_running_command_with_timeout(
cmd, 20)
print kafka_client_output
# TODO check that there are heartbeats
# Verify the kafka client output
# instance id
found = False
for out in kafka_client_output:
if all(ep for ep in expected_pattern if ep in out):
found = True
break
self.assertTrue(found)
# verify docker-compose logs are being produced - just get the
# first work of each line
print "Verify docker compose logs has output from all the services " \
"..."
expected_output = [
'voltha_1', 'fluentd_1', 'consul_1', 'registrator_1',
'kafka_1', 'zookeeper_1', 'chameleon_1', 'ofagent_1',
'netconf_1'
]
cmd = command_defs['docker_compose_logs']
docker_compose_logs = run_long_running_command_with_timeout(
cmd, 5, 0)
intersected_logs = [
l for l in expected_output if l in docker_compose_logs
]
self.assertEqual(len(intersected_logs), len(expected_output))
# TODO: file in /tmp/fluentd/ cannot be found
# # verify fluentd logs are being produced - we will just verify
# that there are "voltha.logging" in the logs
# os.environ["PYTHONPATH"] += os.pathsep + "/tmp/fluentd/"
# os.environ['PATH'] += os.pathsep + "/tmp/fluentd/"
# expected_output=['voltha.logging']
# cmd = command_defs['fluentd_logs']
# fluentd_logs, err = run_command_to_completion_with_raw_stdout(cmd)
# # self.assertIsNone(err)
# print err
# intersected_logs = [l for l in expected_output if
# l in fluentd_logs]
# self.assertEqual(len(intersected_logs), len(expected_output))
# verify docker voltha logs are being produced - we will just verify
# some
# key messages in the logs
print "Verify docker voltha logs are produced ..."
expected_output = [
'kafka_proxy.send_message', 'coordinator._renew_session',
'main.heartbeat'
]
cmd = command_defs['docker_voltha_logs']
docker_voltha_logs = run_long_running_command_with_timeout(
cmd, 0.5, 3)
intersected_logs = [
l for l in expected_output if l in docker_voltha_logs
]
self.assertEqual(len(intersected_logs), len(expected_output))
finally:
print "Stopping all containers ..."
# clean up all created containers for this test
self._stop_and_remove_all_containers()
print "Test_07_start_all_containers_End:------------------ took {}" \
" secs \n\n".format(time.time() - t0)
def test_07_start_all_containers(self):
print "Test_07_start_all_containers_Start:------------------ "
t0 = time.time()
try:
# Pre-test - clean up all running docker containers
print "Pre-test: Removing all running containers ..."
cmd = command_defs['docker_compose_stop']
_, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
cmd = command_defs['docker_compose_rm_f']
_, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# get a list of services in the docker-compose file
print "Getting list of services in docker compose file ..."
cmd = command_defs['docker_compose_services']
services, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
docker_service_list = services.split()
self.assertGreaterEqual(len(docker_service_list),
DOCKER_COMPOSE_FILE_SERVICES_COUNT)
# start all the containers
print "Starting all containers ..."
cmd = command_defs['docker_compose_start_all']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# Instead of using only a fixed timeout:
# 1) wait until the services are ready (polling per second)
# 2) bail out after a longer timeout.
print "Waiting for all containers to be ready ..."
self.wait_till('Not all services are up',
self._is_voltha_ensemble_ready,
interval=1,
timeout=30)
# verify that all containers are running
print "Verify all services are running using docker command ..."
for service in docker_service_list:
cmd = command_defs['docker_compose_ps'] + ' {} | wc -l'.format(
service)
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertGreaterEqual(out, 3) # 2 are for headers
# Verify that 'docker ps' return the same number of running process
cmd = command_defs['docker_ps_count']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertGreaterEqual(out.split(),
[str(len(docker_service_list))])
# Retrieve the list of services from consul and validate against
# the list obtained from docker composed
print "Verify all services are registered in consul ..."
expected_services = [
'consul-rest', 'fluentd-intake', 'voltha-grpc',
'voltha-health', 'consul-8600', 'zookeeper', 'consul', 'kafka'
]
cmd = command_defs['consul_get_services']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
try:
consul_services = json.loads(out)
intersected_services = [
s for s in expected_services if s in consul_services
]
self.assertEqual(len(intersected_services),
len(expected_services))
# services_match = 0
# for d_service in docker_service_list:
# for c_service in consul_services:
# if c_service.find(d_service) != -1:
# services_match += 1
# print d_service, c_service
# break
# self.assertEqual(services_match, len(docker_service_list))
except Exception as e:
self.assertRaises(e)
# Verify the service record of the voltha service
print "Verify the service record of voltha in consul ..."
expected_srv_elements = [
'ModifyIndex', 'CreateIndex', 'ServiceEnableTagOverride',
'Node', 'Address', 'TaggedAddresses', 'ServiceID',
'ServiceName', 'ServiceTags', 'ServiceAddress', 'ServicePort'
]
cmd = command_defs['consul_get_srv_voltha_health']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
try:
srv = json.loads(out)
intersect_elems = [
e for e in srv[0] if e in expected_srv_elements
]
self.assertEqual(len(expected_srv_elements),
len(intersect_elems))
except Exception as e:
self.assertRaises(e)
# Verify kafka client is receiving the messages
print "Verify kafka client has heartbeat topic ..."
expected_pattern = ['voltha.heartbeat']
kafka_endpoint = get_endpoint_from_consul(LOCAL_CONSUL, 'kafka')
cmd = command_defs['kafka_client_run'].format(kafka_endpoint)
kafka_client_output = run_long_running_command_with_timeout(
cmd, 20)
# Verify the kafka client output
# instance id
found = False
for out in kafka_client_output:
if all(ep for ep in expected_pattern if ep in out):
found = True
break
self.assertTrue(found)
# Commented the heartbeat messages from voltha as on Jenkins this
# test fails more often than not. On local or cluster environment
# the kafka event bus works well.
# verify docker-compose logs are being produced - just get the
# first work of each line
print "Verify docker compose logs has output from all the services " \
"..."
expected_output = [
'voltha_1', 'fluentd_1', 'consul_1', 'registrator_1',
'kafka_1', 'zookeeper_1', 'ofagent_1', 'netconf_1'
]
cmd = command_defs['docker_compose_logs']
docker_compose_logs = run_long_running_command_with_timeout(
cmd, 5, 0)
intersected_logs = [
l for l in expected_output if l in docker_compose_logs
]
self.assertEqual(len(intersected_logs), len(expected_output))
# verify docker voltha logs are being produced - we will just verify
# some
# key messages in the logs
print "Verify docker voltha logs are produced ..."
self.wait_till('Basic voltha logs are absent',
self._is_basic_voltha_logs_produced,
interval=1,
timeout=30)
finally:
print "Stopping all containers ..."
# clean up all created containers for this test
#self._stop_and_remove_all_containers()
cmd = command_defs['docker_compose_down']
_, err, rc = run_command_to_completion_with_raw_stdout(cmd)
print "Test_07_start_all_containers_End:------------------ took {}" \
" secs \n\n".format(time.time() - t0)
def test_06_run_voltha_standalone_with_consul_only(self):
print "Test_06_run_voltha_standalone_with_consul_only_Start:----------" \
"-------- "
t0 = time.time()
try:
# run consul first
print "Start consul ..."
self._run_consul()
# get consul ip
print "Get consul IP ..."
cmd = command_defs['docker_get_consul_ip']
consul_ip, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertIsNotNone(consul_ip)
# start voltha now for 15 secs and verify it can now connect to
# consul - following message in the output
print "Start voltha with consul IP ..."
expected_pattern = ['coordinator', 'event: created-consul-session']
cmd = command_defs['docker_start_voltha_with_consul_ip'] + \
'{}:8500'.format(consul_ip.strip())
command_output = run_long_running_command_with_timeout(cmd, 10)
# Verify the output of voltha and get the container instance id
print "Verify voltha is registered with consul ..."
instance_id = None
for out in command_output:
if all(ep for ep in expected_pattern if ep in out):
self.assertTrue(True)
instance_id = re.findall(r'[0-9a-f]+', out)[-1]
break
self.assertIsNotNone(instance_id)
# Verify Voltha's self-registration with consul
expected_output = [
'ModifyIndex', 'CreateIndex', 'Session', 'Value', 'Flags',
'Key', 'LockIndex'
]
cmd = command_defs['consul_verify_voltha_registration']
registration_info, err, rc = \
run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
try:
jr_info = json.loads(registration_info)
intersect_elems = [
e for e in jr_info[0] if e in expected_output
]
self.assertEqual(len(expected_output), len(intersect_elems))
except Exception as e:
self.assertRaises(e)
# stop voltha
print "Stop voltha ..."
self._stop_docker_container_by_id(instance_id)
# check the service has deregistered
print "Verify voltha is no longer registered in consul..."
cmd = command_defs['consul_verify_voltha_registration']
registration_info, err, rc = \
run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertEqual(registration_info, '')
finally:
# clean up all created containers for this test
print "Stop consul ..."
self._stop_and_remove_all_containers()
print "Test_06_run_voltha_standalone_with_consul_only_End:--------" \
"---------- took {} " \
"secs \n\n".format(time.time() - t0)
