def start_all_containers(self):
t0 = time()
# start all the containers
self.pt("Starting all containers ...")
cmd = command_defs['docker_compose_start_all']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.pt("Waiting for voltha and chameleon containers to be ready ...")
self.wait_till('voltha services HEALTHY',
lambda: verify_all_services_healthy(
LOCAL_CONSUL, service_name='voltha-grpc') == True,
timeout=10)
self.wait_till('chameleon services HEALTHY',
lambda: verify_all_services_healthy(
LOCAL_CONSUL,service_name='chameleon-rest') == True,
timeout=10)
# Chameleon takes some time to compile the protos and make them
# available. So let's wait 10 seconds
sleep(10)
def start_voltha(self):
t0 = time()
self.pt("Starting voltha ...")
cmd = command_defs['docker_compose_start_voltha']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.pt("Waiting for voltha to be ready ...")
self.wait_till('voltha service HEALTHY',
lambda: verify_all_services_healthy(
LOCAL_CONSUL, service_name='vcore-grpc') == True,
timeout=30)
self.pt("Voltha is ready ...")
def start_all_containers(self):
t0 = time()
# start all the containers
self.pt("Starting all containers ...")
cmd = command_defs['docker_compose_start_all']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.pt("Waiting for voltha container to be ready ...")
self.wait_till('voltha services HEALTHY',
lambda: verify_all_services_healthy(
LOCAL_CONSUL, service_name='vcore-grpc') == True,
timeout=10)
sleep(10)
def start_all_containers(self):
t0 = time()
# start all the containers
self.pt("Starting all containers ...")
cmd = command_defs['docker_compose_start_all']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.pt("Waiting for voltha container to be ready ...")
self.wait_till('voltha services HEALTHY',
lambda: verify_all_services_healthy(
LOCAL_CONSUL, service_name='voltha-grpc') == True,
timeout=10)
sleep(10)
def test_09_dig_consul_command(self):
print "Test_09_dig_consul_command_Start:------------------"
t0 = time.time()
try:
# start all containers
print "Start all containers..."
self._start_all_containers()
print "Waiting for all containers to be ready ..."
time.sleep(10)
rc = verify_all_services_healthy(LOCAL_CONSUL)
if not rc:
print "Not all services are up"
self.assertEqual(rc, True)
# Get the IP address(es) for voltha's REST interface
print "Get IP of Voltha REST interface..."
cmd = command_defs['consul_get_voltha_rest_a_record']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertGreaterEqual(out.find("voltha-health.service.consul"),
0)
# Get only the ip address
cmd = command_defs['consul_get_voltha_rest_ip']
ip, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertTrue(is_valid_ip(ip))
# Get the exposed service port
print "Get Voltha exposed service port..."
cmd = command_defs['consul_get_voltha_service_port']
port, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# Verify that we can connect to the port using the previously
# acquired ip
print "Verify connectivity with voltha ip and port..."
self.assertTrue(is_open('{}:{}'.format(ip, port)))
finally:
print "Stopping all containers ..."
# clean up all created containers for this test
self._stop_and_remove_all_containers()
print "Test_09_dig_consul_command_Start_End:------------------" \
"took {} secs \n\n".format(time.time() - t0)
def test_09_dig_consul_command(self):
print "Test_09_dig_consul_command_Start:------------------"
t0 = time.time()
try:
# start all containers
print "Start all containers..."
self._start_all_containers()
print "Waiting for all containers to be ready ..."
time.sleep(10)
rc = verify_all_services_healthy(LOCAL_CONSUL)
if not rc:
print "Not all services are up"
self.assertEqual(rc, True)
# Get the IP address(es) for voltha's REST interface
print "Get IP of Voltha REST interface..."
cmd = command_defs['consul_get_voltha_rest_a_record']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertGreaterEqual(out.find("voltha-health.service.consul"),
0)
# Get only the ip address
cmd = command_defs['consul_get_voltha_rest_ip']
ip, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertTrue(is_valid_ip(ip))
# Get the exposed service port
print "Get Voltha exposed service port..."
cmd = command_defs['consul_get_voltha_service_port']
port, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
# Verify that we can connect to the port using the previously
# acquired ip
print "Verify connectivity with voltha ip and port..."
self.assertTrue(is_open('{}:{}'.format(ip, port)))
finally:
print "Stopping all containers ..."
# clean up all created containers for this test
self._stop_and_remove_all_containers()
print "Test_09_dig_consul_command_Start_End:------------------" \
"took {} secs \n\n".format(time.time() - t0)
def stop_remove_start_voltha(self):
self.stop_voltha(remove=True)
self.consume_kafka_message_starting_at = time()
self.start_voltha()
# REST endpoint may have changed after a voltha restart
# Send a basic command to trigger the REST endpoint to refresh itself
try:
self.get_devices()
except Exception as e:
self.pt('get-devices-fail expected')
# Wait for everything to settle
self.wait_till(
'chameleon service HEALTHY',
lambda: verify_all_services_healthy(
LOCAL_CONSUL, service_name='chameleon-rest') == True,
timeout=30)
# Chameleon takes some time to compile the protos and make them
# available. So let's wait 10 seconds
sleep(10)
# Update the REST endpoint info
self.set_rest_endpoint()
def _is_voltha_ensemble_ready(self):
res = verify_all_services_healthy(LOCAL_CONSUL)
if not res:
print "Not all consul services are ready ..."
return res
def verify_all_services_healthy(self,
service_name=None,
number_of_expected_services=None):
return verify_all_services_healthy(self.LOCAL_CONSUL, service_name,
number_of_expected_services)
def test_07_start_all_containers(self):
print "Test_07_start_all_containers_Start:------------------ "
t0 = time.time()
try:
# Pre-test - clean up all running docker containers
print "Pre-test: Removing all running containers ..."
self._stop_and_remove_all_containers()
# get a list of services in the docker-compose file
print "Getting list of services in docker compose file ..."
cmd = command_defs['docker_compose_services']
services, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
docker_service_list = services.split()
self.assertGreaterEqual(len(docker_service_list),
DOCKER_COMPOSE_FILE_SERVICES_COUNT)
# start all the containers
print "Starting all containers ..."
cmd = command_defs['docker_compose_start_all']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
print "Waiting for all containers to be ready ..."
time.sleep(10)
rc = verify_all_services_healthy(LOCAL_CONSUL)
if not rc:
print "Not all services are up"
self.assertEqual(rc, True)
# verify that all containers are running
print "Verify all services are running using docker command ..."
for service in docker_service_list:
cmd = command_defs['docker_compose_ps'] + ' {} | wc -l'.format(
service)
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertGreaterEqual(out, 3) # 2 are for headers
# Verify that 'docker ps' return the same number of running process
cmd = command_defs['docker_ps_count']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
self.assertGreaterEqual(out.split(),
[str(len(docker_service_list))])
# Retrieve the list of services from consul and validate against
# the list obtained from docker composed
print "Verify all services are registered in consul ..."
expected_services = [
'consul-rest', 'fluentd-intake', 'chameleon-rest',
'voltha-grpc', 'voltha-health', 'consul-8600', 'zookeeper',
'consul', 'kafka'
]
cmd = command_defs['consul_get_services']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
try:
consul_services = json.loads(out)
intersected_services = [
s for s in expected_services if s in consul_services
]
self.assertEqual(len(intersected_services),
len(expected_services))
# services_match = 0
# for d_service in docker_service_list:
# for c_service in consul_services:
# if c_service.find(d_service) != -1:
# services_match += 1
# print d_service, c_service
# break
# self.assertEqual(services_match, len(docker_service_list))
except Exception as e:
self.assertRaises(e)
# Verify the service record of the voltha service
print "Verify the service record of voltha in consul ..."
expected_srv_elements = [
'ModifyIndex', 'CreateIndex', 'ServiceEnableTagOverride',
'Node', 'Address', 'TaggedAddresses', 'ServiceID',
'ServiceName', 'ServiceTags', 'ServiceAddress', 'ServicePort'
]
cmd = command_defs['consul_get_srv_voltha_health']
out, err, rc = run_command_to_completion_with_raw_stdout(cmd)
self.assertEqual(rc, 0)
try:
srv = json.loads(out)
intersect_elems = [
e for e in srv[0] if e in expected_srv_elements
]
self.assertEqual(len(expected_srv_elements),
len(intersect_elems))
except Exception as e:
self.assertRaises(e)
# Verify kafka client is receiving the messages
print "Verify kafka client has heartbeat topic ..."
expected_pattern = ['voltha.heartbeat']
kafka_endpoint = get_endpoint_from_consul(LOCAL_CONSUL, 'kafka')
cmd = command_defs['kafka_client_run'].format(kafka_endpoint)
kafka_client_output = run_long_running_command_with_timeout(
cmd, 20)
# TODO check that there are heartbeats
# Verify the kafka client output
# instance id
found = False
for out in kafka_client_output:
if all(ep for ep in expected_pattern if ep in out):
found = True
break
self.assertTrue(found)
print "Verify kafka client is receiving the heartbeat messages from voltha..."
expected_pattern = ['heartbeat', 'compose_voltha_1']
cmd = command_defs['kafka_client_heart_check'].format(
kafka_endpoint)
kafka_client_output = run_long_running_command_with_timeout(
cmd, 20)
print kafka_client_output
# TODO check that there are heartbeats
# Verify the kafka client output
# instance id
found = False
for out in kafka_client_output:
if all(ep for ep in expected_pattern if ep in out):
found = True
break
self.assertTrue(found)
# verify docker-compose logs are being produced - just get the
# first work of each line
print "Verify docker compose logs has output from all the services " \
"..."
expected_output = [
'voltha_1', 'fluentd_1', 'consul_1', 'registrator_1',
'kafka_1', 'zookeeper_1', 'chameleon_1', 'ofagent_1',
'netconf_1'
]
cmd = command_defs['docker_compose_logs']
docker_compose_logs = run_long_running_command_with_timeout(
cmd, 5, 0)
intersected_logs = [
l for l in expected_output if l in docker_compose_logs
]
self.assertEqual(len(intersected_logs), len(expected_output))
# TODO: file in /tmp/fluentd/ cannot be found
# # verify fluentd logs are being produced - we will just verify
# that there are "voltha.logging" in the logs
# os.environ["PYTHONPATH"] += os.pathsep + "/tmp/fluentd/"
# os.environ['PATH'] += os.pathsep + "/tmp/fluentd/"
# expected_output=['voltha.logging']
# cmd = command_defs['fluentd_logs']
# fluentd_logs, err = run_command_to_completion_with_raw_stdout(cmd)
# # self.assertIsNone(err)
# print err
# intersected_logs = [l for l in expected_output if
# l in fluentd_logs]
# self.assertEqual(len(intersected_logs), len(expected_output))
# verify docker voltha logs are being produced - we will just verify
# some
# key messages in the logs
print "Verify docker voltha logs are produced ..."
expected_output = [
'kafka_proxy.send_message', 'coordinator._renew_session',
'main.heartbeat'
]
cmd = command_defs['docker_voltha_logs']
docker_voltha_logs = run_long_running_command_with_timeout(
cmd, 0.5, 3)
intersected_logs = [
l for l in expected_output if l in docker_voltha_logs
]
self.assertEqual(len(intersected_logs), len(expected_output))
finally:
print "Stopping all containers ..."
# clean up all created containers for this test
self._stop_and_remove_all_containers()
print "Test_07_start_all_containers_End:------------------ took {}" \
" secs \n\n".format(time.time() - t0)
def _is_voltha_ensemble_ready(self):
res = verify_all_services_healthy(LOCAL_CONSUL)
if not res:
print "Not all consul services are ready ..."
return res
