def start_all_containers(self): t0 = time() # start all the containers self.pt("Starting all containers ...") cmd = command_defs['docker_compose_start_all'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.pt("Waiting for voltha and chameleon containers to be ready ...") self.wait_till('voltha services HEALTHY', lambda: verify_all_services_healthy( LOCAL_CONSUL, service_name='voltha-grpc') == True, timeout=10) self.wait_till('chameleon services HEALTHY', lambda: verify_all_services_healthy( LOCAL_CONSUL,service_name='chameleon-rest') == True, timeout=10) # Chameleon takes some time to compile the protos and make them # available. So let's wait 10 seconds sleep(10)
def start_voltha(self): t0 = time() self.pt("Starting voltha ...") cmd = command_defs['docker_compose_start_voltha'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.pt("Waiting for voltha to be ready ...") self.wait_till('voltha service HEALTHY', lambda: verify_all_services_healthy( LOCAL_CONSUL, service_name='vcore-grpc') == True, timeout=30) self.pt("Voltha is ready ...")
def start_all_containers(self): t0 = time() # start all the containers self.pt("Starting all containers ...") cmd = command_defs['docker_compose_start_all'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.pt("Waiting for voltha container to be ready ...") self.wait_till('voltha services HEALTHY', lambda: verify_all_services_healthy( LOCAL_CONSUL, service_name='vcore-grpc') == True, timeout=10) sleep(10)
def start_all_containers(self): t0 = time() # start all the containers self.pt("Starting all containers ...") cmd = command_defs['docker_compose_start_all'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.pt("Waiting for voltha container to be ready ...") self.wait_till('voltha services HEALTHY', lambda: verify_all_services_healthy( LOCAL_CONSUL, service_name='voltha-grpc') == True, timeout=10) sleep(10)
def test_09_dig_consul_command(self): print "Test_09_dig_consul_command_Start:------------------" t0 = time.time() try: # start all containers print "Start all containers..." self._start_all_containers() print "Waiting for all containers to be ready ..." time.sleep(10) rc = verify_all_services_healthy(LOCAL_CONSUL) if not rc: print "Not all services are up" self.assertEqual(rc, True) # Get the IP address(es) for voltha's REST interface print "Get IP of Voltha REST interface..." cmd = command_defs['consul_get_voltha_rest_a_record'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertGreaterEqual(out.find("voltha-health.service.consul"), 0) # Get only the ip address cmd = command_defs['consul_get_voltha_rest_ip'] ip, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertTrue(is_valid_ip(ip)) # Get the exposed service port print "Get Voltha exposed service port..." cmd = command_defs['consul_get_voltha_service_port'] port, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # Verify that we can connect to the port using the previously # acquired ip print "Verify connectivity with voltha ip and port..." self.assertTrue(is_open('{}:{}'.format(ip, port))) finally: print "Stopping all containers ..." # clean up all created containers for this test self._stop_and_remove_all_containers() print "Test_09_dig_consul_command_Start_End:------------------" \ "took {} secs \n\n".format(time.time() - t0)
def stop_remove_start_voltha(self): self.stop_voltha(remove=True) self.consume_kafka_message_starting_at = time() self.start_voltha() # REST endpoint may have changed after a voltha restart # Send a basic command to trigger the REST endpoint to refresh itself try: self.get_devices() except Exception as e: self.pt('get-devices-fail expected') # Wait for everything to settle self.wait_till( 'chameleon service HEALTHY', lambda: verify_all_services_healthy( LOCAL_CONSUL, service_name='chameleon-rest') == True, timeout=30) # Chameleon takes some time to compile the protos and make them # available. So let's wait 10 seconds sleep(10) # Update the REST endpoint info self.set_rest_endpoint()
def _is_voltha_ensemble_ready(self): res = verify_all_services_healthy(LOCAL_CONSUL) if not res: print "Not all consul services are ready ..." return res
def verify_all_services_healthy(self, service_name=None, number_of_expected_services=None): return verify_all_services_healthy(self.LOCAL_CONSUL, service_name, number_of_expected_services)
def test_07_start_all_containers(self): print "Test_07_start_all_containers_Start:------------------ " t0 = time.time() try: # Pre-test - clean up all running docker containers print "Pre-test: Removing all running containers ..." self._stop_and_remove_all_containers() # get a list of services in the docker-compose file print "Getting list of services in docker compose file ..." cmd = command_defs['docker_compose_services'] services, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) docker_service_list = services.split() self.assertGreaterEqual(len(docker_service_list), DOCKER_COMPOSE_FILE_SERVICES_COUNT) # start all the containers print "Starting all containers ..." cmd = command_defs['docker_compose_start_all'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) print "Waiting for all containers to be ready ..." time.sleep(10) rc = verify_all_services_healthy(LOCAL_CONSUL) if not rc: print "Not all services are up" self.assertEqual(rc, True) # verify that all containers are running print "Verify all services are running using docker command ..." for service in docker_service_list: cmd = command_defs['docker_compose_ps'] + ' {} | wc -l'.format( service) out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertGreaterEqual(out, 3) # 2 are for headers # Verify that 'docker ps' return the same number of running process cmd = command_defs['docker_ps_count'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertGreaterEqual(out.split(), [str(len(docker_service_list))]) # Retrieve the list of services from consul and validate against # the list obtained from docker composed print "Verify all services are registered in consul ..." expected_services = [ 'consul-rest', 'fluentd-intake', 'chameleon-rest', 'voltha-grpc', 'voltha-health', 'consul-8600', 'zookeeper', 'consul', 'kafka' ] cmd = command_defs['consul_get_services'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) try: consul_services = json.loads(out) intersected_services = [ s for s in expected_services if s in consul_services ] self.assertEqual(len(intersected_services), len(expected_services)) # services_match = 0 # for d_service in docker_service_list: # for c_service in consul_services: # if c_service.find(d_service) != -1: # services_match += 1 # print d_service, c_service # break # self.assertEqual(services_match, len(docker_service_list)) except Exception as e: self.assertRaises(e) # Verify the service record of the voltha service print "Verify the service record of voltha in consul ..." expected_srv_elements = [ 'ModifyIndex', 'CreateIndex', 'ServiceEnableTagOverride', 'Node', 'Address', 'TaggedAddresses', 'ServiceID', 'ServiceName', 'ServiceTags', 'ServiceAddress', 'ServicePort' ] cmd = command_defs['consul_get_srv_voltha_health'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) try: srv = json.loads(out) intersect_elems = [ e for e in srv[0] if e in expected_srv_elements ] self.assertEqual(len(expected_srv_elements), len(intersect_elems)) except Exception as e: self.assertRaises(e) # Verify kafka client is receiving the messages print "Verify kafka client has heartbeat topic ..." expected_pattern = ['voltha.heartbeat'] kafka_endpoint = get_endpoint_from_consul(LOCAL_CONSUL, 'kafka') cmd = command_defs['kafka_client_run'].format(kafka_endpoint) kafka_client_output = run_long_running_command_with_timeout( cmd, 20) # TODO check that there are heartbeats # Verify the kafka client output # instance id found = False for out in kafka_client_output: if all(ep for ep in expected_pattern if ep in out): found = True break self.assertTrue(found) print "Verify kafka client is receiving the heartbeat messages from voltha..." expected_pattern = ['heartbeat', 'compose_voltha_1'] cmd = command_defs['kafka_client_heart_check'].format( kafka_endpoint) kafka_client_output = run_long_running_command_with_timeout( cmd, 20) print kafka_client_output # TODO check that there are heartbeats # Verify the kafka client output # instance id found = False for out in kafka_client_output: if all(ep for ep in expected_pattern if ep in out): found = True break self.assertTrue(found) # verify docker-compose logs are being produced - just get the # first work of each line print "Verify docker compose logs has output from all the services " \ "..." expected_output = [ 'voltha_1', 'fluentd_1', 'consul_1', 'registrator_1', 'kafka_1', 'zookeeper_1', 'chameleon_1', 'ofagent_1', 'netconf_1' ] cmd = command_defs['docker_compose_logs'] docker_compose_logs = run_long_running_command_with_timeout( cmd, 5, 0) intersected_logs = [ l for l in expected_output if l in docker_compose_logs ] self.assertEqual(len(intersected_logs), len(expected_output)) # TODO: file in /tmp/fluentd/ cannot be found # # verify fluentd logs are being produced - we will just verify # that there are "voltha.logging" in the logs # os.environ["PYTHONPATH"] += os.pathsep + "/tmp/fluentd/" # os.environ['PATH'] += os.pathsep + "/tmp/fluentd/" # expected_output=['voltha.logging'] # cmd = command_defs['fluentd_logs'] # fluentd_logs, err = run_command_to_completion_with_raw_stdout(cmd) # # self.assertIsNone(err) # print err # intersected_logs = [l for l in expected_output if # l in fluentd_logs] # self.assertEqual(len(intersected_logs), len(expected_output)) # verify docker voltha logs are being produced - we will just verify # some # key messages in the logs print "Verify docker voltha logs are produced ..." expected_output = [ 'kafka_proxy.send_message', 'coordinator._renew_session', 'main.heartbeat' ] cmd = command_defs['docker_voltha_logs'] docker_voltha_logs = run_long_running_command_with_timeout( cmd, 0.5, 3) intersected_logs = [ l for l in expected_output if l in docker_voltha_logs ] self.assertEqual(len(intersected_logs), len(expected_output)) finally: print "Stopping all containers ..." # clean up all created containers for this test self._stop_and_remove_all_containers() print "Test_07_start_all_containers_End:------------------ took {}" \ " secs \n\n".format(time.time() - t0)