def setUp(self): # Run Voltha,OFAgent,3 ONOS and form ONOS cluster. print "Starting all containers ..." cmd = command_defs['docker_compose_start_all'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) print "Waiting for all containers to be ready ..." sleep(60) cmd = command_defs['onos1_ip'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) onos1_ip = out print "ONOS1 IP is {}".format(onos1_ip) cmd = command_defs['onos2_ip'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) onos2_ip = out print "ONOS2 IP is {}".format(onos2_ip) cmd = command_defs['onos3_ip'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) onos3_ip = out print "ONOS3 IP is {}".format(onos3_ip) cmd = command_defs['onos_form_cluster'] + ' {} {} {}'.format( onos1_ip.strip(), onos2_ip.strip(), onos3_ip.strip()) out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) print "Cluster Output :{} ".format(out) self.get_rest_endpoint()
def test_08_stop_all_containers_started_using_docker_compose(self): print "Test_08_stop_all_containers_started_using_docker_compose_Start:" \ "------------------ " t0 = time.time() try: # commands to stop and clear the docker images cmds = [ command_defs['docker_compose_stop'], command_defs['docker_compose_rm_f'] ] print "Stopping all containers ..." for cmd in cmds: out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # Verify that no docker process is running print "Verify no containers is running..." cmd = command_defs['docker_compose_services_running'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) finally: print "Test_08_stop_all_containers_started_using_docker_compose_:" \ "------------------ took {} secs \n\n".format( time.time() - t0)
def _stop_docker_container_by_id(self, instance_id): # stop cmd = command_defs['docker_stop'] + " {}".format(instance_id) out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # remove cmd = command_defs['docker_rm'] + " {}".format(instance_id) out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0)
def _run_consul(self): # run consul cmd = command_defs['docker_compose_start_consul'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # verify consul is up cmd = command_defs['docker_compose_is_consul_up'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertIn('compose_consul_1', out)
def tearDown(self): # Stopping and Removing Voltha,OFAgent,3 ONOS. print "Stopping and removing all containers ..." cmd = command_defs['docker_compose_stop'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) print "Waiting for all containers to be stopped ..." sleep(1) cmd = command_defs['docker_compose_rm_f'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0)
def stop_voltha(self, remove=False): t0 = time() self.pt("Stopping voltha ...") cmd = command_defs['docker_compose_stop_voltha'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) if remove: cmd = command_defs['docker_compose_remove_voltha'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0)
def test_05_run_consul_only(self): print "Test_05_run_consul_only_Start:------------------ " t0 = time.time() try: # run consul print "Start consul ..." self._run_consul() print "Waiting for consul to be ready ..." rc = self._wait_for_consul_to_be_ready() self.assertEqual(rc, 0) # Get the docker IP address and port number of the consul instance print "Get consul leader IP ..." cmd = command_defs['consul_get_leader_ip_port'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # validate that the returned ip:port is valid and open print "Verify consul IP and port is reachable ..." self.assertTrue(is_open(out)) finally: # clean up all created containers for this test print "Stop consul ..." self._stop_and_remove_all_containers() print "Test_05_run_consul_only_End:------------------ took {} secs" \ "\n\n".format(time.time() - t0)
def test_09_dig_consul_command(self): print "Test_09_dig_consul_command_Start:------------------" t0 = time.time() try: # start all containers print "Start all containers..." self._start_all_containers() print "Waiting for all containers to be ready ..." time.sleep(10) rc = verify_all_services_healthy(LOCAL_CONSUL) if not rc: print "Not all services are up" self.assertEqual(rc, True) # Get the IP address(es) for voltha's REST interface print "Get IP of Voltha REST interface..." cmd = command_defs['consul_get_voltha_rest_a_record'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertGreaterEqual(out.find("voltha-health.service.consul"), 0) # Get only the ip address cmd = command_defs['consul_get_voltha_rest_ip'] ip, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertTrue(is_valid_ip(ip)) # Get the exposed service port print "Get Voltha exposed service port..." cmd = command_defs['consul_get_voltha_service_port'] port, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # Verify that we can connect to the port using the previously # acquired ip print "Verify connectivity with voltha ip and port..." self.assertTrue(is_open('{}:{}'.format(ip, port))) finally: print "Stopping all containers ..." # clean up all created containers for this test self._stop_and_remove_all_containers() print "Test_09_dig_consul_command_Start_End:------------------" \ "took {} secs \n\n".format(time.time() - t0)
def _stop_and_remove_all_containers(self): # check if there are any running containers first cmd = command_defs['docker_ps'] out, err, rc = run_command_to_completion_with_stdout_in_list(cmd) self.assertEqual(rc, 0) if len(out) > 1: # not counting docker ps header cmd = command_defs['docker_stop_and_remove_all_containers'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0)
def start_container(self, container): print "Starting {} ...".format(container) cmd = command_defs['docker_start'].format(container) out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) sleep(10) print "Started {}".format(container)
def test_03_make(self): print "Test_03_make_build_Start:------------------" t0 = time.time() try: cmd = command_defs['make_clean_build'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) finally: print "Test_03_make_build_Start:------------------ took {} secs \n\n" \ .format(time.time() - t0)
def start_voltha(self): t0 = time() self.pt("Starting voltha ...") cmd = command_defs['docker_compose_start_voltha'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.pt("Waiting for voltha to be ready ...") self.wait_till('voltha service HEALTHY', lambda: verify_all_services_healthy( LOCAL_CONSUL, service_name='vcore-grpc') == True, timeout=30) self.pt("Voltha is ready ...")
def get_onos_devices(self): print "Getting ONOS devices ..." cmd = command_defs['get_onos_devices'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) if out is not None: onos_devices = json.loads(out) print "Got ONOS devices" else: onos_devices = None print "Unable to get ONOS devices" return onos_devices
def test_10_scale_voltha(self): print "Test_10_scale_voltha_Start:------------------" t0 = time.time() try: # start all containers print "Start all containers..." self._start_all_containers() # Instead of using only a fixed timeout: # 1) wait until the services are ready (polling per second) # 2) bail out after a longer timeout. print "Waiting for all containers to be ready ..." self.wait_till('Not all services are up', self._is_voltha_ensemble_ready, interval=1, timeout=30) # Scale voltha to 10 instances print "Scale voltha to 10 instances ..." cmd = command_defs['docker_compose_scale_voltha_to_10'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # Verify that 10 instances are running print "Verify 10 instances of voltha are running ..." cmd = command_defs['docker_compose_scaled_voltha_ps'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertEqual(out.split(), ['10']) finally: print "Stopping all containers ..." # clean up all created containers for this test self._stop_and_remove_all_containers() print "Test_10_scale_voltha_End:------------------took {} secs " \ "\n\n".format(time.time() - t0)
def start_all_containers(self): t0 = time() # start all the containers self.pt("Starting all containers ...") cmd = get_command('docker_compose_start_all') out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.pt("Waiting for voltha container to be ready ...") self.wait_till('voltha services HEALTHY', lambda: orch.verify_all_services_healthy( service_name=envoy_svc_name[orch_env]) == True, timeout=10) sleep(10)
def start_all_containers(self): t0 = time() # start all the containers self.pt("Starting all containers ...") cmd = command_defs['docker_compose_start_all'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.pt("Waiting for voltha container to be ready ...") self.wait_till('voltha services HEALTHY', lambda: verify_all_services_healthy( LOCAL_CONSUL, service_name='vcore-grpc') == True, timeout=10) sleep(10)
def test_01_setup(self): print "Test_01_setup_Start:------------------" t0 = time.time() try: # remove the venv-linux directory print "Remove venv-linux ..." cmd = command_defs['remove_env_directory'] rm_venv, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # make clean print "Make clean ..." cmd = command_defs['make_clean'] mk_clean, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # source the env print "Source environment ..." self._source_env() finally: print "Test_01_setup_End:------------------ took {} " \ "secs\n\n".format(time.time() - t0)
def test_02_make_fetch(self): print "Test_02_make_fetch_Start:------------------" t0 = time.time() try: # Get list of images to fetch from the Makefile print "Get list of images to fetch ..." cmd = command_defs['makefile_fetch_images'] makefile_images_to_fetch, err, rc \ = run_command_to_completion_with_stdout_in_list(cmd) self.assertEqual(rc, 0) images_to_fetch = [] for image in makefile_images_to_fetch: tmp = ''.join(image.split()) images_to_fetch.append(tmp[len('dockerpull'):]) # make fetch print "Fetching images {} ...".format(images_to_fetch) cmd = command_defs['make_fetch'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # verify that the images have been downloaded print "Verify images downloaded and present locally ..." cmd = command_defs['docker_images'] local_images, err, rc = \ run_command_to_completion_with_stdout_in_list(cmd) self.assertEqual(rc, 0) local_images_list = [] for local_image in local_images: words = local_image.split() local_images_list.append('{}:{}'.format(words[0], words[1])) intersection_list = [ i for i in images_to_fetch if i in local_images_list ] assert len(intersection_list) == len(images_to_fetch) finally: print "Test_02_make_fetch_End:------------------ took {} " \ "secs \n\n".format(time.time() - t0)
def _wait_for_consul_to_be_ready(self): # Consul is ready when it's leader ip and port is set. The maximum # time to wait of 60 secs as consul should be ready by then max_wait_time = 60 t0 = time.time() while True: # Get the docker IP address and port number of the consul instance print "waiting for consul to be ready ..." cmd = command_defs['consul_get_leader_ip_port'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) out = out.strip() if rc != 0: # Something is wrong, return return -1 # error elif out is not None and out != '': return 0 # found something elif time.time() - t0 > max_wait_time: return -1 # consul should have come up by this time else: time.sleep(2) # constant sleep for testing
def test_ofagent_controller_failover(self): olt_device = self.add_device() print "Output of ADD OLT is {} {} {}".format(olt_device, type(olt_device), olt_device['id']) sleep(5) self.enable_device(olt_device['id']) print "Waiting for OLT device to be activated ..." sleep(80) cmd = command_defs['onos1_devices'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) onos1_devices = json.loads(out) onos1_role = onos1_devices['devices'][0]['role'] print "Role of ONOS1 is {}".format(onos1_role) cmd = command_defs['onos2_devices'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) onos2_devices = json.loads(out) onos2_role = onos2_devices['devices'][0]['role'] print "Role of ONOS2 is {}".format(onos2_role) cmd = command_defs['onos3_devices'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) onos3_devices = json.loads(out) onos3_role = onos3_devices['devices'][0]['role'] print "Role of ONOS3 is {}".format(onos3_role) if onos1_role == "MASTER": cmd = command_defs['docker_stop']+ ' onos1' out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) print "Waiting for ONOS to Elect New Master" sleep(20) cmd = command_defs['onos2_devices'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) onos2_devices = json.loads(out) onos2_role = onos2_devices['devices'][0]['role'] print "Role of ONOS2 is {}".format(onos2_role) cmd = command_defs['onos3_devices'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) onos3_devices = json.loads(out) onos3_role = onos3_devices['devices'][0]['role'] print "Role of ONOS3 is {}".format(onos3_role) assert (onos3_role == "MASTER" or onos2_role == "MASTER"), "Exception,New Master Election Failed" elif onos2_role == "MASTER": cmd = command_defs['docker_stop']+ ' onos2' out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) print "Waiting for ONOS to Elect New Master" sleep(20) cmd = command_defs['onos1_devices'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) onos1_devices = json.loads(out) onos1_role = onos1_devices['devices'][0]['role'] print "Role of ONOS1 is {}".format(onos1_role) cmd = command_defs['onos3_devices'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) onos3_devices = json.loads(out) onos3_role = onos3_devices['devices'][0]['role'] print "Role of ONOS3 is {}".format(onos3_role) assert (onos3_role == "MASTER" or onos1_role == "MASTER"), "Exception,New Master Election Failed" elif onos3_role == "MASTER": cmd = command_defs['docker_stop']+ ' onos3' out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) print "Waiting for ONOS to Elect New Master" sleep(20) cmd = command_defs['onos1_devices'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) onos1_devices = json.loads(out) onos1_role = onos1_devices['devices'][0]['role'] print "Role of ONOS1 is {}".format(onos1_role) cmd = command_defs['onos2_devices'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) onos2_devices = json.loads(out) onos2_role = onos2_devices['devices'][0]['role'] print "Role of ONOS2 is {}".format(onos2_role) assert (onos1_role == "MASTER" or onos2_role == "MASTER"), "Exception,New Master Election Failed"
def _scale_voltha(self, scale=2): self.pt("Scaling voltha ...") cmd = get_command('docker_compose_scale_voltha') + str(scale) out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0)
def test_07_start_all_containers(self): print "Test_07_start_all_containers_Start:------------------ " t0 = time.time() try: # Pre-test - clean up all running docker containers print "Pre-test: Removing all running containers ..." cmd = command_defs['docker_compose_stop'] _, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) cmd = command_defs['docker_compose_rm_f'] _, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # get a list of services in the docker-compose file print "Getting list of services in docker compose file ..." cmd = command_defs['docker_compose_services'] services, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) docker_service_list = services.split() self.assertGreaterEqual(len(docker_service_list), DOCKER_COMPOSE_FILE_SERVICES_COUNT) # start all the containers print "Starting all containers ..." cmd = command_defs['docker_compose_start_all'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # Instead of using only a fixed timeout: # 1) wait until the services are ready (polling per second) # 2) bail out after a longer timeout. print "Waiting for all containers to be ready ..." self.wait_till('Not all services are up', self._is_voltha_ensemble_ready, interval=1, timeout=30) # verify that all containers are running print "Verify all services are running using docker command ..." for service in docker_service_list: cmd = command_defs['docker_compose_ps'] + ' {} | wc -l'.format( service) out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertGreaterEqual(out, 3) # 2 are for headers # Verify that 'docker ps' return the same number of running process cmd = command_defs['docker_ps_count'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertGreaterEqual(out, (len(docker_service_list))) # Retrieve the list of services from consul and validate against # the list obtained from docker composed print "Verify all services are registered in consul ..." expected_services = [ 'consul-rest', 'fluentd-intake', 'voltha-grpc', 'voltha-health', 'consul-8600', 'zookeeper', 'consul', 'kafka' ] cmd = command_defs['consul_get_services'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) try: consul_services = json.loads(out) intersected_services = [ s for s in expected_services if s in consul_services ] self.assertEqual(len(intersected_services), len(expected_services)) # services_match = 0 # for d_service in docker_service_list: # for c_service in consul_services: # if c_service.find(d_service) != -1: # services_match += 1 # print d_service, c_service # break # self.assertEqual(services_match, len(docker_service_list)) except Exception as e: self.assertRaises(e) # Verify the service record of the voltha service print "Verify the service record of voltha in consul ..." expected_srv_elements = [ 'ModifyIndex', 'CreateIndex', 'ServiceEnableTagOverride', 'Node', 'Address', 'TaggedAddresses', 'ServiceID', 'ServiceName', 'ServiceTags', 'ServiceAddress', 'ServicePort' ] cmd = command_defs['consul_get_srv_voltha_health'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) try: srv = json.loads(out) intersect_elems = [ e for e in srv[0] if e in expected_srv_elements ] self.assertEqual(len(expected_srv_elements), len(intersect_elems)) except Exception as e: self.assertRaises(e) # Verify kafka client is receiving the messages print "Verify kafka client has heartbeat topic ..." expected_pattern = ['voltha.heartbeat'] kafka_endpoint = get_endpoint_from_consul(LOCAL_CONSUL, 'kafka') cmd = command_defs['kafka_client_run'].format(kafka_endpoint) kafka_client_output = run_long_running_command_with_timeout( cmd, 20) # Verify the kafka client output # instance id found = False for out in kafka_client_output: if all(ep for ep in expected_pattern if ep in out): found = True break self.assertTrue(found) # Commented the heartbeat messages from voltha as on Jenkins this # test fails more often than not. On local or cluster environment # the kafka event bus works well. # verify docker-compose logs are being produced - just get the # first work of each line print "Verify docker compose logs has output from all the services " \ "..." expected_output = [ 'voltha_1', 'fluentd_1', 'vconsul_1', 'registrator_1', 'kafka_1', 'zookeeper_1', 'ofagent_1', 'netconf_1' ] cmd = command_defs['docker_compose_logs'] docker_compose_logs = run_long_running_command_with_timeout( cmd, 5, 0) intersected_logs = [ l for l in expected_output if l in docker_compose_logs ] self.assertEqual(len(intersected_logs), len(expected_output)) # verify docker voltha logs are being produced - we will just verify # some # key messages in the logs print "Verify docker voltha logs are produced ..." self.wait_till('Basic voltha logs are absent', self._is_basic_voltha_logs_produced, interval=1, timeout=30) finally: print "Stopping all containers ..." # clean up all created containers for this test #self._stop_and_remove_all_containers() cmd = command_defs['docker_compose_down'] _, err, rc = run_command_to_completion_with_raw_stdout(cmd) print "Test_07_start_all_containers_End:------------------ took {}" \ " secs \n\n".format(time.time() - t0)
def test_06_run_voltha_standalone_with_consul_only(self): print "Test_06_run_voltha_standalone_with_consul_only_Start:----------" \ "-------- " t0 = time.time() try: # run consul first print "Start consul ..." self._run_consul() # get consul ip print "Get consul IP ..." cmd = command_defs['docker_get_consul_ip'] consul_ip, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertIsNotNone(consul_ip) # start voltha now for 15 secs and verify it can now connect to # consul - following message in the output print "Start voltha with consul IP ..." expected_pattern = ['coordinator', 'event: created-consul-session'] cmd = command_defs['docker_start_voltha_with_consul_ip'] + \ '{}:8500'.format(consul_ip.strip()) command_output = run_long_running_command_with_timeout(cmd, 10) # Verify the output of voltha and get the container instance id print "Verify voltha is registered with consul ..." instance_id = None for out in command_output: if all(ep for ep in expected_pattern if ep in out): self.assertTrue(True) instance_id = re.findall(r'[0-9a-f]+', out)[-1] break self.assertIsNotNone(instance_id) # Verify Voltha's self-registration with consul expected_output = [ 'ModifyIndex', 'CreateIndex', 'Session', 'Value', 'Flags', 'Key', 'LockIndex' ] cmd = command_defs['consul_verify_voltha_registration'] registration_info, err, rc = \ run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) try: jr_info = json.loads(registration_info) intersect_elems = [ e for e in jr_info[0] if e in expected_output ] self.assertEqual(len(expected_output), len(intersect_elems)) except Exception as e: self.assertRaises(e) # stop voltha print "Stop voltha ..." self._stop_docker_container_by_id(instance_id) # check the service has deregistered print "Verify voltha is no longer registered in consul..." cmd = command_defs['consul_verify_voltha_registration'] registration_info, err, rc = \ run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertEqual(registration_info, '') finally: # clean up all created containers for this test print "Stop consul ..." self._stop_and_remove_all_containers() print "Test_06_run_voltha_standalone_with_consul_only_End:--------" \ "---------- took {} " \ "secs \n\n".format(time.time() - t0)
def _start_all_containers(self): cmd = command_defs['docker_compose_start_all'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0)