def _is_basic_voltha_logs_produced(self): expected_output = ['coordinator._renew_session', 'main.heartbeat'] cmd = command_defs['docker_voltha_logs'] docker_voltha_logs = run_long_running_command_with_timeout(cmd, 10, 5) intersected_logs = [ l for l in expected_output if l in docker_voltha_logs ] return len(intersected_logs) == len(expected_output)
def test_04_run_voltha_standalone_without_consul(self): print "Test_04_run_voltha_standalone_without_consul_Start:------------" \ "------" t0 = time.time() try: # Run voltha for 10 secs and verity the following lines are displayed # (a subset of output messages along with a flag when found) print "Start voltha ..." expected_output_subset = [ 'main.print_banner {event: (to stop: press Ctrl-C), ' 'instance_id:', 'coordinator.__init__ {event: initializing-coordinator,', 'grpc_server.start {event: started', 'main.<lambda> {event: twisted-reactor-started', 'main.startup_components {event: started-internal-services,', 'kafka_proxy.start {event: started,', 'coordinator._backoff {retry_in: 5, event: consul-not-up,' ] cmd = command_defs['docker_start_voltha'] command_output = run_long_running_command_with_timeout(cmd, 10) # There should at least be 1 line in the output self.assertGreater(len(command_output), 0) # Verify that the output contained the expected_output_subset - # save the docker instance id print "Verify voltha started correctly ..." instance_id = None for ext_output in expected_output_subset: match_str = next( (out for out in command_output if ext_output in out), None) self.assertIsNotNone(match_str) if "instance_id" in ext_output: instance_id = re.findall(r'[0-9a-f]+', match_str)[-1] # Now stop the voltha docker that was created print "Stop voltha ..." self._stop_docker_container_by_id(instance_id) finally: # Remove any created container self._stop_and_remove_all_containers() print "Test_04_run_voltha_standalone_without_consul_End" \ ":------------------ took {} secs \n\n".format( time.time() - t0)
def test_04_run_voltha_standalone_without_consul(self): print "Test_04_run_voltha_standalone_without_consul_Start:------------" \ "------" t0 = time.time() try: # Run voltha for 10 secs and verity the following lines are displayed # (a subset of output messages along with a flag when found) print "Start voltha ..." expected_output_subset = [ 'main.print_banner {event: (to stop: press Ctrl-C), ' 'instance_id:', 'coordinator.__init__ {event: initializing-coordinator,', 'grpc_server.start {event: started', 'main.<lambda> {event: twisted-reactor-started', 'main.startup_components {event: started-internal-services,', 'kafka_proxy.start {event: started,', 'coordinator._backoff {retry_in: 5, event: consul-not-up,' ] cmd = command_defs['docker_start_voltha'] command_output = run_long_running_command_with_timeout(cmd, 10) # There should at least be 1 line in the output self.assertGreater(len(command_output), 0) # Verify that the output contained the expected_output_subset - # save the docker instance id print "Verify voltha started correctly ..." instance_id = None for ext_output in expected_output_subset: match_str = next( (out for out in command_output if ext_output in out), None) self.assertIsNotNone(match_str) if "instance_id" in ext_output: instance_id = re.findall(r'[0-9a-f]+', match_str)[-1] # Now stop the voltha docker that was created print "Stop voltha ..." self._stop_docker_container_by_id(instance_id) finally: # Remove any created container self._stop_and_remove_all_containers() print "Test_04_run_voltha_standalone_without_consul_End" \ ":------------------ took {} secs \n\n".format( time.time() - t0)
def test_07_start_all_containers(self): print "Test_07_start_all_containers_Start:------------------ " t0 = time.time() try: # Pre-test - clean up all running docker containers print "Pre-test: Removing all running containers ..." cmd = command_defs['docker_compose_stop'] _, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) cmd = command_defs['docker_compose_rm_f'] _, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # get a list of services in the docker-compose file print "Getting list of services in docker compose file ..." cmd = command_defs['docker_compose_services'] services, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) docker_service_list = services.split() self.assertGreaterEqual(len(docker_service_list), DOCKER_COMPOSE_FILE_SERVICES_COUNT) # start all the containers print "Starting all containers ..." cmd = command_defs['docker_compose_start_all'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # Instead of using only a fixed timeout: # 1) wait until the services are ready (polling per second) # 2) bail out after a longer timeout. print "Waiting for all containers to be ready ..." self.wait_till('Not all services are up', self._is_voltha_ensemble_ready, interval=1, timeout=30) # verify that all containers are running print "Verify all services are running using docker command ..." for service in docker_service_list: cmd = command_defs['docker_compose_ps'] + ' {} | wc -l'.format( service) out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertGreaterEqual(out, 3) # 2 are for headers # Verify that 'docker ps' return the same number of running process cmd = command_defs['docker_ps_count'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertGreaterEqual(out.split(), [str(len(docker_service_list))]) # Retrieve the list of services from consul and validate against # the list obtained from docker composed print "Verify all services are registered in consul ..." expected_services = [ 'consul-rest', 'fluentd-intake', 'voltha-grpc', 'voltha-health', 'consul-8600', 'zookeeper', 'consul', 'kafka' ] cmd = command_defs['consul_get_services'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) try: consul_services = json.loads(out) intersected_services = [ s for s in expected_services if s in consul_services ] self.assertEqual(len(intersected_services), len(expected_services)) # services_match = 0 # for d_service in docker_service_list: # for c_service in consul_services: # if c_service.find(d_service) != -1: # services_match += 1 # print d_service, c_service # break # self.assertEqual(services_match, len(docker_service_list)) except Exception as e: self.assertRaises(e) # Verify the service record of the voltha service print "Verify the service record of voltha in consul ..." expected_srv_elements = [ 'ModifyIndex', 'CreateIndex', 'ServiceEnableTagOverride', 'Node', 'Address', 'TaggedAddresses', 'ServiceID', 'ServiceName', 'ServiceTags', 'ServiceAddress', 'ServicePort' ] cmd = command_defs['consul_get_srv_voltha_health'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) try: srv = json.loads(out) intersect_elems = [ e for e in srv[0] if e in expected_srv_elements ] self.assertEqual(len(expected_srv_elements), len(intersect_elems)) except Exception as e: self.assertRaises(e) # Verify kafka client is receiving the messages print "Verify kafka client has heartbeat topic ..." expected_pattern = ['voltha.heartbeat'] kafka_endpoint = get_endpoint_from_consul(LOCAL_CONSUL, 'kafka') cmd = command_defs['kafka_client_run'].format(kafka_endpoint) kafka_client_output = run_long_running_command_with_timeout( cmd, 20) # Verify the kafka client output # instance id found = False for out in kafka_client_output: if all(ep for ep in expected_pattern if ep in out): found = True break self.assertTrue(found) # Commented the heartbeat messages from voltha as on Jenkins this # test fails more often than not. On local or cluster environment # the kafka event bus works well. # verify docker-compose logs are being produced - just get the # first work of each line print "Verify docker compose logs has output from all the services " \ "..." expected_output = [ 'voltha_1', 'fluentd_1', 'consul_1', 'registrator_1', 'kafka_1', 'zookeeper_1', 'ofagent_1', 'netconf_1' ] cmd = command_defs['docker_compose_logs'] docker_compose_logs = run_long_running_command_with_timeout( cmd, 5, 0) intersected_logs = [ l for l in expected_output if l in docker_compose_logs ] self.assertEqual(len(intersected_logs), len(expected_output)) # verify docker voltha logs are being produced - we will just verify # some # key messages in the logs print "Verify docker voltha logs are produced ..." self.wait_till('Basic voltha logs are absent', self._is_basic_voltha_logs_produced, interval=1, timeout=30) finally: print "Stopping all containers ..." # clean up all created containers for this test #self._stop_and_remove_all_containers() cmd = command_defs['docker_compose_down'] _, err, rc = run_command_to_completion_with_raw_stdout(cmd) print "Test_07_start_all_containers_End:------------------ took {}" \ " secs \n\n".format(time.time() - t0)
def test_06_run_voltha_standalone_with_consul_only(self): print "Test_06_run_voltha_standalone_with_consul_only_Start:----------" \ "-------- " t0 = time.time() try: # run consul first print "Start consul ..." self._run_consul() # get consul ip print "Get consul IP ..." cmd = command_defs['docker_get_consul_ip'] consul_ip, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertIsNotNone(consul_ip) # start voltha now for 15 secs and verify it can now connect to # consul - following message in the output print "Start voltha with consul IP ..." expected_pattern = ['coordinator', 'event: created-consul-session'] cmd = command_defs['docker_start_voltha_with_consul_ip'] + \ '{}:8500'.format(consul_ip.strip()) command_output = run_long_running_command_with_timeout(cmd, 10) # Verify the output of voltha and get the container instance id print "Verify voltha is registered with consul ..." instance_id = None for out in command_output: if all(ep for ep in expected_pattern if ep in out): self.assertTrue(True) instance_id = re.findall(r'[0-9a-f]+', out)[-1] break self.assertIsNotNone(instance_id) # Verify Voltha's self-registration with consul expected_output = [ 'ModifyIndex', 'CreateIndex', 'Session', 'Value', 'Flags', 'Key', 'LockIndex' ] cmd = command_defs['consul_verify_voltha_registration'] registration_info, err, rc = \ run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) try: jr_info = json.loads(registration_info) intersect_elems = [ e for e in jr_info[0] if e in expected_output ] self.assertEqual(len(expected_output), len(intersect_elems)) except Exception as e: self.assertRaises(e) # stop voltha print "Stop voltha ..." self._stop_docker_container_by_id(instance_id) # check the service has deregistered print "Verify voltha is no longer registered in consul..." cmd = command_defs['consul_verify_voltha_registration'] registration_info, err, rc = \ run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertEqual(registration_info, '') finally: # clean up all created containers for this test print "Stop consul ..." self._stop_and_remove_all_containers() print "Test_06_run_voltha_standalone_with_consul_only_End:--------" \ "---------- took {} " \ "secs \n\n".format(time.time() - t0)
def test_07_start_all_containers(self): print "Test_07_start_all_containers_Start:------------------ " t0 = time.time() try: # Pre-test - clean up all running docker containers print "Pre-test: Removing all running containers ..." self._stop_and_remove_all_containers() # get a list of services in the docker-compose file print "Getting list of services in docker compose file ..." cmd = command_defs['docker_compose_services'] services, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) docker_service_list = services.split() self.assertGreaterEqual(len(docker_service_list), DOCKER_COMPOSE_FILE_SERVICES_COUNT) # start all the containers print "Starting all containers ..." cmd = command_defs['docker_compose_start_all'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) print "Waiting for all containers to be ready ..." time.sleep(10) rc = verify_all_services_healthy(LOCAL_CONSUL) if not rc: print "Not all services are up" self.assertEqual(rc, True) # verify that all containers are running print "Verify all services are running using docker command ..." for service in docker_service_list: cmd = command_defs['docker_compose_ps'] + ' {} | wc -l'.format( service) out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertGreaterEqual(out, 3) # 2 are for headers # Verify that 'docker ps' return the same number of running process cmd = command_defs['docker_ps_count'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertGreaterEqual(out.split(), [str(len(docker_service_list))]) # Retrieve the list of services from consul and validate against # the list obtained from docker composed print "Verify all services are registered in consul ..." expected_services = [ 'consul-rest', 'fluentd-intake', 'chameleon-rest', 'voltha-grpc', 'voltha-health', 'consul-8600', 'zookeeper', 'consul', 'kafka' ] cmd = command_defs['consul_get_services'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) try: consul_services = json.loads(out) intersected_services = [ s for s in expected_services if s in consul_services ] self.assertEqual(len(intersected_services), len(expected_services)) # services_match = 0 # for d_service in docker_service_list: # for c_service in consul_services: # if c_service.find(d_service) != -1: # services_match += 1 # print d_service, c_service # break # self.assertEqual(services_match, len(docker_service_list)) except Exception as e: self.assertRaises(e) # Verify the service record of the voltha service print "Verify the service record of voltha in consul ..." expected_srv_elements = [ 'ModifyIndex', 'CreateIndex', 'ServiceEnableTagOverride', 'Node', 'Address', 'TaggedAddresses', 'ServiceID', 'ServiceName', 'ServiceTags', 'ServiceAddress', 'ServicePort' ] cmd = command_defs['consul_get_srv_voltha_health'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) try: srv = json.loads(out) intersect_elems = [ e for e in srv[0] if e in expected_srv_elements ] self.assertEqual(len(expected_srv_elements), len(intersect_elems)) except Exception as e: self.assertRaises(e) # Verify kafka client is receiving the messages print "Verify kafka client has heartbeat topic ..." expected_pattern = ['voltha.heartbeat'] kafka_endpoint = get_endpoint_from_consul(LOCAL_CONSUL, 'kafka') cmd = command_defs['kafka_client_run'].format(kafka_endpoint) kafka_client_output = run_long_running_command_with_timeout( cmd, 20) # TODO check that there are heartbeats # Verify the kafka client output # instance id found = False for out in kafka_client_output: if all(ep for ep in expected_pattern if ep in out): found = True break self.assertTrue(found) print "Verify kafka client is receiving the heartbeat messages from voltha..." expected_pattern = ['heartbeat', 'compose_voltha_1'] cmd = command_defs['kafka_client_heart_check'].format( kafka_endpoint) kafka_client_output = run_long_running_command_with_timeout( cmd, 20) print kafka_client_output # TODO check that there are heartbeats # Verify the kafka client output # instance id found = False for out in kafka_client_output: if all(ep for ep in expected_pattern if ep in out): found = True break self.assertTrue(found) # verify docker-compose logs are being produced - just get the # first work of each line print "Verify docker compose logs has output from all the services " \ "..." expected_output = [ 'voltha_1', 'fluentd_1', 'consul_1', 'registrator_1', 'kafka_1', 'zookeeper_1', 'chameleon_1', 'ofagent_1', 'netconf_1' ] cmd = command_defs['docker_compose_logs'] docker_compose_logs = run_long_running_command_with_timeout( cmd, 5, 0) intersected_logs = [ l for l in expected_output if l in docker_compose_logs ] self.assertEqual(len(intersected_logs), len(expected_output)) # TODO: file in /tmp/fluentd/ cannot be found # # verify fluentd logs are being produced - we will just verify # that there are "voltha.logging" in the logs # os.environ["PYTHONPATH"] += os.pathsep + "/tmp/fluentd/" # os.environ['PATH'] += os.pathsep + "/tmp/fluentd/" # expected_output=['voltha.logging'] # cmd = command_defs['fluentd_logs'] # fluentd_logs, err = run_command_to_completion_with_raw_stdout(cmd) # # self.assertIsNone(err) # print err # intersected_logs = [l for l in expected_output if # l in fluentd_logs] # self.assertEqual(len(intersected_logs), len(expected_output)) # verify docker voltha logs are being produced - we will just verify # some # key messages in the logs print "Verify docker voltha logs are produced ..." expected_output = [ 'kafka_proxy.send_message', 'coordinator._renew_session', 'main.heartbeat' ] cmd = command_defs['docker_voltha_logs'] docker_voltha_logs = run_long_running_command_with_timeout( cmd, 0.5, 3) intersected_logs = [ l for l in expected_output if l in docker_voltha_logs ] self.assertEqual(len(intersected_logs), len(expected_output)) finally: print "Stopping all containers ..." # clean up all created containers for this test self._stop_and_remove_all_containers() print "Test_07_start_all_containers_End:------------------ took {}" \ " secs \n\n".format(time.time() - t0)
def test_07_start_all_containers(self): print "Test_07_start_all_containers_Start:------------------ " t0 = time.time() try: # Pre-test - clean up all running docker containers print "Pre-test: Removing all running containers ..." cmd = command_defs['docker_compose_stop'] _, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) cmd = command_defs['docker_compose_rm_f'] _, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # get a list of services in the docker-compose file print "Getting list of services in docker compose file ..." cmd = command_defs['docker_compose_services'] services, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) docker_service_list = services.split() self.assertGreaterEqual(len(docker_service_list), DOCKER_COMPOSE_FILE_SERVICES_COUNT) # start all the containers print "Starting all containers ..." cmd = command_defs['docker_compose_start_all'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) # Instead of using only a fixed timeout: # 1) wait until the services are ready (polling per second) # 2) bail out after a longer timeout. print "Waiting for all containers to be ready ..." self.wait_till('Not all services are up', self._is_voltha_ensemble_ready, interval=1, timeout=30) # verify that all containers are running print "Verify all services are running using docker command ..." for service in docker_service_list: cmd = command_defs['docker_compose_ps'] + ' {} | wc -l'.format( service) out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertGreaterEqual(out, 3) # 2 are for headers # Verify that 'docker ps' return the same number of running process cmd = command_defs['docker_ps_count'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertGreaterEqual(out.split(), [str(len( docker_service_list))]) # Retrieve the list of services from consul and validate against # the list obtained from docker composed print "Verify all services are registered in consul ..." expected_services = ['consul-rest', 'fluentd-intake', 'voltha-grpc', 'voltha-health', 'consul-8600', 'zookeeper', 'consul', 'kafka'] cmd = command_defs['consul_get_services'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) try: consul_services = json.loads(out) intersected_services = [s for s in expected_services if s in consul_services] self.assertEqual(len(intersected_services), len(expected_services)) # services_match = 0 # for d_service in docker_service_list: # for c_service in consul_services: # if c_service.find(d_service) != -1: # services_match += 1 # print d_service, c_service # break # self.assertEqual(services_match, len(docker_service_list)) except Exception as e: self.assertRaises(e) # Verify the service record of the voltha service print "Verify the service record of voltha in consul ..." expected_srv_elements = ['ModifyIndex', 'CreateIndex', 'ServiceEnableTagOverride', 'Node', 'Address', 'TaggedAddresses', 'ServiceID', 'ServiceName', 'ServiceTags', 'ServiceAddress', 'ServicePort'] cmd = command_defs['consul_get_srv_voltha_health'] out, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) try: srv = json.loads(out) intersect_elems = [e for e in srv[0] if e in expected_srv_elements] self.assertEqual(len(expected_srv_elements), len(intersect_elems)) except Exception as e: self.assertRaises(e) # Verify kafka client is receiving the messages print "Verify kafka client has heartbeat topic ..." expected_pattern = ['voltha.heartbeat'] kafka_endpoint = get_endpoint_from_consul(LOCAL_CONSUL,'kafka') cmd = command_defs['kafka_client_run'].format(kafka_endpoint) kafka_client_output = run_long_running_command_with_timeout(cmd, 20) # Verify the kafka client output # instance id found = False for out in kafka_client_output: if all(ep for ep in expected_pattern if ep in out): found = True break self.assertTrue(found) # Commented the heartbeat messages from voltha as on Jenkins this # test fails more often than not. On local or cluster environment # the kafka event bus works well. # verify docker-compose logs are being produced - just get the # first work of each line print "Verify docker compose logs has output from all the services " \ "..." expected_output = ['voltha_1', 'fluentd_1', 'consul_1', 'registrator_1', 'kafka_1', 'zookeeper_1', 'ofagent_1', 'netconf_1'] cmd = command_defs['docker_compose_logs'] docker_compose_logs = run_long_running_command_with_timeout(cmd, 5, 0) intersected_logs = [l for l in expected_output if l in docker_compose_logs] self.assertEqual(len(intersected_logs), len(expected_output)) # verify docker voltha logs are being produced - we will just verify # some # key messages in the logs print "Verify docker voltha logs are produced ..." expected_output = ['coordinator._renew_session', 'main.heartbeat'] cmd = command_defs['docker_voltha_logs'] docker_voltha_logs = run_long_running_command_with_timeout(cmd, 0.5, 5) intersected_logs = [l for l in expected_output if l in docker_voltha_logs] self.assertEqual(len(intersected_logs), len(expected_output)) finally: print "Stopping all containers ..." # clean up all created containers for this test #self._stop_and_remove_all_containers() cmd = command_defs['docker_compose_down'] _, err, rc = run_command_to_completion_with_raw_stdout(cmd) print "Test_07_start_all_containers_End:------------------ took {}" \ " secs \n\n".format(time.time() - t0)
def test_06_run_voltha_standalone_with_consul_only(self): print "Test_06_run_voltha_standalone_with_consul_only_Start:----------" \ "-------- " t0 = time.time() try: # run consul first print "Start consul ..." self._run_consul() # get consul ip print "Get consul IP ..." cmd = command_defs['docker_get_consul_ip'] consul_ip, err, rc = run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertIsNotNone(consul_ip) # start voltha now for 15 secs and verify it can now connect to # consul - following message in the output print "Start voltha with consul IP ..." expected_pattern = ['coordinator', 'event: created-consul-session'] cmd = command_defs['docker_start_voltha_with_consul_ip'] + \ '{}:8500'.format(consul_ip.strip()) command_output = run_long_running_command_with_timeout(cmd, 10) # Verify the output of voltha and get the container instance id print "Verify voltha is registered with consul ..." instance_id = None for out in command_output: if all(ep for ep in expected_pattern if ep in out): self.assertTrue(True) instance_id = re.findall(r'[0-9a-f]+', out)[-1] break self.assertIsNotNone(instance_id) # Verify Voltha's self-registration with consul expected_output = ['ModifyIndex', 'CreateIndex', 'Session', 'Value', 'Flags', 'Key', 'LockIndex'] cmd = command_defs['consul_verify_voltha_registration'] registration_info, err, rc = \ run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) try: jr_info = json.loads(registration_info) intersect_elems = [e for e in jr_info[0] if e in expected_output] self.assertEqual(len(expected_output), len(intersect_elems)) except Exception as e: self.assertRaises(e) # stop voltha print "Stop voltha ..." self._stop_docker_container_by_id(instance_id) # check the service has deregistered print "Verify voltha is no longer registered in consul..." cmd = command_defs['consul_verify_voltha_registration'] registration_info, err, rc = \ run_command_to_completion_with_raw_stdout(cmd) self.assertEqual(rc, 0) self.assertEqual(registration_info, '') finally: # clean up all created containers for this test print "Stop consul ..." self._stop_and_remove_all_containers() print "Test_06_run_voltha_standalone_with_consul_only_End:--------" \ "---------- took {} " \ "secs \n\n".format(time.time() - t0)