def test_traffic_with_control_node_switchover(self): ''' Stop the control node and check peering with agent fallback to other control node. ''' if len(set(self.inputs.bgp_ips)) < 2: raise self.skipTest( "Skipping Test. At least 2 control node required to run the test" ) result = True fip_pool_name = get_random_name('some-pool1') (self.vn1_name, self.vn1_subnets) = (get_random_name("vn1"), ["11.1.1.0/24"]) (self.vn2_name, self.vn2_subnets) = (get_random_name("vn2"), ["22.1.1.0/24"]) (self.fvn_public_name, self.fvn_public_subnets) = (get_random_name("fip_vn_public"), ['10.204.219.16/28']) (self.fvn1_name, self.fvn1_subnets) = (get_random_name("fip_vn1"), ['100.1.1.0/24']) (self.fvn2_name, self.fvn2_subnets) = (get_random_name("fip_vn2"), ['200.1.1.0/24']) (self.fvn3_name, self.fvn3_subnets) = (get_random_name("fip_vn3"), ['170.1.1.0/29']) (self.vn1_vm1_name, self.vn1_vm2_name) = (get_random_name('vn1_vm1'), get_random_name('vn1_vm2')) (self.vn2_vm1_name, self.vn2_vm2_name) = (get_random_name('vn2_vm1'), get_random_name('vn2_vm2')) (self.fvn_public_vm1_name) = (get_random_name('fvn_public_vm1')) (self.fvn1_vm1_name) = (get_random_name('fvn1_vm1')) (self.fvn2_vm1_name) = (get_random_name('fvn2_vm1')) (self.fvn3_vm1_name) = (get_random_name('fvn3_vm1')) (self.vn1_vm1_traffic_name) = get_random_name('VN1_VM1_traffic') (self.fvn1_vm1_traffic_name) = get_random_name('FVN1_VM1_traffic') fip_pool_name1 = get_random_name('some-pool1') fip_pool_name2 = get_random_name('some-pool2') # Get all compute host host_list = self.connections.nova_h.get_hosts() compute_1 = host_list[0] compute_2 = host_list[0] if len(host_list) > 1: compute_1 = host_list[0] compute_2 = host_list[1] self.fvn1_fixture = self.useFixture( VNFixture(project_name=self.inputs.project_name, connections=self.connections, inputs=self.inputs, vn_name=self.fvn1_name, subnets=self.fvn1_subnets)) self.vn1_fixture = self.useFixture( VNFixture(project_name=self.inputs.project_name, connections=self.connections, inputs=self.inputs, vn_name=self.vn1_name, subnets=self.vn1_subnets)) self.fvn1_vm1_traffic_fixture = self.useFixture( VMFixture(project_name=self.inputs.project_name, connections=self.connections, vn_obj=self.fvn1_fixture.obj, flavor='contrail_flavor_small', image_name='ubuntu-traffic', vm_name=self.fvn1_vm1_traffic_name, node_name=compute_2)) self.vn1_vm1_traffic_fixture = self.useFixture( VMFixture(project_name=self.inputs.project_name, connections=self.connections, vn_obj=self.vn1_fixture.obj, flavor='contrail_flavor_small', image_name='ubuntu-traffic', vm_name=self.vn1_vm1_traffic_name, node_name=compute_1)) fvn1_fixture = self.fvn1_fixture fvn1_vm1_traffic_fixture = self.fvn1_vm1_traffic_fixture fvn1_subnets = self.fvn1_subnets fvn1_vm1_traffic_name = self.fvn1_vm1_traffic_name vn1_fixture = self.vn1_fixture vn1_vm1_traffic_fixture = self.vn1_vm1_traffic_fixture vn1_subnets = self.vn1_subnets vn1_vm1_traffic_name = self.vn1_vm1_traffic_name assert fvn1_fixture.verify_on_setup() assert vn1_fixture.verify_on_setup() assert fvn1_vm1_traffic_fixture.verify_on_setup() assert vn1_vm1_traffic_fixture.verify_on_setup() fip_fixture1 = self.useFixture( FloatingIPFixture(project_name=self.inputs.project_name, inputs=self.inputs, connections=self.connections, pool_name=fip_pool_name1, vn_id=fvn1_fixture.vn_id)) assert fip_fixture1.verify_on_setup() fip_id1 = fip_fixture1.create_and_assoc_fip( fvn1_fixture.vn_id, vn1_vm1_traffic_fixture.vm_id) self.addCleanup(fip_fixture1.disassoc_and_delete_fip, fip_id1) assert fip_fixture1.verify_fip(fip_id1, vn1_vm1_traffic_fixture, fvn1_fixture) if not vn1_vm1_traffic_fixture.ping_with_certainty( fvn1_vm1_traffic_fixture.vm_ip): result = result and False # Figuring the active control node active_controller = None inspect_h = self.agent_inspect[vn1_vm1_traffic_fixture.vm_node_ip] agent_xmpp_status = inspect_h.get_vna_xmpp_connection_status() for entry in agent_xmpp_status: if entry['cfg_controller'] == 'Yes': active_controller = entry['controller_ip'] self.logger.info( 'Active control node from the Agent %s is %s' % (vn1_vm1_traffic_fixture.vm_node_ip, active_controller)) fvn1_vm1_traffic_fixture.wait_till_vm_is_up() vn1_vm1_traffic_fixture.wait_till_vm_is_up() # Install traffic pkg in VM vn1_vm1_traffic_fixture.install_pkg("Traffic") fvn1_vm1_traffic_fixture.install_pkg("Traffic") # Start Traffic traffic_obj = {} startStatus = {} stopStatus = {} traffic_proto_l = ['icmp'] total_streams = {} total_streams['icmp'] = 1 dpi = 9100 proto = 'icmp' for proto in traffic_proto_l: traffic_obj[proto] = {} startStatus[proto] = {} traffic_obj[proto] = self.useFixture( traffic_tests.trafficTestFixture(self.connections)) startStatus[proto] = traffic_obj[proto].startTraffic( num_streams=total_streams[proto], start_port=dpi, tx_vm_fixture=vn1_vm1_traffic_fixture, rx_vm_fixture=fvn1_vm1_traffic_fixture, stream_proto=proto) self.logger.info( "Status of start traffic : %s, %s, %s" % (proto, vn1_vm1_traffic_fixture.vm_ip, startStatus[proto])) if startStatus[proto]['status'] != True: result = False self.logger.info("-" * 80) # Poll live traffic traffic_stats = {} self.logger.info("Poll live traffic and get status..") for proto in traffic_proto_l: traffic_stats = traffic_obj[proto].getLiveTrafficStats() err_msg = "Traffic disruption is seen: details: " #self.assertEqual(traffic_stats['status'], True, err_msg) assert (traffic_stats['status']), err_msg self.logger.info("-" * 80) # Stop on Active node self.logger.info('Stoping the Control service in %s' % (active_controller)) self.inputs.stop_service('contrail-control', [active_controller], container='control') sleep(5) # Check the control node shifted to other control node new_active_controller = None new_active_controller_state = None inspect_h = self.agent_inspect[vn1_vm1_traffic_fixture.vm_node_ip] agent_xmpp_status = inspect_h.get_vna_xmpp_connection_status() for entry in agent_xmpp_status: if entry['cfg_controller'] == 'Yes': new_active_controller = entry['controller_ip'] new_active_controller_state = entry['state'] self.logger.info( 'Active control node from the Agent %s is %s' % (vn1_vm1_traffic_fixture.vm_node_ip, new_active_controller)) if new_active_controller == active_controller: self.logger.error( 'Control node switchover fail. Old Active controlnode was %s and new active control node is %s' % (active_controller, new_active_controller)) result = False if new_active_controller_state != 'Established': self.logger.error( 'Agent does not have Established XMPP connection with Active control node' ) result = result and False # Verify Flow records here inspect_h1 = self.agent_inspect[vn1_vm1_traffic_fixture.vm_node_ip] inspect_h2 = self.agent_inspect[fvn1_vm1_traffic_fixture.vm_node_ip] flow_rec1 = None udp_src = unicode(8000) dpi = unicode(dpi) # Verify Ingress Traffic self.logger.info('Verifying Ingress Flow Record') vn_fq_name = vn1_vm1_traffic_fixture.vn_fq_name compute_node_fixture = self.useFixture( ComputeNodeFixture(self.connections, vn1_vm1_traffic_fixture.vm_node_ip)) fwd_flow, rev_flow = compute_node_fixture.get_flow_entry( source_ip=vn1_vm1_traffic_fixture.vm_ip, dest_ip=fvn1_vm1_traffic_fixture.vm_ip, dest_port='0', proto='1') if fwd_flow: sport = fwd_flow.source_port flow_rec1 = inspect_h1.get_vna_fetchflowrecord( nh=vn1_vm1_traffic_fixture.tap_intf[vn_fq_name] ['flow_key_idx'], sip=vn1_vm1_traffic_fixture.vm_ip, dip=fvn1_vm1_traffic_fixture.vm_ip, sport=sport, dport='0', protocol='1') else: flow_rec1 = None if flow_rec1 is not None: self.logger.info('Verifying NAT in flow records') match = inspect_h1.match_item_in_flowrecord( flow_rec1, 'nat', 'enabled') if match is False: self.logger.error( 'Test Failed. NAT is not enabled in given flow. Flow details %s' % (flow_rec1)) result = result and False self.logger.info('Verifying traffic direction in flow records') match = inspect_h1.match_item_in_flowrecord( flow_rec1, 'direction', 'ingress') if match is False: self.logger.error( 'Test Failed. Traffic direction is wrong should be ingress. Flow details %s' % (flow_rec1)) result = result and False else: self.logger.error( 'Test Failed. Required ingress Traffic flow not found') result = result and False # Verify Egress Traffic # Check VMs are in same agent or not. Need to compute source vrf # accordingly self.logger.info('Verifying Egress Flow Records') if fwd_flow: flow_rec2 = inspect_h1.get_vna_fetchflowrecord( nh=vn1_vm1_traffic_fixture.tap_intf[vn_fq_name] ['flow_key_idx'], sip=fvn1_vm1_traffic_fixture.vm_ip, dip=fip_fixture1.fip[fip_id1], sport=sport, dport='0', protocol='1') else: flow_rec2 = False if flow_rec2 is not None: self.logger.info('Verifying NAT in flow records') match = inspect_h1.match_item_in_flowrecord( flow_rec2, 'nat', 'enabled') if match is False: self.logger.error( 'Test Failed. NAT is not enabled in given flow. Flow details %s' % (flow_rec2)) result = result and False self.logger.info('Verifying traffic direction in flow records') match = inspect_h1.match_item_in_flowrecord( flow_rec2, 'direction', 'egress') if match is False: self.logger.error( 'Test Failed. Traffic direction is wrong should be Egress. Flow details %s' % (flow_rec1)) result = result and False else: self.logger.error( 'Test Failed. Required Egress Traffic flow not found') result = result and False # Stop Traffic self.logger.info("Proceed to stop traffic..") self.logger.info("-" * 80) for proto in traffic_proto_l: stopStatus[proto] = {} stopStatus[proto] = traffic_obj[proto].stopTraffic() #if stopStatus[proto] != []: msg.append(stopStatus[proto]); result= False if stopStatus[proto] != []: result = False self.logger.info("Status of stop traffic for proto %s is %s" % (proto, stopStatus[proto])) self.logger.info("-" * 80) # Start the control node service again self.logger.info('Starting the Control service in %s' % (active_controller)) self.inputs.start_service('contrail-control', [active_controller], container='control') sleep(10) # Check the BGP peering status from the currently active control node self.logger.info( 'Checking the BGP peering from new active controler %s' % (new_active_controller)) cn_bgp_entry = self.cn_inspect[ new_active_controller].get_cn_bgp_neigh_entry() sleep(5) for entry in cn_bgp_entry: if entry['state'] != 'Established': result = result and False self.logger.error( 'With Peer %s peering is not Established. Current State %s ' % (entry['peer'], entry['state'])) # fip_fixture1.disassoc_and_delete_fip(fip_id1) if not result: self.logger.error('Switchover of control node failed') assert result return True
def test_max_vm_flows(self): ''' Test to validate setting up of the max_vm_flows parameter in agent config file has expected effect on the flows in the system. 1. Set VM flow cache time and max_vm_flows to 0.01% of max system flows(512K). 2. Create 2 VN's and connect them using a policy. 3. Launch 2 VM's in the respective VN's. 4. Start traffic with around 20000 flows. 6. Restart vrouter agent service and check the flows are limited 0.01% of max system flows. Pass criteria: Step 6 should pass ''' result = True # Set VM flow cache time to 30 and max_vm_flows to 0.1% of max system # flows(512K). self.comp_node_fixt = {} self.flow_cache_timeout = 10 self.max_system_flows = 0 self.max_vm_flows = 0.01 for cmp_node in self.inputs.compute_ips: self.comp_node_fixt[cmp_node] = self.useFixture(ComputeNodeFixture( self.connections, cmp_node)) self.comp_node_fixt[cmp_node].set_flow_aging_time( self.flow_cache_timeout) self.comp_node_fixt[cmp_node].get_config_per_vm_flow_limit() self.comp_node_fixt[cmp_node].set_per_vm_flow_limit( self.max_vm_flows) self.comp_node_fixt[cmp_node].sup_vrouter_process_restart() if self.max_system_flows < self.comp_node_fixt[ cmp_node].max_system_flows: self.max_system_flows = self.comp_node_fixt[ cmp_node].max_system_flows self.addCleanup(self.cleanup_test_max_vm_flows_vrouter_config, self.inputs.compute_ips, self.comp_node_fixt) # Define resources for this test. vn1_name = get_random_name('VN1') vn1_subnets = ['10.1.1.0/24'] vn2_name = get_random_name('VN2') vn2_subnets = ['10.2.1.0/24'] vn1_vm1_name = get_random_name('VM1') vn2_vm2_name = get_random_name('VM2') policy1_name = 'policy1' policy2_name = 'policy2' rules = [ { 'direction': '<>', 'simple_action': 'pass', 'protocol': 'any', 'source_network': vn1_name, 'dest_network': vn2_name, }, ] rev_rules = [ { 'direction': '<>', 'simple_action': 'pass', 'protocol': 'any', 'source_network': vn2_name, 'dest_network': vn1_name, }, ] # Create 2 VN's and connect them using a policy. vn1_fixture = self.create_vn(vn1_name, vn1_subnets) assert vn1_fixture.verify_on_setup() vn2_fixture = self.create_vn(vn2_name, vn2_subnets) assert vn2_fixture.verify_on_setup() policy1_fixture = self.useFixture( PolicyFixture( policy_name=policy1_name, rules_list=rules, inputs=self.inputs, connections=self.connections)) policy2_fixture = self.useFixture( PolicyFixture( policy_name=policy2_name, rules_list=rev_rules, inputs=self.inputs, connections=self.connections)) vn1_fixture.bind_policies( [policy1_fixture.policy_fq_name], vn1_fixture.vn_id) self.addCleanup(vn1_fixture.unbind_policies, vn1_fixture.vn_id, [policy1_fixture.policy_fq_name]) vn2_fixture.bind_policies( [policy2_fixture.policy_fq_name], vn2_fixture.vn_id) self.addCleanup(vn2_fixture.unbind_policies, vn2_fixture.vn_id, [policy2_fixture.policy_fq_name]) # Launch 2 VM's in the respective VN's. vm1_fixture = self.create_vm(vn1_fixture,vm_name=vn1_vm1_name, flavor='contrail_flavor_small', image_name='ubuntu-traffic') vm2_fixture = self.create_vm(vn2_fixture,vm_name=vn2_vm2_name, flavor='contrail_flavor_small', image_name='ubuntu-traffic') assert vm1_fixture.verify_on_setup(), 'VM1 verifications FAILED' assert vm2_fixture.verify_on_setup(), 'VM2 verifications FAILED' assert vm1_fixture.wait_till_vm_is_up(), 'VM1 does not seem to be up' assert vm2_fixture.wait_till_vm_is_up(), 'VM2 does not seem to be up' assert vm1_fixture.ping_with_certainty(vm2_fixture.vm_ip), \ 'Ping from VM1 to VM2 FAILED' # Set num_flows to fixed, smaller value but > 1% of # system max flows max_system_flows = self.max_system_flows vm_flow_limit = int((self.max_vm_flows/100.0)*max_system_flows) num_flows = vm_flow_limit + 30 generated_flows = 2*num_flows flow_gen_rate = 5 proto = 'udp' # Start Traffic. self.traffic_obj = self.useFixture( traffic_tests.trafficTestFixture(self.connections)) startStatus = self.traffic_obj.startTraffic( total_single_instance_streams=int(num_flows), pps=flow_gen_rate, start_sport=5000, cfg_profile='ContinuousSportRange', tx_vm_fixture=vm1_fixture, rx_vm_fixture=vm2_fixture, stream_proto=proto) msg1 = "Status of start traffic : %s, %s, %s" % ( proto, vm1_fixture.vm_ip, startStatus['status']) self.logger.info(msg1) assert startStatus['status'], msg1 self.logger.info("Wait for 3 sec for flows to be setup.") sleep(3) # 4. Poll live traffic & verify VM flow count flow_cmd = 'flow -l | grep %s -A1 |' % vm1_fixture.vm_ip flow_cmd = flow_cmd + ' grep "Action" | grep -v "Action:D(FlowLim)" | wc -l' sample_time = 2 vm_flow_list=[] for i in range(5): sleep(sample_time) vm_flow_record = self.inputs.run_cmd_on_server( vm1_fixture.vm_node_ip, flow_cmd, self.inputs.host_data[vm1_fixture.vm_node_ip]['username'], self.inputs.host_data[vm1_fixture.vm_node_ip]['password']) vm_flow_record = vm_flow_record.strip() vm_flow_list.append(int(vm_flow_record)) self.logger.info("%s iteration DONE." % i) self.logger.info("VM flow count = %s." % vm_flow_list[i]) self.logger.info("Sleeping for %s sec before next iteration." % sample_time) vm_flow_list.sort(reverse=True) if vm_flow_list[0] > int(1.1*vm_flow_limit): self.logger.error("TEST FAILED.") self.logger.error("VM flow count seen is greater than configured.") result = False elif vm_flow_list[0] < int(0.9*vm_flow_limit): self.logger.error("TEST FAILED.") self.logger.error("VM flow count seen is much lower than config.") self.logger.error("Something is stopping flow creation. Please debug") result = False else: self.logger.info("TEST PASSED") self.logger.info("Expected range of vm flows seen.") self.logger.info("Max VM flows = %s" % vm_flow_list[0]) # Stop Traffic. self.logger.info("Proceed to stop traffic..") try: self.traffic_obj.stopTraffic(wait_for_stop=False) except: self.logger.warn("Failed to get a VM handle and stop traffic.") self.logger.info("Wait for the flows to get purged.") sleep(self.flow_cache_timeout) return result
def test_flow_multi_projects(self): """Tests related to flow setup rate and flow table stability accross various triggers for verification accross VN's and accross multiple projects. """ result = True self.comp_node_fixt = {} for cmp_node in self.inputs.compute_ips: self.comp_node_fixt[cmp_node] = self.useFixture(ComputeNodeFixture( self.connections, cmp_node)) # 1. Start Traffic num_flows = 15000 flow_gen_rate = 1000 proto = 'udp' profile = 'TrafficProfile1' details = self.topo[self.topo.keys()[0]].traffic_profile[profile] self.traffic_setup(profile, details, num_flows, flow_gen_rate, proto) self.traffic_obj = self.useFixture( traffic_tests.trafficTestFixture(self.connections)) # def startTraffic (tx_vm_fixture= None, rx_vm_fixture= None, # stream_proto= 'udp', start_sport= 8000, # total_single_instance_streams= 20): startStatus = self.traffic_obj.startTraffic( total_single_instance_streams=num_flows, pps=self.flow_gen_rate, start_sport=1000, cfg_profile='ContinuousSportRange', tx_vm_fixture=self.src_vm_fixture, rx_vm_fixture=self.dst_vm_fixture, stream_proto=self.proto) msg1 = "Status of start traffic : %s, %s, %s" % ( self.proto, self.src_vm_fixture.vm_ip, startStatus['status']) self.logger.info(msg1) assert startStatus['status'], msg1 # 2. Poll live traffic & verify VM flow count flow_test_utils.verify_node_flow_setup(self) # 3. Stop Traffic self.logger.info("Proceed to stop traffic..") self.traffic_obj.stopTraffic(wait_for_stop=False) start_time = time.time() # 4. Verify flow ageing self.logger.info( "With traffic stopped, wait for flow_cache_timeout to trigger flow ageing") sleep(self.flow_cache_timeout) while True: begin_flow_count = self.comp_node_fixt[ self.cmp_node].get_vrouter_matching_flow_count( self.flow_data) self.logger.debug('begin_flow_count: %s' % (begin_flow_count)) if begin_flow_count['all'] == 0: break flow_teardown_time = math.ceil( flow_test_utils.get_max_flow_removal_time( begin_flow_count['all'], self.flow_cache_timeout)) # flow_teardown_time is not the actual time to remove flows # Based on flow_count at this time, teardown_time is calculated to the value # which will vary with agent's poll, which is done at regular # intervals.. self.logger.info('Sleeping for %s secs' % (flow_teardown_time)) sleep(flow_teardown_time) # at the end of wait, actual_flows should be atleast < 50% of total # flows before start of teardown current_flow_count = self.comp_node_fixt[ self.cmp_node].get_vrouter_matching_flow_count( self.flow_data) self.logger.debug('current_flow_count: %s' % (current_flow_count)) if current_flow_count['all'] > (0.5 * begin_flow_count['all']): msg = [ 'Flow removal not happening as expected in node %s' % self.cmp_node] msg.append( 'Flow count before wait: %s, after wait of %s secs, its: %s' % (begin_flow_count['all'], flow_teardown_time, current_flow_count['all'])) assert False, msg if current_flow_count['all'] < (0.1 * begin_flow_count['all']): break # end of while loop elapsed_time = time.time() - start_time self.logger.info( "Flows aged out as expected in configured flow_cache_timeout") return True
def test_max_vm_flows(self): ''' Test to validate setting up of the max_vm_flows parameter in agent config file has expected effect on the flows in the system. 1. Set VM flow cache time and max_vm_flows to 0.01% of max system flows(512K). 2. Create 2 VN's and connect them using a policy. 3. Launch 2 VM's in the respective VN's. 4. Start traffic with around 20000 flows. 6. Restart vrouter agent service and check the flows are limited 0.01% of max system flows. Pass criteria: Step 6 should pass ''' result = True # Set VM flow cache time to 30 and max_vm_flows to 0.1% of max system # flows(512K). self.comp_node_fixt = {} self.flow_cache_timeout = 10 self.max_system_flows = 0 self.max_vm_flows = 0.01 for cmp_node in self.inputs.compute_ips: self.comp_node_fixt[cmp_node] = self.useFixture( ComputeNodeFixture(self.connections, cmp_node)) self.comp_node_fixt[cmp_node].set_flow_aging_time( self.flow_cache_timeout) self.comp_node_fixt[cmp_node].get_config_per_vm_flow_limit() self.comp_node_fixt[cmp_node].set_per_vm_flow_limit( self.max_vm_flows) self.comp_node_fixt[cmp_node].sup_vrouter_process_restart() if self.max_system_flows < self.comp_node_fixt[ cmp_node].max_system_flows: self.max_system_flows = self.comp_node_fixt[ cmp_node].max_system_flows self.addCleanup(self.cleanup_test_max_vm_flows_vrouter_config, self.inputs.compute_ips, self.comp_node_fixt) # Define resources for this test. vn1_name = get_random_name('VN1') vn1_subnets = ['10.1.1.0/24'] vn2_name = get_random_name('VN2') vn2_subnets = ['10.2.1.0/24'] vn1_vm1_name = get_random_name('VM1') vn2_vm2_name = get_random_name('VM2') policy1_name = 'policy1' policy2_name = 'policy2' rules = [ { 'direction': '<>', 'simple_action': 'pass', 'protocol': 'any', 'source_network': vn1_name, 'dest_network': vn2_name, }, ] rev_rules = [ { 'direction': '<>', 'simple_action': 'pass', 'protocol': 'any', 'source_network': vn2_name, 'dest_network': vn1_name, }, ] # Create 2 VN's and connect them using a policy. vn1_fixture = self.create_vn(vn1_name, vn1_subnets) assert vn1_fixture.verify_on_setup() vn2_fixture = self.create_vn(vn2_name, vn2_subnets) assert vn2_fixture.verify_on_setup() policy1_fixture = self.useFixture( PolicyFixture(policy_name=policy1_name, rules_list=rules, inputs=self.inputs, connections=self.connections)) policy2_fixture = self.useFixture( PolicyFixture(policy_name=policy2_name, rules_list=rev_rules, inputs=self.inputs, connections=self.connections)) vn1_fixture.bind_policies([policy1_fixture.policy_fq_name], vn1_fixture.vn_id) self.addCleanup(vn1_fixture.unbind_policies, vn1_fixture.vn_id, [policy1_fixture.policy_fq_name]) vn2_fixture.bind_policies([policy2_fixture.policy_fq_name], vn2_fixture.vn_id) self.addCleanup(vn2_fixture.unbind_policies, vn2_fixture.vn_id, [policy2_fixture.policy_fq_name]) # Launch 2 VM's in the respective VN's. vm1_fixture = self.create_vm(vn1_fixture, vm_name=vn1_vm1_name, flavor='contrail_flavor_small', image_name='ubuntu-traffic') vm2_fixture = self.create_vm(vn2_fixture, vm_name=vn2_vm2_name, flavor='contrail_flavor_small', image_name='ubuntu-traffic') assert vm1_fixture.verify_on_setup(), 'VM1 verifications FAILED' assert vm2_fixture.verify_on_setup(), 'VM2 verifications FAILED' assert vm1_fixture.wait_till_vm_is_up(), 'VM1 does not seem to be up' assert vm2_fixture.wait_till_vm_is_up(), 'VM2 does not seem to be up' assert vm1_fixture.ping_with_certainty(vm2_fixture.vm_ip), \ 'Ping from VM1 to VM2 FAILED' # Set num_flows to fixed, smaller value but > 1% of # system max flows max_system_flows = self.max_system_flows vm_flow_limit = int((self.max_vm_flows / 100.0) * max_system_flows) num_flows = vm_flow_limit + 30 generated_flows = 2 * num_flows flow_gen_rate = 5 proto = 'udp' # Start Traffic. self.traffic_obj = self.useFixture( traffic_tests.trafficTestFixture(self.connections)) startStatus = self.traffic_obj.startTraffic( total_single_instance_streams=int(num_flows), pps=flow_gen_rate, start_sport=5000, cfg_profile='ContinuousSportRange', tx_vm_fixture=vm1_fixture, rx_vm_fixture=vm2_fixture, stream_proto=proto) msg1 = "Status of start traffic : %s, %s, %s" % ( proto, vm1_fixture.vm_ip, startStatus['status']) self.logger.info(msg1) assert startStatus['status'], msg1 self.logger.info("Wait for 3 sec for flows to be setup.") sleep(3) # 4. Poll live traffic & verify VM flow count flow_cmd = 'flow -l | grep %s -A2 |' % vm1_fixture.vm_ip flow_cmd = flow_cmd + ' grep "Action" | grep -v "Action:D(FlowLim)" | wc -l' sample_time = 2 vm_flow_list = [] for i in range(5): sleep(sample_time) vm_flow_record = self.inputs.run_cmd_on_server( vm1_fixture.vm_node_ip, flow_cmd, self.inputs.host_data[vm1_fixture.vm_node_ip]['username'], self.inputs.host_data[vm1_fixture.vm_node_ip]['password']) vm_flow_record = vm_flow_record.strip() vm_flow_list.append(int(vm_flow_record)) self.logger.info("%s iteration DONE." % i) self.logger.info("VM flow count = %s." % vm_flow_list[i]) self.logger.info("Sleeping for %s sec before next iteration." % sample_time) vm_flow_list.sort(reverse=True) if vm_flow_list[0] > int(1.1 * vm_flow_limit): self.logger.error("TEST FAILED.") self.logger.error("VM flow count seen is greater than configured.") result = False elif vm_flow_list[0] < int(0.9 * vm_flow_limit): self.logger.error("TEST FAILED.") self.logger.error("VM flow count seen is much lower than config.") self.logger.error( "Something is stopping flow creation. Please debug") result = False else: self.logger.info("TEST PASSED") self.logger.info("Expected range of vm flows seen.") self.logger.info("Max VM flows = %s" % vm_flow_list[0]) # Stop Traffic. self.logger.info("Proceed to stop traffic..") try: self.traffic_obj.stopTraffic(wait_for_stop=False) except: self.logger.warn("Failed to get a VM handle and stop traffic.") self.logger.info("Wait for the flows to get purged.") sleep(self.flow_cache_timeout) return result
def test_agent_flow_settings(self): """Basic systest with single project with many features & traffic.. """ # # Check if there are enough nodes i.e. atleast 2 compute nodes to run this test. # else report that minimum 2 compute nodes are needed for this test and # exit. if len(self.inputs.compute_ips) < 2: self.logger.warn( "Minimum 2 compute nodes are needed for this test to run") self.logger.warn( "Exiting since this test can't be run on single compute node") return True # # Get config for test from topology # import mini_flow_test_topo # topology_class_name = mini_flow_test_topo.systest_topo_single_project topology_class_name = flow_test_topo.systest_topo_single_project self.logger.info( "Scenario for the test used is: %s" % (topology_class_name)) topo = topology_class_name( compute_node_list=self.inputs.compute_ips) # # 1. Test setup: Configure policy, VN, & VM # return {'result':result, 'msg': err_msg, 'data': [self.topo, config_topo]} # Returned topo is of following format: # config_topo= {'policy': policy_fixt, 'vn': vn_fixture, 'vm': # vm_fixture} setup_obj = self.useFixture( sdnTopoSetupFixture(self.connections, topo)) out = setup_obj.sdn_topo_setup() assertEqual(out['result'], True, out['msg']) if out['result']: config_topo = out['data'][1] self.proj = list(config_topo.keys())[0] self.topo, self.config_topo = topo, config_topo # 2. set agent flow_cache_timeout to 60s # set max_vm_flows to 1% of 500k, comes to 5000 self.comp_node_fixt = {} self.flow_cache_timeout = 60 for cmp_node in self.inputs.compute_ips: self.comp_node_fixt[cmp_node] = self.useFixture(ComputeNodeFixture( self.connections, cmp_node)) self.comp_node_fixt[cmp_node].set_flow_aging_time( self.flow_cache_timeout) self.comp_node_fixt[cmp_node].sup_vrouter_process_restart() # 3. Start Traffic for profile, details in self.topo.traffic_profile.items(): self.logger.info("Profile under test: %s, details: %s" %(profile, details)) self.src_vm = details['src_vm'] self.dst_vm = details['dst_vm'] self.src_proj = self.proj self.dst_proj = self.proj # Set num_flows to fixed, smaller value but > 1% of # system max flows num_flows = 5555 self.generated_flows = 2*num_flows self.flow_gen_rate = 1000 src_vm_fixture = self.config_topo[self.proj]['vm'][self.src_vm] src_vm_vn = src_vm_fixture.vn_names[0] src_vm_vn_fix = self.config_topo[self.proj]['vn'][src_vm_vn] dst_vm_fixture = self.config_topo[self.proj]['vm'][self.dst_vm] self.proto = 'udp' self.cmp_node = src_vm_fixture.vm_node_ip # 3a. Set max_vm_flows to 1% in TX VM node self.max_vm_flows = 1 self.comp_node_fixt[ self.cmp_node].set_per_vm_flow_limit( self.max_vm_flows) self.comp_node_fixt[self.cmp_node].sup_vrouter_process_restart() self.logger.info( "Wait for 2s for flow setup to start after service restart") sleep(2) flow_test_utils.update_vm_mdata_ip(self.cmp_node, self) self.traffic_obj = self.useFixture( traffic_tests.trafficTestFixture(self.connections)) # def startTraffic (tx_vm_fixture= None, rx_vm_fixture= None, # stream_proto= 'udp', start_sport= 8000, # total_single_instance_streams= 20): startStatus = self.traffic_obj.startTraffic( total_single_instance_streams=num_flows, pps=self.flow_gen_rate, start_sport=1000, cfg_profile='ContinuousSportRange', tx_vm_fixture=src_vm_fixture, rx_vm_fixture=dst_vm_fixture, stream_proto=self.proto) msg1 = "Status of start traffic : %s, %s, %s" % ( self.proto, src_vm_fixture.vm_ip, startStatus['status']) self.logger.info(msg1) assert startStatus['status'], msg1 # 4. Poll live traffic & verify VM flow count self.verify_node_flow_setup() # 5. Increase max_vm_flows to 50% in TX VM node self.max_vm_flows = 50 self.comp_node_fixt[ self.cmp_node].set_per_vm_flow_limit( self.max_vm_flows) self.comp_node_fixt[self.cmp_node].sup_vrouter_process_restart() self.logger.info( "Wait for 2s for flow setup to start after service restart") sleep(2) # 6. Poll live traffic self.verify_node_flow_setup() # 7. Stop Traffic self.logger.info("Proceed to stop traffic..") self.traffic_obj.stopTraffic(wait_for_stop=False) start_time = time.time() # 8. Verify flow ageing self.logger.info( "With traffic stopped, wait for flow_cache_timeout to trigger flow ageing") sleep(self.flow_cache_timeout) retries = 0 retry_wait_time = 10 flow_teardown_time = math.ceil(flow_test_utils.get_max_flow_removal_time(self.generated_flows, self.flow_cache_timeout)) self.logger.debug("flow tear down time based on calcualtion: %s" %flow_teardown_time) max_retries = math.ceil(self.flow_cache_timeout / retry_wait_time) while retries < max_retries: actual_flows = self.comp_node_fixt[ self.cmp_node].get_vrouter_matching_flow_count( self.flow_data) actual_flows = int(actual_flows['all']) if actual_flows > 10: self.logger.info("Waiting for flows to age out") sleep(retry_wait_time) retries += 1 else: break elapsed_time = time.time() - start_time if actual_flows > 50: msg = "Expected flows to age-out as configured, Seeing flows still active after elapsed time %s in node: %s, actual_flows: %s" % ( elapsed_time, self.cmp_node, actual_flows) assert False, msg else: self.logger.info( "Flows aged out as expected in configured flow_cache_timeout") self.logger.info( "elapsed_time after stopping traffic is %s, flow_count is %s" % (elapsed_time, actual_flows))
def test_traffic_with_control_node_switchover(self): ''' Stop the control node and check peering with agent fallback to other control node. ''' if len(set(self.inputs.bgp_ips)) < 2: raise self.skipTest( "Skipping Test. At least 2 control node required to run the test") result = True fip_pool_name = get_random_name('some-pool1') (self.vn1_name, self.vn1_subnets) = ( get_random_name("vn1"), ["11.1.1.0/24"]) (self.vn2_name, self.vn2_subnets) = ( get_random_name("vn2"), ["22.1.1.0/24"]) (self.fvn_public_name, self.fvn_public_subnets) = ( get_random_name("fip_vn_public"), ['10.204.219.16/28']) (self.fvn1_name, self.fvn1_subnets) = ( get_random_name("fip_vn1"), ['100.1.1.0/24']) (self.fvn2_name, self.fvn2_subnets) = ( get_random_name("fip_vn2"), ['200.1.1.0/24']) (self.fvn3_name, self.fvn3_subnets) = ( get_random_name("fip_vn3"), ['170.1.1.0/29']) (self.vn1_vm1_name, self.vn1_vm2_name) = ( get_random_name('vn1_vm1'), get_random_name('vn1_vm2')) (self.vn2_vm1_name, self.vn2_vm2_name) = ( get_random_name('vn2_vm1'), get_random_name('vn2_vm2')) (self.fvn_public_vm1_name) = (get_random_name('fvn_public_vm1')) (self.fvn1_vm1_name) = (get_random_name('fvn1_vm1')) (self.fvn2_vm1_name) = (get_random_name('fvn2_vm1')) (self.fvn3_vm1_name) = (get_random_name('fvn3_vm1')) (self.vn1_vm1_traffic_name) = get_random_name('VN1_VM1_traffic') (self.fvn1_vm1_traffic_name) = get_random_name('FVN1_VM1_traffic') fip_pool_name1 = get_random_name('some-pool1') fip_pool_name2 = get_random_name('some-pool2') # Get all compute host host_list = self.connections.nova_h.get_hosts() compute_1 = host_list[0] compute_2 = host_list[0] if len(host_list) > 1: compute_1 = host_list[0] compute_2 = host_list[1] self.fvn1_fixture = self.useFixture( VNFixture( project_name=self.inputs.project_name, connections=self.connections, inputs=self.inputs, vn_name=self.fvn1_name, subnets=self.fvn1_subnets)) self.vn1_fixture = self.useFixture( VNFixture( project_name=self.inputs.project_name, connections=self.connections, inputs=self.inputs, vn_name=self.vn1_name, subnets=self.vn1_subnets)) self.fvn1_vm1_traffic_fixture = self.useFixture( VMFixture( project_name=self.inputs.project_name, connections=self.connections, vn_obj=self.fvn1_fixture.obj, flavor='contrail_flavor_small', image_name='ubuntu-traffic', vm_name=self.fvn1_vm1_traffic_name, node_name=compute_2)) self.vn1_vm1_traffic_fixture = self.useFixture( VMFixture( project_name=self.inputs.project_name, connections=self.connections, vn_obj=self.vn1_fixture.obj, flavor='contrail_flavor_small', image_name='ubuntu-traffic', vm_name=self.vn1_vm1_traffic_name, node_name=compute_1)) fvn1_fixture = self.fvn1_fixture fvn1_vm1_traffic_fixture = self.fvn1_vm1_traffic_fixture fvn1_subnets = self.fvn1_subnets fvn1_vm1_traffic_name = self.fvn1_vm1_traffic_name vn1_fixture = self.vn1_fixture vn1_vm1_traffic_fixture = self.vn1_vm1_traffic_fixture vn1_subnets = self.vn1_subnets vn1_vm1_traffic_name = self.vn1_vm1_traffic_name assert fvn1_fixture.verify_on_setup() assert vn1_fixture.verify_on_setup() assert fvn1_vm1_traffic_fixture.verify_on_setup() assert vn1_vm1_traffic_fixture.verify_on_setup() fip_fixture1 = self.useFixture( FloatingIPFixture( project_name=self.inputs.project_name, inputs=self.inputs, connections=self.connections, pool_name=fip_pool_name1, vn_id=fvn1_fixture.vn_id)) assert fip_fixture1.verify_on_setup() fip_id1 = fip_fixture1.create_and_assoc_fip( fvn1_fixture.vn_id, vn1_vm1_traffic_fixture.vm_id) self.addCleanup(fip_fixture1.disassoc_and_delete_fip, fip_id1) assert fip_fixture1.verify_fip( fip_id1, vn1_vm1_traffic_fixture, fvn1_fixture) if not vn1_vm1_traffic_fixture.ping_with_certainty( fvn1_vm1_traffic_fixture.vm_ip): result = result and False # Figuring the active control node active_controller = None inspect_h = self.agent_inspect[vn1_vm1_traffic_fixture.vm_node_ip] agent_xmpp_status = inspect_h.get_vna_xmpp_connection_status() for entry in agent_xmpp_status: if entry['cfg_controller'] == 'Yes': active_controller = entry['controller_ip'] self.logger.info( 'Active control node from the Agent %s is %s' % (vn1_vm1_traffic_fixture.vm_node_ip, active_controller)) fvn1_vm1_traffic_fixture.wait_till_vm_is_up() vn1_vm1_traffic_fixture.wait_till_vm_is_up() # Install traffic pkg in VM vn1_vm1_traffic_fixture.install_pkg("Traffic") fvn1_vm1_traffic_fixture.install_pkg("Traffic") # Start Traffic traffic_obj = {} startStatus = {} stopStatus = {} traffic_proto_l = ['icmp'] total_streams = {} total_streams[ 'icmp'] = 1 dpi = 9100 proto = 'icmp' for proto in traffic_proto_l: traffic_obj[proto] = {} startStatus[proto] = {} traffic_obj[proto] = self.useFixture( traffic_tests.trafficTestFixture(self.connections)) startStatus[proto] = traffic_obj[proto].startTraffic( num_streams=total_streams[proto], start_port=dpi, tx_vm_fixture=vn1_vm1_traffic_fixture, rx_vm_fixture=fvn1_vm1_traffic_fixture, stream_proto=proto) self.logger.info( "Status of start traffic : %s, %s, %s" % (proto, vn1_vm1_traffic_fixture.vm_ip, startStatus[proto])) if startStatus[proto]['status'] != True: result = False self.logger.info("-" * 80) # Poll live traffic traffic_stats = {} self.logger.info("Poll live traffic and get status..") for proto in traffic_proto_l: traffic_stats = traffic_obj[proto].getLiveTrafficStats() err_msg = "Traffic disruption is seen: details: " #self.assertEqual(traffic_stats['status'], True, err_msg) assert(traffic_stats['status']), err_msg self.logger.info("-" * 80) # Stop on Active node self.logger.info('Stoping the Control service in %s' % (active_controller)) self.inputs.stop_service('contrail-control', [active_controller], container='control') sleep(5) # Check the control node shifted to other control node new_active_controller = None new_active_controller_state = None inspect_h = self.agent_inspect[vn1_vm1_traffic_fixture.vm_node_ip] agent_xmpp_status = inspect_h.get_vna_xmpp_connection_status() for entry in agent_xmpp_status: if entry['cfg_controller'] == 'Yes': new_active_controller = entry['controller_ip'] new_active_controller_state = entry['state'] self.logger.info( 'Active control node from the Agent %s is %s' % (vn1_vm1_traffic_fixture.vm_node_ip, new_active_controller)) if new_active_controller == active_controller: self.logger.error( 'Control node switchover fail. Old Active controlnode was %s and new active control node is %s' % (active_controller, new_active_controller)) result = False if new_active_controller_state != 'Established': self.logger.error( 'Agent does not have Established XMPP connection with Active control node') result = result and False # Verify Flow records here inspect_h1 = self.agent_inspect[vn1_vm1_traffic_fixture.vm_node_ip] inspect_h2 = self.agent_inspect[fvn1_vm1_traffic_fixture.vm_node_ip] flow_rec1 = None udp_src = unicode(8000) dpi = unicode(dpi) # Verify Ingress Traffic self.logger.info('Verifying Ingress Flow Record') vn_fq_name=vn1_vm1_traffic_fixture.vn_fq_name compute_node_fixture = self.useFixture(ComputeNodeFixture( self.connections, vn1_vm1_traffic_fixture.vm_node_ip)) fwd_flow, rev_flow = compute_node_fixture.get_flow_entry( source_ip = vn1_vm1_traffic_fixture.vm_ip, dest_ip = fvn1_vm1_traffic_fixture.vm_ip, dest_port = '0', proto = '1') if fwd_flow: sport = fwd_flow.source_port flow_rec1 = inspect_h1.get_vna_fetchflowrecord( nh=vn1_vm1_traffic_fixture.tap_intf[vn_fq_name]['flow_key_idx'], sip=vn1_vm1_traffic_fixture.vm_ip, dip=fvn1_vm1_traffic_fixture.vm_ip, sport=sport, dport='0', protocol='1') else: flow_rec1 = None if flow_rec1 is not None: self.logger.info('Verifying NAT in flow records') match = inspect_h1.match_item_in_flowrecord( flow_rec1, 'nat', 'enabled') if match is False: self.logger.error( 'Test Failed. NAT is not enabled in given flow. Flow details %s' % (flow_rec1)) result = result and False self.logger.info('Verifying traffic direction in flow records') match = inspect_h1.match_item_in_flowrecord( flow_rec1, 'direction', 'ingress') if match is False: self.logger.error( 'Test Failed. Traffic direction is wrong should be ingress. Flow details %s' % (flow_rec1)) result = result and False else: self.logger.error( 'Test Failed. Required ingress Traffic flow not found') result = result and False # Verify Egress Traffic # Check VMs are in same agent or not. Need to compute source vrf # accordingly self.logger.info('Verifying Egress Flow Records') if fwd_flow: flow_rec2 = inspect_h1.get_vna_fetchflowrecord( nh=vn1_vm1_traffic_fixture.tap_intf[vn_fq_name]['flow_key_idx'], sip=fvn1_vm1_traffic_fixture.vm_ip, dip=fip_fixture1.fip[fip_id1], sport=sport, dport='0', protocol='1') else: flow_rec2 = False if flow_rec2 is not None: self.logger.info('Verifying NAT in flow records') match = inspect_h1.match_item_in_flowrecord( flow_rec2, 'nat', 'enabled') if match is False: self.logger.error( 'Test Failed. NAT is not enabled in given flow. Flow details %s' % (flow_rec2)) result = result and False self.logger.info('Verifying traffic direction in flow records') match = inspect_h1.match_item_in_flowrecord( flow_rec2, 'direction', 'egress') if match is False: self.logger.error( 'Test Failed. Traffic direction is wrong should be Egress. Flow details %s' % (flow_rec1)) result = result and False else: self.logger.error( 'Test Failed. Required Egress Traffic flow not found') result = result and False # Stop Traffic self.logger.info("Proceed to stop traffic..") self.logger.info("-" * 80) for proto in traffic_proto_l: stopStatus[proto] = {} stopStatus[proto] = traffic_obj[proto].stopTraffic() #if stopStatus[proto] != []: msg.append(stopStatus[proto]); result= False if stopStatus[proto] != []: result = False self.logger.info("Status of stop traffic for proto %s is %s" % (proto, stopStatus[proto])) self.logger.info("-" * 80) # Start the control node service again self.logger.info('Starting the Control service in %s' % (active_controller)) self.inputs.start_service('contrail-control', [active_controller], container='control') sleep(10) # Check the BGP peering status from the currently active control node self.logger.info( 'Checking the BGP peering from new active controler %s' % (new_active_controller)) cn_bgp_entry = self.cn_inspect[ new_active_controller].get_cn_bgp_neigh_entry() sleep(5) for entry in cn_bgp_entry: if entry['state'] != 'Established': result = result and False self.logger.error( 'With Peer %s peering is not Established. Current State %s ' % (entry['peer'], entry['state'])) # fip_fixture1.disassoc_and_delete_fip(fip_id1) if not result: self.logger.error('Switchover of control node failed') assert result return True
def test_flow_multi_projects(self): """Tests related to flow setup rate and flow table stability accross various triggers for verification accross VN's and accross multiple projects""" result = True self.comp_node_fixt = {} for cmp_node in self.inputs.compute_ips: self.comp_node_fixt[cmp_node] = self.useFixture(ComputeNodeFixture( self.connections, cmp_node)) # # Check if there are enough nodes i.e. atleast 2 compute nodes to run this test. # else report that minimum 2 compute nodes are needed for this test and # exit. if len(self.inputs.compute_ips) < 2: self.logger.warn( "Minimum 2 compute nodes are needed for this test to run") self.logger.warn( "Exiting since this test can't be run on single compute node") return True # # Get config for test from topology msg = [] topology_class_name = sdn_flow_test_topo_multiple_projects.multi_project_topo self.logger.info("Scenario for the test used is: %s" % (topology_class_name)) # # Create a list of compute node IP's and pass it to topo if you want to pin # a vm to a particular node topo = topology_class_name( compute_node_list=self.inputs.compute_ips) # # 1. Test setup: Configure policy, VN, & VM # return {'result':result, 'msg': err_msg, 'data': [self.topo, config_topo]} # Returned topo is of following format: # config_topo= {'policy': policy_fixt, 'vn': vn_fixture, 'vm': # vm_fixture} setup_obj = self.useFixture( sdnTopoSetupFixture(self.connections, topo)) out = setup_obj.sdn_topo_setup() assertEqual(out['result'], True, out['msg']) self.topo, self.config_topo = out['data'][0], out['data'][1] self.proj = list(self.topo.keys())[0] # 2. Start Traffic for profile, details in self.topo[self.proj].traffic_profile.items(): self.logger.info("Profile under test: %s, details: %s" %(profile, details)) self.src_vm = details['src_vm'] self.dst_vm = details['dst_vm'] self.src_proj = details['src_proj'] self.dst_proj = details['dst_proj'] # Not flow scaling test, limit num_flows to low number.. num_flows = 15000 self.generated_flows = 2*num_flows self.flow_gen_rate = 1000 src_vm_fixture = self.config_topo[self.src_proj]['vm'][self.src_vm] src_vm_vn = src_vm_fixture.vn_names[0] src_vm_vn_fix = self.config_topo[self.src_proj]['vn'][src_vm_vn] dst_vm_fixture = self.config_topo[self.dst_proj]['vm'][self.dst_vm] self.proto = 'udp' self.cmp_node = src_vm_fixture.vm_node_ip self.comp_node_fixt[self.cmp_node].get_config_per_vm_flow_limit() self.comp_node_fixt[self.cmp_node].get_config_flow_aging_time() self.max_vm_flows = self.comp_node_fixt[self.cmp_node].max_vm_flows self.flow_cache_timeout = self.comp_node_fixt[self.cmp_node].flow_cache_timeout self.traffic_obj = self.useFixture( traffic_tests.trafficTestFixture(self.connections)) # def startTraffic (tx_vm_fixture= None, rx_vm_fixture= None, # stream_proto= 'udp', start_sport= 8000, # total_single_instance_streams= 20): startStatus = self.traffic_obj.startTraffic( total_single_instance_streams=num_flows, pps=self.flow_gen_rate, start_sport=1000, cfg_profile='ContinuousSportRange', tx_vm_fixture=src_vm_fixture, rx_vm_fixture=dst_vm_fixture, stream_proto=self.proto) msg1 = "Status of start traffic : %s, %s, %s" % ( self.proto, src_vm_fixture.vm_ip, startStatus['status']) self.logger.info(msg1) assert startStatus['status'], msg1 # 3. Poll live traffic & verify VM flow count self.verify_node_flow_setup() # 4. Stop Traffic self.logger.info("Proceed to stop traffic..") self.traffic_obj.stopTraffic(wait_for_stop=False) start_time = time.time() # 5. Verify flow ageing self.logger.info( "With traffic stopped, wait for flow_cache_timeout to trigger flow ageing") sleep(self.flow_cache_timeout) while True: begin_flow_count = self.comp_node_fixt[ self.cmp_node].get_vrouter_matching_flow_count( self.flow_data) self.logger.debug('begin_flow_count: %s' %(begin_flow_count)) if begin_flow_count['all'] == 0: break flow_teardown_time = math.ceil(flow_test_utils.get_max_flow_removal_time(begin_flow_count['all'], self.flow_cache_timeout)) # flow_teardown_time is not the actual time to remove flows # Based on flow_count at this time, teardown_time is calculated to the value # which will vary with agent's poll, which is done at regular intervals.. self.logger.info('Sleeping for %s secs' %(flow_teardown_time)) sleep(flow_teardown_time) # at the end of wait, actual_flows should be atleast < 50% of total flows before start of teardown current_flow_count = self.comp_node_fixt[ self.cmp_node].get_vrouter_matching_flow_count( self.flow_data) self.logger.debug('current_flow_count: %s' %(current_flow_count)) if current_flow_count['all'] > (0.5*begin_flow_count['all']): msg = ['Flow removal not happening as expected in node %s' %self.cmp_node] msg.append('Flow count before wait: %s, after wait of %s secs, its: %s' % (begin_flow_count['all'], flow_teardown_time, current_flow_count['all'])) assert False, msg if current_flow_count['all'] < (0.1*begin_flow_count['all']): break # end of while loop elapsed_time = time.time() - start_time self.logger.info( "Flows aged out as expected in configured flow_cache_timeout") # end of profile for loop return True
def test_agent_flow_settings(self): """Basic systest with single project with many features & traffic.. """ # # Check if there are enough nodes i.e. atleast 2 compute nodes to run this test. # else report that minimum 2 compute nodes are needed for this test and # exit. if len(self.inputs.compute_ips) < 2: self.logger.warn( "Minimum 2 compute nodes are needed for this test to run") self.logger.warn( "Exiting since this test can't be run on single compute node") return True # # Get config for test from topology # import mini_flow_test_topo # topology_class_name = mini_flow_test_topo.systest_topo_single_project topology_class_name = flow_test_topo.systest_topo_single_project self.logger.info("Scenario for the test used is: %s" % (topology_class_name)) topo = topology_class_name(compute_node_list=self.inputs.compute_ips) # # 1. Test setup: Configure policy, VN, & VM # return {'result':result, 'msg': err_msg, 'data': [self.topo, config_topo]} # Returned topo is of following format: # config_topo= {'policy': policy_fixt, 'vn': vn_fixture, 'vm': # vm_fixture} setup_obj = self.useFixture(sdnTopoSetupFixture( self.connections, topo)) out = setup_obj.sdn_topo_setup() assertEqual(out['result'], True, out['msg']) if out['result']: config_topo = out['data'][1] self.proj = list(config_topo.keys())[0] self.topo, self.config_topo = topo, config_topo # 2. set agent flow_cache_timeout to 60s # set max_vm_flows to 1% of 500k, comes to 5000 self.comp_node_fixt = {} self.flow_cache_timeout = 60 for cmp_node in self.inputs.compute_ips: self.comp_node_fixt[cmp_node] = self.useFixture( ComputeNodeFixture(self.connections, cmp_node)) self.comp_node_fixt[cmp_node].set_flow_aging_time( self.flow_cache_timeout) self.comp_node_fixt[cmp_node].sup_vrouter_process_restart() # 3. Start Traffic for profile, details in self.topo.traffic_profile.items(): self.logger.info("Profile under test: %s, details: %s" % (profile, details)) self.src_vm = details['src_vm'] self.dst_vm = details['dst_vm'] self.src_proj = self.proj self.dst_proj = self.proj # Set num_flows to fixed, smaller value but > 1% of # system max flows num_flows = 5555 self.generated_flows = 2 * num_flows self.flow_gen_rate = 1000 src_vm_fixture = self.config_topo[self.proj]['vm'][self.src_vm] src_vm_vn = src_vm_fixture.vn_names[0] src_vm_vn_fix = self.config_topo[self.proj]['vn'][src_vm_vn] dst_vm_fixture = self.config_topo[self.proj]['vm'][self.dst_vm] self.proto = 'udp' self.cmp_node = src_vm_fixture.vm_node_ip # 3a. Set max_vm_flows to 1% in TX VM node self.max_vm_flows = 1 self.comp_node_fixt[self.cmp_node].set_per_vm_flow_limit( self.max_vm_flows) self.comp_node_fixt[self.cmp_node].sup_vrouter_process_restart() self.logger.info( "Wait for 2s for flow setup to start after service restart") sleep(2) flow_test_utils.update_vm_mdata_ip(self.cmp_node, self) self.traffic_obj = self.useFixture( traffic_tests.trafficTestFixture(self.connections)) # def startTraffic (tx_vm_fixture= None, rx_vm_fixture= None, # stream_proto= 'udp', start_sport= 8000, # total_single_instance_streams= 20): startStatus = self.traffic_obj.startTraffic( total_single_instance_streams=num_flows, pps=self.flow_gen_rate, start_sport=1000, cfg_profile='ContinuousSportRange', tx_vm_fixture=src_vm_fixture, rx_vm_fixture=dst_vm_fixture, stream_proto=self.proto) msg1 = "Status of start traffic : %s, %s, %s" % ( self.proto, src_vm_fixture.vm_ip, startStatus['status']) self.logger.info(msg1) assert startStatus['status'], msg1 # 4. Poll live traffic & verify VM flow count self.verify_node_flow_setup() # 5. Increase max_vm_flows to 50% in TX VM node self.max_vm_flows = 50 self.comp_node_fixt[self.cmp_node].set_per_vm_flow_limit( self.max_vm_flows) self.comp_node_fixt[self.cmp_node].sup_vrouter_process_restart() self.logger.info( "Wait for 2s for flow setup to start after service restart") sleep(2) # 6. Poll live traffic self.verify_node_flow_setup() # 7. Stop Traffic self.logger.info("Proceed to stop traffic..") self.traffic_obj.stopTraffic(wait_for_stop=False) start_time = time.time() # 8. Verify flow ageing self.logger.info( "With traffic stopped, wait for flow_cache_timeout to trigger flow ageing" ) sleep(self.flow_cache_timeout) retries = 0 retry_wait_time = 10 flow_teardown_time = math.ceil( flow_test_utils.get_max_flow_removal_time( self.generated_flows, self.flow_cache_timeout)) self.logger.debug("flow tear down time based on calcualtion: %s" % flow_teardown_time) max_retries = math.ceil(self.flow_cache_timeout / retry_wait_time) while retries < max_retries: actual_flows = self.comp_node_fixt[ self.cmp_node].get_vrouter_matching_flow_count( self.flow_data) actual_flows = int(actual_flows['all']) if actual_flows > 10: self.logger.info("Waiting for flows to age out") sleep(retry_wait_time) retries += 1 else: break elapsed_time = time.time() - start_time if actual_flows > 50: msg = "Expected flows to age-out as configured, Seeing flows still active after elapsed time %s in node: %s, actual_flows: %s" % ( elapsed_time, self.cmp_node, actual_flows) assert False, msg else: self.logger.info( "Flows aged out as expected in configured flow_cache_timeout" ) self.logger.info( "elapsed_time after stopping traffic is %s, flow_count is %s" % (elapsed_time, actual_flows))
def test_flow_multi_projects(self): """Tests related to flow setup rate and flow table stability accross various triggers for verification accross VN's and accross multiple projects""" result = True self.comp_node_fixt = {} for cmp_node in self.inputs.compute_ips: self.comp_node_fixt[cmp_node] = self.useFixture( ComputeNodeFixture(self.connections, cmp_node)) # # Check if there are enough nodes i.e. atleast 2 compute nodes to run this test. # else report that minimum 2 compute nodes are needed for this test and # exit. if len(self.inputs.compute_ips) < 2: self.logger.warn( "Minimum 2 compute nodes are needed for this test to run") self.logger.warn( "Exiting since this test can't be run on single compute node") return True # # Get config for test from topology msg = [] topology_class_name = sdn_flow_test_topo_multiple_projects.multi_project_topo self.logger.info("Scenario for the test used is: %s" % (topology_class_name)) # # Create a list of compute node IP's and pass it to topo if you want to pin # a vm to a particular node topo = topology_class_name(compute_node_list=self.inputs.compute_ips) # # 1. Test setup: Configure policy, VN, & VM # return {'result':result, 'msg': err_msg, 'data': [self.topo, config_topo]} # Returned topo is of following format: # config_topo= {'policy': policy_fixt, 'vn': vn_fixture, 'vm': # vm_fixture} setup_obj = self.useFixture(sdnTopoSetupFixture( self.connections, topo)) out = setup_obj.sdn_topo_setup() assertEqual(out['result'], True, out['msg']) self.topo, self.config_topo = out['data'][0], out['data'][1] self.proj = list(self.topo.keys())[0] # 2. Start Traffic for profile, details in self.topo[self.proj].traffic_profile.items(): self.logger.info("Profile under test: %s, details: %s" % (profile, details)) self.src_vm = details['src_vm'] self.dst_vm = details['dst_vm'] self.src_proj = details['src_proj'] self.dst_proj = details['dst_proj'] # Not flow scaling test, limit num_flows to low number.. num_flows = 15000 self.generated_flows = 2 * num_flows self.flow_gen_rate = 1000 src_vm_fixture = self.config_topo[self.src_proj]['vm'][self.src_vm] src_vm_vn = src_vm_fixture.vn_names[0] src_vm_vn_fix = self.config_topo[self.src_proj]['vn'][src_vm_vn] dst_vm_fixture = self.config_topo[self.dst_proj]['vm'][self.dst_vm] self.proto = 'udp' self.cmp_node = src_vm_fixture.vm_node_ip self.comp_node_fixt[self.cmp_node].get_config_per_vm_flow_limit() self.comp_node_fixt[self.cmp_node].get_config_flow_aging_time() self.max_vm_flows = self.comp_node_fixt[self.cmp_node].max_vm_flows self.flow_cache_timeout = self.comp_node_fixt[ self.cmp_node].flow_cache_timeout self.traffic_obj = self.useFixture( traffic_tests.trafficTestFixture(self.connections)) # def startTraffic (tx_vm_fixture= None, rx_vm_fixture= None, # stream_proto= 'udp', start_sport= 8000, # total_single_instance_streams= 20): startStatus = self.traffic_obj.startTraffic( total_single_instance_streams=num_flows, pps=self.flow_gen_rate, start_sport=1000, cfg_profile='ContinuousSportRange', tx_vm_fixture=src_vm_fixture, rx_vm_fixture=dst_vm_fixture, stream_proto=self.proto) msg1 = "Status of start traffic : %s, %s, %s" % ( self.proto, src_vm_fixture.vm_ip, startStatus['status']) self.logger.info(msg1) assert startStatus['status'], msg1 # 3. Poll live traffic & verify VM flow count self.verify_node_flow_setup() # 4. Stop Traffic self.logger.info("Proceed to stop traffic..") self.traffic_obj.stopTraffic(wait_for_stop=False) start_time = time.time() # 5. Verify flow ageing self.logger.info( "With traffic stopped, wait for flow_cache_timeout to trigger flow ageing" ) sleep(self.flow_cache_timeout) while True: begin_flow_count = self.comp_node_fixt[ self.cmp_node].get_vrouter_matching_flow_count( self.flow_data) self.logger.debug('begin_flow_count: %s' % (begin_flow_count)) if begin_flow_count['all'] == 0: break flow_teardown_time = math.ceil( flow_test_utils.get_max_flow_removal_time( begin_flow_count['all'], self.flow_cache_timeout)) # flow_teardown_time is not the actual time to remove flows # Based on flow_count at this time, teardown_time is calculated to the value # which will vary with agent's poll, which is done at regular intervals.. self.logger.info('Sleeping for %s secs' % (flow_teardown_time)) sleep(flow_teardown_time) # at the end of wait, actual_flows should be atleast < 50% of total flows before start of teardown current_flow_count = self.comp_node_fixt[ self.cmp_node].get_vrouter_matching_flow_count( self.flow_data) self.logger.debug('current_flow_count: %s' % (current_flow_count)) if current_flow_count['all'] > (0.5 * begin_flow_count['all']): msg = [ 'Flow removal not happening as expected in node %s' % self.cmp_node ] msg.append( 'Flow count before wait: %s, after wait of %s secs, its: %s' % (begin_flow_count['all'], flow_teardown_time, current_flow_count['all'])) assert False, msg if current_flow_count['all'] < (0.1 * begin_flow_count['all']): break # end of while loop elapsed_time = time.time() - start_time self.logger.info( "Flows aged out as expected in configured flow_cache_timeout") # end of profile for loop return True
def test_traffic_across_projects(self): """Traffic test with policy applied across multiple projects""" result = True topology_class_name = None # # Check if there are enough nodes i.e. atleast 2 compute nodes to run this test. # else report that minimum 2 compute nodes are needed for this test and # exit. if len(self.inputs.compute_ips) < 2: self.logger.warn( "Minimum 2 compute nodes are needed for this test to run") self.logger.warn( "Exiting since this test can't be run on single compute node") return True # # Get config for test from topology import sdn_policy_topo_with_multi_project result = True msg = [] if not topology_class_name: topology_class_name = sdn_policy_topo_with_multi_project.sdn_basic_policy_topo_with_fip self.logger.info("Scenario for the test used is: %s" % (topology_class_name)) # # Create a list of compute node IP's and pass it to topo if you want to pin # a vm to a particular node topo_obj = topology_class_name( compute_node_list=self.inputs.compute_ips) # # Test setup: Configure policy, VN, & VM # return {'result':result, 'msg': err_msg, 'data': [self.topo, config_topo]} # Returned topo is of following format: # config_topo= {'policy': policy_fixt, 'vn': vn_fixture, 'vm': vm_fixture} topo = {} topo_objs = {} config_topo = {} setup_obj = self.useFixture( sdnTopoSetupFixture(self.connections, topo_obj)) out = setup_obj.sdn_topo_setup() self.assertEqual(out['result'], True, out['msg']) if out['result'] == True: topo_objs, config_topo, vm_fip_info = out['data'] p_lst = topo_obj.project_list # projects p1vm1 = topo_objs[p_lst[0]].vmc_list[0] # 'vmc1' p2vm2 = topo_objs[p_lst[1]].vmc_list[0] # 'vmc2' p3vm3 = topo_objs[p_lst[2]].vmc_list[0] # 'vmc3' adminvm = topo_objs[p_lst[3]].vmc_list[0] # 'vmc-admin' result = True msg = [] traffic_obj = {} startStatus = {} stopStatus = {} traffic_proto_l = ['tcp', 'icmp'] total_streams = {} total_streams['icmp'] = 1 total_streams['udp'] = 1 total_streams['tcp'] = 1 dpi = 9100 expectedResult = {} for proto in traffic_proto_l: expectedResult[proto] = True traffic_obj[proto] = {} startStatus[proto] = {} if proto == 'icmp': tx_vm_fixt = config_topo[p_lst[0]]['vm'][ p1vm1] rx_vm_fixt = config_topo[p_lst[1]]['vm'][p2vm2] else: tx_vm_fixt = config_topo[p_lst[0]]['vm'][ p1vm1] rx_vm_fixt = config_topo[p_lst[2]]['vm'][p3vm3] traffic_obj[proto] = self.useFixture( traffic_tests.trafficTestFixture(self.connections)) # def startTraffic (self, name, num_streams= 1, start_port= 9100, tx_vm_fixture= None, rx_vm_fixture= None, stream_proto= 'udp', \ # packet_size= 100, start_sport= 8000, # total_single_instance_streams= 20): if vm_fip_info[0]: startStatus[proto] = traffic_obj[proto].startTraffic( num_streams=total_streams[proto], start_port=dpi, tx_vm_fixture=tx_vm_fixt, rx_vm_fixture=rx_vm_fixt, stream_proto=proto, vm_fip_info=vm_fip_info[1]) else: startStatus[proto] = traffic_obj[proto].startTraffic( num_streams=total_streams[proto], start_port=dpi, tx_vm_fixture=tx_vm_fixt, rx_vm_fixture=rx_vm_fixt, stream_proto=proto) msg1 = "Status of start traffic : %s, %s, %s" % ( proto, tx_vm_fixt.vm_ip, startStatus[proto]['status']) if startStatus[proto]['status'] == False: self.logger.error(msg1) msg.extend( [msg1, 'More info on failure: ', startStatus[proto]['msg']]) else: self.logger.info(msg1) self.assertEqual(startStatus[proto]['status'], True, msg) self.logger.info("-" * 80) # Poll live traffic traffic_stats = {} err_msg = [] self.logger.info("Poll live traffic and get status..") for proto in traffic_proto_l: self.logger.info("Poll live traffic %s and get status.." % (proto)) traffic_stats = traffic_obj[proto].getLiveTrafficStats() if not traffic_stats['status']: err_msg.extend( ["Traffic disruption is seen:", traffic_stats['msg']]) self.assertEqual(traffic_stats['status'], expectedResult[proto], err_msg) self.logger.info("-" * 80) # Stop Traffic self.logger.info("Proceed to stop traffic..") self.logger.info("-" * 80) for proto in traffic_proto_l: stopStatus[proto] = traffic_obj[proto].stopTraffic() status = True if stopStatus[proto] == [] else False if status != expectedResult[proto]: msg.append(stopStatus[proto]) result = False self.logger.info("Status of stop traffic for proto %s is %s" % (proto, stopStatus[proto])) self.logger.info("-" * 80) self.assertEqual(result, True, msg) result = True msg = [] dst_vm = p3vm3 # 'vmc3' dst_vm_fixture = config_topo[p_lst[2]]['vm'][p3vm3] dst_vm_ip = dst_vm_fixture.vm_ip src_vm = p1vm1 # 'vmc1' src_vm_fixture = config_topo[p_lst[0]]['vm'][p1vm1] self.logger.info( "With proto tcp allowed between %s and %s, trying to send icmp traffic" % (p1vm1, p3vm3)) expectedResult = False self.logger.info( "Verify ping to vm %s from vm %s, expecting it to fail" % (dst_vm, src_vm)) ret = src_vm_fixture.ping_with_certainty( dst_vm_ip, expectation=expectedResult) result_msg = "vm ping test result to vm %s is: %s" % (dst_vm, ret) self.logger.info(result_msg) if ret != True: result = False msg.extend( ["icmp traffic passed with deny rule:", result_msg]) self.assertEqual(result, True, msg) result = True msg = [] expectedResult = True dst_vm = p2vm2 # 'vmc2' dst_vm_fixture = config_topo[p_lst[1]]['vm'][p2vm2] dst_vm_ip = dst_vm_fixture.vm_ip src_vm_fixture = config_topo[p_lst[3]]['vm'][adminvm] # 'vmc-admin' self.logger.info( "Now will test ICMP traffic between admin VM %s and non-default project VM %s" % (adminvm, p2vm2)) ret = src_vm_fixture.ping_with_certainty( dst_vm_ip, expectation=expectedResult) result_msg = "vm ping test result to vm %s is: %s" % (dst_vm, ret) self.logger.info(result_msg) if ret != True: result = False msg.extend( ["ICMP traffic failed between default and non-default project with policy:", result_msg]) self.assertEqual(result, True, msg) return True
def policy_test_with_scaled_udp_flows(self, topo, num_udp_streams=100, pps=100, wait_time_after_start_traffic=300, vms_on_single_compute=False, setup_only=False): """Pick 2n VM's for testing, have rules affecting udp protocol.. pick 2 VN's, source and destination. each VM in src will send to a unique VM in dest. Generate traffic streams matching policy rules - udp for now.. Check for system stability with flow scaling and traffic behavior as expected. """ result = True msg = [] # # Test setup: Configure policy, VN, & VM setup_obj = self.useFixture( sdnTopoSetupFixture(self.connections, topo)) if vms_on_single_compute: out = setup_obj.topo_setup(vms_on_single_compute=True) else: out = setup_obj.topo_setup() #out= setup_obj.topo_setup(vm_verify='yes', skip_cleanup='yes') self.logger.info("Setup completed with result %s" % (out['result'])) self.assertEqual(out['result'], True, out['msg']) if out['result'] == True: topo, config_topo = out['data'] # Setup/Verify Traffic --- # 1. Define Traffic Params # This will be source_vn for traffic test test_vn = topo.vnet_list[0] dest_vn = topo.vnet_list[1] topo_helper_obj = topology_helper(topo) vms_in_vn = topo_helper_obj.get_vm_of_vn() src_vms = vms_in_vn[test_vn] dest_vms = vms_in_vn[dest_vn] self.logger.info("----" * 20) self.logger.info("num_udp_streams: %s, pps: %s, src_vms: %s" % (num_udp_streams, pps, len(src_vms))) self.logger.info("----" * 20) # using default protocol udp, traffic_proto_l= ['udp'] total_single_instance_streams = num_udp_streams # 2. set expectation to verify.. matching_rule_action = {} # Assumption made here: one policy assigned to test_vn policy = topo.vn_policy[test_vn][0] policy_info = "policy in effect is : " + str(topo.rules[policy]) num_rules = len(topo.rules[policy]) # Assumption made here: one rule for each dest_vn for i in range(num_rules): dvn = topo.rules[policy][i]['dest_network'] matching_rule_action[dvn] = topo.rules[policy][i]['simple_action'] if num_rules == 0: matching_rule_action[dvn] = 'deny' self.logger.info("matching_rule_action: %s" % matching_rule_action) # 3. Start Traffic traffic_obj = {} startStatus = { } stopStatus = {} expectedResult = {} for i in range(len(src_vms)): test_vm1 = src_vms[i] test_vm2 = dest_vms[i] test_vm1_fixture = config_topo['vm'][test_vm1] test_vm2_fixture = config_topo['vm'][test_vm2] expectedResult[i] = True if matching_rule_action[ dest_vn] == 'pass' else False startStatus[i] = {} traffic_obj[i] = self.useFixture( traffic_tests.trafficTestFixture(self.connections)) # def startTraffic (self, name=name, num_streams= 1, start_port= 9100, tx_vm_fixture, rx_vm_fixture, \ # stream_proto, packet_size= 100, start_sport= 8000, # total_single_instance_streams= 20) startStatus[i] = traffic_obj[i].startTraffic( tx_vm_fixture=test_vm1_fixture, rx_vm_fixture=test_vm2_fixture, total_single_instance_streams=total_single_instance_streams, cfg_profile='ContinuousSportRange', pps=pps) msg1 = "Status of start traffic : %s, %s, %s" % ( i, test_vm1_fixture.vm_ip, startStatus[i]['status']) if startStatus[i]['status'] == False: self.logger.error(msg1) msg.extend( [msg1, 'More info on failure: ', startStatus[i]['msg']]) else: self.logger.info(msg1) self.assertEqual(startStatus[i]['status'], True, msg) self.logger.info("-" * 80) sessions = self.tcpdump_on_analyzer(topo.si_list[0]) for svm_name, (session, pcap) in sessions.items(): self.verify_mirror(svm_name, session, pcap) if setup_only: self.logger.info("Test called with setup only..") return True else: # Should be more than 3 mins, aging time, for flows to peak and # stabilise time.sleep(wait_time_after_start_traffic) # 4. Stop Traffic & validate received packets.. self.logger.info("Proceed to stop traffic..") self.logger.info("-" * 80) for i in range(len(src_vms)): stopStatus[i] = traffic_obj[i].stopTraffic(loose='yes') status = True if stopStatus[i] == [] else False if status != expectedResult[i]: msg.append(stopStatus[i]) self.logger.info("Status of stop traffic for instance %s is %s" % (i, stopStatus[i])) if msg != []: result = False self.assertEqual(result, True, msg) self.logger.info("-" * 80) # 5. verify kernel flows after stopping traffic.. we dont expect any # stuck HOLD flows. for compNode in self.inputs.compute_ips: retry = 0 while retry < 5: kflows = self.agent_inspect[compNode].get_vna_kflowresp() wait_time = 60 if len(kflows) > 1000 else 30 if len(kflows) == 0: break else: self.logger.info( "Waiting for Kernel flows to drain in Compute %s, attempt %s, num_flows %s" % (compNode, retry, len(kflows))) time.sleep(wait_time) retry += 1 for compNode in self.inputs.compute_ips: out = self.check_exception_flows_in_kernel(compNode) self.logger.info("out is: %s" % out) self.assertEqual(out['status'], True, out['msg']) # end checking flows return result