def check_pool_state(pool_name, **kwargs):
"""
:param pool_name:
:param kwargs:
:return:
"""
down_servers = kwargs.get('down_servers')
disabled_servers = kwargs.get('disabled_servers')
error_string = kwargs.get('error_string')
retry_timeout = int(kwargs.get('retry_timeout', 10))
retry_interval = float(kwargs.get('retry_interval', 1))
try:
@logger_utils.aretry(delay=retry_interval, period=retry_timeout)
def retry_action():
return is_pool_servers_in_state(pool_name, down_servers,
disabled_servers, error_string)
return retry_action()
except Exception as e:
stats = get_all_pool_stats(pool_name)
logger.debug("Failure case : All pool stats %s" % stats)
logger_utils.error('Did not find pool in expected state after retry '
'timeout of %s, down servers: %s, failed with '
'error: %s' % (retry_timeout, down_servers, e))
def get_all_vnic_flow_create_on_primary_se(vs_name):
se_name = vs_get_primary_se_name(vs_name)
logger.debug('get_all_dispatcher_stats_on_primary_se: %s' % se_name)
se_info = get_se_info(se_name, connected=True)
d_stats = []
for vnic in se_info['data_vnics']:
if_name = vnic['if_name']
params = {'intfname': if_name}
resp_code, json_data = rest.get('serviceengine',
name=se_name,
path='flowtablestat',
params=params)
for dsr in json_data:
if 'dispatch' in dsr:
d_stats.append(dsr['dispatch'][0])
if infra_utils.get_cloud_context_type() == 'baremetal':
vnic = se_info['mgmt_vnic']
if_name = vnic['if_name']
params = {'intfname': if_name}
resp_code, json_data = rest.get('serviceengine',
name=se_name,
path='flowtablestat',
params=params)
for dsr in json_data:
if 'dispatch' in dsr:
d_stats.append(dsr['dispatch'][0])
c = 0
for stats in d_stats:
c = c + stats['flow_rx_create']
return c
def poweroff(self, vm_name=None):
'''
Power off a VM
arguments:
vm_name: vm you wish to resume
'''
if not vm_name:
vm_name = self.vm_json.get('name')
@aretry(retry=10, delay=2, period=2)
def getVm():
vm = self.server.get_vm_by_name(vm_name)
return vm
vm = getVm()
if vm:
vm.power_off()
if vm.get_status() != "POWERED OFF":
fail("VM could not power off. It is in " + vm.get_status() +
" state")
logger.info('vm : %s powered off, status: %s' %
(vm_name, vm.get_status()))
else:
logger.debug('DEBUG: all_vms: %s' %
self.server.get_registered_vms())
fail("Can't find the vm %s" % vm_name)
def server_allow_icmp_pkts(se, svr_vm_hdl, svr_ip):
# Get server side IP for the SE
se_back_ip = se_get_ip_for_server(se, svr_ip)
server_vm = get_server_by_handle(svr_vm_hdl).vm()
cmd = 'iptables -D INPUT -p icmp -s %s -j DROP' % se_back_ip
out = server_vm.execute_command(cmd)
logger.debug('Cmd %s Output: %s' % (cmd, out))
def make_follower_ready_for_cluster(ctrl_vm, **kwargs):
"""
Resets Controller password to admin/admin
"""
config = AviConfig.get_instance()
mode = config.get_mode()
logger.debug("Current Default Mode %s" % mode)
username = mode['user']
current_password = mode['password']
logger.info('Reset controller password for %s' % ctrl_vm.ip)
try:
config.switch_mode(password=ctrl_vm.password)
session = create_session(ctrl_vm)
config.switch_mode(session=session)
# REVIEW password should be original default password
reset_admin_user(username=username,
password='******',
old_password=ctrl_vm.password,
**kwargs)
except Exception as e:
logger.debug("Trying with admin/admin")
config.switch_mode(password='******')
session = create_session(ctrl_vm)
config.switch_mode(session=session)
# REVIEW password shoulde original default password
reset_admin_user(username=username,
password='******',
old_password='******',
**kwargs)
config.switch_mode(session=None, password=current_password)
def get_hmon_stats(pool_name, hm_name, handle, field1='', field2=''):
"""
:param pool_name:
:param hm_name:
:param handle:
:param field1:
:param field2:
:return:
"""
resp_code, resp_data = rest.get('pool',
name=pool_name,
path='/runtime/server/hmonstat')
common.check_response_for_errors(resp_data)
# Check if server is in handle format or name
if ':' in handle:
name = handle
else:
server = infra_utils.get_server_by_handle(handle)
name = server.ip() + ':' + str(server.port())
logger.debug('server_name', name)
shm = resp_data[0].get('server_hm_stat')
for server in shm:
if name == server.get('server_name'):
for hm in server[field1]:
if hm_name == hm.get('health_monitor_name'):
if field2:
return hm[field2]
else:
return hm
def get_vm_ip_for_name(self, vm_name=None, public_ip_address=False):
""" Get IP address for given vm """
vm_ip_addr = None
if not vm_name:
vm_name = self.vm_json.get('name')
try:
vm_obj = self.compute_client.virtual_machines.get(
self.resource_group, vm_name)
for interface in vm_obj.network_profile.network_interfaces:
logger.debug('Got interface details ..: %s' % interface.id)
nic_name = " ".join(interface.id.split('/')[-1:])
#sub="".join(interface.id.split('/')[4])
ip_addr_objs = self.network_client.network_interfaces.get(
self.resource_group, nic_name).ip_configurations
for ip_obj in ip_addr_objs:
logger.info(
" Private IP Address: %s , for Nic: %s, IP Config obj:%s"
% (ip_obj.private_ip_address, nic_name, ip_obj.name))
if ip_obj.primary:
if public_ip_address:
vm_ip_addr = ip_obj.public_ip_address
else:
vm_ip_addr = ip_obj.private_ip_address
logger.info("IP Address: %s , for Nic: %s" %
(vm_ip_addr, nic_name))
return vm_ip_addr
except Exception as e:
fail('Error while getting the ip address for vm, exp: %s' %
e.message)
def error_counters_should_be_under_threshold(shm_runtime, threshold=0):
"""
:param shm_runtime:
:param threshold:
:return:
"""
hm_type = shm_runtime['health_monitor_type']
if hm_type in [
'HEALTH_MONITOR_TCP', 'HEALTH_MONITOR_HTTP',
'HEALTH_MONITOR_HTTPS', 'HEALTH_MONITOR_EXTERNAL',
'HEALTH_MONITOR_UDP', 'HEALTH_MONITOR_DNS', 'HEALTH_MONITOR_PING'
]:
logger.debug('shm_runtime %s' % shm_runtime)
bad_counters = []
if 'curr_count' in shm_runtime and \
len(shm_runtime['curr_count']) > 0:
error_list = shm_runtime['curr_count']
for error in error_list:
if int(error['count']) > threshold:
bad_counters.append("%s: %s" %
(error['type'], error['count']))
if len(bad_counters):
logger_utils.fail('ERROR! Non zero bad connects: %s' %
"".join(bad_counters))
def create_keystone_user(self, username, password, tenant=None):
logger.debug('Creating keystone user %s' %username)
if tenant is None:
tenant_id = self.ks_client.auth_ref.project_id
else:
tenant_id = self.ks_client.tenants.find(name=tenant).id
self.ks_client.users.create(username, password, tenant_id=tenant_id)
def wait_until_n_cluster_nodes_ready_inner():
rsp = None
try:
st, rsp = get('cluster/runtime')
except Exception as ex:
fail('Cluster api runtime exception: %s' % ex)
if rsp and st == 200:
node_states = rsp.get('node_states', [])
cluster_state = rsp.get('cluster_state', {})
cl_state = cluster_state.get('state', 'unknown')
up_nodes = 0
for node in node_states:
if node.get('state') == 'CLUSTER_ACTIVE':
up_nodes += 1
if (up_nodes != n):
logger.debug('Cluster (status:%s) expects %d active nodes '
'but contains %d active nodes' %
(cl_state, n, up_nodes))
elif (n == 1 and cl_state == 'CLUSTER_UP_NO_HA'):
logger.info('Cluster is ready! Cluster state is %s' %
cluster_state)
return
elif (n == 2 and cl_state == 'CLUSTER_UP_HA_COMPROMISED'):
logger.info('Cluster is ready! Cluster state is %s' %
cluster_state)
return
elif (n == 3 and cl_state == 'CLUSTER_UP_HA_ACTIVE'):
logger.info('Cluster is ready! Cluster state is %s' %
cluster_state)
return
fail('Cluster runtime response not as expected %s' %
(rsp if rsp else 'None'))
def verify_no_traffic_errors_on_client_side():
for vm in infra_utils.get_vm_of_type('client'):
vm.execute_command('rm -rf /tmp/httptest_io_*')
time.sleep(10)
resp = vm.execute_command('ls -ltr /tmp/httptest_io_error_* | wc -l')
logger.debug('response is %s' % resp)
if int(resp[0]) > 0:
logger_utils.fail('Errors are generated on client side %s' % resp)
def get_interface_mim_stats_for_vs(vs):
se = vs_get_primary_se_name(vs)
logger.debug('primary: %s' % se)
c = get_interface_mim_stats_for_se(se)
se_list = get_vs_secondary_se_list(vs)
for se in se_list:
logger.info('primary: %s' % se)
c += get_interface_mim_stats_for_se(se)
return c
def create_instance(self, wait=True, **kwargs):
""" Create an instance from AMI """
instance_type = self.vm_json.get('type')
networks = self.vm_json.get('networks')
offset = kwargs.pop('offset', 10)
version_tag = kwargs.pop('version_tag', None)
iam_role = None
if version_tag is None and instance_type in ['se', 'controller']:
logger_utils.fail(
'Please specify version tag of AMI for creating SE/Controller')
if instance_type == 'controller':
ami_name = 'Avi-Controller-%s' % version_tag
iam_role = kwargs.get('iam_role', None)
elif instance_type == 'se':
ami_name = 'Avi-SE-%s' % version_tag
elif instance_type == 'client':
ami_name = 'Jenkins Client'
elif instance_type == 'server':
ami_name = 'jenkins-server'
image_id = self._get_ami_id(ami_name)
sec_grps = self._get_sec_grp_ids(SEC_GROUP[instance_type])
interfaces = self._set_mgmt_interfaces(networks, offset, sec_grps)
try:
# http://boto.readthedocs.org/en/latest/ref/ec2.html#boto.ec2.connection.EC2Connection.run_instances
self.reservation = self.ec2.run_instances(
#placement=REGION + 'a',
image_id=image_id,
instance_type=INSTANCE_SIZE[instance_type],
network_interfaces=interfaces,
instance_profile_name=iam_role) #,
#**kwargs)
except Exception as e:
logger_utils.fail(e)
logger.info('Created reservation for instance: %s' % self.reservation)
if wait:
instance = self.reservation.instances[0]
logger_utils.asleep(delay=10)
logger.info('Wait until instance %s goes to running state' %
instance.id)
ip_addr = self._wait_until_instance_in_state(instance)
try:
vm_name = self.vm_json.get('name')
instance.add_tag('Name', vm_name)
instance.add_tag('Owner', vm_name)
wait_until_vm_ready([ip_addr])
logger.debug('Set data interfaces for instance: %s' %
instance.id)
#offset = self.__set_data_interfaces(
#instance, vm, networks, offset, sec_grps)
except Exception as e:
logger_utils.fail(e)
return instance, offset
def verify_cluster_mode(result, **kwargs):
controllers = infra_utils.get_vm_of_type('controller')
for node in result:
ip_addr = node['ip']['addr']
ctrl_ips = [ctrl.ip for ctrl in controllers]
if ip_addr not in ctrl_ips:
logger.debug(
'Cluster is not up in desired mode(ip). Could not find %s in controller ips %s'
% (ip_addr, ctrl_ips))
return True
return False
def set_workspace():
workspace = ''
thispath = os.path.realpath(__file__)
workspace = dirname(dirname(
dirname(thispath))) # $workspace/test/avitest/conftest.py
#workspace = subprocess.check_output(
# 'git rev-parse --show-toplevel', shell=True)
#workspace = workspace.strip(' \n\t')
if not workspace:
raise Exception('ERROR! Could not setup workspace')
logger.debug('Setting workspace:: %s' % workspace)
return workspace
def sdk_connect(self):
auth_url = self.configuration['auth_url']
user = self.configuration['username']
password = self.configuration['password']
logger.debug('password from config = %s' % password)
# FIXME read from rc
password = '******'
project_id = user
nova = NovaClient('2', user, password,
project_id, auth_url)
return nova
def check_return_code(request_type, rsp, should_pass, ignore_bad_code=False):
if ignore_bad_code and rsp.status_code >= 300:
print('Return bad response: %s - %s , Ignore set to %r' %
(rsp.status_code, rsp.content, ignore_bad_code))
return
logger.debug("%d:: %s :: %r" % (rsp.status_code, rsp.content, should_pass))
if rsp.status_code >= 300 and should_pass and not ignore_bad_code:
fail('%s return bad response: code %s, content %s' %
(request_type, rsp.status_code, rsp.content))
elif rsp.status_code < 300 and not should_pass and not ignore_bad_code:
fail('%s should have failed: code %s, content %s' %
(request_type, rsp.status_code, rsp.content))
def get_internal(obj_type,
obj_name,
core=0,
ret_all=False,
disable_aggregate=None,
**kwargs):
"""
:param obj_type:
:param obj_name:
:param core:
:param ret_all:
:param disable_aggregate:
:param kwargs:
:return:
"""
# REVIEW: re-examine where this should go
path = '/runtime/internal'
if disable_aggregate:
path += '?disable_aggregate=%s' % disable_aggregate
resp_code, resp_data = rest.get(obj_type,
name=obj_name,
path=path,
params=kwargs.get('params', {}))
if disable_aggregate or ret_all:
logger.debug('Requesting disable_aggregate, returning')
return resp_data
for json_data in resp_data:
proc_id_from_get_data = json_data.get('proc_id')
if not proc_id_from_get_data:
continue
if re.search('C' + str(core), proc_id_from_get_data):
return json_data
# Shared memory (not per core)
if obj_type == 'pool':
if re.search('so_pool', proc_id_from_get_data):
return json_data
else:
logger_utils.fail(
'ERROR! internal data NULL for %s proc_id: so_pool' %
obj_type)
logger_utils.fail('ERROR! internal data NULL for %s core %s' %
(path, str(core)))
def get_random_slave_vm():
""" Returns a list of VMs with current role of slave """
slave_vms = []
for vm in infra_utils.get_vm_of_type('controller'):
if suite_vars.dns == False:
vm_ip = vm.ip
else:
vm_ip = vm.name
role = get_node_role(vm_ip)
if role == 'CLUSTER_FOLLOWER':
slave_vms.append(vm)
logger.debug('Role of vm %s is %s' % (vm_ip, role))
if not slave_vms:
return None
return random.choice(slave_vms)
def poweroff(self, vm_name=None):
""" Power off openstack vm """
if not vm_name:
vm_name = self.vm_json.get('name')
vm = None
try:
vm = self.nova.servers.find(name='%s' % vm_name)
logger.info('vm state %s' %vm.status)
if vm.status == 'SHUTOFF':
logger.info('vm is already powered off, return')
return True
except Exception:
error("can't find vm %s in openstack, exp: %s" % (vm_name, e))
if vm:
logger.debug('Found vm: %s to poweroff' % vm_name)
vm.stop()
return (self.check_vm_status(vm_name, 'SHUTOFF'))
def wait_until_vm_is_up(self):
instance = self.reservation.instances[0]
logger_utils.asleep(delay=10)
logger.info('Wait until instance %s goes to running state' %
instance.id)
ip_addr = self._wait_until_instance_in_state(instance)
try:
vm_name = self.vm_json.get('name')
instance.add_tag('Name', vm_name)
instance.add_tag('Owner', vm_name)
wait_until_vm_ready([ip_addr])
logger.debug('Set data interfaces for instance: %s' % instance.id)
#offset = self.__set_data_interfaces(
#instance, vm, networks, offset, sec_grps)
except Exception as e:
logger_utils.fail(e)
return instance
def is_there_IO_error(client_range,
log_file='httptest_io_error*',
raise_exception=False):
"""
While traffic genearation IO errors are generally logged at
/tmp/httptest_<timestamp>.log. The function checks if
the log file is present or not.
"""
if isinstance(log_file, basestring):
logs = [log_file]
else:
logger_utils.fail(
'HttpTest failed. Error - Log file should be of type string, but got : %s'
% log_file)
for _log_file in logs:
logger.info('is_there_IO_error: %s\n' % log_file)
clients = get_clients_from_range(client_range)
vm, ip = traffic_manager.get_client_by_handle(clients[0])
logger.debug('VM IP, NAME, CLIENT: %s, %s, %s' % (vm.ip, vm.name, ip))
cmd = 'tail -5 %s' % log_file
resp = vm.execute_command(cmd)
if len(resp) > 0 and raise_exception:
error_msg = 'Get request failed\n'
for error in resp:
try:
msg = json.loads(error)
except Exception:
# When httptest fails, it doesn't write error log in json
# format.
logger_utils.error('HttpTest failed. Error - %s' % error)
error_msg += 'Client: %s\nValidation: %s\nExpected: %s\nActual: ' \
'%s\n\n' % (msg['client'], msg['error_code'],
msg['expected'], msg['actual'])
# Cleaning up before raising exception
vm.execute_command('rm %s &> /tmp/httptest' % log_file)
logger_utils.error(error_msg)
else:
if len(resp) == 0:
return 'False'
else:
logger.info('Failures: %s' % resp)
return 'True'
def get_node_role(ip, **kwargs):
logger.debug('get_node_role for ip %s' % ip)
nodes = []
nodes = cluster.get_node_config(**kwargs)
nodes_by_ip = {}
# AV-18689 this workaround is for the mesos AWS testbed,
# where the single cluster node has internal ip not matching the vm.ip
if len(nodes) == 1:
nodes_by_ip[ip] = nodes[0]
else:
for node in nodes:
nodes_by_ip[node['ip']['addr']] = node
@aretry(retry=10, delay=5, period = 5)
def get_cluster_runtime():
code, rsp = get('cluster', path='runtime')
return rsp
rsp = get_cluster_runtime()
try:
node_runtimes = rsp['node_states']
except:
logger.info('Error getting the cluster runtime. Got %s from the API' %
rsp)
fail('Error getting the cluster runtime. Got %s from '
'the API' % rsp)
if len(nodes_by_ip) != len(node_runtimes):
logger.info('There are %d configured nodes, but %d active nodes '
'in the cluster runtime.' %
(len(nodes_by_ip), len(node_runtimes)))
fail('There are %d configured nodes, but %d active '
'nodes in the cluster runtime.' %
(len(nodes_by_ip), len(node_runtimes)))
for node in node_runtimes:
if (node['name'] == nodes_by_ip[ip]['name'] or
node['name'] == nodes_by_ip[ip]['vm_uuid']):
logger.info('role for %s is %s' % (ip, node['role']))
return node['role']
fail('get_node_role: Could not find an active node with name '
'%s, so could not get its role' % ip)
def verify_persistence_entries(map):
if not map:
raise RuntimeError('Empty persistence map')
for key in map.keys():
for se, val in map.iteritems():
if key == se:
continue
if map[key] != val:
logger.info('Entries preset in %s and not in %s::' % (key, se))
logger.info('%s' % str(set(val.keys()) - set(map[key].keys())))
logger.info('Entries preset in %s and not in %s::' % (se, key))
logger.info('%s' % str(set(map[key].keys()) - set(val.keys())))
diff = [(k, val[k], v)
for k, v in map[key].iteritems() if k in val and val[k] != v]
logger.info('Client IPs having different server Ips::')
logger.info(diff)
logger.debug('Comparing: %s\nAND\n%s' % (map[key], val))
logger_utils.fail('Persistence entries do not match for %s and %s' % (key, se))
def retry_wait():
rsp = None
try:
status_code, rsp = get('cluster', path='runtime')
except Exception as ex:
fail('Cluster api runtime exception: %s' % ex)
if rsp:
cluster_state = rsp.get('cluster_state', {})
if (detailed_state and
detailed_state in cluster_state.get('reason')):
return True
elif (not detailed_state and 'CLUSTER_UP' in cluster_state.get('state', '')):
logger.info('Controller cluster is ready. It is in %s state' % cluster_state)
return True
elif ('CLUSTER_UP' in cluster_state.get('state', '') and
not 'HA_NOT_READY' in cluster_state.get('state', '')):
return True
logger.debug('cluster state[%s]: %s' % (rsp['node_states'], cluster_state.get('state', 'unknown')))
else:
fail('Cluster api runtime exception: no response.')
def configure_cluster(wait_time=600, **kwargs):
ignore_follower_reset = kwargs.pop('ignore_follower_reset', False)
logger.debug("::ignore_follower_reset:: %s" % ignore_follower_reset)
nodes = []
controllers = infra_utils.get_vm_of_type('controller')
ip_list = [vm.ip for vm in controllers]
leader_ip = ip_list[0]
ctrl_vm = None
for vm in controllers:
if vm.ip == leader_ip:
ctrl_vm = vm
for vm in controllers:
nodes.append({'ip': {'addr': vm.ip, 'type': 'V4'}, 'name': vm.ip})
if vm.ip != ctrl_vm.ip and not ignore_follower_reset:
make_follower_ready_for_cluster(vm, **kwargs)
cluster_obj = {'name': DEFAULT_CLUSTER_NAME, 'nodes': nodes}
logger.info("configure_cluster with nodes %s" % (nodes))
st, rsp = put('cluster', data=json.dumps(cluster_obj))
wait_until_n_cluster_nodes_ready(len(controllers),
wait_time=wait_time,
**kwargs)
def _set_mgmt_interfaces(self, networks, offset, sec_grps):
""" Return a NetworkInterfaceCollection object for an instance """
interfaces = []
logger.debug('networks: %s' % networks)
device_index = 0
vm_ip = self.vm_json.get('ip', None)
for key, network in networks.iteritems():
key_var = '${' + str(key) + '}'
subnet = self._get_subnet(network)
if key == 'management' or 'mgmt':
logger.debug('Adding management network on device: %s' %
device_index)
# Assumption: Everything is DHCP in management network.
logger.debug('subnet: %s, subnet.id: %s' % (subnet, subnet.id))
if vm_ip:
interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(
subnet_id=subnet.id,
groups=sec_grps,
device_index=device_index,
private_ip_address=vm_ip)
else:
interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(
subnet_id=subnet.id,
groups=sec_grps,
description='Mgmt',
device_index=device_index)
interfaces.append(interface)
interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(
*interfaces)
return interfaces
def vrf_add_ibgp_profile_peer(vrf_name, asnum, nw_name, md5):
"""
:param vrf_name:
:param asnum:
:param nw_name:
:param md5:
:return:
"""
json_data = vrf_get(vrf_name)
if 'bgp_profile' not in json_data:
json_data['bgp_profile'] = dict()
json_data['bgp_profile']['local_as'] = int(asnum)
json_data['bgp_profile']['ibgp'] = True
peer = dict()
peer['remote_as'] = int(asnum)
config = infra_utils.get_config()
import lib.network_lib as network_lib
peer_ip = network_lib.get_ip_for_last_octet(nw_name, '1')
peer['peer_ip'] = {'addr': peer_ip, 'type': 'V4'}
peer['subnet'] = {'ip_addr': {'addr': peer_ip, 'type': 'V4'},
'mask': int(network_lib.get_mask_for_network(nw_name))}
peer['network_ref'] = '/api/network?name=' + nw_name
peer['md5_secret'] = md5
peer['bfd'] = True
if 'peers' not in json_data['bgp_profile']:
json_data['bgp_profile']['peers'] = []
for peer_data in json_data['bgp_profile']['peers']:
if peer_data['peer_ip']['addr'] == peer_ip:
logger.debug('Peer %s already configured' % peer_ip)
return
json_data['bgp_profile']['peers'].append(peer)
rest.put('vrfcontext', name=vrf_name, data=json_data)
def vrf_del_ibgp_profile_peers(vrf_name, nw_names):
"""
:param vrf_name:
:param nw_names:
:return:
"""
json_data = vrf_get(vrf_name)
if 'bgp_profile' not in json_data:
logger.debug('No BGP Profile configured')
return
if 'peers' not in json_data['bgp_profile']:
logger.debug('No BGP Peers configured')
return
config = infra_utils.get_config()
import lib.network_lib as network_lib
for nw_name in nw_names:
peer_ip = network_lib.get_ip_for_last_octet(nw_name, '1')
for i, peer_data in enumerate(json_data['bgp_profile']['peers']):
if peer_data['peer_ip']['addr'] == peer_ip:
logger.debug('Peer %s found for delete' % peer_ip)
json_data['bgp_profile']['peers'].pop(i)
break
rest.put('vrfcontext', name=vrf_name, data=json_data)
def get_node_role(ip, **kwargs):
"""
:param ip:
:param kwargs:
:return:
"""
logger.debug('get_node_role for ip %s' % ip)
if rest.get_cloud_type() == 'gcp':
ctrl_vm = infra_utils.get_vm_of_type('controller')[0]
vm_public_ip = ctrl_vm.vm_public_ip
else:
vm_public_ip = ip
if not suite_vars.dns:
vm_public_ip = ip
else:
ctrl_vms = infra_utils.get_vm_of_type('controller')
for vm in ctrl_vms:
if ip == vm.ip:
vm_public_ip = vm.name
ip = vm.name
break
nodes = []
nodes = cluster.get_node_config(vm_public_ip, **kwargs)
nodes_by_ip = {}
# AV-18689 this workaround is for the mesos AWS testbed,
# where the single cluster node has internal ip not matching the vm.ip
if len(nodes) == 1:
nodes_by_ip[vm_public_ip] = nodes[0]
else:
for x in nodes:
nodes_by_ip[x['ip']['addr']] = x
for _ in xrange(10):
path = 'cluster/runtime'
try:
status_code, rsp = rest.get(path)
break
except:
logger.info('Could not get cluster runtime')
logger_utils.asleep(delay=5)
try:
node_runtimes = rsp['node_states']
except:
logger.info('Error getting the cluster runtime. Got %s from the API' %
rsp.text)
logger_utils.fail('Error getting the cluster runtime. Got %s from '
'the API' % rsp.text)
if len(nodes_by_ip) != len(node_runtimes):
logger.info('There are %d configured nodes, but %d active nodes '
'in the cluster runtime.' %
(len(nodes_by_ip), len(node_runtimes)))
logger.info('Waiting for 5 min to try and fix things...')
logger_utils.asleep(delay=60 * 5)
logger_utils.fail('There are %d configured nodes, but %d active '
'nodes in the cluster runtime.' %
(len(nodes_by_ip), len(node_runtimes)))
for node in node_runtimes:
try:
if (node['name'] == nodes_by_ip[ip]['name'] or
node['name'] == nodes_by_ip[ip]['vm_uuid']):
logger.info('role for %s is %s' % (ip, node['role']))
return node['role']
except Exception as e:
# Sometime get_node_config on dns name will returns nodes with ip when set to default
# And so the nodes_by_ip dict will have ip as keys instead of dns names
# And vice versa
if suite_vars.dns:
ctrl_vms = infra_utils.get_vm_of_type('controller')
for vm in ctrl_vms:
if ip == vm.ip:
ip = vm.name
break
elif ip == vm.name:
ip = vm.ip
break
if (node['name'] == nodes_by_ip[ip]['name'] or
node['name'] == nodes_by_ip[ip]['vm_uuid']):
logger.info('role for %s is %s' % (ip, node['role']))
return node['role']
logger_utils.fail('get_node_role: Could not find an active node with name '
'%s, so could not get its role' % ip)
