def wait_for_intelligent_scalein(vs_name):
"""
:param vs_name:
:return:
"""
_, vs_obj = rest.get('virtualservice', name=vs_name)
pool_ref = pool_lib._get_pool_from_vs(vs_obj)
pool_uuid = pool_ref.split('pool/')[1].split('#')[0]
_, pool_obj = rest.get('pool', uuid=pool_uuid)
asp_ref = pool_obj['autoscale_policy_ref']
as_policy_uuid = asp_ref.split('serverautoscalepolicy/')[1].split('#')[0]
_, autoscale_policy = rest.get('serverautoscalepolicy',
uuid=as_policy_uuid)
min_size = autoscale_policy['min_size']
# now wait for time such that num_servers == min_size
for _ in xrange(30):
_, pool_obj = rest.get('pool', uuid=pool_uuid)
num_servers = len(pool_obj['servers'])
if num_servers <= min_size:
break
logger_utils.asleep(delay=15)
if num_servers > min_size:
logger_utils.fail('scalein did not succeed pool %s' %
(str(pool_obj['servers'])))
def create_vm(self, **kwargs):
""" Create Azure VM with given build
"""
self.vm_deployment()
build_dir = kwargs.get('build_dir')
vm_name = self.vm_json.get('name')
# Delete VMs and NIcs if already exists
self.delete_instance(vm_name=vm_name, raise_error=False)
vhd_url, vhd_name = self.get_vhd_url(build_dir)
logger.info("Got VHD URL : %s \n VHD name: %s " % (vhd_url, vhd_name))
if not self.vhd_exists(vhd_name):
logger.info('Deleting previous controller vhd ..')
self.upload_vhd_to_azure(build_dir, vhd_name)
# vNetwork name + Subnet
controller_ip = self.create_nic(vm_name=vm_name)
nic_id = self.get_nic_id(vm_name=vm_name)
try:
result = self.compute_client.virtual_machines.create_or_update(self.resource_group, \
vm_name, self.vm_parameters(vhd_url, nic_id, vhd_name))
asleep(
msg='Creating Virtual in-progress .. internal wait is there.',
delay=10)
except Exception as e:
fail('Error while creating Controller Virtual Machine: %s' %
e.message)
result.wait()
logger.info('Controller : %s Created withIP address: %s' %
(vm_name, controller_ip))
def validate_vs_dns_deleted(vs_dns_name, retries=5, dns_vs_vip=None, **kwargs):
"""
:param vs_dns_name:
:param retries:
:param dns_vs_vip:
:param kwargs:
:return:
"""
if not dns_vs_vip:
logger.info("[SKIPPING] DNS check for VS as no DNS vip provided. Note,"
" controller based DNS is not supported anymore.")
return True
count = retries
while count:
ipl, portl = dns_get_ip_ports_for_fqdn(
dns_get_resolver(dns_vs_vip=dns_vs_vip), vs_dns_name)
if ipl or portl:
count -= 1
logger_utils.asleep(delay=5)
else:
return True
logger_utils.fail("Unexpected[%s]: DNS entries %s, %s found" %
(vs_dns_name, ipl, portl))
def retry_action_detail(action, retry_count=0, retry_interval=0.1):
"""
:param action:
:param retry_count:
:param retry_interval:
:return:
"""
if retry_interval <= 0:
logger_utils.fail('retry_interval <= 0 is not allowed, was: %s' %
retry_interval)
if retry_count < 0:
logger_utils.fail('retry_count < 0 is not allowed, was %s' %
retry_count)
retry_count = int(retry_count)
tries = 1
success, dbg_str = action()
if success:
return dbg_str
for x in xrange(1, retry_count):
tries += 1
success, dbg_str = action()
if success:
return dbg_str
else:
logger_utils.asleep(delay=retry_interval)
logger.trace('Last Retry result: %s' % dbg_str)
logger_utils.fail('%s, failed after %s tries' % (dbg_str, tries))
def main():
args = parse_arguments()
testbed_path = args.testbed_name
testbed_abspath, testbed_dir, testbed_name = __testbed_find(testbed_path)
logger.info("Cleanup Processing for testbed %s" %testbed_name)
try:
testbed_data = open(testbed_abspath).read()
tb_json = simplejson.loads(testbed_data)
logger.info('test_bed loaded')
except Exception as e:
abort('Json load failed for testbed file %s with Exception %s' % (testbed_abspath,e))
aws_clouds = get_aws_clouds_tb_json(tb_json)
if not aws_clouds:
abort("No AWS Clouds Found. Aborting")
for aws_cloud in aws_clouds:
cloud_name = aws_cloud.get('name')
sdk_conn = Aws(cloud_configuration_json= aws_cloud.get('aws_configuration'))
logger.info("AWS SDK connection successfull for cloud %s" %cloud_name)
filters = {'tag-key':'avitest_tb_tag', 'tag-value': testbed_name}
terminate_instances(sdk_conn, filters)
asleep(msg="Settling with Instances delete" , delay = 10)
delete_images(sdk_conn, filters)
delete_network_interfaces(sdk_conn, filters)
asleep(msg="Settling with Network Interfaces Deletion" , delay = 30)
delete_security_groups(sdk_conn, filters)
sdk_conn.disconnect()
clean_secondary_ips_on_client_server(tb_json)
def check_cloud_state(expected_status='CLOUD_STATE_PLACEMENT_READY', **kwargs):
cloud_name = kwargs.get('cloud_name', None)
# config = get_config()
# ctrl_clouds = kwargs.get('clouds', config.testbed[config.site_name].cloud)
asleep(msg='waiting for cloud state', delay=10)
status_code, resp_json = get('cloud-inventory')
#resp_json = resp.json()
#if len(ctrl_clouds) != int(resp_json['count']):
# error("Number of Configured Clouds not as Received. Configured=%s Received=%s" %(len(ctrl_clouds), resp_json['count']))
for cloud_obj in resp_json['results']:
if cloud_name and cloud_name != cloud_obj['config']['name']:
continue
if 'error' in cloud_obj['status']:
error('Received Error in cloud status %s' %
cloud_obj['status']['error'])
cloud_status = cloud_obj['status']['state']
last_reason = cloud_obj['status'].get('reason', '')
# To handle special? case where cloud is up but about to be reconfigured
# REVIEW any other reasons that we need to account for?
if cloud_status != expected_status or 'Pending re-config' in last_reason:
if cloud_obj['config']['vtype'] == 'CLOUD_AWS':
asleep("additional delay for AWS cloud", delay=30)
error(
'Cloud Status is not as expected or reason not null. Expected=%s Received=%s, reason = %s',
expected_status, cloud_status, last_reason)
return True
def validate_vs_dns_info(vs_name, retries=5, **kwargs):
"""
:param vs_name:
:param retries:
:param kwargs:
:return:
"""
dns_vs_vip = kwargs.get('dns_vs_vip', '')
if not dns_vs_vip:
logger.info(
"[SKIPPING] DNS check for VS as no DNS vip provided. Note, "
"controller based DNS is not supported anymore.")
return True
import lib.vs_lib as vs_lib
vs_json = vs_lib.get_vs(vs_name, tenant=kwargs.get('tenant', 'admin'))
if vs_json['type'] == 'VS_TYPE_VH_CHILD':
if rest.get_cloud_type() != 'openshift':
logger.info("[SKIPPING] DNS check for VS as SNI child are not "
"currently supported for non-openshift clouds")
return True
parent_ref = vs_json['vh_parent_vs_ref']
parent_uuid = parent_ref.split('/')[-1]
_, parent_vs = rest.get('virtualservice', uuid=parent_uuid)
parent_vs_name = parent_vs['name']
logger.info('SNI child VS detected; doing DNS on parent VS %s' %
parent_vs_name)
child_fqdn = vs_json['vh_domain_name']
parent_fqdns = [t['fqdn'] for t in parent_vs['dns_info']]
if child_fqdn not in parent_fqdns:
return False
vs_name = parent_vs_name # REVIEW should it be parent or child name?
vs_json = parent_vs
dns_name = child_fqdn
else:
dns_name = vs_json['ipam_dns_records'][0]['fqdn']
logger.trace('vs_json: %s' % vs_json)
if 'floating_ip' in vs_json:
ip = vs_json['vip'][0]['floating_ip']['addr']
else:
ip = vs_json['vip'][0]['ip_address']['addr']
ports = sorted([srv['port'] for srv in vs_json['services']])
logger.info("VS [%s]: IP %s, DNS %s, Ports: %s" %
(vs_name, ip, dns_name, ports))
count = retries
while count:
if vs_lib.vs_check_ip_ports(vs_name,
ip,
dns_name,
ports,
dns_vs_vip=dns_vs_vip):
return True
count -= 1
logger_utils.asleep(delay=5)
logger_utils.fail("DNS check failed!!")
def set_key_rotate_period(**kwargs):
try:
set_controller_properties(**kwargs)
except Exception as e:
if kwargs.get('should_pass', True):
logger_utils.fail("set_controller_properties should fail")
return True # do not sleep if expected error is caught
logger_utils.asleep(msg='wait', delay=61) # sleeping for 1 min, so that
def wait_for_scale_set_success(ss_name):
"""
:param ss_name: Azure scale set name
"""
asleep('Dummy waiting since scale set autoscaling does not take effect instantly', delay=10)
cloud_type, configuration = rest.get_cloud_type(get_configuration = True)
azure = Azure(configuration, 'Default-Cloud')
azure.check_scale_set_status(ss_name)
def autoscale_lower_max_size(pool_name, **kwargs):
"""
pass the autoscale policy settings into the kwargs
:param pool_name:
:param kwargs:
:return:
"""
_, pool_obj = rest.get('pool', name=pool_name)
asp_ref = pool_obj['autoscale_policy_ref']
as_policy_uuid = asp_ref.split('serverautoscalepolicy/')[1].split('#')[0]
_, autoscale_policy = rest.get('serverautoscalepolicy',
uuid=as_policy_uuid)
logger.info('received asp %s type %s' %
(autoscale_policy, type(autoscale_policy)))
orig_max_size = autoscale_policy['max_size']
orig_min_size = autoscale_policy['min_size']
num_servers = len(pool_obj['servers'])
for k, v in kwargs.iteritems():
logger.info('k,v %s,%s' % (k, v))
autoscale_policy[k] = v
if num_servers < 2:
logger_utils.fail('Number of servers is less than required %d' %
num_servers)
autoscale_policy['max_size'] = num_servers - 1
autoscale_policy['min_size'] = min(autoscale_policy['min_size'],
autoscale_policy['max_size'])
asp_json = json.dumps(autoscale_policy)
logger.info('json: %s' % asp_json)
rc, result = rest.put('serverautoscalepolicy',
uuid=as_policy_uuid,
data=asp_json)
logger.info('updating as_policy %s' % autoscale_policy)
as_info = get_autoscale_info(pool_name)
assert as_info
logger_utils.asleep(delay=AS_WAIT_TIME)
for _ in xrange(12):
logger_utils.asleep(delay=10)
_, pool_obj = rest.get('pool', name=pool_name)
new_num_servers = len(pool_obj['servers'])
if new_num_servers <= num_servers:
break
_, autoscale_policy = rest.get('serverautoscalepolicy',
uuid=as_policy_uuid)
autoscale_policy['max_size'] = orig_max_size
autoscale_policy['min_size'] = orig_min_size
asp_json = json.dumps(autoscale_policy)
rc, result = rest.put('serverautoscalepolicy',
uuid=as_policy_uuid,
data=asp_json)
logger.info('json: %s rc: %s results: %s' % (asp_json, rc, result))
return autoscale_policy['max_size']
def stop_controller_process(vm, process_name, update_pids=True):
import lib.cluster_lib as cluster_lib
update_pids = (update_pids or update_pids == 'True')
proc = process_name
logger.info('Stopping process %s on controller with IP %s' % (proc, vm.ip))
vm.stop_upstart_job(proc)
if update_pids:
logger_utils.asleep(delay=5)
cluster_lib.wait_until_cluster_ready()
cluster_lib.update_processes_for_all_controllers()
def create_instance(self, wait=True, **kwargs):
""" Create an instance from AMI """
instance_type = self.vm_json.get('type')
networks = self.vm_json.get('networks')
offset = kwargs.pop('offset', 10)
version_tag = kwargs.pop('version_tag', None)
iam_role = None
if version_tag is None and instance_type in ['se', 'controller']:
logger_utils.fail(
'Please specify version tag of AMI for creating SE/Controller')
if instance_type == 'controller':
ami_name = 'Avi-Controller-%s' % version_tag
iam_role = kwargs.get('iam_role', None)
elif instance_type == 'se':
ami_name = 'Avi-SE-%s' % version_tag
elif instance_type == 'client':
ami_name = 'Jenkins Client'
elif instance_type == 'server':
ami_name = 'jenkins-server'
image_id = self._get_ami_id(ami_name)
sec_grps = self._get_sec_grp_ids(SEC_GROUP[instance_type])
interfaces = self._set_mgmt_interfaces(networks, offset, sec_grps)
try:
# http://boto.readthedocs.org/en/latest/ref/ec2.html#boto.ec2.connection.EC2Connection.run_instances
self.reservation = self.ec2.run_instances(
#placement=REGION + 'a',
image_id=image_id,
instance_type=INSTANCE_SIZE[instance_type],
network_interfaces=interfaces,
instance_profile_name=iam_role) #,
#**kwargs)
except Exception as e:
logger_utils.fail(e)
logger.info('Created reservation for instance: %s' % self.reservation)
if wait:
instance = self.reservation.instances[0]
logger_utils.asleep(delay=10)
logger.info('Wait until instance %s goes to running state' %
instance.id)
ip_addr = self._wait_until_instance_in_state(instance)
try:
vm_name = self.vm_json.get('name')
instance.add_tag('Name', vm_name)
instance.add_tag('Owner', vm_name)
wait_until_vm_ready([ip_addr])
logger.debug('Set data interfaces for instance: %s' %
instance.id)
#offset = self.__set_data_interfaces(
#instance, vm, networks, offset, sec_grps)
except Exception as e:
logger_utils.fail(e)
return instance, offset
def terminate_instance(self, instance_name):
""" Terminate instance on AWS """
self.sdk_connect()
instance_id = self._get_instance_id(instance_name)
logger.info('Terminate instance: %s', instance_name)
self.ec2.terminate_instances(instance_ids=[instance_id])
# TODO: Check instance state as Terminate and should release IP
logger_utils.asleep(
'Sleeping for instance to terminate and release IP',
delay=60,
period=20)
def disconnect_zk(vm, duration):
""" Blocks the port used by zookeeper. Use duration = -1 to indefinitely
block the port """
vm.block_port_range(5000, 5097)
duration = float(duration)
if duration > 0:
logger_utils.asleep(delay=duration)
vm.clear_iptables()
logger_utils.asleep(delay=120)
cluster.wait_until_n_cluster_nodes_ready(
len(infra_utils.get_vm_of_type('controller')))
update_processes_for_all_controllers_and_ses()
def delete_dns_system_configuration():
"""
:return:
"""
status_code, response = rest.get('systemconfiguration')
response['dns_virtualservice_uuids'] = []
if 'dns_virtualservice_refs' in response:
del response['dns_virtualservice_refs']
rest.put('systemconfiguration', data=response)
logger_utils.asleep(delay=5)
def wait_for_next_hs_computation(now=None, step=300, step_times=1):
"""
:param now:
:param step:
:param step_times:
:return:
"""
time_to_wait = hs_next_computation(now, step, int(step_times))
logger.info('Sleeping for hs computation for %s secs' % time_to_wait)
logger_utils.asleep(delay=time_to_wait)
return
def warm_restart_cluster(wait=True):
ctrl_vm = get_cluster_master_vm()
path = os.path.join('http://localhost:%d' % (PROC_SUPERVISOR_PORT),
'service_event?status=warm-restart')
cmd_str = 'curl %s' % path
rsp = ctrl_vm.execute_command(cmd_str)
if wait is True:
logger_utils.asleep(delay=20)
wait_until_cluster_ready()
wait_until_res_mgr_coldstart_complete()
#update_processes_for_all_controllers()
return str(rsp[0])
def wait_until_vm_is_up(self, vm_name=None, timeout=3600):
if not vm_name:
vm_name = self.vm_json.get('name')
vcenter_handle = self.server
num_loops = int(timeout) / 30
i = 0
while i < num_loops:
vm = vcenter_handle.get_vm_by_name(vm_name)
props = vm.get_properties()
if 'net' in props.keys():
logger.info('vm network info: %s' % (props.get('net')))
return
asleep(msg='Waiting for vm to come up', delay=30)
i += 1
def test_traffic_1(self):
# traffic_manager.create_clients(1, 'c', 'net1', 'httperf')
msvc_map = mesos_lib.create_erdos_renyi_graph(
self.app_prefix,
self.num_apps,
None,
self.num_edges,
ip_client='c1',
northsouth=self.northsouth)
mesos_lib.generate_microservice_traffic(msvc_map, load=1)
start_time = mesos_lib.generate_microservice_traffic(msvc_map)
logger_utils.asleep(delay=15)
mesos_lib.validate_microservice_traffic(msvc_map, start_time)
def autoscale_raise_min_size(pool_name, **kwargs):
"""
pass the autoscale policy settings into the kwargs
:param pool_name:
:param kwargs:
:return:
"""
_, pool_obj = rest.get('pool', name=pool_name)
asp_ref = pool_obj['autoscale_policy_ref']
as_policy_uuid = asp_ref.split('serverautoscalepolicy/')[1].split('#')[0]
_, autoscale_policy = rest.get('serverautoscalepolicy',
uuid=as_policy_uuid)
logger.info('received asp %s type %s ' %
(autoscale_policy, type(autoscale_policy)))
as_policy_old = copy.deepcopy(autoscale_policy)
orig_min_size = autoscale_policy['min_size']
orig_max_size = autoscale_policy['max_size']
num_servers = len(pool_obj['servers'])
for k, v in kwargs.iteritems():
logger.info('k,v %s,%s' % (k, v))
autoscale_policy[k] = v
autoscale_policy['min_size'] = num_servers + 1
autoscale_policy['max_size'] = max(autoscale_policy['max_size'],
autoscale_policy['min_size'])
asp_json = json.dumps(autoscale_policy)
logger.info(' json: %s' % asp_json)
rc, result = rest.put('serverautoscalepolicy',
uuid=as_policy_uuid,
data=asp_json)
logger.info('updating as_policy %s %s %s' % (autoscale_policy, rc, result))
logger_utils.asleep(delay=AS_WAIT_TIME)
get_autoscale_info(pool_name)
_, pool_obj = rest.get('pool', name=pool_name)
num_servers = len(pool_obj['servers'])
if num_servers == 0:
logger_utils.fail('Pool %s has no up servers' % pool_name)
_, autoscale_policy = rest.get('serverautoscalepolicy',
uuid=as_policy_uuid)
autoscale_policy['min_size'] = orig_min_size
autoscale_policy['max_size'] = orig_max_size
asp_json = json.dumps(autoscale_policy)
logger.info('json: %s' % asp_json)
rc, result = rest.put('serverautoscalepolicy',
uuid=as_policy_uuid,
data=asp_json)
logger.info('rc: %s result: %s' % (rc, result))
return autoscale_policy['min_size']
def wait_for_server_count_change(vs_name, prev_count, expected_op='>'):
"""
:param vs_name:
:param prev_count:
:param expected_op:
:return:
"""
for _ in xrange(20):
curr_count = vs_lib.get_vs_pool_server_count(vs_name)
if eval('%d %s %d' % (curr_count, expected_op, prev_count)):
return
logger_utils.asleep(delay=15)
logger_utils.fail('vs %s server count did not change prev %s curr %s' %
(vs_name, prev_count, curr_count))
def reboot_clean(update_admin_info=True, **kwargs):
post('cluster', path='reboot', data=json.dumps({'mode': 'REBOOT_CLEAN'}))
asleep(msg="Sleep before cluster wait check", delay=120)
wait_until_cluster_ready()
set_sysadmin_public_key()
set_systemconfiguration()
set_ha_mode_best_effort()
if update_admin_info:
config = AviConfig.get_instance()
mode = config.get_mode()
logger.debug("Current Default Mode %s" % mode)
username = mode['user']
password = '******'
mode["password"] = '******'
update_admin_user(username=username, password=password)
switch_mode(**mode)
def delete_vm(self, vm_name=None, raise_error=True):
""" Delete VM instance """
if not vm_name:
vm_name = self.vm_json.get('name')
try:
result = self.compute_client.virtual_machines.delete(
self.resource_group, vm_name)
asleep(msg='Delete Virtual in-progress .. internal wait is there.',
delay=10)
result.wait()
except Exception as e:
logger.info("Got Exception while deleting VM: %s , Exp: %s" %
(vm_name, e.message))
if raise_error:
fail('Error while Deleting Controller Virtual Machine: %s',
str(e))
def test_token_authenticated_traffic(self):
# traffic_manager.create_clients(1, 'c', 'net1', 'httperf')
msvc_map = mesos_lib.create_erdos_renyi_graph(
'auth1',
self.num_apps,
None,
self.num_edges,
ip_client='c1',
northsouth=self.northsouth)
mesos_lib.generate_microservice_traffic(msvc_map,
auth_type='token',
load=1)
start_time = mesos_lib.generate_microservice_traffic(msvc_map,
auth_type='token')
logger_utils.asleep(delay=15)
mesos_lib.validate_microservice_traffic(msvc_map, start_time)
def wait_until_vm_is_up(self):
instance = self.reservation.instances[0]
logger_utils.asleep(delay=10)
logger.info('Wait until instance %s goes to running state' %
instance.id)
ip_addr = self._wait_until_instance_in_state(instance)
try:
vm_name = self.vm_json.get('name')
instance.add_tag('Name', vm_name)
instance.add_tag('Owner', vm_name)
wait_until_vm_ready([ip_addr])
logger.debug('Set data interfaces for instance: %s' % instance.id)
#offset = self.__set_data_interfaces(
#instance, vm, networks, offset, sec_grps)
except Exception as e:
logger_utils.fail(e)
return instance
def test_traffic_with_security_policy(self):
# traffic_manager.create_clients(1, 'c', 'net1', 'httperf')
adj_list = []
blocked = True
mesos_lib.add_edge(adj_list, 't2-1', 't2-2', blocked=blocked)
mesos_lib.add_edge(adj_list, 't2-1', 't2-3')
mesos_lib.add_edge(adj_list, 't2-1', 't2-4')
mesos_lib.add_edge(adj_list, 't2-2', 't2-1', blocked=blocked)
mesos_lib.add_edge(adj_list, 't2-2', 't2-4')
mesos_lib.add_edge(adj_list, 't2-3', 't2-1', blocked=blocked)
mesos_lib.add_edge(adj_list, 't2-4', 't2-2')
mesos_lib.add_edge(adj_list, 't2-5', 't2-2')
msvc_map = mesos_lib.create_microservice_map(adj_list)
mesos_lib.generate_microservice_traffic(msvc_map, load=1)
start_time = mesos_lib.generate_microservice_traffic(msvc_map)
logger_utils.asleep(delay=15)
mesos_lib.validate_microservice_traffic(msvc_map, start_time)
def test_check_realtime_traffic(self):
# traffic_manager.create_clients(1, 'c', 'net1', 'httperf')
mesos_lib.set_mesos_rt_collection(rt_flag=True)
# sleep for the default collection period until the new period kicks in
logger_utils.asleep(delay=60)
# Generate traffic
adj_list = []
mesos_lib.add_edge(adj_list, 'c1', 'ma1-1', src_type='vm', load=20000)
msvc_map = mesos_lib.create_microservice_map(adj_list)
start_time = mesos_lib.generate_microservice_traffic(msvc_map,
load=20000)
logger_utils.asleep(delay=5)
ma1_1_pool = mesos_lib.get_pool_name_with_tenant('ma1-1')
metrics_lib.metrics_check_poolvm_container(ma1_1_pool,
step=5,
limit=12,
mbaseline=0)
mesos_lib.set_mesos_rt_collection(rt_flag=False)
def test_multiple_marathon_traffic(self):
adj_list = []
mesos_lib.add_edge(adj_list, 'mara1-1', 'mara2-1')
mesos_lib.add_edge(adj_list, 'mara1-3', 'mara2-2')
mesos_lib.add_edge(adj_list, 'mara1-2', 'mara2-3')
mesos_lib.add_edge(adj_list, 'mara2-3', 'mara1-1')
mesos_lib.add_edge(adj_list, 'mara2-2', 'mara1-2')
mesos_lib.add_edge(adj_list, 'mara2-1', 'mara1-3')
mesos_lib.add_edge(adj_list, 'c1', 'mara1-1', src_type='vm')
mesos_lib.add_edge(adj_list, 'c1', 'mara2-1', src_type='vm')
msvc_map = mesos_lib.create_microservice_map(adj_list)
mesos_lib.generate_microservice_traffic(msvc_map,
auth_type='hybrid',
load=1)
start_time = mesos_lib.generate_microservice_traffic(
msvc_map, auth_type='hybrid')
logger_utils.asleep(delay=15)
mesos_lib.validate_microservice_traffic(msvc_map, start_time)
def create_nic(self, vm_name=None):
""" Create Network interface """
if not vm_name:
vm_name = self.vm_json.get('name')
controller_ip = self.get_ip_from_azure()
nic_name = "%s-NIC" % vm_name
try:
nic = self.network_client.network_interfaces.create_or_update(
self.resource_group,
nic_name,
parameters=self.get_nic_params(controller_ip))
asleep(
msg='Creating NIC in-progress .. internal wait is there ...',
delay=10)
nic.wait()
return controller_ip
except Exception as e:
fail('Error while creating a NIC %s' % str(e))
def verify_traffic_can_flow(vs_name, vport, num_conns, retry=5):
"""
:param vs_name:
:param vport:
:param num_conns:
:param retry:
:return:
"""
num_connections = int(num_conns)
retry = int(retry)
try:
tcpstat_pre = tcp_lib.tcpstat(vs_name)
except Exception:
logger_utils.fail('tcpstat not got before flow starts')
request('get',
vs_name,
vport,
'/',
concurrent_conn_per_client=num_connections,
skip_exception=1)
time.sleep(3)
# retry another 10 sec to wait for connection finished
while retry > 0:
try:
tcpstat_post = tcp_lib.tcpstat(vs_name)
except Exception:
logger_utils.fail('tcpstat not got after flow starts')
tcps_num_conns = json_utils.json_diff(tcpstat_post, tcpstat_pre,
'connections_closed')
logger.info('conn diff:', tcps_num_conns, '--- expected:',
str(num_connections * 2))
if tcps_num_conns < num_connections * 2:
logger.info('Not all connections went through. Retrying..')
retry -= 1
logger_utils.asleep(delay=10)
continue
else:
break
if retry <= 0:
logger_utils.fail('Not all connections went through!')
