def test_multiLB(self):
""" test of multiple lb's per haproxy device
"""
# wait until our lb is ACTIVE before trying to update it
# test / validate lb1
lbaas_utils.wait_for_active_status(self, self.lb_id1)
lbaas_utils.validate_loadBalancer( self
, multi=True
, multi_id=self.lb_id1
, multi_name=self.lb_name1
, multi_nodes=self.nodes1)
result_data = self.driver.list_lb_detail(self.lb_id1)
if 'virtualIps' in result_data:
self.vip = result_data['virtualIps'][0]['id']
else: # client has different key
self.vip = ast.literal_eval(result_data['ips'])[0]['id']
# create lb2
self.logging.info('Creating load balancer2...')
self.create_result, self.actual_status, self.lb_id2, self.lb_addr = self.driver.create_lb(self.lb_name2, self.nodes2, self.algorithm, self.bad_statuses, self.vip)
if self.args.verbose:
self.logging.info("STATUS: %s" %self.actual_status)
self.logging.info("RESULT: %s" %self.create_result.text)
lbaas_utils.wait_for_active_status(self, self.lb_id2)
self.logging.info('load balancer2 id: %s' %(self.lb_id2))
lbaas_utils.validate_loadBalancer( self
, multi=True
, multi_id=self.lb_id2
, multi_name=self.lb_name2
, multi_nodes=self.nodes2)
def test_multiLB(self):
""" test of multiple lb's per haproxy device
"""
# wait until our lb is ACTIVE before trying to update it
# test / validate lb1
lbaas_utils.wait_for_active_status(self, self.lb_id1)
lbaas_utils.validate_loadBalancer(self,
multi=True,
multi_id=self.lb_id1,
multi_name=self.lb_name1,
multi_nodes=self.nodes1)
result_data = self.driver.list_lb_detail(self.lb_id1)
if 'virtualIps' in result_data:
self.vip = result_data['virtualIps'][0]['id']
else: # client has different key
self.vip = ast.literal_eval(result_data['ips'])[0]['id']
# create lb2
self.logging.info('Creating load balancer2...')
self.create_result, self.actual_status, self.lb_id2, self.lb_addr = self.driver.create_lb(
self.lb_name2, self.nodes2, self.algorithm, self.bad_statuses,
self.vip)
if self.args.verbose:
self.logging.info("STATUS: %s" % self.actual_status)
self.logging.info("RESULT: %s" % self.create_result.text)
lbaas_utils.wait_for_active_status(self, self.lb_id2)
self.logging.info('load balancer2 id: %s' % (self.lb_id2))
lbaas_utils.validate_loadBalancer(self,
multi=True,
multi_id=self.lb_id2,
multi_name=self.lb_name2,
multi_nodes=self.nodes2)
def test_Monitoring(self):
""" multiple clients beat on a loadbalancer
"""
# Create our loadbalancer
if not self.args.lbid:
self.create_result, self.actual_status, self.lb_id, self.lb_addr = self.driver.create_lb(self.lb_name, self.nodes, self.algorithm, self.bad_statuses)
self.logging.info('load balancer id: %s' %self.lb_id)
self.logging.info('load balancer ip addr: %s' %self.lb_addr)
lbaas_utils.validate_loadBalancer(self)
else:
self.logging.info("Using user-supplied loadbalancer: %s" %self.args.lbid)
self.lb_id = self.args.lbid
self.lb_addr = self.get_floating_ip(self.lb_id)
# validate the loadbalancer
lbaas_utils.validate_loadBalancer(self)
self.logging.info("Getting monitor data for default monitor...")
output, status = self.driver.get_monitor(self.lb_id)
self.assertEqual(status,'200',msg="ERROR: problem w/ loadbalancer: %s monitor. Received status: %s: %s" %(self.lb_id, status, output))
self.assertEqual(output, self.default_monitor, msg="ERROR: problem with default monitor. Expected: %s, Actual: %s" %(self.default_monitor, output))
# updating the monitor...
for monitor_set in self.monitor_data:
description = monitor_set['description']
monitor = monitor_set['monitor']
self.logging.info("Testing monitor variant: %s..." %description)
output, status = self.driver.update_monitor(self.lb_id, monitor)
self.logging.info("\tStatus: %s || output: %s" %(status, output))
current_monitor, current_status = self.driver.get_monitor(self.lb_id)
if status == '202': # we have a positive result!
self.assertEqual(current_monitor, monitor, msg= "Loadbalancer: %s monitor not matching expected value: Expected: %s || Actual %s" %(self.lb_id, monitor, current_monitor))
self.logging.info("\tTest monitor: %s" %monitor)
self.logging.info("\tCurrent monitor: %s" %current_monitor)
lbaas_utils.wait_for_active_status(self)
def test_monitorLoadBalancer(self):
""" test monitoring of loadbalancers for libra
"""
# Create our loadbalancer
self.create_result, self.actual_status, self.lb_id, self.lb_addr = self.driver.create_lb(self.lb_name, self.nodes, self.algorithm, self.bad_statuses)
self.logging.info('load balancer id: %s' %self.lb_id)
self.logging.info('load balancer ip addr: %s' %self.lb_addr)
lbaas_utils.validate_loadBalancer(self)
# use fabric to stop libra_worker
self.logging.info("Stopping libra_worker on lb: %s address: %s" %(self.lb_id, self.lb_addr))
cmd = "fab --no-pty -H %s stop_libra_worker" %(self.lb_addr)
status, output = commands.getstatusoutput(cmd)
#if self.args.verbose:
self.logging.info("Command: %s" %cmd)
self.logging.info("Status: %s" %status)
self.logging.info("Output: %s" %output)
# check api to see if ERROR state is set / libra_worker failure is detected
self.logging.info("Wait / poll api server for loadbalancer to be set to ERROR status...")
lbaas_utils.wait_for_active_status(self, self.lb_id, active_wait_time=240, desired_status='ERROR')
# restart libra_worker
self.logging.info("Starting libra_worker on lb: %s address: %s" %(self.lb_id, self.lb_addr))
cmd = "fab --no-pty -H %s start_libra_worker" %(self.lb_addr)
status, output = commands.getstatusoutput(cmd)
#if self.args.verbose:
self.logging.info("Command: %s" %cmd)
self.logging.info("Status: %s" %status)
self.logging.info("Output: %s" %output)
# check api to ensure lb state is properly reset
self.logging.info("Wait / poll api server for loadbalancer to be set to ACTIVE status...")
lbaas_utils.wait_for_active_status(self, self.lb_id, active_wait_time=240, desired_status='ACTIVE')
# one final lb validation
lbaas_utils.validate_loadBalancer(self)
def test_modifyNodes(self):
""" test modifying loadbalancer nodes for libra
"""
# wait until our lb is ACTIVE before trying to update it
lbaas_utils.wait_for_active_status(self)
# modify/disable a node on our loadbalancer
self.logging.info("Testing node disable...")
nodes = self.driver.list_lb_nodes(self.lb_id)
mod_node = nodes['nodes'][0]
mod_node_id = mod_node['id']
mod_node_addr = mod_node['address']
mod_node_data = {'condition':'DISABLED'}
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
lbaas_utils.validate_loadBalancer(self, [mod_node_addr])
# re-enable the node
self.logging.info("Testing re-enable of node...")
mod_node_data = {'condition':'ENABLED'}
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
lbaas_utils.validate_loadBalancer(self)
# modify ip / address
self.logging.info("Testing update of node ip...")
mod_node_data = {'address': '127.0.0.1'}
expected_status = '400'
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
self.assertEqual(str(self.actual_status), expected_status, msg = "ERROR: Attempt to update node ip address succeeded with status: %s. Expected status: %s" %(self.actual_status, expected_status))
# modify port
self.logging.info("Testing update of node port...")
mod_node_data = {'port': '443'}
expected_status = '400'
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
self.assertEqual(str(self.actual_status), expected_status, msg = "ERROR: Attempt to update node port succeeded with status: %s. Expected status: %s" %(self.actual_status, expected_status))
# update of non-existent node
self.logging.info("Testing update of non-existent node...")
mod_node_data = {'condition':"DISABLED"}
expected_status = '404'
self.actual_status = self.driver.modify_node(self.lb_id, '0', mod_node_data)
self.assertEqual(str(self.actual_status), expected_status, msg = "ERROR: Attempt to update node port succeeded with status: %s. Expected status: %s" %(self.actual_status, expected_status))
# lower-case condition
#self.logging.info("Testing lowercase condition...")
#mod_node_data = {'condition':'disabled'}
#self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
#lbaas_utils.validate_loadBalancer(self, [mod_node_addr])
#self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
#lbaas_utils.validate_loadBalancer(self, [mod_node_addr])
# lower-case condition
self.logging.info("Testing bad condition...")
mod_node_data = {'condition':'TASERED_BADLY'}
expected_statuses = ['400','512']
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
self.assertTrue(str(self.actual_status) in expected_statuses, msg = "ERROR: Attempt to update to bad condition succeeded with status: %s. Expected status: %s" %(self.actual_status, expected_status))
def tearDown(self):
##########################
# delete the load balancer
##########################
self.logging.info("Deleting loadbalancer: %s" %self.lb_id)
lbaas_utils.wait_for_active_status(self, active_wait_time=180)
if self.args.cleanupoff:
self.logging.info("NOT deleting loadbalancer: %s per user-specified flag..." %self.lb_id)
else:
self.logging.info("Deleting loadbalancer: %s" %self.lb_id)
result = self.driver.delete_lb(self.lb_id)
def test_updateLoadBalancer(self):
""" test update of loadbalancers for libra
"""
# wait until our lb is ACTIVE before trying to update it
lbaas_utils.wait_for_active_status(self)
# update our loadbalancer
self.actual_status = self.driver.update_lb(self.lb_id, self.update_data)
if 'name' in self.update_data:
self.lb_name = self.update_data['name']
lbaas_utils.validate_loadBalancer(self)
def test_updateLoadBalancer(self):
""" test update of loadbalancers for libra
"""
# wait until our lb is ACTIVE before trying to update it
lbaas_utils.wait_for_active_status(self)
# update our loadbalancer
self.actual_status = self.driver.update_lb(self.lb_id,
self.update_data)
if 'name' in self.update_data:
self.lb_name = self.update_data['name']
lbaas_utils.validate_loadBalancer(self)
def test_Monitoring(self):
""" multiple clients beat on a loadbalancer
"""
# Create our loadbalancer
if not self.args.lbid:
self.create_result, self.actual_status, self.lb_id, self.lb_addr = self.driver.create_lb(
self.lb_name, self.nodes, self.algorithm, self.bad_statuses)
self.logging.info('load balancer id: %s' % self.lb_id)
self.logging.info('load balancer ip addr: %s' % self.lb_addr)
lbaas_utils.validate_loadBalancer(self)
else:
self.logging.info("Using user-supplied loadbalancer: %s" %
self.args.lbid)
self.lb_id = self.args.lbid
self.lb_addr = self.get_floating_ip(self.lb_id)
# validate the loadbalancer
lbaas_utils.validate_loadBalancer(self)
self.logging.info("Getting monitor data for default monitor...")
output, status = self.driver.get_monitor(self.lb_id)
self.assertEqual(
status,
'200',
msg=
"ERROR: problem w/ loadbalancer: %s monitor. Received status: %s: %s"
% (self.lb_id, status, output))
self.assertEqual(
output,
self.default_monitor,
msg="ERROR: problem with default monitor. Expected: %s, Actual: %s"
% (self.default_monitor, output))
# updating the monitor...
for monitor_set in self.monitor_data:
description = monitor_set['description']
monitor = monitor_set['monitor']
self.logging.info("Testing monitor variant: %s..." % description)
output, status = self.driver.update_monitor(self.lb_id, monitor)
self.logging.info("\tStatus: %s || output: %s" % (status, output))
current_monitor, current_status = self.driver.get_monitor(
self.lb_id)
if status == '202': # we have a positive result!
self.assertEqual(
current_monitor,
monitor,
msg=
"Loadbalancer: %s monitor not matching expected value: Expected: %s || Actual %s"
% (self.lb_id, monitor, current_monitor))
self.logging.info("\tTest monitor: %s" % monitor)
self.logging.info("\tCurrent monitor: %s" % current_monitor)
lbaas_utils.wait_for_active_status(self)
def test_LoadBalancerLogs(self):
""" test creation of loadbalancers for libra
"""
# Create our loadbalancer
self.create_result, self.actual_status, self.lb_id, self.lb_addr = self.driver.create_lb(self.lb_name, self.nodes, self.algorithm, self.bad_statuses)
self.logging.info('load balancer id: %s' %self.lb_id)
self.logging.info('load balancer ip addr: %s' %self.lb_addr)
lbaas_utils.wait_for_active_status(self, active_wait_time=180)
self.logging.info('gathering logs, 1st attempt...')
self.actual_status = self.driver.get_logs(self.lb_id, auth_token = self.swift_auth_token, obj_endpoint = self.swift_endpoint, obj_basepath = self.args.swiftbasepath)
self.assertEqual(str(self.actual_status), str(self.expected_status), msg = "ERROR: Attempt to gather lb logs produced status: %s. Expected status: %s" %(self.actual_status, self.expected_status))
lbaas_utils.validate_loadBalancer(self)
lbaas_utils.wait_for_active_status(self, active_wait_time=180)
self.logging.info('gathering logs, 2nd attempt...')
self.actual_status = self.driver.get_logs(self.lb_id, auth_token = self.swift_auth_token, obj_endpoint = self.swift_endpoint, obj_basepath = self.args.swiftbasepath)
self.assertEqual(str(self.actual_status), str(self.expected_status), msg = "ERROR: Attempt to gather lb logs produced status: %s. Expected status: %s" %(self.actual_status, self.expected_status))
def test_addNodes(self):
""" test update of loadbalancers for libra
"""
# wait until our lb is ACTIVE before trying to update it
lbaas_utils.wait_for_active_status(self)
# add nodes to our loadbalancer
self.add_node_result, self.actual_status = self.driver.add_nodes(self.lb_id, self.add_node_data)
disabled_list = []
if self.actual_status in self.good_statuses:
# good update, we need to update our expected nodes
self.nodes = self.init_nodes + self.add_node_data
for node in self.nodes:
if "condition" in node and node["condition"] == "DISABLED":
if "address" in node:
disabled_list.append(node["address"])
lbaas_utils.validate_loadBalancer(self, disabled_list)
def test_monitorLoadBalancer(self):
""" test monitoring of loadbalancers for libra
"""
# Create our loadbalancer
self.create_result, self.actual_status, self.lb_id, self.lb_addr = self.driver.create_lb(
self.lb_name, self.nodes, self.algorithm, self.bad_statuses)
self.logging.info('load balancer id: %s' % self.lb_id)
self.logging.info('load balancer ip addr: %s' % self.lb_addr)
lbaas_utils.validate_loadBalancer(self)
# use fabric to stop libra_worker
self.logging.info("Stopping libra_worker on lb: %s address: %s" %
(self.lb_id, self.lb_addr))
cmd = "fab --no-pty -H %s stop_libra_worker" % (self.lb_addr)
status, output = commands.getstatusoutput(cmd)
#if self.args.verbose:
self.logging.info("Command: %s" % cmd)
self.logging.info("Status: %s" % status)
self.logging.info("Output: %s" % output)
# check api to see if ERROR state is set / libra_worker failure is detected
self.logging.info(
"Wait / poll api server for loadbalancer to be set to ERROR status..."
)
lbaas_utils.wait_for_active_status(self,
self.lb_id,
active_wait_time=240,
desired_status='ERROR')
# restart libra_worker
self.logging.info("Starting libra_worker on lb: %s address: %s" %
(self.lb_id, self.lb_addr))
cmd = "fab --no-pty -H %s start_libra_worker" % (self.lb_addr)
status, output = commands.getstatusoutput(cmd)
#if self.args.verbose:
self.logging.info("Command: %s" % cmd)
self.logging.info("Status: %s" % status)
self.logging.info("Output: %s" % output)
# check api to ensure lb state is properly reset
self.logging.info(
"Wait / poll api server for loadbalancer to be set to ACTIVE status..."
)
lbaas_utils.wait_for_active_status(self,
self.lb_id,
active_wait_time=240,
desired_status='ACTIVE')
# one final lb validation
lbaas_utils.validate_loadBalancer(self)
def test_addNodes(self):
""" test update of loadbalancers for libra
"""
# wait until our lb is ACTIVE before trying to update it
lbaas_utils.wait_for_active_status(self)
# add nodes to our loadbalancer
self.add_node_result, self.actual_status = self.driver.add_nodes(
self.lb_id, self.add_node_data)
disabled_list = []
if self.actual_status in self.good_statuses:
# good update, we need to update our expected nodes
self.nodes = self.init_nodes + self.add_node_data
for node in self.nodes:
if 'condition' in node and node['condition'] == 'DISABLED':
if 'address' in node:
disabled_list.append(node['address'])
lbaas_utils.validate_loadBalancer(self, disabled_list)
def test_createLoadBalancer(self):
""" gathering time until we have a good loadbalancer and counting bad devices
"""
iterations = []
bad_iterations = []
failed_iterations = []
bad_count = 0
fail_count = 0
test_iterations = 25
for i in range(test_iterations):
self.logging.info("Iteration: %d" % i)
# Create our loadbalancer
lb_ready = False
suspected_bad = False
time_wait = 1
attempts_remain = 100
max_time = 300
start_time = time.time()
self.create_result, self.actual_status, self.lb_id, self.lb_addr = self.driver.create_lb(
self.lb_name, self.nodes, self.algorithm, self.bad_statuses)
self.logging.info('load balancer id: %s' % self.lb_id)
self.logging.info('load balancer ip addr: %s' % self.lb_addr)
lbaas_utils.wait_for_active_status(self, must_pass=False)
# make sure we can get traffic from our loadbalancer
self.logging.info(time.time() - start_time)
self.logging.info((time.time() - start_time) < max_time)
while not lb_ready and attempts_remain and (
(time.time() - start_time) <= max_time):
try:
if attempts_remain % 10 == 0:
self.logging.info("Attempts remaining: %d" %
attempts_remain)
lb_url = 'http://%s' % (self.lb_addr)
result = requests.get(lb_url, verify=False)
result.connection.close()
if result:
lb_ready = True
except Exception, e:
if not suspected_bad:
self.logging.info(Exception)
self.logging.info(e)
self.logging.info(
"loadbalancer id: %s not yet ready. Suspected bad haproxy device"
% (self.lb_id))
self.logging.info(
"Will try up to: %d times for the loadbalancer to be functional (~10 minutes), please be patient..."
% (attempts_remain * time_wait))
suspected_bad = True
bad_count += 1
time.sleep(time_wait)
attempts_remain -= 1
stop_time = time.time()
expended_time = stop_time - start_time
self.logging.info("Time for loadbalancer: %s to be ready: %f" %
(self.lb_id, expended_time))
if suspected_bad:
if ((expended_time) <= max_time):
bad_iterations.append(expended_time)
else:
failed_iterations.append(expended_time)
else:
iterations.append(expended_time)
if attempts_remain and ((expended_time) <= max_time):
lbaas_utils.validate_loadBalancer(self)
else:
self.logging.info(
"WARN: loadbalancer: %s suspected still not ready after %d seconds"
% (self.lb_id, expended_time))
fail_count += 1
self.logging.info("Deleting loadbalancer: %s" % self.lb_id)
result = self.driver.delete_lb(self.lb_id)
time.sleep(10)
def test_loadBalancerFuncs(self):
""" test creation of loadbalancers and their functions for libra
"""
# Create our loadbalancer
self.create_result, self.actual_status, self.lb_id, self.lb_addr = self.driver.create_lb(
self.lb_name, self.nodes, self.algorithm, self.bad_statuses)
self.logging.info('load balancer id: %s' % self.lb_id)
self.logging.info('load balancer ip addr: %s' % self.lb_addr)
lbaas_utils.validate_loadBalancer(self)
##############
# test updates
##############
self.logging.info("Testing update variants...")
if self.functional_inputs and 'update_variants' in self.functional_inputs and self.actual_status not in self.bad_statuses:
for update_variant in self.functional_inputs['update_variants']:
self.logging.info("Update variant description: %s" %
update_variant['description'])
self.update_data = update_variant['update_data']
if 'expected_status' in update_variant:
self.expected_status = update_variant['expected_status']
else:
self.expected_status = 202
if 'algorithm' in update_variant and update_variant[
'algorithm'] in self.good_algorithms:
self.algorithm = update_variant['algorithm']
# wait until our lb is ACTIVE before trying to update it
lbaas_utils.wait_for_active_status(self)
# update our loadbalancer
self.actual_status = self.driver.update_lb(
self.lb_id, self.update_data)
if 'name' in self.update_data and self.actual_status not in self.bad_statuses:
self.lb_name = self.update_data['name']
lbaas_utils.validate_loadBalancer(self)
################
# test add_nodes
################
self.expected_status = '202'
if self.functional_inputs and 'add_node_variants' in self.functional_inputs:
# get list of original lb nodes (each node = dict)
original_nodes = self.driver.list_lb_nodes(self.lb_id)['nodes']
self.original_nodes = copy.deepcopy(self.nodes)
for variant in self.functional_inputs['add_node_variants']:
self.logging.info("Testing add / remove node functionality...")
self.logging.info("Add node variant: %s" %
(variant['description']))
add_nodes = []
if 'nodes' in variant:
add_nodes = variant['nodes']
else:
node_count = variant['node_count']
if str(variant['node_count']).startswith(
'MAX_BACKEND_COUNT'):
node_count = int(self.args.maxbackendnodes) - len(
self.nodes)
if str(variant['node_count']).endswith('+1'):
node_count += 1
node_pool = self.test_nodes
# we have a node_count value and pull from default_values['nodes']
if node_count < len(node_pool):
add_nodes = node_pool[:node_count]
else:
add_nodes = []
idx = 0
while len(add_nodes) < node_count:
add_nodes.append(node_pool[idx])
idx += 1
if idx == len(node_pool):
idx = 0
# add nodes to our loadbalancer
if self.args.verbose:
self.logging.info("Pre-add node list:")
nodes = self.driver.list_lb_nodes(self.lb_id)
self.logging.info(nodes)
self.add_node_result, self.actual_status = self.driver.add_nodes(
self.lb_id, add_nodes)
disabled_list = []
if self.actual_status == '202':
# good update, we need to update our expected nodes
self.nodes += add_nodes
for node in self.nodes:
if 'condition' in node and node[
'condition'] == 'DISABLED':
if 'address' in node:
disabled_list.append(node['address'])
self.logging.info("Post-add node list:")
nodes = self.driver.list_lb_nodes(self.lb_id)
self.logging.info(nodes)
# maybe delete this wait!
lbaas_utils.wait_for_active_status(self)
lbaas_utils.validate_loadBalancer(self, disabled_list)
# remove nodes / reset to original set up
current_nodes = self.driver.list_lb_nodes(
self.lb_id)['nodes']
for orig_node in original_nodes:
if orig_node in current_nodes:
current_nodes.remove(orig_node)
for current_node in current_nodes:
attempts_remain = 60
time_wait = 1
node_exists = True
node_id = current_node['id']
if self.args.verbose:
self.logging.info("Removing node id: %s" % node_id)
while attempts_remain and node_exists:
result = self.driver.delete_lb_node(
self.lb_id, node_id)
if result != '202':
attempts_remain -= 1
time.sleep(time_wait)
else:
node_exists = False
self.assertEqual(
result,
'202',
msg=
"ERROR: Node id: %s deletion on loadbalancer id: %s failed"
% (node_id, self.lb_id))
self.nodes = copy.deepcopy(self.original_nodes)
###################
# test modify_nodes
###################
if self.original_nodes:
self.nodes = self.original_nodes
if self.functional_inputs and 'modify_variants' in self.functional_inputs:
self.logging.info("Testing modify nodes")
# wait until our lb is ACTIVE before trying to update it
lbaas_utils.wait_for_active_status(self)
# modify/disable a node on our loadbalancer
self.logging.info("Testing node disable...")
self.expected_status = 202
nodes = self.driver.list_lb_nodes(self.lb_id)
mod_node = nodes['nodes'][0]
mod_node_id = mod_node['id']
mod_node_addr = mod_node['address']
mod_node_data = {'condition': 'DISABLED'}
self.actual_status = self.driver.modify_node(
self.lb_id, mod_node_id, mod_node_data)
lbaas_utils.validate_loadBalancer(self, [mod_node_addr])
# re-enable the node
self.logging.info("Testing re-enable of node...")
mod_node_data = {'condition': 'ENABLED'}
self.actual_status = self.driver.modify_node(
self.lb_id, mod_node_id, mod_node_data)
lbaas_utils.validate_loadBalancer(self)
# DISABLED -
# libra currently does not actually update ip addrs or ports of nodes
# but it does return a 202 / incorrect status code
"""
# modify ip / address
self.logging.info("Testing update of node ip...")
mod_node_data = {'address': '127.0.0.1'}
expected_status = '400'
if self.args.badstatus:
expected_status = self.args.badstatus
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
nodes = self.driver.list_lb_nodes(self.lb_id)
self.assertEqual(str(self.actual_status), expected_status, msg = "ERROR: Attempt to update node ip address succeeded with status: %s. Expected status: %s" %(self.actual_status, expected_status))
# modify port
self.logging.info("Testing update of node port...")
mod_node_data = {'port': '443'}
expected_status = '400'
if self.args.badstatus:
expected_status = self.args.badstatus
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
self.assertEqual(str(self.actual_status), expected_status, msg = "ERROR: Attempt to update node port succeeded with status: %s. Expected status: %s" %(self.actual_status, expected_status))
"""
# update of non-existent node
self.logging.info("Testing update of non-existent node...")
mod_node_data = {'condition': "DISABLED"}
expected_status = '404'
if self.args.badstatus:
expected_status = self.args.badstatus
self.actual_status = self.driver.modify_node(
self.lb_id, '0', mod_node_data)
self.assertEqual(
str(self.actual_status),
expected_status,
msg=
"ERROR: Attempt to update non-existent node succeeded with status: %s. Expected status: %s"
% (self.actual_status, expected_status))
# lower-case condition
expected_status = '400'
if self.args.badstatus:
expected_status = self.args.badstatus
self.logging.info("Testing lowercase condition...")
mod_node_data = {'condition': 'disabled'}
self.actual_status = self.driver.modify_node(
self.lb_id, mod_node_id, mod_node_data)
self.assertEqual(
str(self.actual_status),
expected_status,
msg=
"ERROR: Usage of lowercase node condition succeeded with status: %s. Expected status: %s"
% (self.actual_status, expected_status))
self.expected_status = '202'
# negative / bad node condition
self.logging.info("Testing bad condition...")
mod_node_data = {'condition': 'TASERED_BADLY'}
expected_statuses = ['400', '512', '404']
self.actual_status = self.driver.modify_node(
self.lb_id, mod_node_id, mod_node_data)
self.assertTrue(
str(self.actual_status) in expected_statuses,
msg=
"ERROR: Attempt to update to bad condition succeeded with status: %s. Expected status: %s"
% (self.actual_status, expected_statuses))
################
# test multi_lb
################
""" test of multiple lb's per haproxy device
"""
if self.functional_inputs and 'multiLB_variants' in self.functional_inputs:
self.logging.info(
"Testing multiple loadbalancers on one device...")
self.expected_status = '202'
self.main_lb_id = copy.deepcopy(self.lb_id)
for multi_variant in self.functional_inputs['multiLB_variants']:
self.lb_name2 = multi_variant['name2']
self.nodes2 = multi_variant['nodes2']
# wait until our lb is ACTIVE before trying to update it
# test / validate lb1
self.actual_status = '202'
lbaas_utils.wait_for_active_status(self, self.lb_id)
lbaas_utils.validate_loadBalancer(self,
multi=True,
multi_id=self.lb_id,
multi_name=self.lb_name,
multi_nodes=self.nodes)
result_data = self.driver.list_lb_detail(self.lb_id)
if 'virtualIps' in result_data:
self.vip = result_data['virtualIps'][0]['id']
else: # client has different key
self.vip = ast.literal_eval(result_data['ips'])[0]['id']
# create lb2
self.logging.info('Creating load balancer2...')
self.create_result, self.actual_status, self.lb_id2, self.lb_addr = self.driver.create_lb(
self.lb_name2, self.nodes2, self.algorithm,
self.bad_statuses, self.vip)
if self.args.verbose:
self.logging.info("STATUS: %s" % self.actual_status)
self.logging.info("RESULT: %s" % self.create_result.text)
lbaas_utils.wait_for_active_status(self, self.lb_id2)
self.logging.info('load balancer2 id: %s' % (self.lb_id2))
lbaas_utils.validate_loadBalancer(self,
multi=True,
multi_id=self.lb_id2,
multi_name=self.lb_name2,
multi_nodes=self.nodes2)
self.logging.info("Deleting loadbalancer: %s" % self.lb_id2)
result = self.driver.delete_lb(self.lb_id2)
def test_createLoadBalancer(self):
""" gathering time until we have a good loadbalancer and counting bad devices
"""
iterations = []
bad_iterations = []
failed_iterations = []
bad_count = 0
fail_count = 0
test_iterations=25
for i in range(test_iterations):
self.logging.info("Iteration: %d" %i)
# Create our loadbalancer
lb_ready = False
suspected_bad = False
time_wait = 1
attempts_remain = 100
max_time = 300
start_time = time.time()
self.create_result, self.actual_status, self.lb_id, self.lb_addr = self.driver.create_lb(self.lb_name, self.nodes, self.algorithm, self.bad_statuses)
self.logging.info('load balancer id: %s' %self.lb_id)
self.logging.info('load balancer ip addr: %s' %self.lb_addr)
lbaas_utils.wait_for_active_status(self, must_pass=False)
# make sure we can get traffic from our loadbalancer
self.logging.info(time.time()-start_time)
self.logging.info((time.time()-start_time) < max_time)
while not lb_ready and attempts_remain and ((time.time()-start_time) <= max_time):
try:
if attempts_remain%10 ==0:
self.logging.info("Attempts remaining: %d" %attempts_remain)
lb_url = 'http://%s' %(self.lb_addr)
result = requests.get(lb_url, verify= False)
result.connection.close()
if result:
lb_ready=True
except Exception, e:
if not suspected_bad:
self.logging.info(Exception)
self.logging.info(e)
self.logging.info("loadbalancer id: %s not yet ready. Suspected bad haproxy device" %(self.lb_id))
self.logging.info("Will try up to: %d times for the loadbalancer to be functional (~10 minutes), please be patient..." %(attempts_remain*time_wait))
suspected_bad = True
bad_count += 1
time.sleep(time_wait)
attempts_remain -= 1
stop_time = time.time()
expended_time = stop_time - start_time
self.logging.info("Time for loadbalancer: %s to be ready: %f" %(self.lb_id, expended_time))
if suspected_bad:
if ((expended_time) <= max_time):
bad_iterations.append(expended_time)
else:
failed_iterations.append(expended_time)
else:
iterations.append(expended_time)
if attempts_remain and ((expended_time) <= max_time):
lbaas_utils.validate_loadBalancer(self)
else:
self.logging.info("WARN: loadbalancer: %s suspected still not ready after %d seconds" %(self.lb_id, expended_time))
fail_count += 1
self.logging.info("Deleting loadbalancer: %s" %self.lb_id)
result = self.driver.delete_lb(self.lb_id)
time.sleep(10)
def test_modifyNodes(self):
""" test modifying loadbalancer nodes for libra
"""
# wait until our lb is ACTIVE before trying to update it
lbaas_utils.wait_for_active_status(self)
# modify/disable a node on our loadbalancer
self.logging.info("Testing node disable...")
nodes = self.driver.list_lb_nodes(self.lb_id)
mod_node = nodes['nodes'][0]
mod_node_id = mod_node['id']
mod_node_addr = mod_node['address']
mod_node_data = {'condition': 'DISABLED'}
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id,
mod_node_data)
lbaas_utils.validate_loadBalancer(self, [mod_node_addr])
# re-enable the node
self.logging.info("Testing re-enable of node...")
mod_node_data = {'condition': 'ENABLED'}
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id,
mod_node_data)
lbaas_utils.validate_loadBalancer(self)
# modify ip / address
self.logging.info("Testing update of node ip...")
mod_node_data = {'address': '127.0.0.1'}
expected_status = '400'
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id,
mod_node_data)
self.assertEqual(
str(self.actual_status),
expected_status,
msg=
"ERROR: Attempt to update node ip address succeeded with status: %s. Expected status: %s"
% (self.actual_status, expected_status))
# modify port
self.logging.info("Testing update of node port...")
mod_node_data = {'port': '443'}
expected_status = '400'
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id,
mod_node_data)
self.assertEqual(
str(self.actual_status),
expected_status,
msg=
"ERROR: Attempt to update node port succeeded with status: %s. Expected status: %s"
% (self.actual_status, expected_status))
# update of non-existent node
self.logging.info("Testing update of non-existent node...")
mod_node_data = {'condition': "DISABLED"}
expected_status = '404'
self.actual_status = self.driver.modify_node(self.lb_id, '0',
mod_node_data)
self.assertEqual(
str(self.actual_status),
expected_status,
msg=
"ERROR: Attempt to update node port succeeded with status: %s. Expected status: %s"
% (self.actual_status, expected_status))
# lower-case condition
#self.logging.info("Testing lowercase condition...")
#mod_node_data = {'condition':'disabled'}
#self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
#lbaas_utils.validate_loadBalancer(self, [mod_node_addr])
#self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
#lbaas_utils.validate_loadBalancer(self, [mod_node_addr])
# lower-case condition
self.logging.info("Testing bad condition...")
mod_node_data = {'condition': 'TASERED_BADLY'}
expected_statuses = ['400', '512']
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id,
mod_node_data)
self.assertTrue(
str(self.actual_status) in expected_statuses,
msg=
"ERROR: Attempt to update to bad condition succeeded with status: %s. Expected status: %s"
% (self.actual_status, expected_status))
def test_loadBalancerFuncs(self):
""" test creation of loadbalancers and their functions for libra
"""
# Create our loadbalancer
self.create_result, self.actual_status, self.lb_id, self.lb_addr = self.driver.create_lb(self.lb_name, self.nodes, self.algorithm, self.bad_statuses)
self.logging.info('load balancer id: %s' %self.lb_id)
self.logging.info('load balancer ip addr: %s' %self.lb_addr)
lbaas_utils.validate_loadBalancer(self)
##############
# test updates
##############
self.logging.info("Testing update variants...")
if self.functional_inputs and 'update_variants' in self.functional_inputs and self.actual_status not in self.bad_statuses:
for update_variant in self.functional_inputs['update_variants']:
self.logging.info("Update variant description: %s" %update_variant['description'])
self.update_data = update_variant['update_data']
if 'expected_status' in update_variant:
self.expected_status = update_variant['expected_status']
else:
self.expected_status = 202
if 'algorithm' in update_variant and update_variant['algorithm'] in self.good_algorithms:
self.algorithm = update_variant['algorithm']
# wait until our lb is ACTIVE before trying to update it
lbaas_utils.wait_for_active_status(self)
# update our loadbalancer
self.actual_status = self.driver.update_lb(self.lb_id, self.update_data)
if 'name' in self.update_data and self.actual_status not in self.bad_statuses:
self.lb_name = self.update_data['name']
lbaas_utils.validate_loadBalancer(self)
################
# test add_nodes
################
self.expected_status='202'
if self.functional_inputs and 'add_node_variants' in self.functional_inputs:
# get list of original lb nodes (each node = dict)
original_nodes = self.driver.list_lb_nodes(self.lb_id)['nodes']
self.original_nodes = copy.deepcopy(self.nodes)
for variant in self.functional_inputs['add_node_variants']:
self.logging.info("Testing add / remove node functionality...")
self.logging.info("Add node variant: %s" %(variant['description']))
add_nodes = []
if 'nodes' in variant:
add_nodes = variant['nodes']
else:
node_count = variant['node_count']
if str(variant['node_count']).startswith('MAX_BACKEND_COUNT'):
node_count = int(self.args.maxbackendnodes) - len(self.nodes)
if str(variant['node_count']).endswith('+1'):
node_count += 1
node_pool = self.test_nodes
# we have a node_count value and pull from default_values['nodes']
if node_count < len(node_pool):
add_nodes = node_pool[:node_count]
else:
add_nodes = []
idx = 0
while len(add_nodes) < node_count:
add_nodes.append(node_pool[idx])
idx += 1
if idx == len(node_pool):
idx = 0
# add nodes to our loadbalancer
if self.args.verbose:
self.logging.info("Pre-add node list:")
nodes = self.driver.list_lb_nodes(self.lb_id)
self.logging.info(nodes)
self.add_node_result, self.actual_status = self.driver.add_nodes(self.lb_id, add_nodes)
disabled_list = []
if self.actual_status =='202':
# good update, we need to update our expected nodes
self.nodes += add_nodes
for node in self.nodes:
if 'condition' in node and node['condition'] == 'DISABLED':
if 'address' in node:
disabled_list.append(node['address'])
self.logging.info("Post-add node list:")
nodes = self.driver.list_lb_nodes(self.lb_id)
self.logging.info(nodes)
# maybe delete this wait!
lbaas_utils.wait_for_active_status(self)
lbaas_utils.validate_loadBalancer(self, disabled_list)
# remove nodes / reset to original set up
current_nodes = self.driver.list_lb_nodes(self.lb_id)['nodes']
for orig_node in original_nodes:
if orig_node in current_nodes:
current_nodes.remove(orig_node)
for current_node in current_nodes:
attempts_remain = 60
time_wait = 1
node_exists = True
node_id = current_node['id']
if self.args.verbose:
self.logging.info("Removing node id: %s" %node_id)
while attempts_remain and node_exists:
result = self.driver.delete_lb_node(self.lb_id, node_id)
if result != '202':
attempts_remain -= 1
time.sleep(time_wait)
else:
node_exists = False
self.assertEqual(result, '202', msg="ERROR: Node id: %s deletion on loadbalancer id: %s failed" %(node_id, self.lb_id))
self.nodes = copy.deepcopy(self.original_nodes)
###################
# test modify_nodes
###################
if self.original_nodes:
self.nodes = self.original_nodes
if self.functional_inputs and 'modify_variants' in self.functional_inputs:
self.logging.info("Testing modify nodes")
# wait until our lb is ACTIVE before trying to update it
lbaas_utils.wait_for_active_status(self)
# modify/disable a node on our loadbalancer
self.logging.info("Testing node disable...")
self.expected_status=202
nodes = self.driver.list_lb_nodes(self.lb_id)
mod_node = nodes['nodes'][0]
mod_node_id = mod_node['id']
mod_node_addr = mod_node['address']
mod_node_data = {'condition':'DISABLED'}
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
lbaas_utils.validate_loadBalancer(self, [mod_node_addr])
# re-enable the node
self.logging.info("Testing re-enable of node...")
mod_node_data = {'condition':'ENABLED'}
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
lbaas_utils.validate_loadBalancer(self)
# DISABLED -
# libra currently does not actually update ip addrs or ports of nodes
# but it does return a 202 / incorrect status code
"""
# modify ip / address
self.logging.info("Testing update of node ip...")
mod_node_data = {'address': '127.0.0.1'}
expected_status = '400'
if self.args.badstatus:
expected_status = self.args.badstatus
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
nodes = self.driver.list_lb_nodes(self.lb_id)
self.assertEqual(str(self.actual_status), expected_status, msg = "ERROR: Attempt to update node ip address succeeded with status: %s. Expected status: %s" %(self.actual_status, expected_status))
# modify port
self.logging.info("Testing update of node port...")
mod_node_data = {'port': '443'}
expected_status = '400'
if self.args.badstatus:
expected_status = self.args.badstatus
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
self.assertEqual(str(self.actual_status), expected_status, msg = "ERROR: Attempt to update node port succeeded with status: %s. Expected status: %s" %(self.actual_status, expected_status))
"""
# update of non-existent node
self.logging.info("Testing update of non-existent node...")
mod_node_data = {'condition':"DISABLED"}
expected_status = '404'
if self.args.badstatus:
expected_status = self.args.badstatus
self.actual_status = self.driver.modify_node(self.lb_id, '0', mod_node_data)
self.assertEqual(str(self.actual_status), expected_status, msg = "ERROR: Attempt to update non-existent node succeeded with status: %s. Expected status: %s" %(self.actual_status, expected_status))
# lower-case condition
expected_status='400'
if self.args.badstatus:
expected_status = self.args.badstatus
self.logging.info("Testing lowercase condition...")
mod_node_data = {'condition':'disabled'}
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
self.assertEqual(str(self.actual_status), expected_status, msg = "ERROR: Usage of lowercase node condition succeeded with status: %s. Expected status: %s" %(self.actual_status, expected_status))
self.expected_status='202'
# negative / bad node condition
self.logging.info("Testing bad condition...")
mod_node_data = {'condition':'TASERED_BADLY'}
expected_statuses = ['400','512', '404']
self.actual_status = self.driver.modify_node(self.lb_id, mod_node_id, mod_node_data)
self.assertTrue(str(self.actual_status) in expected_statuses, msg = "ERROR: Attempt to update to bad condition succeeded with status: %s. Expected status: %s" %(self.actual_status, expected_statuses))
################
# test multi_lb
################
""" test of multiple lb's per haproxy device
"""
if self.functional_inputs and 'multiLB_variants' in self.functional_inputs:
self.logging.info("Testing multiple loadbalancers on one device...")
self.expected_status = '202'
self.main_lb_id = copy.deepcopy(self.lb_id)
for multi_variant in self.functional_inputs['multiLB_variants']:
self.lb_name2 = multi_variant['name2']
self.nodes2 = multi_variant['nodes2']
# wait until our lb is ACTIVE before trying to update it
# test / validate lb1
self.actual_status='202'
lbaas_utils.wait_for_active_status(self, self.lb_id)
lbaas_utils.validate_loadBalancer( self
, multi=True
, multi_id=self.lb_id
, multi_name=self.lb_name
, multi_nodes=self.nodes)
result_data = self.driver.list_lb_detail(self.lb_id)
if 'virtualIps' in result_data:
self.vip = result_data['virtualIps'][0]['id']
else: # client has different key
self.vip = ast.literal_eval(result_data['ips'])[0]['id']
# create lb2
self.logging.info('Creating load balancer2...')
self.create_result, self.actual_status, self.lb_id2, self.lb_addr = self.driver.create_lb(self.lb_name2, self.nodes2, self.algorithm, self.bad_statuses, self.vip)
if self.args.verbose:
self.logging.info("STATUS: %s" %self.actual_status)
self.logging.info("RESULT: %s" %self.create_result.text)
lbaas_utils.wait_for_active_status(self, self.lb_id2)
self.logging.info('load balancer2 id: %s' %(self.lb_id2))
lbaas_utils.validate_loadBalancer( self
, multi=True
, multi_id=self.lb_id2
, multi_name=self.lb_name2
, multi_nodes=self.nodes2)
self.logging.info("Deleting loadbalancer: %s" %self.lb_id2)
result = self.driver.delete_lb(self.lb_id2)
