def run_cmd(self, command, _connect=False):
"""Execute command over ssh on remote system.
Wait for remote command completion.
Return command output (combined stdout and stderr).
_connect argument is reserved for connect() method.
"""
cmd = self._build_ssh(command)
try:
return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
RapidLog.exception('ssh returned exit status %d:\n%s' %
(ex.returncode, ex.output.strip()))
def connect_socket(self):
attempts = 1
RapidLog.debug("Trying to connect to PROX (just launched) on %s, \
attempt: %d" % (self._ip, attempts))
sock = None
while True:
sock = self.prox_sock()
if sock is not None:
break
attempts += 1
if attempts > 20:
RapidLog.exception("Failed to connect to PROX on %s after %d \
attempts" % (self._ip, attempts))
time.sleep(2)
RapidLog.debug("Trying to connect to PROX (just launched) on %s, \
attempt: %d" % (self._ip, attempts))
RapidLog.info("Connected to PROX on %s" % self._ip)
return sock
def connect(self):
attempts = 1
RapidLog.debug("Trying to connect to machine \
on %s, attempt: %d" % (self._ip, attempts))
while True:
try:
self.test_connect()
break
except RuntimeWarning as ex:
RapidLog.debug("RuntimeWarning %d:\n%s" %
(ex.returncode, ex.output.strip()))
attempts += 1
if attempts > 20:
RapidLog.exception("Failed to connect to instance after %d\
attempts:\n%s" % (attempts, ex))
time.sleep(2)
RapidLog.debug("Trying to connect to machine \
on %s, attempt: %d" % (self._ip, attempts))
RapidLog.debug("Connected to machine on %s" % self._ip)
def connect(self):
attempts = 1
RapidLog.debug("Trying to connect to instance which was just launched \
on %s, attempt: %d" % (self._ip, attempts))
while True:
try:
self.test_connect()
break
except RuntimeWarning as ex:
attempts += 1
if attempts > 20:
RapidLog.exception("Failed to connect to instance after %d\
attempts:\n%s" % (attempts, ex))
raise Exception("Failed to connect to instance after %d \
attempts:\n%s" % (attempts, ex))
time.sleep(2)
RapidLog.debug("Trying to connect to instance which was just \
launched on %s, attempt: %d" % (self._ip, attempts))
RapidLog.debug("Connected to instance on %s" % self._ip)
def create_stack(self, stack_name, stack_file_path, heat_parameters):
files, template = template_utils.process_template_path(stack_file_path)
stack_created = self.heatclient.stacks.create(
stack_name=stack_name,
template=template,
parameters=heat_parameters,
files=files)
stack = self.heatclient.stacks.get(stack_created['stack']['id'],
resolve_outputs=True)
# Poll at 5 second intervals, until the status is no longer 'BUILD'
while stack.stack_status == 'CREATE_IN_PROGRESS':
print('waiting..')
time.sleep(5)
stack = self.heatclient.stacks.get(stack_created['stack']['id'],
resolve_outputs=True)
if stack.stack_status == 'CREATE_COMPLETE':
return stack
else:
RapidLog.exception('Error in stack deployment')
def scp_get(self, src, dst):
"""Copy src file from remote system to dst on local system."""
cmd = [
'scp', '-B', '-oStrictHostKeyChecking=no',
'-oUserKnownHostsFile=/dev/null', '-oLogLevel=ERROR'
]
if self._key is not None:
cmd.extend(['-i', self._key])
remote = ''
if self._user is not None:
remote += self._user + '@'
remote += self._ip + ':/home/' + self._user + src
cmd.append(remote)
cmd.append(dst)
try:
# Actually ignore output on success, but capture stderr on failure
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
RapidLog.exception('scp returned exit status %d:\n%s' %
(ex.returncode, ex.output.strip()))
def parse_config(test_params):
testconfig = configparser.RawConfigParser()
testconfig.read(test_params['test_file'])
test_params['required_number_of_test_machines'] = int(
testconfig.get('TestParameters', 'total_number_of_test_machines'))
test_params['number_of_tests'] = int(
testconfig.get('TestParameters', 'number_of_tests'))
test_params['TestName'] = testconfig.get('TestParameters', 'name')
if testconfig.has_option('TestParameters', 'lat_percentile'):
test_params['lat_percentile'] = old_div(
float(testconfig.get('TestParameters', 'lat_percentile')),
100.0)
else:
test_params['lat_percentile'] = 0.99
RapidLog.info('Latency percentile at {:.0f}%'.format(
test_params['lat_percentile'] * 100))
if testconfig.has_option('TestParameters', 'ipv6'):
test_params['ipv6'] = testconfig.getboolean(
'TestParameters', 'ipv6')
else:
test_params['ipv6'] = False
config = configparser.RawConfigParser()
config.read(test_params['environment_file'])
test_params['vim_type'] = config.get('Varia', 'vim')
test_params['key'] = config.get('ssh', 'key')
test_params['user'] = config.get('ssh', 'user')
test_params['total_number_of_machines'] = int(
config.get('rapid', 'total_number_of_machines'))
tests = []
test = {}
for test_index in range(1, test_params['number_of_tests'] + 1):
test.clear()
section = 'test%d' % test_index
options = testconfig.options(section)
for option in options:
if option in ['imix', 'imixs', 'flows']:
test[option] = ast.literal_eval(
testconfig.get(section, option))
elif option in [
'maxframespersecondallingress', 'stepsize', 'flowsize'
]:
test[option] = int(testconfig.get(section, option))
elif option in [
'startspeed', 'step', 'drop_rate_threshold',
'lat_avg_threshold', 'lat_perc_threshold',
'lat_max_threshold', 'accuracy', 'maxr', 'maxz',
'pass_threshold', 'ramp_step'
]:
test[option] = float(testconfig.get(section, option))
else:
test[option] = testconfig.get(section, option)
tests.append(dict(test))
for test in tests:
if test['test'] in ['flowsizetest', 'TST009test']:
if 'drop_rate_threshold' not in test.keys():
test['drop_rate_threshold'] = 0
latency_thresholds = [
'lat_avg_threshold', 'lat_perc_threshold',
'lat_max_threshold'
]
for threshold in latency_thresholds:
if threshold not in test.keys():
test[threshold] = 'inf'
test_params['tests'] = tests
if test_params['required_number_of_test_machines'] > test_params[
'total_number_of_machines']:
RapidLog.exception(
"Not enough VMs for this test: %d needed and only %d available"
% (required_number_of_test_machines, total_number_of_machines))
raise Exception(
"Not enough VMs for this test: %d needed and only %d available"
% (required_number_of_test_machines, total_number_of_machines))
machine_map = configparser.RawConfigParser()
machine_map.read(test_params['machine_map_file'])
machines = []
machine = {}
for test_machine in range(
1, test_params['required_number_of_test_machines'] + 1):
machine.clear()
section = 'TestM%d' % test_machine
options = testconfig.options(section)
for option in options:
if option in ['prox_socket', 'prox_launch_exit', 'monitor']:
machine[option] = testconfig.getboolean(section, option)
elif option in ['cores', 'gencores', 'latcores']:
machine[option] = ast.literal_eval(
testconfig.get(section, option))
elif option in ['bucket_size_exp']:
machine[option] = int(testconfig.get(section, option))
else:
machine[option] = testconfig.get(section, option)
for key in ['prox_socket', 'prox_launch_exit']:
if key not in machine.keys():
machine[key] = True
if 'monitor' not in machine.keys():
machine['monitor'] = True
index = int(
machine_map.get('TestM%d' % test_machine, 'machine_index'))
section = 'M%d' % index
options = config.options(section)
for option in options:
machine[option] = config.get(section, option)
machines.append(dict(machine))
for machine in machines:
dp_ports = []
if 'dest_vm' in machine.keys():
index = 1
while True:
dp_ip_key = 'dp_ip{}'.format(index)
dp_mac_key = 'dp_mac{}'.format(index)
if dp_ip_key in machines[int(machine['dest_vm'])-1].keys() and \
dp_mac_key in machines[int(machine['dest_vm'])-1].keys():
dp_port = {
'ip':
machines[int(machine['dest_vm']) - 1][dp_ip_key],
'mac':
machines[int(machine['dest_vm']) - 1][dp_mac_key]
}
dp_ports.append(dict(dp_port))
index += 1
else:
break
machine['dest_ports'] = list(dp_ports)
gw_ips = []
if 'gw_vm' in machine.keys():
index = 1
while True:
gw_ip_key = 'dp_ip{}'.format(index)
if gw_ip_key in machines[int(machine['gw_vm']) - 1].keys():
gw_ip = machines[int(machine['gw_vm']) - 1][gw_ip_key]
gw_ips.append(gw_ip)
index += 1
else:
break
machine['gw_ips'] = list(gw_ips)
test_params['machines'] = machines
return (test_params)
def run_tests(self, test_params):
test_params = RapidConfigParser.parse_config(test_params)
RapidLog.debug(test_params)
monitor_gen = monitor_sut = False
background_machines = []
sut_machine = gen_machine = None
configonly = test_params['configonly']
for machine_params in test_params['machines']:
if 'gencores' in machine_params.keys():
machine = RapidGeneratorMachine(
test_params['key'], test_params['user'],
test_params['password'], test_params['vim_type'],
test_params['rundir'], test_params['resultsdir'],
machine_params, configonly, test_params['ipv6'])
if machine_params['monitor']:
if monitor_gen:
RapidLog.exception("Can only monitor 1 generator")
raise Exception("Can only monitor 1 generator")
else:
monitor_gen = True
gen_machine = machine
else:
background_machines.append(machine)
else:
machine = RapidMachine(test_params['key'], test_params['user'],
test_params['password'],
test_params['vim_type'],
test_params['rundir'],
test_params['resultsdir'],
machine_params, configonly)
if machine_params['monitor']:
if monitor_sut:
RapidLog.exception("Can only monitor 1 sut")
raise Exception("Can only monitor 1 sut")
else:
monitor_sut = True
if machine_params['prox_socket']:
sut_machine = machine
self.machines.append(machine)
try:
prox_executor = concurrent.futures.ThreadPoolExecutor(
max_workers=len(self.machines))
self.future_to_prox = {
prox_executor.submit(machine.start_prox): machine
for machine in self.machines
}
if configonly:
concurrent.futures.wait(self.future_to_prox,
return_when=ALL_COMPLETED)
sys.exit()
socket_executor = concurrent.futures.ThreadPoolExecutor(
max_workers=len(self.machines))
future_to_connect_prox = {
socket_executor.submit(machine.connect_prox): machine
for machine in self.machines
}
concurrent.futures.wait(future_to_connect_prox,
return_when=ALL_COMPLETED)
result = 0
for test_param in test_params['tests']:
RapidLog.info(test_param['test'])
if test_param['test'] in [
'flowsizetest', 'TST009test', 'fixed_rate',
'increment_till_fail'
]:
test = FlowSizeTest(
test_param, test_params['lat_percentile'],
test_params['runtime'], test_params['TestName'],
test_params['environment_file'], gen_machine,
sut_machine, background_machines,
test_params['sleep_time'])
elif test_param['test'] in ['corestatstest']:
test = CoreStatsTest(test_param, test_params['runtime'],
test_params['TestName'],
test_params['environment_file'],
self.machines)
elif test_param['test'] in ['portstatstest']:
test = PortStatsTest(test_param, test_params['runtime'],
test_params['TestName'],
test_params['environment_file'],
self.machines)
elif test_param['test'] in ['impairtest']:
test = ImpairTest(
test_param, test_params['lat_percentile'],
test_params['runtime'], test_params['TestName'],
test_params['environment_file'], gen_machine,
sut_machine, background_machines)
elif test_param['test'] in ['irqtest']:
test = IrqTest(test_param, test_params['runtime'],
test_params['TestName'],
test_params['environment_file'],
self.machines)
elif test_param['test'] in ['warmuptest']:
test = WarmupTest(test_param, gen_machine)
else:
RapidLog.debug('Test name ({}) is not valid:'.format(
test_param['test']))
single_test_result, result_details = test.run()
result = result + single_test_result
for machine in self.machines:
machine.close_prox()
concurrent.futures.wait(self.future_to_prox,
return_when=ALL_COMPLETED)
except (ConnectionError, KeyboardInterrupt) as e:
result = result_details = None
socket_executor.shutdown(wait=False)
socket_executor._threads.clear()
prox_executor.shutdown(wait=False)
prox_executor._threads.clear()
concurrent.futures.thread._threads_queues.clear()
RapidLog.error("Test interrupted: {} {}".format(
type(e).__name__, e))
return (result, result_details)
def parse_config(test_params):
testconfig = configparser.RawConfigParser()
testconfig.read(test_params['test_file'])
test_params['required_number_of_test_machines'] = int(testconfig.get(
'TestParameters', 'total_number_of_test_machines'))
test_params['number_of_tests'] = int(testconfig.get('TestParameters',
'number_of_tests'))
test_params['TestName'] = testconfig.get('TestParameters', 'name')
if testconfig.has_option('TestParameters', 'lat_percentile'):
test_params['lat_percentile'] = old_div(float(
testconfig.get('TestParameters', 'lat_percentile')),100.0)
else:
test_params['lat_percentile'] = 0.99
RapidLog.info('Latency percentile at {:.0f}%'.format(
test_params['lat_percentile']*100))
if testconfig.has_option('TestParameters', 'sleep_time'):
test_params['sleep_time'] = int(testconfig.get('TestParameters', 'sleep_time'))
if test_params['sleep_time'] < 2:
test_params['sleep_time'] = 2
else:
test_params['sleep_time'] = 2
if testconfig.has_option('TestParameters', 'ipv6'):
test_params['ipv6'] = testconfig.getboolean('TestParameters','ipv6')
else:
test_params['ipv6'] = False
config = configparser.RawConfigParser()
config.read(test_params['environment_file'])
test_params['vim_type'] = config.get('Varia', 'vim')
test_params['user'] = config.get('ssh', 'user')
if config.has_option('ssh', 'key'):
test_params['key'] = config.get('ssh', 'key')
if test_params['user'] in ['rapid']:
if test_params['key'] != 'rapid_rsa_key':
RapidLog.debug(("Key file {} for user {} overruled by key file:"
" rapid_rsa_key").format(test_params['key'],
test_params['user']))
test_params['key'] = 'rapid_rsa_key'
else:
test_params['key'] = None
if config.has_option('ssh', 'password'):
test_params['password'] = config.get('ssh', 'password')
else:
test_params['password'] = None
test_params['total_number_of_machines'] = int(config.get('rapid',
'total_number_of_machines'))
tests = []
test = {}
for test_index in range(1, test_params['number_of_tests']+1):
test.clear()
section = 'test%d'%test_index
options = testconfig.options(section)
for option in options:
if option in ['imix','imixs','flows', 'warmupimix']:
test[option] = ast.literal_eval(testconfig.get(section,
option))
elif option in ['maxframespersecondallingress','stepsize',
'flowsize','warmupflowsize','warmuptime', 'steps']:
test[option] = int(testconfig.get(section, option))
elif option in ['startspeed', 'step', 'drop_rate_threshold',
'lat_avg_threshold','lat_perc_threshold',
'lat_max_threshold','accuracy','maxr','maxz',
'ramp_step','warmupspeed','mis_ordered_threshold']:
test[option] = float(testconfig.get(section, option))
else:
test[option] = testconfig.get(section, option)
tests.append(dict(test))
for test in tests:
if test['test'] in ['flowsizetest','TST009test']:
if 'drop_rate_threshold' not in test.keys():
test['drop_rate_threshold'] = 0
latency_thresholds = ['lat_avg_threshold','lat_perc_threshold','lat_max_threshold','mis_ordered_threshold']
for threshold in latency_thresholds:
if threshold not in test.keys():
test[threshold] = inf
test_params['tests'] = tests
if test_params['required_number_of_test_machines'] > test_params[
'total_number_of_machines']:
RapidLog.exception("Not enough VMs for this test: %d needed and only %d available" % (required_number_of_test_machines,total_number_of_machines))
raise Exception("Not enough VMs for this test: %d needed and only %d available" % (required_number_of_test_machines,total_number_of_machines))
map_info = test_params['machine_map_file'].strip('[]').split(',')
map_info_length = len(map_info)
# If map_info is a list where the first entry is numeric, we assume we
# are dealing with a list of machines and NOT the machine.map file
if map_info[0].isnumeric():
if map_info_length < test_params[
'required_number_of_test_machines']:
RapidLog.exception('Not enough machine indices in --map \
parameter: {}. Needing {} entries'.format(map_info,
test_params['required_number_of_test_machines']))
machine_index = list(map(int,map_info))
else:
machine_map = configparser.RawConfigParser()
machine_map.read(test_params['machine_map_file'])
machine_index = []
for test_machine in range(1,
test_params['required_number_of_test_machines']+1):
machine_index.append(int(machine_map.get(
'TestM%d'%test_machine, 'machine_index')))
machine_map = configparser.RawConfigParser()
machine_map.read(test_params['machine_map_file'])
machines = []
machine = {}
for test_machine in range(1, test_params[
'required_number_of_test_machines']+1):
machine.clear()
section = 'TestM%d'%test_machine
options = testconfig.options(section)
for option in options:
if option in ['prox_socket','prox_launch_exit','monitor']:
machine[option] = testconfig.getboolean(section, option)
elif option in ['mcore', 'cores', 'gencores','latcores']:
machine[option] = ast.literal_eval(testconfig.get(
section, option))
elif option in ['bucket_size_exp']:
machine[option] = int(testconfig.get(section, option))
if machine[option] < 11:
RapidLog.exception(
"Minimum Value for bucket_size_exp is 11")
else:
machine[option] = testconfig.get(section, option)
for key in ['prox_socket','prox_launch_exit']:
if key not in machine.keys():
machine[key] = True
if 'monitor' not in machine.keys():
machine['monitor'] = True
section = 'M%d'%machine_index[test_machine-1]
options = config.options(section)
for option in options:
machine[option] = config.get(section, option)
machines.append(dict(machine))
for machine in machines:
dp_ports = []
if 'dest_vm' in machine.keys():
index = 1
while True:
dp_ip_key = 'dp_ip{}'.format(index)
dp_mac_key = 'dp_mac{}'.format(index)
if dp_ip_key in machines[int(machine['dest_vm'])-1].keys() and \
dp_mac_key in machines[int(machine['dest_vm'])-1].keys():
dp_port = {'ip': machines[int(machine['dest_vm'])-1][dp_ip_key],
'mac' : machines[int(machine['dest_vm'])-1][dp_mac_key]}
dp_ports.append(dict(dp_port))
index += 1
else:
break
machine['dest_ports'] = list(dp_ports)
gw_ips = []
if 'gw_vm' in machine.keys():
index = 1
while True:
gw_ip_key = 'dp_ip{}'.format(index)
if gw_ip_key in machines[int(machine['gw_vm'])-1].keys():
gw_ip = machines[int(machine['gw_vm'])-1][gw_ip_key]
gw_ips.append(gw_ip)
index += 1
else:
break
machine['gw_ips'] = list(gw_ips)
test_params['machines'] = machines
return (test_params)
def run_tests(self, test_params):
test_params = RapidConfigParser.parse_config(test_params)
RapidLog.debug(test_params)
monitor_gen = monitor_sut = False
background_machines = []
sut_machine = gen_machine = None
self.machines = []
for machine_params in test_params['machines']:
if 'gencores' in machine_params.keys():
machine = RapidGeneratorMachine(test_params['key'],
test_params['user'],
test_params['vim_type'],
test_params['rundir'],
machine_params,
test_params['ipv6'])
if machine_params['monitor']:
if monitor_gen:
RapidLog.exception("Can only monitor 1 generator")
raise Exception("Can only monitor 1 generator")
else:
monitor_gen = True
gen_machine = machine
else:
background_machines.append(machine)
else:
machine = RapidMachine(test_params['key'], test_params['user'],
test_params['vim_type'],
test_params['rundir'], machine_params)
if machine_params['monitor']:
if monitor_sut:
RapidLog.exception("Can only monitor 1 sut")
raise Exception("Can only monitor 1 sut")
else:
monitor_sut = True
if machine_params['prox_socket']:
sut_machine = machine
self.machines.append(machine)
prox_executor = concurrent.futures.ThreadPoolExecutor(
max_workers=len(self.machines))
self.future_to_prox = {
prox_executor.submit(machine.start_prox,
test_params['configonly']): machine
for machine in self.machines
}
if test_params['configonly']:
concurrent.futures.wait(self.future_to_prox,
return_when=ALL_COMPLETED)
sys.exit()
with concurrent.futures.ThreadPoolExecutor(
max_workers=len(self.machines)) as executor:
future_to_connect_prox = {
executor.submit(machine.connect_prox): machine
for machine in self.machines
}
concurrent.futures.wait(future_to_connect_prox,
return_when=ALL_COMPLETED)
result = True
for test_param in test_params['tests']:
RapidLog.info(test_param['test'])
if test_param['test'] in [
'flowsizetest', 'TST009test', 'fixed_rate',
'increment_till_fail'
]:
test = FlowSizeTest(test_param, test_params['lat_percentile'],
test_params['runtime'],
test_params['TestName'],
test_params['environment_file'],
gen_machine, sut_machine,
background_machines)
elif test_param['test'] in ['corestats']:
test = CoreStatsTest(test_param, test_params['runtime'],
test_params['TestName'],
test_params['environment_file'],
self.machines)
elif test_param['test'] in ['portstats']:
test = PortStatsTest(test_param, test_params['runtime'],
test_params['TestName'],
test_params['environment_file'],
self.machines)
elif test_param['test'] in ['impairtest']:
test = ImpairTest(test_param, test_params['lat_percentile'],
test_params['runtime'],
test_params['TestName'],
test_params['environment_file'], gen_machine,
sut_machine)
elif test_param['test'] in ['irqtest']:
test = IrqTest(test_param, test_params['runtime'],
test_params['TestName'],
test_params['environment_file'], self.machines)
elif test_param['test'] in ['warmuptest']:
test = WarmupTest(test_param, gen_machine)
else:
RapidLog.debug('Test name ({}) is not valid:'.format(
test_param['test']))
single_test_result = test.run()
if not single_test_result:
result = False
return (result)
def run_tests(test_params):
test_params = RapidConfigParser.parse_config(test_params)
RapidLog.debug(test_params)
monitor_gen = monitor_sut = False
background_machines = []
sut_machine = gen_machine = None
machines = []
for machine_params in test_params['machines']:
if 'gencores' in machine_params.keys():
machine = RapidGeneratorMachine(test_params['key'],
test_params['user'], test_params['vim_type'],
test_params['rundir'], machine_params,
test_params['ipv6'])
if machine_params['monitor']:
if monitor_gen:
RapidLog.exception("Can only monitor 1 generator")
raise Exception("Can only monitor 1 generator")
else:
monitor_gen = True
gen_machine = machine
else:
background_machines.append(machine)
else:
machine = RapidMachine(test_params['key'], test_params['user'],
test_params['vim_type'], test_params['rundir'],
machine_params)
if machine_params['monitor']:
if monitor_sut:
RapidLog.exception("Can only monitor 1 sut")
raise Exception("Can only monitor 1 sut")
else:
monitor_sut = True
sut_machine = machine
machines.append(machine)
if test_params['configonly']:
sys.exit()
for machine in machines:
machine.start_prox()
result = True
for test_param in test_params['tests']:
RapidLog.info(test_param['test'])
if test_param['test'] in ['flowsizetest', 'TST009test',
'fixed_rate', 'increment_till_fail']:
test = FlowSizeTest(test_param, test_params['lat_percentile'],
test_params['runtime'],
test_params['TestName'],
test_params['environment_file'], gen_machine,
sut_machine, background_machines)
elif test_param['test'] in ['corestats']:
test = CoreStatsTest(test_param, test_params['runtime'],
test_params['TestName'],
test_params['environment_file'], machines)
elif test_param['test'] in ['portstats']:
test = PortStatsTest(test_param, test_params['runtime'],
test_params['TestName'],
test_params['environment_file'], machines)
elif test_param['test'] in ['impairtest']:
test = ImpairTest(test_param, test_params['lat_percentile'],
test_params['runtime'],
test_params['TestName'],
test_params['environment_file'], gen_machine,
sut_machine)
elif test_param['test'] in ['irqtest']:
test = IrqTest(test_param, test_params['runtime'],
test_params['TestName'],
test_params['environment_file'], machines)
elif test_param['test'] in ['warmuptest']:
test = WarmupTest(test_param, gen_machine)
else:
RapidLog.debug('Test name ({}) is not valid:'.format(
test_param['test']))
single_test_result = test.run()
if not single_test_result:
result = False
return (result)