def run(self, **kwargs):
try:
test_params = RapidTestManager.get_defaults()
for key in kwargs:
test_params[key] = kwargs[key]
os.makedirs(self.res_dir, exist_ok=True)
test_params['resultsdir'] = self.res_dir
_, test_file_name = os.path.split(test_params['test_file'])
_, environment_file_name = os.path.split(
test_params['environment_file'])
log_file = '{}/RUN{}.{}.log'.format(self.res_dir,
environment_file_name,
test_file_name)
RapidLog.log_init(log_file, test_params['loglevel'],
test_params['screenloglevel'],
test_params['version'])
test_manager = RapidTestManager()
self.start_time = time.time()
self.result, self.details = test_manager.run_tests(test_params)
self.stop_time = time.time()
RapidLog.log_close()
except Exception: # pylint: disable=broad-except
print("Unexpected error:", sys.exc_info()[0])
self.result = 0
self.stop_time = time.time()
def IsDeployed(self, stack_name):
for stack in self.heatclient.stacks.list():
if stack.stack_name == stack_name:
RapidLog.info('Stack already existing: {}'.format(stack_name))
self.stack = stack
return True
return False
def post_data(self, test, variables):
var = copy.deepcopy(self.data_format)
self.parse_data_format_dict(var, variables)
if 'URL' not in var.keys():
return
if test not in var.keys():
return
URL = ''
for value in var['URL'].values():
URL = URL + value
HEADERS = {
'X-Requested-With': 'Python requests',
'Content-type': 'application/rapid'
}
if 'Format' in var.keys():
if var['Format'] == 'PushGateway':
data = "\n".join("{} {}".format(k, v)
for k, v in var[test].items()) + "\n"
response = requests.post(url=URL, data=data, headers=HEADERS)
elif var['Format'] == 'Xtesting':
data = var[test]
response = requests.post(url=URL, json=data)
else:
return
else:
return
if (response.status_code != 202) and (response.status_code != 200):
RapidLog.info('Cannot send metrics to {}'.format(URL))
RapidLog.info(data)
def set_udp_packet_size(self, imix_frame_sizes):
# We should check the gen.cfg to make sure we only send UDP packets
# If only 1 packet size, still using the 'old' way of setting the
# packet sizes in PROX. Otherwise, using the 'new' way which
# automatically sets IP and UDP sizes. We should switch to the new way
# eventually for all cases.
if len(imix_frame_sizes) == 1:
# Frame size = PROX pkt size + 4 bytes CRC
# The set_size function takes the PROX packet size as a parameter
self.socket.set_size(self.machine_params['gencores'], 0,
imix_frame_sizes[0] - 4)
# Writing length in the ip header
self.socket.set_value(
self.machine_params['gencores'], 0, self.ip_length_offset,
imix_frame_sizes[0] - self.frame_size_minus_ip_size, 2)
# Writing length in the udp header
self.socket.set_value(
self.machine_params['gencores'], 0, self.udp_length_offset,
imix_frame_sizes[0] -
self.frame_size_minus_udp_header_and_content, 2)
else:
if self.ipv6:
RapidLog.critical('IMIX not supported for IPV6')
prox_sizes = [frame_size - 4 for frame_size in imix_frame_sizes]
self.socket.set_imix(self.machine_params['gencores'], 0,
prox_sizes)
def start_prox(self, autostart=''):
if self.machine_params['prox_socket']:
self._client = prox_ctrl(self.ip, self.key, self.user,
self.password)
self._client.test_connection()
if self.vim in ['OpenStack']:
self.devbind()
if self.vim in ['kubernetes']:
self.read_cpuset()
self.read_cpuset_mems()
self.remap_all_cpus()
_, prox_config_file_name = os.path.split(
self.machine_params['config_file'])
if self.machine_params['prox_launch_exit']:
self.generate_lua()
self._client.scp_put(
self.machine_params['config_file'],
'{}/{}'.format(self.rundir, prox_config_file_name))
if not self.configonly:
cmd = 'sudo {}/prox {} -t -o cli -f {}/{}'.format(
self.rundir, autostart, self.rundir,
prox_config_file_name)
RapidLog.debug("Starting PROX on {}: {}".format(
self.name, cmd))
result = self._client.run_cmd(cmd)
RapidLog.debug("Finished PROX on {}: {}".format(
self.name, cmd))
def create_key(self):
keypair = self.nova_client.keypairs.create(name=self.key_name)
# Create a file for writing that can only be read and written by owner
fp = os.open(self.private_key_filename, os.O_WRONLY | os.O_CREAT,
0o600)
with os.fdopen(fp, 'w') as f:
f.write(keypair.private_key)
RapidLog.info('Keypair {} created'.format(self.key_name))
def _send(self, cmd):
"""Append LF and send command to the PROX instance."""
if self._sock is None:
raise RuntimeError("PROX socket closed, cannot send '%s'" % cmd)
try:
self._sock.sendall(cmd.encode() + b'\n')
except ConnectionResetError as e:
RapidLog.error('Pipe reset by Prox instance: traffic too high?')
raise
def start_prox(self, autostart=''):
if self.machine_params['prox_launch_exit']:
cmd = 'sudo {}/prox {} -t -o cli -f {}/{}'.format(
self.rundir, autostart, self.rundir,
self.machine_params['config_file'])
result = self._client.fork_cmd(
cmd, 'PROX Testing on {}'.format(self.name))
RapidLog.debug("Starting PROX on {}: {}, {}".format(
self.name, cmd, result))
self.socket = self._client.connect_socket()
def read_cpuset_mems(self):
"""Read list of NUMA nodes on which we allowed to allocate memory
"""
cmd = 'cat /sys/fs/cgroup/cpuset/cpuset.mems'
cpuset_mems = self._client.run_cmd(cmd).decode().rstrip()
RapidLog.debug('{} ({}): Allowed NUMA nodes: {}'.format(
self.name, self.ip, cpuset_mems))
self.numa_nodes = self.expand_list_format(cpuset_mems)
RapidLog.debug('{} ({}): Expanded allowed NUMA nodes: {}'.format(
self.name, self.ip, self.numa_nodes))
def main():
rapid_stack_params = {}
RapidStackManager.parse_config(rapid_stack_params)
log_file = 'CREATE{}.log'.format(rapid_stack_params['stack_name'])
RapidLog.log_init(log_file, 'DEBUG', 'INFO', '2021.03.15')
#cloud_name = 'openstackL6'
#stack_name = 'rapid'
#heat_template = 'openstack-rapid.yaml'
#heat_param = 'params_rapid.yaml'
#user = '******'
RapidStackManager.deploy_stack(rapid_stack_params)
def lat_stats(self, cores, tasks=[0]):
result = {}
result['lat_min'] = 999999999
result['lat_max'] = result['lat_avg'] = 0
result['buckets'] = [0] * 128
result['mis_ordered'] = 0
result['extent'] = 0
result['duplicate'] = 0
number_tasks_returning_stats = 0
self._send('lat all stats %s %s' %
(','.join(map(str, cores)), ','.join(map(str, tasks))))
for core in cores:
for task in tasks:
stats = self._recv().split(',')
if 'is not measuring' in stats[0]:
continue
if stats[0].startswith('error'):
RapidLog.critical("lat stats error: unexpected reply from PROX\
(potential incompatibility between scripts and PROX)")
raise Exception("lat stats error")
number_tasks_returning_stats += 1
result['lat_min'] = min(int(stats[0]), result['lat_min'])
result['lat_max'] = max(int(stats[1]), result['lat_max'])
result['lat_avg'] += int(stats[2])
#min_since begin = int(stats[3])
#max_since_begin = int(stats[4])
result['lat_tsc'] = int(
stats[5]) # Taking the last tsc as the timestamp since
# PROX will return the same tsc for each
# core/task combination
result['lat_hz'] = int(stats[6])
#coreid = int(stats[7])
#taskid = int(stats[8])
result['mis_ordered'] += int(stats[9])
result['extent'] += int(stats[10])
result['duplicate'] += int(stats[11])
stats = self._recv().split(':')
if stats[0].startswith('error'):
RapidLog.critical("lat stats error: unexpected lat bucket \
reply (potential incompatibility between scripts \
and PROX)")
raise Exception("lat bucket reply error")
result['buckets'][0] = int(stats[1])
for i in range(1, 128):
stats = self._recv().split(':')
result['buckets'][i] = int(stats[1])
result['lat_avg'] = old_div(result['lat_avg'],
number_tasks_returning_stats)
self._send('stats latency(0).used')
used = float(self._recv())
self._send('stats latency(0).total')
total = float(self._recv())
result['lat_used'] = old_div(used, total)
return (result)
def devbind(self):
# Script to bind the right network interface to the poll mode driver
for index, dp_port in enumerate(self.dp_ports, start = 1):
DevBindFileName = self.rundir + '/devbind-{}-port{}.sh'.format(self.ip, index)
self._client.scp_put('./devbind.sh', DevBindFileName)
cmd = 'sed -i \'s/MACADDRESS/' + dp_port['mac'] + '/\' ' + DevBindFileName
result = self._client.run_cmd(cmd)
RapidLog.debug('devbind.sh MAC updated for port {} on {} {}'.format(index, self.name, result))
if ((not self.configonly) and self.machine_params['prox_launch_exit']):
result = self._client.run_cmd(DevBindFileName)
RapidLog.debug('devbind.sh running for port {} on {} {}'.format(index, self.name, result))
def run_cmd(self, command, _connect=False):
"""Execute command over ssh on remote system.
Wait for remote command completion.
Return command output (combined stdout and stderr).
_connect argument is reserved for connect() method.
"""
cmd = self._build_ssh(command)
try:
return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
RapidLog.exception('ssh returned exit status %d:\n%s' %
(ex.returncode, ex.output.strip()))
def main():
"""Main function.
"""
test_params = RapidTestManager.get_defaults()
# When no cli is used, the process_cli can be replaced by code modifying
# test_params
test_params = RapidCli.process_cli(test_params)
log_file = 'RUN{}.{}.log'.format(test_params['environment_file'],
test_params['test_file'])
RapidLog.log_init(log_file, test_params['loglevel'],
test_params['screenloglevel'] , test_params['version'] )
test_result = RapidTestManager.run_tests(test_params)
RapidLog.info('Test result is : {}'.format(test_result))
def start_prox(self, configonly=False, autostart=''):
if self.machine_params['prox_socket']:
self._client = prox_ctrl(self.ip, self.key, self.user)
self._client.connect()
if self.vim in ['OpenStack']:
self.devbind(configonly)
self.generate_lua(self.vim)
self._client.scp_put(self.machine_params['config_file'], '{}/{}'.format(self.rundir, self.machine_params['config_file']))
if ((not configonly) and self.machine_params['prox_launch_exit']):
cmd = 'sudo {}/prox {} -t -o cli -f {}/{}'.format(self.rundir, autostart, self.rundir, self.machine_params['config_file'])
RapidLog.debug("Starting PROX on {}: {}".format(self.name, cmd))
result = self._client.run_cmd(cmd, 'PROX Testing on {}'.format(self.name))
#RapidLog.debug("Finished PROX on {}: {}, {}".format(self.name, cmd, result))
RapidLog.debug("Finished PROX on {}: {}".format(self.name, cmd))
def generate_lua(self, appendix=''):
self.LuaFileName = 'parameters-{}.lua'.format(self.ip)
with open(self.LuaFileName, "w") as LuaFile:
LuaFile.write('require "helper"\n')
LuaFile.write('name="%s"\n' % self.name)
for index, dp_port in enumerate(self.dp_ports, start=1):
LuaFile.write('local_ip{}="{}"\n'.format(index, dp_port['ip']))
LuaFile.write(
'local_hex_ip{}=convertIPToHex(local_ip{})\n'.format(
index, index))
if self.vim in ['kubernetes']:
socket_mem_str = self.get_prox_socket_mem_str()
RapidLog.debug('{} ({}): PROX socket mem str: {}'.format(
self.name, self.ip, socket_mem_str))
LuaFile.write(
"eal=\"--socket-mem=%s --file-prefix %s --pci-whitelist %s\"\n"
% (socket_mem_str, self.name,
self.machine_params['dp_pci_dev']))
else:
LuaFile.write("eal=\"\"\n")
if 'mcore' in self.machine_params.keys():
LuaFile.write('mcore="%s"\n' %
','.join(map(str, self.machine_params['mcore'])))
if 'cores' in self.machine_params.keys():
LuaFile.write('cores="%s"\n' %
','.join(map(str, self.machine_params['cores'])))
if 'ports' in self.machine_params.keys():
LuaFile.write('ports="%s"\n' %
','.join(map(str, self.machine_params['ports'])))
if 'dest_ports' in self.machine_params.keys():
for index, dest_port in enumerate(
self.machine_params['dest_ports'], start=1):
LuaFile.write('dest_ip{}="{}"\n'.format(
index, dest_port['ip']))
LuaFile.write(
'dest_hex_ip{}=convertIPToHex(dest_ip{})\n'.format(
index, index))
LuaFile.write('dest_hex_mac{}="{}"\n'.format(
index, dest_port['mac'].replace(':', ' ')))
if 'gw_vm' in self.machine_params.keys():
for index, gw_ip in enumerate(self.machine_params['gw_ips'],
start=1):
LuaFile.write('gw_ip{}="{}"\n'.format(index, gw_ip))
LuaFile.write(
'gw_hex_ip{}=convertIPToHex(gw_ip{})\n'.format(
index, index))
LuaFile.write(appendix)
self._client.scp_put(self.LuaFileName, self.rundir + '/parameters.lua')
self._client.scp_put('helper.lua', self.rundir + '/helper.lua')
def _recv(self):
"""Receive response from PROX instance, return it with LF removed."""
if self._sock is None:
raise RuntimeError("PROX socket closed, cannot receive anymore")
try:
pos = self._rcvd.find(b'\n')
while pos == -1:
self._rcvd += self._sock.recv(256)
pos = self._rcvd.find(b'\n')
rsp = self._rcvd[:pos]
self._rcvd = self._rcvd[pos + 1:]
except ConnectionResetError as e:
RapidLog.error('Pipe reset by Prox instance: traffic too high?')
raise
return rsp.decode()
def main():
"""Main function.
"""
test_params = RapidTestManager.get_defaults()
# When no cli is used, the process_cli can be replaced by code modifying
# test_params
test_params = RapidCli.process_cli(test_params)
_, test_file_name = os.path.split(test_params['test_file'])
_, environment_file_name = os.path.split(test_params['environment_file'])
log_file = 'RUN{}.{}.log'.format(environment_file_name, test_file_name)
RapidLog.log_init(log_file, test_params['loglevel'],
test_params['screenloglevel'], test_params['version'])
test_manager = RapidTestManager()
test_result, _ = test_manager.run_tests(test_params)
RapidLog.log_close()
def lat_stats(self, cores, tasks=[0]):
min_lat = 999999999
max_lat = avg_lat = 0
number_tasks_returning_stats = 0
buckets = [0] * 128
self._send('lat all stats %s %s' %
(','.join(map(str, cores)), ','.join(map(str, tasks))))
for core in cores:
for task in tasks:
stats = self._recv().split(',')
if 'is not measuring' in stats[0]:
continue
if stats[0].startswith('error'):
RapidLog.critical("lat stats error: unexpected reply from PROX\
(potential incompatibility between scripts and PROX)")
raise Exception("lat stats error")
number_tasks_returning_stats += 1
min_lat = min(int(stats[0]), min_lat)
max_lat = max(int(stats[1]), max_lat)
avg_lat += int(stats[2])
#min_since begin = int(stats[3])
#max_since_begin = int(stats[4])
tsc = int(stats[5]) # Taking the last tsc as the timestamp since
# PROX will return the same tsc for each
# core/task combination
hz = int(stats[6])
#coreid = int(stats[7])
#taskid = int(stats[8])
stats = self._recv().split(':')
if stats[0].startswith('error'):
RapidLog.critical("lat stats error: unexpected lat bucket \
reply (potential incompatibility between scripts \
and PROX)")
raise Exception("lat bucket reply error")
buckets[0] = int(stats[1])
for i in range(1, 128):
stats = self._recv().split(':')
buckets[i] = int(stats[1])
avg_lat = old_div(avg_lat, number_tasks_returning_stats)
self._send('stats latency(0).used')
used = float(self._recv())
self._send('stats latency(0).total')
total = float(self._recv())
return (min_lat, max_lat, avg_lat, (old_div(used,
total)), tsc, hz, buckets)
def create_key(self):
if os.path.exists(self.key_name):
public_key_file = "{}.pub".format(self.key_name)
if not os.path.exists(public_key_file):
RapidLog.critical('Keypair {}.pub does not exist'.format(
self.key_name))
with open(public_key_file, mode='rb') as public_file:
public_key = public_file.read()
else:
public_key = None
keypair = self.nova_client.keypairs.create(name=self.key_name,
public_key=public_key)
# Create a file for writing that can only be read and written by owner
if not os.path.exists(self.key_name):
fp = os.open(self.key_name, os.O_WRONLY | os.O_CREAT, 0o600)
with os.fdopen(fp, 'w') as f:
f.write(keypair.private_key)
RapidLog.info('Keypair {} created'.format(self.key_name))
def multi_port_stats(self, ports=[0]):
rx = tx = port_id = tsc = no_mbufs = errors = 0
self._send('multi port stats %s' % (','.join(map(str, ports))))
result = self._recv().split(';')
if result[0].startswith('error'):
RapidLog.critical("multi port stats error: unexpected invalid \
syntax (potential incompatibility between scripts and \
PROX)")
raise Exception("multi port stats error")
for statistics in result:
stats = statistics.split(',')
port_id = int(stats[0])
rx += int(stats[1])
tx += int(stats[2])
no_mbufs += int(stats[3])
errors += int(stats[4])
tsc = int(stats[5])
return rx, tx, no_mbufs, errors, tsc
def connect(self):
attempts = 1
RapidLog.debug("Trying to connect to machine \
on %s, attempt: %d" % (self._ip, attempts))
while True:
try:
self.test_connect()
break
except RuntimeWarning as ex:
RapidLog.debug("RuntimeWarning %d:\n%s" %
(ex.returncode, ex.output.strip()))
attempts += 1
if attempts > 20:
RapidLog.exception("Failed to connect to instance after %d\
attempts:\n%s" % (attempts, ex))
time.sleep(2)
RapidLog.debug("Trying to connect to machine \
on %s, attempt: %d" % (self._ip, attempts))
RapidLog.debug("Connected to machine on %s" % self._ip)
def create_stack(self, stack_name, stack_file_path, heat_parameters):
files, template = template_utils.process_template_path(stack_file_path)
stack_created = self.heatclient.stacks.create(
stack_name=stack_name,
template=template,
parameters=heat_parameters,
files=files)
stack = self.heatclient.stacks.get(stack_created['stack']['id'],
resolve_outputs=True)
# Poll at 5 second intervals, until the status is no longer 'BUILD'
while stack.stack_status == 'CREATE_IN_PROGRESS':
print('waiting..')
time.sleep(5)
stack = self.heatclient.stacks.get(stack_created['stack']['id'],
resolve_outputs=True)
if stack.stack_status == 'CREATE_COMPLETE':
return stack
else:
RapidLog.exception('Error in stack deployment')
def scp_get(self, src, dst):
"""Copy src file from remote system to dst on local system."""
cmd = [
'scp', '-B', '-oStrictHostKeyChecking=no',
'-oUserKnownHostsFile=/dev/null', '-oLogLevel=ERROR'
]
if self._key is not None:
cmd.extend(['-i', self._key])
remote = ''
if self._user is not None:
remote += self._user + '@'
remote += self._ip + ':/home/' + self._user + src
cmd.append(remote)
cmd.append(dst)
try:
# Actually ignore output on success, but capture stderr on failure
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
RapidLog.exception('scp returned exit status %d:\n%s' %
(ex.returncode, ex.output.strip()))
def remap_all_cpus(self):
"""Convert relative cpu ids for different parameters (gencores, latcores)
"""
super().remap_all_cpus()
if self.cpu_mapping is None:
return
if 'gencores' in self.machine_params.keys():
cpus_remapped = super().remap_cpus(self.machine_params['gencores'])
RapidLog.debug('{} ({}): gencores {} remapped to {}'.format(
self.name, self.ip, self.machine_params['gencores'],
cpus_remapped))
self.machine_params['gencores'] = cpus_remapped
if 'latcores' in self.machine_params.keys():
cpus_remapped = super().remap_cpus(self.machine_params['latcores'])
RapidLog.debug('{} ({}): latcores {} remapped to {}'.format(
self.name, self.ip, self.machine_params['latcores'],
cpus_remapped))
self.machine_params['latcores'] = cpus_remapped
def core_stats(self, cores, tasks=[0]):
rx = tx = drop = tsc = hz = rx_non_dp = tx_non_dp = tx_fail = 0
self._send('dp core stats %s %s' %
(','.join(map(str, cores)), ','.join(map(str, tasks))))
for core in cores:
for task in tasks:
stats = self._recv().split(',')
if stats[0].startswith('error'):
if stats[0].startswith('error: invalid syntax'):
RapidLog.critical("dp core stats error: unexpected \
invalid syntax (potential incompatibility \
between scripts and PROX)")
raise Exception("dp core stats error")
continue
rx += int(stats[0])
tx += int(stats[1])
rx_non_dp += int(stats[2])
tx_non_dp += int(stats[3])
drop += int(stats[4])
tx_fail += int(stats[5])
tsc = int(stats[6])
hz = int(stats[7])
return rx, rx_non_dp, tx, tx_non_dp, drop, tx_fail, tsc, hz
def warm_up(self):
# Running at low speed to make sure the ARP messages can get through.
# If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
# Note however that if we would run the test steps during a very long time, the ARP would expire in the switch.
# PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
imix = self.test['warmupimix']
FLOWSIZE = self.test['warmupflowsize']
WARMUPSPEED = self.test['warmupspeed']
WARMUPTIME = self.test['warmuptime']
RapidLog.info(("Warming up during {} seconds..., packet size = {},"
" flows = {}, speed = {}").format(WARMUPTIME, imix, FLOWSIZE,
WARMUPSPEED))
self.gen_machine.set_generator_speed(WARMUPSPEED)
self.set_background_speed(self.background_machines, WARMUPSPEED)
self.gen_machine.set_udp_packet_size(imix)
self.set_background_size(self.background_machines, imix)
_ = self.gen_machine.set_flows(FLOWSIZE)
self.set_background_flows(self.background_machines, FLOWSIZE)
self.gen_machine.start()
self.start_background_traffic(self.background_machines)
time.sleep(WARMUPTIME)
self.stop_background_traffic(self.background_machines)
self.gen_machine.stop()
def connect_socket(self):
attempts = 1
RapidLog.debug("Trying to connect to PROX (just launched) on %s, \
attempt: %d" % (self._ip, attempts))
sock = None
while True:
sock = self.prox_sock()
if sock is not None:
break
attempts += 1
if attempts > 20:
RapidLog.exception("Failed to connect to PROX on %s after %d \
attempts" % (self._ip, attempts))
time.sleep(2)
RapidLog.debug("Trying to connect to PROX (just launched) on %s, \
attempt: %d" % (self._ip, attempts))
RapidLog.info("Connected to PROX on %s" % self._ip)
return sock
def connect(self):
attempts = 1
RapidLog.debug("Trying to connect to instance which was just launched \
on %s, attempt: %d" % (self._ip, attempts))
while True:
try:
self.test_connect()
break
except RuntimeWarning as ex:
attempts += 1
if attempts > 20:
RapidLog.exception("Failed to connect to instance after %d\
attempts:\n%s" % (attempts, ex))
raise Exception("Failed to connect to instance after %d \
attempts:\n%s" % (attempts, ex))
time.sleep(2)
RapidLog.debug("Trying to connect to instance which was just \
launched on %s, attempt: %d" % (self._ip, attempts))
RapidLog.debug("Connected to instance on %s" % self._ip)
def read_cpuset(self):
"""Read list of cpus on which we allowed to execute
"""
cmd = 'cat /sys/fs/cgroup/cpuset/cpuset.cpus'
cpuset_cpus = self._client.run_cmd(cmd).decode().rstrip()
RapidLog.debug('{} ({}): Allocated cpuset: {}'.format(self.name, self.ip, cpuset_cpus))
self.cpu_mapping = self.expand_cpuset(cpuset_cpus)
RapidLog.debug('{} ({}): Expanded cpuset: {}'.format(self.name, self.ip, self.cpu_mapping))
# Log CPU core mapping for user information
cpu_mapping_str = ''
for i in range(len(self.cpu_mapping)):
cpu_mapping_str = cpu_mapping_str + '[' + str(i) + '->' + str(self.cpu_mapping[i]) + '], '
cpu_mapping_str = cpu_mapping_str[:-2]
RapidLog.debug('{} ({}): CPU mapping: {}'.format(self.name, self.ip, cpu_mapping_str))