def main():
opts, args = getopt.getopt(sys.argv[1:], 'vt:xq')
has_xplot = False
open_page = True
for opt, val in opts:
if opt == '-v':
continue
elif opt == '-x':
has_xplot = True
elif opt == '-q':
open_page = False
else:
print_usage()
return -1
log.setup_logging(opts)
data_dir = 'LATEST' if not args else args[0]
threads = []
if has_xplot:
t = threading.Thread(target=gen_xplots, args=[data_dir])
threads.append(t)
t.start()
for t in threads:
t.join()
return ret
def main():
ip_mode = socket.AF_INET6
opts, _ = getopt.getopt(sys.argv[1:], 'vp:n:s',
['ifacecfg=', 'ip_mode=',
'no_pcap', 'no_kern_debug',
'hosts='])
listenport = 6324
node = None # Optional; None means we use InterfaceConfig.default_cfg.
ifacecfg = None
singlesrv_mode = False
save_pcap = True
save_kern_debug = True
hosts = None
for opt, val in opts:
if opt == '-v':
continue
elif opt == '-p':
listenport = int(val)
elif opt == '--ifacecfg':
ifacecfg = os.path.abspath(os.path.expanduser(val))
elif opt == '-n':
node = val
elif opt == '-s':
singlesrv_mode = True
elif opt == '--ip_mode':
key = int(val)
assert key in ip_modes, 'ip_mode must be in %s' % str(
ip_modes.keys())
ip_mode = ip_modes[key]
elif opt == '--no_pcap':
save_pcap = False
elif opt == '--no_kern_debug':
save_kern_debug = False
elif opt == '--hosts':
hosts = os.path.abspath(os.path.expanduser(val))
else:
print_usage()
return -1
log.setup_logging(opts)
addr = (listen_addrs[ip_mode], listenport)
LOG.info('starting sender on %s:%d', addr[0], addr[1])
LOG.info('socket.gethostname() = %s', socket.gethostname())
ifaces = transperf.InterfaceConfig.default_cfg
if ifacecfg is not None:
ifaces = transperf.InterfaceConfig.node_config(ifacecfg, node, LOG)
sender = Sender(ifaces, singlesrv_mode, ip_mode, save_pcap, save_kern_debug,
hosts)
sender.listen(addr)
LOG.info('sender stopped')
return 0
def main():
output = 'diff.html'
opts, args = getopt.getopt(sys.argv[1:], 'vo:')
for opt, val in opts:
if opt == '-v':
continue
elif opt == '-o':
output = val
else:
print_usage()
return -1
if len(args) < 2:
print_usage()
return -1
log.setup_logging(opts)
_gen_diff(args, output)
def main():
invocation_dir = os.getcwd()
source_dir = os.path.dirname(os.path.realpath(__file__))
out_dir_rel = _timestamp_dirname()
out_dir = os.path.join(invocation_dir, out_dir_rel)
opts, args = getopt.getopt(sys.argv[1:], 'vo:b:c:s:nt:xyhq:V', [
'rport=',
'sport=',
'help',
'skip_pcap_scan',
'sslog_interval=',
'ifacecfg=',
'bridge=',
'ssrv=',
'ssrv_local',
'save=',
'genonly',
'gen_from_file=',
'virtcleanup',
'ip_mode=',
'no_pcap',
'no_kern_debug',
])
# Setup logging early.
log.setup_logging(opts)
# These are arbitrary ports that must be open between test machines.
ip_mode = 4
rport, sport = 6200, 6300
sync = True
has_xplot = False
binary_dirs = [invocation_dir]
if invocation_dir != source_dir:
binary_dirs.append(source_dir)
data_search_path = []
open_page = True
skip_pcap_scan = False
sslog_interval = 0.1
singlesrv_cfg = {
'enabled': False,
'bridge': virtsetup.Constants.DEFAULT_BRIDGE,
'host': None,
'local': False,
'nodes': [],
'cleanup': False
}
ifacecfg = None
save_dir = None
gen_only = False
gen_from_file = None
save_pcap = True
save_kern_debug = True
# cfgs includes all configs both in files and in command line.
cfgs = []
for opt, val in opts:
if opt == '-V':
print("transperf {}".format(transperf.__version__))
return
if opt == '-v':
continue
elif opt == '--ip_mode':
ip_mode = int(val)
assert ip_mode in [4, 6], '--ip_mode must be in [4 (default), 6]'
elif opt == '--virtcleanup':
singlesrv_cfg['cleanup'] = True
elif opt == '-o':
out_dir_rel = val
out_dir = abspath(out_dir_rel)
elif opt == '-b':
binary_dirs.extend([abspath(path) for path in val.split(',')])
data_search_path.extend(binary_dirs)
elif opt == '-c':
cfg_paths = [abspath(path) for path in val.split(',')]
for path in cfg_paths:
if os.path.isfile(path):
data_search_path.append(os.path.dirname(path))
cfgs.append(_read_file(path))
continue
data_search_path.append(path)
cfgs += [
_read_file(cfg_file)
for cfg_file in transperf.path.list_files(path)
]
elif opt == '-s':
cfgs.append(val)
elif opt == '-n':
sync = False
elif opt == '--rport':
rport = int(val)
elif opt == '--sport':
sport = int(val)
elif opt == '-x':
has_xplot = True
elif opt == '-q':
open_page = False
elif opt == '--skip_pcap_scan':
skip_pcap_scan = True
elif opt == '--no_pcap':
save_pcap = False
skip_pcap_scan = True # Since we have no other way to get metrics.
elif opt == '--no_kern_debug':
save_kern_debug = False
elif opt == '--genonly':
gen_only = True
elif opt == '--gen_from_file':
gen_from_file = '-' if val == '-' else (os.path.abspath(
os.path.expanduser(val)))
elif opt == '--sslog_interval':
sslog_interval = float(val)
elif opt == '--ifacecfg':
ifacecfg = abspath(val)
elif opt == '--ssrv':
if '_' in val:
assert False, 'Cannot have underscore in hostname for --ssrv.'
if singlesrv_cfg['local']:
assert False, 'Cannot set both --ssrv and --ssrv_local at once.'
singlesrv_cfg['enabled'] = True
singlesrv_cfg['host'] = val
elif opt == '--ssrv_local':
if singlesrv_cfg['host']:
assert False, 'Cannot set both --ssrv and --ssrv_local at once.'
singlesrv_cfg['enabled'] = True
singlesrv_cfg['local'] = True
elif opt == '--save':
save_dir = abspath(val)
elif opt == '--bridge':
curr_val = singlesrv_cfg['bridge']
singlesrv_cfg['bridge'] = val if val is not None else curr_val
elif (opt == '-h' or opt == '--help'):
print_usage()
return -1
else: # Catch-all for unexpected flags.
print_usage()
return -1
# After processing the input paths, we change directory so we can
# stage/invoke other source files within transperf. Special case though: we
# may be invoking from a zip file. In that case, it's hard to know what
# directory contains the unzipped source, so we just don't bother.
if os.path.isdir(source_dir):
os.chdir(source_dir)
# Short circuit if we're generating an output webpage for previous test run.
if gen_only:
_mkdir_if_not_exists(out_dir)
return _process_output(out_dir, has_xplot, open_page, skip_pcap_scan,
gen_from_file)
else:
assert gen_from_file is None, ('--gen_from_file only meaningful '
'if --genonly specified.')
if not args:
print_usage()
sys.exit(-1)
if not cfgs:
raise RuntimeError('no configuration found')
LOG.debug('%d config(s) loaded: %s', len(cfgs), cfgs)
_mkdir_if_not_exists(out_dir)
_dump_cfgs(cfgs, data_search_path, out_dir)
ifacecfg_dir = os.path.join(out_dir, transperf.path.IFACE_CFG_DIR)
_mkdir_if_not_exists(ifacecfg_dir)
ifacecfg_rel = _validate_and_stage_ifacecfg(ifacecfg, ifacecfg_dir)
# Grab receiver and sender hostnames.
recvh, _, recvh_internal = args[0].partition('/')
sendhs, _, sendhs_internal = [
list(t) for t in zip(*[_.partition('/') for _ in args[1:]])
]
# Check for duplicates.
nodeset = set([recvh] + sendhs)
if len(nodeset) != len(sendhs) + 1:
# There was repetition, which we do not support.
raise RuntimeError('There are repeated nodes in the arguments!')
# Are we using ssh or are we local? For debug statements.
session_type = 'ssh'
if singlesrv_cfg['enabled']:
if singlesrv_cfg['local']:
session_type = 'local'
else:
assert singlesrv_cfg['host']
if singlesrv_cfg['enabled']:
# Strip usernames for single server mode; we must use root.
recvh = recvh.split('@')[-1]
sendhs = [sendh.split('@')[-1] for sendh in sendhs]
nodes = [recvh] + sendhs
singlesrv_cfg['nodes'] = nodes
singlesrv_cfg['scratchd'] = (os.path.join(
transperf.path.get_transperf_home(), 'containers'))
singlesrv_cfg['out_dir'] = os.path.join(singlesrv_cfg['scratchd'],
out_dir_rel)
node_exec_cfgs = get_container_node_exec_cfgs(singlesrv_cfg, nodes)
else:
nodes = [recvh] + sendhs
nodes_internal = [recvh_internal] + sendhs_internal
node_exec_cfgs = {
node: {
'ssh': node,
'int_ip': ip,
'cfg': None
}
for node, ip in zip(nodes, nodes_internal)
}
LOG.info('creating %s session to %s', session_type, recvh)
r_exec = executor.Executor(node_exec_cfgs[recvh]['ssh'],
internal_ip=node_exec_cfgs[recvh]['int_ip'],
container_params=node_exec_cfgs[recvh]['cfg'])
LOG.info('creating %s sessions to %s', session_type, sendhs)
s_execs = [
executor.Executor(node_exec_cfgs[sendh]['ssh'],
internal_ip=node_exec_cfgs[sendh]['int_ip'],
container_params=node_exec_cfgs[sendh]['cfg'])
for sendh in sendhs
]
_, staged_src = _stage_transperf_src()
if singlesrv_cfg['enabled']:
# In this case all executor sessions just point to the same box,
# so we can just reuse r_exec in use_rootns mode.
_init_containers(
r_exec, singlesrv_cfg, nodes, staged_src,
os.path.join(ifacecfg_dir, ifacecfg_rel)
if ifacecfg_rel is not None else None, ip_mode)
# Copy the container hosts file over to the test output directory.
singlesrv_cfg['hosts'] = os.path.join(singlesrv_cfg['out_dir'],
'hosts')
if singlesrv_cfg['enabled']:
if singlesrv_cfg['local']:
# launch.py and the send/recv/orch processes are all on the same
# node, and out_dir is accessible from all of them.
out_dir_for_servers = out_dir
else:
# launch.py is local but send/recv/orch are remote; point them to
# their own directories.
out_dir_for_servers = os.path.join(
singlesrv_cfg['out_dir'], 'fs', '{node}',
transperf.path.EXP_OUT_DIR.lstrip('/'))
else:
out_dir_for_servers = os.path.join(transperf.path.TRANSPERF_TMP)
_init_servers(r_exec, s_execs, binary_dirs, out_dir, sync, staged_src,
singlesrv_cfg)
_start_servers(r_exec, s_execs, rport, sport, sslog_interval, ifacecfg_rel,
singlesrv_cfg, ip_mode, save_pcap, save_kern_debug,
out_dir_for_servers)
_collect_results(r_exec, s_execs, out_dir, singlesrv_cfg)
retcode = _process_output(out_dir, has_xplot, open_page, skip_pcap_scan)
# Save a copy of the results (e.g. for debugging, wher outdir vanishes).
if save_dir is not None:
LOG.info('Saving a copy of results to %s', save_dir)
_mkdir_if_not_exists(save_dir)
shell.run('cp -r {out} {save}'.format(out=out_dir,
save=save_dir + os.path.sep))
else:
LOG.info('Saving a copy of results not requested, skipping.')
# Cleanup virtual environment if specified/relevant.
if singlesrv_cfg['enabled'] and singlesrv_cfg['cleanup']:
tgt_dir = singlesrv_cfg['out_dir']
cmd = shell.py_cmd([], 'virtcleanup.py', '-v', '-d', tgt_dir)
exec_ctx = shell if singlesrv_cfg['local'] else r_exec
out, err, returncode = exec_ctx.run(cmd)
LOG.info('Cleanup output: [%d] [%s] stderr: [%s]', returncode, out,
err)
return retcode
def main():
ip_mode = socket.AF_INET6
raddr_val = None
out_dir = None
hosts = None
opts, _ = getopt.getopt(sys.argv[1:], 'vr:s:', ['sslog_interval=',
'ip_mode=',
'out_dir=',
'hosts='])
for opt, val in opts:
if opt == '-v':
continue
elif opt == '-r':
raddr_val = val
elif opt == '-s':
saddrs = val.split(',')
elif opt == '--out_dir':
out_dir = os.path.abspath(os.path.expanduser(val))
elif opt == '--sslog_interval':
sslog_interval = float(val)
elif opt == '--ip_mode':
key = int(val)
assert key in ip_modes, 'ip_mode must be in %s' % str(
ip_modes.keys())
ip_mode = ip_modes[key]
elif opt == '--hosts':
hosts = os.path.abspath(os.path.expanduser(val))
else:
print_usage()
return -1
log.setup_logging(opts)
assert out_dir is not None, 'Missing output directory'
assert raddr_val is not None, 'Missing receiver address.'
if hosts is not None:
ip_map = parse_ip_map(hosts)
LOG.info('IP map: %s', ip_map)
else:
ip_map = {}
LOG.info('No hosts provided to orchestrator.')
raddr = _replace_host_with_ip(raddr_val, 3, ip_mode, ip_map)
cfgs = _load_configs()
rproxy = _create_proxy_with_retry('http://%s/' % (raddr), 10)
sproxies = []
for addr in saddrs:
addr = _replace_host_with_ip(addr, 3, ip_mode, ip_map)
sproxy = _create_proxy_with_retry('http://%s/' % (addr), 10)
receiver = ':'.join(raddr.split(':')[:-1])
sender = ':'.join(addr.split(':')[:-1])
sproxy.register_receiver(receiver)
rproxy.register_sender(sender)
sproxies.append((addr, sproxy))
for i, cfg in enumerate(cfgs):
for exp in cfg.experiments():
out_dir_rel = os.path.join(str(i), exp.get_dir_name())
_run_experiment(exp, out_dir, out_dir_rel, rproxy, sproxies,
sslog_interval)
return 0
def main():
"""main() initializes a container environment for singleserver transperf.
launch.py will call main() over ssh to set up a singleserver
transperf environment if flagged to do so. main() can also be
invoked manually to provide a quick flat L2 container based environment to
play around in.
Returns:
0 upon successful exit.
"""
# Setup CLI args
parser = argparse.ArgumentParser(description='transperf container setup.')
parser.add_argument('rcvnode', nargs=1, help='Receiver node (1)')
parser.add_argument('sndnodes', nargs='+', help='Sender nodes (>=1)')
parser.add_argument('-b',
nargs='?',
const=Constants.DEFAULT_BRIDGE,
default=Constants.DEFAULT_BRIDGE,
dest='brdev',
help='Specify bridge device')
parser.add_argument('-d',
nargs='?',
const=None,
default=None,
dest='out_dir',
help='Specify out directory')
parser.add_argument('--demo',
action='store_true',
default=False,
dest='demo',
help='Setup basic NICs')
parser.add_argument('-v',
action='store_true',
default=False,
dest='debug',
help='Enabled debug output')
parser.add_argument('--ifacecfg',
nargs='?',
dest='ifacecfg',
help='Node interfaces config.')
parser.add_argument('--ip_mode',
nargs='?',
dest='ip_mode',
help='IP mode (4 or 6 (default)).',
default=6)
# Get args
args = parser.parse_args()
# Setup logging.
log.setup_logging([(arg, getattr(args, arg)) for arg in vars(args)])
ip_mode = transperf.ip_modes[int(args.ip_mode)]
brdev = args.brdev
demo_mode = args.demo
nodes = args.rcvnode + args.sndnodes
if args.ifacecfg is not None:
iface_cfgname = args.ifacecfg
ifacecfg = os.path.abspath(os.path.expanduser(iface_cfgname))
node_cfg = transperf.InterfaceConfig.validate_config(ifacecfg, LOG)
else:
iface_cfgname = 'default'
node_cfg = {node: dict(Constants.DEFAULT_NODE_CFG) for node in nodes}
# For receiver node, we need to disable NIC offloads.
for rcvr in args.rcvnode:
LOG.debug('Disabling root/container offloads for receiver %s', rcvr)
node_cfg[rcvr]['root_nic_offloads_enabled'] = False
node_cfg[rcvr]['container_nic_offloads_enabled'] = False
# Create output directory.
out_dir = args.out_dir
if out_dir is not None:
out_dir = os.path.abspath(os.path.expanduser(args.out_dir))
else:
out_dir = os.path.join(Constants.DEFAULT_OUTDIR_BASE,
Utils.timestamp_dirname())
try:
os.makedirs(out_dir)
except OSError:
assert os.path.isdir(out_dir), ('Output dir %s does not exist '
'and cannot be created.' % out_dir)
# Setup container environment.
ctx = ContainerCtx(brdev, nodes, out_dir, ip_mode)
ctx.setup_container_environment()
# Setup basic connectivity in demo mode.
if demo_mode:
demo_cfg = ctx.default_node_iface_setup(nodes)
raw_input('Press enter to remove all interfaces for test.')
ctx.cleanup_all_container_interfaces(demo_cfg)
raw_input('Press enter to re-add all interfaces for test.')
ctx.setup_all_container_interfaces(demo_cfg)
print('\nDemo mode complete; leaving containers running.')
else:
# Setup requested connectivity.
ctx.setup_all_container_interfaces(node_cfg)
return 0