def get_coverage_stats(coverage_map):
"""
Collect stats of coverage counters. In every sampling iteration, these stats are stored
in corresponding sampling slots.
Parameters
----------
coverage_map : dict
mapping of coverage class and its coverage object.
Raises
------
OsCommandExc
if the given OS command did not succeed for some reason.
"""
nlog = Context.nlog
# retrieve required data from the vswitch.
cmd = "ovs-appctl coverage/show"
data = util.exec_host_command(cmd)
if not data:
raise OsCommandExc("unable to collect data")
for line in data.splitlines():
# In below matching line, retrieve upcall_flow_limit_hit count
if line.startswith("upcall_flow_limit_hit"):
linesre = re.split(r'\s{2,}', line)
name, count = linesre[-1].split(": ")
count = int(count)
# if coverage object exists modify it by adding new sample
if "coverage" in coverage_map:
coverage = coverage_map["coverage"]
coverage.index = (coverage.index + 1) % config.ncd_samples_max
# else create new coverage object and add sample to it
else:
coverage = Dataif_Coverage()
Context.coverage_map["coverage"] = coverage
coverage.upcall[coverage.index] = count
return coverage_map
def ncd_kill(signal, frame):
ctx = dataif.Context
rctx = RebalContext
tctx = TraceContext
nlog.critical("Got signal %s, doing required clean up .." % signal)
# reset rebalance settings in ports
cmd = ""
for port_name, port in ctx.port_to_cls.items():
# skip port that we did not rebalance.
if not port.rebalance:
continue
cmd += "-- remove Interface %s other_config pmd-rxq-affinity " % (
port_name)
if cmd:
ret = util.exec_host_command("ovs-vsctl --no-wait %s" % cmd)
if ret == 0:
nlog.info("removed pmd-rxq-affinity in rebalanced ports.")
else:
nlog.warn("removing pmd-rxq-affinity failed for some ports.")
nlog.warn("you may check ovs-vsctl --no-wait %s" % cmd)
os.makedirs(os.path.dirname(config.ncd_dump_file), exist_ok=True)
with open(config.ncd_dump_file, 'w') as f:
fh = ctx.log_handler
json.dump(
{
"time": "%s" % datetime.now(),
"version": netcontrold.__version__,
"events": ctx.events,
"config": {
"trace_mode": tctx.trace_mode,
"rebalance_mode": rctx.rebal_mode,
"verbose_log": fh.level == logging.DEBUG,
},
}, f)
nlog.info("saved current configuration and data in %s" %
config.ncd_dump_file)
raise error.NcdShutdownExc
def ncd_kill(signal, frame):
ctx = dataif.Context
nlog.critical("Got signal %s, doing required clean up .." % signal)
# reset rebalance settings in ports
cmd = ""
for port_name, port in ctx.port_to_cls.items():
# skip port that we did not rebalance.
if not port.rebalance:
continue
cmd += "-- remove Interface %s other_config pmd-rxq-affinity " % (
port_name)
if cmd:
ret = util.exec_host_command("ovs-vsctl --no-wait %s" % cmd)
if ret == 0:
nlog.info("removed pmd-rxq-affinity in rebalanced ports.")
else:
nlog.warn("removing pmd-rxq-affinity failed for some ports.")
nlog.warn("you may check ovs-vsctl --no-wait %s" % cmd)
raise error.NcdShutdownExc
def run(self):
sock_file = config.ncd_socket
try:
os.unlink(sock_file)
except OSError:
if os.path.exists(sock_file):
raise
os.makedirs(os.path.dirname(sock_file), exist_ok=True)
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
nlog.info("starting ctld on %s" % sock_file)
sock.bind(sock_file)
sock.listen(1)
while (not self.ncd_shutdown.is_set()):
conn, client = sock.accept()
try:
cmd = conn.recv(24).decode()
ctx = dataif.Context
rctx = RebalContext
tctx = TraceContext
if cmd == 'CTLD_TRACE_ON':
if not tctx.trace_mode:
nlog.info("turning on trace mode ..")
tctx.trace_mode = True
else:
nlog.info("trace mode already on ..!")
conn.sendall(b"CTLD_ACK")
elif cmd == 'CTLD_TRACE_OFF':
if tctx.trace_mode:
nlog.info("turning off trace mode ..")
tctx.trace_mode = False
else:
nlog.info("trace mode already off ..!")
conn.sendall(b"CTLD_ACK")
elif cmd == 'CTLD_REBAL_ON':
if not rctx.rebal_mode:
nlog.info("turning on rebalance mode ..")
rctx.rebal_mode = True
else:
nlog.info("rebalance mode already on ..!")
conn.sendall(b"CTLD_ACK")
elif cmd == 'CTLD_REBAL_OFF':
if rctx.rebal_mode:
nlog.info("turning off rebalance mode ..")
rctx.rebal_mode = False
else:
nlog.info("rebalance mode already off ..!")
conn.sendall(b"CTLD_ACK")
elif cmd == 'CTLD_REBAL_QUICK_ON':
if not rctx.rebal_quick:
nlog.info("turning on rebalance quick mode ..")
rctx.rebal_quick = True
else:
nlog.info("rebalance quick mode already on ..!")
conn.sendall(b"CTLD_ACK")
elif cmd == 'CTLD_REBAL_QUICK_OFF':
if rctx.rebal_quick:
nlog.info("turning off rebalance quick mode ..")
rctx.rebal_quick = False
else:
nlog.info("rebalance quick mode already off ..!")
conn.sendall(b"CTLD_ACK")
elif cmd == 'CTLD_VERBOSE_ON':
fh = ctx.log_handler
if fh.level == logging.INFO:
nlog.info("turning on verbose mode ..")
fh.setLevel(logging.DEBUG)
else:
nlog.info("verbose mode already on ..!")
conn.sendall(b"CTLD_ACK")
elif cmd == 'CTLD_VERBOSE_OFF':
fh = ctx.log_handler
if fh.level == logging.DEBUG:
nlog.info("turning off verbose mode ..")
fh.setLevel(logging.INFO)
else:
nlog.info("verbose mode already off ..!")
conn.sendall(b"CTLD_ACK")
elif cmd == 'CTLD_REBAL_CNT':
n = 0
if rctx.rebal_mode:
n = len(rctx.rebal_stat)
for (x, y, z) in ctx.events:
if y == 'rebalance':
n += 1
conn.sendall(b"CTLD_DATA_ACK %6d" % (len(str(n))))
conn.sendall(str(n).encode())
elif cmd == 'CTLD_CONFIG':
status = "trace mode:"
if tctx.trace_mode:
status += " on\n"
else:
status += " off\n"
status += "rebalance mode:"
if rctx.rebal_mode:
status += " on\n"
else:
status += " off\n"
status += "rebalance quick:"
if rctx.rebal_quick:
status += " on\n"
else:
status += " off\n"
status += "verbose log:"
fh = ctx.log_handler
if fh.level == logging.DEBUG:
status += " on\n"
else:
status += " off\n"
conn.sendall(b"CTLD_DATA_ACK %6d" % (len(status)))
conn.sendall(status.encode())
elif cmd == 'CTLD_STATUS':
status = "%-16s | %-12s | %s\n" % ('Interface', 'Event',
'Time stamp')
status += ('-' * 17) + '+' + ('-' * 14) + '+' + ('-' * 28)
status += '\n'
for (x, y, z) in ctx.events:
status += "%-16s | %-12s | %s\n" % (x, y, z)
conn.sendall(b"CTLD_DATA_ACK %6d" % (len(status)))
conn.sendall(status.encode())
elif cmd == 'CTLD_VERSION':
status = "netcontrold v%s\n" % netcontrold.__version__
ret = util.exec_host_command("ovs-vsctl -V")
if ret == 1:
status += "openvswitch (unknown)\n"
else:
parse = re.match("ovs-vsctl \(Open vSwitch\) (.*?)\n",
ret)
status += "openvswitch v%s\n" % parse[1]
conn.sendall(b"CTLD_DATA_ACK %6d" % (len(status)))
conn.sendall(status.encode())
else:
nlog.info("unknown control command %s" % cmd)
finally:
conn.close()
return
def ncd_main(argv):
# input options
argpobj = argparse.ArgumentParser(
prog='ncd.py', description='control network load on pmd')
argpobj.add_argument('-s',
'--sample-interval',
type=int,
default=10,
help='seconds between each sampling (default: 10)')
argpobj.add_argument(
'-t',
'--trace',
required=False,
action='store_true',
default=False,
help='operate in trace mode',
)
argpobj.add_argument('--trace-cb',
type=str,
default='ncd_cb_pktdrop',
help='trace mode callback '
'(default: ncd_cb_pktdrop)')
argpobj.add_argument(
'-r',
'--rebalance',
required=False,
action='store_true',
default=True,
help="operate in rebalance mode",
)
argpobj.add_argument('--rebalance-interval',
type=int,
default=60,
help='seconds between each re-balance '
'(default: 60)')
argpobj.add_argument('--rebalance-n',
type=int,
default=1,
help='rebalance dry-runs at the max (default: 1)')
argpobj.add_argument('--rebalance-iq',
action='store_true',
default=False,
help='rebalance by iterative queues logic '
'(default: False)')
argpobj.add_argument('-q',
'--quiet',
action='store_true',
default=False,
help='no logging in terminal (default: False)')
argpobj.add_argument('-v',
'--verbose',
action='store_true',
default=False,
help='trace logging (default: False)')
args = argpobj.parse_args(argv)
# check input to ncd
ncd_trace = args.trace
ncd_trace_cb = args.trace_cb
ncd_rebal = args.rebalance
if ncd_trace and not util.exists(ncd_trace_cb):
print("no such program %s exists!" % ncd_trace_cb)
sys.exit(1)
# set verbose level
os.makedirs(os.path.dirname(config.ncd_log_file), exist_ok=True)
fh = RotatingFileHandler(config.ncd_log_file,
maxBytes=(config.ncd_log_max_KB * 1024),
backupCount=config.ncd_log_max_backup_n)
fh_fmt = logging.Formatter(
"%(asctime)s|%(name)s|%(levelname)s|%(message)s")
fh.setFormatter(fh_fmt)
if args.verbose:
fh.setLevel(logging.DEBUG)
else:
fh.setLevel(logging.INFO)
ch = logging.StreamHandler(sys.stdout)
ch_fmt = logging.Formatter("%(message)s")
ch.setFormatter(ch_fmt)
ch.setLevel(logging.INFO)
global nlog
nlog = logging.getLogger('ncd')
nlog.setLevel(logging.DEBUG)
nlog.addHandler(fh)
if not args.quiet:
nlog.addHandler(ch)
ctx = dataif.Context
ctx.nlog = nlog
ctx.log_handler = fh
pmd_map = ctx.pmd_map
# set sampling interval to collect data
ncd_sample_interval = args.sample_interval
# set interval between each re-balance
ncd_rebal_interval = args.rebalance_interval
# set rebalance dryrun count
ncd_rebal_n = args.rebalance_n
# set iterative queue rebalance algorithm
ncd_iq_rebal = args.rebalance_iq
# set rebalance method.
if ncd_iq_rebal:
rebalance_dryrun = dataif.rebalance_dryrun_by_iq
else:
# round robin logic to rebalance.
rebalance_dryrun = dataif.rebalance_dryrun_by_cyc
# restrict only one dry run for cycles based mode.
ncd_rebal_n = 1
# set check point to call rebalance in vswitch
rctx = RebalContext
rctx.rebal_tick_n = ncd_rebal_interval / ncd_sample_interval
# rebalance dryrun iteration
rebal_i = 0
if ncd_rebal:
# adjust length of the samples counter
config.ncd_samples_max = min(ncd_rebal_interval / ncd_sample_interval,
config.ncd_samples_max)
rctx.rebal_mode = True
rctx.rebal_quick = True
config.ncd_samples_max = int(config.ncd_samples_max)
# set signal handler to abort ncd
signal.signal(signal.SIGINT, ncd_kill)
signal.signal(signal.SIGTERM, ncd_kill)
tctx = TraceContext
if ncd_trace:
tctx.trace_mode = True
# start ctld thread to monitor control command and dispatch
# necessary action.
shutdown_event = threading.Event()
tobj = CtlDThread(shutdown_event)
tobj.daemon = True
try:
tobj.start()
except threading.ThreadError:
nlog.info("failed to start ctld thread ..")
sys.exit(1)
prev_var = 0
cur_var = 0
ncd_samples_max = config.ncd_samples_max
min_sample_i = 0
# begin rebalance dry run
while (1):
try:
collect_data(ncd_samples_max, ncd_sample_interval)
min_sample_i += ncd_samples_max
nlog.info("current pmd load:")
for pmd_id in sorted(pmd_map.keys()):
pmd = pmd_map[pmd_id]
nlog.info("pmd id %d load %d" % (pmd_id, pmd.pmd_load))
cur_var = dataif.pmd_load_variance(pmd_map)
nlog.info("current pmd load variance: %d" % cur_var)
# do not trace if rebalance dry-run in progress.
if tctx.trace_mode and not rebal_i:
pmd_cb_list = []
for pname in sorted(ctx.port_to_cls.keys()):
port = ctx.port_to_cls[pname]
drop = dataif.port_drop_ppm(port)
drop_min = config.ncd_cb_pktdrop_min
tx_retry = dataif.port_tx_retry(port)
do_cb = False
if drop[0] > drop_min:
nlog.info("port %s drop_rx %d ppm above %d ppm" %
(port.name, drop[0], drop_min))
ctx.events.append((port.name, "rx_drop", ctx.last_ts))
do_cb = True
if drop[1] > drop_min:
nlog.info("port %s drop_tx %d ppm above %d ppm" %
(port.name, drop[1], drop_min))
ctx.events.append((port.name, "tx_drop", ctx.last_ts))
do_cb = True
if tx_retry > config.ncd_samples_max:
nlog.info(
"port %s tx_retry %d above %d" %
(port.name, tx_retry, config.ncd_samples_max))
ctx.events.append((port.name, "tx_retry", ctx.last_ts))
do_cb = True
if not do_cb:
# no pmd needs to be traceged.
continue
for pmd_id in sorted(pmd_map.keys()):
pmd = pmd_map[pmd_id]
if (pmd.find_port_by_name(port.name)):
pmd_cb_list.insert(0, pmd_id)
if (len(pmd_cb_list) > 0):
pmds = " ".join(list(map(str, set(pmd_cb_list))))
cmd = "%s %s" % (ncd_trace_cb, pmds)
nlog.info("executing callback %s" % cmd)
data = util.exec_host_command(cmd)
nlog.info(data)
if not rctx.rebal_mode:
continue
# At the minimum for deriving current load on pmds, all of
# the sampling counters (of size config.ncd_samples_max) have
# to be filled "every time" before other evaluations done.
#
# However, for quick rebalance, we fill all the counters
# once, and then keep rolling with one counter across old
# stats so that, we reduce time to sample before kicking off
# rebalance (from sampling config.ncd_samples_max counters
# to only one.
#
# As ovs internally refers all its 6 sample counters for any
# stats we query, it is absolutely fine we roll with one
# new sample and retain old n-1 samples to check for current
# state of pmd and rxqs.
#
if rctx.rebal_quick and (min_sample_i >= config.ncd_samples_max):
ncd_samples_max = 1
min_sample_i = (config.ncd_samples_max - 1)
else:
ncd_samples_max = config.ncd_samples_max
# dry-run pmd rebalance.
if pmd_map and rebalance_dryrun(pmd_map):
rebal_i += 1
# restart sampling when no dry-run performed.
if not rebal_i:
nlog.info("no dryrun performed.")
continue
else:
# compare previous and current state of pmds.
prev_var = cur_var
collect_data(ncd_samples_max, ncd_sample_interval)
cur_var = dataif.pmd_load_variance(pmd_map)
nlog.info("pmd load in dry run(%d):" % rebal_i)
for pmd_id in sorted(pmd_map.keys()):
pmd = pmd_map[pmd_id]
nlog.info("pmd id %d load %d" % (pmd_id, pmd.pmd_load))
nlog.info("pmd load variance: previous %d, in dry run(%d) %d" %
(prev_var, rebal_i, cur_var))
if (cur_var < prev_var):
diff = (prev_var - cur_var) * 100 / prev_var
if diff > config.ncd_pmd_load_improve_min:
rctx.apply_rebal = True
# check if we reached maximum allowable dry-runs.
if rebal_i < ncd_rebal_n:
# continue for more dry runs.
continue
# check if balance state of all pmds is reached
if rctx.apply_rebal:
# check if rebalance call needed really.
if (rctx.rebal_tick > rctx.rebal_tick_n):
rctx.rebal_tick = 0
cmd = rebalance_switch(pmd_map)
ctx.events.append(("pmd", "rebalance", ctx.last_ts))
nlog.info(
"vswitch command for current optimization is: %s" %
cmd)
rctx.apply_rebal = False
if (util.exec_host_command(cmd) == 1):
nlog.info("problem running this command.. "
"check vswitch!")
now = datetime.now()
now_ts = now.strftime("%Y-%m-%d %H:%M:%S")
ctx.events.append(("switch", "error", now_ts))
# sleep for few seconds before thrashing current
# dry-run
nlog.info("waiting for %d seconds "
"before new dry runs begin.." %
config.ncd_vsw_wait_min)
time.sleep(config.ncd_vsw_wait_min)
else:
nlog.info("minimum rebalance interval not met!"
" now at %d sec" %
(rctx.rebal_tick * ncd_sample_interval))
else:
nlog.info("no new optimization found ..")
# reset collected data
pmd_map.clear()
ctx.port_to_cls.clear()
ctx.port_to_id.clear()
ncd_samples_max = config.ncd_samples_max
rebal_i = 0
min_sample_i = 0
nlog.info("dry-run reset.")
except error.NcdShutdownExc:
nlog.info("Exiting NCD ..")
tobj.ncd_shutdown.set()
sys.exit(1)
tobj.join()
def get_port_stats():
"""
Collect stats on every port in the datapath.
In every sampling iteration, these stats are stored
in corresponding sampling slots.
Raises
------
OsCommandExc
if the given OS command did not succeed for some reason.
ObjModleExc
if state of ports in switch differ.
"""
nlog = Context.nlog
# retrieve required data from the vswitch.
cmd = "ovs-appctl dpctl/show -s"
data = util.exec_host_command(cmd)
if not data:
raise OsCommandExc("unable to collect data")
# current state of ports
cur_port_l = sorted(Context.port_to_cls.keys())
# current port object to be used in every line under parse.
port = None
for line in data.splitlines():
if re.match(r'\s.*port\s(\d+):\s([A-Za-z0-9_-]+) *', line):
# In below matching line, we retrieve port id and name.
linesre = re.search(r'\s.*port\s(\d+):\s([A-Za-z0-9_-]+) *', line)
(pid, pname) = linesre.groups()
Context.port_to_id[pname] = int(pid)
# If in mid of sampling, we should have port_to_cls having
# entry for this port name.
if pname in Context.port_to_cls:
port = Context.port_to_cls[pname]
assert (port.id == pid)
# Store following stats in new sampling slot.
port.cyc_idx = (port.cyc_idx + 1) % config.ncd_samples_max
nlog.debug("port %s in iteration %d" %
(port.name, port.cyc_idx))
else:
# create new entry in port_to_cls for this port.
port = make_dataif_port(pname)
port.id = pid
nlog.debug("added port %s stats.." % pname)
elif re.match(r'\s.*RX packets:(\d+) .*? dropped:(\d+) *', line):
# From other lines, we retrieve stats of the port.
linesre = re.search(r'\s.*RX packets:(\d+) .*? dropped:(\d+) *',
line)
(
rx,
drop,
) = linesre.groups()
port.rx_cyc[port.cyc_idx] = int(rx)
port.rx_drop_cyc[port.cyc_idx] = int(drop)
elif re.match(r'\s.*TX packets:(\d+) .*? dropped:(\d+) *', line):
# From other lines, we retrieve stats of the port.
linesre = re.search(r'\s.*TX packets:(\d+) .*? dropped:(\d+) *',
line)
(
tx,
drop,
) = linesre.groups()
port.tx_cyc[port.cyc_idx] = int(tx)
port.tx_drop_cyc[port.cyc_idx] = int(drop)
# new state of ports.
new_port_l = sorted(Context.port_to_cls.keys())
# skip modelling this object if states differ.
if len(cur_port_l) > 0 and cur_port_l != new_port_l:
raise ObjModelExc("ports count differ")
# current port object to be used in every line under parse.
return None
def get_pmd_rxqs(pmd_map):
"""
Collect info on how rxq is pinned with pmd, from the vswitch.
Parameters
----------
pmd_map : dict
mapping of pmd id and its Dataif_Pmd object.
Raises
------
OsCommandExc
if the given OS command did not succeed for some reason.
ObjConsistencyExc
if state of pmds in ncd differ.
ObjParseExc
if unable to retrieve info from switch.
ObjModleExc
if state of pmds in switch differ.
"""
nlog = Context.nlog
# retrieve required data from the vswitch.
cmd = "ovs-appctl dpif-netdev/pmd-rxq-show"
data = util.exec_host_command(cmd)
if not data:
raise OsCommandExc("unable to collect data")
# current state of pmds
cur_pmd_l = sorted(pmd_map.keys())
# sname and sval stores parsed string's key and value.
sname, sval = None, None
# current pmd object to be used in every line under parse.
pmd = None
for line in data.splitlines():
if line.startswith('pmd thread'):
# In below matching line, we retrieve core id (aka pmd id)
# and core id.
linesre = re.search(r'pmd thread numa_id (\d+) core_id (\d+):',
line)
numa_id = int(linesre.groups()[0])
core_id = int(linesre.groups()[1])
if core_id not in pmd_map:
raise ObjConsistencyExc(
"trying to add new pmd %d in mid of ncd!.. aborting! ")
pmd = pmd_map[core_id]
assert (pmd.numa_id == numa_id)
nlog.debug("pmd %d in iteration %d" % (pmd.id, pmd.cyc_idx))
elif re.match(r'\s.*port: .*', line):
# From this line, we retrieve cpu usage of rxq.
linesre = re.search(
r'\s.*port:\s([A-Za-z0-9_-]+)\s*'
r'queue-id:\s*(\d+)\s*(?=\((enabled)\))?.*'
r'pmd usage:\s*(\d+|NOT AVAIL)\s*?', line)
pname = linesre.groups()[0]
qid = int(linesre.groups()[1])
enabled_flag = linesre.groups()[2]
try:
qcpu = int(linesre.groups()[3])
except ValueError:
qcpu = linesre.groups()[3]
if (qcpu == 'NOT AVAIL'):
raise ObjParseExc("pmd usage unavailable for now")
else:
raise ObjParseExc("error parsing line %s" % line)
# get the Dataif_Port owning this rxq.
port = pmd.find_port_by_name(pname)
if not port:
port = pmd.add_port(pname)
# update port attributes now.
port.id = Context.port_to_id[pname]
port.numa_id = pmd.numa_id
port_cls = Context.port_to_cls[pname]
port_cls.rebalance = True
# check whether this rxq was being rebalanced.
if qid in port.rxq_rebalanced:
raise ObjConsistencyExc(
"stale %s object found while parsing rxq in pmd ..")
else:
# skip updating zero index as rxq counters works only
# with the differences between pmd stats in every
# sampling slot.
if pmd.cyc_idx == 0:
continue
# port not in rebalancing state, so update rxq for its
# cpu cycles consumed by it.
rxq = (port.find_rxq_by_id(qid) or port.add_rxq(qid))
rxq.pmd = pmd
rxq.port = port
cur_idx = pmd.cyc_idx
prev_idx = (cur_idx - 1) % config.ncd_samples_max
rx_diff = pmd.rx_cyc[cur_idx] - pmd.rx_cyc[prev_idx]
pcpu_diff = pmd.proc_cpu_cyc[cur_idx] - pmd.proc_cpu_cyc[
prev_idx]
icpu_diff = pmd.idle_cpu_cyc[cur_idx] - pmd.idle_cpu_cyc[
prev_idx]
cpu_diff = pcpu_diff + icpu_diff
qcpu_diff = int((qcpu * cpu_diff) / 100)
qrx_diff = int((qcpu * rx_diff) / 100)
rxq.cpu_cyc[pmd.cyc_idx] = qcpu_diff
rxq.rx_cyc[pmd.cyc_idx] = qrx_diff
if (enabled_flag == "enabled"):
rxq.enabled = True
else:
rxq.enabled = False
else:
# From other line, we retrieve isolated flag.
(sname, sval) = line.split(":")
sname = re.sub(r"^\s+", "", sname)
assert (sname == 'isolated ')
pmd.isolated = {'true': True, 'false': False}[sval[1:]]
# new state of pmds.
new_pmd_l = sorted(pmd_map.keys())
# skip modelling this object if states differ.
if len(cur_pmd_l) > 0 and cur_pmd_l != new_pmd_l:
raise ObjModelExc("pmds count differ")
return pmd_map
def get_pmd_stats(pmd_map):
"""
Collect stats on every pmd running in the system and update
pmd_map. In every sampling iteration, these stats are stored
in corresponding sampling slots.
Parameters
----------
pmd_map : dict
mapping of pmd id and its Dataif_Pmd object.
Raises
------
OsCommandExc
if the given OS command did not succeed for some reason.
ObjConsistencyExc
if state of pmds in ncd differ.
ObjModleExc
if state of pmds in switch differ.
"""
nlog = Context.nlog
# retrieve required data from the vswitch.
cmd = "ovs-appctl dpif-netdev/pmd-stats-show"
data = util.exec_host_command(cmd)
if not data:
raise OsCommandExc("unable to collect data")
# current state of pmds
cur_pmd_l = sorted(pmd_map.keys())
# sname and sval stores parsed string's key and value.
sname, sval = None, None
# current pmd object to be used in every line under parse.
pmd = None
for line in data.splitlines():
if line.startswith("pmd thread"):
# In below matching line, we retrieve core id (aka pmd id)
# and core id.
linesre = re.search(r'pmd thread numa_id (\d+) core_id (\d+):',
line)
numa_id = int(linesre.groups()[0])
core_id = int(linesre.groups()[1])
# If in mid of sampling, we should have pmd_map having
# entry for this core id.
if core_id in pmd_map:
pmd = pmd_map[core_id]
# Check to ensure we are good to go as local should
# always be used.
assert (pmd.numa_id == numa_id)
# Store following stats in new sampling slot.
pmd.cyc_idx = (pmd.cyc_idx + 1) % config.ncd_samples_max
nlog.debug("pmd %d in iteration %d" % (pmd.id, pmd.cyc_idx))
else:
# Very first sampling for each pmd occur in this
# clause. Just ensure, no new pmd is added from system
# reconfiguration.
if len(pmd_map) != 0 and not pmd:
raise ObjConsistencyExc(
"trying to add new pmd %d in mid of ncd!.. aborting! ")
# create new entry in pmd_map for this pmd.
pmd = Dataif_Pmd(core_id)
pmd_map[pmd.id] = pmd
nlog.debug("added pmd %s stats.." % pmd.id)
# numa id of pmd is of core's.
pmd.numa_id = numa_id
elif line.startswith("main thread"):
# end of pmd stats
break
else:
# From other lines, we retrieve stats of the pmd.
(sname, sval) = line.split(":")
sname = re.sub(r"^\s+", "", sname)
sval = sval[1:].split()
if sname == "packets received":
pmd.rx_cyc[pmd.cyc_idx] = int(sval[0])
elif sname == "idle cycles":
pmd.idle_cpu_cyc[pmd.cyc_idx] = int(sval[0])
elif sname == "processing cycles":
pmd.proc_cpu_cyc[pmd.cyc_idx] = int(sval[0])
# new state of pmds.
new_pmd_l = sorted(pmd_map.keys())
# skip modelling this object if states differ.
if len(cur_pmd_l) > 0 and cur_pmd_l != new_pmd_l:
raise ObjModelExc("pmds count differ")
return pmd_map
def get_interface_stats():
"""
Collect retry stats on every applicable port in the datapath.
In every sampling iteration, these stats are stored
in corresponding sampling slots.
Raises
------
OsCommandExc
if the given OS command did not succeed for some reason.
ObjModleExc
if state of ports in switch differ.
"""
nlog = Context.nlog
# retrieve required data from the vswitch.
cmd = "ovs-vsctl list interface"
data = util.exec_host_command(cmd)
if not data:
raise OsCommandExc("unable to collect data")
# current state of ports
cur_port_l = sorted(Context.port_to_cls.keys())
# current port object to be used in every line under parse.
port = None
for line in data.splitlines():
if re.match(r'\s*name\s.*:\s"*([A-Za-z0-9_-]+)"*', line):
# In below matching line, we retrieve port id and name.
linesre = re.search(r'\s*name\s.*:\s"*([A-Za-z0-9_-]+)"*', line)
(pname, ) = linesre.groups()
# If in mid of sampling, we should have port_to_cls having
# entry for this port name.
if pname in Context.port_to_cls:
port = Context.port_to_cls[pname]
nlog.debug("port %s in iteration %d" %
(port.name, port.cyc_idx))
elif re.match(r'\s*type\s.*:\s([a-z]+)', line):
if not port:
continue
# From other lines, we retrieve stats of the port.
linesre = re.search(r'\s*type\s.*:\s([a-z]+)', line)
(type, ) = linesre.groups()
port.type = type
port = None
elif re.match(r'\s*statistics\s.*:\s{(.*)}', line):
if not port:
continue
# From other lines, we retrieve stats of the port.
linesre = re.search(r'\s*statistics\s.*:\s{(.*)}', line)
(sval, ) = linesre.groups()
dval = {
sub.split("=")[0]: sub.split("=")[1]
for sub in sval.split(", ")
}
if 'tx_retries' in dval:
port.tx_retry_cyc[port.cyc_idx] = int(dval['tx_retries'])
# new state of ports.
new_port_l = sorted(Context.port_to_cls.keys())
# skip modelling this object if states differ.
if len(cur_port_l) > 0 and cur_port_l != new_port_l:
raise ObjModelExc("ports count differ")
return None
def get_pmd_rxqs(pmd_map):
"""
Collect info on how rxq is pinned with pmd, from the vswitch.
Parameters
----------
pmd_map : dict
mapping of pmd id and its Dataif_Pmd object.
Raises
------
OsCommandExc
if the given OS command did not succeed for some reason.
ObjConsistencyExc
if state of pmds in ncd differ.
ObjParseExc
if unable to retrieve info from switch.
ObjModleExc
if state of pmds in switch differ.
"""
nlog = Context.nlog
# retrieve required data from the vswitch.
cmd = "ovs-appctl dpif-netdev/pmd-rxq-show"
data = util.exec_host_command(cmd)
if not data:
raise OsCommandExc("unable to collect data")
# current state of pmds
cur_pmd_l = sorted(pmd_map.keys())
# sname and sval stores parsed string's key and value.
sname, sval = None, None
# current pmd object to be used in every line under parse.
pmd = None
for line in data.splitlines():
if line.startswith('pmd thread'):
# In below matching line, we retrieve core id (aka pmd id)
# and core id.
linesre = re.search(r'pmd thread numa_id (\d+) core_id (\d+):',
line)
numa_id = int(linesre.groups()[0])
core_id = int(linesre.groups()[1])
if core_id not in pmd_map:
raise ObjConsistencyExc(
"trying to add new pmd %d in mid of ncd!.. aborting! ")
pmd = pmd_map[core_id]
assert(pmd.numa_id == numa_id)
nlog.debug("pmd %d in iteration %d" % (pmd.id, pmd.cyc_idx))
elif re.match(r'\s.*port: .*', line):
# From this line, we retrieve cpu usage of rxq.
linesre = re.search(r'\s.*port:\s([A-Za-z0-9_-]+)\s*'
r'queue-id:\s*(\d+)\s*'
r'pmd usage:\s*(\d+|NOT AVAIL)\s*?',
line)
pname = linesre.groups()[0]
qid = int(linesre.groups()[1])
try:
qcpu = int(linesre.groups()[2])
except ValueError:
qcpu = linesre.groups()[2]
if (qcpu == 'NOT AVAIL'):
raise ObjParseExc("pmd usage unavailable for now")
else:
raise ObjParseExc("error parsing line %s" % line)
# get the Dataif_Port owning this rxq.
port = pmd.find_port_by_name(pname)
if not port:
port = pmd.add_port(pname)
# update port attributes now.
port.id = Context.port_to_id[pname]
port.numa_id = pmd.numa_id
port_cls = Context.port_to_cls[pname]
port_cls.rebalance = True
# check whether this rxq was being rebalanced.
if qid in port.rxq_rebalanced:
# In dry-run, we need to update cpu cycles consumed by
# this rxq (through current pmd), into the processing
# cycles of the rebalancing pmd. Then the load of the
# rebalancing pmd could be estimated appropriately.
reb_pmd_id = port.rxq_rebalanced[qid]
reb_pmd = pmd_map[reb_pmd_id]
reb_port = reb_pmd.find_port_by_name(port.name)
rxq = reb_port.find_rxq_by_id(qid)
# qcpu is in percentage in this data, so we convert it
# into actual cycles using processing cycles that this
# pmd consumed.
# qrx is approximate count of packets that this rxq
# received.
cur_idx = pmd.cyc_idx
prev_idx = (cur_idx - 1) % config.ncd_samples_max
rx_diff = pmd.rx_cyc[cur_idx] - pmd.rx_cyc[prev_idx]
cpu_diff = pmd.proc_cpu_cyc[
cur_idx] - pmd.proc_cpu_cyc[prev_idx]
qrx = (qcpu * rx_diff) / 100
qcpu = (qcpu * cpu_diff) / 100
# update rebalancing pmd for cpu cycles and rx count.
reb_pmd.proc_cpu_cyc[cur_idx] += qcpu
reb_pmd.idle_cpu_cyc[cur_idx] -= qcpu
reb_pmd.rx_cyc[pmd.cyc_idx] += qrx
# update current pmd for cpu cycles and rx count.
pmd.proc_cpu_cyc[pmd.cyc_idx] -= qcpu
pmd.idle_cpu_cyc[pmd.cyc_idx] += qcpu
pmd.rx_cyc[pmd.cyc_idx] -= qrx
else:
# port not in rebalancing state, so update rxq for its
# cpu cycles consumed by it.
rxq = (port.find_rxq_by_id(qid) or port.add_rxq(qid))
rxq.pmd = pmd
rxq.port = port
cur_idx = pmd.cyc_idx
prev_idx = (cur_idx - 1) % config.ncd_samples_max
rx_diff = pmd.rx_cyc[cur_idx] - pmd.rx_cyc[prev_idx]
cpu_diff = pmd.proc_cpu_cyc[
cur_idx] - pmd.proc_cpu_cyc[prev_idx]
qcpu = (qcpu * cpu_diff) / 100
qrx = (qcpu * rx_diff) / 100
rxq.cpu_cyc[pmd.cyc_idx] = qcpu
else:
# From other line, we retrieve isolated flag.
(sname, sval) = line.split(":")
sname = re.sub("^\s+", "", sname)
assert(sname == 'isolated ')
pmd.isolated = {'true': True, 'false': False}[sval[1:]]
# new state of pmds.
new_pmd_l = sorted(pmd_map.keys())
# skip modelling this object if states differ.
if len(cur_pmd_l) > 0 and cur_pmd_l != new_pmd_l:
raise ObjModelExc("pmds count differ")
return pmd_map