def exec_command(self, cmd, timeout=10):
"""Execute SSH command on a new channel on the connected Node.
Returns (return_code, stdout, stderr).
"""
start = time()
chan = self._ssh.get_transport().open_session()
if timeout is not None:
chan.settimeout(int(timeout))
chan.exec_command(cmd)
end = time()
logger.trace('exec_command "{0}" on {1} took {2} seconds'.format(cmd,
self._hostname, end-start))
stdout = ""
while True:
buf = chan.recv(self.__MAX_RECV_BUF)
stdout += buf
if not buf:
break
stderr = ""
while True:
buf = chan.recv_stderr(self.__MAX_RECV_BUF)
stderr += buf
if not buf:
break
return_code = chan.recv_exit_status()
logger.trace('chan_recv/_stderr took {} seconds'.format(time()-end))
return (return_code, stdout, stderr)
def get(self, message_template, timeout=None, header_filter=None):
header_fields = message_template.header_parameters
logger.trace("Get message with params %s" % header_fields)
msg = self._get_from_cache(message_template, header_fields, header_filter)
if msg:
logger.trace("Cache hit. Cache currently has %s messages" % len(self._cache))
return msg
while True:
header, pdu_bytes = self._protocol.read(self._stream, timeout=timeout)
if self._matches(header, header_fields, header_filter):
return self._to_msg(message_template, header, pdu_bytes)
self._cache.append((header, pdu_bytes))
def setup_framework(self, nodes):
"""Pack the whole directory and extract in temp on each node."""
tarball = self.__pack_framework_dir()
logger.console('Framework packed to {0}'.format(tarball))
remote_tarball = "/tmp/{0}".format(basename(tarball))
for node in nodes.values():
self.__copy_tarball_to_node(tarball, node)
self.__extract_tarball_at_node(remote_tarball, node)
logger.trace('Test framework copied to all topology nodes')
self.__delete_local_tarball(tarball)
def show_statistics_on_all_duts(nodes, sleeptime=5):
"""Show VPP statistics on all DUTs.
:param nodes: VPP nodes.
:type nodes: dict
:param sleeptime: Time to wait for traffic to arrive back to TG.
:type sleeptime: int
"""
logger.trace('Waiting for statistics to be collected')
time.sleep(sleeptime)
for node in nodes.values():
if node['type'] == NodeType.DUT:
VppCounters.show_vpp_statistics(node)
def setup_dut(self, node):
ssh = SSH()
ssh.connect(node)
ssh.scp('resources/libraries/bash/dut_setup.sh', '/tmp/dut_setup.sh')
(ret_code, stdout, stderr) = \
ssh.exec_command('sudo -Sn bash /tmp/dut_setup.sh')
logger.trace(stdout)
if 0 != int(ret_code):
logger.error('DUT {0} setup script failed: "{1}"'.
format(node['host'], stdout + stderr))
raise Exception('DUT test setup script failed at node {}'.
format(node['host']))
def stop_timer(self, benchmark_name='default'):
logger.trace("Timer::stop_timer({})".format(benchmark_name))
"""
Stops a single timer
Parameters:
- ``benchmark_name`` Name of the benchmark, optional. Defaults to 'default'
Example:
| Stop Timer | mytimer |
"""
if benchmark_name not in self.benchmarks:
raise DataError('Benchmark "%s" not started.' % benchmark_name)
self.benchmarks[benchmark_name]['stop'] = timer()
def run_teardown_only_once(self, keyword, *args):
"""
Runs a keyword only once after all executions have gone throught this step in the last possible moment.
[https://pabot.org/PabotLib.html?ref=log#run-teardown-only-once|Open online docs.]
"""
last_level = BuiltIn().get_variable_value('${%s}' % PABOT_LAST_LEVEL)
if last_level is None:
BuiltIn().run_keyword(keyword, *args)
return
logger.trace('Current path "%s" and last level "%s"' %
(self._path, last_level))
if not self._path.startswith(last_level):
logger.info("Teardown skipped in this item")
return
queue_index = int(
BuiltIn().get_variable_value('${%s}' % PABOT_QUEUE_INDEX) or 0)
logger.trace("Queue index (%d)" % queue_index)
if self._remotelib:
while self.get_parallel_value_for_key(
PABOT_MIN_QUEUE_INDEX_EXECUTING_PARALLEL_VALUE
) < queue_index:
logger.trace(
self.get_parallel_value_for_key(
PABOT_MIN_QUEUE_INDEX_EXECUTING_PARALLEL_VALUE))
time.sleep(0.3)
logger.trace("Teardown conditions met. Executing keyword.")
BuiltIn().run_keyword(keyword, *args)
def create_session(self, node, hello, time_out=10):
"""Create an SSH session, connect to Honeycomb on the specified node,
open a communication channel to the Netconf subsystem and exchange hello
messages.
:param node: Honeycomb node.
:param hello: Hello message and capability list to be sent to Honeycomb.
:param time_out: Timeout value for the connection in seconds.
:type node: dict
:type hello: str
:type time_out: int
"""
start = time()
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(node['host'],
username=node['honeycomb']['user'],
password=node['honeycomb']['passwd'],
pkey=None,
port=node['honeycomb']['netconf_port'],
timeout=time_out)
logger.trace('Connect took {0} seconds'.format(time() - start))
logger.debug('New ssh: {0}'.format(client))
logger.debug('Connect peer: {0}'.
format(client.get_transport().getpeername()))
logger.debug(client)
channel = client.get_transport().open_session()
channel.settimeout(time_out)
channel.set_combine_stderr(True)
channel.get_pty()
channel.invoke_subsystem("netconf")
logger.debug(channel)
self.client = client
self.channel = channel
# read OpenDaylight's hello message and capability list
self.get_response(
size=131072,
time_out=time_out,
err="Timeout on getting hello message."
)
self.channel.send(hello)
if not self.channel.active:
raise HoneycombError("Channel closed on capabilities exchange.")
def run_traffic_script_on_node(script_file_name, node, script_args,
timeout=60):
"""Run traffic script on the TG node.
:param script_file_name: Traffic script name.
:param node: Node to run traffic script on.
:param script_args: Traffic scripts arguments.
:param timeout: Timeout (optional).
:type script_file_name: str
:type node: dict
:type script_args: str
:type timeout: int
:raises RuntimeError: ICMP echo Rx timeout.
:raises RuntimeError: DHCP REQUEST Rx timeout.
:raises RuntimeError: DHCP DISCOVER Rx timeout.
:raises RuntimeError: TCP/UDP Rx timeout.
:raises RuntimeError: ARP reply timeout.
:raises RuntimeError: Traffic script execution failed.
"""
logger.trace("{}".format(timeout))
ssh = SSH()
ssh.connect(node)
cmd = ("cd {}; " +
"virtualenv --system-site-packages --never-download env && " +
"export PYTHONPATH=${{PWD}}; " +
". ${{PWD}}/env/bin/activate; " +
"resources/traffic_scripts/{} {}") \
.format(Constants.REMOTE_FW_DIR, script_file_name,
script_args)
(ret_code, stdout, stderr) = ssh.exec_command_sudo(
'sh -c "{}"'.format(TrafficScriptExecutor._escape(cmd)),
timeout=timeout)
logger.debug("stdout: {}".format(stdout))
logger.debug("stderr: {}".format(stderr))
logger.debug("ret_code: {}".format(ret_code))
if ret_code != 0:
if "RuntimeError: ICMP echo Rx timeout" in stderr:
raise RuntimeError("ICMP echo Rx timeout")
elif "RuntimeError: DHCP REQUEST Rx timeout" in stderr:
raise RuntimeError("DHCP REQUEST Rx timeout")
elif "RuntimeError: DHCP DISCOVER Rx timeout" in stderr:
raise RuntimeError("DHCP DISCOVER Rx timeout")
elif "RuntimeError: TCP/UDP Rx timeout" in stderr:
raise RuntimeError("TCP/UDP Rx timeout")
elif "Error occurred: ARP reply timeout" in stdout:
raise RuntimeError("ARP reply timeout")
elif "RuntimeError: ESP packet Rx timeout" in stderr:
raise RuntimeError("ESP packet Rx timeout")
else:
raise RuntimeError("Traffic script execution failed")
def get_webelements_in_active_area(xpath, **kwargs):
"""Find element under another element.
If ${ACTIVE_AREA_FUNC} returns an element then the xpath is searched from
that element. Otherwise the element is searched under body element.
Parameters
----------
xpath : str
Xpath expression without xpath= prefix.
Returns
-------
:obj:`list` of :obj:`WebElement`
List of visible WebElements.
"""
active_area_xpath = CONFIG["ActiveAreaXpath"]
if ACTIVE_AREA_FUNCTION is not None:
active_area = ACTIVE_AREA_FUNCTION()
if active_area:
xpath = xpath.replace('//', './/', 1)
else:
driver = browser.get_current_browser()
active_area = driver.find_element(By.XPATH, active_area_xpath)
else:
driver = browser.get_current_browser()
try:
active_area = driver.find_element(By.XPATH, active_area_xpath)
if active_area is None:
logger.debug('Got None for active area. Is page still loading '
'or is it missing body tag?')
return None
# //body not found, is page still loading? Return None to continue looping
except NoSuchElementException:
logger.debug(
"Cannot locate //body element. Is page still loading?")
return None
try:
webelements = active_area.find_elements(By.XPATH, xpath)
logger.trace('XPath {} matched {} webelements'.format(
xpath, len(webelements)))
webelements = get_visible_elements_from_elements(webelements, **kwargs)
except StaleElementReferenceException as se:
raise QWebStalingElementError('Got StaleElementException') from se
except (JavascriptException, InvalidSelectorException) as e:
logger.debug('Got {}, returning None'.format(e))
webelements = None
return webelements
def search_from_frames(driver=None, current_frame=None):
keep_frame = kwargs.get('stay_in_current_frame',
CONFIG['StayInCurrentFrame'])
if keep_frame:
return fn(*args, **kwargs)
err = None
if not driver:
driver = browser.get_current_browser()
driver.switch_to.default_content()
if current_frame:
try:
driver.switch_to.frame(current_frame)
logger.debug('switching to childframe {}'.format(str(fn)))
except (StaleElementReferenceException,
WebDriverException) as e:
logger.warn(e)
driver.switch_to.default_content()
raise e
try:
web_element = fn(*args, **kwargs)
except QWebElementNotFoundError as e:
err = e
web_element = None
if is_valid(web_element):
return web_element
start = time.time()
timeout = CONFIG['FrameTimeout']
while time.time() < timeout + start:
frames = fc.check_frames(driver)
for frame in frames:
web_element = search_from_frames(driver=driver,
current_frame=frame)
if is_valid(web_element):
logger.debug(
'Found webelement = {}'.format(web_element))
return web_element
try:
driver.switch_to.parent_frame()
except WebDriverException as e:
driver.switch_to.default_content()
raise e
config.set_config('FrameTimeout',
float(timeout + start - time.time()))
logger.trace('Frame timeout: {}'.format(timeout))
if err:
raise err
return web_element
driver.switch_to.default_content()
raise QWebTimeoutError(
'From frame decorator: Unable to locate element in given time')
def scp(self, local_path, remote_path):
"""Copy files from local_path to remote_path.
connect() method has to be called first!
"""
logger.trace('SCP {0} to {1}:{2}'.format(
local_path, self._ssh.get_transport().getpeername(), remote_path))
# SCPCLient takes a paramiko transport as its only argument
scp = SCPClient(self._ssh.get_transport(), socket_timeout=10)
start = time()
scp.put(local_path, remote_path)
scp.close()
end = time()
logger.trace('SCP took {0} seconds'.format(end-start))
def scp(self, local_path, remote_path):
"""Copy files from local_path to remote_path.
connect() method has to be called first!
"""
logger.trace('SCP {0} to {1}:{2}'.format(
local_path, self._hostname, remote_path))
# SCPCLient takes a paramiko transport as its only argument
scp = SCPClient(self._ssh.get_transport())
start = time()
scp.put(local_path, remote_path)
scp.close()
end = time()
logger.trace('SCP took {0} seconds'.format(end-start))
def process_vxlan_dump(data):
"""
Process API reply acl_dump and return dictionary of usable values.
:param data: API reply from acl_dump call,
:type data: dict
:return: Values ready for comparison with Agent or ETCD values.
:rtype: dict
"""
if len(data) > 1:
logger.debug(len(data))
logger.trace(data)
raise RuntimeError("Data contains more than one API reply.")
data = data["vxlan_tunnel_details"]
index = int(data["sw_if_index"])
mcast_index = int(data["mcast_sw_if_index"])
ipv6 = int(data["is_ipv6"])
if ipv6:
destination_address = data["dst_address"]["ipv6"]
source_address = data["src_address"]["ipv6"]
else:
destination_address = data["dst_address"]["ipv4"]
source_address = data["src_address"]["ipv4"]
vrf_id = int(data["encap_vrf_id"])
vni = int(data["vni"])
next_index = int(data["decap_next_index"])
instance = int(data["instance"])
output = {
"sw_if_index": index,
"ipv6": ipv6,
"vrf": vrf_id,
"vni": vni,
"dst_address": destination_address,
"src_address": source_address,
"next_index": next_index,
"mcast_index": mcast_index,
"instance": instance
}
return output
def connect(self, node):
"""Connect to node prior to running exec_command or scp.
If there already is a connection to the node, this method reuses it.
"""
self._hostname = node['host']
node_hash = self._node_hash(node)
if node_hash in self.__existing_connections:
self._ssh = self.__existing_connections[node_hash]
else:
start = time()
self._ssh.connect(node['host'], username=node['username'],
password=node['password'])
self.__existing_connections[node_hash] = self._ssh
logger.trace('connect took {} seconds'.format(time() - start))
def get_dropdown_element_by_css_selector(locator, anchor, index, **kwargs):
"""Get Dropdown element using css selectors.
Parameters
----------
locator : str
Label text or attribute that points to the dropdown.
Looking for placeholder and commonly used tooltip-attributes first.
If locator is label text, finds input element by it's for attribute.
if for attribute is not available, then finds element by doing some
DOM traversing.
anchor : str
Using if locator is not an XPath.
index : int
If multiple elements use index to pick correct one.
Returns
-------
WebElement
"""
dropdown_elements = []
partial_matches = []
css = 'select'
if 'qweb_old' not in kwargs:
full_matches, partial_matches = element.get_elements_by_css(
locator, css, **kwargs)
if full_matches:
if index != 0:
try:
return full_matches[index]
except IndexError:
raise QWebInstanceDoesNotExistError(
f'Found {len(full_matches)} elements. Given index was {index}'
)
correct_element = text.get_element_using_anchor(
full_matches, anchor)
return correct_element
try:
locator_element = text.get_text_using_anchor(locator, anchor)
dropdown_elements = list(
dict.fromkeys(
element.get_element_from_childnodes(locator_element, css, **
kwargs) + partial_matches))
except QWebElementNotFoundError:
logger.trace(
'Element not found by visible text. Trying with partial match')
dropdown_elements = partial_matches
if dropdown_elements:
return dropdown_elements[index]
return None
def _qemu_qga_flush(self):
"""Flush the QGA parser state
"""
qga_cmd = '(printf "\xFF"; sleep 1) | sudo -S socat - UNIX-CONNECT:' + \
self._qga_sock
#TODO: probably need something else
(ret_code, stdout, stderr) = self._ssh.exec_command(qga_cmd)
if int(ret_code) != 0:
logger.debug('QGA execute failed {0}'.format(stderr))
raise RuntimeError('QGA execute "{0}" '
'failed on {1}'.format(qga_cmd,
self._node['host']))
logger.trace(stdout)
if not stdout:
return {}
return json.loads(stdout.split('\n', 1)[0])
def connect(self, node):
"""Connect to node prior to running exec_command or scp.
If there already is a connection to the node, this method reuses it.
"""
self._hostname = node['host']
node_hash = self._node_hash(node)
if node_hash in self.__existing_connections:
self._ssh = self.__existing_connections[node_hash]
else:
start = time()
self._ssh.connect(node['host'],
username=node['username'],
password=node['password'])
self.__existing_connections[node_hash] = self._ssh
logger.trace('connect took {} seconds'.format(time() - start))
def vpp_show_threads(node):
"""Show VPP threads on node.
:param node: Node to run command on.
:type node: dict
:returns: VPP thread data.
:rtype: list
"""
cmd = u"show_threads"
with PapiSocketExecutor(node) as papi_exec:
reply = papi_exec.add(cmd).get_reply()
threads_data = reply[u"thread_data"]
logger.trace(f"show threads:\n{threads_data}")
return threads_data
def verify_all_timers(self, fail_on_errors=True):
"""
Verifies all timers within a testsuite. Timer's must be done, eg `Start Timer` and `Stop Timer` keywords must have been called for it and it has to have been configured with `Configure Timer` keyword and lower_than parameter.
Keyword will also write a html table into the logs that shows all finished timers and their status.
Parameters:
- ``fail_on_errors`` Should we throw an error if any timers are not within given ranges. Defaults to True
Example:
| Verify All Timers | fail_on_errors=False |
"""
logger.trace("Timer::verify_all_timers({})".format(fail_on_errors))
failures = []
fail_on_errors = is_truthy(fail_on_errors)
html = [
'<table class="statistics"><tr><th>Timer</th><th>Lower than</th><th>Execution Time</th><th>Higher Than</th></tr>'
]
for item in filter(lambda timer: timer_done(timer[1]),
self.benchmarks.items()):
benchmark_name = item[0]
benchmark_data = item[1]
difference = int(
(benchmark_data['stop'] - benchmark_data['start']) * 1000)
lower_than = benchmark_data['lower_than']
higher_than = benchmark_data['higher_than']
if not _is_within_range(difference, lower_than, higher_than):
html.append(
html_row("fail", benchmark_name, lower_than, difference,
higher_than))
failures.append(
assert_string(benchmark_name, difference, lower_than,
higher_than))
else:
html.append(
html_row("pass", benchmark_name, lower_than, difference,
higher_than))
html.append("</table")
logger.info("".join(html), html=True)
if failures:
if fail_on_errors:
raise AssertionError("\n".join(failures))
else:
logger.warn("\n".join(failures))
if failures:
return False
return True
def _wait_until_vm_boot(self, timeout=60):
"""Wait until QEMU VM is booted.
First try to flush qga until there is output.
Then ping QEMU guest agent each 5s until VM booted or timeout.
:param timeout: Waiting timeout in seconds (optional, default 60s).
:type timeout: int
"""
start = time()
while True:
if time() - start > timeout:
raise RuntimeError('timeout, VM {disk} not booted on {host}'.
format(disk=self._qemu_opt['disk_image'],
host=self._node['host']))
out = None
try:
out = self._qemu_qga_flush()
except ValueError:
logger.trace('QGA qga flush unexpected output {out}'.
format(out=out))
# Empty output - VM not booted yet
if not out:
sleep(5)
else:
break
while True:
if time() - start > timeout:
raise RuntimeError('timeout, VM with {disk} not booted '
'on {host}'.
format(disk=self._qemu_opt['disk_image'],
host=self._node['host']))
out = None
try:
out = self._qemu_qga_exec('guest-ping')
except ValueError:
logger.trace('QGA guest-ping unexpected output {out}'.
format(out=out))
# Empty output - VM not booted yet
if not out:
sleep(5)
# Non-error return - VM booted
elif out.get('return') is not None:
break
# Skip error and wait
elif out.get('error') is not None:
sleep(5)
else:
# If there is an unexpected output from QGA guest-info, try
# again until timeout.
logger.trace('QGA guest-ping unexpected output {out}'.
format(out=out))
logger.trace('VM with {disk_image} booted on {host}'.
format(disk_image=self._qemu_opt['disk_image'],
host=self._node['host']))
def get_sw_if_index(self, err_msg=u"Failed to get reply."):
"""Get sw_if_index from reply from VPP Python API.
Frequently, the caller is only interested in sw_if_index field
of the reply, this wrapper makes such call sites shorter.
TODO: Discuss exception types to raise, unify with inner methods.
:param err_msg: The message used if the PAPI command(s) execution fails.
:type err_msg: str
:returns: Response, sw_if_index value of the reply.
:rtype: int
:raises AssertionError: If retval is nonzero, parsing or ssh error.
"""
reply = self.get_reply(err_msg=err_msg)
logger.trace(f"Getting index from {reply!r}")
return reply[u"sw_if_index"]
def packet_operation(expression_str, deviation_str=None, reason=None):
"""
Provide logical and mathematical operation with packet
- expression_str: operand1 operation operand2
| Example | Comments |
| 1M * 2 | Multiple Packet size 2M |
| 1M + 10K | Return packet size 1.01M |
| 1M + 10% | Return packet size 1.1M |
Options:
- deviation: add on for comparison verifications (eq, ,gt, lt, ge, le)
Equality Examples:
| Operand1 Operation Operand2 | Deviation | Result | Comments |
| 10M == 12M | 25% | TRUE | (10M - 25%) < 12M < (10M + 25%) |
| 10M == 12M | 0.25 | TRUE | (10M - 25%) < 12M < (10M + 25%) |
| 10M == 12M | -25% | FALSE | (10M - 25%) < 12M < 10M |
| 10M == 12M | +25% | TRUE | 10M < 12M < (10M + 25%) |
- reason: Custom fail reason
return: TRUE/FALSE for logical operation and value for math operation
"""
try:
logger.trace(
f"{expression_str}{', ' + deviation_str if deviation_str else ''}"
)
_deviation = format_factory(deviation_str,
Percent) if deviation_str else None
operand1, operation, operand2 = _parse_line(expression_str, DataPacket,
Percent)
result = _type_evaluation(operand1=operand1,
operand2=operand2,
operation=operation,
deviation=_deviation,
special_eq=packet_eq)
assert result is not False, f"{operand1} {operation.__name__} {operand2} False" if reason is None else reason
return result
except AssertionError as e:
raise e
except Exception as e:
raise FrameworkError(e)
def set_traffic_input(self, iput):
""" Assign all given traffic data input into this class instance.
Example:
| Set Traffic Input | @{iput} |.
"""
self.traffic.data = iput
# Format 'vlan' key in entity list
for index in xrange(len(self.traffic.data['entities'])):
if 'vlan' in self.traffic.data['entities'][index]:
self.traffic.data['entities'][index]['vlan'] = self._format_vlan_key(
self.traffic.data['entities'][index]['vlan'])
logger.trace(json.dumps(self.traffic.data['serversAndCredentials'], indent=4, separators=(',', ': ')))
return True
def perform(*args, **kwargs):
params = signature(fn).parameters
args, kwargs = _args_to_kwargs(params, args, kwargs)
timeout = get_timeout(**kwargs)
err = None
msg = None
performed = False
logger.debug('time to run {}'.format(timeout))
start = time.time()
while time.time() < timeout + start:
try:
return fn(*args, **kwargs)
except QWebValueMismatchError as mismatch:
if 'text_appearance' not in str(
fn) and 'get_or_compare_text' not in str(fn):
err = QWebValueError
msg = mismatch
logger.trace('Value mismatch: {}'.format(mismatch))
continue
except (QWebElementNotFoundError, UnexpectedAlertPresentException):
logger.debug('Not found')
time.sleep(SHORT_DELAY)
except QWebValueError as ve:
if performed:
break
raise ve
except (QWebStalingElementError,
StaleElementReferenceException) as S:
if 'execute_click' in str(fn) or 'text_appearance' in str(fn):
logger.info('Got staling element err from retry click.'
'Action is probably triggered.')
raise QWebUnexpectedConditionError(S)
raise QWebStalingElementError('Staling element')
except (WebDriverException, QWebDriverError) as wde:
if 'alert' in str(fn):
time.sleep(LONG_DELAY)
logger.info(
"Got webdriver exception..{}. Retrying..".format(wde))
err = QWebDriverError
msg = wde
else:
raise QWebDriverError(wde)
if msg:
raise err(msg)
raise QWebTimeoutError('Timeout exceeded')
def measure_loss(self, rate, frame_size, loss_acceptance,
loss_acceptance_type, traffic_type):
"""Runs the traffic and evaluate the measured results.
:param rate: Offered traffic load.
:param frame_size: Size of frame.
:param loss_acceptance: Permitted drop ratio or frames count.
:param loss_acceptance_type: Type of permitted loss.
:param traffic_type: Traffic profile ([2,3]-node-L[2,3], ...).
:type rate: int
:type frame_size: str
:type loss_acceptance: float
:type loss_acceptance_type: LossAcceptanceType
:type traffic_type: str
:returns: Drop threshold exceeded? (True/False)
:rtype: bool
:raises: NotImplementedError if TG is not supported.
:raises: RuntimeError if TG is not specified.
"""
# we need instance of TrafficGenerator instantiated by Robot Framework
# to be able to use trex_stl-*()
tg_instance = BuiltIn().get_library_instance(
'resources.libraries.python.TrafficGenerator')
if tg_instance.node['subtype'] is None:
raise RuntimeError('TG subtype not defined')
elif tg_instance.node['subtype'] == NodeSubTypeTG.TREX:
unit_rate = str(rate) + self.get_rate_type_str()
tg_instance.trex_stl_start_remote_exec(self.get_duration(),
unit_rate, frame_size,
traffic_type)
loss = tg_instance.get_loss()
sent = tg_instance.get_sent()
if self.loss_acceptance_type_is_percentage():
loss = (float(loss) / float(sent)) * 100
logger.trace("comparing: {} < {} {}".format(loss,
loss_acceptance,
loss_acceptance_type))
if float(loss) > float(loss_acceptance):
return False
else:
return True
else:
raise NotImplementedError("TG subtype not supported")
def check_vxlan_tunnel_presence_from_api(data, src, dst, vni):
for iface in data:
if iface["src_address"] == src and iface[
"dst_address"] == dst and iface["vni"] == int(vni):
logger.trace(
"matched interface: {interface}".format(interface=iface))
return True, iface["sw_if_index"]
else:
logger.debug("interface with:\n"
"src_addr: {src_address}\n"
"dst_addr: {dst_address}\n"
"vni: {vni}\n"
"not found in dump. Full dump:\n"
"{data}".format(src_address=src,
dst_address=dst,
vni=vni,
data=data))
def run_dirs3arch(self, url, extensions, wordlist, results_path):
self.url = url
self.extensions = extensions
self.wordlist = wordlist
self.results_path = results_path
results_mount = Mount("/dirs3arch_results",
self.results_path,
type="bind")
wordlist_mount = Mount("/wordlist.txt", self.wordlist, type="bind")
mounts = [results_mount, wordlist_mount]
command = '-u {0} -e {1} -w {2} --json-report=/dirs3arch_results/dirs3arch_results.json'.format(
self.url, self.extensions, '/wordlist.txt')
self.client.containers.run(self.dirs3arch_docker,
mounts=mounts,
command=command)
logger.trace(
"Successfully ran dirs3arch against the target URL {0}. Please find the *.json output in the results directory"
.format(self.url))
def process_bridge_domain_dump(data):
"""Process API reply bridge_domain_dump and return dictionary of usable values.
:param data: API reply from bridge_domain_dump call,
:type data: dict
:return: Values ready for comparison with Agent or ETCD values.
:rtype: dict
:raises RuntimeError: If the data is in an unexpceted format,
"""
if len(data) > 1:
logger.debug(len(data))
logger.trace(data)
raise RuntimeError("Data contains more than one API reply.")
data = data["bridge_domain_details"]
return data
def setup_framework(nodes):
"""Pack the whole directory and extract in temp on each node.
:param nodes: Topology nodes.
:type nodes: dict
:raises RuntimeError: If setup framework failed.
"""
tarball = pack_framework_dir()
msg = f"Framework packed to {tarball}"
logger.console(msg)
logger.trace(msg)
remote_tarball = f"{tarball}"
results = list()
logs = list()
threads = list()
for node in nodes.values():
args = node, tarball, remote_tarball, results, logs
thread = threading.Thread(target=setup_node, args=args)
thread.start()
threads.append(thread)
logger.info(
u"Executing node setups in parallel, waiting for threads to end.")
for thread in threads:
thread.join()
logger.info(f"Results: {results}")
for log in logs:
logger.trace(log)
delete_local_tarball(tarball)
if all(results):
logger.console(u"All nodes are ready.")
for node in nodes.values():
logger.info(
f"Setup of node {node[u'type']} host {node[u'host']}, "
f"port {node[u'port']} done.")
else:
raise RuntimeError(u"Failed to setup framework.")
def crypto_device_verify(node, force_init=False, numvfs=32):
"""Verify if Crypto QAT device virtual functions are initialized on all
DUTs. If parameter force initialization is set to True, then try to
initialize or disable QAT.
:param node: DUT node.
:param force_init: If True then try to initialize to specific value.
:param numvfs: Number of VFs to initialize, 0 - disable the VFs.
:type node: dict
:type force_init: bool
:type numvfs: int
:returns: nothing
:raises RuntimeError: If QAT is not initialized or failed to initialize.
"""
ssh = SSH()
ssh.connect(node)
cryptodev = Topology.get_cryptodev(node)
cmd = 'cat /sys/bus/pci/devices/{0}/sriov_numvfs'.\
format(cryptodev.replace(':', r'\:'))
# Try to read number of VFs from PCI address of QAT device
for _ in range(3):
ret_code, stdout, _ = ssh.exec_command(cmd)
if int(ret_code) == 0:
try:
sriov_numvfs = int(stdout)
except ValueError:
logger.trace(
'Reading sriov_numvfs info failed on {0}'.format(
node['host']))
else:
if sriov_numvfs != numvfs:
if force_init:
# QAT is not initialized and we want to initialize
# with numvfs
DUTSetup.crypto_device_init(node, numvfs)
else:
raise RuntimeError(
'QAT device {0} is not '
'initialized to {1} on host {2}'.format(
cryptodev, numvfs, node['host']))
break
def connect(self, node, attempts=5):
"""Connect to node prior to running exec_command or scp.
If there already is a connection to the node, this method reuses it.
"""
try:
self._node = node
node_hash = self._node_hash(node)
if node_hash in SSH.__existing_connections:
self._ssh = SSH.__existing_connections[node_hash]
logger.debug('reusing ssh: {0}'.format(self._ssh))
else:
start = time()
pkey = None
if 'priv_key' in node:
pkey = RSAKey.from_private_key(
StringIO.StringIO(node['priv_key']))
self._ssh = paramiko.SSHClient()
self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self._ssh.connect(node['host'],
username=node['username'],
password=node.get('password'),
pkey=pkey,
port=node['port'])
self._ssh.get_transport().set_keepalive(10)
SSH.__existing_connections[node_hash] = self._ssh
logger.trace('connect took {} seconds'.format(time() - start))
logger.debug('new ssh: {0}'.format(self._ssh))
logger.debug('Connect peer: {0}'.format(
self._ssh.get_transport().getpeername()))
logger.debug('Connections: {0}'.format(
str(SSH.__existing_connections)))
except:
if attempts > 0:
self._reconnect(attempts - 1)
else:
raise
def put_honeycomb_data(node,
url_file,
data,
path="",
data_representation=DataRepresentation.JSON):
"""Send configuration data using PUT request and return the status code
and response content.
:param node: Honeycomb node.
:param url_file: URL file. The argument contains only the name of file
without extension, not the full path.
:param data: Configuration data to be sent to Honeycomb.
:param path: Path which is added to the base path to identify the data.
:param data_representation: How the data is represented.
:type node: dict
:type url_file: str
:type data: dict, str
:type path: str
:type data_representation: DataRepresentation
:returns: Status code and content of response.
:rtype: tuple
:raises HoneycombError: If the given data representation is not defined
in HEADERS.
"""
try:
header = HEADERS[data_representation]
except AttributeError as err:
raise HoneycombError(
"Wrong data representation: {0}.".format(data_representation),
repr(err))
if data_representation == DataRepresentation.JSON:
data = dumps(data)
logger.trace(data)
base_path = HoneycombUtil.read_path_from_url_file(url_file)
path = base_path + path
logger.trace(path)
return HTTPRequest.put(node=node,
path=path,
headers=header,
payload=data)
def __init__(self, node, json_param=True):
json_text = u" json" if json_param else u""
self.json = json_param
self._node = node
self._ssh = SSH()
self._ssh.connect(self._node)
try:
self._tty = self._ssh.interactive_terminal_open()
except Exception:
raise RuntimeError(
f"Cannot open interactive terminal on node "
f"{self._node[u'host']}"
)
for _ in range(3):
try:
self._ssh.interactive_terminal_exec_command(
self._tty, f"sudo -S {Constants.VAT_BIN_NAME}{json_text}",
self.__VAT_PROMPT
)
except Exception:
continue
else:
break
else:
vpp_pid = get_vpp_pid(self._node)
if vpp_pid:
if isinstance(vpp_pid, int):
logger.trace(f"VPP running on node {self._node[u'host']}")
else:
logger.error(
f"More instances of VPP running "
f"on node {self._node[u'host']}."
)
else:
logger.error(f"VPP not running on node {self._node[u'host']}.")
raise RuntimeError(
f"Failed to open VAT console on node {self._node[u'host']}"
)
self._exec_failure = False
self.vat_stdout = None
def get_pci_dev_driver(node, pci_addr):
"""Get current PCI device driver on node.
:param node: DUT node.
:param pci_addr: PCI device address.
:type node: dict
:type pci_addr: str
:returns: Driver or None
:raises RuntimeError: If PCI rescan or lspci command execution failed.
"""
ssh = SSH()
ssh.connect(node)
for i in range(3):
logger.trace('Try {0}: Get interface driver'.format(i))
cmd = 'sh -c "echo 1 > /sys/bus/pci/rescan"'
ret_code, _, _ = ssh.exec_command_sudo(cmd)
if int(ret_code) != 0:
raise RuntimeError("'{0}' failed on '{1}'".format(
cmd, node['host']))
cmd = 'lspci -vmmks {0}'.format(pci_addr)
ret_code, stdout, _ = ssh.exec_command(cmd)
if int(ret_code) != 0:
raise RuntimeError("'{0}' failed on '{1}'".format(
cmd, node['host']))
for line in stdout.splitlines():
if len(line) == 0:
continue
name = None
value = None
try:
name, value = line.split("\t", 1)
except ValueError:
if name == "Driver:":
return None
if name == 'Driver:':
return value
else:
return None
def __init__(self, node, json_param=True):
json_text = ' json' if json_param else ''
self.json = json_param
self._node = node
self._ssh = SSH()
self._ssh.connect(self._node)
try:
self._tty = self._ssh.interactive_terminal_open()
except Exception:
raise RuntimeError(
"Cannot open interactive terminal on node {0}".format(
self._node))
for _ in range(3):
try:
self._ssh.interactive_terminal_exec_command(
self._tty, 'sudo -S {0}{1}'.format(Constants.VAT_BIN_NAME,
json_text),
self.__VAT_PROMPT)
except Exception:
continue
else:
break
else:
vpp_pid = get_vpp_pid(self._node)
if vpp_pid:
if isinstance(vpp_pid, int):
logger.trace("VPP running on node {0}".format(
self._node['host']))
else:
logger.error(
"More instances of VPP running on node {0}.".format(
self._node['host']))
else:
logger.error("VPP not running on node {0}.".format(
self._node['host']))
raise RuntimeError("Failed to open VAT console on node {0}".format(
self._node['host']))
self._exec_failure = False
self.vat_stdout = None
def execute_script(self, local_path, node, timeout=10, json_out=True):
"""Copy local_path script to node, execute it and return result.
Returns (rc, stdout, stderr tuple).
"""
ssh = SSH()
ssh.connect(node)
local_basename = os.path.basename(local_path)
remote_file_path = self.__TMP_DIR + local_basename
remote_file_out = remote_file_path + ".out"
ssh.scp(local_path, remote_file_path)
cmd = "sudo -S {vat} {json} < {input}".format(vat=self.__VAT_BIN,
json="json" if json_out == True else "",
input=remote_file_path)
(ret_code, stdout, stderr) = ssh.exec_command(cmd, timeout)
self._ret_code = ret_code
self._stdout = stdout
self._stderr = stderr
logger.trace("Command '{0}' returned {1}'".format(cmd, self._ret_code))
logger.trace("stdout: '{0}'".format(self._stdout))
logger.trace("stderr: '{0}'".format(self._stderr))
#TODO: download vpp_api_test output file
self._delete_files(node, remote_file_path, remote_file_out)
def _trace(self, message):
logger.trace(msg=message)
def test_logged_to_python(self):
logger.info("Foo")
logger.debug("Boo")
logger.trace("Goo")
logger.write("Doo", 'INFO')
self.assertEquals(self.handler.messages, ['Foo', 'Boo', 'Goo', 'Doo'])
def msg(self, msg, *args):
# Forward telnetlib's debug messages to log
logger.trace(msg % args)
