def _scan_t_subcases(self, path, prefix):
# we just create here the list of parameters we'll use to
# create the subtestcase on every execution of the testcase
# itself (done in configure_10 when the TC is executed). Read
# that for reasons...
with open(path) as tf:
subcases = re.findall(self._regex_t_subcases, tf.read())
for name in subcases:
# make sure to remove leading/trailing whitespace and then
# trailing _--this allows it to match what the bats tool
# scans, which we'll need to match in the output of
# tap_parse_output() in _eval_one()
_name = commonl.name_make_safe(name.strip()).rstrip("_")
t_file_name = os.path.basename(path)
self.subtc_list.append((
# note we'll key on the .t file basename and the subtest name
# inside the .t file, otherwise if two different .t files
# in the dir have same subtest names, they'd override.
# In _eval_one() we'll refer to this dict
# We keep this, even if is basically the same because
# it will easily make configure_10 later print a
# shorter list for reference in reports...
t_file_name + "." + _name,
# testcase full name / unique ID
prefix + t_file_name.replace(".t", "") + "." + _name,
self.kws['thisfile'], self.origin
))
def _eval_one(self, target, t_file, prefix = ""):
result = tcfl.tc.result_c(0, 0, 0, 0, 0)
self.report_info("running %s%s" % (prefix, t_file))
self.kw_set("t_file", t_file)
output = target.shell.run(
# remember we cd'ed into the directory
"bats --tap %s || echo FAI''LED-%s"
% (t_file, self.kws['tc_hash']),
output = True)
if 'bats: command not found' in output:
raise tcfl.tc.blocked_e(
"'bats' tool not installed in the target",
dict(target = target, output = output))
# top level result
if 'FAILED-%(tc_hash)s' % self.kws in output:
result.failed += 1
else:
result.passed += 1
# seems we had execution, so let's parse the output and
# make subcases
tcs = tap_parse_output(output)
for name, data in tcs.iteritems():
_name = commonl.name_make_safe(name)
t_file_name = t_file.replace(".t", "")
subcase = tc_taps_subcase_c_base(
prefix + t_file_name + "." + _name,
self.kws['thisfile'], self.origin,
self, data)
self.post_tc_append(subcase)
result += subcase.result
return result
def __init__(self, path, t_file_path):
tcfl.tc.tc_c.__init__(self, commonl.name_make_safe(path),
# these two count as the ones that started this
path, t_file_path)
# t_file_path goes to self.kws['thisfile'], name to self.name
# and self.kws['tc_name']
self.image = None
self.rel_path_in_target = None
self.t_files = [ t_file_path ]
def __init__(self, target, template_image_filename, capturer,
in_area, merge_similar, min_width, min_height,
poll_period, timeout, raise_on_timeout, raise_on_found,
name = None):
if not image_expectation_works:
raise RuntimeError("Image matching won't work; need packages"
" cv2, imutils, numpy")
assert isinstance(target, tc.target_c)
assert isinstance(capturer, basestring)
assert in_area == None \
or isinstance(in_area, collections.Iterable) \
and len(in_area) == 4 \
and all(i >= 0 and i <= 1 for i in in_area), \
'in_area parameter must be a tuple of four numbers ' \
'from 0 to 1 describing the upper-left and lower-right ' \
'of the area where to look for the image, ' \
'eg: (0, 0, 0.5, 0.5) means in the top left quarter'
assert merge_similar >= 0.0 and merge_similar <= 1.0, \
"merge_similar has to be a float from 0.0 to 1.0"
assert name == None or isinstance(name, basestring)
tc.expectation_c.__init__(self, target, poll_period, timeout,
raise_on_timeout = raise_on_timeout,
raise_on_found = raise_on_found)
self.capturer = capturer
self.in_area = in_area
self.merge_similar = merge_similar
# if the image is relative, we make it relative to the
# filename of the caller--so we can do short names
if name:
self.name = name
else:
self.name = commonl.name_make_safe(template_image_filename)
if os.path.isabs(template_image_filename):
self.template_image_filename = template_image_filename
else:
self.template_image_filename = os.path.join(
# 2 up the stack is the guy who called
# target.capture.image_on_screeshot()
os.path.dirname(inspect.stack()[2][1]),
template_image_filename)
with open(self.template_image_filename) as _f:
# try to open it, cv2.imread() is quite crappy at giving
# errors on file not found
pass
self.template_img = cv2.imread(self.template_image_filename,
cv2.IMREAD_GRAYSCALE)
# FIXME: raise exception if too small
self.min_width = min_width
self.min_height = min_height
def _scan_t_subcases(self, path, prefix):
# we just create here the list of parameters we'll use to
# create the subtestcase on every execution of the testcase
# itself (done in configure_10 when the TC is executed). Read
# that for reasons...
with open(path) as tf:
subcases = re.findall(self._regex_t_subcases, tf.read())
for name in subcases:
# here we need to be careful to treat the scanned name
# exactly the same way the bats tool will do it, otherwise
# when we scan them from the bats output, they won't match
name = name.strip()
# strip the simple or double quotes from the name
# -> "Starting somethings --id=\"blah\""
# <- Starting somethings --id=\"blah\"
name = name[1:-1]
# ok, now somethings might have been escaped with \...we
# can ignore it, since we are not affected by it...
# -> Starting somethings --id=\"blah\"
# <- Starting somethings --id="blah"
name = name.replace("\\", "")
# strip here, as BATS will do too
_name = commonl.name_make_safe(name.strip())
logging.debug("%s contains subcase %s", path, _name)
t_file_name = os.path.basename(path)
self.subtc_list.append((
# note we'll key on the .t file basename and the subtest name
# inside the .t file, otherwise if two different .t files
# in the dir have same subtest names, they'd override.
# In _eval_one() we'll refer to this dict
# We keep this, even if is basically the same because
# it will easily make configure_10 later print a
# shorter list for reference in reports...
t_file_name + "." + _name,
# testcase full name / unique ID
prefix + t_file_name.replace(".t", "") + "." + _name,
self.kws['thisfile'],
self.origin))
def _eval_one(self, target, t_file, prefix):
result = tcfl.tc.result_c(0, 0, 0, 0, 0)
rel_file_path = os.path.join(prefix, t_file)
if self._ts_ignore(rel_file_path):
target.report_skip(
"%s: skipped due to configuration "
"(tcfl.tc_clear_bbt.ignore_ts or BBT_IGNORE_TS environment)"
% rel_file_path)
for name, subtc in self.subtc.iteritems():
if name.startswith(t_file):
subtc.result = tcfl.tc.result_c(0, 0, 0, 0, 1)
subtc.data = dict(result = "skipped")
subtc.update(
tcfl.tc.result_c(0, 0, 0, 0, 1),
"skipped due to configuration"
" (tcfl.tc_clear_bbt.ignore_ts or"
" BBT_IGNORE_TS environment)",
"")
result.skipped += 1
else:
self.report_info("running %s%s" % (prefix, t_file))
self.kw_set("t_file", t_file)
# patch any execution hot fixes
pre_sh_l = []
if self.test_bundle_name in bundle_run_pre_sh:
self.report_info("adding configured pre_sh steps from "
"tcfl.tc_clear_bbt.bundle_run_pre_sh[%s]"
% self.test_bundle_name)
pre_sh_l += bundle_run_pre_sh[self.test_bundle_name]
if t_file in bundle_run_pre_sh:
self.report_info("adding configured pre_sh steps from "
"tcfl.tc_clear_bbt.bundle_run_pre_sh[%s]"
% t_file)
pre_sh_l += bundle_run_pre_sh[t_file]
for pre_sh in pre_sh_l:
target.shell.run(pre_sh % self.kws)
# Run the t_file
# remember we cd'ed into the directory, the way these
# BBTs are written, it is expected
if self.bats_parallel and 'use_parallel' in t_file:
# note we set CUPS in the target in start()
parallel = "-j $CPUS"
else:
parallel = ""
run_timeout = _bundle_run_timeout(self, self.test_bundle_name, t_file)
output = target.shell.run(
"bats --tap %s %s || echo FAILED''-%s"
% (t_file, parallel, self.kws['tc_hash']),
output = True, timeout = run_timeout)
# top level result
if 'bats: command not found' in output:
self.report_error(
"'bats' tool not installed in the target",
dict(target = target, output = output))
result.errors += 1
elif 'FAILED-%(tc_hash)s' % self.kws in output:
result.failed += 1
else:
result.passed += 1
# seems we had execution, so let's parse the output and
# make subcases of the .t -- if anything fails, catch and
# convert to a TCF exception so it only affects *this*
# testcase in the result accounting--note
# report_from_exception() will report exceptio data so we
# can debug if it is an infra or TC problem.
try:
tcs = tap_parse_output(output)
except Exception as e:
tcs = dict()
result += tcfl.tc.result_c.report_from_exception(self, e)
for name, data in tcs.iteritems():
# get the subtc; see _scan_t_subcases() were we keyed
# them in
_name = commonl.name_make_safe(name.strip())
tc_name = t_file + "." + _name
if tc_name in bundle_t_map:
_tc_name = bundle_t_map[tc_name]
self.report_info("subtestcase name %s mapped to %s "
"per configuration "
"tcfl.tc_clear_bbt.bundle_t_map"
% (tc_name, _tc_name))
else:
_tc_name = tc_name
subtc = self.subtc[_tc_name]
if self._ts_ignore(subtc.name):
_result = tcfl.tc.result_c(0, 0, 0, 0, 1)
summary = "result skipped due to configuration " \
"(tcfl.tc_clear_bbt.ignore_ts or " \
"BBT_IGNORE_TS environment)"
log = data['output']
else:
# translate the taps result to a TCF result, record it
_result = self.mapping[data['result']]
log = data['output']
summary = log.split('\n', 1)[0]
subtc.update(_result, summary, log, )
result += subtc.result
return result
def _output_parse(self, testsuite, suite_tc, lf):
#
# Parse the log file out of running a single test suite; the
# file format is more or less
#
# START: ptest-runner
# <timestamp>
# BEGIN: <suite>
# <output for subtc>
# FAIL|SKIP|PASS: <subtc>
# <output for subtc>
# FAIL|SKIP|PASS: <subtc>
# <output for subtc>
# FAIL|SKIP|PASS: <subtc>
# DURATION: <n>
# END: <suite>
# <timestamp>
# STOP: ptest-runner
#
result = tcfl.tc.result_c()
cnt = -1
start_seen = False
suite = None
log = ""
date_regex = re.compile("^[-:T0-9]+$") # lazy YYYY-MM-DDTHH:MM
for line in lf:
line = line.rstrip()
cnt += 1
if line.startswith("START: "):
_, name = line.split(None, 1)
assert name == 'ptest-runner', \
"%d: Found START for %s, not ptest-runner" % (cnt, name)
start_seen = True
elif line.startswith("STOP:"):
assert start_seen, "%d: Found STOP: without START" % cnt
_, name = line.split(None, 1)
assert name == 'ptest-runner', \
"%d: Found STOP for %s, not ptest-runner" % (cnt, name)
start_seen = False
elif date_regex.search(line):
pass # ignored
elif line.startswith("BEGIN:"):
assert start_seen, "%d: Found BEGIN: without START" % cnt
_, suite = line.split(None, 1)
if suite != testsuite \
and testsuite not in suite: # /usr/lib/SUITE/ptest HACK
raise AssertionError(
"%d: Found BEGIN: for %s, expected %s" \
% (cnt, suite, testsuite))
log = ""
elif line.startswith("END:"):
assert start_seen, "%d: Found END: without START" % cnt
assert suite, "%d: Found END: without suite" % cnt
_, _suite = line.split(None, 1)
assert suite == _suite, \
"%d: Found END: for suite %s different to" \
" START:'s %s" % (cnt, _suite, suite)
suite = None
elif line.startswith("PASS:"******"FAIL:") \
or line.startswith("SKIP:"):
tag, subcase = line.split(None, 1)
assert start_seen, "%d: Found %s without START" % cnt
assert suite, "%d: Found %s without suite" % cnt
# some tests print names that are ...hummm...useless
subcase_safe = commonl.name_make_safe(subcase)
if tag == "PASS:"******"FAIL:":
_result = tcfl.tc.result_c(failed = 1)
elif tag == "SKIP:":
_result = tcfl.tc.result_c(skipped = 1)
else:
raise AssertionError("unknown tag from command output '%s'"
% tag)
#
# Create a subtestcase with the information we found
# about the execution of this SUBCASE of this SUITE
#
name = self.name + "##%s/%s" % (testsuite, subcase_safe)
subtc = tcfl.tc.subtc_c(name, self.kws['thisfile'],
suite, self)
subtc.update(_result, line, log)
result += _result
self.subtc[subcase] = subtc
log = ""
elif line.startswith("DURATION:"):
pass # ignored
else:
log += line + "\n"
# well, we done -- check thinks are as they should be, test
# closed and all
if suite_tc.result:
# no wait, we have already updated the suite testcase
# data; this was probably a timeout, so we know bad stuff
# happened
return
if suite:
# this should have been closed by a END: tag
result.errors += 1
suite_tc.update(result,
"testsuite probably didn't complete"
" execution, no END tag", log)
elif start_seen:
# this should have been closed by a STOP tag
result.errors += 1
suite_tc.update(result,
"testsuite probably didn't complete"
" execution, no STOP tag", log)
else:
result.passed += 1
lf.seek(0, 0)
suite_tc.update(result, "testsuite completed execution",
lf.read())
def flash_spec_parse(self, flash_image_s=None):
"""Parse a images to flash specification in a string (that might be
taken from the environment
The string describing what to flash is in the form::
[[no-]soft] [[no-]upload] IMAGE:NAME[ IMAGE:NAME[..]]]
- *soft*: flash in soft mode (default *False) (see
:meth:`target.images.flash
<tcfl.target_ext_images.extension.flash>`) the image(s) will
only be flashed if the image to be flashed is different than the
last image that was flashed.
- *upload*: flash in soft mode (default *True*) (see
:meth:`target.images.flash
<tcfl.target_ext_images.extension.flash>`). The file will be
uploaded to the server first or it will be assumed it is already
present in the server.
- *IMAGETYPE:FILENAME* flash file *FILENAME* in flash destination
*IMAGETYPE*; *IMAGETYPE* being any of the image
destinations the target can flash; can be found with::
$ tcf image-ls TARGETNAME
or from the inventory::
$ tcf get TARGETNAME -p interfaces.images
The string specification will be taken, in this order from the
following list
- the *image_flash* parameter
- environment *IMAGE_FLASH_<TYPE>*
- environment *IMAGE_FLASH_<FULLID>*
- environment *IMAGE_FLASH_<ID>*
- environment *IMAGE_FLASH*
With *TYPE*, *FULLID* and *ID* sanitized to any character outside
of the ranges *[a-zA-Z0-9_]* replaced with an underscore (*_*).
**Example**
::
$ export IMAGE_FLASH_TYPE1="soft bios:path/to/bios.bin"
$ tcf run test_SOMESCRIPT.py
if *test_SOMESCRIPT.py* uses this template, every invocation of it
on a machine of type TYPE1 will result on the file
*path/to/bios.bin* being flashed on the *bios* location (however,
because of *soft*, if it was already flashed before, it will be
skipped).
:return: a tuple of
>>> ( DICT, UPLOAD, SOFT )
- *DICT*: Dictionary is a dictionary of IMAGETYPE/file name to flash:
>>> {
>>> IMGTYPE1: IMGFILE1,
>>> IMGTYPE2: IMGFILE2,
>>> ...
>>> }
- *UPLOAD*: boolean indicating if the user wants the files to be
uploaded (*True*, default) or to assume they are already in
the server (*False*).
- *SOFT*: flash in soft mode (*True*) or not (*False*, default
if not given).
"""
target = self.target
if not flash_image_s:
# empty, load from environment
target_id_safe = commonl.name_make_safe(
target.id, string.ascii_letters + string.digits)
target_fullid_safe = commonl.name_make_safe(
target.fullid, string.ascii_letters + string.digits)
target_type_safe = commonl.name_make_safe(
target.type, string.ascii_letters + string.digits)
source = None # keep pylint happy
sourcel = [
# go from most specifcy to most generic
"IMAGE_FLASH_%s" % target_fullid_safe,
"IMAGE_FLASH_%s" % target_id_safe,
"IMAGE_FLASH_%s" % target_type_safe,
"IMAGE_FLASH",
]
for source in sourcel:
flash_image_s = os.environ.get(source, None)
if flash_image_s != None:
break
else:
target.report_info(
"skipping image flashing (no function argument nor environment: %s)"
% " ".join(sourcel))
return {}, False, False
else:
source = "function argument"
# verify the format
if not self._image_flash_regex.search(flash_image_s):
raise tc.blocked_e(
"image specification in %s does not conform to the form"
" [[no-]soft] [[no-]upload] IMAGE:NAME[ IMAGE:NAME[..]]]" %
source, dict(target=target))
image_flash = {}
soft = False
upload = True
for entry in flash_image_s.split(" "):
if not entry: # empty spaces...welp
continue
if entry == "soft":
soft = True
continue
if entry == "no-soft":
soft = False
continue
if entry == "upload":
upload = True
continue
if entry == "no-upload":
upload = False
continue
name, value = entry.split(":", 1)
image_flash[name] = value
return image_flash, upload, soft
def flash_spec_parse(self, flash_image_s=None, env_prefix="IMAGE_FLASH"):
"""Parse a images to flash specification in a string (that might be
taken from the environment
The string describing what to flash is in the form::
[[no-]soft] [[no-]upload] IMAGE:NAME[ IMAGE:NAME[..]]]
- *soft*: flash in soft mode (default *False) (see
:meth:`target.images.flash
<tcfl.target_ext_images.extension.flash>`) the image(s) will
only be flashed if the image to be flashed is different than the
last image that was flashed.
- *upload*: flash in soft mode (default *True*) (see
:meth:`target.images.flash
<tcfl.target_ext_images.extension.flash>`). The file will be
uploaded to the server first or it will be assumed it is already
present in the server.
- *IMAGETYPE:FILENAME* flash file *FILENAME* in flash destination
*IMAGETYPE*; *IMAGETYPE* being any of the image
destinations the target can flash; can be found with::
$ tcf image-ls TARGETNAME
or from the inventory::
$ tcf get TARGETNAME -p interfaces.images
Note that the specification can include strings *%(FIELD)s* to be
replaced by values extracted from the inventory and
formatted using Python % formatting. This resolution will
recurse up to five times (eg: if string *some-%(FIELD1)s* resolves
into *some-thing-%(FIELD2)s* and expansion of *FIELD2*
yields *some-thing-nice*).
The string specification will be taken, in this order from the
following list
- the *image_flash* parameter
- environment *IMAGE_FLASH_<TYPE>*
- environment *IMAGE_FLASH_<FULLID>*
- environment *IMAGE_FLASH_<ID>*
- environment *IMAGE_FLASH*
With *TYPE*, *FULLID* and *ID* sanitized to any character outside
of the ranges *[a-zA-Z0-9_]* replaced with an underscore (*_*).
**Example**
::
$ export IMAGE_FLASH_TYPE1="soft bios:path/to/bios.bin"
$ tcf run test_SOMESCRIPT.py
if *test_SOMESCRIPT.py* uses this template, every invocation of it
on a machine of type TYPE1 will result on the file
*path/to/bios.bin* being flashed on the *bios* location (however,
because of *soft*, if it was already flashed before, it will be
skipped).
:return: a tuple of
>>> ( DICT, UPLOAD, SOFT )
- *DICT*: Dictionary is a dictionary of IMAGETYPE/file name to flash:
>>> {
>>> IMGTYPE1: IMGFILE1,
>>> IMGTYPE2: IMGFILE2,
>>> ...
>>> }
- *UPLOAD*: boolean indicating if the user wants the files to be
uploaded (*True*, default) or to assume they are already in
the server (*False*).
- *SOFT*: flash in soft mode (*True*) or not (*False*, default
if not given).
"""
target = self.target
if not flash_image_s:
# empty, load from environment
target_id_safe = commonl.name_make_safe(
target.id, string.ascii_letters + string.digits)
target_fullid_safe = commonl.name_make_safe(
target.fullid, string.ascii_letters + string.digits)
target_type_safe = commonl.name_make_safe(
target.type, string.ascii_letters + string.digits)
source = None # keep pylint happy
sourcel = [
# go from most specifcy to most generic
# IMAGE_FLASH_SERVER_NAME
# IMAGE_FLASH_NAME
# IMAGE_FLASH_TYPE
# IMAGE_FLASH
f"{env_prefix}_{target_fullid_safe}",
f"{env_prefix}_{target_id_safe}",
f"{env_prefix}_{target_type_safe}",
env_prefix,
]
for source in sourcel:
flash_image_s = os.environ.get(source, None)
if flash_image_s != None:
break
else:
target.report_info(
"skipping image flashing (no function argument nor environment: %s)"
% " ".join(sourcel))
return {}, False, False
else:
source = "function argument"
# verify the format
if not self._image_flash_regex.search(flash_image_s):
raise tc.blocked_e(
"image specification in %s does not conform to the form"
" [[no-]soft] [[no-]upload] [IMAGE:]NAME[ [IMAGE:]NAME[..]]]" %
source, dict(target=target))
image_flash = {}
soft = False
upload = True
for entry in flash_image_s.split(" "):
if not entry: # empty spaces...welp
continue
if entry == "soft":
soft = True
continue
if entry == "no-soft":
soft = False
continue
if entry == "upload":
upload = True
continue
if entry == "no-upload":
upload = False
continue
# Expand keywords, so we can do things like
# bios:/BIOSDIR/%(type)s/LATEST-%(bsps.x86_64.cpu-CPU0.family)s
entry = commonl.kws_expand(entry, target.kws)
# see if we can assume this is in the form
# [SOMEPATH/]FILENAME[.EXT]; if FILENAME matches a known
# flashing destination, we take it
if ":" in entry:
name, value = entry.split(":", 1)
image_flash[name] = value
else:
basename, _ext = os.path.splitext(os.path.basename(entry))
# if the inventory publishes a
# interfaces.images.basename, this means we have a
# flasher where this can go
image_typel = list(
target.rt.get("interfaces", {}).get("images", {}))
for image_type in image_typel:
if basename.startswith(image_type):
image_flash[image_type] = entry
break
else:
raise RuntimeError(
"%s: can't auto-guess destination for this "
"file, please prefix IMAGETYPE: "
"(known are: %s)" % (entry, " ".join(image_typel)))
return image_flash, upload, soft
def eval_00(self, ic, target):
target.shell.prompt_regex = re.compile(r"KTS-PROMPT[\$%#]")
target.shell.run(
f"echo $PS1 | grep -q KTS-PROMPT"
f" || export PS1='TCF-{self.ticket}:KTS-PROMPT$ '"
f" # a simple prompt is harder to confuse with general output")
if not self.subcases:
raise RuntimeError("FIXME: no subcases")
tcfl.tl.linux_time_set(target)
# check if stdbuf is available to use it if so
output = target.shell.run(
"stdbuf --help > /dev/null || echo N''OPE # stdbuf available??",
output=True,
trim=True)
if 'NOPE' in output:
stdbuf_prefix = ""
else:
# run the command prefixing this to reduce the
# buffering; otherwise suites that take too long will
# see their ouput buffered in tee and
# resulting in timeouts expecting progress
# stdbuf is part of GNU coreutils, so most places have
# it installed.
stdbuf_prefix = "stdbuf -e0 -o0"
target.shell.run("rm -rf logs && mkdir logs")
target.shell.run("set -o pipefail")
# FIXME: running serial -> add way to run parallel
for testsuite in self.subcases:
timeout = timeouts.get(
testsuite, int(os.environ.get("TIMEOUT", timeout_default)))
# suite_tc -> overall result of the whole testsuite execution
with self.subcase(testsuite):
try:
target.report_info("running testsuite %s" % testsuite)
target.shell.run(
f"mkdir -p 'logs/{os.path.dirname(testsuite)}'")
# run the testsuite and capture the output to a log file
# HACK: ignore errors, so we can cancel on timeout
# without it becoming a mess -- still
target.shell.run(
f"podman kill --signal KILL {self.runid_hashid} || true"
)
target.shell.run(
f"podman rm -f {self.runid_hashid} || true")
target.shell.run(
f"{stdbuf_prefix} podman run --name {self.runid_hashid} {testsuite} 2>&1"
f" | tee -a logs/{testsuite}.log",
timeout=timeout)
except tcfl.tc.error_e as e:
# FIXME: this is a hack, we need a proper timeout exception
if "NOT FOUND after" not in str(e):
raise
# if we timeout this one, just cancel it and go for the next
target.report_error(f"stopping after {timeout}s timeout")
# suspend -- Ctrl-Z & kill like really harshly
target.console_tx("\x1a\x1a\x1a")
target.expect("SIGTSTP")
target.shell.run(
f"podman kill --signal KILL {self.runid_hashid} || true"
) # might be dez already
# bring the log files home; SSH is way faster than over the console
target.shell.run('tar cjf logs.tar.bz2 logs/')
target.tunnel.ip_addr = target.addr_get(ic, "ipv4")
target.ssh.copy_from("logs.tar.bz2", self.tmpdir)
subprocess.check_output(["tar", "xf", "logs.tar.bz2"],
stderr=subprocess.STDOUT,
cwd=self.tmpdir)
# cat each log file to tell what happened? we know the log
# file names, so we can just iterate in python
for testsuite in self.subcases:
dirname = os.path.dirname(testsuite)
name = os.path.basename(testsuite)
result = tcfl.result_c()
with self.subcase(testsuite), \
open(os.path.join(self.tmpdir, "logs", dirname, name + ".log")) as lf:
target.report_info("parsing logs")
try:
# FIXME: support multiple formats, for now hard coded to TAP
d = tcfl.tl.tap_parse_output(lf)
except Exception as e:
d = dict()
result += tcfl.result_c.report_from_exception(self, e)
for name, data in d.items():
_tc_name = commonl.name_make_safe(name)
with self.subcase(_tc_name):
self.subtc[tcfl.msgid_c.subcase()].update(
tcfl.tl.tap_mapping_result_c[data['result']],
data.get('subject', str(data['plan_count'])),
data['output'])
def on(self, target, component):
# Start QEMU
#
# We first assign a port for GDB debugging, if someone takes
# it before we do, start will fail and paranoid mode (see
# above) will restart it.
# FIXME: allocate ports for VNC only if there is a vnc= in the
# command line
base = ttbl.config.tcp_port_range[0]
top = ttbl.config.tcp_port_range[1]
if base < 5900: # we need ports >= 5900 for VNC
base = 5900
tcp_port_base = commonl.tcp_port_assigner(2, port_range=(base, top))
target.fsdb.set("qemu-gdb-tcp-port", "%s" % tcp_port_base)
# This might not be used at all, but we allocate and declare
# it in case the implementation will use it; allocating in a
# higher layer makes it more complicated.
# this one is the port number based on 5900 (VNC default 0)
if top < 5900:
logging.warning(
"ttbl.config.tcp_port_range %s doesn't include ports "
"between above 5900, needed for VNC services. "
"QEMU targets needing VNC support will fail to start "
"complaining about 'vnc-port' not defined",
ttbl.config.tcp_port_range)
else:
# FIXME: move to vnc.vnc0.{host,port,tcp-port}
# set this for general information; the VNC screenshotter
# also uses it
target.fsdb.set("vnc-host", "localhost")
target.fsdb.set("vnc-port", "%s" % (tcp_port_base + 1 - 5900))
# this one is the raw port number
target.fsdb.set("vnc-tcp-port", "%s" % (tcp_port_base + 1))
# New form
target.fsdb.set("vnc.vnc0.host", "localhost")
target.fsdb.set("vnc.vnc0.port", "%s" % (tcp_port_base + 1 - 5900))
# this one is the raw port number
target.fsdb.set("vnc.vnc0.tcp_port", "%s" % (tcp_port_base + 1))
self.cmdline_extra = []
image_keys = target.fsdb.keys("qemu-image-*")
#
# Images interface: flash() below has been used to set images
# to run, this will feed them into QEMU
#
if 'qemu-image-bios' in image_keys:
self.cmdline_extra += ["-bios", "%(qemu-image-bios)s"]
if 'qemu-image-kernel' in image_keys:
self.cmdline_extra += ["-kernel", "%(qemu-image-kernel)s"]
if 'qemu-image-kernel-args' in image_keys:
self.cmdline_extra += ["-append", "%(qemu-image-kernel-args)s"]
if 'qemu-image-initrd' in image_keys:
self.cmdline_extra += ["-initrd", "%(qemu-image-initrd)s"]
#
# Network support bits--add command line options to connect
# this VM to the networks the target declares
#
# For each network we are connected to, there must be a
# network_tap_pc power rail controller that sets up the
# network devices for us before starting QEMU.
#
# We look at it to setup the QEMU side of things as extra
# command line options.
#
# - network name/icname: comes from the tuntap-ICNAME property
# created by the network_tap_pc power rail controller that
# the configuration must put on before the qemu power rail
# controller.
#
# - model: comes from what HW it has to emulate; each arch has a
# default or you can set for all with property qemu-model or
# qemu-model-ICNAME (for that specifc interconnect).
#
# It has to be a valid QEMU model or QEMU will reject it
# with an error. You can find valid models with::
#
# qemu-system-ARCH -net nic,model=help
#
# Zephyr, e.g., supports a few: lan9118 stellaris e1000 (from)::
#
# $ grep --color -nH --null -rie ETH_NIC_MODEL
# drivers/ethernet/Kconfig.smsc911x:13:config ETH_NIC_MODEL$
# drivers/ethernet/Kconfig.stellaris:15:config ETH_NIC_MODEL$
# drivers/ethernet/Kconfig.e1000:16:config ETH_NIC_MODEL$
for tap, if_name in target.fsdb.get_as_dict("tuntap-*").items():
# @tap is tuntap-NETWORK, the property set by
# powe rail component network_tap_pc.
_, ic_name = tap.split("-", 1)
mac_addr = target.tags['interconnects'][ic_name]['mac_addr']
model = commonl.name_make_safe(
target.fsdb.get("qemu-model-" + ic_name,
target.fsdb.get("qemu-model", self.nic_model)))
self.cmdline_extra += [
"-device",
#
# romfile= UNSET for using virtio-net
# - https://wiki.syslinux.org/wiki/index.php?title=Development/Testing
# Workaround using virtio
f"{model},netdev={ic_name},mac={mac_addr},romfile=",
"-netdev",
f"tap,id={ic_name},script=no,ifname={if_name}"
]
#
# Ok, so do actually start
#
# if the debug port is snatched before we start by someone
# else, this will fail and we'll have to retry--this is
# why the power component is set to paranoid mode, so the
# power interface automatically retries it if it fails.
ttbl.power.daemon_c.on(self, target, component)
# run console/on steps
self._qemu_console_on(target, component)
#
# Debugging interface--if the target is not in debugging mode,
# tell QEMU to start right away
#
if target.fsdb.get("debug") == None:
self.debug_resume(target, component)
def on(self, target, component):
# Start QEMU
#
# We first assign a port for GDB debugging, if someone takes
# it before we do, start will fail and paranoid mode (see
# above) will restart it.
base = ttbl.config.tcp_port_range[0]
top = ttbl.config.tcp_port_range[1]
if base < 5900: # we need ports >= 5900 for VNC
base = 5900
tcp_port_base = commonl.tcp_port_assigner(2, port_range=(base, top))
target.fsdb.set("qemu-gdb-tcp-port", "%s" % tcp_port_base)
# This might not be used at all, but we allocate and declare
# it in case the implementation will use it; allocating in a
# higher layer makes it more complicated.
# this one is the port number based on 5900 (VNC default 0)
if top < 5900:
logging.warning(
"ttbl.config.tcp_port_range %s doesn't include ports "
"between above 5900, needed for VNC services. "
"QEMU targets needing VNC support will fail to start "
"complaining about 'vnc-port' not defined",
ttbl.config.tcp_port_range)
else:
# set this for general information; the VNC screenshotter
# also uses it
target.fsdb.set("vnc-host", "localhost")
target.fsdb.set("vnc-port", "%s" % (tcp_port_base + 1 - 5900))
# this one is the raw port number
target.fsdb.set("vnc-tcp-port", "%s" % (tcp_port_base + 1))
self.cmdline_extra = []
image_keys = target.fsdb.keys("qemu-image-*")
#
# Images interface: flash() below has been used to set images
# to run, this will feed them into QEMU
#
if 'qemu-image-bios' in image_keys:
self.cmdline_extra += ["-bios", "%(qemu-image-bios)s"]
if 'qemu-image-kernel' in image_keys:
self.cmdline_extra += ["-kernel", "%(qemu-image-kernel)s"]
if 'qemu-image-kernel-args' in image_keys:
self.cmdline_extra += ["-append", "%(qemu-image-kernel-args)s"]
if 'qemu-image-initrd' in image_keys:
self.cmdline_extra += ["-initrd", "%(qemu-image-initrd)s"]
#
# Network support bits--add command line options to connect
# this VM to the networks the target declares
#
# For each network we are connected to, there must be a
# network_tap_pc power rail controller that sets up the
# network devices for us before starting QEMU.
#
# We look at it to setup the QEMU side of things as extra
# command line options.
#
# - network name/icname: comes from the tuntap-ICNAME property
# created by the network_tap_pc power rail controller that
# the configuration must put on before the qemu power rail
# controller.
#
# - model: comes from what HW it has to emulate; each arch has a
# default or you can set for all with property qemu-model or
# qemu-model-ICNAME (for that specifc interconnect).
#
# It has to be a valid QEMU model or QEMU will reject it
# with an error. You can find valid models with::
#
# qemu-system-ARCH -net nic,model=help
#
# Zephyr, e.g., supports a few: lan9118 stellaris e1000 (from)::
#
# $ grep --color -nH --null -rie ETH_NIC_MODEL
# drivers/ethernet/Kconfig.smsc911x:13:config ETH_NIC_MODEL$
# drivers/ethernet/Kconfig.stellaris:15:config ETH_NIC_MODEL$
# drivers/ethernet/Kconfig.e1000:16:config ETH_NIC_MODEL$
fds = []
try:
for tap, if_name in target.fsdb.get_as_dict(
"tuntap-*").iteritems():
# @tap is tuntap-NETWORK, the property set by
# powe rail component network_tap_pc.
_, ic_name = tap.split("-", 1)
mac_addr = target.tags['interconnects'][ic_name]['mac_addr']
model = commonl.name_make_safe(
target.fsdb.get(
"qemu-model-" + ic_name,
target.fsdb.get("qemu-model", self.nic_model)))
# There is no way to cahole QEMU to say: hey, we
# already have a tap interface configured for you, use
# it...other than to open it and pass the file
# descriptor to it.
# So this finds the /dev/tap* file which matches the
# interface created with tuntap- (class
# network_tap_pc), opens it and passes the descriptor
# along to qemu, then closes the FD, since we don't
# need it.
tapindex = commonl.if_index(if_name)
# Need to wait for udev to reconfigure this for us to have
# access; this is done by a udev rule installed
# (/usr/lib/udev/rules.d/80-ttbd.rules)
tapdevname = "/dev/tap%d" % tapindex
ts0 = time.time()
ts = ts0
timeout = 5
fd = None
while ts - ts0 < timeout:
try:
fd = os.open(tapdevname, os.O_RDWR, 0)
fds.append(fd)
break
except OSError as e:
target.log.error(
"%s: couldn't open %s after"
" +%.1f/%.1fs, retrying: %s", ic_name, tapdevname,
ts - ts0, timeout, e)
time.sleep(0.25)
ts = time.time()
else:
raise RuntimeError(
"%s: timedout (%.1fs) opening /dev/tap%d's;"
" udev's /usr/lib/udev/rules.d/80-ttbd.rules"
" didn't set permissions?" %
(ic_name, timeout, tapindex))
self.cmdline_extra += [
"-nic",
"tap,fd=%d,id=%s,model=%s,mac=%s,br=_b%s" \
% (fd, ic_name, model, mac_addr, ic_name )
]
if not fds:
# No tuntap network attachments -> no network
# support. Cap it.
self.cmdline_extra += ['-nic', 'none']
#
# Ok, so do actually start
#
# if the debug port is snatched before we start by someone
# else, this will fail and we'll have to retry--this is
# why the power component is set to paranoid mode, so the
# power interface automatically retries it if it fails.
ttbl.power.daemon_c.on(self, target, component)
# Those consoles? update their generational counts so the
# clients can know they restarted
if hasattr(target, "console"):
for console in target.console.impls.keys():
ttbl.console.generation_set(target, console)
finally:
for fd in fds: # close fds we opened for ...
os.close(fd) # ... networking, unneeded now
#
# Debugging interface--if the target is not in debugging mode,
# tell QEMU to start right away
#
if target.fsdb.get("debug") == None:
self.debug_resume(target, component)
def _eval_one(self, target, t_file, prefix):
result = tcfl.tc.result_c(0, 0, 0, 0, 0)
rel_file_path = os.path.join(prefix, t_file)
if self._ts_ignore(rel_file_path):
target.report_skip(
"%s: skipped due to configuration "
"(tcfl.tc_clear_bbt.ignore_ts or BBT_IGNORE_TS environment)"
% rel_file_path)
for name, subtc in self.subtcs.iteritems():
if name.startswith(t_file):
subtc.result = tcfl.tc.result_c(0, 0, 0, 0, 1)
subtc.data = dict(result = "skipped")
result.skipped += 1
else:
self.report_info("running %s%s" % (prefix, t_file))
self.kw_set("t_file", t_file)
output = target.shell.run(
# remember we cd'ed into the directory, the way these
# BBTs are written, it is expected
"bats --tap %s || echo FAILED''-%s"
% (t_file, self.kws['tc_hash']),
output = True)
# top level result
if 'bats: command not found' in output:
self.report_error(
"'bats' tool not installed in the target",
dict(target = target, output = output))
result.errors += 1
elif 'FAILED-%(tc_hash)s' % self.kws in output:
result.failed += 1
else:
result.passed += 1
# seems we had execution, so let's parse the output and
# make subcases of the .t -- if anything fails, catch and
# convert to a TCF exception so it only affects *this*
# testcase in the result accounting--note
# report_from_exception() will report exceptio data so we
# can debug if it is an infra or TC problem.
try:
tcs = tap_parse_output(output)
except Exception as e:
tcs = dict()
result += tcfl.tc.result_c.report_from_exception(self, e)
for name, data in tcs.iteritems():
# get the subtc; see _scan_t_subcases() were we keyed
# them in
_name = commonl.name_make_safe(name.strip()).rstrip("_")
tc_name = t_file + "." + _name
subtc = self.subtcs[tc_name]
if self._ts_ignore(subtc.name):
data['result'] += \
"result skipped due to configuration " \
"(tcfl.tc_clear_bbt.ignore_ts or " \
"BBT_IGNORE_TS environment)"
subtc.update(tcfl.tc.result_c(0, 0, 0, 0, 1), data)
else:
# translate the taps result to a TCF result, record it
subtc.update(self.mapping[data['result']], data)
result += subtc.result
return result
