def _zephyr_testcase_patcher_for_components(testcase):
for tag in testcase._tags:
if tag.startswith('component/'):
# there are already assigned components
testcase.log.debug("component info found, no need to guess")
return
# Is this a Zephyr related testcase? Does it declare any target
# that will run Zephyr?
# yeah, wouldn't cover all the basis, but then properly tag the
# components in your testcase, this is just a guesser for backward
# compat
for _target_name, target_data in testcase._targets.iteritems():
kws = target_data.get('kws', {})
if 'app_zephyr' in kws:
# yay, at least one Zephyr target, we takes
break
else:
# of all the targets in this testcase (if any), none declares
# an App Zephyr, so we can't be sure this is a Zephyr-related
# testcase
return
# There are not assigned components, so let's guess based on paths
ZEPHYR_BASE = os.environ.get('ZEPHYR_BASE', None)
ZEPHYR_TESTS_BASE = os.environ.get('ZEPHYR_TESTS_BASE', None)
roots = []
if ZEPHYR_BASE:
roots += [
os.path.join(ZEPHYR_BASE, "tests"),
os.path.join(ZEPHYR_BASE, "samples"),
]
if ZEPHYR_TESTS_BASE:
roots += [
os.path.join(ZEPHYR_TESTS_BASE, "tests"),
os.path.join(ZEPHYR_TESTS_BASE, "testlib"),
]
origin = commonl.origin_get()
filename = testcase.kws.get('thisfile', None)
if filename == None:
testcase.log.debug("component info n/a, can't find source file")
return
filename = os.path.normpath(os.path.realpath(filename))
componentl = _zephyr_components_from_path(os.path.abspath(filename), roots)
for component in componentl:
component_tag = "component/" + component
testcase.tag_set(component_tag, component, origin)
components = " ".join(componentl)
testcase.log.info("component info guessed: %s", components)
def _zephyr_testcase_patcher_for_components(testcase):
for tag in testcase._tags:
if tag.startswith('component/'):
# there are already assigned components
testcase.log.debug("component info found, no need to guess")
return
# Is this a Zephyr related testcase? Does it declare any target
# that will run Zephyr?
# yeah, wouldn't cover all the basis, but then properly tag the
# components in your testcase, this is just a guesser for backward
# compat
for _target_name, target_data in testcase._targets.items():
kws = target_data.get('kws', {})
if 'app_zephyr' in kws:
# yay, at least one Zephyr target, we takes
break
else:
# of all the targets in this testcase (if any), none declares
# an App Zephyr, so we can't be sure this is a Zephyr-related
# testcase
return
# There are not assigned components, so let's guess based on paths
ZEPHYR_BASE = os.environ.get('ZEPHYR_BASE', None)
ZEPHYR_TESTS_BASE = os.environ.get('ZEPHYR_TESTS_BASE', None)
roots = []
if ZEPHYR_BASE:
roots += [
os.path.join(ZEPHYR_BASE, "tests"),
os.path.join(ZEPHYR_BASE, "samples"),
]
if ZEPHYR_TESTS_BASE:
roots += [
os.path.join(ZEPHYR_TESTS_BASE, "tests"),
os.path.join(ZEPHYR_TESTS_BASE, "testlib"),
]
origin = commonl.origin_get()
filename = testcase.kws.get('thisfile', None)
if filename == None:
testcase.log.debug("component info n/a, can't find source file")
return
filename = os.path.normpath(os.path.realpath(filename))
componentl = _zephyr_components_from_path(os.path.abspath(filename), roots)
for component in componentl:
component_tag = "component/" + component
testcase.tag_set(component_tag, component, origin)
components = " ".join(componentl)
testcase.log.info("component info guessed: %s", components)
def check_filter(self, _objdir, arch, board, _filter, origin=None):
"""
This is going to be called by the App Builder's build function
to evaluate if we need to filter out a build of a testcase. In
any other case, it will be ignored.
:param str objdir: name of the Zephyr's object directory
where to find configuration files
:param str arch: name of the architecture we are building
:param str board: Zephyr's board we are building
:param str _filter: Zephyr's sanity Check style filter
expression
:param str origin: where does this come from?
"""
if not origin:
origin = commonl.origin_get()
if _filter == None or _filter == "":
return
self.target.report_info("filter: processing '%s' @%s" %
(_filter, origin),
dlevel=1)
self.target.report_info("filter: reading defconfig", dlevel=2)
_defconfig = self.config_file_read()
defconfig = {}
for key, value in _defconfig.iteritems():
# The testcase.ini filter language doesn't prefix the
# CONFIG_ stuff, so we are going to strip it with [7:]
if key.startswith("CONFIG_"):
defconfig[key[7:]] = value
# The testcase.yaml filter language prefixes with
# CONFIG_ stuff, so we don't strip it
defconfig[key] = value
self.target.report_info("filter: evaluating", dlevel=2)
try:
res = commonl.expr_parser.parse(_filter, defconfig)
self.target.report_info("filter: evaluated defconfig: %s" % res,
dlevel=1)
if res == False:
raise tc.skip_e("filter '%s' @ %s causes TC to be skipped" %
(_filter, origin))
else:
self.target.report_info(
"filter '%s' @ %s causes TC to be continued" %
(_filter, origin),
dlevel=2)
except SyntaxError as se:
raise tc.error_e("filter: failed processing '%s' @ %s: %s" %
(_filter, origin, se))
def check_filter(self, _objdir, arch, board, _filter, origin = None):
"""
This is going to be called by the App Builder's build function
to evaluate if we need to filter out a build of a testcase. In
any other case, it will be ignored.
:param str objdir: name of the Zephyr's object directory
where to find configuration files
:param str arch: name of the architecture we are building
:param str board: Zephyr's board we are building
:param str _filter: Zephyr's sanity Check style filter
expression
:param str origin: where does this come from?
"""
if not origin:
origin = commonl.origin_get()
if _filter == None or _filter == "":
return
self.target.report_info("filter: processing '%s' @%s"
% (_filter, origin), dlevel = 1)
self.target.report_info("filter: reading defconfig", dlevel = 2)
_defconfig = self.config_file_read()
defconfig = {}
for key, value in _defconfig.iteritems():
# The testcase.ini filter language doesn't prefix the
# CONFIG_ stuff, so we are going to strip it with [7:]
if key.startswith("CONFIG_"):
defconfig[key[7:]] = value
# The testcase.yaml filter language prefixes with
# CONFIG_ stuff, so we don't strip it
defconfig[key] = value
self.target.report_info("filter: evaluating", dlevel = 2)
try:
res = commonl.expr_parser.parse(_filter, defconfig)
self.target.report_info("filter: evaluated defconfig: %s" % res,
dlevel = 1)
if res == False:
raise tc.skip_e("filter '%s' @ %s causes TC to be skipped"
% (_filter, origin))
else:
self.target.report_info(
"filter '%s' @ %s causes TC to be continued"
% (_filter, origin), dlevel = 2)
except SyntaxError as se:
raise tc.error_e("filter: failed processing '%s' @ %s: %s"
% (_filter, origin, se))
def driver_add(cls, name=None):
"""
Add a new driver for app building
Note the driver will be called as the class name; it is
recommended to call then *app_something*.
"""
assert issubclass(cls, app_c)
if name == None:
name = cls.__name__
else:
assert isinstance(name, basestring)
if name in _drivers:
raise ValueError('%s: already registered by @%s' %
(name, _drivers[name][1]))
_drivers[name] = (cls, commonl.origin_get(2))
def driver_add(cls, name = None):
"""
Add a new driver for app building
Note the driver will be called as the class name; it is
recommended to call then *app_something*.
"""
assert issubclass(cls, app_c)
if name == None:
name = cls.__name__
else:
assert isinstance(name, basestring)
if name in _drivers:
raise ValueError('%s: already registered by @%s'
% (name, _drivers[name][1]))
_drivers[name] = (cls, commonl.origin_get(2))
def url_add(url, ssl_ignore=False, aka=None, ca_path=None, origin=None):
"""
Add a TTBD server
:param str url: server's URL (``http://SERVER:PORT``); it can be
*https*; port is most commonly *5000*.
:param bool ssl_ignore: if True, skips verifying SSL certificates
:param str aka: Short form for this server, to use in display messages
"""
# FIXME: move this to the object construction
u = urllib.parse.urlparse(url)
if u.scheme == "" or u.netloc == "":
raise Exception("%s: malformed URL?" % url)
if not origin:
origin = "configured " + commonl.origin_get(2)
logger.info("%s: Added server URL %s", origin, url)
urls.append((url, ssl_ignore, aka, ca_path)) # COMPAT
urls_data[url]['origin'] = origin
def config_file_write(self, name, data, bsp = None):
"""\
Write an extra config file called *NAME*.conf in the Zephyr's
App build directory.
Note this takes care to only write it if the data is new or
the file is unexistant, to avoid unnecesary rebuilds.
:param str name: Name for the configuration file; this has to
be a valid filename; *.conf* will be added by the function.
:param str data: Data to include in the configuration file;
this is (currently) valid kconfig data, which are lines of
text with # acting as comment character; for example::
CONFIG_UART_CONSOLE_ON_DEV_NAME="UART_1"
:param str bsp: (optional) BSP on which to operate; when the
target is configured for a :term:`BSP model` which contains
multiple Zephyr BSPs, you will need to specify which one to
modify.
This parameter can be omitted if only one BSP is available
in the current BSP Model.
*Example*
>>> if something:
>>> target.zephyr.conf_file_write("mytweaks",
>>> 'CONFIG_SOMEVAR=1\\n'
>>> 'CONFIG_ANOTHER="VALUE"\\n')
"""
target = self.target
bsp = self._bsp_select(target, bsp)
# Ensure the config directory is there
outdir = target._kws_bsp[bsp]['zephyr_objdir']
try:
os.makedirs(outdir)
except OSError as e:
if e.errno != errno.EEXIST:
raise RuntimeError("%s: Cannot create outdir directory: %s"
% (outdir, str(e)))
# Now create a .new file
if not name.endswith(".conf"):
name += ".conf"
existing_filename = os.path.join(outdir, name)
new_filename = existing_filename + ".new"
with codecs.open(new_filename, "w+", encoding = 'utf-8',
errors = 'ignore') as f:
f.write("""\
# Config file automatically generated by TCF's:
#
# %s
#
# because of instructions from
#
# %s
#
# Do not edit by hand
""" % (commonl.origin_get(), commonl.origin_get(2)))
f.write(data)
# report the config file we wrote, so reproduction
# instructions will carry it in the report file; we report
# the data without the header to make it easier -- note we
# use report_pass() [vs report_info()] as _info might get
# filtered as too verbose info, where as pass is
# information important for passing.
target.report_pass("enabled Zephyr config file %s at %s"
% (name, outdir),
{ 'config file': data })
if not os.path.exists(existing_filename):
shutil.move(new_filename, existing_filename)
else:
# Check if there are changes before updating it, to avoid
# unnecesary rebuilds
_new_hash = hashlib.sha256()
new_hash = commonl.hash_file(_new_hash, new_filename)
_old_hash = hashlib.sha256()
old_hash = commonl.hash_file(_old_hash, existing_filename)
if new_hash.digest() != old_hash.digest():
shutil.move(new_filename, existing_filename)
else:
os.unlink(new_filename)
def _run(self,
cmd=None,
expect=None,
prompt_regex=None,
output=False,
output_filter_crlf=True,
trim=False,
console=None,
origin=None):
if cmd:
assert isinstance(cmd, basestring)
assert expect == None \
or isinstance(expect, basestring) \
or isinstance(expect, re._pattern_type) \
or isinstance(expect, list)
assert prompt_regex == None \
or isinstance(prompt_regex, basestring) \
or isinstance(prompt_regex, re._pattern_type)
if origin == None:
origin = commonl.origin_get(3)
else:
assert isinstance(origin, basestring)
target = self.target
if output:
offset = self.target.console.size(console=console)
if cmd:
self.target.send(cmd, console=console)
if expect:
if isinstance(expect, list):
for expectation in expect:
assert isinstance(expectation, basestring) \
or isinstance(expectation, re._pattern_type)
target.expect(expectation,
name="command output",
console=console,
origin=origin)
else:
target.expect(expect,
name="command output",
console=console,
origin=origin)
if prompt_regex == None:
self.target.expect(self.shell_prompt_regex,
name="shell prompt",
console=console,
origin=origin)
else:
self.target.expect(prompt_regex,
name="shell prompt",
console=console,
origin=origin)
if output:
output = self.target.console.read(offset=offset, console=console)
if output_filter_crlf:
# replace \r\n, \r\r\n, \r\r\r\r\n... it happens
output = re.sub(self.crnl_regex, self.target.crlf, output)
if trim:
# When we can run(), it usually prints in the console:
## <command-echo from our typing>
## <command output>
## <prompt>
#
# So to trim we just remove the first and last
# lines--won't work well without output_filter_crlf
# and it is quite a hack.
first_nl = output.find(self.target.crlf)
last_nl = output.rfind(self.target.crlf)
output = output[first_nl + 1:last_nl + 1]
return output
return None
def __init__(self,
config_text=None,
config_files=None,
use_ssl=False,
tmpdir=None,
keep_temp=True,
errors_ignore=None,
warnings_ignore=None,
aka=None,
local_auth=True):
# Force all assertions, when running like this, to fail the TC
tcfl.tc.tc_c.exception_to_result[AssertionError] = tcfl.tc.failed_e
# If no aka is defined, we make one out of the place when this
# object is being created, so it is always the same *and* thus
# the report hashes are always identical with each run
if aka == None:
self.aka = "ttbd-" + commonl.mkid(commonl.origin_get(2), 4)
else:
self.aka = aka
if config_files == None:
config_files = []
self.keep_temp = keep_temp
self.port = commonl.tcp_port_assigner()
self.use_ssl = use_ssl
if use_ssl == True:
self.url = "https://localhost:%d" % self.port
ssl_context = ""
else:
self.url = "http://localhost:%d" % self.port
ssl_context = "--no-ssl"
self.url_spec = "fullid:'^%s'" % self.aka
if tmpdir:
self.tmpdir = tmpdir
else:
# default to place the server's dir in the tempdir for
# testcases
self.tmpdir = os.path.join(tcfl.tc.tc_c.tmpdir, "server", self.aka)
shutil.rmtree(self.tmpdir, ignore_errors=True)
commonl.makedirs_p(self.tmpdir)
self.etc_dir = os.path.join(self.tmpdir, "etc")
self.files_dir = os.path.join(self.tmpdir, "files")
self.lib_dir = os.path.join(self.tmpdir, "lib")
self.state_dir = os.path.join(self.tmpdir, "state")
os.mkdir(self.etc_dir)
os.mkdir(self.files_dir)
os.mkdir(self.lib_dir)
os.mkdir(self.state_dir)
self.stdout = self.tmpdir + "/stdout"
self.stderr = self.tmpdir + "/stderr"
for fn in config_files:
shutil.copy(fn, self.etc_dir)
with open(os.path.join(self.etc_dir, "conf_00_base.py"), "w") as cfgf:
cfgf.write(r"""
import ttbl.config
ttbl.config.processes = 2
host = '127.0.0.1'
""")
# We don't define here the port, so we see it in the
# command line
if config_text:
cfgf.write(config_text)
self.srcdir = os.path.realpath(
os.path.join(os.path.dirname(__file__), ".."))
self.cmdline = [
"stdbuf",
"-o0",
"-e0",
# This allows us to default to the source location,when
# running from source, or the installed when running from
# the system
os.environ.get("TTBD_PATH", self.srcdir + "/ttbd/ttbd"),
"--port",
"%d" % self.port,
ssl_context,
"-vvvvv",
"--files-path",
self.files_dir,
"--state-path",
self.state_dir,
"--config-path",
"", # This empty one is to clear them all
"--config-path",
self.etc_dir
]
self.local_auth = local_auth
if local_auth:
self.cmdline.append("--local-auth")
self.p = None
#: Exclude these regexes / strings from triggering an error
#: message check
self.errors_ignore = [] if errors_ignore == None else errors_ignore
#: Exclude these regexes / strings from triggering an warning
#: message check
self.warnings_ignore = [re.compile('daemon lacks CAP_NET_ADMIN')]
if warnings_ignore:
self.warnings_ignore += warnings_ignore
def _preexec_fn():
stdout_fd = os.open(
self.stdout,
# O_CREAT: Always a new file, so
# we can check for errors and not
# get confused with previous runs
os.O_WRONLY | os.O_EXCL | os.O_CREAT,
0o0644)
stderr_fd = os.open(
self.stderr,
# O_CREAT: Always a new file, so
# we can check for errors and not
# get confused with previous runs
os.O_WRONLY | os.O_EXCL | os.O_CREAT,
0o0644)
os.dup2(stdout_fd, 1)
os.dup2(stderr_fd, 2)
logging.info("Launching: %s", " ".join(self.cmdline))
self.p = subprocess.Popen(self.cmdline,
shell=False,
cwd=self.tmpdir,
close_fds=True,
preexec_fn=_preexec_fn,
bufsize=0)
try:
self._check_if_alive()
finally:
self.check_log_for_issues()
# if we call self.terminate() from __del__, the garbage
# collector has started to wipe things, so we can't use, ie:
# open() to check the log file
atexit.register(self.terminate)
def __init__(self, config_text = None, config_files = None,
use_ssl = False, tmpdir = None, keep_temp = True,
errors_ignore = None, warnings_ignore = None,
aka = None):
# Force all assertions, when running like this, to fail the TC
tcfl.tc.tc_c.exception_to_result[AssertionError] = tcfl.tc.failed_e
# If no aka is defined, we make one out of the place when this
# object is being created, so it is always the same *and* thus
# the report hashes are always identical with each run
if aka == None:
self.aka = "ttbd-" + commonl.mkid(commonl.origin_get(2), 4)
else:
self.aka = aka
if config_files == None:
config_files = []
self.keep_temp = keep_temp
self.port = commonl.tcp_port_assigner()
self.use_ssl = use_ssl
if use_ssl == True:
self.url = "https://localhost:%d" % self.port
ssl_context = ""
else:
self.url = "http://localhost:%d" % self.port
ssl_context = "--no-ssl"
self.url_spec = "url:'^%s'" % self.url
if tmpdir:
self.tmpdir = tmpdir
else:
# Don't use colon on the name, or it'll thing it is a path
self.tmpdir = tempfile.mkdtemp(prefix = "test-ttbd-%d."
% self.port)
self.etc_dir = os.path.join(self.tmpdir, "etc")
self.files_dir = os.path.join(self.tmpdir, "files")
self.lib_dir = os.path.join(self.tmpdir, "lib")
self.state_dir = os.path.join(self.tmpdir, "state")
os.mkdir(self.etc_dir)
os.mkdir(self.files_dir)
os.mkdir(self.lib_dir)
os.mkdir(self.state_dir)
self.stdout = self.tmpdir + "/stdout"
self.stderr = self.tmpdir + "/stderr"
for fn in config_files:
shutil.copy(fn, self.etc_dir)
with open(os.path.join(self.etc_dir,
"conf_00_base.py"), "w") as cfgf:
cfgf.write(r"""
import ttbl.config
ttbl.config.processes = 2
host = '127.0.0.1'
""")
# We don't define here the port, so we see it in the
# command line
if config_text:
cfgf.write(config_text)
srcdir = os.path.realpath(
os.path.join(os.path.dirname(__file__), ".."))
self.cmdline = [
# This allows us to default to the source location,when
# running from source, or the installed when running from
# the system
os.environ.get("TTBD_PATH", srcdir + "/ttbd/ttbd"),
"--port", "%d" % self.port,
ssl_context,
"--local-auth", "-vvvvv",
"--files-path", self.files_dir,
"--state-path", self.state_dir,
"--var-lib-path", self.lib_dir,
"--config-path", "", # This empty one is to clear them all
"--config-path", self.etc_dir
]
self.p = None
#: Exclude these regexes / strings from triggering an error
#: message check
self.errors_ignore = [] if errors_ignore == None else errors_ignore
#: Exclude these regexes / strings from triggering an warning
#: message check
self.warnings_ignore = [ re.compile('daemon lacks CAP_NET_ADMIN') ]
if warnings_ignore:
self.warnings_ignore =+ warnings_ignore
def _preexec_fn():
stdout_fd = os.open(self.stdout,
# O_CREAT: Always a new file, so
# we can check for errors and not
# get confused with previous runs
os.O_WRONLY | os.O_EXCL |os.O_CREAT, 0o0644)
stderr_fd = os.open(self.stderr,
# O_CREAT: Always a new file, so
# we can check for errors and not
# get confused with previous runs
os.O_WRONLY | os.O_EXCL |os.O_CREAT, 0o0644)
os.dup2(stdout_fd, 1)
os.dup2(stderr_fd, 2)
logging.info("Launching: %s", " ".join(self.cmdline))
self.p = subprocess.Popen(
self.cmdline, shell = False, cwd = self.tmpdir,
close_fds = True, preexec_fn = _preexec_fn)
self._check_if_alive()
self.check_log_for_issues()
def config_file_write(self, name, data, bsp = None):
"""\
Write an extra config file called *NAME*.conf in the Zephyr's
App build directory.
Note this takes care to only write it if the data is new or
the file is unexistant, to avoid unnecesary rebuilds.
:param str name: Name for the configuration file; this has to
be a valid filename; *.conf* will be added by the function.
:param str data: Data to include in the configuration file;
this is (currently) valid kconfig data, which are lines of
text with # acting as comment character; for example::
CONFIG_UART_CONSOLE_ON_DEV_NAME="UART_1"
:param str bsp: (optional) BSP on which to operate; when the
target is configured for a :term:`BSP model` which contains
multiple Zephyr BSPs, you will need to specify which one to
modify.
This parameter can be omitted if only one BSP is available
in the current BSP Model.
*Example*
>>> if something:
>>> target.zephyr.conf_file_write("mytweaks",
>>> 'CONFIG_SOMEVAR=1\\n'
>>> 'CONFIG_ANOTHER="VALUE"\\n')
"""
target = self.target
bsp = self._bsp_select(target, bsp)
# Ensure the config directory is there
outdir = target._kws_bsp[bsp]['zephyr_objdir']
try:
os.makedirs(outdir)
except OSError as e:
if e.errno != errno.EEXIST:
raise RuntimeError("%s: Cannot create outdir directory: %s"
% (outdir, e.message))
# Now create a .new file
if not name.endswith(".conf"):
name += ".conf"
existing_filename = os.path.join(outdir, name)
new_filename = existing_filename + ".new"
with codecs.open(new_filename, "w+", encoding = 'utf-8',
errors = 'ignore') as f:
f.write("""\
# Config file automatically generated by TCF's:
#
# %s
#
# because of instructions from
#
# %s
#
# Do not edit by hand
""" % (commonl.origin_get(), commonl.origin_get(2)))
f.write(data)
# report the config file we wrote, so reproduction
# instructions will carry it in the report file; we report
# the data without the header to make it easier -- note we
# use report_pass() [vs report_info()] as _info might get
# filtered as too verbose info, where as pass is
# information important for passing.
target.report_pass("enabled Zephyr config file %s at %s"
% (name, outdir),
{ 'config file': data })
if not os.path.exists(existing_filename):
shutil.move(new_filename, existing_filename)
else:
# Check if there are changes before updating it, to avoid
# unnecesary rebuilds
_new_hash = hashlib.sha256()
new_hash = commonl.hash_file(_new_hash, new_filename)
_old_hash = hashlib.sha256()
old_hash = commonl.hash_file(_old_hash, existing_filename)
if new_hash.digest() != old_hash.digest():
shutil.move(new_filename, existing_filename)
else:
os.unlink(new_filename)
def setup(*args,
report_drivers=None,
verbosity=2,
logfile_name="run.log",
name="toplevel",
runid=None,
hashid="standalone",
skip_reports=False,
**kwargs):
"""
Setup and Load the TCF Library configuration for standalone execution
This is needed before you can access from your client program any
other module.
:param int verbosity: (optional, default 2) verbosity of output to
the console
:param str logfile_name: (optional, default *run.log*) where to
log the detailed output to.
:param list(tcfl.tc.report_driver_c) report_drivers: (optional)
list of drivers for reporting execution data.
By default, drivers that logs to a logfile, to report files and
to json files are loaded for you.
Other arguments as :func:`load`.
"""
assert runid == None or isinstance(runid, str)
assert hashid == None or isinstance(hashid, str)
assert isinstance(skip_reports, bool)
tcfl.tc.tc_c.runid = runid
if runid == None:
tcfl.tc.tc_c.runid_visible = ""
else:
tcfl.tc.tc_c.runid_visible = runid
# Do a partial initialzation of the testcase management system so
# the only testcase object declared, tcfl.tc.tc_global reflects
# all the info
tcfl.tc.tc_c.tmpdir = "tmp"
# reinitialize this one up, so that we have minimal hash printing
tcfl.tc.tc_global = tcfl.tc.tc_c(name,
"",
commonl.origin_get(1),
hashid=hashid)
tcfl.tc.tc_global.skip_reports = skip_reports
if not report_drivers:
tcfl.tc.report_driver_c.add(tcfl.tc.report_jinja2.driver("."),
name="jinja2")
tcfl.tc.report_driver_c.add(tcfl.tc.report_console.driver(
verbosity, logfile_name, verbosity_logf=100),
name="console")
tcfl.tc.report_driver_c.add(tcfl.tc.report_data_json.driver(),
name="json")
else:
for report_driver in report_drivers:
tcfl.tc.report_driver_c.add(report_driver)
load(*args, **kwargs)
tcfl.msgid_c.tls.msgid_lifo.append(tcfl.msgid_c(""))
def mount_fs(target, image, boot_dev):
"""
Boots a root filesystem on /mnt
The partition used as a root filesystem is picked up based on the
image that is going to be installed; we look for one that has the
most similar image already installed and pick that.
:returns: name of the root partition device
"""
pos_reinitialize = target.property_get("pos_reinitialize", False)
if pos_reinitialize:
# Need to reinit the partition table (we were told to by
# setting pos_repartition to anything
target.report_info("POS: repartitioning per pos_reinitialize "
"property")
for tag in target.rt.keys():
# remove pos_root_*, as they don't apply anymore
if tag.startswith("pos_root_"):
target.property_set(tag, None)
_disk_partition(target)
target.property_set('pos_reinitialize', None)
root_part_dev = _rootfs_guess(target, image, boot_dev)
# save for other functions called later
target.root_part_dev = root_part_dev
root_part_dev_base = os.path.basename(root_part_dev)
image_prev = target.property_get("pos_root_" + root_part_dev_base,
"nothing")
target.report_info("POS: will use %s for root partition (had %s before)"
% (root_part_dev, image_prev))
# fsinfo looks like described in target.pos._fsinfo_load()
dev_info = None
for blockdevice in target.pos.fsinfo.get('blockdevices', []):
for child in blockdevice.get('children', []):
if child['name'] == root_part_dev_base:
dev_info = child
if dev_info == None:
# it cannot be we might have to repartition because at this
# point *we* have partitoned.
raise tc.error_e(
"Can't find information for root device %s in FSinfo array"
% root_part_dev_base,
dict(fsinfo = target.pos.fsinfo))
# what format does it currently have?
current_fstype = dev_info.get('fstype', 'ext4')
# What format does it have to have?
#
# Ok, here we need to note that we can't have multiple root
# filesystems with the same UUID or LABEL, so the image can't rely
# on UUIDs
#
img_fss = target.pos.metadata.get('filesystems', {})
if '/' in img_fss:
# a common origin is ok because the YAML schema forces both
# fstype and mkfs_opts to be specified
origin = "image's /.tcf.metadata.yaml"
fsdata = img_fss.get('/', {})
else:
origin = "defaults @" + commonl.origin_get(0)
fsdata = {}
fstype = fsdata.get('fstype', 'ext4')
mkfs_opts = fsdata.get('mkfs_opts', '-Fj')
# do they match?
if fstype != current_fstype:
target.report_info(
"POS: reformatting %s because current format is '%s' and "
"'%s' is needed (per %s)"
% (root_part_dev, current_fstype, fstype, origin))
_mkfs(target, root_part_dev, fstype, mkfs_opts)
else:
target.report_info(
"POS: no need to reformat %s because current format is '%s' and "
"'%s' is needed (per %s)"
% (root_part_dev, current_fstype, fstype, origin), dlevel = 1)
for try_count in range(3):
target.report_info("POS: mounting root partition %s onto /mnt "
"to image [%d/3]" % (root_part_dev, try_count))
# don't let it fail or it will raise an exception, so we
# print FAILED in that case to look for stuff; note the
# double apostrophe trick so the regex finder doens't trip
# on the command
output = target.shell.run(
"mount %s /mnt || echo FAI''LED" % root_part_dev,
output = True)
# What did we get?
if 'FAILED' in output:
if 'special device ' + root_part_dev \
+ ' does not exist.' in output:
_disk_partition(target)
else:
# ok, this probably means probably the partitions are not
# formatted; so let's just reformat and retry
_mkfs(target, root_part_dev, fstype, mkfs_opts)
else:
target.report_info("POS: mounted %s onto /mnt to image"
% root_part_dev)
return root_part_dev # it worked, we are done
# fall through, retry
else:
raise tc.blocked_e(
"POS: Tried to mount too many times and failed",
dict(target = target))
def config_watch_add(bus_name, driver_name, device_name, actions):
r"""
:param str bus_name: name of bus in */sys/bus* to watch
:param str driver_name: name of driver in
*/sys/bus/BUS_NAME/drivers* to watch
:param str device_name: device under
/sys/bus/BUS_NAME/drivers/DRIVER_NAME to watch; if *None*, watch
all of them
:param dict actions: dictionary describing actions to do; key is a
substring of a message, value is a function to call or a tuple
that starts with a function to call and the rest are arguments
to add
The action function has to follow this prototype:
>>> def action_function(bus_name, driver_name, device_name,
condition, entry, *args, **kwargs:
thus, when called, bus_name, driver_name and device_name are all
the names of the entity that is causing it; condition is the
condition string that was matched (the key) and *entry* is the
journal entry which matched. *\*args* and *\*\*kwargs* are the
extra arguments given in the *actions* value tuple.
"""
assert isinstance(bus_name, basestring)
assert isinstance(driver_name, basestring)
if device_name:
if isinstance(device_name, basestring):
_device_name = "/" + device_name
elif isinstance(device_name, re._pattern_type):
_device_name = "/" + device_name.pattern
else:
raise AssertionError(
"'device_name' must be string or regex, found %s",
type(device_name).__name__)
else:
_device_name = ""
assert isinstance(actions, dict)
global _watch_rules
_actions = {}
origin = commonl.origin_get(2)
# verify arguments and transform all the actions to a unique
# form (all have to be a list)
for condition, action in actions.iteritems():
assert isinstance(condition, basestring), \
"Key passed as condition is not a string"
try:
action_fn = action[0]
_actions[condition] = action
except TypeError:
action_fn = action
_actions[condition] = [action_fn]
assert callable(action_fn), \
"Argument passed as action function to condition '%s' " \
"is not callable" % condition
driver_path = os.path.join("/sys/bus", bus_name, "drivers", driver_name)
if not os.path.isdir(driver_path):
logging.warning(
"%s/%s%s @%s: driver path does not exist, will not monitor",
bus_name, driver_name, _device_name, origin)
return
_watch_rules.append((bus_name, driver_name, device_name, _actions, origin))
logging.info("%s/%s%s @%s: will monitor", bus_name, driver_name,
_device_name, origin)
def mount_fs(target, image, boot_dev):
"""
Boots a root filesystem on /mnt
The partition used as a root filesystem is picked up based on the
image that is going to be installed; we look for one that has the
most similar image already installed and pick that.
:returns: name of the root partition device
"""
# does the disk have a partition scheme we recognize?
pos_partsizes = target.rt['pos_partsizes']
# the name we'll give to the boot partition; see
# _disk_partition(); if we can't find it, we assume the things
# needs to be repartitioned. Note it includes the sizes, so if we
# change the sizes in the config it reformats automatically. #
# TCF-multiroot-NN:NN:NN:NN
target._boot_label_name = "TCF-multiroot-" + pos_partsizes
pos_reinitialize_force = True
boot_dev_base = os.path.basename(boot_dev)
child = target.pos.fsinfo_get_child_by_partlabel(boot_dev_base,
target._boot_label_name)
if child:
pos_reinitialize_force = False
else:
target.report_info("POS: repartitioning due to different"
" partition schema")
pos_reinitialize = target.property_get("pos_reinitialize", False)
if pos_reinitialize:
target.report_info("POS: repartitioning per pos_reinitialize "
"property")
if pos_reinitialize or pos_reinitialize_force:
# Need to reinit the partition table (we were told to by
# setting pos_repartition to anything or we didn't recognize
# the existing partitioning scheme)
for tag in target.rt.keys():
# remove pos_root_*, as they don't apply anymore
if tag.startswith("pos_root_"):
target.property_set(tag, None)
_disk_partition(target)
target.pos.fsinfo_read(target._boot_label_name)
target.property_set('pos_reinitialize', None)
root_part_dev = _rootfs_guess(target, image, boot_dev)
# save for other functions called later
target.root_part_dev = root_part_dev
root_part_dev_base = os.path.basename(root_part_dev)
image_prev = target.property_get("pos_root_" + root_part_dev_base,
"nothing")
target.report_info("POS: will use %s for root partition (had %s before)" %
(root_part_dev, image_prev))
# fsinfo looks like described in target.pos.fsinfo_read()
dev_info = None
for blockdevice in target.pos.fsinfo.get('blockdevices', []):
for child in blockdevice.get('children', []):
if child['name'] == root_part_dev_base:
dev_info = child
if dev_info == None:
# it cannot be we might have to repartition because at this
# point *we* have partitioned.
raise tc.error_e(
"Can't find information for root device %s in FSinfo array" %
root_part_dev_base, dict(fsinfo=target.pos.fsinfo))
# what format does it currently have?
current_fstype = dev_info.get('fstype', 'ext4')
# What format does it have to have?
#
# Ok, here we need to note that we can't have multiple root
# filesystems with the same UUID or LABEL, so the image can't rely
# on UUIDs
#
img_fss = target.pos.metadata.get('filesystems', {})
if '/' in img_fss:
# a common origin is ok because the YAML schema forces both
# fstype and mkfs_opts to be specified
origin = "image's /.tcf.metadata.yaml"
fsdata = img_fss.get('/', {})
else:
origin = "defaults @" + commonl.origin_get(0)
fsdata = {}
fstype = fsdata.get('fstype', 'ext4')
mkfs_opts = fsdata.get('mkfs_opts', '-Fj')
# do they match?
if fstype != current_fstype:
target.report_info(
"POS: reformatting %s because current format is '%s' and "
"'%s' is needed (per %s)" %
(root_part_dev, current_fstype, fstype, origin))
_mkfs(target, root_part_dev, fstype, mkfs_opts)
else:
target.report_info(
"POS: no need to reformat %s because current format is '%s' and "
"'%s' is needed (per %s)" %
(root_part_dev, current_fstype, fstype, origin),
dlevel=1)
for try_count in range(3):
target.report_info("POS: mounting root partition %s onto /mnt "
"to image [%d/3]" % (root_part_dev, try_count))
# don't let it fail or it will raise an exception, so we
# print FAILED in that case to look for stuff; note the
# double apostrophe trick so the regex finder doens't trip
# on the command
output = target.shell.run("mount %s /mnt || echo FAI''LED" %
root_part_dev,
output=True)
# What did we get?
if 'FAILED' in output:
if 'special device ' + root_part_dev \
+ ' does not exist.' in output:
_disk_partition(target)
target.pos.fsinfo_read(target._boot_label_name)
else:
# ok, this probably means probably the partitions are not
# formatted; so let's just reformat and retry
_mkfs(target, root_part_dev, fstype, mkfs_opts)
else:
target.report_info("POS: mounted %s onto /mnt to image" %
root_part_dev)
return root_part_dev # it worked, we are done
# fall through, retry
else:
raise tc.blocked_e("POS: Tried to mount too many times and failed",
dict(target=target))
def _run(self,
cmd=None,
expect=None,
prompt_regex=None,
output=False,
output_filter_crlf=True,
trim=False,
console=None,
origin=None):
if cmd:
assert isinstance(cmd, str)
assert expect == None \
or isinstance(expect, str) \
or isinstance(expect, typing.Pattern) \
or isinstance(expect, list)
assert prompt_regex == None \
or isinstance(prompt_regex, str) \
or isinstance(prompt_regex, typing.Pattern)
if origin == None:
origin = commonl.origin_get(3)
else:
assert isinstance(origin, str)
target = self.target
testcase = target.testcase
if output:
offset = self.target.console.size(console=console)
# the protocol needs UTF-8 anyway
cmd = commonl.str_cast_maybe(cmd)
if cmd and not self._fixups:
self.target.send(cmd, console=console)
if cmd and self._fixups:
# we have to handle CRLF at the end ourselves here, we
# can't defer to target.send() -- see that for how we get
# crlf.
if console == None:
console = target.console.default
crlf = target.console.crlf.get(console, None)
cmd += crlf
try:
# send the command, doing echo verification if
if origin == None:
origin = commonl.origin_get(2)
testcase.tls.echo_cmd = cmd
testcase.tls.echo_waiter = self.target.console.text(
cmd,
name="shell echo",
console=console,
timeout=10,
)
while True:
testcase.tls.echo_cmd_leftover = None
self.target.send(testcase.tls.echo_cmd,
crlf=None,
console=console)
testcase.tls.echo_waiter.regex_set(testcase.tls.echo_cmd)
testcase.expect(
testcase.tls.echo_waiter,
**self._fixups,
)
if testcase.tls.echo_cmd_leftover == None:
break
self.target.report_info(
"shell/fixup: resuming partially interrupted command"
" by message printed in console; sending: " +
commonl.str_bytes_cast(testcase.tls.echo_cmd, str))
testcase.tls.echo_cmd = testcase.tls.echo_cmd_leftover
continue
finally:
testcase.tls.echo_cmd = None
console_name = console if console else "default"
self.target.report_info(f"shell/{console_name}: sent command: {cmd}")
if expect:
if isinstance(expect, list):
for expectation in expect:
assert isinstance(expectation, str) \
or isinstance(expectation, typing.Pattern)
r = target.expect(expectation,
name="command output",
console=console,
origin=origin)
else:
r = target.expect(expect,
name="command output",
console=console,
origin=origin)
else:
r = None
if prompt_regex == None:
self.target.expect(self.prompt_regex,
name="shell prompt",
console=console,
origin=origin)
else:
self.target.expect(prompt_regex,
name="shell prompt",
console=console,
origin=origin)
if not output:
return r
# we need to return output, postprocess it
if console == None:
console = target.console.default
if output_filter_crlf:
newline = None
else:
newline = ''
output = self.target.console.read(offset=offset,
console=console,
newline=newline)
if trim:
# When we can run(), it usually prints in the console:
## <command-echo from our typing>
## <command output>
## <prompt>
#
# So to trim we just remove the first and last
# lines--won't work well without output_filter_crlf
# and it is quite a hack.
first_nl = output.find("\n")
last_nl = output.rfind("\n")
output = output[first_nl + 1:last_nl + 1]
return output
