def on(self, target, component):
self.log = target.log
self.log.action = "openocd start"
# Well, reusing the TCP port range is creating plenty of
# problems, as when we kill it and try to restart it, the
# sockets are lingering and it fails to reopen it...
#
# So we'll go random -- if it fails, it'll be restarted
# with another one
tcp_port_base = commonl.tcp_port_assigner(
2 + len(self.board['targets']) - 1, ttbl.config.tcp_port_range)
# these are so the command line can be substituted
target.fsdb.set("openocd-serial-string", self.serial)
target.fsdb.set("openocd-tcp-port", tcp_port_base + 1)
target.fsdb.set("openocd-telnet-port", tcp_port_base)
target.fsdb.set("openocd-gdb-port", tcp_port_base + 2)
self.cmdline_extra = []
# configuration text for the board itself
#
# this can be read anytime, but can only be written once we
# know the target and thus it has to happen in the on() method
if self.board['config']:
name = os.path.join(target.state_dir,
"openocd-board-%s.cfg" % component)
with open(name, "w") as cfgf:
cfgf.write(self.board['config'] % kws)
self.cmdline_extra += ["-f", name]
ttbl.power.daemon_c.on(self, target, component)
def _power_on_do_openocd_start(self):
self.log.action = "openocd start"
kws = {}
if self.serial != None:
kws["serial_string"] = self.serial
else:
kws["serial_string"] = "MISCONFIGURATION? SERIAL-NUMBER NOT SPECIFIED"
# Well, reusing the TCP port range is creating plenty of
# problems, as when we kill it and try to restart it, the
# sockets are lingering and it fails to reopen it...
#tcp_port_base = ttbl.tcp_port_assigner(2 + self.max_target)
# Schew it, let's go random -- if it fails, it'll be restarted
# with another one
tcp_port_base = commonl.tcp_port_assigner(2 + self.max_target,
ttbl.config.tcp_port_range)
self.log.debug("port base %d" % tcp_port_base)
self.tt.fsdb.set("openocd.port", "%d" % tcp_port_base)
self.log.debug("port base read: %s" % self.tt.fsdb.get("openocd.port"))
args = [
self.openocd_path, "-c",
'tcl_port %d' % (tcp_port_base + 1), "-c",
'telnet_port %d' % tcp_port_base, "--log_output", self.log_name,
"-c",
'gdb_port %d' % (tcp_port_base + 2)
]
if self.debug:
args.append("-d")
if self.openocd_scripts:
args += ["-s", self.openocd_scripts]
if 'interface' in self.board and self.board['interface'] != None:
args += ["-f", self.board['interface']]
if 'board' in self.board and self.board['board'] != None:
if self.openocd_scripts == None:
self.openocd_scripts = ""
args += [
"-f",
os.path.join(self.openocd_scripts, "board",
self.board['board'] + ".cfg")
]
if self.board['config']:
with open(os.path.join(self.tt.state_dir, "openocd.cfg"), "w") \
as cfgf:
cfgf.write(self.board['config'] % kws)
args += ["-f", cfgf.name]
self.log.info("OpenOCD command line %s" % " ".join(args))
self.tt.fsdb.set("openocd.path", commonl.which(self.openocd_path))
p = subprocess.Popen(args,
shell=False,
cwd=self.tt.state_dir,
close_fds=True)
# loop for a while until we can connect to it, to prove it's
# done initializing
self.pid = p.pid
self.pid_s = "%d" % p.pid
ttbl.daemon_pid_add(self.pid) # FIXME: race condition if it died?
# Write a pidfile, as openocd can't do it himself :/ [daemon 101]
self.tt.fsdb.set("openocd.pid", self.pid_s)
def setUpClass(
cls, # pylint: disable = arguments-differ, dangerous-default-value, too-many-arguments
config_text=None,
use_ssl=False,
ttbd_config_files=[],
tcf_config_text=None,
tcf_config_files=[]):
test_tcf_mixin.setUpClass()
srcdir = os.path.realpath(
os.path.join(os.path.dirname(inspect.getsourcefile(cls)), ".."))
cls.ttbd_etc_dir = os.path.join(cls.wd, "ttbd-etc")
os.mkdir(cls.ttbd_etc_dir)
cls.ttbd_files_dir = os.path.join(cls.wd, "ttbd-files")
os.mkdir(cls.ttbd_files_dir)
cls.ttbd_state_dir = os.path.join(cls.wd, "ttbd-state")
os.mkdir(cls.ttbd_state_dir)
for fn in ttbd_config_files:
shutil.copy(fn, cls.ttbd_etc_dir)
cls.ttbd_config = open(
os.path.join(cls.ttbd_etc_dir, "conf_00_base.py"), "w")
if config_text:
cls.ttbd_config.write(config_text)
cls.ttbd_config.flush()
cls.ttbd_config.seek(0) # So we can read it
cls.port = commonl.tcp_port_assigner()
# FIXME: establish a random port that is not used
if use_ssl == True:
cls.url = "https://localhost:%d" % cls.port
ssl_context = ""
else:
cls.url = "http://localhost:%d" % cls.port
ssl_context = "--no-ssl"
try:
# This allows us to default to the source location,when
# running from source, or the installed when running from
# the system
ttbd_path = os.environ.get("TTBD_PATH", srcdir + "/ttbd/ttbd")
cls.ttbd_sp = ttbd_start(
ttbd_path + " --local-auth -vvvvv --host localhost "
"%s --port %d --files-path %s --state-path %s "
"--config-path %s " %
(ssl_context, cls.port, cls.ttbd_files_dir, cls.ttbd_files_dir,
cls.ttbd_etc_dir),
url=cls.url,
stdout_name=os.path.join(cls.wd, "ttbd-stdout.log"),
stderr_name=os.path.join(cls.wd, "ttbd-stderr.log"))
except Exception as e:
logging.error(e)
raise
# Leave TCF client library ready to use with this server
tcfl.ttb_client.rest_init(cls.ttbd_state_dir, cls.url, use_ssl, None)
def setUpClass(cls, # pylint: disable = arguments-differ, dangerous-default-value, too-many-arguments
config_text = None, use_ssl = False, ttbd_config_files = [],
tcf_config_text = None, tcf_config_files = []):
test_tcf_mixin.setUpClass()
srcdir = os.path.realpath(os.path.join(
os.path.dirname(inspect.getsourcefile(cls)), ".."))
cls.ttbd_etc_dir = os.path.join(cls.wd, "ttbd-etc")
os.mkdir(cls.ttbd_etc_dir)
cls.ttbd_files_dir = os.path.join(cls.wd, "ttbd-files")
os.mkdir(cls.ttbd_files_dir)
cls.ttbd_state_dir = os.path.join(cls.wd, "ttbd-state")
os.mkdir(cls.ttbd_state_dir)
for fn in ttbd_config_files:
shutil.copy(fn, cls.ttbd_etc_dir)
cls.ttbd_config = open(
os.path.join(cls.ttbd_etc_dir, "conf_00_base.py"), "w")
if config_text:
cls.ttbd_config.write(config_text)
cls.ttbd_config.flush()
cls.ttbd_config.seek(0) # So we can read it
cls.port = commonl.tcp_port_assigner()
# FIXME: establish a random port that is not used
if use_ssl == True:
cls.url = "https://localhost:%d" % cls.port
ssl_context = ""
else:
cls.url = "http://localhost:%d" % cls.port
ssl_context = "--no-ssl"
try:
# This allows us to default to the source location,when
# running from source, or the installed when running from
# the system
ttbd_path = os.environ.get("TTBD_PATH", srcdir + "/ttbd/ttbd")
cls.ttbd_sp = ttbd_start(
ttbd_path + " --local-auth -vvvvv --host localhost "
"%s --port %d --files-path %s --state-path %s "
"--config-path %s "
% (ssl_context, cls.port,
cls.ttbd_files_dir, cls.ttbd_files_dir,
cls.ttbd_etc_dir),
url = cls.url,
stdout_name = os.path.join(cls.wd, "ttbd-stdout.log"),
stderr_name = os.path.join(cls.wd, "ttbd-stderr.log")
)
except Exception as e:
logging.error(e)
raise
# Leave TCF client library ready to use with this server
tcfl.ttb_client.rest_init(cls.ttbd_state_dir,
cls.url, use_ssl, None)
def _power_on_do_openocd_start(self):
self.log.action = "openocd start"
kws = {}
if self.serial != None:
kws["serial_string"] = self.serial
else:
kws["serial_string"] = "MISCONFIGURATION? SERIAL-NUMBER NOT SPECIFIED"
# Well, reusing the TCP port range is creating plenty of
# problems, as when we kill it and try to restart it, the
# sockets are lingering and it fails to reopen it...
#tcp_port_base = ttbl.tcp_port_assigner(2 + self.max_target)
# Schew it, let's go random -- if it fails, it'll be restarted
# with another one
tcp_port_base = commonl.tcp_port_assigner(2 + self.max_target,
ttbl.config.tcp_port_range)
self.log.debug("port base %d" % tcp_port_base)
self.tt.fsdb.set("openocd.port", "%d" % tcp_port_base)
self.log.debug("port base read: %s" % self.tt.fsdb.get("openocd.port"))
args = [ self.openocd_path,
"-c", 'tcl_port %d' % (tcp_port_base + 1),
"-c", 'telnet_port %d' % tcp_port_base,
"--log_output", self.log_name,
"-c", 'gdb_port %d' % (tcp_port_base + 2) ]
if self.debug:
args.append("-d")
if self.openocd_scripts:
args += [ "-s", self.openocd_scripts ]
if 'interface' in self.board and self.board['interface'] != None:
args += [ "-f", self.board['interface'] ]
if 'board' in self.board and self.board['board'] != None:
if self.openocd_scripts == None:
self.openocd_scripts = ""
args += [ "-f", os.path.join(self.openocd_scripts, "board",
self.board['board'] + ".cfg")]
if self.board['config']:
with open(os.path.join(self.tt.state_dir, "openocd.cfg"), "w") \
as cfgf:
cfgf.write(self.board['config'] % kws)
args += [ "-f", cfgf.name ]
self.log.info("OpenOCD command line %s" % " ".join(args))
self.tt.fsdb.set("openocd.path", commonl.which(self.openocd_path))
p = subprocess.Popen(args, shell = False, cwd = self.tt.state_dir,
close_fds = True)
# loop for a while until we can connect to it, to prove it's
# done initializing
self.pid = p.pid
self.pid_s = "%d" % p.pid
ttbl.daemon_pid_add(self.pid) # FIXME: race condition if it died?
# Write a pidfile, as openocd can't do it himself :/ [daemon 101]
self.tt.fsdb.set("openocd.pid", self.pid_s)
def _qemu_launch(self, bsp, kws):
gdb_tcp_port = commonl.tcp_port_assigner(
1, port_range=ttbl.config.tcp_port_range)
self.fsdb.set("debug-%s-gdb-tcp-port" % bsp, "%s" % gdb_tcp_port)
console_out_fname = os.path.join(self.state_dir,
"console-%s.log" % bsp)
errfname = os.path.join(self.state_dir, "%s-stderr.log" % bsp)
try:
# Make sure we wipe the PID file -- sometimes a pidfile is
# left over and it seems to override it, so the reading
# becomes corrupt
commonl.rm_f(self.pidfile[bsp])
qemu_cmdline = \
(self.qemu_cmdlines[bsp] % kws).split() \
+ [
# Don't add -daemonize! This way this is part of
# the process tree and killed when we kill the
# parent process
# We use QMP to find the PTY assigned
"-qmp", "unix:%s.qmp,server,nowait" % self.pidfile[bsp],
# Always start in debug mode -- this way the
# whole thing is stopped until we unleash it
# with QMP; this allows us to first start
# daemons that we might need to start
"-S",
"-pidfile", self.pidfile[bsp],
"-gdb", "tcp:0.0.0.0:%d" % gdb_tcp_port,
]
self.fsdb.set("qemu-cmdline-%s" % bsp, qemu_cmdline[0])
except KeyError as e:
msg = "bad QEMU command line specification: " \
"uninitialized key %s" % e
self.log.error(msg)
commonl.raise_from(RuntimeError(msg), e)
self.log.debug("QEMU cmdline %s" % " ".join(qemu_cmdline))
self.tags['bsps'][bsp]['cmdline'] = " ".join(qemu_cmdline)
try:
_preexec_fn = getattr(self, "qemu_preexec_fn", None)
def _local_preexec_fn():
if _preexec_fn:
_preexec_fn()
# Open file descriptors for stdout/stderr to a
# logfile, because -D is not really working. We
# need it to check startup errors for things that
# need a retry.
commonl.rm_f(errfname)
logfd = os.open(
errfname,
# O_CREAT: Always a new file, so
# we can check for errors and not
# get confused with previous runs
os.O_WRONLY | os.O_EXCL | os.O_CREAT,
0o0644)
os.dup2(logfd, 1)
os.dup2(logfd, 2)
os.close(logfd)
p = subprocess.Popen(qemu_cmdline,
shell=False,
cwd=self.state_dir,
close_fds=True,
preexec_fn=_local_preexec_fn)
self.log.debug("QEMU %s: console @ %s" % (bsp, console_out_fname))
# Give it a few secs to start, the pidfile has been
# deleted before starting -- note 4 was found by
# ad-hoc experimentation, sometimes depending on system load it
# takes more or less.
timeout = 10
ts0 = time.time()
while True:
if time.time() - ts0 > timeout:
lines = []
with open(errfname) as f:
count = 0
for line in f:
lines.append("log: " + line)
if count > 5:
lines.append("log: ...")
break
raise RuntimeError("QEMU %s: did not start after %.0fs\n"
"%s" % (bsp, timeout, "\n".join(lines)))
try:
if self._qmp_running(bsp):
# FIXME: race condition
ttbl.daemon_pid_add(p.pid)
return True
except RuntimeError as e:
self.log.warning("QEMU %s: can't read QMP: %s" %
(bsp, str(e)))
# fall through, let it retry
# Check errors during startup
with open(errfname, "r") as logf:
causes_for_retry = [
# bah, race condition: since we chose the
# port we wanted to use and until we
# started using it someone took it. Retry
'Failed to bind socket: Address already in use',
]
for line in logf:
for cause in causes_for_retry:
if cause in line:
self.log.info(
"QEMU %s: retrying because found in "
"logfile: %s", bsp, cause)
return False
time.sleep(0.25)
# nothing runs after this, either it returns or raises
except (OSError, ValueError) as e:
self.log.debug("QEMU %s: launch failure: %s", bsp, e)
raise
#! /usr/bin/python3
#
# Copyright (c) 2017 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
import commonl
import ttbl.power
port_base = commonl.tcp_port_assigner(1)
commonl.buildah_image_create("ttbd",
ttbl.power.daemon_podman_container_c.dockerfile,
capture_output=False)
target = ttbl.test_target("t0")
target.interface_add(
"power",
ttbl.power.interface(c0=ttbl.power.rpyc_c(
"ttbd",
port_base,
run_files={
"runthis.sh":
"""
#! /bin/sh -xe
ls -l /etc/ttbd
ls -l /etc/ttbd/run
echo "ein belegtes Brot mit Schinken" > /tmp/runthisexecuted
"""
},
def _qemu_launch(self, bsp, kws):
gdb_tcp_port = commonl.tcp_port_assigner(
1, port_range = ttbl.config.tcp_port_range)
self.fsdb.set("debug-%s-gdb-tcp-port" % bsp, "%s" % gdb_tcp_port)
console_out_fname = os.path.join(
self.state_dir, "console-%s.log" % bsp)
errfname = os.path.join(
self.state_dir, "%s-stderr.log" % bsp)
try:
# Make sure we wipe the PID file -- sometimes a pidfile is
# left over and it seems to override it, so the reading
# becomes corrupt
commonl.rm_f(self.pidfile[bsp])
qemu_cmdline = \
(self.qemu_cmdlines[bsp] % kws).split() \
+ [
# Don't add -daemonize! This way this is part of
# the process tree and killed when we kill the
# parent process
# We use QMP to find the PTY assigned
"-qmp", "unix:%s.qmp,server,nowait" % self.pidfile[bsp],
# Always start in debug mode -- this way the
# whole thing is stopped until we unleash it
# with QMP; this allows us to first start
# daemons that we might need to start
"-S",
"-pidfile", self.pidfile[bsp],
"-gdb", "tcp:0.0.0.0:%d" % gdb_tcp_port,
]
self.fsdb.set("qemu-cmdline-%s" % bsp, qemu_cmdline[0])
except KeyError as e:
msg = "bad QEMU command line specification: " \
"uninitialized key %s" % e
self.log.error(msg)
commonl.raise_from(RuntimeError(msg), e)
self.log.debug("QEMU cmdline %s" % " ".join(qemu_cmdline))
self.tags['bsps'][bsp]['cmdline'] = " ".join(qemu_cmdline)
try:
_preexec_fn = getattr(self, "qemu_preexec_fn", None)
def _local_preexec_fn():
if _preexec_fn:
_preexec_fn()
# Open file descriptors for stdout/stderr to a
# logfile, because -D is not really working. We
# need it to check startup errors for things that
# need a retry.
commonl.rm_f(errfname)
logfd = os.open(errfname,
# O_CREAT: Always a new file, so
# we can check for errors and not
# get confused with previous runs
os.O_WRONLY | os.O_EXCL |os.O_CREAT, 0o0644)
os.dup2(logfd, 1)
os.dup2(logfd, 2)
os.close(logfd)
p = subprocess.Popen(qemu_cmdline,
shell = False, cwd = self.state_dir,
close_fds = True,
preexec_fn = _local_preexec_fn)
self.log.debug("QEMU %s: console @ %s" % (bsp, console_out_fname))
# Give it a few secs to start, the pidfile has been
# deleted before starting -- note 4 was found by
# ad-hoc experimentation, sometimes depending on system load it
# takes more or less.
timeout = 10
ts0 = time.time()
while True:
if time.time() - ts0 > timeout:
lines = []
with open(errfname) as f:
count = 0
for line in f:
lines.append("log: " + line)
if count > 5:
lines.append("log: ...")
break
raise RuntimeError("QEMU %s: did not start after %.0fs\n"
"%s" % (bsp, timeout, "\n".join(lines)))
try:
if self._qmp_running(bsp):
# FIXME: race condition
ttbl.daemon_pid_add(p.pid)
return True
except RuntimeError as e:
self.log.warning("QEMU %s: can't read QMP: %s"
% (bsp, str(e)))
# fall through, let it retry
# Check errors during startup
with open(errfname, "r") as logf:
causes_for_retry = [
# bah, race condition: since we chose the
# port we wanted to use and until we
# started using it someone took it. Retry
'Failed to bind socket: Address already in use',
]
for line in logf:
for cause in causes_for_retry:
if cause in line:
self.log.info(
"QEMU %s: retrying because found in "
"logfile: %s", bsp, cause)
return False
time.sleep(0.25)
# nothing runs after this, either it returns or raises
except (OSError, ValueError) as e:
self.log.debug("QEMU %s: launch failure: %s", bsp, e)
raise
def __init__(self,
config_text=None,
config_files=None,
use_ssl=False,
tmpdir=None,
keep_temp=True,
errors_ignore=None,
warnings_ignore=None,
aka=None,
local_auth=True):
# Force all assertions, when running like this, to fail the TC
tcfl.tc.tc_c.exception_to_result[AssertionError] = tcfl.tc.failed_e
# If no aka is defined, we make one out of the place when this
# object is being created, so it is always the same *and* thus
# the report hashes are always identical with each run
if aka == None:
self.aka = "ttbd-" + commonl.mkid(commonl.origin_get(2), 4)
else:
self.aka = aka
if config_files == None:
config_files = []
self.keep_temp = keep_temp
self.port = commonl.tcp_port_assigner()
self.use_ssl = use_ssl
if use_ssl == True:
self.url = "https://localhost:%d" % self.port
ssl_context = ""
else:
self.url = "http://localhost:%d" % self.port
ssl_context = "--no-ssl"
self.url_spec = "fullid:'^%s'" % self.aka
if tmpdir:
self.tmpdir = tmpdir
else:
# default to place the server's dir in the tempdir for
# testcases
self.tmpdir = os.path.join(tcfl.tc.tc_c.tmpdir, "server", self.aka)
shutil.rmtree(self.tmpdir, ignore_errors=True)
commonl.makedirs_p(self.tmpdir)
self.etc_dir = os.path.join(self.tmpdir, "etc")
self.files_dir = os.path.join(self.tmpdir, "files")
self.lib_dir = os.path.join(self.tmpdir, "lib")
self.state_dir = os.path.join(self.tmpdir, "state")
os.mkdir(self.etc_dir)
os.mkdir(self.files_dir)
os.mkdir(self.lib_dir)
os.mkdir(self.state_dir)
self.stdout = self.tmpdir + "/stdout"
self.stderr = self.tmpdir + "/stderr"
for fn in config_files:
shutil.copy(fn, self.etc_dir)
with open(os.path.join(self.etc_dir, "conf_00_base.py"), "w") as cfgf:
cfgf.write(r"""
import ttbl.config
ttbl.config.processes = 2
host = '127.0.0.1'
""")
# We don't define here the port, so we see it in the
# command line
if config_text:
cfgf.write(config_text)
self.srcdir = os.path.realpath(
os.path.join(os.path.dirname(__file__), ".."))
self.cmdline = [
"stdbuf",
"-o0",
"-e0",
# This allows us to default to the source location,when
# running from source, or the installed when running from
# the system
os.environ.get("TTBD_PATH", self.srcdir + "/ttbd/ttbd"),
"--port",
"%d" % self.port,
ssl_context,
"-vvvvv",
"--files-path",
self.files_dir,
"--state-path",
self.state_dir,
"--config-path",
"", # This empty one is to clear them all
"--config-path",
self.etc_dir
]
self.local_auth = local_auth
if local_auth:
self.cmdline.append("--local-auth")
self.p = None
#: Exclude these regexes / strings from triggering an error
#: message check
self.errors_ignore = [] if errors_ignore == None else errors_ignore
#: Exclude these regexes / strings from triggering an warning
#: message check
self.warnings_ignore = [re.compile('daemon lacks CAP_NET_ADMIN')]
if warnings_ignore:
self.warnings_ignore += warnings_ignore
def _preexec_fn():
stdout_fd = os.open(
self.stdout,
# O_CREAT: Always a new file, so
# we can check for errors and not
# get confused with previous runs
os.O_WRONLY | os.O_EXCL | os.O_CREAT,
0o0644)
stderr_fd = os.open(
self.stderr,
# O_CREAT: Always a new file, so
# we can check for errors and not
# get confused with previous runs
os.O_WRONLY | os.O_EXCL | os.O_CREAT,
0o0644)
os.dup2(stdout_fd, 1)
os.dup2(stderr_fd, 2)
logging.info("Launching: %s", " ".join(self.cmdline))
self.p = subprocess.Popen(self.cmdline,
shell=False,
cwd=self.tmpdir,
close_fds=True,
preexec_fn=_preexec_fn,
bufsize=0)
try:
self._check_if_alive()
finally:
self.check_log_for_issues()
# if we call self.terminate() from __del__, the garbage
# collector has started to wipe things, so we can't use, ie:
# open() to check the log file
atexit.register(self.terminate)
def on(self, target, component):
# Start QEMU
#
# We first assign a port for GDB debugging, if someone takes
# it before we do, start will fail and paranoid mode (see
# above) will restart it.
# FIXME: allocate ports for VNC only if there is a vnc= in the
# command line
base = ttbl.config.tcp_port_range[0]
top = ttbl.config.tcp_port_range[1]
if base < 5900: # we need ports >= 5900 for VNC
base = 5900
tcp_port_base = commonl.tcp_port_assigner(2, port_range=(base, top))
target.fsdb.set("qemu-gdb-tcp-port", "%s" % tcp_port_base)
# This might not be used at all, but we allocate and declare
# it in case the implementation will use it; allocating in a
# higher layer makes it more complicated.
# this one is the port number based on 5900 (VNC default 0)
if top < 5900:
logging.warning(
"ttbl.config.tcp_port_range %s doesn't include ports "
"between above 5900, needed for VNC services. "
"QEMU targets needing VNC support will fail to start "
"complaining about 'vnc-port' not defined",
ttbl.config.tcp_port_range)
else:
# FIXME: move to vnc.vnc0.{host,port,tcp-port}
# set this for general information; the VNC screenshotter
# also uses it
target.fsdb.set("vnc-host", "localhost")
target.fsdb.set("vnc-port", "%s" % (tcp_port_base + 1 - 5900))
# this one is the raw port number
target.fsdb.set("vnc-tcp-port", "%s" % (tcp_port_base + 1))
# New form
target.fsdb.set("vnc.vnc0.host", "localhost")
target.fsdb.set("vnc.vnc0.port", "%s" % (tcp_port_base + 1 - 5900))
# this one is the raw port number
target.fsdb.set("vnc.vnc0.tcp_port", "%s" % (tcp_port_base + 1))
self.cmdline_extra = []
image_keys = target.fsdb.keys("qemu-image-*")
#
# Images interface: flash() below has been used to set images
# to run, this will feed them into QEMU
#
if 'qemu-image-bios' in image_keys:
self.cmdline_extra += ["-bios", "%(qemu-image-bios)s"]
if 'qemu-image-kernel' in image_keys:
self.cmdline_extra += ["-kernel", "%(qemu-image-kernel)s"]
if 'qemu-image-kernel-args' in image_keys:
self.cmdline_extra += ["-append", "%(qemu-image-kernel-args)s"]
if 'qemu-image-initrd' in image_keys:
self.cmdline_extra += ["-initrd", "%(qemu-image-initrd)s"]
#
# Network support bits--add command line options to connect
# this VM to the networks the target declares
#
# For each network we are connected to, there must be a
# network_tap_pc power rail controller that sets up the
# network devices for us before starting QEMU.
#
# We look at it to setup the QEMU side of things as extra
# command line options.
#
# - network name/icname: comes from the tuntap-ICNAME property
# created by the network_tap_pc power rail controller that
# the configuration must put on before the qemu power rail
# controller.
#
# - model: comes from what HW it has to emulate; each arch has a
# default or you can set for all with property qemu-model or
# qemu-model-ICNAME (for that specifc interconnect).
#
# It has to be a valid QEMU model or QEMU will reject it
# with an error. You can find valid models with::
#
# qemu-system-ARCH -net nic,model=help
#
# Zephyr, e.g., supports a few: lan9118 stellaris e1000 (from)::
#
# $ grep --color -nH --null -rie ETH_NIC_MODEL
# drivers/ethernet/Kconfig.smsc911x:13:config ETH_NIC_MODEL$
# drivers/ethernet/Kconfig.stellaris:15:config ETH_NIC_MODEL$
# drivers/ethernet/Kconfig.e1000:16:config ETH_NIC_MODEL$
for tap, if_name in target.fsdb.get_as_dict("tuntap-*").items():
# @tap is tuntap-NETWORK, the property set by
# powe rail component network_tap_pc.
_, ic_name = tap.split("-", 1)
mac_addr = target.tags['interconnects'][ic_name]['mac_addr']
model = commonl.name_make_safe(
target.fsdb.get("qemu-model-" + ic_name,
target.fsdb.get("qemu-model", self.nic_model)))
self.cmdline_extra += [
"-device",
#
# romfile= UNSET for using virtio-net
# - https://wiki.syslinux.org/wiki/index.php?title=Development/Testing
# Workaround using virtio
f"{model},netdev={ic_name},mac={mac_addr},romfile=",
"-netdev",
f"tap,id={ic_name},script=no,ifname={if_name}"
]
#
# Ok, so do actually start
#
# if the debug port is snatched before we start by someone
# else, this will fail and we'll have to retry--this is
# why the power component is set to paranoid mode, so the
# power interface automatically retries it if it fails.
ttbl.power.daemon_c.on(self, target, component)
# run console/on steps
self._qemu_console_on(target, component)
#
# Debugging interface--if the target is not in debugging mode,
# tell QEMU to start right away
#
if target.fsdb.get("debug") == None:
self.debug_resume(target, component)
def on(self, target, component):
# Start QEMU
#
# We first assign a port for GDB debugging, if someone takes
# it before we do, start will fail and paranoid mode (see
# above) will restart it.
base = ttbl.config.tcp_port_range[0]
top = ttbl.config.tcp_port_range[1]
if base < 5900: # we need ports >= 5900 for VNC
base = 5900
tcp_port_base = commonl.tcp_port_assigner(2, port_range=(base, top))
target.fsdb.set("qemu-gdb-tcp-port", "%s" % tcp_port_base)
# This might not be used at all, but we allocate and declare
# it in case the implementation will use it; allocating in a
# higher layer makes it more complicated.
# this one is the port number based on 5900 (VNC default 0)
if top < 5900:
logging.warning(
"ttbl.config.tcp_port_range %s doesn't include ports "
"between above 5900, needed for VNC services. "
"QEMU targets needing VNC support will fail to start "
"complaining about 'vnc-port' not defined",
ttbl.config.tcp_port_range)
else:
# set this for general information; the VNC screenshotter
# also uses it
target.fsdb.set("vnc-host", "localhost")
target.fsdb.set("vnc-port", "%s" % (tcp_port_base + 1 - 5900))
# this one is the raw port number
target.fsdb.set("vnc-tcp-port", "%s" % (tcp_port_base + 1))
self.cmdline_extra = []
image_keys = target.fsdb.keys("qemu-image-*")
#
# Images interface: flash() below has been used to set images
# to run, this will feed them into QEMU
#
if 'qemu-image-bios' in image_keys:
self.cmdline_extra += ["-bios", "%(qemu-image-bios)s"]
if 'qemu-image-kernel' in image_keys:
self.cmdline_extra += ["-kernel", "%(qemu-image-kernel)s"]
if 'qemu-image-kernel-args' in image_keys:
self.cmdline_extra += ["-append", "%(qemu-image-kernel-args)s"]
if 'qemu-image-initrd' in image_keys:
self.cmdline_extra += ["-initrd", "%(qemu-image-initrd)s"]
#
# Network support bits--add command line options to connect
# this VM to the networks the target declares
#
# For each network we are connected to, there must be a
# network_tap_pc power rail controller that sets up the
# network devices for us before starting QEMU.
#
# We look at it to setup the QEMU side of things as extra
# command line options.
#
# - network name/icname: comes from the tuntap-ICNAME property
# created by the network_tap_pc power rail controller that
# the configuration must put on before the qemu power rail
# controller.
#
# - model: comes from what HW it has to emulate; each arch has a
# default or you can set for all with property qemu-model or
# qemu-model-ICNAME (for that specifc interconnect).
#
# It has to be a valid QEMU model or QEMU will reject it
# with an error. You can find valid models with::
#
# qemu-system-ARCH -net nic,model=help
#
# Zephyr, e.g., supports a few: lan9118 stellaris e1000 (from)::
#
# $ grep --color -nH --null -rie ETH_NIC_MODEL
# drivers/ethernet/Kconfig.smsc911x:13:config ETH_NIC_MODEL$
# drivers/ethernet/Kconfig.stellaris:15:config ETH_NIC_MODEL$
# drivers/ethernet/Kconfig.e1000:16:config ETH_NIC_MODEL$
fds = []
try:
for tap, if_name in target.fsdb.get_as_dict(
"tuntap-*").iteritems():
# @tap is tuntap-NETWORK, the property set by
# powe rail component network_tap_pc.
_, ic_name = tap.split("-", 1)
mac_addr = target.tags['interconnects'][ic_name]['mac_addr']
model = commonl.name_make_safe(
target.fsdb.get(
"qemu-model-" + ic_name,
target.fsdb.get("qemu-model", self.nic_model)))
# There is no way to cahole QEMU to say: hey, we
# already have a tap interface configured for you, use
# it...other than to open it and pass the file
# descriptor to it.
# So this finds the /dev/tap* file which matches the
# interface created with tuntap- (class
# network_tap_pc), opens it and passes the descriptor
# along to qemu, then closes the FD, since we don't
# need it.
tapindex = commonl.if_index(if_name)
# Need to wait for udev to reconfigure this for us to have
# access; this is done by a udev rule installed
# (/usr/lib/udev/rules.d/80-ttbd.rules)
tapdevname = "/dev/tap%d" % tapindex
ts0 = time.time()
ts = ts0
timeout = 5
fd = None
while ts - ts0 < timeout:
try:
fd = os.open(tapdevname, os.O_RDWR, 0)
fds.append(fd)
break
except OSError as e:
target.log.error(
"%s: couldn't open %s after"
" +%.1f/%.1fs, retrying: %s", ic_name, tapdevname,
ts - ts0, timeout, e)
time.sleep(0.25)
ts = time.time()
else:
raise RuntimeError(
"%s: timedout (%.1fs) opening /dev/tap%d's;"
" udev's /usr/lib/udev/rules.d/80-ttbd.rules"
" didn't set permissions?" %
(ic_name, timeout, tapindex))
self.cmdline_extra += [
"-nic",
"tap,fd=%d,id=%s,model=%s,mac=%s,br=_b%s" \
% (fd, ic_name, model, mac_addr, ic_name )
]
if not fds:
# No tuntap network attachments -> no network
# support. Cap it.
self.cmdline_extra += ['-nic', 'none']
#
# Ok, so do actually start
#
# if the debug port is snatched before we start by someone
# else, this will fail and we'll have to retry--this is
# why the power component is set to paranoid mode, so the
# power interface automatically retries it if it fails.
ttbl.power.daemon_c.on(self, target, component)
# Those consoles? update their generational counts so the
# clients can know they restarted
if hasattr(target, "console"):
for console in target.console.impls.keys():
ttbl.console.generation_set(target, console)
finally:
for fd in fds: # close fds we opened for ...
os.close(fd) # ... networking, unneeded now
#
# Debugging interface--if the target is not in debugging mode,
# tell QEMU to start right away
#
if target.fsdb.get("debug") == None:
self.debug_resume(target, component)
def put_tunnel(self, target, who, args, _files, _user_path):
"""
Setup a TCP/UDP/SCTP v4 or v5 tunnel to the target
Parameters are same as :meth:`ttbl.tt_interface.request_process`
Parameters specified in the *args* dictionary from the HTTP
interface:
:param str ip_addr: target's IP address to use (it must be
listed on the targets's tags *ipv4_address* or
*ipv6_address*).
:param int port: port to redirect to
:param str protocol: Protocol to tunnel: {udp,sctp,tcp}[{4,6}]
:returns dict: dicionary with a single key *result* set ot the
*local_port* where to TCP connect to reach the tunnel.
"""
ip_addr, port, protocol, tunnel_id = self._check_args(
self.arg_get(args, 'ip_addr', basestring),
self.arg_get(args, 'port', int),
self.arg_get(args, 'protocol', basestring),
)
self._ip_addr_validate(target, ip_addr)
with target.target_owned_and_locked(who):
for tunnel_id in target.fsdb.keys("interfaces.tunnel.*.protocol"):
prefix = tunnel_id[:-len(".protocol")]
_ip_addr = target.fsdb.get(prefix + ".ip_addr")
_protocol = target.fsdb.get(prefix + ".protocol")
_port = target.fsdb.get(prefix + ".port")
_pid = target.fsdb.get(prefix + ".__id")
_lport = prefix[len("interfaces.tunnel."):]
if _ip_addr == ip_addr \
and _protocol == protocol \
and _port == port \
and commonl.process_alive(_pid, "/usr/bin/socat"):
# there is already an active tunnel for this port
# and it is alive, so use that
return dict(result=int(_lport))
local_port = commonl.tcp_port_assigner(
port_range=ttbl.config.tcp_port_range)
ip_addr = ipaddress.ip_address(unicode(ip_addr))
if isinstance(ip_addr, ipaddress.IPv6Address):
# beacause socat (and most others) likes it like that
ip_addr = "[%s]" % ip_addr
# this could be refactored using daemon_c, but it'd be
# harder to follow the code and it is not really needed.
p = subprocess.Popen([
"/usr/bin/socat", "-ly", "-lp", tunnel_id,
"%s-LISTEN:%d,fork,reuseaddr" % (protocol, local_port),
"%s:%s:%s" % (protocol, ip_addr, port)
],
shell=False,
cwd=target.state_dir,
close_fds=True)
pid = commonl.process_started(p.pid,
"/usr/bin/socat",
verification_f=commonl.tcp_port_busy,
verification_f_args=(local_port, ),
tag="socat-" + tunnel_id,
log=target.log)
if p.returncode != None:
raise RuntimeError("TUNNEL %s: socat exited with %d" %
(tunnel_id, p.returncode))
ttbl.daemon_pid_add(p.pid) # FIXME: race condition if it # died?
target.fsdb.set("interfaces.tunnel.%s.__id" % local_port, p.pid)
target.fsdb.set("interfaces.tunnel.%s.ip_addr" % local_port,
str(ip_addr))
target.fsdb.set("interfaces.tunnel.%s.protocol" % local_port,
protocol)
target.fsdb.set("interfaces.tunnel.%s.port" % local_port, port)
return dict(result=local_port)
def __init__(self, config_text = None, config_files = None,
use_ssl = False, tmpdir = None, keep_temp = True,
errors_ignore = None, warnings_ignore = None,
aka = None):
# Force all assertions, when running like this, to fail the TC
tcfl.tc.tc_c.exception_to_result[AssertionError] = tcfl.tc.failed_e
# If no aka is defined, we make one out of the place when this
# object is being created, so it is always the same *and* thus
# the report hashes are always identical with each run
if aka == None:
self.aka = "ttbd-" + commonl.mkid(commonl.origin_get(2), 4)
else:
self.aka = aka
if config_files == None:
config_files = []
self.keep_temp = keep_temp
self.port = commonl.tcp_port_assigner()
self.use_ssl = use_ssl
if use_ssl == True:
self.url = "https://localhost:%d" % self.port
ssl_context = ""
else:
self.url = "http://localhost:%d" % self.port
ssl_context = "--no-ssl"
self.url_spec = "url:'^%s'" % self.url
if tmpdir:
self.tmpdir = tmpdir
else:
# Don't use colon on the name, or it'll thing it is a path
self.tmpdir = tempfile.mkdtemp(prefix = "test-ttbd-%d."
% self.port)
self.etc_dir = os.path.join(self.tmpdir, "etc")
self.files_dir = os.path.join(self.tmpdir, "files")
self.lib_dir = os.path.join(self.tmpdir, "lib")
self.state_dir = os.path.join(self.tmpdir, "state")
os.mkdir(self.etc_dir)
os.mkdir(self.files_dir)
os.mkdir(self.lib_dir)
os.mkdir(self.state_dir)
self.stdout = self.tmpdir + "/stdout"
self.stderr = self.tmpdir + "/stderr"
for fn in config_files:
shutil.copy(fn, self.etc_dir)
with open(os.path.join(self.etc_dir,
"conf_00_base.py"), "w") as cfgf:
cfgf.write(r"""
import ttbl.config
ttbl.config.processes = 2
host = '127.0.0.1'
""")
# We don't define here the port, so we see it in the
# command line
if config_text:
cfgf.write(config_text)
srcdir = os.path.realpath(
os.path.join(os.path.dirname(__file__), ".."))
self.cmdline = [
# This allows us to default to the source location,when
# running from source, or the installed when running from
# the system
os.environ.get("TTBD_PATH", srcdir + "/ttbd/ttbd"),
"--port", "%d" % self.port,
ssl_context,
"--local-auth", "-vvvvv",
"--files-path", self.files_dir,
"--state-path", self.state_dir,
"--var-lib-path", self.lib_dir,
"--config-path", "", # This empty one is to clear them all
"--config-path", self.etc_dir
]
self.p = None
#: Exclude these regexes / strings from triggering an error
#: message check
self.errors_ignore = [] if errors_ignore == None else errors_ignore
#: Exclude these regexes / strings from triggering an warning
#: message check
self.warnings_ignore = [ re.compile('daemon lacks CAP_NET_ADMIN') ]
if warnings_ignore:
self.warnings_ignore =+ warnings_ignore
def _preexec_fn():
stdout_fd = os.open(self.stdout,
# O_CREAT: Always a new file, so
# we can check for errors and not
# get confused with previous runs
os.O_WRONLY | os.O_EXCL |os.O_CREAT, 0o0644)
stderr_fd = os.open(self.stderr,
# O_CREAT: Always a new file, so
# we can check for errors and not
# get confused with previous runs
os.O_WRONLY | os.O_EXCL |os.O_CREAT, 0o0644)
os.dup2(stdout_fd, 1)
os.dup2(stderr_fd, 2)
logging.info("Launching: %s", " ".join(self.cmdline))
self.p = subprocess.Popen(
self.cmdline, shell = False, cwd = self.tmpdir,
close_fds = True, preexec_fn = _preexec_fn)
self._check_if_alive()
self.check_log_for_issues()
