def on(self, target, component): self.log = target.log self.log.action = "openocd start" # Well, reusing the TCP port range is creating plenty of # problems, as when we kill it and try to restart it, the # sockets are lingering and it fails to reopen it... # # So we'll go random -- if it fails, it'll be restarted # with another one tcp_port_base = commonl.tcp_port_assigner( 2 + len(self.board['targets']) - 1, ttbl.config.tcp_port_range) # these are so the command line can be substituted target.fsdb.set("openocd-serial-string", self.serial) target.fsdb.set("openocd-tcp-port", tcp_port_base + 1) target.fsdb.set("openocd-telnet-port", tcp_port_base) target.fsdb.set("openocd-gdb-port", tcp_port_base + 2) self.cmdline_extra = [] # configuration text for the board itself # # this can be read anytime, but can only be written once we # know the target and thus it has to happen in the on() method if self.board['config']: name = os.path.join(target.state_dir, "openocd-board-%s.cfg" % component) with open(name, "w") as cfgf: cfgf.write(self.board['config'] % kws) self.cmdline_extra += ["-f", name] ttbl.power.daemon_c.on(self, target, component)
def _power_on_do_openocd_start(self): self.log.action = "openocd start" kws = {} if self.serial != None: kws["serial_string"] = self.serial else: kws["serial_string"] = "MISCONFIGURATION? SERIAL-NUMBER NOT SPECIFIED" # Well, reusing the TCP port range is creating plenty of # problems, as when we kill it and try to restart it, the # sockets are lingering and it fails to reopen it... #tcp_port_base = ttbl.tcp_port_assigner(2 + self.max_target) # Schew it, let's go random -- if it fails, it'll be restarted # with another one tcp_port_base = commonl.tcp_port_assigner(2 + self.max_target, ttbl.config.tcp_port_range) self.log.debug("port base %d" % tcp_port_base) self.tt.fsdb.set("openocd.port", "%d" % tcp_port_base) self.log.debug("port base read: %s" % self.tt.fsdb.get("openocd.port")) args = [ self.openocd_path, "-c", 'tcl_port %d' % (tcp_port_base + 1), "-c", 'telnet_port %d' % tcp_port_base, "--log_output", self.log_name, "-c", 'gdb_port %d' % (tcp_port_base + 2) ] if self.debug: args.append("-d") if self.openocd_scripts: args += ["-s", self.openocd_scripts] if 'interface' in self.board and self.board['interface'] != None: args += ["-f", self.board['interface']] if 'board' in self.board and self.board['board'] != None: if self.openocd_scripts == None: self.openocd_scripts = "" args += [ "-f", os.path.join(self.openocd_scripts, "board", self.board['board'] + ".cfg") ] if self.board['config']: with open(os.path.join(self.tt.state_dir, "openocd.cfg"), "w") \ as cfgf: cfgf.write(self.board['config'] % kws) args += ["-f", cfgf.name] self.log.info("OpenOCD command line %s" % " ".join(args)) self.tt.fsdb.set("openocd.path", commonl.which(self.openocd_path)) p = subprocess.Popen(args, shell=False, cwd=self.tt.state_dir, close_fds=True) # loop for a while until we can connect to it, to prove it's # done initializing self.pid = p.pid self.pid_s = "%d" % p.pid ttbl.daemon_pid_add(self.pid) # FIXME: race condition if it died? # Write a pidfile, as openocd can't do it himself :/ [daemon 101] self.tt.fsdb.set("openocd.pid", self.pid_s)
def setUpClass( cls, # pylint: disable = arguments-differ, dangerous-default-value, too-many-arguments config_text=None, use_ssl=False, ttbd_config_files=[], tcf_config_text=None, tcf_config_files=[]): test_tcf_mixin.setUpClass() srcdir = os.path.realpath( os.path.join(os.path.dirname(inspect.getsourcefile(cls)), "..")) cls.ttbd_etc_dir = os.path.join(cls.wd, "ttbd-etc") os.mkdir(cls.ttbd_etc_dir) cls.ttbd_files_dir = os.path.join(cls.wd, "ttbd-files") os.mkdir(cls.ttbd_files_dir) cls.ttbd_state_dir = os.path.join(cls.wd, "ttbd-state") os.mkdir(cls.ttbd_state_dir) for fn in ttbd_config_files: shutil.copy(fn, cls.ttbd_etc_dir) cls.ttbd_config = open( os.path.join(cls.ttbd_etc_dir, "conf_00_base.py"), "w") if config_text: cls.ttbd_config.write(config_text) cls.ttbd_config.flush() cls.ttbd_config.seek(0) # So we can read it cls.port = commonl.tcp_port_assigner() # FIXME: establish a random port that is not used if use_ssl == True: cls.url = "https://localhost:%d" % cls.port ssl_context = "" else: cls.url = "http://localhost:%d" % cls.port ssl_context = "--no-ssl" try: # This allows us to default to the source location,when # running from source, or the installed when running from # the system ttbd_path = os.environ.get("TTBD_PATH", srcdir + "/ttbd/ttbd") cls.ttbd_sp = ttbd_start( ttbd_path + " --local-auth -vvvvv --host localhost " "%s --port %d --files-path %s --state-path %s " "--config-path %s " % (ssl_context, cls.port, cls.ttbd_files_dir, cls.ttbd_files_dir, cls.ttbd_etc_dir), url=cls.url, stdout_name=os.path.join(cls.wd, "ttbd-stdout.log"), stderr_name=os.path.join(cls.wd, "ttbd-stderr.log")) except Exception as e: logging.error(e) raise # Leave TCF client library ready to use with this server tcfl.ttb_client.rest_init(cls.ttbd_state_dir, cls.url, use_ssl, None)
def setUpClass(cls, # pylint: disable = arguments-differ, dangerous-default-value, too-many-arguments config_text = None, use_ssl = False, ttbd_config_files = [], tcf_config_text = None, tcf_config_files = []): test_tcf_mixin.setUpClass() srcdir = os.path.realpath(os.path.join( os.path.dirname(inspect.getsourcefile(cls)), "..")) cls.ttbd_etc_dir = os.path.join(cls.wd, "ttbd-etc") os.mkdir(cls.ttbd_etc_dir) cls.ttbd_files_dir = os.path.join(cls.wd, "ttbd-files") os.mkdir(cls.ttbd_files_dir) cls.ttbd_state_dir = os.path.join(cls.wd, "ttbd-state") os.mkdir(cls.ttbd_state_dir) for fn in ttbd_config_files: shutil.copy(fn, cls.ttbd_etc_dir) cls.ttbd_config = open( os.path.join(cls.ttbd_etc_dir, "conf_00_base.py"), "w") if config_text: cls.ttbd_config.write(config_text) cls.ttbd_config.flush() cls.ttbd_config.seek(0) # So we can read it cls.port = commonl.tcp_port_assigner() # FIXME: establish a random port that is not used if use_ssl == True: cls.url = "https://localhost:%d" % cls.port ssl_context = "" else: cls.url = "http://localhost:%d" % cls.port ssl_context = "--no-ssl" try: # This allows us to default to the source location,when # running from source, or the installed when running from # the system ttbd_path = os.environ.get("TTBD_PATH", srcdir + "/ttbd/ttbd") cls.ttbd_sp = ttbd_start( ttbd_path + " --local-auth -vvvvv --host localhost " "%s --port %d --files-path %s --state-path %s " "--config-path %s " % (ssl_context, cls.port, cls.ttbd_files_dir, cls.ttbd_files_dir, cls.ttbd_etc_dir), url = cls.url, stdout_name = os.path.join(cls.wd, "ttbd-stdout.log"), stderr_name = os.path.join(cls.wd, "ttbd-stderr.log") ) except Exception as e: logging.error(e) raise # Leave TCF client library ready to use with this server tcfl.ttb_client.rest_init(cls.ttbd_state_dir, cls.url, use_ssl, None)
def _power_on_do_openocd_start(self): self.log.action = "openocd start" kws = {} if self.serial != None: kws["serial_string"] = self.serial else: kws["serial_string"] = "MISCONFIGURATION? SERIAL-NUMBER NOT SPECIFIED" # Well, reusing the TCP port range is creating plenty of # problems, as when we kill it and try to restart it, the # sockets are lingering and it fails to reopen it... #tcp_port_base = ttbl.tcp_port_assigner(2 + self.max_target) # Schew it, let's go random -- if it fails, it'll be restarted # with another one tcp_port_base = commonl.tcp_port_assigner(2 + self.max_target, ttbl.config.tcp_port_range) self.log.debug("port base %d" % tcp_port_base) self.tt.fsdb.set("openocd.port", "%d" % tcp_port_base) self.log.debug("port base read: %s" % self.tt.fsdb.get("openocd.port")) args = [ self.openocd_path, "-c", 'tcl_port %d' % (tcp_port_base + 1), "-c", 'telnet_port %d' % tcp_port_base, "--log_output", self.log_name, "-c", 'gdb_port %d' % (tcp_port_base + 2) ] if self.debug: args.append("-d") if self.openocd_scripts: args += [ "-s", self.openocd_scripts ] if 'interface' in self.board and self.board['interface'] != None: args += [ "-f", self.board['interface'] ] if 'board' in self.board and self.board['board'] != None: if self.openocd_scripts == None: self.openocd_scripts = "" args += [ "-f", os.path.join(self.openocd_scripts, "board", self.board['board'] + ".cfg")] if self.board['config']: with open(os.path.join(self.tt.state_dir, "openocd.cfg"), "w") \ as cfgf: cfgf.write(self.board['config'] % kws) args += [ "-f", cfgf.name ] self.log.info("OpenOCD command line %s" % " ".join(args)) self.tt.fsdb.set("openocd.path", commonl.which(self.openocd_path)) p = subprocess.Popen(args, shell = False, cwd = self.tt.state_dir, close_fds = True) # loop for a while until we can connect to it, to prove it's # done initializing self.pid = p.pid self.pid_s = "%d" % p.pid ttbl.daemon_pid_add(self.pid) # FIXME: race condition if it died? # Write a pidfile, as openocd can't do it himself :/ [daemon 101] self.tt.fsdb.set("openocd.pid", self.pid_s)
def _qemu_launch(self, bsp, kws): gdb_tcp_port = commonl.tcp_port_assigner( 1, port_range=ttbl.config.tcp_port_range) self.fsdb.set("debug-%s-gdb-tcp-port" % bsp, "%s" % gdb_tcp_port) console_out_fname = os.path.join(self.state_dir, "console-%s.log" % bsp) errfname = os.path.join(self.state_dir, "%s-stderr.log" % bsp) try: # Make sure we wipe the PID file -- sometimes a pidfile is # left over and it seems to override it, so the reading # becomes corrupt commonl.rm_f(self.pidfile[bsp]) qemu_cmdline = \ (self.qemu_cmdlines[bsp] % kws).split() \ + [ # Don't add -daemonize! This way this is part of # the process tree and killed when we kill the # parent process # We use QMP to find the PTY assigned "-qmp", "unix:%s.qmp,server,nowait" % self.pidfile[bsp], # Always start in debug mode -- this way the # whole thing is stopped until we unleash it # with QMP; this allows us to first start # daemons that we might need to start "-S", "-pidfile", self.pidfile[bsp], "-gdb", "tcp:0.0.0.0:%d" % gdb_tcp_port, ] self.fsdb.set("qemu-cmdline-%s" % bsp, qemu_cmdline[0]) except KeyError as e: msg = "bad QEMU command line specification: " \ "uninitialized key %s" % e self.log.error(msg) commonl.raise_from(RuntimeError(msg), e) self.log.debug("QEMU cmdline %s" % " ".join(qemu_cmdline)) self.tags['bsps'][bsp]['cmdline'] = " ".join(qemu_cmdline) try: _preexec_fn = getattr(self, "qemu_preexec_fn", None) def _local_preexec_fn(): if _preexec_fn: _preexec_fn() # Open file descriptors for stdout/stderr to a # logfile, because -D is not really working. We # need it to check startup errors for things that # need a retry. commonl.rm_f(errfname) logfd = os.open( errfname, # O_CREAT: Always a new file, so # we can check for errors and not # get confused with previous runs os.O_WRONLY | os.O_EXCL | os.O_CREAT, 0o0644) os.dup2(logfd, 1) os.dup2(logfd, 2) os.close(logfd) p = subprocess.Popen(qemu_cmdline, shell=False, cwd=self.state_dir, close_fds=True, preexec_fn=_local_preexec_fn) self.log.debug("QEMU %s: console @ %s" % (bsp, console_out_fname)) # Give it a few secs to start, the pidfile has been # deleted before starting -- note 4 was found by # ad-hoc experimentation, sometimes depending on system load it # takes more or less. timeout = 10 ts0 = time.time() while True: if time.time() - ts0 > timeout: lines = [] with open(errfname) as f: count = 0 for line in f: lines.append("log: " + line) if count > 5: lines.append("log: ...") break raise RuntimeError("QEMU %s: did not start after %.0fs\n" "%s" % (bsp, timeout, "\n".join(lines))) try: if self._qmp_running(bsp): # FIXME: race condition ttbl.daemon_pid_add(p.pid) return True except RuntimeError as e: self.log.warning("QEMU %s: can't read QMP: %s" % (bsp, str(e))) # fall through, let it retry # Check errors during startup with open(errfname, "r") as logf: causes_for_retry = [ # bah, race condition: since we chose the # port we wanted to use and until we # started using it someone took it. Retry 'Failed to bind socket: Address already in use', ] for line in logf: for cause in causes_for_retry: if cause in line: self.log.info( "QEMU %s: retrying because found in " "logfile: %s", bsp, cause) return False time.sleep(0.25) # nothing runs after this, either it returns or raises except (OSError, ValueError) as e: self.log.debug("QEMU %s: launch failure: %s", bsp, e) raise
#! /usr/bin/python3 # # Copyright (c) 2017 Intel Corporation # # SPDX-License-Identifier: Apache-2.0 import commonl import ttbl.power port_base = commonl.tcp_port_assigner(1) commonl.buildah_image_create("ttbd", ttbl.power.daemon_podman_container_c.dockerfile, capture_output=False) target = ttbl.test_target("t0") target.interface_add( "power", ttbl.power.interface(c0=ttbl.power.rpyc_c( "ttbd", port_base, run_files={ "runthis.sh": """ #! /bin/sh -xe ls -l /etc/ttbd ls -l /etc/ttbd/run echo "ein belegtes Brot mit Schinken" > /tmp/runthisexecuted """ },
def _qemu_launch(self, bsp, kws): gdb_tcp_port = commonl.tcp_port_assigner( 1, port_range = ttbl.config.tcp_port_range) self.fsdb.set("debug-%s-gdb-tcp-port" % bsp, "%s" % gdb_tcp_port) console_out_fname = os.path.join( self.state_dir, "console-%s.log" % bsp) errfname = os.path.join( self.state_dir, "%s-stderr.log" % bsp) try: # Make sure we wipe the PID file -- sometimes a pidfile is # left over and it seems to override it, so the reading # becomes corrupt commonl.rm_f(self.pidfile[bsp]) qemu_cmdline = \ (self.qemu_cmdlines[bsp] % kws).split() \ + [ # Don't add -daemonize! This way this is part of # the process tree and killed when we kill the # parent process # We use QMP to find the PTY assigned "-qmp", "unix:%s.qmp,server,nowait" % self.pidfile[bsp], # Always start in debug mode -- this way the # whole thing is stopped until we unleash it # with QMP; this allows us to first start # daemons that we might need to start "-S", "-pidfile", self.pidfile[bsp], "-gdb", "tcp:0.0.0.0:%d" % gdb_tcp_port, ] self.fsdb.set("qemu-cmdline-%s" % bsp, qemu_cmdline[0]) except KeyError as e: msg = "bad QEMU command line specification: " \ "uninitialized key %s" % e self.log.error(msg) commonl.raise_from(RuntimeError(msg), e) self.log.debug("QEMU cmdline %s" % " ".join(qemu_cmdline)) self.tags['bsps'][bsp]['cmdline'] = " ".join(qemu_cmdline) try: _preexec_fn = getattr(self, "qemu_preexec_fn", None) def _local_preexec_fn(): if _preexec_fn: _preexec_fn() # Open file descriptors for stdout/stderr to a # logfile, because -D is not really working. We # need it to check startup errors for things that # need a retry. commonl.rm_f(errfname) logfd = os.open(errfname, # O_CREAT: Always a new file, so # we can check for errors and not # get confused with previous runs os.O_WRONLY | os.O_EXCL |os.O_CREAT, 0o0644) os.dup2(logfd, 1) os.dup2(logfd, 2) os.close(logfd) p = subprocess.Popen(qemu_cmdline, shell = False, cwd = self.state_dir, close_fds = True, preexec_fn = _local_preexec_fn) self.log.debug("QEMU %s: console @ %s" % (bsp, console_out_fname)) # Give it a few secs to start, the pidfile has been # deleted before starting -- note 4 was found by # ad-hoc experimentation, sometimes depending on system load it # takes more or less. timeout = 10 ts0 = time.time() while True: if time.time() - ts0 > timeout: lines = [] with open(errfname) as f: count = 0 for line in f: lines.append("log: " + line) if count > 5: lines.append("log: ...") break raise RuntimeError("QEMU %s: did not start after %.0fs\n" "%s" % (bsp, timeout, "\n".join(lines))) try: if self._qmp_running(bsp): # FIXME: race condition ttbl.daemon_pid_add(p.pid) return True except RuntimeError as e: self.log.warning("QEMU %s: can't read QMP: %s" % (bsp, str(e))) # fall through, let it retry # Check errors during startup with open(errfname, "r") as logf: causes_for_retry = [ # bah, race condition: since we chose the # port we wanted to use and until we # started using it someone took it. Retry 'Failed to bind socket: Address already in use', ] for line in logf: for cause in causes_for_retry: if cause in line: self.log.info( "QEMU %s: retrying because found in " "logfile: %s", bsp, cause) return False time.sleep(0.25) # nothing runs after this, either it returns or raises except (OSError, ValueError) as e: self.log.debug("QEMU %s: launch failure: %s", bsp, e) raise
def __init__(self, config_text=None, config_files=None, use_ssl=False, tmpdir=None, keep_temp=True, errors_ignore=None, warnings_ignore=None, aka=None, local_auth=True): # Force all assertions, when running like this, to fail the TC tcfl.tc.tc_c.exception_to_result[AssertionError] = tcfl.tc.failed_e # If no aka is defined, we make one out of the place when this # object is being created, so it is always the same *and* thus # the report hashes are always identical with each run if aka == None: self.aka = "ttbd-" + commonl.mkid(commonl.origin_get(2), 4) else: self.aka = aka if config_files == None: config_files = [] self.keep_temp = keep_temp self.port = commonl.tcp_port_assigner() self.use_ssl = use_ssl if use_ssl == True: self.url = "https://localhost:%d" % self.port ssl_context = "" else: self.url = "http://localhost:%d" % self.port ssl_context = "--no-ssl" self.url_spec = "fullid:'^%s'" % self.aka if tmpdir: self.tmpdir = tmpdir else: # default to place the server's dir in the tempdir for # testcases self.tmpdir = os.path.join(tcfl.tc.tc_c.tmpdir, "server", self.aka) shutil.rmtree(self.tmpdir, ignore_errors=True) commonl.makedirs_p(self.tmpdir) self.etc_dir = os.path.join(self.tmpdir, "etc") self.files_dir = os.path.join(self.tmpdir, "files") self.lib_dir = os.path.join(self.tmpdir, "lib") self.state_dir = os.path.join(self.tmpdir, "state") os.mkdir(self.etc_dir) os.mkdir(self.files_dir) os.mkdir(self.lib_dir) os.mkdir(self.state_dir) self.stdout = self.tmpdir + "/stdout" self.stderr = self.tmpdir + "/stderr" for fn in config_files: shutil.copy(fn, self.etc_dir) with open(os.path.join(self.etc_dir, "conf_00_base.py"), "w") as cfgf: cfgf.write(r""" import ttbl.config ttbl.config.processes = 2 host = '127.0.0.1' """) # We don't define here the port, so we see it in the # command line if config_text: cfgf.write(config_text) self.srcdir = os.path.realpath( os.path.join(os.path.dirname(__file__), "..")) self.cmdline = [ "stdbuf", "-o0", "-e0", # This allows us to default to the source location,when # running from source, or the installed when running from # the system os.environ.get("TTBD_PATH", self.srcdir + "/ttbd/ttbd"), "--port", "%d" % self.port, ssl_context, "-vvvvv", "--files-path", self.files_dir, "--state-path", self.state_dir, "--config-path", "", # This empty one is to clear them all "--config-path", self.etc_dir ] self.local_auth = local_auth if local_auth: self.cmdline.append("--local-auth") self.p = None #: Exclude these regexes / strings from triggering an error #: message check self.errors_ignore = [] if errors_ignore == None else errors_ignore #: Exclude these regexes / strings from triggering an warning #: message check self.warnings_ignore = [re.compile('daemon lacks CAP_NET_ADMIN')] if warnings_ignore: self.warnings_ignore += warnings_ignore def _preexec_fn(): stdout_fd = os.open( self.stdout, # O_CREAT: Always a new file, so # we can check for errors and not # get confused with previous runs os.O_WRONLY | os.O_EXCL | os.O_CREAT, 0o0644) stderr_fd = os.open( self.stderr, # O_CREAT: Always a new file, so # we can check for errors and not # get confused with previous runs os.O_WRONLY | os.O_EXCL | os.O_CREAT, 0o0644) os.dup2(stdout_fd, 1) os.dup2(stderr_fd, 2) logging.info("Launching: %s", " ".join(self.cmdline)) self.p = subprocess.Popen(self.cmdline, shell=False, cwd=self.tmpdir, close_fds=True, preexec_fn=_preexec_fn, bufsize=0) try: self._check_if_alive() finally: self.check_log_for_issues() # if we call self.terminate() from __del__, the garbage # collector has started to wipe things, so we can't use, ie: # open() to check the log file atexit.register(self.terminate)
def on(self, target, component): # Start QEMU # # We first assign a port for GDB debugging, if someone takes # it before we do, start will fail and paranoid mode (see # above) will restart it. # FIXME: allocate ports for VNC only if there is a vnc= in the # command line base = ttbl.config.tcp_port_range[0] top = ttbl.config.tcp_port_range[1] if base < 5900: # we need ports >= 5900 for VNC base = 5900 tcp_port_base = commonl.tcp_port_assigner(2, port_range=(base, top)) target.fsdb.set("qemu-gdb-tcp-port", "%s" % tcp_port_base) # This might not be used at all, but we allocate and declare # it in case the implementation will use it; allocating in a # higher layer makes it more complicated. # this one is the port number based on 5900 (VNC default 0) if top < 5900: logging.warning( "ttbl.config.tcp_port_range %s doesn't include ports " "between above 5900, needed for VNC services. " "QEMU targets needing VNC support will fail to start " "complaining about 'vnc-port' not defined", ttbl.config.tcp_port_range) else: # FIXME: move to vnc.vnc0.{host,port,tcp-port} # set this for general information; the VNC screenshotter # also uses it target.fsdb.set("vnc-host", "localhost") target.fsdb.set("vnc-port", "%s" % (tcp_port_base + 1 - 5900)) # this one is the raw port number target.fsdb.set("vnc-tcp-port", "%s" % (tcp_port_base + 1)) # New form target.fsdb.set("vnc.vnc0.host", "localhost") target.fsdb.set("vnc.vnc0.port", "%s" % (tcp_port_base + 1 - 5900)) # this one is the raw port number target.fsdb.set("vnc.vnc0.tcp_port", "%s" % (tcp_port_base + 1)) self.cmdline_extra = [] image_keys = target.fsdb.keys("qemu-image-*") # # Images interface: flash() below has been used to set images # to run, this will feed them into QEMU # if 'qemu-image-bios' in image_keys: self.cmdline_extra += ["-bios", "%(qemu-image-bios)s"] if 'qemu-image-kernel' in image_keys: self.cmdline_extra += ["-kernel", "%(qemu-image-kernel)s"] if 'qemu-image-kernel-args' in image_keys: self.cmdline_extra += ["-append", "%(qemu-image-kernel-args)s"] if 'qemu-image-initrd' in image_keys: self.cmdline_extra += ["-initrd", "%(qemu-image-initrd)s"] # # Network support bits--add command line options to connect # this VM to the networks the target declares # # For each network we are connected to, there must be a # network_tap_pc power rail controller that sets up the # network devices for us before starting QEMU. # # We look at it to setup the QEMU side of things as extra # command line options. # # - network name/icname: comes from the tuntap-ICNAME property # created by the network_tap_pc power rail controller that # the configuration must put on before the qemu power rail # controller. # # - model: comes from what HW it has to emulate; each arch has a # default or you can set for all with property qemu-model or # qemu-model-ICNAME (for that specifc interconnect). # # It has to be a valid QEMU model or QEMU will reject it # with an error. You can find valid models with:: # # qemu-system-ARCH -net nic,model=help # # Zephyr, e.g., supports a few: lan9118 stellaris e1000 (from):: # # $ grep --color -nH --null -rie ETH_NIC_MODEL # drivers/ethernet/Kconfig.smsc911x:13:config ETH_NIC_MODEL$ # drivers/ethernet/Kconfig.stellaris:15:config ETH_NIC_MODEL$ # drivers/ethernet/Kconfig.e1000:16:config ETH_NIC_MODEL$ for tap, if_name in target.fsdb.get_as_dict("tuntap-*").items(): # @tap is tuntap-NETWORK, the property set by # powe rail component network_tap_pc. _, ic_name = tap.split("-", 1) mac_addr = target.tags['interconnects'][ic_name]['mac_addr'] model = commonl.name_make_safe( target.fsdb.get("qemu-model-" + ic_name, target.fsdb.get("qemu-model", self.nic_model))) self.cmdline_extra += [ "-device", # # romfile= UNSET for using virtio-net # - https://wiki.syslinux.org/wiki/index.php?title=Development/Testing # Workaround using virtio f"{model},netdev={ic_name},mac={mac_addr},romfile=", "-netdev", f"tap,id={ic_name},script=no,ifname={if_name}" ] # # Ok, so do actually start # # if the debug port is snatched before we start by someone # else, this will fail and we'll have to retry--this is # why the power component is set to paranoid mode, so the # power interface automatically retries it if it fails. ttbl.power.daemon_c.on(self, target, component) # run console/on steps self._qemu_console_on(target, component) # # Debugging interface--if the target is not in debugging mode, # tell QEMU to start right away # if target.fsdb.get("debug") == None: self.debug_resume(target, component)
def on(self, target, component): # Start QEMU # # We first assign a port for GDB debugging, if someone takes # it before we do, start will fail and paranoid mode (see # above) will restart it. base = ttbl.config.tcp_port_range[0] top = ttbl.config.tcp_port_range[1] if base < 5900: # we need ports >= 5900 for VNC base = 5900 tcp_port_base = commonl.tcp_port_assigner(2, port_range=(base, top)) target.fsdb.set("qemu-gdb-tcp-port", "%s" % tcp_port_base) # This might not be used at all, but we allocate and declare # it in case the implementation will use it; allocating in a # higher layer makes it more complicated. # this one is the port number based on 5900 (VNC default 0) if top < 5900: logging.warning( "ttbl.config.tcp_port_range %s doesn't include ports " "between above 5900, needed for VNC services. " "QEMU targets needing VNC support will fail to start " "complaining about 'vnc-port' not defined", ttbl.config.tcp_port_range) else: # set this for general information; the VNC screenshotter # also uses it target.fsdb.set("vnc-host", "localhost") target.fsdb.set("vnc-port", "%s" % (tcp_port_base + 1 - 5900)) # this one is the raw port number target.fsdb.set("vnc-tcp-port", "%s" % (tcp_port_base + 1)) self.cmdline_extra = [] image_keys = target.fsdb.keys("qemu-image-*") # # Images interface: flash() below has been used to set images # to run, this will feed them into QEMU # if 'qemu-image-bios' in image_keys: self.cmdline_extra += ["-bios", "%(qemu-image-bios)s"] if 'qemu-image-kernel' in image_keys: self.cmdline_extra += ["-kernel", "%(qemu-image-kernel)s"] if 'qemu-image-kernel-args' in image_keys: self.cmdline_extra += ["-append", "%(qemu-image-kernel-args)s"] if 'qemu-image-initrd' in image_keys: self.cmdline_extra += ["-initrd", "%(qemu-image-initrd)s"] # # Network support bits--add command line options to connect # this VM to the networks the target declares # # For each network we are connected to, there must be a # network_tap_pc power rail controller that sets up the # network devices for us before starting QEMU. # # We look at it to setup the QEMU side of things as extra # command line options. # # - network name/icname: comes from the tuntap-ICNAME property # created by the network_tap_pc power rail controller that # the configuration must put on before the qemu power rail # controller. # # - model: comes from what HW it has to emulate; each arch has a # default or you can set for all with property qemu-model or # qemu-model-ICNAME (for that specifc interconnect). # # It has to be a valid QEMU model or QEMU will reject it # with an error. You can find valid models with:: # # qemu-system-ARCH -net nic,model=help # # Zephyr, e.g., supports a few: lan9118 stellaris e1000 (from):: # # $ grep --color -nH --null -rie ETH_NIC_MODEL # drivers/ethernet/Kconfig.smsc911x:13:config ETH_NIC_MODEL$ # drivers/ethernet/Kconfig.stellaris:15:config ETH_NIC_MODEL$ # drivers/ethernet/Kconfig.e1000:16:config ETH_NIC_MODEL$ fds = [] try: for tap, if_name in target.fsdb.get_as_dict( "tuntap-*").iteritems(): # @tap is tuntap-NETWORK, the property set by # powe rail component network_tap_pc. _, ic_name = tap.split("-", 1) mac_addr = target.tags['interconnects'][ic_name]['mac_addr'] model = commonl.name_make_safe( target.fsdb.get( "qemu-model-" + ic_name, target.fsdb.get("qemu-model", self.nic_model))) # There is no way to cahole QEMU to say: hey, we # already have a tap interface configured for you, use # it...other than to open it and pass the file # descriptor to it. # So this finds the /dev/tap* file which matches the # interface created with tuntap- (class # network_tap_pc), opens it and passes the descriptor # along to qemu, then closes the FD, since we don't # need it. tapindex = commonl.if_index(if_name) # Need to wait for udev to reconfigure this for us to have # access; this is done by a udev rule installed # (/usr/lib/udev/rules.d/80-ttbd.rules) tapdevname = "/dev/tap%d" % tapindex ts0 = time.time() ts = ts0 timeout = 5 fd = None while ts - ts0 < timeout: try: fd = os.open(tapdevname, os.O_RDWR, 0) fds.append(fd) break except OSError as e: target.log.error( "%s: couldn't open %s after" " +%.1f/%.1fs, retrying: %s", ic_name, tapdevname, ts - ts0, timeout, e) time.sleep(0.25) ts = time.time() else: raise RuntimeError( "%s: timedout (%.1fs) opening /dev/tap%d's;" " udev's /usr/lib/udev/rules.d/80-ttbd.rules" " didn't set permissions?" % (ic_name, timeout, tapindex)) self.cmdline_extra += [ "-nic", "tap,fd=%d,id=%s,model=%s,mac=%s,br=_b%s" \ % (fd, ic_name, model, mac_addr, ic_name ) ] if not fds: # No tuntap network attachments -> no network # support. Cap it. self.cmdline_extra += ['-nic', 'none'] # # Ok, so do actually start # # if the debug port is snatched before we start by someone # else, this will fail and we'll have to retry--this is # why the power component is set to paranoid mode, so the # power interface automatically retries it if it fails. ttbl.power.daemon_c.on(self, target, component) # Those consoles? update their generational counts so the # clients can know they restarted if hasattr(target, "console"): for console in target.console.impls.keys(): ttbl.console.generation_set(target, console) finally: for fd in fds: # close fds we opened for ... os.close(fd) # ... networking, unneeded now # # Debugging interface--if the target is not in debugging mode, # tell QEMU to start right away # if target.fsdb.get("debug") == None: self.debug_resume(target, component)
def put_tunnel(self, target, who, args, _files, _user_path): """ Setup a TCP/UDP/SCTP v4 or v5 tunnel to the target Parameters are same as :meth:`ttbl.tt_interface.request_process` Parameters specified in the *args* dictionary from the HTTP interface: :param str ip_addr: target's IP address to use (it must be listed on the targets's tags *ipv4_address* or *ipv6_address*). :param int port: port to redirect to :param str protocol: Protocol to tunnel: {udp,sctp,tcp}[{4,6}] :returns dict: dicionary with a single key *result* set ot the *local_port* where to TCP connect to reach the tunnel. """ ip_addr, port, protocol, tunnel_id = self._check_args( self.arg_get(args, 'ip_addr', basestring), self.arg_get(args, 'port', int), self.arg_get(args, 'protocol', basestring), ) self._ip_addr_validate(target, ip_addr) with target.target_owned_and_locked(who): for tunnel_id in target.fsdb.keys("interfaces.tunnel.*.protocol"): prefix = tunnel_id[:-len(".protocol")] _ip_addr = target.fsdb.get(prefix + ".ip_addr") _protocol = target.fsdb.get(prefix + ".protocol") _port = target.fsdb.get(prefix + ".port") _pid = target.fsdb.get(prefix + ".__id") _lport = prefix[len("interfaces.tunnel."):] if _ip_addr == ip_addr \ and _protocol == protocol \ and _port == port \ and commonl.process_alive(_pid, "/usr/bin/socat"): # there is already an active tunnel for this port # and it is alive, so use that return dict(result=int(_lport)) local_port = commonl.tcp_port_assigner( port_range=ttbl.config.tcp_port_range) ip_addr = ipaddress.ip_address(unicode(ip_addr)) if isinstance(ip_addr, ipaddress.IPv6Address): # beacause socat (and most others) likes it like that ip_addr = "[%s]" % ip_addr # this could be refactored using daemon_c, but it'd be # harder to follow the code and it is not really needed. p = subprocess.Popen([ "/usr/bin/socat", "-ly", "-lp", tunnel_id, "%s-LISTEN:%d,fork,reuseaddr" % (protocol, local_port), "%s:%s:%s" % (protocol, ip_addr, port) ], shell=False, cwd=target.state_dir, close_fds=True) pid = commonl.process_started(p.pid, "/usr/bin/socat", verification_f=commonl.tcp_port_busy, verification_f_args=(local_port, ), tag="socat-" + tunnel_id, log=target.log) if p.returncode != None: raise RuntimeError("TUNNEL %s: socat exited with %d" % (tunnel_id, p.returncode)) ttbl.daemon_pid_add(p.pid) # FIXME: race condition if it # died? target.fsdb.set("interfaces.tunnel.%s.__id" % local_port, p.pid) target.fsdb.set("interfaces.tunnel.%s.ip_addr" % local_port, str(ip_addr)) target.fsdb.set("interfaces.tunnel.%s.protocol" % local_port, protocol) target.fsdb.set("interfaces.tunnel.%s.port" % local_port, port) return dict(result=local_port)
def __init__(self, config_text = None, config_files = None, use_ssl = False, tmpdir = None, keep_temp = True, errors_ignore = None, warnings_ignore = None, aka = None): # Force all assertions, when running like this, to fail the TC tcfl.tc.tc_c.exception_to_result[AssertionError] = tcfl.tc.failed_e # If no aka is defined, we make one out of the place when this # object is being created, so it is always the same *and* thus # the report hashes are always identical with each run if aka == None: self.aka = "ttbd-" + commonl.mkid(commonl.origin_get(2), 4) else: self.aka = aka if config_files == None: config_files = [] self.keep_temp = keep_temp self.port = commonl.tcp_port_assigner() self.use_ssl = use_ssl if use_ssl == True: self.url = "https://localhost:%d" % self.port ssl_context = "" else: self.url = "http://localhost:%d" % self.port ssl_context = "--no-ssl" self.url_spec = "url:'^%s'" % self.url if tmpdir: self.tmpdir = tmpdir else: # Don't use colon on the name, or it'll thing it is a path self.tmpdir = tempfile.mkdtemp(prefix = "test-ttbd-%d." % self.port) self.etc_dir = os.path.join(self.tmpdir, "etc") self.files_dir = os.path.join(self.tmpdir, "files") self.lib_dir = os.path.join(self.tmpdir, "lib") self.state_dir = os.path.join(self.tmpdir, "state") os.mkdir(self.etc_dir) os.mkdir(self.files_dir) os.mkdir(self.lib_dir) os.mkdir(self.state_dir) self.stdout = self.tmpdir + "/stdout" self.stderr = self.tmpdir + "/stderr" for fn in config_files: shutil.copy(fn, self.etc_dir) with open(os.path.join(self.etc_dir, "conf_00_base.py"), "w") as cfgf: cfgf.write(r""" import ttbl.config ttbl.config.processes = 2 host = '127.0.0.1' """) # We don't define here the port, so we see it in the # command line if config_text: cfgf.write(config_text) srcdir = os.path.realpath( os.path.join(os.path.dirname(__file__), "..")) self.cmdline = [ # This allows us to default to the source location,when # running from source, or the installed when running from # the system os.environ.get("TTBD_PATH", srcdir + "/ttbd/ttbd"), "--port", "%d" % self.port, ssl_context, "--local-auth", "-vvvvv", "--files-path", self.files_dir, "--state-path", self.state_dir, "--var-lib-path", self.lib_dir, "--config-path", "", # This empty one is to clear them all "--config-path", self.etc_dir ] self.p = None #: Exclude these regexes / strings from triggering an error #: message check self.errors_ignore = [] if errors_ignore == None else errors_ignore #: Exclude these regexes / strings from triggering an warning #: message check self.warnings_ignore = [ re.compile('daemon lacks CAP_NET_ADMIN') ] if warnings_ignore: self.warnings_ignore =+ warnings_ignore def _preexec_fn(): stdout_fd = os.open(self.stdout, # O_CREAT: Always a new file, so # we can check for errors and not # get confused with previous runs os.O_WRONLY | os.O_EXCL |os.O_CREAT, 0o0644) stderr_fd = os.open(self.stderr, # O_CREAT: Always a new file, so # we can check for errors and not # get confused with previous runs os.O_WRONLY | os.O_EXCL |os.O_CREAT, 0o0644) os.dup2(stdout_fd, 1) os.dup2(stderr_fd, 2) logging.info("Launching: %s", " ".join(self.cmdline)) self.p = subprocess.Popen( self.cmdline, shell = False, cwd = self.tmpdir, close_fds = True, preexec_fn = _preexec_fn) self._check_if_alive() self.check_log_for_issues()