def power_on_do(self, target): pidfile = os.path.join(target.state_dir, "socat-" + self.tunnel_id + ".pid") cmdline = [ self.path, "-ly", "-lp", self.tunnel_id, "%s-LISTEN:%d,bind=%s,fork,reuseaddr" % ( self.proto, self.local_port, self.local_addr), "%s:%s:%s" % (self.proto, self.remote_addr, self.remote_port) ] try: p = subprocess.Popen(cmdline, shell = False, cwd = target.state_dir, close_fds = True, stderr = subprocess.STDOUT) with open(pidfile, "w+") as pidf: pidf.write("%s" % p.pid) except OSError as e: raise self.start_e("socat failed to start: %s", e) pid = commonl.process_started( pidfile, self.path, verification_f = commonl.tcp_port_busy, verification_f_args = (self.local_port,), tag = "socat", log = target.log) # systemd might complain with # # Supervising process PID which is not our child. We'll most # likely not notice when it exits. # # Can be ignored if pid == None: raise self.start_e("socat failed to start") ttbl.daemon_pid_add(pid) # FIXME: race condition if it died?
def power_on_do(self, target): pidfile = os.path.join(target.state_dir, "socat-" + self.tunnel_id + ".pid") cmdline = [ self.path, "-ly", "-lp", self.tunnel_id, "%s-LISTEN:%d,bind=%s,fork,reuseaddr" % (self.proto, self.local_port, self.local_addr), "%s:%s:%s" % (self.proto, self.remote_addr, self.remote_port) ] try: p = subprocess.Popen(cmdline, shell=False, cwd=target.state_dir, close_fds=True, stderr=subprocess.STDOUT) with open(pidfile, "w+") as pidf: pidf.write("%s" % p.pid) except OSError as e: raise self.start_e("socat failed to start: %s", e) pid = commonl.process_started(pidfile, self.path, verification_f=commonl.tcp_port_busy, verification_f_args=(self.local_port, ), tag="socat", log=target.log) # systemd might complain with # # Supervising process PID which is not our child. We'll most # likely not notice when it exits. # # Can be ignored if pid == None: raise self.start_e("socat failed to start") ttbl.daemon_pid_add(pid) # FIXME: race condition if it died?
def start(self, target, capturer, path): stream_filename = capturer + ".data.json" log_filename = capturer + ".capture.log" pidfile = "%s/capture-%s.pid" % (target.state_dir, capturer) logf = open(os.path.join(path, log_filename), "w+") p = subprocess.Popen( [ "stdbuf", "-e0", "-o0", self.capture_program, "%s://%s@%s" % (self.url.scheme, self.url.username, self.url.hostname), "environment", # the indexes the command line tool expects are # 1-based, whereas we stored zero based (what the API likes) str(self.outlet_number + 1), os.path.join(path, stream_filename), ], env = { 'RARITAN_PASSWORD': self.password }, bufsize = -1, close_fds = True, shell = False, stderr = subprocess.STDOUT, stdout = logf.buffer, ) with open(pidfile, "w+") as pidf: pidf.write("%s" % p.pid) ttbl.daemon_pid_add(p.pid) return True, { "default": stream_filename, "log": log_filename }
def _power_on_do_openocd_start(self): self.log.action = "openocd start" kws = {} if self.serial != None: kws["serial_string"] = self.serial else: kws["serial_string"] = "MISCONFIGURATION? SERIAL-NUMBER NOT SPECIFIED" # Well, reusing the TCP port range is creating plenty of # problems, as when we kill it and try to restart it, the # sockets are lingering and it fails to reopen it... #tcp_port_base = ttbl.tcp_port_assigner(2 + self.max_target) # Schew it, let's go random -- if it fails, it'll be restarted # with another one tcp_port_base = commonl.tcp_port_assigner(2 + self.max_target, ttbl.config.tcp_port_range) self.log.debug("port base %d" % tcp_port_base) self.tt.fsdb.set("openocd.port", "%d" % tcp_port_base) self.log.debug("port base read: %s" % self.tt.fsdb.get("openocd.port")) args = [ self.openocd_path, "-c", 'tcl_port %d' % (tcp_port_base + 1), "-c", 'telnet_port %d' % tcp_port_base, "--log_output", self.log_name, "-c", 'gdb_port %d' % (tcp_port_base + 2) ] if self.debug: args.append("-d") if self.openocd_scripts: args += ["-s", self.openocd_scripts] if 'interface' in self.board and self.board['interface'] != None: args += ["-f", self.board['interface']] if 'board' in self.board and self.board['board'] != None: if self.openocd_scripts == None: self.openocd_scripts = "" args += [ "-f", os.path.join(self.openocd_scripts, "board", self.board['board'] + ".cfg") ] if self.board['config']: with open(os.path.join(self.tt.state_dir, "openocd.cfg"), "w") \ as cfgf: cfgf.write(self.board['config'] % kws) args += ["-f", cfgf.name] self.log.info("OpenOCD command line %s" % " ".join(args)) self.tt.fsdb.set("openocd.path", commonl.which(self.openocd_path)) p = subprocess.Popen(args, shell=False, cwd=self.tt.state_dir, close_fds=True) # loop for a while until we can connect to it, to prove it's # done initializing self.pid = p.pid self.pid_s = "%d" % p.pid ttbl.daemon_pid_add(self.pid) # FIXME: race condition if it died? # Write a pidfile, as openocd can't do it himself :/ [daemon 101] self.tt.fsdb.set("openocd.pid", self.pid_s)
def _dhcpd_start(self): # Fire up the daemons dhcpd_leases_name = os.path.join(self.state_dir, "dhcpd.leases") # Create the leases file if it doesn't exist with open(dhcpd_leases_name, 'a'): # touch the access/modify time to now os.utime(dhcpd_leases_name, None) if self.ip_mode == 4: ip_mode = "-4" else: ip_mode = "-6" args = [ # Requires CAP_NET_BIND_SERVICE CAP_NET_ADMIN #"strace", "-f", "-s2048", "-o/tmp/kk.log", "dhcpd", "-d", "-q", # Run it in foreground, so the process group owns it and # kills it when exiting "-f", ip_mode, "-cf", os.path.join(self.state_dir, "dhcpd.conf"), "-lf", dhcpd_leases_name, "-pf", self.dhcpd_pidfile, self._params['if_name'], ] logfile_name = os.path.join(self.state_dir, "dhcpd.log") so = open(logfile_name, "wb") try: subprocess.Popen(args, shell=False, cwd=self.state_dir, close_fds=True, stdout=so, stderr=subprocess.STDOUT) except OSError as e: raise self.start_e("DHCPD failed to start: %s", e) pid = commonl.process_started( self.dhcpd_pidfile, self.dhcpd_path, verification_f=os.path.exists, verification_f_args=(self.dhcpd_pidfile, ), tag="dhcpd", log=self.log) # systemd might complain with # # Supervising process PID which is not our child. We'll most # likely not notice when it exits. # # Can be ignored if pid == None: raise self.start_e("dhcpd failed to start") ttbl.daemon_pid_add(pid) # FIXME: race condition if it died?
def power_on_do(self, target): pidfile = os.path.join(target.state_dir, "adb-" + self._id + ".pid") cmdline = [self.path] if self.target_serial_number: # If the thing is connected via USB cmdline += ["-s", self.target_serial_number] cmdline += [ # we are going to listen on this port on all interfaces "-a", "-P", str(self.server_port), "nodaemon", "server" ] try: target.log.error('DEBUG %s' % cmdline) env = dict(os.environ) if self.debug: env['ADB_TRACE'] = "all" p = subprocess.Popen(cmdline, shell=False, cwd=target.state_dir, env=env, close_fds=True, stderr=subprocess.STDOUT) with open(pidfile, "w+") as pidf: pidf.write("%s" % p.pid) except OSError as e: raise self.start_e("adb failed to start: %s", e) pid = commonl.process_started(pidfile, self.path, verification_f=commonl.tcp_port_busy, verification_f_args=(self.server_port, ), tag="adb", log=target.log) # systemd might complain with # # Supervising process PID which is not our child. We'll most # likely not notice when it exits. # # Can be ignored if pid == None: raise self.start_e("adb failed to start") ttbl.daemon_pid_add(pid) # FIXME: race condition if it died? # Connected via TCP/IP? tell the daemon to connect if self.target_port: subprocess.check_output([ self.path, "-H", "localhost", "-P", str(self.server_port), "connect", "%s:%d" % (self.target_port, self.target_port) ]) target.property_set("adb.port", str(self.server_port))
def _power_on_do_openocd_start(self): self.log.action = "openocd start" kws = {} if self.serial != None: kws["serial_string"] = self.serial else: kws["serial_string"] = "MISCONFIGURATION? SERIAL-NUMBER NOT SPECIFIED" # Well, reusing the TCP port range is creating plenty of # problems, as when we kill it and try to restart it, the # sockets are lingering and it fails to reopen it... #tcp_port_base = ttbl.tcp_port_assigner(2 + self.max_target) # Schew it, let's go random -- if it fails, it'll be restarted # with another one tcp_port_base = commonl.tcp_port_assigner(2 + self.max_target, ttbl.config.tcp_port_range) self.log.debug("port base %d" % tcp_port_base) self.tt.fsdb.set("openocd.port", "%d" % tcp_port_base) self.log.debug("port base read: %s" % self.tt.fsdb.get("openocd.port")) args = [ self.openocd_path, "-c", 'tcl_port %d' % (tcp_port_base + 1), "-c", 'telnet_port %d' % tcp_port_base, "--log_output", self.log_name, "-c", 'gdb_port %d' % (tcp_port_base + 2) ] if self.debug: args.append("-d") if self.openocd_scripts: args += [ "-s", self.openocd_scripts ] if 'interface' in self.board and self.board['interface'] != None: args += [ "-f", self.board['interface'] ] if 'board' in self.board and self.board['board'] != None: if self.openocd_scripts == None: self.openocd_scripts = "" args += [ "-f", os.path.join(self.openocd_scripts, "board", self.board['board'] + ".cfg")] if self.board['config']: with open(os.path.join(self.tt.state_dir, "openocd.cfg"), "w") \ as cfgf: cfgf.write(self.board['config'] % kws) args += [ "-f", cfgf.name ] self.log.info("OpenOCD command line %s" % " ".join(args)) self.tt.fsdb.set("openocd.path", commonl.which(self.openocd_path)) p = subprocess.Popen(args, shell = False, cwd = self.tt.state_dir, close_fds = True) # loop for a while until we can connect to it, to prove it's # done initializing self.pid = p.pid self.pid_s = "%d" % p.pid ttbl.daemon_pid_add(self.pid) # FIXME: race condition if it died? # Write a pidfile, as openocd can't do it himself :/ [daemon 101] self.tt.fsdb.set("openocd.pid", self.pid_s)
def power_on_do(self, target): pidfile = os.path.join(target.state_dir, "adb-" + self._id + ".pid") cmdline = [ self.path ] if self.target_serial_number: # If the thing is connected via USB cmdline += [ "-s", self.target_serial_number ] cmdline += [ # we are going to listen on this port on all interfaces "-a", "-P", str(self.server_port), "nodaemon", "server" ] try: target.log.error('DEBUG %s' % cmdline) env = dict(os.environ) if self.debug: env['ADB_TRACE'] = "all" p = subprocess.Popen(cmdline, shell = False, cwd = target.state_dir, env = env, close_fds = True, stderr = subprocess.STDOUT) with open(pidfile, "w+") as pidf: pidf.write("%s" % p.pid) except OSError as e: raise self.start_e("adb failed to start: %s", e) pid = commonl.process_started( pidfile, self.path, verification_f = commonl.tcp_port_busy, verification_f_args = (self.server_port,), tag = "adb", log = target.log) # systemd might complain with # # Supervising process PID which is not our child. We'll most # likely not notice when it exits. # # Can be ignored if pid == None: raise self.start_e("adb failed to start") ttbl.daemon_pid_add(pid) # FIXME: race condition if it died? # Connected via TCP/IP? tell the daemon to connect if self.target_port: subprocess.check_output([ self.path, "-H", "localhost", "-P", str(self.server_port), "connect", "%s:%d" % (self.target_port, self.target_port) ]) target.property_set("adb.port", str(self.server_port))
def _tftpd_start(self): # install stuff we need shutil.copy("/usr/share/syslinux/pxelinux.0", self.pxe_dir) shutil.copy("/usr/share/syslinux/ldlinux.c32", self.pxe_dir) args = [ #"strace", "-f", "-s2048", "-o/tmp/tftpd.log", "in.tftpd", "--foreground", "--secure", "--address", self._params['if_addr'], "--verbosity", "4", # Ensure we run as us, otherwise it tries to go nobody "--user", pwd.getpwuid(os.getuid()).pw_name, "--pid", self.tftpd_pidfile, self.pxe_dir, ] try: so = open(os.path.join(self.state_dir, "tftpd.log"), "wb+") subprocess.Popen(args, shell=False, cwd=self.pxe_dir, close_fds=True, stdout=so, stderr=subprocess.STDOUT) except OSError as e: raise self.tftpd_start_e("TFTPD failed to start: %s", e) pid = commonl.process_started( self.tftpd_pidfile, self.tftpd_path, verification_f=os.path.exists, verification_f_args=(self.tftpd_pidfile, ), tag="tftpd", log=self.log) # systemd might complain with # # Supervising process PID which is not our child. We'll most # likely not notice when it exits. # # Can be ignored if pid == None: raise self.tftpd_start_e("tftpd failed to start") ttbl.daemon_pid_add(pid) # FIXME: race condition if it died?
def _dhcpd_start(self): # Fire up the daemons dhcpd_leases_name = os.path.join(self.state_dir, "dhcpd.leases") # Create the leases file if it doesn't exist with open(dhcpd_leases_name, 'a'): # touch the access/modify time to now os.utime(dhcpd_leases_name, None) if self.ip_mode == 4: ip_mode = "-4" else: ip_mode = "-6" args = [ # Requires CAP_NET_BIND_SERVICE CAP_NET_ADMIN #"strace", "-f", "-s2048", "-o/tmp/kk.log", "dhcpd", "-d", "-q", # Run it in foreground, so the process group owns it and # kills it when exiting "-f", ip_mode, "-cf", os.path.join(self.state_dir, "dhcpd.conf"), "-lf", dhcpd_leases_name, "-pf", self.dhcpd_pidfile, self._params['if_name'], ] logfile_name = os.path.join(self.state_dir, "dhcpd.log") so = open(logfile_name, "wb") try: subprocess.Popen(args, shell = False, cwd = self.state_dir, close_fds = True, stdout = so, stderr = subprocess.STDOUT) except OSError as e: raise self.start_e("DHCPD failed to start: %s", e) pid = commonl.process_started( self.dhcpd_pidfile, self.dhcpd_path, verification_f = os.path.exists, verification_f_args = (self.dhcpd_pidfile,), tag = "dhcpd", log = self.log) # systemd might complain with # # Supervising process PID which is not our child. We'll most # likely not notice when it exits. # # Can be ignored if pid == None: raise self.start_e("dhcpd failed to start") ttbl.daemon_pid_add(pid) # FIXME: race condition if it died?
def on(self, target, component): pidfile = os.path.join(target.state_dir, "socat-" + self.tunnel_id + ".pid") # kill anything that might be left lingering # FIXME: this should use daemon_c commonl.process_terminate(pidfile, path = self.path, tag = "socat") cmdline = [ self.path, "-ly", "-lp", self.tunnel_id, "%s-LISTEN:%d,bind=%s,fork,reuseaddr" % ( self.proto, self.local_port, self.local_addr), "%s:%s:%s" % (self.proto, self.remote_addr, self.remote_port) ] if self.kill_before_on: pids = commonl.kill_by_cmdline(" ".join(cmdline)) if pids: target.log.error( f"BUG? {component}/on: killed PIDs '{pids}'" f" with the same command line: {cmdline}") try: p = subprocess.Popen(cmdline, shell = False, cwd = target.state_dir, close_fds = True, stderr = subprocess.STDOUT) with open(pidfile, "w+") as pidf: pidf.write("%s" % p.pid) except OSError as e: raise self.start_e("socat failed to start: %s", e) pid = commonl.process_started( pidfile, self.path, verification_f = commonl.tcp_port_busy, verification_f_args = (self.local_port,), tag = "socat", log = target.log) # systemd might complain with # # Supervising process PID which is not our child. We'll most # likely not notice when it exits. # # Can be ignored if pid == None: raise self.start_e("socat failed to start") ttbl.daemon_pid_add(pid) # FIXME: race condition if it died?
def _qemu_launch(self, bsp, kws): gdb_tcp_port = commonl.tcp_port_assigner( 1, port_range=ttbl.config.tcp_port_range) self.fsdb.set("debug-%s-gdb-tcp-port" % bsp, "%s" % gdb_tcp_port) console_out_fname = os.path.join(self.state_dir, "console-%s.log" % bsp) errfname = os.path.join(self.state_dir, "%s-stderr.log" % bsp) try: # Make sure we wipe the PID file -- sometimes a pidfile is # left over and it seems to override it, so the reading # becomes corrupt commonl.rm_f(self.pidfile[bsp]) qemu_cmdline = \ (self.qemu_cmdlines[bsp] % kws).split() \ + [ # Don't add -daemonize! This way this is part of # the process tree and killed when we kill the # parent process # We use QMP to find the PTY assigned "-qmp", "unix:%s.qmp,server,nowait" % self.pidfile[bsp], # Always start in debug mode -- this way the # whole thing is stopped until we unleash it # with QMP; this allows us to first start # daemons that we might need to start "-S", "-pidfile", self.pidfile[bsp], "-gdb", "tcp:0.0.0.0:%d" % gdb_tcp_port, ] self.fsdb.set("qemu-cmdline-%s" % bsp, qemu_cmdline[0]) except KeyError as e: msg = "bad QEMU command line specification: " \ "uninitialized key %s" % e self.log.error(msg) commonl.raise_from(RuntimeError(msg), e) self.log.debug("QEMU cmdline %s" % " ".join(qemu_cmdline)) self.tags['bsps'][bsp]['cmdline'] = " ".join(qemu_cmdline) try: _preexec_fn = getattr(self, "qemu_preexec_fn", None) def _local_preexec_fn(): if _preexec_fn: _preexec_fn() # Open file descriptors for stdout/stderr to a # logfile, because -D is not really working. We # need it to check startup errors for things that # need a retry. commonl.rm_f(errfname) logfd = os.open( errfname, # O_CREAT: Always a new file, so # we can check for errors and not # get confused with previous runs os.O_WRONLY | os.O_EXCL | os.O_CREAT, 0o0644) os.dup2(logfd, 1) os.dup2(logfd, 2) os.close(logfd) p = subprocess.Popen(qemu_cmdline, shell=False, cwd=self.state_dir, close_fds=True, preexec_fn=_local_preexec_fn) self.log.debug("QEMU %s: console @ %s" % (bsp, console_out_fname)) # Give it a few secs to start, the pidfile has been # deleted before starting -- note 4 was found by # ad-hoc experimentation, sometimes depending on system load it # takes more or less. timeout = 10 ts0 = time.time() while True: if time.time() - ts0 > timeout: lines = [] with open(errfname) as f: count = 0 for line in f: lines.append("log: " + line) if count > 5: lines.append("log: ...") break raise RuntimeError("QEMU %s: did not start after %.0fs\n" "%s" % (bsp, timeout, "\n".join(lines))) try: if self._qmp_running(bsp): # FIXME: race condition ttbl.daemon_pid_add(p.pid) return True except RuntimeError as e: self.log.warning("QEMU %s: can't read QMP: %s" % (bsp, str(e))) # fall through, let it retry # Check errors during startup with open(errfname, "r") as logf: causes_for_retry = [ # bah, race condition: since we chose the # port we wanted to use and until we # started using it someone took it. Retry 'Failed to bind socket: Address already in use', ] for line in logf: for cause in causes_for_retry: if cause in line: self.log.info( "QEMU %s: retrying because found in " "logfile: %s", bsp, cause) return False time.sleep(0.25) # nothing runs after this, either it returns or raises except (OSError, ValueError) as e: self.log.debug("QEMU %s: launch failure: %s", bsp, e) raise
def put_tunnel(self, target, who, args, _files, _user_path): """ Setup a TCP/UDP/SCTP v4 or v5 tunnel to the target Parameters are same as :meth:`ttbl.tt_interface.request_process` Parameters specified in the *args* dictionary from the HTTP interface: :param str ip_addr: target's IP address to use (it must be listed on the targets's tags *ipv4_address* or *ipv6_address*). :param int port: port to redirect to :param str protocol: Protocol to tunnel: {udp,sctp,tcp}[{4,6}] :returns dict: dicionary with a single key *result* set ot the *local_port* where to TCP connect to reach the tunnel. """ ip_addr, port, protocol, tunnel_id = self._check_args( self.arg_get(args, 'ip_addr', basestring), self.arg_get(args, 'port', int), self.arg_get(args, 'protocol', basestring), ) self._ip_addr_validate(target, ip_addr) with target.target_owned_and_locked(who): for tunnel_id in target.fsdb.keys("interfaces.tunnel.*.protocol"): prefix = tunnel_id[:-len(".protocol")] _ip_addr = target.fsdb.get(prefix + ".ip_addr") _protocol = target.fsdb.get(prefix + ".protocol") _port = target.fsdb.get(prefix + ".port") _pid = target.fsdb.get(prefix + ".__id") _lport = prefix[len("interfaces.tunnel."):] if _ip_addr == ip_addr \ and _protocol == protocol \ and _port == port \ and commonl.process_alive(_pid, "/usr/bin/socat"): # there is already an active tunnel for this port # and it is alive, so use that return dict(result=int(_lport)) local_port = commonl.tcp_port_assigner( port_range=ttbl.config.tcp_port_range) ip_addr = ipaddress.ip_address(unicode(ip_addr)) if isinstance(ip_addr, ipaddress.IPv6Address): # beacause socat (and most others) likes it like that ip_addr = "[%s]" % ip_addr # this could be refactored using daemon_c, but it'd be # harder to follow the code and it is not really needed. p = subprocess.Popen([ "/usr/bin/socat", "-ly", "-lp", tunnel_id, "%s-LISTEN:%d,fork,reuseaddr" % (protocol, local_port), "%s:%s:%s" % (protocol, ip_addr, port) ], shell=False, cwd=target.state_dir, close_fds=True) pid = commonl.process_started(p.pid, "/usr/bin/socat", verification_f=commonl.tcp_port_busy, verification_f_args=(local_port, ), tag="socat-" + tunnel_id, log=target.log) if p.returncode != None: raise RuntimeError("TUNNEL %s: socat exited with %d" % (tunnel_id, p.returncode)) ttbl.daemon_pid_add(p.pid) # FIXME: race condition if it # died? target.fsdb.set("interfaces.tunnel.%s.__id" % local_port, p.pid) target.fsdb.set("interfaces.tunnel.%s.ip_addr" % local_port, str(ip_addr)) target.fsdb.set("interfaces.tunnel.%s.protocol" % local_port, protocol) target.fsdb.set("interfaces.tunnel.%s.port" % local_port, port) return dict(result=local_port)
def power_on_do(self, target): """ Start the daemon, generating first the config file """ file_prefix = os.path.join( target.state_dir, "rsync-%s:%d" % (self.address, self.port)) pidfile = file_prefix + ".pid" with open(file_prefix + ".conf", "w+") as conff: conff.write("""\ # We run the daemon as root and need to run as root so we can access # folders that have root-only weird permissions # FIXME: we could also CAP_DAC_READ_SEARCH or similar [images] path = {0.share_path} read only = {0.read_only} timeout = 300 """.format(self)) if self.uid: conff.write("uid = %s" % self.uid) if self.gid: conff.write("gid = %s" % self.gid) def _preexec_fn(): # We need this to access image files to serve that are # owned by root (because that's what the image is and we # want to share them with the same bits--even if we mapped # user to something else, some attributes and different # user bits would force us to do something like this) prctl.cap_effective.dac_read_search = True # rsync chroots for safety prctl.cap_effective.sys_chroot = True return cmdline = [ "rsync", "--daemon", "--no-detach", "--address", self.address, "--port", str(self.port), "--config", file_prefix + ".conf" ] try: p = subprocess.Popen( cmdline, shell = False, cwd = target.state_dir, close_fds = True, stderr = subprocess.STDOUT, preexec_fn = _preexec_fn) with open(pidfile, "w+") as pidf: pidf.write("%s" % p.pid) except OSError as e: raise self.start_e("rsync failed to start: %s" % e) pid = commonl.process_started( pidfile, self.path, verification_f = commonl.tcp_port_busy, verification_f_args = (self.port,), tag = "rsync", log = target.log) # systemd might complain with # # Supervising process PID which is not our child. We'll most # likely not notice when it exits. # # Can be ignored if pid == None: raise self.start_e("rsync failed to start") ttbl.daemon_pid_add(pid) # FIXME: race condition if it died?
def on(self, target, component): stderrf_name = os.path.join(target.state_dir, component + "-" + self.name + ".stderr") kws = dict(target.kws) kws.update(self.kws) kws['component'] = component # render the real commandline against kws _cmdline = [] for i in self.cmdline: # some older Linux distros complain if this string is unicode _cmdline.append(str(i % kws)) target.log.info("%s: command line: %s" % (component, " ".join(_cmdline))) if self.env_add: env = dict(os.environ) env.update(self.env_add) else: env = os.environ pidfile = self.pidfile % kws commonl.rm_f(pidfile) stderrf = open(stderrf_name, "w+") try: p = subprocess.Popen(_cmdline, env=env, cwd=target.state_dir, stderr=stderrf, bufsize=0, shell=False, universal_newlines=False) if self.mkpidfile: with open(pidfile, "w+") as pidf: pidf.write("%s" % p.pid) except TypeError as e: # This happens on misconfiguration ## TypeError: execve() arg 3 contains a non-string value if 'execve() arg 3' in str(e): target.log.exception( "Ensure environment settings are not set to None", e) if 'execve()' in str(e): target.log.exception("Possible target misconfiguration: %s", e) count = 0 for i in _cmdline: target.log.error("cmdline %d: [%s] %s", count, type(i).__name__, i) count += 1 for key, val in env.iteritems(): target.log.error("env %s: [%s] %s", key, type(val).__name__, val) raise except OSError as e: raise self.start_e("%s: %s failed to start: %s" % (component, self.name, e)) del stderrf # we don't care for this file here if self.precheck_wait: time.sleep(self.precheck_wait) pid = commonl.process_started(pidfile, self.path, component + "-" + self.name, target.log, self.verify, (_cmdline, )) if pid == None: raise self.start_e("%s: %s failed to start" % (component, self.name)) ttbl.daemon_pid_add(pid)
def _qemu_launch(self, bsp, kws): gdb_tcp_port = commonl.tcp_port_assigner( 1, port_range = ttbl.config.tcp_port_range) self.fsdb.set("debug-%s-gdb-tcp-port" % bsp, "%s" % gdb_tcp_port) console_out_fname = os.path.join( self.state_dir, "console-%s.log" % bsp) errfname = os.path.join( self.state_dir, "%s-stderr.log" % bsp) try: # Make sure we wipe the PID file -- sometimes a pidfile is # left over and it seems to override it, so the reading # becomes corrupt commonl.rm_f(self.pidfile[bsp]) qemu_cmdline = \ (self.qemu_cmdlines[bsp] % kws).split() \ + [ # Don't add -daemonize! This way this is part of # the process tree and killed when we kill the # parent process # We use QMP to find the PTY assigned "-qmp", "unix:%s.qmp,server,nowait" % self.pidfile[bsp], # Always start in debug mode -- this way the # whole thing is stopped until we unleash it # with QMP; this allows us to first start # daemons that we might need to start "-S", "-pidfile", self.pidfile[bsp], "-gdb", "tcp:0.0.0.0:%d" % gdb_tcp_port, ] self.fsdb.set("qemu-cmdline-%s" % bsp, qemu_cmdline[0]) except KeyError as e: msg = "bad QEMU command line specification: " \ "uninitialized key %s" % e self.log.error(msg) commonl.raise_from(RuntimeError(msg), e) self.log.debug("QEMU cmdline %s" % " ".join(qemu_cmdline)) self.tags['bsps'][bsp]['cmdline'] = " ".join(qemu_cmdline) try: _preexec_fn = getattr(self, "qemu_preexec_fn", None) def _local_preexec_fn(): if _preexec_fn: _preexec_fn() # Open file descriptors for stdout/stderr to a # logfile, because -D is not really working. We # need it to check startup errors for things that # need a retry. commonl.rm_f(errfname) logfd = os.open(errfname, # O_CREAT: Always a new file, so # we can check for errors and not # get confused with previous runs os.O_WRONLY | os.O_EXCL |os.O_CREAT, 0o0644) os.dup2(logfd, 1) os.dup2(logfd, 2) os.close(logfd) p = subprocess.Popen(qemu_cmdline, shell = False, cwd = self.state_dir, close_fds = True, preexec_fn = _local_preexec_fn) self.log.debug("QEMU %s: console @ %s" % (bsp, console_out_fname)) # Give it a few secs to start, the pidfile has been # deleted before starting -- note 4 was found by # ad-hoc experimentation, sometimes depending on system load it # takes more or less. timeout = 10 ts0 = time.time() while True: if time.time() - ts0 > timeout: lines = [] with open(errfname) as f: count = 0 for line in f: lines.append("log: " + line) if count > 5: lines.append("log: ...") break raise RuntimeError("QEMU %s: did not start after %.0fs\n" "%s" % (bsp, timeout, "\n".join(lines))) try: if self._qmp_running(bsp): # FIXME: race condition ttbl.daemon_pid_add(p.pid) return True except RuntimeError as e: self.log.warning("QEMU %s: can't read QMP: %s" % (bsp, str(e))) # fall through, let it retry # Check errors during startup with open(errfname, "r") as logf: causes_for_retry = [ # bah, race condition: since we chose the # port we wanted to use and until we # started using it someone took it. Retry 'Failed to bind socket: Address already in use', ] for line in logf: for cause in causes_for_retry: if cause in line: self.log.info( "QEMU %s: retrying because found in " "logfile: %s", bsp, cause) return False time.sleep(0.25) # nothing runs after this, either it returns or raises except (OSError, ValueError) as e: self.log.debug("QEMU %s: launch failure: %s", bsp, e) raise
def start(self, target, capturer, path): commonl.makedirs_p(path) stream_filename = "%s%s" % (capturer, self.extension) log_filename = "%s.log" % capturer pidfile = "%s/capture-%s.pid" % (target.state_dir, capturer) kws = target.kws_collect(self) kws['output_file_name'] = os.path.join(path, stream_filename) # LEGACY kws['stream_filename'] = os.path.join(path, stream_filename) # LEGACY kws['_impl.stream_filename'] = os.path.join(path, stream_filename) kws['_impl.log_filename'] = os.path.join(path, log_filename) kws['_impl.capturer'] = capturer kws['_impl.timestamp'] = str(datetime.datetime.utcnow()) with open(kws['_impl.log_filename'], "w+") as logf: logf.write( commonl.kws_expand( """\ INFO: ttbd running generic_stream capture for '%(_impl.capturer)s' at %(_impl.timestamp)s INFO: log_file (this file): %(_impl.log_filename)s INFO: stream_file: %(_impl.stream_filename)s """, kws)) try: for command in self.pre_commands: # yup, run with shell -- this is not a user level # command, the configurator has full control pre_command = commonl.kws_expand(command, kws) logf.write("INFO: calling pre-command: %s\n" % pre_command) logf.flush() subprocess.check_call(pre_command, shell=True, close_fds=True, cwd="/tmp", stdout=logf, stderr=subprocess.STDOUT) cmdline = [] for i in self.cmdline: cmdline.append(commonl.kws_expand(i, kws)) target.log.info("%s: stream command: %s" % (capturer, " ".join(cmdline))) logf.write("INFO: calling commandline: %s\n" % " ".join(cmdline)) logf.flush() p = subprocess.Popen(cmdline, cwd="/tmp", shell=False, close_fds=True, stdout=logf, stderr=subprocess.STDOUT) target.log.info("%s: generic streaming started" % capturer) time.sleep(1) # let it settle or fail if p.poll() != None: logf.close() return False, {"log": log_filename} except subprocess.CalledProcessError as e: target.log.error( "%s: capturing of '%s' with '%s' failed: (%d) %s" % (capturer, self.name, " ".join( e.cmd), e.returncode, e.output)) logf.write("ERROR: capture failed\n") raise with open(pidfile, "w+") as pidf: pidf.write("%s" % p.pid) ttbl.daemon_pid_add(p.pid) return True, {"default": stream_filename, "log": log_filename}
def on(self, target, _component): # Bring up the lower network interface; lower is called # whatever (if it is a physical device) or _bNAME; bring it # up, make it promiscuous mode = self._get_mode(target) if mode == 'vlan': # our lower is a physical device, our upper is a device # which till tag for eth vlan %(vlan) ifname = commonl.if_find_by_mac(target.tags['mac_addr'], physical=True) commonl.if_remove_maybe("b%(id)s" % target.kws) kws = dict(target.kws) kws['ifname'] = ifname subprocess.check_call( "/usr/sbin/ip link add" " link %(ifname)s name b%(id)s" " type vlan id %(vlan)s" #" protocol VLAN_PROTO" #" reorder_hdr on|off" #" gvrp on|off mvrp on|off loose_binding on|off" % kws, shell=True) subprocess.check_call( # bring lower up "/usr/sbin/ip link set dev %s up promisc on" % ifname, shell=True) elif mode == 'physical': ifname = commonl.if_find_by_mac(target.tags['mac_addr']) subprocess.check_call( # bring lower up "/usr/sbin/ip link set dev %s up promisc on" % ifname, shell=True) self._if_rename(target) elif mode == 'virtual': # We do not have a physical device, a bridge, to serve as # lower commonl.if_remove_maybe("_b%(id)s" % target.kws) subprocess.check_call("/usr/sbin/ip link add" " name _b%(id)s" " type bridge" % target.kws, shell=True) subprocess.check_call("/usr/sbin/ip link add" " link _b%(id)s name b%(id)s" " type macvlan mode bridge; " % target.kws, shell=True) subprocess.check_call( # bring lower up "/usr/sbin/ip link set" " dev _b%(id)s" " up promisc on" % target.kws, shell=True) else: raise AssertionError("Unknown mode %s" % mode) # Configure the IP addresses for the top interface subprocess.check_call( # clean up existing address "/usr/sbin/ip add flush dev b%(id)s " % target.kws, shell=True) subprocess.check_call( # add IPv6 # if this fails, check Network Manager hasn't disabled ipv6 # sysctl -a | grep disable_ipv6 must show all to 0 "/usr/sbin/ip addr add" " %(ipv6_addr)s/%(ipv6_prefix_len)s dev b%(id)s " % target.kws, shell=True) subprocess.check_call( # add IPv4 "/usr/sbin/ip addr add" " %(ipv4_addr)s/%(ipv4_prefix_len)d" " dev b%(id)s" % target.kws, shell=True) # Bring up the top interface, which sets up ther outing subprocess.check_call( "/usr/sbin/ip link set dev b%(id)s up promisc on" % target.kws, shell=True) target.fsdb.set('power_state', 'on') # Start tcpdump on the network? # # The value of the tcpdump property, if not None, is the # filename we'll capture to. tcpdump = target.fsdb.get('tcpdump') if tcpdump: assert not os.path.sep in tcpdump \ and tcpdump != "" \ and tcpdump != os.path.pardir \ and tcpdump != os.path.curdir, \ "Bad filename for TCP dump capture '%s' specified as " \ " value to property *tcpdump*: must not include" % tcpdump # per ttbd:make_ticket(), colon splits the real username # from the ticket owner = target.owner_get().split(":")[0] assert owner, "BUG? target not owned on power on?" capfile = os.path.join(target.files_path, owner, tcpdump) # Because it is in the user's area, # we assume the user knows what he is doing to overwrite it, # so we'll remove any first commonl.rm_f(capfile) pidfile = os.path.join(target.state_dir, "tcpdump.pid") logfile = os.path.join(target.state_dir, "tcpdump.log") cmdline = [ "/usr/sbin/tcpdump", "-U", "-i", "_b%(id)s" % target.kws, "-w", capfile ] try: logf = open(logfile, "a") target.log.info("Starting tcpdump with: %s", " ".join(cmdline)) p = subprocess.Popen(cmdline, shell=False, cwd=target.state_dir, close_fds=True, stdout=logf, stderr=subprocess.STDOUT) except OSError as e: raise RuntimeError("tcpdump failed to start: %s" % e) ttbl.daemon_pid_add(p.pid) # FIXME: race condition if it died? with open(pidfile, "w") as pidfilef: pidfilef.write("%d" % p.pid) pid = commonl.process_started( # Verify it started pidfile, "/usr/sbin/tcpdump", verification_f=os.path.exists, verification_f_args=(capfile, ), timeout=20, tag="tcpdump", log=target.log) if pid == None: raise RuntimeError("tcpdump failed to start after 5s")