def execute_timeout(instr, timeout, *popenargs, **popenargsk): '''Popen wrapper with timeout supervision If instr is given, it is fed into stdin, otherwise stdin will be /dev/null. Return (status, stdout, stderr) ''' adtlog.debug('execute-timeout: ' + ' '.join(popenargs[0])) if instr is None: popenargsk['stdin'] = devnull_read else: instr = instr.encode('UTF-8') sp = subprocess.Popen(*popenargs, **popenargsk) timeout_start(timeout) try: (out, err) = sp.communicate(instr) if out is not None: out = out.decode('UTF-8', 'replace') if err is not None: err = err.decode('UTF-8', 'replace') except Timeout: try: sp.kill() sp.wait() except OSError as e: adtlog.error('WARNING: Cannot kill timed out process %s: %s' % (popenargs[0], e)) raise timeout_stop() status = sp.wait() return (status, out, err)
def cmd_shell(c, ce): cmdnumargs(c, ce, 1, None) if not downtmp: bomb("`shell' when not open") # runners can provide a hook if they need a special treatment try: caller.hook_shell(*c[1:]) except AttributeError: adtlog.debug('cmd_shell: using default shell command, dir %s' % c[1]) cmd = 'cd "%s"; ' % c[1] for e in c[2:]: cmd += 'export "%s"; ' % e # use host's $TERM to provide a sane shell try: cmd += 'export TERM="%s"; ' % os.environ['TERM'] except KeyError: pass cmd += 'bash -i' try: with open('/dev/tty', 'rb') as sin: with open('/dev/tty', 'wb') as sout: with open('/dev/tty', 'wb') as serr: subprocess.call(auxverb + ['sh', '-c', cmd], stdin=sin, stdout=sout, stderr=serr) except (OSError, IOError) as e: adtlog.error('Cannot run shell: %s' % e)
def publish(self): if not self.registered: adtlog.debug('Binaries: no registered binaries, not publishing anything') return adtlog.debug('Binaries: publish') try: with open(os.path.join(self.dir.host, 'Packages'), 'w') as f: subprocess.check_call(['apt-ftparchive', 'packages', '.'], cwd=self.dir.host, stdout=f) with open(os.path.join(self.dir.host, 'Release'), 'w') as f: subprocess.call(['apt-ftparchive', 'release', '.'], cwd=self.dir.host, stdout=f) except subprocess.CalledProcessError as e: adtlog.bomb('apt-ftparchive failed: %s' % e) # copy binaries directory to testbed; self.dir.tb might have changed # since last time due to a reset, so update it self.dir.tb = os.path.join(self.testbed.scratch, 'binaries') self.testbed.check_exec(['rm', '-rf', self.dir.tb]) self.dir.copydown() aptupdate_out = adt_testbed.TempPath(self.testbed, 'apt-update.out') script = ''' printf 'Package: *\\nPin: origin ""\\nPin-Priority: 1002\\n' > /etc/apt/preferences.d/90autopkgtest echo "deb [trusted=yes] file://%(d)s /" >/etc/apt/sources.list.d/autopkgtest.list if [ "x`ls /var/lib/dpkg/updates`" != x ]; then echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1 fi apt-get --quiet --no-list-cleanup -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/autopkgtest.list -o Dir::Etc::sourceparts=/dev/null update 2>&1 cp /var/lib/dpkg/status %(o)s ''' % {'d': self.dir.tb, 'o': aptupdate_out.tb} self.need_apt_reset = True self.testbed.check_exec(['sh', '-ec', script], kind='install') aptupdate_out.copyup() adtlog.debug('Binaries: publish reinstall checking...') pkgs_reinstall = set() pkg = None for l in open(aptupdate_out.host, encoding='UTF-8'): if l.startswith('Package: '): pkg = l[9:].rstrip() elif l.startswith('Status: install '): if pkg in self.registered: pkgs_reinstall.add(pkg) adtlog.debug('Binaries: publish reinstall needs ' + pkg) if pkgs_reinstall: rc = self.testbed.execute( ['apt-get', '--quiet', '-o', 'Debug::pkgProblemResolver=true', '-o', 'APT::Get::force-yes=true', '-o', 'APT::Get::Assume-Yes=true', '--reinstall', 'install'] + list(pkgs_reinstall), kind='install')[0] if rc: adtlog.badpkg('installation of basic binaries failed, exit code %d' % rc) adtlog.debug('Binaries: publish done')
def cleanup(): global downtmp, cleaning adtlog.debug("cleanup...") sethandlers(signal.SIG_DFL) # avoid recursion if something bomb()s in hook_cleanup() if not cleaning: cleaning = True if downtmp: caller.hook_cleanup() cleaning = False downtmp = None
def cleanup(): global downtmp, cleaning adtlog.debug("cleanup...") sethandlers(signal.SIG_DFL) # avoid recursion if something bomb()s in hook_cleanup() if not cleaning: cleaning = True if downtmp: caller.hook_cleanup() cleaning = False downtmp = None
def cmd_open(c, ce): global auxverb, downtmp, downtmp_open cmdnumargs(c, ce) if downtmp: bomb("`open' when already open") caller.hook_open() adtlog.debug("auxverb = %s, downtmp = %s" % (str(auxverb), downtmp)) downtmp = caller.hook_downtmp(downtmp_open) if downtmp_open and downtmp_open != downtmp: bomb('virt-runner failed to restore downtmp path %s, gave %s instead' % (downtmp_open, downtmp)) downtmp_open = downtmp return [downtmp]
def cmd_open(c, ce): global auxverb, downtmp, downtmp_open cmdnumargs(c, ce) if downtmp: bomb("`open' when already open") caller.hook_open() adtlog.debug("auxverb = %s, downtmp = %s" % (str(auxverb), downtmp)) downtmp = caller.hook_downtmp(downtmp_open) if downtmp_open and downtmp_open != downtmp: bomb('virt-runner failed to restore downtmp path %s, gave %s instead' % (downtmp_open, downtmp)) downtmp_open = downtmp return [downtmp]
def cmd_reboot(c, ce): global downtmp cmdnumargs(c, ce, 0, 1) if not downtmp: bomb("`reboot' when not open") if 'reboot' not in caller.hook_capabilities(): bomb("`reboot' when `reboot' not advertised") # save current downtmp; try a few locations, as /var/cache might be r/o # (argh Ubuntu touch) directories = '/var/cache /home' check_exec([ 'sh', '-ec', 'for d in %s; do if [ -w $d ]; then ' ' tar --warning=none --create --absolute-names ' ''' -f $d/autopkgtest-tmpdir.tar '%s'; ''' ' rm -f /run/autopkgtest-reboot-prepare-mark; ' ' exit 0; fi; done; exit 1' '' % (directories, downtmp) ], downp=True, timeout=copy_timeout) adtlog.debug('cmd_reboot: saved current downtmp, rebooting') try: caller.hook_prepare_reboot() except AttributeError: pass # reboot if len(c) > 1 and c[1] == 'prepare-only': adtlog.info('state saved, waiting for testbed to reboot...') else: execute_timeout( None, 30, auxverb + ['sh', '-c', '(sleep 3; reboot) >/dev/null 2>&1 &']) caller.hook_wait_reboot() # restore downtmp check_exec([ 'sh', '-ec', 'for d in %s; do ' 'if [ -e $d/autopkgtest-tmpdir.tar ]; then ' ' tar --warning=none --extract --absolute-names ' ' -f $d/autopkgtest-tmpdir.tar;' ' rm $d/autopkgtest-tmpdir.tar; exit 0; ' 'fi; done; exit 1' % directories ], downp=True, timeout=copy_timeout) adtlog.debug('cmd_reboot: restored downtmp after reboot')
def expect(sock, search_bytes, timeout_sec, description=None): adtlog.debug('expect: "%s"' % search_bytes.decode()) what = '"%s"' % (description or search_bytes or 'data') out = b'' with timeout(timeout_sec, description and ('timed out waiting for %s' % what) or None): while True: time.sleep(0.1) block = sock.recv(4096) # adtlog.debug('expect: got block: %s' % block) out += block if search_bytes is None or search_bytes in out: adtlog.debug('expect: found "%s"' % what) break
def copydown_shareddir(host, tb, is_dir, downtmp_host): adtlog.debug( 'copydown_shareddir: host %s tb %s is_dir %s downtmp_host %s' % (host, tb, is_dir, downtmp_host)) host = os.path.normpath(host) tb = os.path.normpath(tb) downtmp_host = os.path.normpath(downtmp_host) timeout_start(copy_timeout) try: host_tmp = None if host.startswith(downtmp_host): # translate into tb path host = downtmp + host[len(downtmp_host):] else: host_tmp = os.path.join(downtmp_host, os.path.basename(tb)) if is_dir: if os.path.exists(host_tmp): try: shutil.rmtree(host_tmp) except OSError as e: adtlog.warning('cannot remove old %s, moving it ' 'instead: %s' % (host_tmp, e)) # some undeletable files? hm, move it aside instead counter = 0 while True: p = host_tmp + '.old%i' % counter if not os.path.exists(p): os.rename(host_tmp, p) break counter += 1 shutil.copytree(host, host_tmp, symlinks=True) else: shutil.copy(host, host_tmp) # translate into tb path host = os.path.join(downtmp, os.path.basename(tb)) if host == tb: host_tmp = None else: check_exec(['rm', '-rf', tb], downp=True) check_exec(['cp', '-r', '--preserve=timestamps,links', host, tb], downp=True) if host_tmp: (is_dir and shutil.rmtree or os.unlink)(host_tmp) finally: timeout_stop()
def cmd_revert(c, ce): global auxverb, downtmp, downtmp_open cmdnumargs(c, ce) if not downtmp: bomb("`revert' when not open") if 'revert' not in caller.hook_capabilities(): bomb("`revert' when `revert' not advertised") caller.hook_revert() downtmp = caller.hook_downtmp(downtmp_open) if downtmp_open and downtmp_open != downtmp: bomb('virt-runner failed to restore downtmp path %s, gave %s instead' % (downtmp_open, downtmp)) adtlog.debug("auxverb = %s, downtmp = %s" % (str(auxverb), downtmp)) return [downtmp]
def cmd_revert(c, ce): global auxverb, downtmp, downtmp_open cmdnumargs(c, ce) if not downtmp: bomb("`revert' when not open") if 'revert' not in caller.hook_capabilities(): bomb("`revert' when `revert' not advertised") caller.hook_revert() downtmp = caller.hook_downtmp(downtmp_open) if downtmp_open and downtmp_open != downtmp: bomb('virt-runner failed to restore downtmp path %s, gave %s instead' % (downtmp_open, downtmp)) adtlog.debug("auxverb = %s, downtmp = %s" % (str(auxverb), downtmp)) return [downtmp]
def copydown_shareddir(host, tb, is_dir, downtmp_host): adtlog.debug('copydown_shareddir: host %s tb %s is_dir %s downtmp_host %s' % (host, tb, is_dir, downtmp_host)) host = os.path.normpath(host) tb = os.path.normpath(tb) downtmp_host = os.path.normpath(downtmp_host) timeout_start(copy_timeout) try: host_tmp = None if host.startswith(downtmp_host): # translate into tb path host = downtmp + host[len(downtmp_host):] else: host_tmp = os.path.join(downtmp_host, os.path.basename(tb)) if is_dir: if os.path.exists(host_tmp): try: shutil.rmtree(host_tmp) except OSError as e: adtlog.warning('cannot remove old %s, moving it ' 'instead: %s' % (host_tmp, e)) # some undeletable files? hm, move it aside instead counter = 0 while True: p = host_tmp + '.old%i' % counter if not os.path.exists(p): os.rename(host_tmp, p) break counter += 1 shutil.copytree(host, host_tmp, symlinks=True) else: shutil.copy(host, host_tmp) # translate into tb path host = os.path.join(downtmp, os.path.basename(tb)) if host == tb: host_tmp = None else: check_exec(['rm', '-rf', tb], downp=True) check_exec(['cp', '-r', '--preserve=timestamps,links', host, tb], downp=True) if host_tmp: (is_dir and shutil.rmtree or os.unlink)(host_tmp) finally: timeout_stop()
def cmd_shell(c, ce): cmdnumargs(c, ce, 1, None) if not downtmp: bomb("`shell' when not open") # runners can provide a hook if they need a special treatment try: caller.hook_shell(*c[1:]) except AttributeError: adtlog.debug('cmd_shell: using default shell command, dir %s' % c[1]) cmd = 'cd "%s"; ' % c[1] for e in c[2:]: cmd += 'export "%s"; ' % e cmd += 'bash -i' with open('/dev/tty', 'rb') as sin: with open('/dev/tty', 'wb') as sout: with open('/dev/tty', 'wb') as serr: subprocess.call(auxverb + ['sh', '-c', cmd], stdin=sin, stdout=sout, stderr=serr)
def __init__(self, testbed, output_dir): adtlog.debug('Binaries: initialising') self.testbed = testbed self.output_dir = output_dir # the binary dir must exist across testbed reopenings, so don't use a # TempPath self.dir = adt_testbed.Path( self.testbed, os.path.join(self.output_dir, 'binaries'), os.path.join(self.testbed.scratch, 'binaries'), is_dir=True) os.mkdir(self.dir.host) self.registered = set() # clean up an empty binaries output dir atexit.register(lambda: os.path.exists(self.dir.host) and ( os.listdir(self.dir.host) or os.rmdir(self.dir.host))) self.need_apt_reset = False
def expect(sock, search_bytes, timeout_sec, description=None, echo=False): adtlog.debug('expect: "%s"' % (search_bytes or b'<none>').decode()) what = '"%s"' % (description or search_bytes or 'data') out = b'' with timeout(timeout_sec, description and ('timed out waiting for %s' % what) or None): while True: block = sock.recv(4096) if not block: time.sleep(0.1) continue if echo: sys.stderr.buffer.write(block) out += block if search_bytes is None or search_bytes in out: adtlog.debug('expect: found "%s"' % what) break return out
def _autodep8(srcdir): '''Generate control file with autodep8''' f = tempfile.NamedTemporaryFile(prefix='autodep8.') try: autodep8 = subprocess.Popen(['autodep8'], cwd=srcdir, stdout=f, stderr=subprocess.PIPE) except OSError as e: adtlog.debug('autodep8 not available (%s)' % e) return None err = autodep8.communicate()[1].decode() if autodep8.returncode == 0: f.flush() f.seek(0) ctrl = f.read().decode() adtlog.debug('autodep8 generated control: -----\n%s\n-------' % ctrl) return f f.close() adtlog.debug('autodep8 failed to generate control (exit status %i): %s' % (autodep8.returncode, err)) return None
def _parse_debian_depends(testname, dep_str, srcdir): '''Parse Depends: line in a Debian package Split dependencies (comma separated), validate their syntax, and expand @ and @builddeps@. Return a list of dependencies. This may raise an InvalidControl exception if there are invalid dependencies. ''' deps = [] for alt_group_str in dep_str.split(','): alt_group_str = alt_group_str.strip() if not alt_group_str: # happens for empty depends or trailing commas continue adtlog.debug('processing dependency %s' % alt_group_str) if alt_group_str == '@': for d in _debian_packages_from_source(srcdir): adtlog.debug('synthesised dependency %s' % d) deps.append(d) elif alt_group_str == '@builddeps@': for d in _debian_build_deps_from_source(srcdir): adtlog.debug('synthesised dependency %s' % d) deps.append(d) else: for dep in alt_group_str.split('|'): _debian_check_dep(testname, dep) deps.append(alt_group_str) return deps
def _parse_debian_depends(testname, dep_str, srcdir, testbed_arch): '''Parse Depends: line in a Debian package Split dependencies (comma separated), validate their syntax, and expand @ and @builddeps@. Return a list of dependencies. This may raise an InvalidControl exception if there are invalid dependencies. ''' deps = [] for alt_group_str in dep_str.split(','): alt_group_str = alt_group_str.strip() if not alt_group_str: # happens for empty depends or trailing commas continue adtlog.debug('processing dependency %s' % alt_group_str) if alt_group_str == '@': for d in _debian_packages_from_source(srcdir): adtlog.debug('synthesised dependency %s' % d) deps.append(d) elif alt_group_str == '@builddeps@': for d in _debian_build_deps_from_source(srcdir, testbed_arch): adtlog.debug('synthesised dependency %s' % d) deps.append(d) else: for dep in alt_group_str.split('|'): _debian_check_dep(testname, dep) deps.append(alt_group_str) return deps
def __init__(self, name, path, command, restrictions, features, depends, clicks, installed_clicks): '''Create new test description A test must have either "path" or "command", the respective other value must be None. @name: Test name @path: path to the test's executable, relative to source tree @command: shell command for the test code @restrictions, @features: string lists, as in README.package-tests @depends: string list of test dependencies (packages) @clicks: path list of click packages to install for this test @installed_clicks: names of already installed clicks for this test ''' if '/' in name: raise Unsupported(name, 'test name may not contain / character') for r in restrictions: if r not in known_restrictions: raise Unsupported(name, 'unknown restriction %s' % r) if not ((path is None) ^ (command is None)): raise InvalidControl(name, 'Test must have either path or command') self.name = name self.path = path self.command = command self.restrictions = restrictions self.features = features self.depends = depends self.clicks = clicks self.installed_clicks = installed_clicks # None while test hasn't run yet; True: pass, False: fail self.result = None adtlog.debug('Test defined: name %s path %s command "%s" ' 'restrictions %s features %s depends %s clicks %s ' 'installed clicks %s' % (name, path, command, restrictions, features, depends, clicks, installed_clicks))
def register(self, path, pkgname): adtlog.debug('Binaries: register deb=%s pkgname=%s ' % (path, pkgname)) dest = os.path.join(self.dir.host, pkgname + '.deb') # link or copy to self.dir try: os.remove(dest) except (IOError, OSError) as oe: if oe.errno != errno.ENOENT: raise oe try: os.link(path, dest) except (IOError, OSError) as oe: if oe.errno != errno.EXDEV: raise oe shutil.copy(path, dest) # clean up locally built debs (what=ubtreeN) to keep a clean # --output-dir, but don't clean up --binary arguments if path.startswith(self.output_dir): atexit.register(lambda f: os.path.exists(f) and os.unlink(f), path) self.registered.add(pkgname)
def __init__(self, name, path, command, restrictions, features, depends, clicks, installed_clicks): '''Create new test description A test must have either "path" or "command", the respective other value must be None. @name: Test name @path: path to the test's executable, relative to source tree @command: shell command for the test code @restrictions, @features: string lists, as in README.package-tests @depends: string list of test dependencies (packages) @clicks: path list of click packages to install for this test @installed_clicks: names of already installed clicks for this test ''' if '/' in name: raise Unsupported(name, 'test name may not contain / character') for r in restrictions: if r not in known_restrictions: raise Unsupported(name, 'unknown restriction %s' % r) if not ((path is None) ^ (command is None)): raise InvalidControl(name, 'Test must have either path or command') self.name = name self.path = path self.command = command self.restrictions = restrictions self.features = features self.depends = depends self.clicks = clicks self.installed_clicks = installed_clicks # None while test hasn't run yet; True: pass, False: fail self.result = None adtlog.debug('Test defined: name %s path %s command "%s" ' 'restrictions %s features %s depends %s clicks %s ' 'installed clicks %s' % (name, path, command, restrictions, features, depends, clicks, installed_clicks))
def command(): sys.stdout.flush() while True: try: ce = sys.stdin.readline().strip() # FIXME: This usually means EOF (as checked below), but with Python # 3 we often get empty strings here even though this is supposed to # block for new input. if ce == '': time.sleep(0.1) continue break except IOError as e: if e.errno == errno.EAGAIN: time.sleep(0.1) continue else: raise if not ce: bomb('end of file - caller quit?') ce = ce.rstrip().split() c = list(map(url_unquote, ce)) if not c: bomb('empty commands are not permitted') adtlog.debug('executing ' + ' '.join(ce)) c_lookup = c[0].replace('-', '_') try: f = globals()['cmd_' + c_lookup] except KeyError: bomb("unknown command `%s'" % ce[0]) try: r = f(c, ce) if not r: r = [] r.insert(0, 'ok') except FailedCmd as fc: r = fc.e print(' '.join(r))
def command(): sys.stdout.flush() while True: try: ce = sys.stdin.readline().strip() # FIXME: This usually means EOF (as checked below), but with Python # 3 we often get empty strings here even though this is supposed to # block for new input. if ce == '': time.sleep(0.1) continue break except IOError as e: if e.errno == errno.EAGAIN: time.sleep(0.1) continue else: raise if not ce: bomb('end of file - caller quit?') ce = ce.rstrip().split() c = list(map(url_unquote, ce)) if not c: bomb('empty commands are not permitted') adtlog.debug('executing ' + ' '.join(ce)) c_lookup = c[0].replace('-', '_') try: f = globals()['cmd_' + c_lookup] except KeyError: bomb("unknown command `%s'" % ce[0]) try: r = f(c, ce) if not r: r = [] r.insert(0, 'ok') except FailedCmd as fc: r = fc.e print(' '.join(r))
def cmd_reboot(c, ce): global downtmp cmdnumargs(c, ce) if not downtmp: bomb("`reboot' when not open") if 'reboot' not in caller.hook_capabilities(): bomb("`reboot' when `reboot' not advertised") # save current downtmp check_exec(['sh', '-ec', '''rm -f /var/cache/autopkgtest/tmpdir.tar mkdir -p /var/cache/autopkgtest/ tar --create --absolute-names -f /var/cache/autopkgtest/tmpdir.tar '%s' ''' % downtmp], downp=True, timeout=copy_timeout) adtlog.debug('cmd_reboot: saved current downtmp, rebooting') caller.hook_reboot() # restore downtmp check_exec(['sh', '-ec', ''' tar --extract --absolute-names -f /var/cache/autopkgtest/tmpdir.tar rm -r /var/cache/autopkgtest/'''], downp=True, timeout=copy_timeout) adtlog.debug('cmd_reboot: saved current downtmp, rebooting')
def execute_timeout(instr, timeout, *popenargs, **popenargsk): '''Popen wrapper with timeout supervision If instr is given, it is fed into stdin, otherwise stdin will be /dev/null. Return (status, stdout, stderr) ''' adtlog.debug('execute-timeout: ' + ' '.join(popenargs[0])) sp = subprocess.Popen(*popenargs, preexec_fn=preexecfn, universal_newlines=True, **popenargsk) if instr is None: popenargsk['stdin'] = devnull_read timeout_start(timeout) try: (out, err) = sp.communicate(instr) except Timeout: sp.kill() sp.wait() raise timeout_stop() status = sp.wait() return (status, out, err)
def copyup_shareddir(tb, host, is_dir, downtmp_host, follow_symlinks=True): adtlog.debug('copyup_shareddir: tb %s host %s is_dir %s downtmp_host %s' % (tb, host, is_dir, downtmp_host)) host = os.path.normpath(host) tb = os.path.normpath(tb) downtmp_host = os.path.normpath(downtmp_host) timeout_start(copy_timeout) try: tb_tmp = None if tb.startswith(downtmp): # translate into host path tb = downtmp_host + tb[len(downtmp):] else: tb_tmp = os.path.join(downtmp, os.path.basename(host)) adtlog.debug('copyup_shareddir: tb path %s is not already in ' 'downtmp, copying to %s' % (tb, tb_tmp)) check_exec(['cp', '-r', '--preserve=timestamps,links', tb, tb_tmp], downp=True) # translate into host path tb = os.path.join(downtmp_host, os.path.basename(host)) if tb == host: tb_tmp = None else: adtlog.debug('copyup_shareddir: tb(host) %s is not already at ' 'destination %s, copying' % (tb, host)) if is_dir: copytree(tb, host) else: shutil.copy(tb, host, follow_symlinks=follow_symlinks) if tb_tmp: adtlog.debug('copyup_shareddir: rm intermediate copy: %s' % tb) check_exec(['rm', '-rf', tb_tmp], downp=True) finally: timeout_stop()
def copyup_shareddir(tb, host, is_dir, downtmp_host): adtlog.debug('copyup_shareddir: tb %s host %s is_dir %s downtmp_host %s' % (tb, host, is_dir, downtmp_host)) host = os.path.normpath(host) tb = os.path.normpath(tb) downtmp_host = os.path.normpath(downtmp_host) timeout_start(copy_timeout) try: tb_tmp = None if tb.startswith(downtmp): # translate into host path tb = downtmp_host + tb[len(downtmp):] else: tb_tmp = os.path.join(downtmp, os.path.basename(host)) adtlog.debug('copyup_shareddir: tb path %s is not already in ' 'downtmp, copying to %s' % (tb, tb_tmp)) check_exec(['cp', '-r', '--preserve=timestamps,links', tb, tb_tmp], downp=True) # translate into host path tb = os.path.join(downtmp_host, os.path.basename(host)) if tb == host: tb_tmp = None else: adtlog.debug('copyup_shareddir: tb(host) %s is not already at ' 'destination %s, copying' % (tb, host)) if is_dir: copytree(tb, host) else: shutil.copy(tb, host) if tb_tmp: adtlog.debug('copyup_shareddir: rm intermediate copy: %s' % tb) check_exec(['rm', '-rf', tb_tmp], downp=True) finally: timeout_stop()
def _autodep8(srcdir): '''Generate control file with autodep8''' f = tempfile.NamedTemporaryFile(prefix='autodep8.') try: autodep8 = subprocess.Popen(['autodep8'], cwd=srcdir, stdout=f, stderr=subprocess.PIPE) except OSError as e: adtlog.debug('autodep8 not available (%s)' % e) return None err = autodep8.communicate()[1].decode() if autodep8.returncode == 0: f.flush() f.seek(0) ctrl = f.read().decode() adtlog.debug('autodep8 generated control: -----\n%s\n-------' % ctrl) return f f.close() adtlog.debug('autodep8 failed to generate control (exit status %i): %s' % (autodep8.returncode, err)) return None
def parse_args(arglist=None): '''Parse autopkgtest command line arguments. Return (options, actions, virt-server-args). ''' global actions actions = [] usage = '%(prog)s [options] [testbinary ...] testsrc -- virt-server [options]' description = '''Test installed binary packages using the tests in testsrc. testsrc can be one of a: - Debian *.dsc source package - Debian *.changes file containing a .dsc source package (and possibly binaries to test) - Debian source package directory - click source directory (optional if a *.click binary is given whose manifest points to the source) - apt source package name (through apt-get source) - Debian source package in git (url#branchname) You can specify local *.deb packages or a single *.click package to test.''' epilog = '''The -- argument separates the autopkgtest actions and options from the virt-server which provides the testbed. See e. g. man autopkgtest-schroot for details.''' parser = argparse.ArgumentParser( usage=usage, description=description, formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog, add_help=False) # test specification g_test = parser.add_argument_group( 'arguments for specifying and modifying the test') g_test.add_argument( '--override-control', metavar='PATH', help='run tests from control file/manifest PATH instead ' 'of the source/click package') # Don't display the deprecated argument name in the --help output. g_test.add_argument('--testname', help=argparse.SUPPRESS) g_test.add_argument('--test-name', dest='testname', help='run only given test name. ' 'This replaces --testname, which is deprecated.') g_test.add_argument( '-B', '--no-built-binaries', dest='built_binaries', action='store_false', default=True, help='do not build/use binaries from .dsc, git source, or unbuilt tree' ) g_test.add_argument('--installed-click', metavar='CLICKNAME', help='Run tests from already installed click package ' '(e. g. "com.example.myapp"), from specified click ' 'source directory or manifest\'s x-source.') g_test.add_argument( 'packages', nargs='*', help='testsrc source package and testbinary packages as above') # logging g_log = parser.add_argument_group('logging options') g_log.add_argument('-o', '--output-dir', help='Write test artifacts (stdout/err, log, debs, etc)' ' to OUTPUT-DIR (must not exist or be empty)') g_log.add_argument('-l', '--log-file', dest='logfile', help='Write the log LOGFILE, emptying it beforehand,' ' instead of using OUTPUT-DIR/log') g_log.add_argument('--summary-file', dest='summary', help='Write a summary report to SUMMARY, emptying it ' 'beforehand') g_log.add_argument('-q', '--quiet', action='store_const', dest='verbosity', const=0, default=1, help='Suppress all messages from %(prog)s itself ' 'except for the test results') # test bed setup g_setup = parser.add_argument_group('test bed setup options') g_setup.add_argument('--setup-commands', metavar='COMMANDS_OR_PATH', action='append', default=[], help='Run these commands after opening the testbed ' '(e. g. "apt-get update" or adding apt sources); ' 'can be a string with the commands, or a file ' 'containing the commands') # Ensure that this fails with something other than 100 in most error cases, # as apt-get update failures are usually transient; but if we find a # nonexisting apt source (404 Not Found) we *do* want 100, as otherwise # we'd get eternally looping tests. g_setup.add_argument( '-U', '--apt-upgrade', dest='setup_commands', action='append_const', const= '''(O=$(bash -o pipefail -ec 'apt-get update | tee /proc/self/fd/2') ||''' '{ [ "${O%404*Not Found*}" = "$O" ] || exit 100; sleep 15; apt-get update; }' '' ' || { sleep 60; apt-get update; } || false)' ' && $(which eatmydata || true) apt-get dist-upgrade -y -o ' 'Dpkg::Options::="--force-confnew"', help='Run apt update/dist-upgrade before the tests') g_setup.add_argument('--setup-commands-boot', metavar='COMMANDS_OR_PATH', action='append', default=[], help='Run these commands after --setup-commands, ' 'and also every time the testbed is rebooted') g_setup.add_argument('--apt-pocket', action='append', metavar='POCKETNAME[=pkgname,src:srcname,...]', default=[], help='Enable additional apt source for POCKETNAME. ' 'If packages are given, set up apt pinning to use ' 'only those packages from POCKETNAME; src:srcname ' ' expands to all binaries of srcname') g_setup.add_argument('--copy', metavar='HOSTFILE:TESTBEDFILE', action='append', default=[], help='Copy file or dir from host into testbed after ' 'opening') g_setup.add_argument( '--env', metavar='VAR=value', action='append', default=[], help='Set arbitrary environment variable for builds and test') # privileges g_priv = parser.add_argument_group('user/privilege handling options') g_priv.add_argument('-u', '--user', help='run tests as USER (needs root on testbed)') g_priv.add_argument('--gain-root', dest='gainroot', help='Command to gain root during package build, ' 'passed to dpkg-buildpackage -r') # debugging g_dbg = parser.add_argument_group('debugging options') g_dbg.add_argument('-d', '--debug', action='store_const', dest='verbosity', const=2, help='Show lots of internal autopkgtest debug messages') g_dbg.add_argument('-s', '--shell-fail', action='store_true', help='Run a shell in the testbed after any failed ' 'build or test') g_dbg.add_argument('--shell', action='store_true', help='Run a shell in the testbed after every test') # timeouts g_time = parser.add_argument_group('timeout options') for k, v in adt_testbed.timeouts.items(): g_time.add_argument('--timeout-' + k, type=int, dest='timeout_' + k, metavar='T', help='set %s timeout to T seconds (default: %us)' % (k, v)) g_time.add_argument('--timeout-factor', type=float, metavar='FACTOR', default=1.0, help='multiply all default timeouts by FACTOR') # locale g_loc = parser.add_argument_group('locale options') g_loc.add_argument('--set-lang', metavar='LANGVAL', help='set LANG on testbed to LANGVAL ' '(default: C.UTF-8') # misc g_misc = parser.add_argument_group('other options') g_misc.add_argument('--no-auto-control', dest='auto_control', action='store_false', default=True, help='Disable automatic test generation with autodep8') g_misc.add_argument('--build-parallel', metavar='N', help='Set "parallel=N" DEB_BUILD_OPTION for building ' 'packages (default: number of available processors)') g_misc.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS, help='show this help message and exit') # first, expand argument files file_parser = ArgumentParser(add_help=False) arglist = file_parser.parse_known_args(arglist)[1] # deprecation warning if '--testname' in arglist: adtlog.warning('--testname is deprecated; use --test-name') # split off virt-server args try: sep = arglist.index('--') except ValueError: # backwards compatibility: allow three dashes try: sep = arglist.index('---') adtlog.warning( 'Using --- to separate virt server arguments is deprecated; use -- instead' ) except ValueError: # still allow --help sep = None virt_args = None if sep is not None: virt_args = arglist[sep + 1:] arglist = arglist[:sep] # parse autopkgtest options args = parser.parse_args(arglist) adtlog.verbosity = args.verbosity adtlog.debug('autopkgtest options: %s' % args) adtlog.debug('virt-runner arguments: %s' % virt_args) if not virt_args: parser.error('You must specify -- <virt-server>...') # autopkgtest-virt-* prefix can be skipped if virt_args and '/' not in virt_args[0] and not virt_args[0].startswith( 'autopkgtest-virt-'): virt_args[0] = 'autopkgtest-virt-' + virt_args[0] process_package_arguments(parser, args) # verify --env validity for e in args.env: if '=' not in e: parser.error('--env must be KEY=value') if args.set_lang: args.env.append('LANG=' + args.set_lang) # set (possibly adjusted) timeout defaults for k in adt_testbed.timeouts: v = getattr(args, 'timeout_' + k) if v is None: adt_testbed.timeouts[k] = int(adt_testbed.timeouts[k] * args.timeout_factor) else: adt_testbed.timeouts[k] = v # this timeout is for the virt server, so pass it down via environment os.environ['AUTOPKGTEST_VIRT_COPY_TIMEOUT'] = str( adt_testbed.timeouts['copy']) # if we have --setup-commands and it points to a file, read its contents for i, c in enumerate(args.setup_commands): # shortcut for shipped scripts if '/' not in c: shipped = os.path.join('/usr/share/autopkgtest/setup-commands', c) if os.path.exists(shipped): c = shipped if os.path.exists(c): with open(c, encoding='UTF-8') as f: args.setup_commands[i] = f.read().strip() for i, c in enumerate(args.setup_commands_boot): if '/' not in c: shipped = os.path.join('/usr/share/autopkgtest/setup-commands', c) if os.path.exists(shipped): c = shipped if os.path.exists(c): with open(c, encoding='UTF-8') as f: args.setup_commands_boot[i] = f.read().strip() # parse --copy arguments copy_pairs = [] for arg in args.copy: try: (host, tb) = arg.split(':', 1) except ValueError: parser.error('--copy argument must be HOSTPATH:TESTBEDPATH: %s' % arg) if not os.path.exists(host): parser.error('--copy host path %s does not exist' % host) copy_pairs.append((host, tb)) args.copy = copy_pairs return (args, actions, virt_args)
def copyupdown_internal(wh, sd, upp, follow_symlinks=True): '''Copy up/down a file or dir. wh: 'copyup' or 'copydown' sd: (source, destination) paths upp: True for copyup, False for copydown ''' if not downtmp: bomb("%s when not open" % wh) if not sd[0] or not sd[1]: bomb("%s paths must be nonempty" % wh) dirsp = sd[0][-1] == '/' if dirsp != (sd[1][-1] == '/'): bomb("%s paths must agree about directoryness" " (presence or absence of trailing /)" % wh) # if we have a shared directory, we just need to copy it from/to there; in # most cases, it's testbed end is already in the downtmp dir downtmp_host = get_downtmp_host() if downtmp_host: try: if upp: copyup_shareddir(sd[0], sd[1], dirsp, downtmp_host, follow_symlinks) else: copydown_shareddir(sd[0], sd[1], dirsp, downtmp_host) return except Timeout: raise FailedCmd(['timeout']) except (shutil.Error, subprocess.CalledProcessError) as e: adtlog.debug( 'Cannot copy %s to %s through shared dir: %s, falling back to tar' % (sd[0], sd[1], str(e))) isrc = 0 idst = 1 ilocal = 0 + upp iremote = 1 - upp deststdout = devnull_read srcstdin = devnull_read remfileq = pipes.quote(sd[iremote]) if not dirsp: rune = 'cat %s%s' % ('><'[upp], remfileq) if upp: deststdout = open(sd[idst], 'wb') else: srcstdin = open(sd[isrc], 'rb') status = os.fstat(srcstdin.fileno()) if status.st_mode & 0o111: rune += '; chmod +x -- %s' % (remfileq) localcmdl = ['cat'] else: taropts = [None, None] taropts[isrc] = '--warning=none -c .' taropts[idst] = '--warning=none --preserve-permissions --extract ' \ '--no-same-owner' rune = 'cd %s; tar %s -f -' % (remfileq, taropts[iremote]) if upp: try: os.mkdir(sd[ilocal]) except (IOError, OSError) as oe: if oe.errno != errno.EEXIST: raise else: rune = ('if ! test -d %s; then mkdir -- %s; fi; ' % (remfileq, remfileq)) + rune localcmdl = ['tar', '--directory', sd[ilocal]] + ( ('%s -f -' % taropts[ilocal]).split()) downcmdl = auxverb + ['sh', '-ec', rune] if upp: cmdls = (downcmdl, localcmdl) else: cmdls = (localcmdl, downcmdl) adtlog.debug(str(["cmdls", str(cmdls)])) adtlog.debug( str([ "srcstdin", str(srcstdin), "deststdout", str(deststdout), "devnull_read", devnull_read ])) subprocs = [None, None] adtlog.debug(" +< %s" % ' '.join(cmdls[0])) subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin, stdout=subprocess.PIPE) adtlog.debug(" +> %s" % ' '.join(cmdls[1])) subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout, stdout=deststdout) subprocs[0].stdout.close() try: timeout_start(copy_timeout) for sdn in [1, 0]: adtlog.debug(" +" + "<>"[sdn] + "?") status = subprocs[sdn].wait() if not (status == 0 or (sdn == 0 and status == -13)): timeout_stop() bomb("%s %s failed, status %d" % (wh, ['source', 'destination'][sdn], status)) timeout_stop() except Timeout: for sdn in [1, 0]: subprocs[sdn].kill() subprocs[sdn].wait() raise FailedCmd(['timeout'])
def parse_click_manifest(manifest, testbed_caps, clickdeps, use_installed, srcdir=None): '''Parse test descriptions from a click manifest. @manifest: String with the click manifest @testbed_caps: List of testbed capabilities @clickdeps: paths of click packages that these tests need @use_installed: True if test expects the described click to be installed already Return (source_dir, list of Test objects, some_skipped). If this encounters any invalid restrictions, fields, or test restrictions which cannot be met by the given testbed capabilities, the test will be skipped (and reported so), and not be included in the result. If srcdir is given, use that as source for the click package, and return that as first return value. Otherwise, locate and download the source from the click's manifest into a temporary directory and use that. This may raise an InvalidControl exception. ''' try: manifest_j = json.loads(manifest) test_j = manifest_j.get('x-test', {}) except ValueError as e: raise InvalidControl( '*', 'click manifest is not valid JSON: %s' % str(e)) if not isinstance(test_j, dict): raise InvalidControl( '*', 'click manifest x-test key must be a dictionary') installed_clicks = [] if use_installed: installed_clicks.append(manifest_j.get('name')) some_skipped = False tests = [] # It's a dictionary and thus does not have a predictable ordering; sort it # to get a predictable list for name in sorted(test_j): desc = test_j[name] adtlog.debug('parsing click manifest test %s: %s' % (name, desc)) # simple string is the same as { "path": <desc> } without any # restrictions, or the special "autopilot" case if isinstance(desc, str): if name == 'autopilot' and re.match('^[a-z_][a-z0-9_]+$', desc): desc = {'autopilot_module': desc} else: desc = {'path': desc} if not isinstance(desc, dict): raise InvalidControl(name, 'click manifest x-test dictionary ' 'entries must be strings or dicts') # autopilot special case: dict with extra depends if 'autopilot_module' in desc: desc['command'] = \ 'PYTHONPATH=app/tests/autopilot:tests/autopilot:$PYTHONPATH '\ 'python3 -m autopilot.run run -v -f subunit -o ' \ '$AUTOPKGTEST_ARTIFACTS/%s.subunit ' % name + os.environ.get( 'AUTOPKGTEST_AUTOPILOT_MODULE', os.environ.get('ADT_AUTOPILOT_MODULE', desc['autopilot_module'])) desc.setdefault('depends', []).insert( 0, 'ubuntu-ui-toolkit-autopilot') desc['depends'].insert(0, 'autopilot-touch') if 'allow-stderr' not in desc.setdefault('restrictions', []): desc['restrictions'].append('allow-stderr') try: test = Test(name, desc.get('path'), desc.get('command'), desc.get('restrictions', []), desc.get('features', []), desc.get('depends', []), clickdeps, installed_clicks) test.check_testbed_compat(testbed_caps) tests.append(test) except Unsupported as u: u.report() some_skipped = True if srcdir is None: # do we have an x-source/vcs-bzr link? if 'x-source' in manifest_j: try: repo = manifest_j['x-source']['vcs-bzr'] adtlog.info('checking out click source from %s' % repo) d = tempfile.mkdtemp(prefix='autopkgtest.clicksrc.') atexit.register(shutil.rmtree, d, ignore_errors=True) try: subprocess.check_call(['bzr', 'checkout', '--lightweight', repo, d]) srcdir = d except subprocess.CalledProcessError as e: adtlog.error('Failed to check out click source from %s: %s' % (repo, str(e))) except KeyError: adtlog.error('Click source download from x-source only ' 'supports "vcs-bzr" repositories') else: adtlog.error('cannot download click source: manifest does not ' 'have "x-source"') return (srcdir, tests, some_skipped)
def copyupdown_internal(wh, sd, upp): '''Copy up/down a file or dir. wh: 'copyup' or 'copydown' sd: (source, destination) paths upp: True for copyup, False for copydown ''' if not downtmp: bomb("%s when not open" % wh) if not sd[0] or not sd[1]: bomb("%s paths must be nonempty" % wh) dirsp = sd[0][-1] == '/' if dirsp != (sd[1][-1] == '/'): bomb("% paths must agree about directoryness" " (presence or absence of trailing /)" % wh) # if we have a shared directory, we just need to copy it from/to there; in # most cases, it's testbed end is already in the downtmp dir downtmp_host = get_downtmp_host() if downtmp_host: try: if upp: copyup_shareddir(sd[0], sd[1], dirsp, downtmp_host) else: copydown_shareddir(sd[0], sd[1], dirsp, downtmp_host) except Timeout: raise FailedCmd(['timeout']) return isrc = 0 idst = 1 ilocal = 0 + upp iremote = 1 - upp deststdout = devnull_read srcstdin = devnull_read remfileq = pipes.quote(sd[iremote]) if not dirsp: rune = 'cat %s%s' % ('><'[upp], remfileq) if upp: deststdout = open(sd[idst], 'w') else: srcstdin = open(sd[isrc], 'r') status = os.fstat(srcstdin.fileno()) if status.st_mode & 0o111: rune += '; chmod +x -- %s' % (remfileq) localcmdl = ['cat'] else: taropts = [None, None] taropts[isrc] = '-c .' taropts[idst] = '--preserve-permissions --extract --no-same-owner' rune = 'cd %s; tar %s -f -' % (remfileq, taropts[iremote]) if upp: try: os.mkdir(sd[ilocal]) except (IOError, OSError) as oe: if oe.errno != errno.EEXIST: raise else: rune = ('if ! test -d %s; then mkdir -- %s; fi; ' % ( remfileq, remfileq) ) + rune localcmdl = ['tar', '--directory', sd[ilocal]] + ( ('%s -f -' % taropts[ilocal]).split() ) downcmdl = auxverb + ['sh', '-ec', rune] if upp: cmdls = (downcmdl, localcmdl) else: cmdls = (localcmdl, downcmdl) adtlog.debug(str(["cmdls", str(cmdls)])) adtlog.debug(str(["srcstdin", str(srcstdin), "deststdout", str(deststdout), "devnull_read", devnull_read])) subprocs = [None, None] adtlog.debug(" +< %s" % ' '.join(cmdls[0])) subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin, stdout=subprocess.PIPE, preexec_fn=preexecfn) adtlog.debug(" +> %s" % ' '.join(cmdls[1])) subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout, stdout=deststdout, preexec_fn=preexecfn) subprocs[0].stdout.close() try: timeout_start(copy_timeout) for sdn in [1, 0]: adtlog.debug(" +" + "<>"[sdn] + "?") status = subprocs[sdn].wait() if not (status == 0 or (sdn == 0 and status == -13)): timeout_stop() bomb("%s %s failed, status %d" % (wh, ['source', 'destination'][sdn], status)) timeout_stop() except Timeout: for sdn in [1, 0]: subprocs[sdn].kill() subprocs[sdn].wait() raise FailedCmd(['timeout'])
def parse_click_manifest(manifest, testbed_caps, clickdeps, use_installed, srcdir=None): '''Parse test descriptions from a click manifest. @manifest: String with the click manifest @testbed_caps: List of testbed capabilities @clickdeps: paths of click packages that these tests need @use_installed: True if test expects the described click to be installed already Return (source_dir, list of Test objects, some_skipped). If this encounters any invalid restrictions, fields, or test restrictions which cannot be met by the given testbed capabilities, the test will be skipped (and reported so), and not be included in the result. If srcdir is given, use that as source for the click package, and return that as first return value. Otherwise, locate and download the source from the click's manifest into a temporary directory and use that. This may raise an InvalidControl exception. ''' try: manifest_j = json.loads(manifest) test_j = manifest_j.get('x-test', {}) except ValueError as e: raise InvalidControl( '*', 'click manifest is not valid JSON: %s' % str(e)) if not isinstance(test_j, dict): raise InvalidControl( '*', 'click manifest x-test key must be a dictionary') installed_clicks = [] if use_installed: installed_clicks.append(manifest_j.get('name')) some_skipped = False tests = [] # It's a dictionary and thus does not have a predictable ordering; sort it # to get a predictable list for name in sorted(test_j): desc = test_j[name] adtlog.debug('parsing click manifest test %s: %s' % (name, desc)) # simple string is the same as { "path": <desc> } without any # restrictions, or the special "autopilot" case if isinstance(desc, str): if name == 'autopilot' and re.match('^[a-z_][a-z0-9_]+$', desc): desc = {'autopilot_module': desc} else: desc = {'path': desc} if not isinstance(desc, dict): raise InvalidControl(name, 'click manifest x-test dictionary ' 'entries must be strings or dicts') # autopilot special case: dict with extra depends if 'autopilot_module' in desc: desc['command'] = 'PYTHONPATH=tests/autopilot:$PYTHONPATH ' \ 'python3 -m autopilot.run run -v -f subunit -o ' \ '$ADT_ARTIFACTS/%s.subunit ' % name + os.environ.get( 'ADT_AUTOPILOT_MODULE', desc['autopilot_module']) desc.setdefault('depends', []).insert( 0, 'ubuntu-ui-toolkit-autopilot') desc['depends'].insert(0, 'autopilot-touch') if 'allow-stderr' not in desc.setdefault('restrictions', []): desc['restrictions'].append('allow-stderr') try: test = Test(name, desc.get('path'), desc.get('command'), desc.get('restrictions', []), desc.get('features', []), desc.get('depends', []), clickdeps, installed_clicks) test.check_testbed_compat(testbed_caps) tests.append(test) except Unsupported as u: u.report() some_skipped = True if srcdir is None: # do we have an x-source/vcs-bzr link? if 'x-source' in manifest_j: try: repo = manifest_j['x-source']['vcs-bzr'] adtlog.info('checking out click source from %s' % repo) d = tempfile.mkdtemp(prefix='adt.clicksrc.') atexit.register(shutil.rmtree, d, ignore_errors=True) try: subprocess.check_call(['bzr', 'checkout', '--lightweight', repo, d]) srcdir = d except subprocess.CalledProcessError as e: adtlog.error('Failed to check out click source from %s: %s' % (repo, str(e))) except KeyError: adtlog.error('Click source download from x-source only ' 'supports "vcs-bzr" repositories') else: adtlog.error('cannot download click source: manifest does not ' 'have "x-source"') return (srcdir, tests, some_skipped)
def parse_args(arglist=None): '''Parse adt-run command line arguments. Return (options, actions, virt-server-args). ''' global actions, built_binaries actions = [] built_binaries = True # action parser; instantiated first to use generated help action_parser = argparse.ArgumentParser(usage=argparse.SUPPRESS, add_help=False) action_parser.add_argument( '--unbuilt-tree', action=ActionArg, metavar='DIR or DIR//', help='run tests from unbuilt Debian source tree DIR') action_parser.add_argument( '--built-tree', action=ActionArg, metavar='DIR or DIR/', help='run tests from built Debian source tree DIR') action_parser.add_argument( '--source', action=ActionArg, metavar='DSC or some/pkg.dsc', help='build DSC and use its tests and/or generated binary packages') action_parser.add_argument( '--git-source', action=ActionArg, metavar='GITURL [branchname]', help='check out git URL (optionally a non-default branch), build it ' 'if necessary, and run its tests') action_parser.add_argument( '--binary', action=ActionArg, metavar='DEB or some/pkg.deb', help='use binary package DEB for subsequent tests') action_parser.add_argument( '--changes', action=ActionArg, metavar='CHANGES or some/pkg.changes', help='run tests from dsc and binary debs from a .changes file') action_parser.add_argument( '--apt-source', action=ActionArg, metavar='SRCPKG or somesrc', help='download with apt-get source in testbed and use its tests') action_parser.add_argument( '--click-source', action=ActionArg, metavar='CLICKSRC or some/src', help='click source tree for subsequent --click package') action_parser.add_argument( '--click', action=ActionArg, metavar='CLICKPKG or some/pkg.click', help='install click package into testbed (path to *.click) or ' 'use an already installed click package ("com.example.myapp") ' 'and run its tests (from manifest\'s x-source or preceding ' '--click-source)') action_parser.add_argument( '--override-control', action=ActionArg, metavar='CONTROL', help='run tests from control file/manifest CONTROL' ' instead in the next package') action_parser.add_argument( '--testname', action=ActionArg, help='run only given test name in the next package') action_parser.add_argument( '-B', '--no-built-binaries', nargs=0, action=BinariesArg, help='do not use any binaries from subsequent --source, ' '--git-source, or --unbuilt-tree actions') action_parser.add_argument( '--built-binaries', nargs=0, action=BinariesArg, help='use binaries from subsequent --source, --git-source, or ' '--unbuilt-tree actions') # main / options parser usage = '%(prog)s [options] action [action ...] --- virt-server [options]' description = '''Test installed binary packages using the tests in the source package. Actions specify the source and binary packages to test, or change what happens with package arguments: %s ''' % action_parser.format_help().split('\n', 1)[1] epilog = '''The --- argument separates the adt-run actions and options from the virt-server which provides the testbed. See e. g. man autopkgtest-schroot for details.''' parser = argparse.ArgumentParser( usage=usage, description=description, formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog, add_help=False) # logging g_log = parser.add_argument_group('logging options') g_log.add_argument('-o', '--output-dir', help='Write test artifacts (stdout/err, log, debs, etc)' ' to OUTPUT-DIR (must not exist or be empty)') g_log.add_argument('-l', '--log-file', dest='logfile', help='Write the log LOGFILE, emptying it beforehand,' ' instead of using OUTPUT-DIR/log') g_log.add_argument('--summary-file', dest='summary', help='Write a summary report to SUMMARY, emptying it ' 'beforehand') g_log.add_argument('-q', '--quiet', action='store_const', dest='verbosity', const=0, default=1, help='Suppress all messages from %(prog)s itself ' 'except for the test results') # test bed setup g_setup = parser.add_argument_group('test bed setup options') g_setup.add_argument('--setup-commands', metavar='COMMANDS_OR_PATH', action='append', default=[], help='Run these commands after opening the testbed ' '(e. g. "apt-get update" or adding apt sources); ' 'can be a string with the commands, or a file ' 'containing the commands') g_setup.add_argument('--setup-commands-boot', metavar='COMMANDS_OR_PATH', action='append', default=[], help='Run these commands after --setup-commands, ' 'and also every time the testbed is rebooted') # ensure that this fails with something other than 100, as apt-get update # failures are usually transient g_setup.add_argument( '-U', '--apt-upgrade', dest='setup_commands', action='append_const', const='(apt-get update || (sleep 15; apt-get update)' ' || (sleep 60; apt-get update) || false)' ' && $(which eatmydata || true) apt-get dist-upgrade -y -o ' 'Dpkg::Options::="--force-confnew"', help='Run apt update/dist-upgrade before the tests') g_setup.add_argument('--apt-pocket', action='append', metavar='POCKETNAME[=pkgname,src:srcname,...]', default=[], help='Enable additional apt source for POCKETNAME. ' 'If packages are given, set up apt pinning to use ' 'only those packages from POCKETNAME; src:srcname ' ' expands to all binaries of srcname') g_setup.add_argument('--copy', metavar='HOSTFILE:TESTBEDFILE', action='append', default=[], help='Copy file or dir from host into testbed after ' 'opening') g_setup.add_argument( '--env', metavar='VAR=value', action='append', default=[], help='Set arbitrary environment variable for builds and test') # privileges g_priv = parser.add_argument_group('user/privilege handling options') g_priv.add_argument('-u', '--user', help='run tests as USER (needs root on testbed)') g_priv.add_argument('--gain-root', dest='gainroot', help='Command to gain root during package build, ' 'passed to dpkg-buildpackage -r') # debugging g_dbg = parser.add_argument_group('debugging options') g_dbg.add_argument('-d', '--debug', action='store_const', dest='verbosity', const=2, help='Show lots of internal adt-run debug messages') g_dbg.add_argument('-s', '--shell-fail', action='store_true', help='Run a shell in the testbed after any failed ' 'build or test') g_dbg.add_argument('--shell', action='store_true', help='Run a shell in the testbed after every test') # timeouts g_time = parser.add_argument_group('timeout options') for k, v in adt_testbed.timeouts.items(): g_time.add_argument('--timeout-' + k, type=int, dest='timeout_' + k, metavar='T', help='set %s timeout to T seconds (default: %us)' % (k, v)) g_time.add_argument('--timeout-factor', type=float, metavar='FACTOR', default=1.0, help='multiply all default timeouts by FACTOR') # locale g_loc = parser.add_argument_group('locale options') g_loc.add_argument('--set-lang', metavar='LANGVAL', help='set LANG on testbed to LANGVAL ' '(default: C.UTF-8') # misc g_misc = parser.add_argument_group('other options') g_misc.add_argument('--no-auto-control', dest='auto_control', action='store_false', default=True, help='Disable automatic test generation with autodep8') g_misc.add_argument('--build-parallel', metavar='N', help='Set "parallel=N" DEB_BUILD_OPTION for building ' 'packages (default: number of available processors)') g_misc.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS, help='show this help message and exit') # first, expand argument files file_parser = ArgumentParser(add_help=False) arglist = file_parser.parse_known_args(arglist)[1] # split off virt-server args try: sep = arglist.index('---') virt_args = arglist[sep + 1:] arglist = arglist[:sep] except ValueError: # still allow --help virt_args = None # parse options first (args, action_args) = parser.parse_known_args(arglist) adtlog.verbosity = args.verbosity adtlog.debug('Parsed options: %s' % args) adtlog.debug('Remaining arguments: %s' % action_args) # now turn implicit "bare" args into option args, so that we can parse them # with argparse, and split off the virt-server args action_args = interpret_implicit_args(parser, action_args) adtlog.debug('Interpreted actions: %s' % action_args) adtlog.debug('Virt runner arguments: %s' % virt_args) if not virt_args: parser.error('You must specify --- <virt-server>...') if virt_args and '/' not in virt_args[0]: # for backwards compat, vserver can be given with "adt-virt-" prefix if virt_args[0].startswith('adt-virt-'): virt_args[0] = virt_args[0][9:] # autopkgtest-virt-* prefix can be skipped if not virt_args[0].startswith('autopkgtest-virt-'): virt_args[0] = 'autopkgtest-virt-' + virt_args[0] action_parser.parse_args(action_args) # verify --env validity for e in args.env: if '=' not in e: parser.error('--env must be KEY=value') if args.set_lang: args.env.append('LANG=' + args.set_lang) # set (possibly adjusted) timeout defaults for k in adt_testbed.timeouts: v = getattr(args, 'timeout_' + k) if v is None: adt_testbed.timeouts[k] = int(adt_testbed.timeouts[k] * args.timeout_factor) else: adt_testbed.timeouts[k] = v # this timeout is for the virt server, so pass it down via environment os.environ['AUTOPKGTEST_VIRT_COPY_TIMEOUT'] = str( adt_testbed.timeouts['copy']) if not actions: parser.error('You must specify at least one action') # if we have --setup-commands and it points to a file, read its contents for i, c in enumerate(args.setup_commands): # shortcut for shipped scripts if '/' not in c: shipped = os.path.join('/usr/share/autopkgtest/setup-commands', c) if os.path.exists(shipped): c = shipped if os.path.exists(c): with open(c, encoding='UTF-8') as f: args.setup_commands[i] = f.read().strip() # parse --copy arguments copy_pairs = [] for arg in args.copy: try: (host, tb) = arg.split(':', 1) except ValueError: parser.error('--copy argument must be HOSTPATH:TESTBEDPATH: %s' % arg) if not os.path.exists(host): parser.error('--copy host path %s does not exist' % host) copy_pairs.append((host, tb)) args.copy = copy_pairs return (args, actions, virt_args)
def parse_debian_source(srcdir, testbed_caps, testbed_arch, control_path=None, auto_control=True): '''Parse test descriptions from a Debian DEP-8 source dir You can specify an alternative path for the control file (default: srcdir/debian/tests/control). Return (list of Test objects, some_skipped). If this encounters any invalid restrictions, fields, or test restrictions which cannot be met by the given testbed capabilities, the test will be skipped (and reported so), and not be included in the result. This may raise an InvalidControl exception. ''' some_skipped = False command_counter = 0 tests = [] if not control_path: control_path = os.path.join(srcdir, 'debian', 'tests', 'control') if not os.path.exists(control_path): if auto_control: control = _autodep8(srcdir) if control is None: return ([], False) control_path = control.name else: adtlog.debug('auto_control is disabled, no tests') return ([], False) for record in parse_rfc822(control_path): command = None try: restrictions = record.get('Restrictions', '').replace( ',', ' ').split() feature_test_name = None features = [] record_features = record.get('Features', '').replace( ',', ' ').split() for feature in record_features: details = feature.split('=', 1) if details[0] != 'test-name': features.append(feature) continue if len(details) != 2: # No value, i.e. a bare 'test-name' raise InvalidControl( '*', 'test-name feature with no argument') if feature_test_name is not None: raise InvalidControl( '*', 'only one test-name feature allowed') feature_test_name = details[1] features.append(feature) if 'Tests' in record: test_names = record['Tests'].replace(',', ' ').split() depends = _parse_debian_depends(test_names[0], record.get('Depends', '@'), srcdir, testbed_arch) if 'Test-command' in record: raise InvalidControl('*', 'Only one of "Tests" or ' '"Test-Command" may be given') if feature_test_name is not None: raise InvalidControl( '*', 'test-name feature incompatible with Tests') test_dir = record.get('Tests-directory', 'debian/tests') _debian_check_unknown_fields(test_names[0], record) for n in test_names: test = Test(n, os.path.join(test_dir, n), None, restrictions, features, depends, [], []) test.check_testbed_compat(testbed_caps) tests.append(test) elif 'Test-command' in record: command = record['Test-command'] depends = _parse_debian_depends(command, record.get('Depends', '@'), srcdir, testbed_arch) if feature_test_name is None: command_counter += 1 name = 'command%i' % command_counter else: name = feature_test_name _debian_check_unknown_fields(name, record) test = Test(name, None, command, restrictions, features, depends, [], []) test.check_testbed_compat(testbed_caps) tests.append(test) else: raise InvalidControl('*', 'missing "Tests" or "Test-Command"' ' field') except Unsupported as u: u.report() some_skipped = True return (tests, some_skipped)
def parse_args(arglist=None): '''Parse adt-run command line arguments. Return (options, actions, virt-server-args). ''' global actions, built_binaries actions = [] built_binaries = True # action parser; instantiated first to use generated help action_parser = argparse.ArgumentParser(usage=argparse.SUPPRESS, add_help=False) action_parser.add_argument( '--unbuilt-tree', action=ActionArg, metavar='DIR or DIR//', help='run tests from unbuilt Debian source tree DIR') action_parser.add_argument( '--built-tree', action=ActionArg, metavar='DIR or DIR/', help='run tests from built Debian source tree DIR') action_parser.add_argument( '--source', action=ActionArg, metavar='DSC or some/pkg.dsc', help='build DSC and use its tests and/or generated binary packages') action_parser.add_argument( '--binary', action=ActionArg, metavar='DEB or some/pkg.deb', help='use binary package DEB for subsequent tests') action_parser.add_argument( '--changes', action=ActionArg, metavar='CHANGES or some/pkg.changes', help='run tests from dsc and binary debs from a .changes file') action_parser.add_argument( '--apt-source', action=ActionArg, metavar='SRCPKG or somesrc', help='download with apt-get source in testbed and use its tests') action_parser.add_argument( '--click-source', action=ActionArg, metavar='CLICKSRC or some/src', help='click source tree for subsequent --click package') action_parser.add_argument( '--click', action=ActionArg, metavar='CLICKPKG or some/pkg.click', help='install click package into testbed (path to *.click) or ' 'use an already installed click package ("com.example.myapp") ' 'and run its tests (from manifest\'s x-source or preceding ' '--click-source)') action_parser.add_argument( '--override-control', action=ActionArg, metavar='CONTROL', help='run tests from control file/manifest CONTROL' ' instead, (applies to next Debian/click test suite only)') action_parser.add_argument( '-B', '--no-built-binaries', nargs=0, action=BinariesArg, help='do not use any binaries from subsequent --source or ' '--unbuilt-tree actions') action_parser.add_argument( '--built-binaries', nargs=0, action=BinariesArg, help='use binaries from subsequent --source or --unbuilt-tree actions') # main / options parser usage = '%(prog)s [options] action [action ...] --- virt-server [options]' description = '''Test installed binary packages using the tests in the source package. Actions specify the source and binary packages to test, or change what happens with package arguments: %s ''' % action_parser.format_help().split('\n', 1)[1] epilog = '''The --- argument separates the adt-run actions and options from the virt-server which provides the testbed. See e. g. man adt-virt-schroot for details.''' parser = argparse.ArgumentParser( usage=usage, description=description, formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog, add_help=False) # logging g_log = parser.add_argument_group('logging options') g_log.add_argument('-o', '--output-dir', help='Write test artifacts (stdout/err, log, debs, etc)' ' to OUTPUT-DIR, emptying it beforehand') # backwards compatible alias g_log.add_argument('--tmp-dir', dest='output_dir', help='Alias for --output-dir for backwards ' 'compatibility') g_log.add_argument('-l', '--log-file', dest='logfile', help='Write the log LOGFILE, emptying it beforehand,' ' instead of using OUTPUT-DIR/log') g_log.add_argument('--summary-file', dest='summary', help='Write a summary report to SUMMARY, emptying it ' 'beforehand') g_log.add_argument('-q', '--quiet', action='store_const', dest='verbosity', const=0, default=1, help='Suppress all messages from %(prog)s itself ' 'except for the test results') # test bed setup g_setup = parser.add_argument_group('test bed setup options') g_setup.add_argument('--setup-commands', metavar='COMMANDS_OR_PATH', action='append', default=[], help='Run these commands after opening the testbed ' '(e. g. "apt-get update" or adding apt sources); ' 'can be a string with the commands, or a file ' 'containing the commands') g_setup.add_argument('-U', '--apt-upgrade', dest='setup_commands', action='append_const', const='(apt-get update || (sleep 15; apt-get update)' ' || (sleep 60; apt-get update))' ' && apt-get dist-upgrade -y -o ' 'Dpkg::Options::="--force-confnew"', help='Run apt update/dist-upgrade before the tests') g_setup.add_argument('--apt-pocket', metavar='POCKETNAME', action='append', default=[], help='Enable additional apt source for POCKETNAME') g_setup.add_argument('--copy', metavar='HOSTFILE:TESTBEDFILE', action='append', default=[], help='Copy file or dir from host into testbed after ' 'opening') # privileges g_priv = parser.add_argument_group('user/privilege handling options') g_priv.add_argument('-u', '--user', help='run tests as USER (needs root on testbed)') g_priv.add_argument('--gain-root', dest='gainroot', help='Command to gain root during package build, ' 'passed to dpkg-buildpackage -r') # debugging g_dbg = parser.add_argument_group('debugging options') g_dbg.add_argument('-d', '--debug', action='store_const', dest='verbosity', const=2, help='Show lots of internal adt-run debug messages') g_dbg.add_argument('-s', '--shell-fail', action='store_true', help='Run a shell in the testbed after any failed ' 'build or test') g_dbg.add_argument('--shell', action='store_true', help='Run a shell in the testbed after every test') # timeouts g_time = parser.add_argument_group('timeout options') for k in timeouts: g_time.add_argument( '--timeout-' + k, type=int, dest='timeout_' + k, metavar='T', default=timeouts[k], help='set %s timeout to T seconds (default: %%(default)s)' % k) g_time.add_argument( '--timeout-factor', type=float, metavar='FACTOR', default=1.0, help='multiply all default timeouts by FACTOR') # locale g_loc = parser.add_argument_group('locale options') g_loc.add_argument('--leave-lang', dest='set_lang', action='store_false', default='C.UTF-8', help="leave LANG on testbed set to testbed's default") g_loc.add_argument('--set-lang', metavar='LANGVAL', default='C.UTF-8', help='set LANG on testbed to LANGVAL ' '(default: %(default)s)') # misc g_misc = parser.add_argument_group('other options') # keep backwards compatible path gnupghome_default = '~/.autopkgtest/gpg' if not os.path.isdir(os.path.expanduser(gnupghome_default)): gnupghome_default = '~/.cache/autopkgtest' g_misc.add_argument( '--gnupg-home', dest='gnupghome', metavar='DIR', default=gnupghome_default, help='use DIR rather than %(default)s (for signing private ' 'apt archive)') g_misc.add_argument( '-h', '--help', action='help', default=argparse.SUPPRESS, help='show this help message and exit') # first, expand argument files file_parser = argparse.ArgumentParser(fromfile_prefix_chars='@', add_help=False) arglist = file_parser.parse_known_args(arglist)[1] # split off virt-server args try: sep = arglist.index('---') virt_args = arglist[sep + 1:] arglist = arglist[:sep] except ValueError: # still allow --help virt_args = None # parse options first (args, action_args) = parser.parse_known_args(arglist) adtlog.verbosity = args.verbosity adtlog.debug('Parsed options: %s' % args) adtlog.debug('Remaining arguments: %s' % action_args) # now turn implicit "bare" args into option args, so that we can parse them # with argparse, and split off the virt-server args action_args = interpret_implicit_args(parser, action_args) adtlog.debug('Interpreted actions: %s' % action_args) adtlog.debug('Virt runner arguments: %s' % virt_args) if not virt_args: parser.error('You must specify --- <virt-server>...') action_parser.parse_args(action_args) # this timeout is for adt-virt-*, so pass it down via environment os.environ['ADT_VIRT_COPY_TIMEOUT'] = str(args.timeout_copy) if not actions: parser.error('You must specify at least one action') # if we have --setup-commands and it points to a file, read its contents for i, c in enumerate(args.setup_commands): # shortcut for shipped scripts if '/' not in c: shipped = os.path.join('/usr/share/autopkgtest/setup-commands', c) if os.path.exists(shipped): c = shipped if os.path.exists(c): with open(c, encoding='UTF-8') as f: args.setup_commands[i] = f.read().strip() # parse --copy arguments copy_pairs = [] for arg in args.copy: try: (host, tb) = arg.split(':', 1) except ValueError: parser.error('--copy argument must be HOSTPATH:TESTBEDPATH: %s' % arg) if not os.path.exists(host): parser.error('--copy host path %s does not exist' % host) copy_pairs.append((host, tb)) args.copy = copy_pairs if args.gnupghome.startswith('~/'): args.gnupghome = os.path.expanduser(args.gnupghome) return (args, actions, virt_args)
def process_package_arguments(parser, args): '''Check positional arguments and produce adt_run_args compatible actions list''' # TODO: This should be simplified once the old adt_run_args CLI gets # dropped. # Sort action list by deb << dsc and click-source << click, for a "do what # I mean" compatible adt_run_args action list global actions debsrc_action = None has_debs = False has_click = False has_clicksrc = False # expand .changes files packages = [] for p in args.packages: if p.endswith('.changes'): packages += read_changes(parser, p) else: packages.append(p) def set_debsrc(p, kind, built_bin=None): nonlocal debsrc_action if has_clicksrc or debsrc_action: parser.error('You must specify only one source package to test') debsrc_action = (kind, p, built_bin) for p in packages: if p.endswith('.deb') and os.path.exists(p): actions.append(('binary', p, None)) has_debs = True elif p.endswith('.dsc') and os.path.exists(p): set_debsrc(p, 'source') elif p.endswith('.click') and os.path.exists(p): if has_click: parser.error( 'You must specify at most one tested click package') actions.append(('click', p, None)) has_click = True elif is_click_src(p): set_debsrc(p, 'click-source') has_clicksrc = True elif re.match('[0-9a-z][0-9a-z.+-]+$', p): set_debsrc(p, 'apt-source', False) elif os.path.isfile(os.path.join(p, 'debian', 'control')): if os.path.exists(os.path.join(p, 'debian', 'files')): set_debsrc(p, 'built-tree', False) else: set_debsrc(p, 'unbuilt-tree') elif os.path.isfile(os.path.join(p, 'debian', 'tests', 'control')): # degenerate Debian source tree with only debian/tests set_debsrc(p, 'built-tree', False) elif '://' in p: set_debsrc(p, 'git-source') else: parser.error('%s is not a valid test package' % p) # translate --installed-click option into an action if args.installed_click: if has_click: parser.error('You must specify at most one tested click package') actions.append(('click', args.installed_click, None)) has_click = True # if no source is given, check if the current directory is a source tree if not debsrc_action and not has_click and os.path.isfile( 'debian/control'): if os.path.exists('debian/files'): set_debsrc('.', 'built-tree', False) else: set_debsrc('.', 'unbuilt-tree') if not debsrc_action and not has_clicksrc and not has_click: parser.error('You must specify source or click package to test') if has_debs or has_click: args.built_binaries = False if debsrc_action: # some actions above disable built binaries, for the rest use the CLI option if debsrc_action[2] is None: debsrc_action = (debsrc_action[0], debsrc_action[1], args.built_binaries) if has_clicksrc: actions.insert(0, debsrc_action) else: actions.append(debsrc_action) adtlog.debug('actions: %s' % actions) adtlog.debug('build binaries: %s' % args.built_binaries)