def main(): from gevent import monkey monkey.patch_all() from .orchestra import monkey monkey.patch_all() import logging log = logging.getLogger(__name__) ctx = parse_args() loglevel = logging.INFO if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig(level=loglevel, ) from teuthology.misc import read_config read_config(ctx) log.info('\n '.join([ 'targets:', ] + yaml.safe_dump(ctx.config['targets'], default_flow_style=False).splitlines())) if ctx.owner is None: from teuthology.misc import get_user ctx.owner = get_user() nuke(ctx, log)
def main(self): """ Entry point implementing the teuthology-openstack command. """ self.setup_logs() misc.read_config(self.args) self.key_filename = self.args.key_filename self.verify_openstack() ip = self.setup() if self.args.suite: self.run_suite() if self.args.key_filename: identity = '-i ' + self.args.key_filename + ' ' else: identity = '' if self.args.upload: upload = 'upload to : ' + self.args.archive_upload else: upload = '' log.info(""" web interface: http://{ip}:8081/ ssh access : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/html {upload}""".format(ip=ip, username=self.username, identity=identity, upload=upload)) if self.args.teardown: self.teardown()
def update_hostkeys(): parser = argparse.ArgumentParser(description=""" Update any hostkeys that have changed. You can list specific machines to run on, or use -a to check all of them automatically. """) parser.add_argument( '-t', '--targets', default=None, help='input yaml containing targets to check', ) parser.add_argument( 'machines', metavar='MACHINES', default=[], nargs='*', help='hosts to check for updated keys', ) parser.add_argument( '-v', '--verbose', action='store_true', default=False, help='be more verbose', ) parser.add_argument( '-a', '--all', action='store_true', default=False, help='update hostkeys of all machines in the db', ) ctx = parser.parse_args() loglevel = logging.ERROR if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig(level=loglevel, ) teuthology.read_config(ctx) assert ctx.all or ctx.targets or ctx.machines, 'You must specify machines to update' if ctx.all: assert not ctx.targets and not ctx.machines, \ 'You can\'t specify machines with the --all option' machines = ctx.machines if ctx.targets: try: with file(ctx.targets) as f: g = yaml.safe_load_all(f) for new in g: if 'targets' in new: for t in new['targets'].iterkeys(): machines.append(t) except IOError, e: raise argparse.ArgumentTypeError(str(e))
def main(ctx): if ctx.owner is None: ctx.owner = 'scheduled_{user}'.format(user=get_user()) read_config(ctx) beanstalk = teuthology.queue.connect(ctx) tube = ctx.worker beanstalk.use(tube) if ctx.show: for job_id in ctx.show: job = beanstalk.peek(job_id) if job is None and ctx.verbose: print 'job {jid} is not in the queue'.format(jid=job_id) else: print '--- job {jid} priority {prio} ---\n'.format( jid=job_id, prio=job.stats()['pri']), job.body return if ctx.delete: for job_id in ctx.delete: job = beanstalk.peek(job_id) if job is None: print 'job {jid} is not in the queue'.format(jid=job_id) else: job.delete() return # strip out targets; the worker will allocate new ones when we run # the job with --lock. if ctx.config.get('targets'): del ctx.config['targets'] job_config = dict( name=ctx.name, last_in_suite=ctx.last_in_suite, email=ctx.email, description=ctx.description, owner=ctx.owner, verbose=ctx.verbose, machine_type=ctx.worker, ) # Merge job_config and ctx.config job_config.update(ctx.config) if ctx.timeout is not None: job_config['results_timeout'] = ctx.timeout job = yaml.safe_dump(job_config) num = ctx.num while num > 0: jid = beanstalk.put( job, ttr=60 * 60 * 24, priority=ctx.priority, ) print 'Job scheduled with name {name} and ID {jid}'.format( name=ctx.name, jid=jid) num -= 1
def analyze(): parser = argparse.ArgumentParser(description=""" Analyze the coverage of a suite of test runs, generating html output with lcov. """) parser.add_argument( '-o', '--lcov-output', help='the directory in which to store results', required=True, ) parser.add_argument( '--html-output', help='the directory in which to store html output', ) parser.add_argument( '--cov-tools-dir', help='the location of coverage scripts (cov-init and cov-analyze)', default='../../coverage', ) parser.add_argument( '--skip-init', help='skip initialization (useful if a run stopped partway through)', action='store_true', default=False, ) parser.add_argument( '-v', '--verbose', help='be more verbose', action='store_true', default=False, ) parser.add_argument( 'test_dir', help='the location of the test results', ) args = parser.parse_args() loglevel = logging.INFO if args.verbose: loglevel = logging.DEBUG logging.basicConfig(level=loglevel, ) teuthology.read_config(args) handler = logging.FileHandler(filename=os.path.join( args.test_dir, 'coverage.log'), ) formatter = logging.Formatter( fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s', datefmt='%Y-%m-%dT%H:%M:%S', ) handler.setFormatter(formatter) logging.getLogger().addHandler(handler) try: _analyze(args) except Exception: log.exception('error generating coverage') raise
def main(): from gevent import monkey; monkey.patch_all() from .orchestra import monkey; monkey.patch_all() import logging log = logging.getLogger(__name__) ctx = parse_args() loglevel = logging.INFO if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, ) from teuthology.misc import read_config read_config(ctx) log.info('\n '.join(['targets:', ] + yaml.safe_dump(ctx.config['targets'], default_flow_style=False).splitlines())) if ctx.owner is None: from teuthology.misc import get_user ctx.owner = get_user() nuke(ctx, log, ctx.unlock, ctx.synch_clocks, ctx.reboot_all)
def main(self): """ Entry point implementing the teuthology-openstack command. """ self.setup_logs() misc.read_config(self.args) self.key_filename = self.args.key_filename self.verify_openstack() ip = self.setup() if self.args.suite: self.run_suite() if self.args.key_filename: identity = '-i ' + self.args.key_filename + ' ' else: identity = '' if self.args.upload: upload = 'upload to : ' + self.args.archive_upload else: upload = '' log.info(""" pulpito web interface: http://{ip}:8081/ ssh access : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/html {upload}""".format(ip=ip, username=self.username, identity=identity, upload=upload)) if self.args.teardown: self.teardown()
def update_hostkeys(): parser = argparse.ArgumentParser(description=""" Update any hostkeys that have changed. You can list specific machines to run on, or use -a to check all of them automatically. """) parser.add_argument( '-t', '--targets', default=None, help='input yaml containing targets to check', ) parser.add_argument( 'machines', metavar='MACHINES', default=[], nargs='*', help='hosts to check for updated keys', ) parser.add_argument( '-v', '--verbose', action='store_true', default=False, help='be more verbose', ) parser.add_argument( '-a', '--all', action='store_true', default=False, help='update hostkeys of all machines in the db', ) ctx = parser.parse_args() loglevel = logging.ERROR if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, ) teuthology.read_config(ctx) assert ctx.all or ctx.targets or ctx.machines, 'You must specify machines to update' if ctx.all: assert not ctx.targets and not ctx.machines, \ 'You can\'t specify machines with the --all option' machines = [canonicalize_hostname(m) for m in ctx.machines] if ctx.targets: try: with file(ctx.targets) as f: g = yaml.safe_load_all(f) for new in g: if 'targets' in new: for t in new['targets'].iterkeys(): machines.append(t) except IOError, e: raise argparse.ArgumentTypeError(str(e))
def main(): from gevent import monkey monkey.patch_all(dns=False) from .orchestra import monkey monkey.patch_all() from teuthology.run import config_file import os import logging log = logging.getLogger(__name__) ctx = parse_args() loglevel = logging.INFO if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig(level=loglevel, ) info = {} if ctx.archive: ctx.config = config_file(ctx.archive + '/config.yaml') ifn = os.path.join(ctx.archive, 'info.yaml') if os.path.exists(ifn): with file(ifn, 'r') as fd: info = yaml.load(fd.read()) if not ctx.pid: ctx.pid = info.get('pid') if not ctx.pid: ctx.pid = int(open(ctx.archive + '/pid').read().rstrip('\n')) if not ctx.owner: ctx.owner = info.get('owner') if not ctx.owner: ctx.owner = open(ctx.archive + '/owner').read().rstrip('\n') ctx.name = info.get('name') from teuthology.misc import read_config read_config(ctx) log.info('\n '.join([ 'targets:', ] + yaml.safe_dump(ctx.config['targets'], default_flow_style=False).splitlines())) if ctx.owner is None: from teuthology.misc import get_user ctx.owner = get_user() if ctx.pid: if ctx.archive: log.info('Killing teuthology process at pid %d', ctx.pid) os.system('grep -q %s /proc/%d/cmdline && sudo kill %d' % (ctx.archive, ctx.pid, ctx.pid)) else: import subprocess subprocess.check_call(["kill", "-9", str(ctx.pid)]) nuke(ctx, log, ctx.unlock, ctx.synch_clocks, ctx.reboot_all, ctx.noipmi)
def main(): from gevent import monkey; monkey.patch_all(dns=False) from .orchestra import monkey; monkey.patch_all() from teuthology.run import config_file import os import logging log = logging.getLogger(__name__) ctx = parse_args() loglevel = logging.INFO if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, ) info = {} if ctx.archive: ctx.config = config_file(ctx.archive + '/config.yaml') ifn = os.path.join(ctx.archive, 'info.yaml') if os.path.exists(ifn): with file(ifn, 'r') as fd: info = yaml.load(fd.read()) if not ctx.pid: ctx.pid = info.get('pid') if not ctx.pid: ctx.pid = int(open(ctx.archive + '/pid').read().rstrip('\n')) if not ctx.owner: ctx.owner = info.get('owner') if not ctx.owner: ctx.owner = open(ctx.archive + '/owner').read().rstrip('\n') ctx.name = info.get('name') from teuthology.misc import read_config read_config(ctx) log.info('\n '.join(['targets:', ] + yaml.safe_dump(ctx.config['targets'], default_flow_style=False).splitlines())) if ctx.owner is None: from teuthology.misc import get_user ctx.owner = get_user() if ctx.pid: if ctx.archive: log.info('Killing teuthology process at pid %d', ctx.pid) os.system('grep -q %s /proc/%d/cmdline && sudo kill %d' % ( ctx.archive, ctx.pid, ctx.pid)) else: import subprocess subprocess.check_call(["kill", "-9", str(ctx.pid)]); nuke(ctx, log, ctx.unlock, ctx.synch_clocks, ctx.reboot_all, ctx.noipmi)
def results(): parser = argparse.ArgumentParser( description='Email teuthology suite results') parser.add_argument( '--email', help='address to email test failures to', ) parser.add_argument( '--timeout', help= 'how many seconds to wait for all tests to finish (default no wait)', type=int, default=0, ) parser.add_argument( '--archive-dir', metavar='DIR', help='path under which results for the suite are stored', required=True, ) parser.add_argument( '--name', help='name of the suite', required=True, ) parser.add_argument( '-v', '--verbose', action='store_true', default=False, help='be more verbose', ) args = parser.parse_args() loglevel = logging.INFO if args.verbose: loglevel = logging.DEBUG logging.basicConfig(level=loglevel, ) misc.read_config(args) handler = logging.FileHandler(filename=os.path.join( args.archive_dir, 'results.log'), ) formatter = logging.Formatter( fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s', datefmt='%Y-%m-%dT%H:%M:%S', ) handler.setFormatter(formatter) logging.getLogger().addHandler(handler) try: _results(args) except Exception: log.exception('error generating results') raise
def results(): parser = argparse.ArgumentParser(description='Email teuthology suite results') parser.add_argument( '--email', help='address to email test failures to', ) parser.add_argument( '--timeout', help='how many seconds to wait for all tests to finish (default no wait)', type=int, default=0, ) parser.add_argument( '--archive-dir', metavar='DIR', help='path under which results for the suite are stored', required=True, ) parser.add_argument( '--name', help='name of the suite', required=True, ) parser.add_argument( '-v', '--verbose', action='store_true', default=False, help='be more verbose', ) args = parser.parse_args() loglevel = logging.INFO if args.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, ) misc.read_config(args) handler = logging.FileHandler( filename=os.path.join(args.archive_dir, 'results.log'), ) formatter = logging.Formatter( fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s', datefmt='%Y-%m-%dT%H:%M:%S', ) handler.setFormatter(formatter) logging.getLogger().addHandler(handler) try: _results(args) except Exception: log.exception('error generating results') raise
def main(): from gevent import monkey monkey.patch_all() from .orchestra import monkey monkey.patch_all() from teuthology.run import config_file import logging log = logging.getLogger(__name__) ctx = parse_args() loglevel = logging.INFO if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig(level=loglevel) if ctx.archive: ctx.config = config_file(ctx.archive + "/config.yaml") if not ctx.pid: ctx.pid = int(open(ctx.archive + "/pid").read().rstrip("\n")) if not ctx.owner: ctx.owner = open(ctx.archive + "/owner").read().rstrip("\n") from teuthology.misc import read_config read_config(ctx) log.info("\n ".join(["targets:"] + yaml.safe_dump(ctx.config["targets"], default_flow_style=False).splitlines())) if ctx.owner is None: from teuthology.misc import get_user ctx.owner = get_user() if ctx.pid: if ctx.archive: import os log.info("Killing teuthology process at pid %d", ctx.pid) os.system("grep -q %s /proc/%d/cmdline && sudo kill %d" % (ctx.archive, ctx.pid, ctx.pid)) else: subprocess.check_call(["kill", "-9", str(ctx.pid)]) nuke(ctx, log, ctx.unlock, ctx.synch_clocks, ctx.reboot_all)
def main(args): log = logging.getLogger(__name__) if args.verbose: teuthology.log.setLevel(logging.DEBUG) misc.read_config(args) log_path = os.path.join(args.archive_dir, 'results.log') teuthology.setup_log_file(log, log_path) try: results(args) except Exception: log.exception('error generating results') raise
def main(args): if args.verbose: teuthology.log.setLevel(logging.DEBUG) log = logging.getLogger(__name__) read_config(args) log_path = os.path.join(args.test_dir, 'coverage.log') teuthology.setup_log_file(log, log_path) try: analyze(args) except Exception: log.exception('error generating coverage') raise
def main(args): if args.verbose: teuthology.log.setLevel(logging.DEBUG) log = logging.getLogger(__name__) read_config(args) log_path = os.path.join(args.test_dir, "coverage.log") teuthology.setup_log_file(log, log_path) try: analyze(args) except Exception: log.exception("error generating coverage") raise
def main(args): if args.verbose: teuthology.log.setLevel(logging.DEBUG) log = logging.getLogger(__name__) read_config(args) handler = logging.FileHandler(filename=os.path.join( args.test_dir, 'coverage.log'), ) formatter = logging.Formatter( fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s', datefmt='%Y-%m-%dT%H:%M:%S', ) handler.setFormatter(formatter) logging.getLogger().addHandler(handler) try: analyze(args) except Exception: log.exception('error generating coverage') raise
def main(args): if args.verbose: teuthology.log.setLevel(logging.DEBUG) log = logging.getLogger(__name__) read_config(args) handler = logging.FileHandler( filename=os.path.join(args.test_dir, 'coverage.log'), ) formatter = logging.Formatter( fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s', datefmt='%Y-%m-%dT%H:%M:%S', ) handler.setFormatter(formatter) logging.getLogger().addHandler(handler) try: analyze(args) except Exception: log.exception('error generating coverage') raise
def main(): from gevent import monkey; monkey.patch_all() from .orchestra import monkey; monkey.patch_all() import time import logging log = logging.getLogger(__name__) ctx = parse_args() loglevel = logging.INFO if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, ) if ctx.block: assert ctx.lock, \ 'the --block option is only supported with the --lock option' from teuthology.misc import read_config read_config(ctx) if ctx.archive is not None: os.mkdir(ctx.archive) handler = logging.FileHandler( filename=os.path.join(ctx.archive, 'teuthology.log'), ) formatter = logging.Formatter( fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s', datefmt='%Y-%m-%dT%H:%M:%S', ) handler.setFormatter(formatter) logging.getLogger().addHandler(handler) with file(os.path.join(ctx.archive, 'pid'), 'w') as f: f.write('%d' % os.getpid()) log.debug('\n '.join(['Config:', ] + yaml.safe_dump(ctx.config, default_flow_style=False).splitlines())) ctx.summary = dict(success=True) if ctx.owner is None: from teuthology.misc import get_user ctx.owner = get_user() ctx.summary['owner'] = ctx.owner if ctx.description is not None: ctx.summary['description'] = ctx.description for task in ctx.config['tasks']: assert 'kernel' not in task, \ 'kernel installation shouldn be a base-level item, not part of the tasks list' init_tasks = [] if ctx.lock: assert 'targets' not in ctx.config, \ 'You cannot specify targets in a config file when using the --lock option' init_tasks.append({'internal.lock_machines': len(ctx.config['roles'])}) init_tasks.extend([ {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.connect': None}, {'internal.check_conflict': None}, ]) if 'kernel' in ctx.config: init_tasks.append({'kernel': ctx.config['kernel']}) init_tasks.extend([ {'internal.base': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.syslog': None}, ]) ctx.config['tasks'][:0] = init_tasks start_time = time.time() from teuthology.run_tasks import run_tasks try: run_tasks(tasks=ctx.config['tasks'], ctx=ctx) finally: end_time = time.time() duration = end_time - start_time ctx.summary['duration'] = duration log.info("Duration was %f seconds" % duration) if not ctx.summary.get('success') and ctx.config.get('nuke-on-error'): from teuthology.parallel import parallel with parallel() as p: for target, hostkey in ctx.config['targets'].iteritems(): p.spawn( nuke, targets={target: hostkey}, owner=ctx.owner, log=log, teuth_config=ctx.teuthology_config, # only unlock if we locked them in the first place should_unlock=ctx.lock, ) if ctx.archive is not None: with file(os.path.join(ctx.archive, 'summary.yaml'), 'w') as f: yaml.safe_dump(ctx.summary, f, default_flow_style=False)
def results(): parser = argparse.ArgumentParser(description='Email teuthology suite results') parser.add_argument( '--email', help='address to email test failures to', ) parser.add_argument( '--timeout', help='how many seconds to wait for all tests to finish (default no wait)', type=int, default=0, ) parser.add_argument( '--archive-dir', metavar='DIR', help='path under which results for the suite are stored', required=True, ) parser.add_argument( '--name', help='name of the suite', required=True, ) parser.add_argument( '-v', '--verbose', action='store_true', default=False, help='be more verbose', ) args = parser.parse_args() loglevel = logging.INFO if args.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, ) teuthology.read_config(args) running_tests = [ f for f in sorted(os.listdir(args.archive_dir)) if not f.startswith('.') and not os.path.exists(os.path.join(args.archive_dir, f, 'summary.yaml')) ] starttime = time.time() while running_tests and args.timeout > 0: if os.path.exists(os.path.join( args.archive_dir, running_tests[-1], 'summary.yaml')): running_tests.pop() else: if time.time() - starttime > args.timeout: log.warn('test(s) did not finish before timeout of %d seconds', args.timeout) break time.sleep(10) subprocess.Popen( args=[ os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-coverage'), '-v', '-o', os.path.join(args.teuthology_config['coverage_output_dir'], args.name), '--html-output', os.path.join(args.teuthology_config['coverage_html_dir'], args.name), '--cov-tools-dir', args.teuthology_config['coverage_tools_dir'], args.archive_dir, ], ) descriptions = [] failures = [] num_failures = 0 unfinished = [] passed = [] all_jobs = sorted(os.listdir(args.archive_dir)) for j in all_jobs: if j.startswith('.'): continue summary_fn = os.path.join(args.archive_dir, j, 'summary.yaml') if not os.path.exists(summary_fn): unfinished.append(j) continue summary = {} with file(summary_fn) as f: g = yaml.safe_load_all(f) for new in g: summary.update(new) desc = '{test}: {desc}'.format( desc=summary['description'], test=j, ) descriptions.append(desc) if summary['success']: passed.append(desc) else: failures.append(desc) num_failures += 1 if 'failure_reason' in summary: failures.append(' {reason}'.format( reason=summary['failure_reason'], )) if not args.email: return if failures or unfinished: subject = ('{num_failed} failed, {num_hung} possibly hung, ' 'and {num_passed} passed tests in {suite}'.format( num_failed=num_failures, num_hung=len(unfinished), num_passed=len(passed), suite=args.name, )) body = """ The following tests failed: {failures} These tests may be hung (did not finish in {timeout} seconds after the last test in the suite): {unfinished} These tests passed: {passed}""".format( failures='\n'.join(failures), unfinished='\n'.join(unfinished), passed='\n'.join(passed), timeout=args.timeout, ) else: subject = 'All tests passed in {suite}!'.format(suite=args.name) body = '\n'.join(descriptions) import smtplib from email.mime.text import MIMEText msg = MIMEText(body) msg['Subject'] = subject msg['From'] = args.teuthology_config['results_sending_email'] msg['To'] = args.email log.debug('sending email %s', msg.as_string()) smtp = smtplib.SMTP('localhost') smtp.sendmail(msg['From'], [msg['To']], msg.as_string()) smtp.quit()
def worker(): parser = argparse.ArgumentParser(description=""" Grab jobs from a beanstalk queue and run the teuthology tests they describe. One job is run at a time. """) parser.add_argument( '-v', '--verbose', action='store_true', default=None, help='be more verbose', ) parser.add_argument( '--archive-dir', metavar='DIR', help='path under which to archive results', required=True, ) parser.add_argument( '-l', '--log-dir', help='path in which to store logs', required=True, ) parser.add_argument( '-t', '--tube', help='which beanstalk tube to read jobs from', required=True, ) ctx = parser.parse_args() loglevel = logging.INFO if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, filename=os.path.join( ctx.log_dir, 'worker.{tube}.{pid}'.format( pid=os.getpid(), tube=ctx.tube, )), format='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s', datefmt='%Y-%m-%dT%H:%M:%S', ) if not os.path.isdir(ctx.archive_dir): sys.exit("{prog}: archive directory must exist: {path}".format( prog=os.path.basename(sys.argv[0]), path=ctx.archive_dir, )) from teuthology.misc import read_config read_config(ctx) beanstalk = connect(ctx) beanstalk.watch(ctx.tube) beanstalk.ignore('default') while True: job = beanstalk.reserve(timeout=60) if job is None: continue # bury the job so it won't be re-run if it fails job.bury() log.debug('Reserved job %d', job.jid) log.debug('Config is: %s', job.body) job_config = yaml.safe_load(job.body) job_config['job_id'] = job.jid safe_archive = safepath.munge(job_config['name']) archive_path_full = os.path.join(ctx.archive_dir, safe_archive, str(job.jid)) job_config['archive_path'] = archive_path_full # If the teuthology branch was not specified, default to master and # store that value. teuthology_branch = job_config.get('teuthology_branch', 'master') job_config['teuthology_branch'] = teuthology_branch teuth_path = os.path.join(os.getenv("HOME"), 'teuthology-' + teuthology_branch) fetch_teuthology_branch(path=teuth_path, branch=teuthology_branch) teuth_bin_path = os.path.join(teuth_path, 'virtualenv', 'bin') if not os.path.isdir(teuth_bin_path): raise RuntimeError("teuthology branch %s at %s not bootstrapped!" % (teuthology_branch, teuth_bin_path)) if job_config.get('last_in_suite'): log.debug('Generating coverage for %s', job_config['name']) args = [ os.path.join(teuth_bin_path, 'teuthology-results'), '--timeout', str(job_config.get('results_timeout', 21600)), '--email', job_config['email'], '--archive-dir', os.path.join(ctx.archive_dir, safe_archive), '--name', job_config['name'], ] subprocess.Popen(args=args) else: log.debug('Creating archive dir...') safepath.makedirs(ctx.archive_dir, safe_archive) log.info('Running job %d', job.jid) run_job(job_config, teuth_bin_path) job.delete()
def main(): from gevent import monkey; monkey.patch_all(dns=False) from .orchestra import monkey; monkey.patch_all() import logging log = logging.getLogger(__name__) ctx = parse_args() loglevel = logging.INFO if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, ) if 'targets' in ctx.config and 'roles' in ctx.config: targets = len(ctx.config['targets']) roles = len(ctx.config['roles']) assert targets >= roles, \ '%d targets are needed for all roles but found %d listed.' % (roles, targets) if ctx.block: assert ctx.lock, \ 'the --block option is only supported with the --lock option' from teuthology.misc import read_config read_config(ctx) log.debug('\n '.join(['Config:', ] + yaml.safe_dump(ctx.config, default_flow_style=False).splitlines())) ctx.summary = dict(success=True) if ctx.owner is None: from teuthology.misc import get_user ctx.owner = get_user() ctx.summary['owner'] = ctx.owner if ctx.description is not None: ctx.summary['description'] = ctx.description if ctx.archive is not None: os.mkdir(ctx.archive) handler = logging.FileHandler( filename=os.path.join(ctx.archive, 'teuthology.log'), ) formatter = logging.Formatter( fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s', datefmt='%Y-%m-%dT%H:%M:%S', ) handler.setFormatter(formatter) logging.getLogger().addHandler(handler) with file(os.path.join(ctx.archive, 'pid'), 'w') as f: f.write('%d' % os.getpid()) with file(os.path.join(ctx.archive, 'owner'), 'w') as f: f.write(ctx.owner + '\n') with file(os.path.join(ctx.archive, 'orig.config.yaml'), 'w') as f: yaml.safe_dump(ctx.config, f, default_flow_style=False) for task in ctx.config['tasks']: assert 'kernel' not in task, \ 'kernel installation shouldn be a base-level item, not part of the tasks list' init_tasks = [] if ctx.lock: assert 'targets' not in ctx.config, \ 'You cannot specify targets in a config file when using the --lock option' init_tasks.append({'internal.lock_machines': len(ctx.config['roles'])}) init_tasks.extend([ {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.connect': None}, {'internal.check_conflict': None}, ]) if 'kernel' in ctx.config: init_tasks.append({'kernel': ctx.config['kernel']}) init_tasks.extend([ {'internal.base': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.syslog': None}, {'internal.timer': None}, ]) ctx.config['tasks'][:0] = init_tasks from teuthology.run_tasks import run_tasks try: run_tasks(tasks=ctx.config['tasks'], ctx=ctx) finally: if not ctx.summary.get('success') and ctx.config.get('nuke-on-error'): from teuthology.nuke import nuke # only unlock if we locked them in the first place nuke(ctx, log, ctx.lock) if ctx.archive is not None: with file(os.path.join(ctx.archive, 'summary.yaml'), 'w') as f: yaml.safe_dump(ctx.summary, f, default_flow_style=False) if not ctx.summary.get('success', True): import sys sys.exit(1)
def schedule(): parser = argparse.ArgumentParser(description='Schedule ceph integration tests') parser.add_argument( 'config', metavar='CONFFILE', nargs='*', type=config_file, action=MergeConfig, default={}, help='config file to read', ) parser.add_argument( '--name', help='name of suite run the job is part of', ) parser.add_argument( '--last-in-suite', action='store_true', default=False, help='mark the last job in a suite so suite post-processing can be run', ) parser.add_argument( '--email', help='where to send the results of a suite (only applies to the last job in a suite)', ) parser.add_argument( '--timeout', help='how many seconds to wait for jobs to finish before emailing results (only applies to the last job in a suite', type=int, ) parser.add_argument( '--description', help='job description', ) parser.add_argument( '--owner', help='job owner', ) parser.add_argument( '--delete', metavar='JOBID', type=int, nargs='*', help='list of jobs to remove from the queue', ) parser.add_argument( '-n', '--num', default=1, type=int, help='number of times to run/queue the job' ) parser.add_argument( '-v', '--verbose', action='store_true', default=False, help='be more verbose', ) parser.add_argument( '-b', '--branch', default='master', help='which branch of teuthology to use', ) parser.add_argument( '-s', '--show', metavar='JOBID', type=int, nargs='*', help='output the contents of specified jobs in the queue', ) ctx = parser.parse_args() if not ctx.last_in_suite: assert not ctx.email, '--email is only applicable to the last job in a suite' assert not ctx.timeout, '--timeout is only applicable to the last job in a suite' from teuthology.misc import read_config, get_user if ctx.owner is None: ctx.owner = 'scheduled_{user}'.format(user=get_user()) read_config(ctx) import teuthology.queue beanstalk = teuthology.queue.connect(ctx) tube = 'teuthology' if ctx.branch != 'master': tube += '-' + ctx.branch beanstalk.use(tube) if ctx.show: for jobid in ctx.show: job = beanstalk.peek(jobid) if job is None and ctx.verbose: print 'job {jid} is not in the queue'.format(jid=jobid) else: print 'job {jid} contains: '.format(jid=jobid), job.body return if ctx.delete: for jobid in ctx.delete: job = beanstalk.peek(jobid) if job is None: print 'job {jid} is not in the queue'.format(jid=jobid) else: job.delete() return job_config = dict( config=ctx.config, name=ctx.name, last_in_suite=ctx.last_in_suite, email=ctx.email, description=ctx.description, owner=ctx.owner, verbose=ctx.verbose, ) if ctx.timeout is not None: job_config['results_timeout'] = ctx.timeout job = yaml.safe_dump(job_config) num = ctx.num while num > 0: jid = beanstalk.put(job, ttr=60*60*24) print 'Job scheduled with ID {jid}'.format(jid=jid) num -= 1
def worker(): parser = argparse.ArgumentParser( description=""" Grab jobs from a beanstalk queue and run the teuthology tests they describe. One job is run at a time. """ ) parser.add_argument("-v", "--verbose", action="store_true", default=None, help="be more verbose") parser.add_argument("--archive-dir", metavar="DIR", help="path under which to archive results", required=True) parser.add_argument("-l", "--log-dir", help="path in which to store logs", required=True) parser.add_argument("-t", "--tube", help="which beanstalk tube to read jobs from", required=True) ctx = parser.parse_args() loglevel = logging.INFO if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, filename=os.path.join(ctx.log_dir, "worker.{pid}".format(pid=os.getpid())), format="%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s", datefmt="%Y-%m-%dT%H:%M:%S", ) if not os.path.isdir(ctx.archive_dir): sys.exit( "{prog}: archive directory must exist: {path}".format( prog=os.path.basename(sys.argv[0]), path=ctx.archive_dir ) ) from teuthology.misc import read_config read_config(ctx) beanstalk = connect(ctx) beanstalk.watch(ctx.tube) beanstalk.ignore("default") while True: job = beanstalk.reserve(timeout=60) if job is None: continue # bury the job so it won't be re-run if it fails job.bury() log.debug("Reserved job %d", job.jid) log.debug("Config is: %s", job.body) job_config = yaml.safe_load(job.body) safe_archive = safepath.munge(job_config["name"]) if job_config.get("last_in_suite", False): log.debug("Generating coverage for %s", job_config["name"]) args = [ os.path.join(os.path.dirname(sys.argv[0]), "teuthology-results"), "--timeout", str(job_config.get("results_timeout", 21600)), "--email", job_config["email"], "--archive-dir", os.path.join(ctx.archive_dir, safe_archive), "--name", job_config["name"], ] subprocess.Popen(args=args) else: log.debug("Creating archive dir...") safepath.makedirs(ctx.archive_dir, safe_archive) archive_path = os.path.join(ctx.archive_dir, safe_archive, str(job.jid)) log.info("Running job %d", job.jid) run_job(job_config, archive_path) job.delete()
def analyze(): parser = argparse.ArgumentParser(description=""" Analyze the coverage of a suite of test runs, generating html output with lcov. """) parser.add_argument( '-o', '--lcov-output', help='the directory in which to store results', required=True, ) parser.add_argument( '--html-output', help='the directory in which to store html output', ) parser.add_argument( '--cov-tools-dir', help='the location of coverage scripts (cov-init and cov-analyze)', default='../../coverage', ) parser.add_argument( '--skip-init', help='skip initialization (useful if a run stopped partway through)', action='store_true', default=False, ) parser.add_argument( '-v', '--verbose', help='be more verbose', action='store_true', default=False, ) parser.add_argument( 'test_dir', help='the location of the test results', ) args = parser.parse_args() loglevel = logging.INFO if args.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, ) teuthology.read_config(args) handler = logging.FileHandler( filename=os.path.join(args.test_dir, 'coverage.log'), ) formatter = logging.Formatter( fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s', datefmt='%Y-%m-%dT%H:%M:%S', ) handler.setFormatter(formatter) logging.getLogger().addHandler(handler) try: _analyze(args) except Exception: log.exception('error generating coverage') raise
def main(): from gevent import monkey monkey.patch_all(dns=False) from .orchestra import monkey monkey.patch_all() import logging ctx = parse_args() set_up_logging(ctx) log = logging.getLogger(__name__) if ctx.owner is None: from teuthology.misc import get_user ctx.owner = get_user() write_initial_metadata(ctx) if 'targets' in ctx.config and 'roles' in ctx.config: targets = len(ctx.config['targets']) roles = len(ctx.config['roles']) assert targets >= roles, \ '%d targets are needed for all roles but found %d listed.' % (roles, targets) machine_type = ctx.machine_type if machine_type is None: fallback_default = ctx.config.get('machine_type', 'plana') machine_type = ctx.config.get('machine-type', fallback_default) if ctx.block: assert ctx.lock, \ 'the --block option is only supported with the --lock option' from teuthology.misc import read_config read_config(ctx) log.debug('\n '.join(['Config:', ] + yaml.safe_dump(ctx.config, default_flow_style=False).splitlines())) ctx.summary = dict(success=True) ctx.summary['owner'] = ctx.owner if ctx.description is not None: ctx.summary['description'] = ctx.description for task in ctx.config['tasks']: assert 'kernel' not in task, \ 'kernel installation shouldn be a base-level item, not part of the tasks list' init_tasks = [] if ctx.lock: assert 'targets' not in ctx.config, \ 'You cannot specify targets in a config file when using the --lock option' init_tasks.append({'internal.lock_machines': (len(ctx.config['roles']), machine_type)}) init_tasks.extend([ {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.connect': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, ]) if 'kernel' in ctx.config: from teuthology.misc import get_distro distro = get_distro(ctx) if distro == 'ubuntu': init_tasks.append({'kernel': ctx.config['kernel']}) init_tasks.extend([ {'internal.base': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, ]) ctx.config['tasks'][:0] = init_tasks from teuthology.run_tasks import run_tasks try: run_tasks(tasks=ctx.config['tasks'], ctx=ctx) finally: if not ctx.summary.get('success') and ctx.config.get('nuke-on-error'): from teuthology.nuke import nuke # only unlock if we locked them in the first place nuke(ctx, log, ctx.lock) if ctx.archive is not None: with file(os.path.join(ctx.archive, 'summary.yaml'), 'w') as f: yaml.safe_dump(ctx.summary, f, default_flow_style=False) with contextlib.closing(StringIO.StringIO()) as f: yaml.safe_dump(ctx.summary, f) log.info('Summary data:\n%s' % f.getvalue()) with contextlib.closing(StringIO.StringIO()) as f: if 'email-on-error' in ctx.config and not ctx.summary.get('success', False): yaml.safe_dump(ctx.summary, f) yaml.safe_dump(ctx.config, f) emsg = f.getvalue() subject = "Teuthology error -- %s" % ctx.summary['failure_reason'] from teuthology.suite import email_results email_results(subject,"Teuthology",ctx.config['email-on-error'],emsg) if ctx.summary.get('success', True): log.info('pass') else: log.info('FAIL') import sys sys.exit(1)
def main(ctx): if ctx.owner is None: ctx.owner = 'scheduled_{user}'.format(user=get_user()) read_config(ctx) beanstalk = teuthology.beanstalk.connect() tube = ctx.worker beanstalk.use(tube) if ctx.show: for job_id in ctx.show: job = beanstalk.peek(job_id) if job is None and ctx.verbose: print 'job {jid} is not in the queue'.format(jid=job_id) else: print '--- job {jid} priority {prio} ---\n'.format( jid=job_id, prio=job.stats()['pri']), job.body return if ctx.delete: for job_id in ctx.delete: job = beanstalk.peek(job_id) if job is None: print 'job {jid} is not in the queue'.format(jid=job_id) else: job.delete() name = yaml.safe_load(job.body).get('name') if name: report.try_delete_jobs(name, job_id) return # strip out targets; the worker will allocate new ones when we run # the job with --lock. if ctx.config.get('targets'): del ctx.config['targets'] job_config = dict( name=ctx.name, last_in_suite=ctx.last_in_suite, email=ctx.email, description=ctx.description, owner=ctx.owner, verbose=ctx.verbose, machine_type=ctx.worker, ) # Merge job_config and ctx.config job_config.update(ctx.config) if ctx.timeout is not None: job_config['results_timeout'] = ctx.timeout job = yaml.safe_dump(job_config) num = ctx.num while num > 0: jid = beanstalk.put( job, ttr=60 * 60 * 24, priority=ctx.priority, ) print 'Job scheduled with name {name} and ID {jid}'.format( name=ctx.name, jid=jid) job_config['job_id'] = str(jid) report.try_push_job_info(job_config, dict(status='queued')) num -= 1
def main(): from gevent import monkey monkey.patch_all(dns=False) from .orchestra import monkey monkey.patch_all() import logging ctx = parse_args() set_up_logging(ctx) log = logging.getLogger(__name__) if ctx.owner is None: from teuthology.misc import get_user ctx.owner = get_user() write_initial_metadata(ctx) if 'targets' in ctx.config and 'roles' in ctx.config: targets = len(ctx.config['targets']) roles = len(ctx.config['roles']) assert targets >= roles, \ '%d targets are needed for all roles but found %d listed.' % (roles, targets) machine_type = ctx.machine_type if machine_type is None: fallback_default = ctx.config.get('machine_type', 'plana') machine_type = ctx.config.get('machine-type', fallback_default) if ctx.block: assert ctx.lock, \ 'the --block option is only supported with the --lock option' from teuthology.misc import read_config read_config(ctx) log.debug('\n '.join([ 'Config:', ] + yaml.safe_dump(ctx.config, default_flow_style=False).splitlines())) ctx.summary = dict(success=True) ctx.summary['owner'] = ctx.owner if ctx.description is not None: ctx.summary['description'] = ctx.description for task in ctx.config['tasks']: assert 'kernel' not in task, \ 'kernel installation shouldn be a base-level item, not part of the tasks list' init_tasks = [] if ctx.lock: assert 'targets' not in ctx.config, \ 'You cannot specify targets in a config file when using the --lock option' init_tasks.append({ 'internal.lock_machines': (len(ctx.config['roles']), machine_type) }) init_tasks.extend([ { 'internal.save_config': None }, { 'internal.check_lock': None }, { 'internal.connect': None }, { 'internal.check_conflict': None }, { 'internal.check_ceph_data': None }, { 'internal.vm_setup': None }, ]) if 'kernel' in ctx.config: from teuthology.misc import get_distro distro = get_distro(ctx) if distro == 'ubuntu': init_tasks.append({'kernel': ctx.config['kernel']}) init_tasks.extend([ { 'internal.base': None }, { 'internal.archive': None }, { 'internal.coredump': None }, { 'internal.sudo': None }, { 'internal.syslog': None }, { 'internal.timer': None }, ]) ctx.config['tasks'][:0] = init_tasks from teuthology.run_tasks import run_tasks try: run_tasks(tasks=ctx.config['tasks'], ctx=ctx) finally: if not ctx.summary.get('success') and ctx.config.get('nuke-on-error'): from teuthology.nuke import nuke # only unlock if we locked them in the first place nuke(ctx, log, ctx.lock) if ctx.archive is not None: with file(os.path.join(ctx.archive, 'summary.yaml'), 'w') as f: yaml.safe_dump(ctx.summary, f, default_flow_style=False) with contextlib.closing(StringIO.StringIO()) as f: yaml.safe_dump(ctx.summary, f) log.info('Summary data:\n%s' % f.getvalue()) with contextlib.closing(StringIO.StringIO()) as f: if 'email-on-error' in ctx.config and not ctx.summary.get( 'success', False): yaml.safe_dump(ctx.summary, f) yaml.safe_dump(ctx.config, f) emsg = f.getvalue() subject = "Teuthology error -- %s" % ctx.summary[ 'failure_reason'] from teuthology.suite import email_results email_results(subject, "Teuthology", ctx.config['email-on-error'], emsg) if ctx.summary.get('success', True): log.info('pass') else: log.info('FAIL') import sys sys.exit(1)
def analyze(): parser = argparse.ArgumentParser(description=""" Analyze the coverage of a suite of test runs, generating html output with lcov. """) parser.add_argument( '-o', '--lcov-output', help='the directory in which to store results', required=True, ) parser.add_argument( '--html-output', help='the directory in which to store html output', ) parser.add_argument( '--cov-tools-dir', help='the location of coverage scripts (cov-init and cov-analyze)', default='../../coverage', ) parser.add_argument( '--skip-init', help='skip initialization (useful if a run stopped partway through)', action='store_true', default=False, ) parser.add_argument( '-v', '--verbose', help='be more verbose', action='store_true', default=False, ) parser.add_argument( 'test_dir', help='the location of the test results', ) args = parser.parse_args() loglevel = logging.INFO if args.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, ) teuthology.read_config(args) tests = [ f for f in sorted(os.listdir(args.test_dir)) if not f.startswith('.') and os.path.exists(os.path.join(args.test_dir, f, 'summary.yaml')) and os.path.exists(os.path.join(args.test_dir, f, 'ceph-sha1'))] test_summaries = {} for test in tests: summary = {} with file(os.path.join(args.test_dir, test, 'summary.yaml')) as f: g = yaml.safe_load_all(f) for new in g: summary.update(new) if summary['flavor'] != 'gcov': log.debug('Skipping %s, since it does not include coverage', test) continue test_summaries[test] = summary assert len(test_summaries) > 0 suite = os.path.basename(args.test_dir) # only run cov-init once. # this only works if all tests were run against the same version. if not args.skip_init: log.info('initializing coverage data...') subprocess.check_call( args=[ os.path.join(args.cov_tools_dir, 'cov-init.sh'), os.path.join(args.test_dir, tests[0]), args.lcov_output, os.path.join( args.teuthology_config['ceph_build_output_dir'], '{suite}.tgz'.format(suite=suite), ), ]) shutil.copy( os.path.join(args.lcov_output, 'base.lcov'), os.path.join(args.lcov_output, 'total.lcov') ) test_coverage = {} for test, summary in test_summaries.iteritems(): lcov_file = '{name}.lcov'.format(name=test) log.info('analyzing coverage for %s', test) proc = subprocess.Popen( args=[ os.path.join(args.cov_tools_dir, 'cov-analyze.sh'), '-t', os.path.join(args.test_dir, test), '-d', args.lcov_output, '-o', test, ], stdout=subprocess.PIPE, ) output, _ = proc.communicate() desc = summary.get('description', test) test_coverage[desc] = read_coverage(output) log.info('adding %s data to total', test) proc = subprocess.Popen( args=[ 'lcov', '-a', os.path.join(args.lcov_output, lcov_file), '-a', os.path.join(args.lcov_output, 'total.lcov'), '-o', os.path.join(args.lcov_output, 'total_tmp.lcov'), ], stdout=subprocess.PIPE, ) output, _ = proc.communicate() os.rename( os.path.join(args.lcov_output, 'total_tmp.lcov'), os.path.join(args.lcov_output, 'total.lcov') ) coverage = read_coverage(output) test_coverage['total for {suite}'.format(suite=suite)] = coverage log.debug('total coverage is %s', str(coverage)) if args.html_output: subprocess.check_call( args=[ 'genhtml', '-s', '-o', os.path.join(args.html_output, 'total'), '-t', 'Total for {suite}'.format(suite=suite), '--', os.path.join(args.lcov_output, 'total.lcov'), ]) store_coverage(args, test_coverage, summary['ceph-sha1'], suite)
def worker(): parser = argparse.ArgumentParser(description=""" Grab jobs from a beanstalk queue and run the teuthology tests they describe. One job is run at a time. """) parser.add_argument( '-v', '--verbose', action='store_true', default=None, help='be more verbose', ) parser.add_argument( '--archive-dir', metavar='DIR', help='path under which to archive results', required=True, ) parser.add_argument( '-l', '--log-dir', help='path in which to store logs', required=True, ) ctx = parser.parse_args() loglevel = logging.INFO if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, filename=os.path.join(ctx.log_dir, 'worker.{pid}'.format(pid=os.getpid())), format='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s', datefmt='%Y-%m-%dT%H:%M:%S', ) if not os.path.isdir(ctx.archive_dir): sys.exit("{prog}: archive directory must exist: {path}".format( prog=os.path.basename(sys.argv[0]), path=ctx.archive_dir, )) from teuthology.misc import read_config read_config(ctx) beanstalk = connect(ctx) beanstalk.watch('teuthology') beanstalk.ignore('default') while True: job = beanstalk.reserve(timeout=60) if job is None: continue # bury the job so it won't be re-run if it fails job.bury() log.debug('Reserved job %d', job.jid) log.debug('Config is: %s', job.body) job_config = yaml.safe_load(job.body) safe_archive = safepath.munge(job_config['name']) if job_config.get('last_in_suite', False): log.debug('Generating coverage for %s', job_config['name']) args = [ os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-results'), '--timeout', str(job_config.get('results_timeout', 21600)), '--email', job_config['email'], '--archive-dir', os.path.join(ctx.archive_dir, safe_archive), '--name', job_config['name'], ] subprocess.Popen(args=args) else: log.debug('Creating archive dir...') safepath.makedirs(ctx.archive_dir, safe_archive) archive_path = os.path.join(ctx.archive_dir, safe_archive, str(job.jid)) log.info('Running job %d', job.jid) run_job(job_config, archive_path) job.delete()
def schedule(): parser = argparse.ArgumentParser(description='Schedule ceph integration tests') parser.add_argument( 'config', metavar='CONFFILE', nargs='*', type=config_file, action=MergeConfig, default={}, help='config file to read', ) parser.add_argument( '--name', help='name of suite run the job is part of', ) parser.add_argument( '--last-in-suite', action='store_true', default=False, help='mark the last job in a suite so suite post-processing can be run', ) parser.add_argument( '--email', help='where to send the results of a suite (only applies to the last job in a suite)', ) parser.add_argument( '--timeout', help='how many seconds to wait for jobs to finish before emailing results (only applies to the last job in a suite', type=int, ) parser.add_argument( '--description', help='job description', ) parser.add_argument( '--owner', help='job owner', ) parser.add_argument( '--delete', metavar='JOBID', type=int, nargs='*', help='list of jobs to remove from the queue', ) parser.add_argument( '-v', '--verbose', action='store_true', default=False, help='be more verbose', ) ctx = parser.parse_args() if not ctx.last_in_suite: assert not ctx.email, '--email is only applicable to the last job in a suite' assert not ctx.timeout, '--timeout is only applicable to the last job in a suite' from teuthology.misc import read_config, get_user if ctx.owner is None: ctx.owner = 'scheduled_{user}'.format(user=get_user()) read_config(ctx) import teuthology.queue beanstalk = teuthology.queue.connect(ctx) beanstalk.use('teuthology') if ctx.delete: for jobid in ctx.delete: job = beanstalk.peek(jobid) if job is None: print 'job {jid} is not in the queue'.format(jid=jobid) else: job.delete() return job_config = dict( config=ctx.config, name=ctx.name, last_in_suite=ctx.last_in_suite, email=ctx.email, description=ctx.description, owner=ctx.owner, verbose=ctx.verbose, ) if ctx.timeout is not None: job_config['results_timeout'] = ctx.timeout job = yaml.safe_dump(job_config) jid = beanstalk.put(job, ttr=60*60*24) print 'Job scheduled with ID {jid}'.format(jid=jid)
def schedule(): parser = argparse.ArgumentParser( description='Schedule ceph integration tests') parser.add_argument( 'config', metavar='CONFFILE', nargs='*', type=config_file, action=MergeConfig, default={}, help='config file to read', ) parser.add_argument( '--name', help='name of suite run the job is part of', ) parser.add_argument( '--last-in-suite', action='store_true', default=False, help='mark the last job in a suite so suite post-processing can be run', ) parser.add_argument( '--email', help= 'where to send the results of a suite (only applies to the last job in a suite)', ) parser.add_argument( '--timeout', help= 'how many seconds to wait for jobs to finish before emailing results (only applies to the last job in a suite', type=int, ) parser.add_argument( '--description', help='job description', ) parser.add_argument( '--owner', help='job owner', ) parser.add_argument( '--delete', metavar='JOBID', type=int, nargs='*', help='list of jobs to remove from the queue', ) parser.add_argument('-n', '--num', default=1, type=int, help='number of times to run/queue the job') parser.add_argument('-p', '--priority', default=1000, type=int, help='beanstalk priority (lower is sooner)') parser.add_argument( '-v', '--verbose', action='store_true', default=False, help='be more verbose', ) parser.add_argument( '-w', '--worker', default='plana', help='which worker to use (type of machine)', ) parser.add_argument( '-s', '--show', metavar='JOBID', type=int, nargs='*', help='output the contents of specified jobs in the queue', ) ctx = parser.parse_args() if not ctx.last_in_suite: assert not ctx.email, '--email is only applicable to the last job in a suite' assert not ctx.timeout, '--timeout is only applicable to the last job in a suite' from teuthology.misc import read_config, get_user if ctx.owner is None: ctx.owner = 'scheduled_{user}'.format(user=get_user()) read_config(ctx) import teuthology.queue beanstalk = teuthology.queue.connect(ctx) tube = ctx.worker beanstalk.use(tube) if ctx.show: for job_id in ctx.show: job = beanstalk.peek(job_id) if job is None and ctx.verbose: print 'job {jid} is not in the queue'.format(jid=job_id) else: print '--- job {jid} priority {prio} ---\n'.format( jid=job_id, prio=job.stats()['pri']), job.body return if ctx.delete: for job_id in ctx.delete: job = beanstalk.peek(job_id) if job is None: print 'job {jid} is not in the queue'.format(jid=job_id) else: job.delete() return # strip out targets; the worker will allocate new ones when we run # the job with --lock. if ctx.config.get('targets'): del ctx.config['targets'] job_config = dict( name=ctx.name, last_in_suite=ctx.last_in_suite, email=ctx.email, description=ctx.description, owner=ctx.owner, verbose=ctx.verbose, ) # Merge job_config and ctx.config job_config.update(ctx.config) if ctx.timeout is not None: job_config['results_timeout'] = ctx.timeout job = yaml.safe_dump(job_config) num = ctx.num while num > 0: jid = beanstalk.put( job, ttr=60 * 60 * 24, priority=ctx.priority, ) print 'Job scheduled with ID {jid}'.format(jid=jid) num -= 1
def main(): parser = argparse.ArgumentParser(description=""" Lock, unlock, or query lock status of machines. """) parser.add_argument( '-v', '--verbose', action='store_true', default=False, help='be more verbose', ) group = parser.add_mutually_exclusive_group(required=True) group.add_argument( '--list', action='store_true', default=False, help= 'Show lock info for machines owned by you, or only machines specified. Can be restricted by --owner and --status.', ) group.add_argument( '--list-targets', action='store_true', default=False, help= 'Show lock info for all machines, or only machines specified, in targets: yaml format. Can be restricted by --owner and --status.', ) group.add_argument( '--lock', action='store_true', default=False, help='lock particular machines', ) group.add_argument( '--unlock', action='store_true', default=False, help='unlock particular machines', ) group.add_argument( '--lock-many', dest='num_to_lock', type=_positive_int, help='lock this many machines', ) group.add_argument( '--update', action='store_true', default=False, help='update the description or status of some machines', ) parser.add_argument( '-a', '--all', action='store_true', default=False, help='list all machines, not just those owned by you', ) parser.add_argument( '--owner', default=None, help='owner of the lock(s) (must match to unlock a machine)', ) parser.add_argument( '-f', action='store_true', default=False, help= 'don\'t exit after the first error, continue locking or unlocking other machines', ) parser.add_argument( '--desc', default=None, help='update description', ) parser.add_argument( '--status', default=None, choices=['up', 'down'], help='whether a machine is usable for testing', ) parser.add_argument( '-t', '--targets', dest='targets', default=None, help='input yaml containing targets', ) parser.add_argument( 'machines', metavar='MACHINE', default=[], nargs='*', help='machines to operate on', ) ctx = parser.parse_args() loglevel = logging.ERROR if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig(level=loglevel, ) teuthology.read_config(ctx) ret = 0 user = ctx.owner machines = ctx.machines machines_to_update = [] if ctx.targets: try: with file(ctx.targets) as f: g = yaml.safe_load_all(f) for new in g: if 'targets' in new: for t in new['targets'].iterkeys(): machines.append(t) except IOError, e: raise argparse.ArgumentTypeError(str(e))
def main(): parser = argparse.ArgumentParser(description=""" Lock, unlock, or query lock status of machines. """) parser.add_argument( '-v', '--verbose', action='store_true', default=False, help='be more verbose', ) group = parser.add_mutually_exclusive_group(required=True) group.add_argument( '--list', action='store_true', default=False, help='Show lock info for machines owned by you, or only machines specified. Can be restricted by --owner, --status, and --locked.', ) group.add_argument( '--list-targets', action='store_true', default=False, help='Show lock info for all machines, or only machines specified, in targets: yaml format. Can be restricted by --owner, --status, and --locked.', ) group.add_argument( '--lock', action='store_true', default=False, help='lock particular machines', ) group.add_argument( '--unlock', action='store_true', default=False, help='unlock particular machines', ) group.add_argument( '--lock-many', dest='num_to_lock', type=_positive_int, help='lock this many machines', ) group.add_argument( '--update', action='store_true', default=False, help='update the description or status of some machines', ) group.add_argument( '--summary', action='store_true', default=False, help='summarize locked-machine counts by owner', ) parser.add_argument( '-a', '--all', action='store_true', default=False, help='list all machines, not just those owned by you', ) parser.add_argument( '--owner', default=None, help='owner of the lock(s) (must match to unlock a machine)', ) parser.add_argument( '-f', action='store_true', default=False, help='don\'t exit after the first error, continue locking or unlocking other machines', ) parser.add_argument( '--desc', default=None, help='update description', ) parser.add_argument( '--status', default=None, choices=['up', 'down'], help='whether a machine is usable for testing', ) parser.add_argument( '--locked', default=None, choices=['true', 'false'], help='whether a machine is locked', ) parser.add_argument( '--brief', action='store_true', default=False, help='Shorten information reported from --list', ) parser.add_argument( '-t', '--targets', dest='targets', default=None, help='input yaml containing targets', ) parser.add_argument( 'machines', metavar='MACHINE', default=[], nargs='*', help='machines to operate on', ) ctx = parser.parse_args() loglevel = logging.ERROR if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, ) teuthology.read_config(ctx) ret = 0 user = ctx.owner machines = [canonicalize_hostname(m) for m in ctx.machines] machines_to_update = [] if ctx.targets: try: with file(ctx.targets) as f: g = yaml.safe_load_all(f) for new in g: if 'targets' in new: for t in new['targets'].iterkeys(): machines.append(t) except IOError, e: raise argparse.ArgumentTypeError(str(e))
def analyze(): parser = argparse.ArgumentParser(description=""" Analyze the coverage of a suite of test runs, generating html output with lcov. """) parser.add_argument( '-o', '--lcov-output', help='the directory in which to store results', required=True, ) parser.add_argument( '--html-output', help='the directory in which to store html output', ) parser.add_argument( '--cov-tools-dir', help='the location of coverage scripts (cov-init and cov-analyze)', default='../../coverage', ) parser.add_argument( '--skip-init', help='skip initialization (useful if a run stopped partway through)', action='store_true', default=False, ) parser.add_argument( '-v', '--verbose', help='be more verbose', action='store_true', default=False, ) parser.add_argument( 'test_dir', help='the location of the test results', ) args = parser.parse_args() loglevel = logging.INFO if args.verbose: loglevel = logging.DEBUG logging.basicConfig(level=loglevel, ) teuthology.read_config(args) tests = [ f for f in sorted(os.listdir(args.test_dir)) if not f.startswith('.') and os.path.exists(os.path.join(args.test_dir, f, 'summary.yaml')) and os.path.exists(os.path.join(args.test_dir, f, 'ceph-sha1')) ] test_summaries = {} for test in tests: summary = {} with file(os.path.join(args.test_dir, test, 'summary.yaml')) as f: g = yaml.safe_load_all(f) for new in g: summary.update(new) if summary['flavor'] != 'gcov': log.debug('Skipping %s, since it does not include coverage', test) continue test_summaries[test] = summary assert len(test_summaries) > 0 suite = os.path.basename(args.test_dir) # only run cov-init once. # this only works if all tests were run against the same version. if not args.skip_init: log.info('initializing coverage data...') subprocess.check_call(args=[ os.path.join(args.cov_tools_dir, 'cov-init.sh'), os.path.join(args.test_dir, tests[0]), args.lcov_output, os.path.join( args.teuthology_config['ceph_build_output_dir'], '{suite}.tgz'.format(suite=suite), ), ]) shutil.copy(os.path.join(args.lcov_output, 'base.lcov'), os.path.join(args.lcov_output, 'total.lcov')) test_coverage = {} for test, summary in test_summaries.iteritems(): lcov_file = '{name}.lcov'.format(name=test) log.info('analyzing coverage for %s', test) proc = subprocess.Popen( args=[ os.path.join(args.cov_tools_dir, 'cov-analyze.sh'), '-t', os.path.join(args.test_dir, test), '-d', args.lcov_output, '-o', test, ], stdout=subprocess.PIPE, ) output, _ = proc.communicate() desc = summary.get('description', test) test_coverage[desc] = read_coverage(output) log.info('adding %s data to total', test) proc = subprocess.Popen( args=[ 'lcov', '-a', os.path.join(args.lcov_output, lcov_file), '-a', os.path.join(args.lcov_output, 'total.lcov'), '-o', os.path.join(args.lcov_output, 'total_tmp.lcov'), ], stdout=subprocess.PIPE, ) output, _ = proc.communicate() os.rename(os.path.join(args.lcov_output, 'total_tmp.lcov'), os.path.join(args.lcov_output, 'total.lcov')) coverage = read_coverage(output) test_coverage['total for {suite}'.format(suite=suite)] = coverage log.debug('total coverage is %s', str(coverage)) if args.html_output: subprocess.check_call(args=[ 'genhtml', '-s', '-o', os.path.join(args.html_output, 'total'), '-t', 'Total for {suite}'.format(suite=suite), '--', os.path.join(args.lcov_output, 'total.lcov'), ]) store_coverage(args, test_coverage, summary['ceph-sha1'], suite)
def worker(): parser = argparse.ArgumentParser(description=""" Grab jobs from a beanstalk queue and run the teuthology tests they describe. One job is run at a time. """) parser.add_argument( '-v', '--verbose', action='store_true', default=None, help='be more verbose', ) parser.add_argument( '--archive-dir', metavar='DIR', help='path under which to archive results', required=True, ) parser.add_argument( '-l', '--log-dir', help='path in which to store logs', required=True, ) parser.add_argument( '-t', '--tube', help='which beanstalk tube to read jobs from', required=True, ) ctx = parser.parse_args() loglevel = logging.INFO if ctx.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, filename=os.path.join(ctx.log_dir, 'worker.{pid}'.format(pid=os.getpid())), format='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s', datefmt='%Y-%m-%dT%H:%M:%S', ) if not os.path.isdir(ctx.archive_dir): sys.exit("{prog}: archive directory must exist: {path}".format( prog=os.path.basename(sys.argv[0]), path=ctx.archive_dir, )) from teuthology.misc import read_config read_config(ctx) beanstalk = connect(ctx) beanstalk.watch(ctx.tube) beanstalk.ignore('default') while True: job = beanstalk.reserve(timeout=60) if job is None: continue # bury the job so it won't be re-run if it fails job.bury() log.debug('Reserved job %d', job.jid) log.debug('Config is: %s', job.body) job_config = yaml.safe_load(job.body) safe_archive = safepath.munge(job_config['name']) teuthology_branch = job_config.get('config', {}).get('teuthology_branch', 'master') teuth_path = os.path.join(os.getenv("HOME"), 'teuthology-' + teuthology_branch, 'virtualenv', 'bin') if not os.path.isdir(teuth_path): raise Exception('Teuthology branch ' + teuthology_branch + ' not found at ' + teuth_path) if job_config.get('last_in_suite'): log.debug('Generating coverage for %s', job_config['name']) args = [ os.path.join(teuth_path, 'teuthology-results'), '--timeout', str(job_config.get('results_timeout', 21600)), '--email', job_config['email'], '--archive-dir', os.path.join(ctx.archive_dir, safe_archive), '--name', job_config['name'], ] subprocess.Popen(args=args) else: log.debug('Creating archive dir...') safepath.makedirs(ctx.archive_dir, safe_archive) archive_path = os.path.join(ctx.archive_dir, safe_archive, str(job.jid)) log.info('Running job %d', job.jid) run_job(job_config, archive_path, teuth_path) job.delete()
def schedule(): parser = argparse.ArgumentParser(description='Schedule ceph integration tests') parser.add_argument( 'config', metavar='CONFFILE', nargs='*', type=config_file, action=MergeConfig, default={}, help='config file to read', ) parser.add_argument( '--name', help='name of suite run the job is part of', ) parser.add_argument( '--last-in-suite', action='store_true', default=False, help='mark the last job in a suite so suite post-processing can be run', ) parser.add_argument( '--email', help='where to send the results of a suite (only applies to the last job in a suite)', ) parser.add_argument( '--timeout', help='how many seconds to wait for jobs to finish before emailing results (only applies to the last job in a suite', type=int, ) parser.add_argument( '--description', help='job description', ) parser.add_argument( '--owner', help='job owner', ) parser.add_argument( '--delete', metavar='JOBID', type=int, nargs='*', help='list of jobs to remove from the queue', ) parser.add_argument( '-n', '--num', default=1, type=int, help='number of times to run/queue the job' ) parser.add_argument( '-p', '--priority', default=1000, type=int, help='beanstalk priority (lower is sooner)' ) parser.add_argument( '-v', '--verbose', action='store_true', default=False, help='be more verbose', ) parser.add_argument( '-w', '--worker', default='plana', help='which worker to use (type of machine)', ) parser.add_argument( '-s', '--show', metavar='JOBID', type=int, nargs='*', help='output the contents of specified jobs in the queue', ) ctx = parser.parse_args() if not ctx.last_in_suite: assert not ctx.email, '--email is only applicable to the last job in a suite' assert not ctx.timeout, '--timeout is only applicable to the last job in a suite' from teuthology.misc import read_config, get_user if ctx.owner is None: ctx.owner = 'scheduled_{user}'.format(user=get_user()) read_config(ctx) import teuthology.queue beanstalk = teuthology.queue.connect(ctx) tube=ctx.worker beanstalk.use(tube) if ctx.show: for job_id in ctx.show: job = beanstalk.peek(job_id) if job is None and ctx.verbose: print 'job {jid} is not in the queue'.format(jid=job_id) else: print '--- job {jid} priority {prio} ---\n'.format( jid=job_id, prio=job.stats()['pri']), job.body return if ctx.delete: for job_id in ctx.delete: job = beanstalk.peek(job_id) if job is None: print 'job {jid} is not in the queue'.format(jid=job_id) else: job.delete() return # strip out targets; the worker will allocate new ones when we run # the job with --lock. if ctx.config.get('targets'): del ctx.config['targets'] job_config = dict( name=ctx.name, last_in_suite=ctx.last_in_suite, email=ctx.email, description=ctx.description, owner=ctx.owner, verbose=ctx.verbose, ) # Merge job_config and ctx.config job_config.update(ctx.config) if ctx.timeout is not None: job_config['results_timeout'] = ctx.timeout job = yaml.safe_dump(job_config) num = ctx.num while num > 0: jid = beanstalk.put( job, ttr=60*60*24, priority=ctx.priority, ) print 'Job scheduled with ID {jid}'.format(jid=jid) num -= 1