def main():
from gevent import monkey
monkey.patch_all()
from .orchestra import monkey
monkey.patch_all()
import logging
log = logging.getLogger(__name__)
ctx = parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(level=loglevel, )
from teuthology.misc import read_config
read_config(ctx)
log.info('\n '.join([
'targets:',
] + yaml.safe_dump(ctx.config['targets'],
default_flow_style=False).splitlines()))
if ctx.owner is None:
from teuthology.misc import get_user
ctx.owner = get_user()
nuke(ctx, log)
def main(self):
"""
Entry point implementing the teuthology-openstack command.
"""
self.setup_logs()
misc.read_config(self.args)
self.key_filename = self.args.key_filename
self.verify_openstack()
ip = self.setup()
if self.args.suite:
self.run_suite()
if self.args.key_filename:
identity = '-i ' + self.args.key_filename + ' '
else:
identity = ''
if self.args.upload:
upload = 'upload to : ' + self.args.archive_upload
else:
upload = ''
log.info("""
web interface: http://{ip}:8081/
ssh access : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/html
{upload}""".format(ip=ip,
username=self.username,
identity=identity,
upload=upload))
if self.args.teardown:
self.teardown()
def update_hostkeys():
parser = argparse.ArgumentParser(description="""
Update any hostkeys that have changed. You can list specific machines
to run on, or use -a to check all of them automatically.
""")
parser.add_argument(
'-t',
'--targets',
default=None,
help='input yaml containing targets to check',
)
parser.add_argument(
'machines',
metavar='MACHINES',
default=[],
nargs='*',
help='hosts to check for updated keys',
)
parser.add_argument(
'-v',
'--verbose',
action='store_true',
default=False,
help='be more verbose',
)
parser.add_argument(
'-a',
'--all',
action='store_true',
default=False,
help='update hostkeys of all machines in the db',
)
ctx = parser.parse_args()
loglevel = logging.ERROR
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(level=loglevel, )
teuthology.read_config(ctx)
assert ctx.all or ctx.targets or ctx.machines, 'You must specify machines to update'
if ctx.all:
assert not ctx.targets and not ctx.machines, \
'You can\'t specify machines with the --all option'
machines = ctx.machines
if ctx.targets:
try:
with file(ctx.targets) as f:
g = yaml.safe_load_all(f)
for new in g:
if 'targets' in new:
for t in new['targets'].iterkeys():
machines.append(t)
except IOError, e:
raise argparse.ArgumentTypeError(str(e))
def main(ctx):
if ctx.owner is None:
ctx.owner = 'scheduled_{user}'.format(user=get_user())
read_config(ctx)
beanstalk = teuthology.queue.connect(ctx)
tube = ctx.worker
beanstalk.use(tube)
if ctx.show:
for job_id in ctx.show:
job = beanstalk.peek(job_id)
if job is None and ctx.verbose:
print 'job {jid} is not in the queue'.format(jid=job_id)
else:
print '--- job {jid} priority {prio} ---\n'.format(
jid=job_id,
prio=job.stats()['pri']), job.body
return
if ctx.delete:
for job_id in ctx.delete:
job = beanstalk.peek(job_id)
if job is None:
print 'job {jid} is not in the queue'.format(jid=job_id)
else:
job.delete()
return
# strip out targets; the worker will allocate new ones when we run
# the job with --lock.
if ctx.config.get('targets'):
del ctx.config['targets']
job_config = dict(
name=ctx.name,
last_in_suite=ctx.last_in_suite,
email=ctx.email,
description=ctx.description,
owner=ctx.owner,
verbose=ctx.verbose,
machine_type=ctx.worker,
)
# Merge job_config and ctx.config
job_config.update(ctx.config)
if ctx.timeout is not None:
job_config['results_timeout'] = ctx.timeout
job = yaml.safe_dump(job_config)
num = ctx.num
while num > 0:
jid = beanstalk.put(
job,
ttr=60 * 60 * 24,
priority=ctx.priority,
)
print 'Job scheduled with name {name} and ID {jid}'.format(
name=ctx.name, jid=jid)
num -= 1
def analyze():
parser = argparse.ArgumentParser(description="""
Analyze the coverage of a suite of test runs, generating html output with lcov.
""")
parser.add_argument(
'-o',
'--lcov-output',
help='the directory in which to store results',
required=True,
)
parser.add_argument(
'--html-output',
help='the directory in which to store html output',
)
parser.add_argument(
'--cov-tools-dir',
help='the location of coverage scripts (cov-init and cov-analyze)',
default='../../coverage',
)
parser.add_argument(
'--skip-init',
help='skip initialization (useful if a run stopped partway through)',
action='store_true',
default=False,
)
parser.add_argument(
'-v',
'--verbose',
help='be more verbose',
action='store_true',
default=False,
)
parser.add_argument(
'test_dir',
help='the location of the test results',
)
args = parser.parse_args()
loglevel = logging.INFO
if args.verbose:
loglevel = logging.DEBUG
logging.basicConfig(level=loglevel, )
teuthology.read_config(args)
handler = logging.FileHandler(filename=os.path.join(
args.test_dir, 'coverage.log'), )
formatter = logging.Formatter(
fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
)
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
try:
_analyze(args)
except Exception:
log.exception('error generating coverage')
raise
def main():
from gevent import monkey; monkey.patch_all()
from .orchestra import monkey; monkey.patch_all()
import logging
log = logging.getLogger(__name__)
ctx = parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
)
from teuthology.misc import read_config
read_config(ctx)
log.info('\n '.join(['targets:', ] + yaml.safe_dump(ctx.config['targets'], default_flow_style=False).splitlines()))
if ctx.owner is None:
from teuthology.misc import get_user
ctx.owner = get_user()
nuke(ctx, log, ctx.unlock, ctx.synch_clocks, ctx.reboot_all)
def main(self):
"""
Entry point implementing the teuthology-openstack command.
"""
self.setup_logs()
misc.read_config(self.args)
self.key_filename = self.args.key_filename
self.verify_openstack()
ip = self.setup()
if self.args.suite:
self.run_suite()
if self.args.key_filename:
identity = '-i ' + self.args.key_filename + ' '
else:
identity = ''
if self.args.upload:
upload = 'upload to : ' + self.args.archive_upload
else:
upload = ''
log.info("""
pulpito web interface: http://{ip}:8081/
ssh access : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/html
{upload}""".format(ip=ip,
username=self.username,
identity=identity,
upload=upload))
if self.args.teardown:
self.teardown()
def update_hostkeys():
parser = argparse.ArgumentParser(description="""
Update any hostkeys that have changed. You can list specific machines
to run on, or use -a to check all of them automatically.
""")
parser.add_argument(
'-t', '--targets',
default=None,
help='input yaml containing targets to check',
)
parser.add_argument(
'machines',
metavar='MACHINES',
default=[],
nargs='*',
help='hosts to check for updated keys',
)
parser.add_argument(
'-v', '--verbose',
action='store_true',
default=False,
help='be more verbose',
)
parser.add_argument(
'-a', '--all',
action='store_true',
default=False,
help='update hostkeys of all machines in the db',
)
ctx = parser.parse_args()
loglevel = logging.ERROR
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
)
teuthology.read_config(ctx)
assert ctx.all or ctx.targets or ctx.machines, 'You must specify machines to update'
if ctx.all:
assert not ctx.targets and not ctx.machines, \
'You can\'t specify machines with the --all option'
machines = [canonicalize_hostname(m) for m in ctx.machines]
if ctx.targets:
try:
with file(ctx.targets) as f:
g = yaml.safe_load_all(f)
for new in g:
if 'targets' in new:
for t in new['targets'].iterkeys():
machines.append(t)
except IOError, e:
raise argparse.ArgumentTypeError(str(e))
def main():
from gevent import monkey
monkey.patch_all(dns=False)
from .orchestra import monkey
monkey.patch_all()
from teuthology.run import config_file
import os
import logging
log = logging.getLogger(__name__)
ctx = parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(level=loglevel, )
info = {}
if ctx.archive:
ctx.config = config_file(ctx.archive + '/config.yaml')
ifn = os.path.join(ctx.archive, 'info.yaml')
if os.path.exists(ifn):
with file(ifn, 'r') as fd:
info = yaml.load(fd.read())
if not ctx.pid:
ctx.pid = info.get('pid')
if not ctx.pid:
ctx.pid = int(open(ctx.archive + '/pid').read().rstrip('\n'))
if not ctx.owner:
ctx.owner = info.get('owner')
if not ctx.owner:
ctx.owner = open(ctx.archive + '/owner').read().rstrip('\n')
ctx.name = info.get('name')
from teuthology.misc import read_config
read_config(ctx)
log.info('\n '.join([
'targets:',
] + yaml.safe_dump(ctx.config['targets'],
default_flow_style=False).splitlines()))
if ctx.owner is None:
from teuthology.misc import get_user
ctx.owner = get_user()
if ctx.pid:
if ctx.archive:
log.info('Killing teuthology process at pid %d', ctx.pid)
os.system('grep -q %s /proc/%d/cmdline && sudo kill %d' %
(ctx.archive, ctx.pid, ctx.pid))
else:
import subprocess
subprocess.check_call(["kill", "-9", str(ctx.pid)])
nuke(ctx, log, ctx.unlock, ctx.synch_clocks, ctx.reboot_all, ctx.noipmi)
def main():
from gevent import monkey; monkey.patch_all(dns=False)
from .orchestra import monkey; monkey.patch_all()
from teuthology.run import config_file
import os
import logging
log = logging.getLogger(__name__)
ctx = parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
)
info = {}
if ctx.archive:
ctx.config = config_file(ctx.archive + '/config.yaml')
ifn = os.path.join(ctx.archive, 'info.yaml')
if os.path.exists(ifn):
with file(ifn, 'r') as fd:
info = yaml.load(fd.read())
if not ctx.pid:
ctx.pid = info.get('pid')
if not ctx.pid:
ctx.pid = int(open(ctx.archive + '/pid').read().rstrip('\n'))
if not ctx.owner:
ctx.owner = info.get('owner')
if not ctx.owner:
ctx.owner = open(ctx.archive + '/owner').read().rstrip('\n')
ctx.name = info.get('name')
from teuthology.misc import read_config
read_config(ctx)
log.info('\n '.join(['targets:', ] + yaml.safe_dump(ctx.config['targets'], default_flow_style=False).splitlines()))
if ctx.owner is None:
from teuthology.misc import get_user
ctx.owner = get_user()
if ctx.pid:
if ctx.archive:
log.info('Killing teuthology process at pid %d', ctx.pid)
os.system('grep -q %s /proc/%d/cmdline && sudo kill %d' % (
ctx.archive,
ctx.pid,
ctx.pid))
else:
import subprocess
subprocess.check_call(["kill", "-9", str(ctx.pid)]);
nuke(ctx, log, ctx.unlock, ctx.synch_clocks, ctx.reboot_all, ctx.noipmi)
def results():
parser = argparse.ArgumentParser(
description='Email teuthology suite results')
parser.add_argument(
'--email',
help='address to email test failures to',
)
parser.add_argument(
'--timeout',
help=
'how many seconds to wait for all tests to finish (default no wait)',
type=int,
default=0,
)
parser.add_argument(
'--archive-dir',
metavar='DIR',
help='path under which results for the suite are stored',
required=True,
)
parser.add_argument(
'--name',
help='name of the suite',
required=True,
)
parser.add_argument(
'-v',
'--verbose',
action='store_true',
default=False,
help='be more verbose',
)
args = parser.parse_args()
loglevel = logging.INFO
if args.verbose:
loglevel = logging.DEBUG
logging.basicConfig(level=loglevel, )
misc.read_config(args)
handler = logging.FileHandler(filename=os.path.join(
args.archive_dir, 'results.log'), )
formatter = logging.Formatter(
fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
)
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
try:
_results(args)
except Exception:
log.exception('error generating results')
raise
def results():
parser = argparse.ArgumentParser(description='Email teuthology suite results')
parser.add_argument(
'--email',
help='address to email test failures to',
)
parser.add_argument(
'--timeout',
help='how many seconds to wait for all tests to finish (default no wait)',
type=int,
default=0,
)
parser.add_argument(
'--archive-dir',
metavar='DIR',
help='path under which results for the suite are stored',
required=True,
)
parser.add_argument(
'--name',
help='name of the suite',
required=True,
)
parser.add_argument(
'-v', '--verbose',
action='store_true', default=False,
help='be more verbose',
)
args = parser.parse_args()
loglevel = logging.INFO
if args.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
)
misc.read_config(args)
handler = logging.FileHandler(
filename=os.path.join(args.archive_dir, 'results.log'),
)
formatter = logging.Formatter(
fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
)
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
try:
_results(args)
except Exception:
log.exception('error generating results')
raise
def main():
from gevent import monkey
monkey.patch_all()
from .orchestra import monkey
monkey.patch_all()
from teuthology.run import config_file
import logging
log = logging.getLogger(__name__)
ctx = parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(level=loglevel)
if ctx.archive:
ctx.config = config_file(ctx.archive + "/config.yaml")
if not ctx.pid:
ctx.pid = int(open(ctx.archive + "/pid").read().rstrip("\n"))
if not ctx.owner:
ctx.owner = open(ctx.archive + "/owner").read().rstrip("\n")
from teuthology.misc import read_config
read_config(ctx)
log.info("\n ".join(["targets:"] + yaml.safe_dump(ctx.config["targets"], default_flow_style=False).splitlines()))
if ctx.owner is None:
from teuthology.misc import get_user
ctx.owner = get_user()
if ctx.pid:
if ctx.archive:
import os
log.info("Killing teuthology process at pid %d", ctx.pid)
os.system("grep -q %s /proc/%d/cmdline && sudo kill %d" % (ctx.archive, ctx.pid, ctx.pid))
else:
subprocess.check_call(["kill", "-9", str(ctx.pid)])
nuke(ctx, log, ctx.unlock, ctx.synch_clocks, ctx.reboot_all)
def main(args):
log = logging.getLogger(__name__)
if args.verbose:
teuthology.log.setLevel(logging.DEBUG)
misc.read_config(args)
log_path = os.path.join(args.archive_dir, 'results.log')
teuthology.setup_log_file(log, log_path)
try:
results(args)
except Exception:
log.exception('error generating results')
raise
def main(args):
if args.verbose:
teuthology.log.setLevel(logging.DEBUG)
log = logging.getLogger(__name__)
read_config(args)
log_path = os.path.join(args.test_dir, 'coverage.log')
teuthology.setup_log_file(log, log_path)
try:
analyze(args)
except Exception:
log.exception('error generating coverage')
raise
def main(args):
if args.verbose:
teuthology.log.setLevel(logging.DEBUG)
log = logging.getLogger(__name__)
read_config(args)
log_path = os.path.join(args.test_dir, "coverage.log")
teuthology.setup_log_file(log, log_path)
try:
analyze(args)
except Exception:
log.exception("error generating coverage")
raise
def main(args):
if args.verbose:
teuthology.log.setLevel(logging.DEBUG)
log = logging.getLogger(__name__)
read_config(args)
handler = logging.FileHandler(filename=os.path.join(
args.test_dir, 'coverage.log'), )
formatter = logging.Formatter(
fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
)
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
try:
analyze(args)
except Exception:
log.exception('error generating coverage')
raise
def main(args):
if args.verbose:
teuthology.log.setLevel(logging.DEBUG)
log = logging.getLogger(__name__)
read_config(args)
handler = logging.FileHandler(
filename=os.path.join(args.test_dir, 'coverage.log'),
)
formatter = logging.Formatter(
fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
)
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
try:
analyze(args)
except Exception:
log.exception('error generating coverage')
raise
def main():
from gevent import monkey; monkey.patch_all()
from .orchestra import monkey; monkey.patch_all()
import time
import logging
log = logging.getLogger(__name__)
ctx = parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
)
if ctx.block:
assert ctx.lock, \
'the --block option is only supported with the --lock option'
from teuthology.misc import read_config
read_config(ctx)
if ctx.archive is not None:
os.mkdir(ctx.archive)
handler = logging.FileHandler(
filename=os.path.join(ctx.archive, 'teuthology.log'),
)
formatter = logging.Formatter(
fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
)
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
with file(os.path.join(ctx.archive, 'pid'), 'w') as f:
f.write('%d' % os.getpid())
log.debug('\n '.join(['Config:', ] + yaml.safe_dump(ctx.config, default_flow_style=False).splitlines()))
ctx.summary = dict(success=True)
if ctx.owner is None:
from teuthology.misc import get_user
ctx.owner = get_user()
ctx.summary['owner'] = ctx.owner
if ctx.description is not None:
ctx.summary['description'] = ctx.description
for task in ctx.config['tasks']:
assert 'kernel' not in task, \
'kernel installation shouldn be a base-level item, not part of the tasks list'
init_tasks = []
if ctx.lock:
assert 'targets' not in ctx.config, \
'You cannot specify targets in a config file when using the --lock option'
init_tasks.append({'internal.lock_machines': len(ctx.config['roles'])})
init_tasks.extend([
{'internal.save_config': None},
{'internal.check_lock': None},
{'internal.connect': None},
{'internal.check_conflict': None},
])
if 'kernel' in ctx.config:
init_tasks.append({'kernel': ctx.config['kernel']})
init_tasks.extend([
{'internal.base': None},
{'internal.archive': None},
{'internal.coredump': None},
{'internal.syslog': None},
])
ctx.config['tasks'][:0] = init_tasks
start_time = time.time()
from teuthology.run_tasks import run_tasks
try:
run_tasks(tasks=ctx.config['tasks'], ctx=ctx)
finally:
end_time = time.time()
duration = end_time - start_time
ctx.summary['duration'] = duration
log.info("Duration was %f seconds" % duration)
if not ctx.summary.get('success') and ctx.config.get('nuke-on-error'):
from teuthology.parallel import parallel
with parallel() as p:
for target, hostkey in ctx.config['targets'].iteritems():
p.spawn(
nuke,
targets={target: hostkey},
owner=ctx.owner,
log=log,
teuth_config=ctx.teuthology_config,
# only unlock if we locked them in the first place
should_unlock=ctx.lock,
)
if ctx.archive is not None:
with file(os.path.join(ctx.archive, 'summary.yaml'), 'w') as f:
yaml.safe_dump(ctx.summary, f, default_flow_style=False)
def results():
parser = argparse.ArgumentParser(description='Email teuthology suite results')
parser.add_argument(
'--email',
help='address to email test failures to',
)
parser.add_argument(
'--timeout',
help='how many seconds to wait for all tests to finish (default no wait)',
type=int,
default=0,
)
parser.add_argument(
'--archive-dir',
metavar='DIR',
help='path under which results for the suite are stored',
required=True,
)
parser.add_argument(
'--name',
help='name of the suite',
required=True,
)
parser.add_argument(
'-v', '--verbose',
action='store_true', default=False,
help='be more verbose',
)
args = parser.parse_args()
loglevel = logging.INFO
if args.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
)
teuthology.read_config(args)
running_tests = [
f for f in sorted(os.listdir(args.archive_dir))
if not f.startswith('.')
and not os.path.exists(os.path.join(args.archive_dir, f, 'summary.yaml'))
]
starttime = time.time()
while running_tests and args.timeout > 0:
if os.path.exists(os.path.join(
args.archive_dir,
running_tests[-1], 'summary.yaml')):
running_tests.pop()
else:
if time.time() - starttime > args.timeout:
log.warn('test(s) did not finish before timeout of %d seconds',
args.timeout)
break
time.sleep(10)
subprocess.Popen(
args=[
os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-coverage'),
'-v',
'-o',
os.path.join(args.teuthology_config['coverage_output_dir'], args.name),
'--html-output',
os.path.join(args.teuthology_config['coverage_html_dir'], args.name),
'--cov-tools-dir',
args.teuthology_config['coverage_tools_dir'],
args.archive_dir,
],
)
descriptions = []
failures = []
num_failures = 0
unfinished = []
passed = []
all_jobs = sorted(os.listdir(args.archive_dir))
for j in all_jobs:
if j.startswith('.'):
continue
summary_fn = os.path.join(args.archive_dir, j, 'summary.yaml')
if not os.path.exists(summary_fn):
unfinished.append(j)
continue
summary = {}
with file(summary_fn) as f:
g = yaml.safe_load_all(f)
for new in g:
summary.update(new)
desc = '{test}: {desc}'.format(
desc=summary['description'],
test=j,
)
descriptions.append(desc)
if summary['success']:
passed.append(desc)
else:
failures.append(desc)
num_failures += 1
if 'failure_reason' in summary:
failures.append(' {reason}'.format(
reason=summary['failure_reason'],
))
if not args.email:
return
if failures or unfinished:
subject = ('{num_failed} failed, {num_hung} possibly hung, '
'and {num_passed} passed tests in {suite}'.format(
num_failed=num_failures,
num_hung=len(unfinished),
num_passed=len(passed),
suite=args.name,
))
body = """
The following tests failed:
{failures}
These tests may be hung (did not finish in {timeout} seconds after the last test in the suite):
{unfinished}
These tests passed:
{passed}""".format(
failures='\n'.join(failures),
unfinished='\n'.join(unfinished),
passed='\n'.join(passed),
timeout=args.timeout,
)
else:
subject = 'All tests passed in {suite}!'.format(suite=args.name)
body = '\n'.join(descriptions)
import smtplib
from email.mime.text import MIMEText
msg = MIMEText(body)
msg['Subject'] = subject
msg['From'] = args.teuthology_config['results_sending_email']
msg['To'] = args.email
log.debug('sending email %s', msg.as_string())
smtp = smtplib.SMTP('localhost')
smtp.sendmail(msg['From'], [msg['To']], msg.as_string())
smtp.quit()
def worker():
parser = argparse.ArgumentParser(description="""
Grab jobs from a beanstalk queue and run the teuthology tests they
describe. One job is run at a time.
""")
parser.add_argument(
'-v',
'--verbose',
action='store_true',
default=None,
help='be more verbose',
)
parser.add_argument(
'--archive-dir',
metavar='DIR',
help='path under which to archive results',
required=True,
)
parser.add_argument(
'-l',
'--log-dir',
help='path in which to store logs',
required=True,
)
parser.add_argument(
'-t',
'--tube',
help='which beanstalk tube to read jobs from',
required=True,
)
ctx = parser.parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
filename=os.path.join(
ctx.log_dir, 'worker.{tube}.{pid}'.format(
pid=os.getpid(),
tube=ctx.tube,
)),
format='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
)
if not os.path.isdir(ctx.archive_dir):
sys.exit("{prog}: archive directory must exist: {path}".format(
prog=os.path.basename(sys.argv[0]),
path=ctx.archive_dir,
))
from teuthology.misc import read_config
read_config(ctx)
beanstalk = connect(ctx)
beanstalk.watch(ctx.tube)
beanstalk.ignore('default')
while True:
job = beanstalk.reserve(timeout=60)
if job is None:
continue
# bury the job so it won't be re-run if it fails
job.bury()
log.debug('Reserved job %d', job.jid)
log.debug('Config is: %s', job.body)
job_config = yaml.safe_load(job.body)
job_config['job_id'] = job.jid
safe_archive = safepath.munge(job_config['name'])
archive_path_full = os.path.join(ctx.archive_dir, safe_archive,
str(job.jid))
job_config['archive_path'] = archive_path_full
# If the teuthology branch was not specified, default to master and
# store that value.
teuthology_branch = job_config.get('teuthology_branch', 'master')
job_config['teuthology_branch'] = teuthology_branch
teuth_path = os.path.join(os.getenv("HOME"),
'teuthology-' + teuthology_branch)
fetch_teuthology_branch(path=teuth_path, branch=teuthology_branch)
teuth_bin_path = os.path.join(teuth_path, 'virtualenv', 'bin')
if not os.path.isdir(teuth_bin_path):
raise RuntimeError("teuthology branch %s at %s not bootstrapped!" %
(teuthology_branch, teuth_bin_path))
if job_config.get('last_in_suite'):
log.debug('Generating coverage for %s', job_config['name'])
args = [
os.path.join(teuth_bin_path, 'teuthology-results'),
'--timeout',
str(job_config.get('results_timeout', 21600)),
'--email',
job_config['email'],
'--archive-dir',
os.path.join(ctx.archive_dir, safe_archive),
'--name',
job_config['name'],
]
subprocess.Popen(args=args)
else:
log.debug('Creating archive dir...')
safepath.makedirs(ctx.archive_dir, safe_archive)
log.info('Running job %d', job.jid)
run_job(job_config, teuth_bin_path)
job.delete()
def main():
from gevent import monkey; monkey.patch_all(dns=False)
from .orchestra import monkey; monkey.patch_all()
import logging
log = logging.getLogger(__name__)
ctx = parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
)
if 'targets' in ctx.config and 'roles' in ctx.config:
targets = len(ctx.config['targets'])
roles = len(ctx.config['roles'])
assert targets >= roles, \
'%d targets are needed for all roles but found %d listed.' % (roles, targets)
if ctx.block:
assert ctx.lock, \
'the --block option is only supported with the --lock option'
from teuthology.misc import read_config
read_config(ctx)
log.debug('\n '.join(['Config:', ] + yaml.safe_dump(ctx.config, default_flow_style=False).splitlines()))
ctx.summary = dict(success=True)
if ctx.owner is None:
from teuthology.misc import get_user
ctx.owner = get_user()
ctx.summary['owner'] = ctx.owner
if ctx.description is not None:
ctx.summary['description'] = ctx.description
if ctx.archive is not None:
os.mkdir(ctx.archive)
handler = logging.FileHandler(
filename=os.path.join(ctx.archive, 'teuthology.log'),
)
formatter = logging.Formatter(
fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
)
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
with file(os.path.join(ctx.archive, 'pid'), 'w') as f:
f.write('%d' % os.getpid())
with file(os.path.join(ctx.archive, 'owner'), 'w') as f:
f.write(ctx.owner + '\n')
with file(os.path.join(ctx.archive, 'orig.config.yaml'), 'w') as f:
yaml.safe_dump(ctx.config, f, default_flow_style=False)
for task in ctx.config['tasks']:
assert 'kernel' not in task, \
'kernel installation shouldn be a base-level item, not part of the tasks list'
init_tasks = []
if ctx.lock:
assert 'targets' not in ctx.config, \
'You cannot specify targets in a config file when using the --lock option'
init_tasks.append({'internal.lock_machines': len(ctx.config['roles'])})
init_tasks.extend([
{'internal.save_config': None},
{'internal.check_lock': None},
{'internal.connect': None},
{'internal.check_conflict': None},
])
if 'kernel' in ctx.config:
init_tasks.append({'kernel': ctx.config['kernel']})
init_tasks.extend([
{'internal.base': None},
{'internal.archive': None},
{'internal.coredump': None},
{'internal.syslog': None},
{'internal.timer': None},
])
ctx.config['tasks'][:0] = init_tasks
from teuthology.run_tasks import run_tasks
try:
run_tasks(tasks=ctx.config['tasks'], ctx=ctx)
finally:
if not ctx.summary.get('success') and ctx.config.get('nuke-on-error'):
from teuthology.nuke import nuke
# only unlock if we locked them in the first place
nuke(ctx, log, ctx.lock)
if ctx.archive is not None:
with file(os.path.join(ctx.archive, 'summary.yaml'), 'w') as f:
yaml.safe_dump(ctx.summary, f, default_flow_style=False)
if not ctx.summary.get('success', True):
import sys
sys.exit(1)
def schedule():
parser = argparse.ArgumentParser(description='Schedule ceph integration tests')
parser.add_argument(
'config',
metavar='CONFFILE',
nargs='*',
type=config_file,
action=MergeConfig,
default={},
help='config file to read',
)
parser.add_argument(
'--name',
help='name of suite run the job is part of',
)
parser.add_argument(
'--last-in-suite',
action='store_true',
default=False,
help='mark the last job in a suite so suite post-processing can be run',
)
parser.add_argument(
'--email',
help='where to send the results of a suite (only applies to the last job in a suite)',
)
parser.add_argument(
'--timeout',
help='how many seconds to wait for jobs to finish before emailing results (only applies to the last job in a suite',
type=int,
)
parser.add_argument(
'--description',
help='job description',
)
parser.add_argument(
'--owner',
help='job owner',
)
parser.add_argument(
'--delete',
metavar='JOBID',
type=int,
nargs='*',
help='list of jobs to remove from the queue',
)
parser.add_argument(
'-n', '--num',
default=1,
type=int,
help='number of times to run/queue the job'
)
parser.add_argument(
'-v', '--verbose',
action='store_true',
default=False,
help='be more verbose',
)
parser.add_argument(
'-b', '--branch',
default='master',
help='which branch of teuthology to use',
)
parser.add_argument(
'-s', '--show',
metavar='JOBID',
type=int,
nargs='*',
help='output the contents of specified jobs in the queue',
)
ctx = parser.parse_args()
if not ctx.last_in_suite:
assert not ctx.email, '--email is only applicable to the last job in a suite'
assert not ctx.timeout, '--timeout is only applicable to the last job in a suite'
from teuthology.misc import read_config, get_user
if ctx.owner is None:
ctx.owner = 'scheduled_{user}'.format(user=get_user())
read_config(ctx)
import teuthology.queue
beanstalk = teuthology.queue.connect(ctx)
tube = 'teuthology'
if ctx.branch != 'master':
tube += '-' + ctx.branch
beanstalk.use(tube)
if ctx.show:
for jobid in ctx.show:
job = beanstalk.peek(jobid)
if job is None and ctx.verbose:
print 'job {jid} is not in the queue'.format(jid=jobid)
else:
print 'job {jid} contains: '.format(jid=jobid), job.body
return
if ctx.delete:
for jobid in ctx.delete:
job = beanstalk.peek(jobid)
if job is None:
print 'job {jid} is not in the queue'.format(jid=jobid)
else:
job.delete()
return
job_config = dict(
config=ctx.config,
name=ctx.name,
last_in_suite=ctx.last_in_suite,
email=ctx.email,
description=ctx.description,
owner=ctx.owner,
verbose=ctx.verbose,
)
if ctx.timeout is not None:
job_config['results_timeout'] = ctx.timeout
job = yaml.safe_dump(job_config)
num = ctx.num
while num > 0:
jid = beanstalk.put(job, ttr=60*60*24)
print 'Job scheduled with ID {jid}'.format(jid=jid)
num -= 1
def worker():
parser = argparse.ArgumentParser(
description="""
Grab jobs from a beanstalk queue and run the teuthology tests they
describe. One job is run at a time.
"""
)
parser.add_argument("-v", "--verbose", action="store_true", default=None, help="be more verbose")
parser.add_argument("--archive-dir", metavar="DIR", help="path under which to archive results", required=True)
parser.add_argument("-l", "--log-dir", help="path in which to store logs", required=True)
parser.add_argument("-t", "--tube", help="which beanstalk tube to read jobs from", required=True)
ctx = parser.parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
filename=os.path.join(ctx.log_dir, "worker.{pid}".format(pid=os.getpid())),
format="%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s",
datefmt="%Y-%m-%dT%H:%M:%S",
)
if not os.path.isdir(ctx.archive_dir):
sys.exit(
"{prog}: archive directory must exist: {path}".format(
prog=os.path.basename(sys.argv[0]), path=ctx.archive_dir
)
)
from teuthology.misc import read_config
read_config(ctx)
beanstalk = connect(ctx)
beanstalk.watch(ctx.tube)
beanstalk.ignore("default")
while True:
job = beanstalk.reserve(timeout=60)
if job is None:
continue
# bury the job so it won't be re-run if it fails
job.bury()
log.debug("Reserved job %d", job.jid)
log.debug("Config is: %s", job.body)
job_config = yaml.safe_load(job.body)
safe_archive = safepath.munge(job_config["name"])
if job_config.get("last_in_suite", False):
log.debug("Generating coverage for %s", job_config["name"])
args = [
os.path.join(os.path.dirname(sys.argv[0]), "teuthology-results"),
"--timeout",
str(job_config.get("results_timeout", 21600)),
"--email",
job_config["email"],
"--archive-dir",
os.path.join(ctx.archive_dir, safe_archive),
"--name",
job_config["name"],
]
subprocess.Popen(args=args)
else:
log.debug("Creating archive dir...")
safepath.makedirs(ctx.archive_dir, safe_archive)
archive_path = os.path.join(ctx.archive_dir, safe_archive, str(job.jid))
log.info("Running job %d", job.jid)
run_job(job_config, archive_path)
job.delete()
def analyze():
parser = argparse.ArgumentParser(description="""
Analyze the coverage of a suite of test runs, generating html output with lcov.
""")
parser.add_argument(
'-o', '--lcov-output',
help='the directory in which to store results',
required=True,
)
parser.add_argument(
'--html-output',
help='the directory in which to store html output',
)
parser.add_argument(
'--cov-tools-dir',
help='the location of coverage scripts (cov-init and cov-analyze)',
default='../../coverage',
)
parser.add_argument(
'--skip-init',
help='skip initialization (useful if a run stopped partway through)',
action='store_true',
default=False,
)
parser.add_argument(
'-v', '--verbose',
help='be more verbose',
action='store_true',
default=False,
)
parser.add_argument(
'test_dir',
help='the location of the test results',
)
args = parser.parse_args()
loglevel = logging.INFO
if args.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
)
teuthology.read_config(args)
handler = logging.FileHandler(
filename=os.path.join(args.test_dir, 'coverage.log'),
)
formatter = logging.Formatter(
fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
)
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
try:
_analyze(args)
except Exception:
log.exception('error generating coverage')
raise
def main():
from gevent import monkey
monkey.patch_all(dns=False)
from .orchestra import monkey
monkey.patch_all()
import logging
ctx = parse_args()
set_up_logging(ctx)
log = logging.getLogger(__name__)
if ctx.owner is None:
from teuthology.misc import get_user
ctx.owner = get_user()
write_initial_metadata(ctx)
if 'targets' in ctx.config and 'roles' in ctx.config:
targets = len(ctx.config['targets'])
roles = len(ctx.config['roles'])
assert targets >= roles, \
'%d targets are needed for all roles but found %d listed.' % (roles, targets)
machine_type = ctx.machine_type
if machine_type is None:
fallback_default = ctx.config.get('machine_type', 'plana')
machine_type = ctx.config.get('machine-type', fallback_default)
if ctx.block:
assert ctx.lock, \
'the --block option is only supported with the --lock option'
from teuthology.misc import read_config
read_config(ctx)
log.debug('\n '.join(['Config:', ] + yaml.safe_dump(ctx.config, default_flow_style=False).splitlines()))
ctx.summary = dict(success=True)
ctx.summary['owner'] = ctx.owner
if ctx.description is not None:
ctx.summary['description'] = ctx.description
for task in ctx.config['tasks']:
assert 'kernel' not in task, \
'kernel installation shouldn be a base-level item, not part of the tasks list'
init_tasks = []
if ctx.lock:
assert 'targets' not in ctx.config, \
'You cannot specify targets in a config file when using the --lock option'
init_tasks.append({'internal.lock_machines': (len(ctx.config['roles']), machine_type)})
init_tasks.extend([
{'internal.save_config': None},
{'internal.check_lock': None},
{'internal.connect': None},
{'internal.check_conflict': None},
{'internal.check_ceph_data': None},
{'internal.vm_setup': None},
])
if 'kernel' in ctx.config:
from teuthology.misc import get_distro
distro = get_distro(ctx)
if distro == 'ubuntu':
init_tasks.append({'kernel': ctx.config['kernel']})
init_tasks.extend([
{'internal.base': None},
{'internal.archive': None},
{'internal.coredump': None},
{'internal.sudo': None},
{'internal.syslog': None},
{'internal.timer': None},
])
ctx.config['tasks'][:0] = init_tasks
from teuthology.run_tasks import run_tasks
try:
run_tasks(tasks=ctx.config['tasks'], ctx=ctx)
finally:
if not ctx.summary.get('success') and ctx.config.get('nuke-on-error'):
from teuthology.nuke import nuke
# only unlock if we locked them in the first place
nuke(ctx, log, ctx.lock)
if ctx.archive is not None:
with file(os.path.join(ctx.archive, 'summary.yaml'), 'w') as f:
yaml.safe_dump(ctx.summary, f, default_flow_style=False)
with contextlib.closing(StringIO.StringIO()) as f:
yaml.safe_dump(ctx.summary, f)
log.info('Summary data:\n%s' % f.getvalue())
with contextlib.closing(StringIO.StringIO()) as f:
if 'email-on-error' in ctx.config and not ctx.summary.get('success', False):
yaml.safe_dump(ctx.summary, f)
yaml.safe_dump(ctx.config, f)
emsg = f.getvalue()
subject = "Teuthology error -- %s" % ctx.summary['failure_reason']
from teuthology.suite import email_results
email_results(subject,"Teuthology",ctx.config['email-on-error'],emsg)
if ctx.summary.get('success', True):
log.info('pass')
else:
log.info('FAIL')
import sys
sys.exit(1)
def main(ctx):
if ctx.owner is None:
ctx.owner = 'scheduled_{user}'.format(user=get_user())
read_config(ctx)
beanstalk = teuthology.beanstalk.connect()
tube = ctx.worker
beanstalk.use(tube)
if ctx.show:
for job_id in ctx.show:
job = beanstalk.peek(job_id)
if job is None and ctx.verbose:
print 'job {jid} is not in the queue'.format(jid=job_id)
else:
print '--- job {jid} priority {prio} ---\n'.format(
jid=job_id, prio=job.stats()['pri']), job.body
return
if ctx.delete:
for job_id in ctx.delete:
job = beanstalk.peek(job_id)
if job is None:
print 'job {jid} is not in the queue'.format(jid=job_id)
else:
job.delete()
name = yaml.safe_load(job.body).get('name')
if name:
report.try_delete_jobs(name, job_id)
return
# strip out targets; the worker will allocate new ones when we run
# the job with --lock.
if ctx.config.get('targets'):
del ctx.config['targets']
job_config = dict(
name=ctx.name,
last_in_suite=ctx.last_in_suite,
email=ctx.email,
description=ctx.description,
owner=ctx.owner,
verbose=ctx.verbose,
machine_type=ctx.worker,
)
# Merge job_config and ctx.config
job_config.update(ctx.config)
if ctx.timeout is not None:
job_config['results_timeout'] = ctx.timeout
job = yaml.safe_dump(job_config)
num = ctx.num
while num > 0:
jid = beanstalk.put(
job,
ttr=60 * 60 * 24,
priority=ctx.priority,
)
print 'Job scheduled with name {name} and ID {jid}'.format(
name=ctx.name, jid=jid)
job_config['job_id'] = str(jid)
report.try_push_job_info(job_config, dict(status='queued'))
num -= 1
def main():
from gevent import monkey
monkey.patch_all(dns=False)
from .orchestra import monkey
monkey.patch_all()
import logging
ctx = parse_args()
set_up_logging(ctx)
log = logging.getLogger(__name__)
if ctx.owner is None:
from teuthology.misc import get_user
ctx.owner = get_user()
write_initial_metadata(ctx)
if 'targets' in ctx.config and 'roles' in ctx.config:
targets = len(ctx.config['targets'])
roles = len(ctx.config['roles'])
assert targets >= roles, \
'%d targets are needed for all roles but found %d listed.' % (roles, targets)
machine_type = ctx.machine_type
if machine_type is None:
fallback_default = ctx.config.get('machine_type', 'plana')
machine_type = ctx.config.get('machine-type', fallback_default)
if ctx.block:
assert ctx.lock, \
'the --block option is only supported with the --lock option'
from teuthology.misc import read_config
read_config(ctx)
log.debug('\n '.join([
'Config:',
] + yaml.safe_dump(ctx.config, default_flow_style=False).splitlines()))
ctx.summary = dict(success=True)
ctx.summary['owner'] = ctx.owner
if ctx.description is not None:
ctx.summary['description'] = ctx.description
for task in ctx.config['tasks']:
assert 'kernel' not in task, \
'kernel installation shouldn be a base-level item, not part of the tasks list'
init_tasks = []
if ctx.lock:
assert 'targets' not in ctx.config, \
'You cannot specify targets in a config file when using the --lock option'
init_tasks.append({
'internal.lock_machines': (len(ctx.config['roles']), machine_type)
})
init_tasks.extend([
{
'internal.save_config': None
},
{
'internal.check_lock': None
},
{
'internal.connect': None
},
{
'internal.check_conflict': None
},
{
'internal.check_ceph_data': None
},
{
'internal.vm_setup': None
},
])
if 'kernel' in ctx.config:
from teuthology.misc import get_distro
distro = get_distro(ctx)
if distro == 'ubuntu':
init_tasks.append({'kernel': ctx.config['kernel']})
init_tasks.extend([
{
'internal.base': None
},
{
'internal.archive': None
},
{
'internal.coredump': None
},
{
'internal.sudo': None
},
{
'internal.syslog': None
},
{
'internal.timer': None
},
])
ctx.config['tasks'][:0] = init_tasks
from teuthology.run_tasks import run_tasks
try:
run_tasks(tasks=ctx.config['tasks'], ctx=ctx)
finally:
if not ctx.summary.get('success') and ctx.config.get('nuke-on-error'):
from teuthology.nuke import nuke
# only unlock if we locked them in the first place
nuke(ctx, log, ctx.lock)
if ctx.archive is not None:
with file(os.path.join(ctx.archive, 'summary.yaml'), 'w') as f:
yaml.safe_dump(ctx.summary, f, default_flow_style=False)
with contextlib.closing(StringIO.StringIO()) as f:
yaml.safe_dump(ctx.summary, f)
log.info('Summary data:\n%s' % f.getvalue())
with contextlib.closing(StringIO.StringIO()) as f:
if 'email-on-error' in ctx.config and not ctx.summary.get(
'success', False):
yaml.safe_dump(ctx.summary, f)
yaml.safe_dump(ctx.config, f)
emsg = f.getvalue()
subject = "Teuthology error -- %s" % ctx.summary[
'failure_reason']
from teuthology.suite import email_results
email_results(subject, "Teuthology",
ctx.config['email-on-error'], emsg)
if ctx.summary.get('success', True):
log.info('pass')
else:
log.info('FAIL')
import sys
sys.exit(1)
def analyze():
parser = argparse.ArgumentParser(description="""
Analyze the coverage of a suite of test runs, generating html output with lcov.
""")
parser.add_argument(
'-o', '--lcov-output',
help='the directory in which to store results',
required=True,
)
parser.add_argument(
'--html-output',
help='the directory in which to store html output',
)
parser.add_argument(
'--cov-tools-dir',
help='the location of coverage scripts (cov-init and cov-analyze)',
default='../../coverage',
)
parser.add_argument(
'--skip-init',
help='skip initialization (useful if a run stopped partway through)',
action='store_true',
default=False,
)
parser.add_argument(
'-v', '--verbose',
help='be more verbose',
action='store_true',
default=False,
)
parser.add_argument(
'test_dir',
help='the location of the test results',
)
args = parser.parse_args()
loglevel = logging.INFO
if args.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
)
teuthology.read_config(args)
tests = [
f for f in sorted(os.listdir(args.test_dir))
if not f.startswith('.')
and os.path.exists(os.path.join(args.test_dir, f, 'summary.yaml'))
and os.path.exists(os.path.join(args.test_dir, f, 'ceph-sha1'))]
test_summaries = {}
for test in tests:
summary = {}
with file(os.path.join(args.test_dir, test, 'summary.yaml')) as f:
g = yaml.safe_load_all(f)
for new in g:
summary.update(new)
if summary['flavor'] != 'gcov':
log.debug('Skipping %s, since it does not include coverage', test)
continue
test_summaries[test] = summary
assert len(test_summaries) > 0
suite = os.path.basename(args.test_dir)
# only run cov-init once.
# this only works if all tests were run against the same version.
if not args.skip_init:
log.info('initializing coverage data...')
subprocess.check_call(
args=[
os.path.join(args.cov_tools_dir, 'cov-init.sh'),
os.path.join(args.test_dir, tests[0]),
args.lcov_output,
os.path.join(
args.teuthology_config['ceph_build_output_dir'],
'{suite}.tgz'.format(suite=suite),
),
])
shutil.copy(
os.path.join(args.lcov_output, 'base.lcov'),
os.path.join(args.lcov_output, 'total.lcov')
)
test_coverage = {}
for test, summary in test_summaries.iteritems():
lcov_file = '{name}.lcov'.format(name=test)
log.info('analyzing coverage for %s', test)
proc = subprocess.Popen(
args=[
os.path.join(args.cov_tools_dir, 'cov-analyze.sh'),
'-t', os.path.join(args.test_dir, test),
'-d', args.lcov_output,
'-o', test,
],
stdout=subprocess.PIPE,
)
output, _ = proc.communicate()
desc = summary.get('description', test)
test_coverage[desc] = read_coverage(output)
log.info('adding %s data to total', test)
proc = subprocess.Popen(
args=[
'lcov',
'-a', os.path.join(args.lcov_output, lcov_file),
'-a', os.path.join(args.lcov_output, 'total.lcov'),
'-o', os.path.join(args.lcov_output, 'total_tmp.lcov'),
],
stdout=subprocess.PIPE,
)
output, _ = proc.communicate()
os.rename(
os.path.join(args.lcov_output, 'total_tmp.lcov'),
os.path.join(args.lcov_output, 'total.lcov')
)
coverage = read_coverage(output)
test_coverage['total for {suite}'.format(suite=suite)] = coverage
log.debug('total coverage is %s', str(coverage))
if args.html_output:
subprocess.check_call(
args=[
'genhtml',
'-s',
'-o', os.path.join(args.html_output, 'total'),
'-t', 'Total for {suite}'.format(suite=suite),
'--',
os.path.join(args.lcov_output, 'total.lcov'),
])
store_coverage(args, test_coverage, summary['ceph-sha1'], suite)
def worker():
parser = argparse.ArgumentParser(description="""
Grab jobs from a beanstalk queue and run the teuthology tests they
describe. One job is run at a time.
""")
parser.add_argument(
'-v',
'--verbose',
action='store_true',
default=None,
help='be more verbose',
)
parser.add_argument(
'--archive-dir',
metavar='DIR',
help='path under which to archive results',
required=True,
)
parser.add_argument(
'-l',
'--log-dir',
help='path in which to store logs',
required=True,
)
ctx = parser.parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
filename=os.path.join(ctx.log_dir,
'worker.{pid}'.format(pid=os.getpid())),
format='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
)
if not os.path.isdir(ctx.archive_dir):
sys.exit("{prog}: archive directory must exist: {path}".format(
prog=os.path.basename(sys.argv[0]),
path=ctx.archive_dir,
))
from teuthology.misc import read_config
read_config(ctx)
beanstalk = connect(ctx)
beanstalk.watch('teuthology')
beanstalk.ignore('default')
while True:
job = beanstalk.reserve(timeout=60)
if job is None:
continue
# bury the job so it won't be re-run if it fails
job.bury()
log.debug('Reserved job %d', job.jid)
log.debug('Config is: %s', job.body)
job_config = yaml.safe_load(job.body)
safe_archive = safepath.munge(job_config['name'])
if job_config.get('last_in_suite', False):
log.debug('Generating coverage for %s', job_config['name'])
args = [
os.path.join(os.path.dirname(sys.argv[0]),
'teuthology-results'),
'--timeout',
str(job_config.get('results_timeout', 21600)),
'--email',
job_config['email'],
'--archive-dir',
os.path.join(ctx.archive_dir, safe_archive),
'--name',
job_config['name'],
]
subprocess.Popen(args=args)
else:
log.debug('Creating archive dir...')
safepath.makedirs(ctx.archive_dir, safe_archive)
archive_path = os.path.join(ctx.archive_dir, safe_archive,
str(job.jid))
log.info('Running job %d', job.jid)
run_job(job_config, archive_path)
job.delete()
def schedule():
parser = argparse.ArgumentParser(description='Schedule ceph integration tests')
parser.add_argument(
'config',
metavar='CONFFILE',
nargs='*',
type=config_file,
action=MergeConfig,
default={},
help='config file to read',
)
parser.add_argument(
'--name',
help='name of suite run the job is part of',
)
parser.add_argument(
'--last-in-suite',
action='store_true',
default=False,
help='mark the last job in a suite so suite post-processing can be run',
)
parser.add_argument(
'--email',
help='where to send the results of a suite (only applies to the last job in a suite)',
)
parser.add_argument(
'--timeout',
help='how many seconds to wait for jobs to finish before emailing results (only applies to the last job in a suite',
type=int,
)
parser.add_argument(
'--description',
help='job description',
)
parser.add_argument(
'--owner',
help='job owner',
)
parser.add_argument(
'--delete',
metavar='JOBID',
type=int,
nargs='*',
help='list of jobs to remove from the queue',
)
parser.add_argument(
'-v', '--verbose',
action='store_true',
default=False,
help='be more verbose',
)
ctx = parser.parse_args()
if not ctx.last_in_suite:
assert not ctx.email, '--email is only applicable to the last job in a suite'
assert not ctx.timeout, '--timeout is only applicable to the last job in a suite'
from teuthology.misc import read_config, get_user
if ctx.owner is None:
ctx.owner = 'scheduled_{user}'.format(user=get_user())
read_config(ctx)
import teuthology.queue
beanstalk = teuthology.queue.connect(ctx)
beanstalk.use('teuthology')
if ctx.delete:
for jobid in ctx.delete:
job = beanstalk.peek(jobid)
if job is None:
print 'job {jid} is not in the queue'.format(jid=jobid)
else:
job.delete()
return
job_config = dict(
config=ctx.config,
name=ctx.name,
last_in_suite=ctx.last_in_suite,
email=ctx.email,
description=ctx.description,
owner=ctx.owner,
verbose=ctx.verbose,
)
if ctx.timeout is not None:
job_config['results_timeout'] = ctx.timeout
job = yaml.safe_dump(job_config)
jid = beanstalk.put(job, ttr=60*60*24)
print 'Job scheduled with ID {jid}'.format(jid=jid)
def schedule():
parser = argparse.ArgumentParser(
description='Schedule ceph integration tests')
parser.add_argument(
'config',
metavar='CONFFILE',
nargs='*',
type=config_file,
action=MergeConfig,
default={},
help='config file to read',
)
parser.add_argument(
'--name',
help='name of suite run the job is part of',
)
parser.add_argument(
'--last-in-suite',
action='store_true',
default=False,
help='mark the last job in a suite so suite post-processing can be run',
)
parser.add_argument(
'--email',
help=
'where to send the results of a suite (only applies to the last job in a suite)',
)
parser.add_argument(
'--timeout',
help=
'how many seconds to wait for jobs to finish before emailing results (only applies to the last job in a suite',
type=int,
)
parser.add_argument(
'--description',
help='job description',
)
parser.add_argument(
'--owner',
help='job owner',
)
parser.add_argument(
'--delete',
metavar='JOBID',
type=int,
nargs='*',
help='list of jobs to remove from the queue',
)
parser.add_argument('-n',
'--num',
default=1,
type=int,
help='number of times to run/queue the job')
parser.add_argument('-p',
'--priority',
default=1000,
type=int,
help='beanstalk priority (lower is sooner)')
parser.add_argument(
'-v',
'--verbose',
action='store_true',
default=False,
help='be more verbose',
)
parser.add_argument(
'-w',
'--worker',
default='plana',
help='which worker to use (type of machine)',
)
parser.add_argument(
'-s',
'--show',
metavar='JOBID',
type=int,
nargs='*',
help='output the contents of specified jobs in the queue',
)
ctx = parser.parse_args()
if not ctx.last_in_suite:
assert not ctx.email, '--email is only applicable to the last job in a suite'
assert not ctx.timeout, '--timeout is only applicable to the last job in a suite'
from teuthology.misc import read_config, get_user
if ctx.owner is None:
ctx.owner = 'scheduled_{user}'.format(user=get_user())
read_config(ctx)
import teuthology.queue
beanstalk = teuthology.queue.connect(ctx)
tube = ctx.worker
beanstalk.use(tube)
if ctx.show:
for job_id in ctx.show:
job = beanstalk.peek(job_id)
if job is None and ctx.verbose:
print 'job {jid} is not in the queue'.format(jid=job_id)
else:
print '--- job {jid} priority {prio} ---\n'.format(
jid=job_id, prio=job.stats()['pri']), job.body
return
if ctx.delete:
for job_id in ctx.delete:
job = beanstalk.peek(job_id)
if job is None:
print 'job {jid} is not in the queue'.format(jid=job_id)
else:
job.delete()
return
# strip out targets; the worker will allocate new ones when we run
# the job with --lock.
if ctx.config.get('targets'):
del ctx.config['targets']
job_config = dict(
name=ctx.name,
last_in_suite=ctx.last_in_suite,
email=ctx.email,
description=ctx.description,
owner=ctx.owner,
verbose=ctx.verbose,
)
# Merge job_config and ctx.config
job_config.update(ctx.config)
if ctx.timeout is not None:
job_config['results_timeout'] = ctx.timeout
job = yaml.safe_dump(job_config)
num = ctx.num
while num > 0:
jid = beanstalk.put(
job,
ttr=60 * 60 * 24,
priority=ctx.priority,
)
print 'Job scheduled with ID {jid}'.format(jid=jid)
num -= 1
def main():
parser = argparse.ArgumentParser(description="""
Lock, unlock, or query lock status of machines.
""")
parser.add_argument(
'-v',
'--verbose',
action='store_true',
default=False,
help='be more verbose',
)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
'--list',
action='store_true',
default=False,
help=
'Show lock info for machines owned by you, or only machines specified. Can be restricted by --owner and --status.',
)
group.add_argument(
'--list-targets',
action='store_true',
default=False,
help=
'Show lock info for all machines, or only machines specified, in targets: yaml format. Can be restricted by --owner and --status.',
)
group.add_argument(
'--lock',
action='store_true',
default=False,
help='lock particular machines',
)
group.add_argument(
'--unlock',
action='store_true',
default=False,
help='unlock particular machines',
)
group.add_argument(
'--lock-many',
dest='num_to_lock',
type=_positive_int,
help='lock this many machines',
)
group.add_argument(
'--update',
action='store_true',
default=False,
help='update the description or status of some machines',
)
parser.add_argument(
'-a',
'--all',
action='store_true',
default=False,
help='list all machines, not just those owned by you',
)
parser.add_argument(
'--owner',
default=None,
help='owner of the lock(s) (must match to unlock a machine)',
)
parser.add_argument(
'-f',
action='store_true',
default=False,
help=
'don\'t exit after the first error, continue locking or unlocking other machines',
)
parser.add_argument(
'--desc',
default=None,
help='update description',
)
parser.add_argument(
'--status',
default=None,
choices=['up', 'down'],
help='whether a machine is usable for testing',
)
parser.add_argument(
'-t',
'--targets',
dest='targets',
default=None,
help='input yaml containing targets',
)
parser.add_argument(
'machines',
metavar='MACHINE',
default=[],
nargs='*',
help='machines to operate on',
)
ctx = parser.parse_args()
loglevel = logging.ERROR
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(level=loglevel, )
teuthology.read_config(ctx)
ret = 0
user = ctx.owner
machines = ctx.machines
machines_to_update = []
if ctx.targets:
try:
with file(ctx.targets) as f:
g = yaml.safe_load_all(f)
for new in g:
if 'targets' in new:
for t in new['targets'].iterkeys():
machines.append(t)
except IOError, e:
raise argparse.ArgumentTypeError(str(e))
def main():
parser = argparse.ArgumentParser(description="""
Lock, unlock, or query lock status of machines.
""")
parser.add_argument(
'-v', '--verbose',
action='store_true',
default=False,
help='be more verbose',
)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
'--list',
action='store_true',
default=False,
help='Show lock info for machines owned by you, or only machines specified. Can be restricted by --owner, --status, and --locked.',
)
group.add_argument(
'--list-targets',
action='store_true',
default=False,
help='Show lock info for all machines, or only machines specified, in targets: yaml format. Can be restricted by --owner, --status, and --locked.',
)
group.add_argument(
'--lock',
action='store_true',
default=False,
help='lock particular machines',
)
group.add_argument(
'--unlock',
action='store_true',
default=False,
help='unlock particular machines',
)
group.add_argument(
'--lock-many',
dest='num_to_lock',
type=_positive_int,
help='lock this many machines',
)
group.add_argument(
'--update',
action='store_true',
default=False,
help='update the description or status of some machines',
)
group.add_argument(
'--summary',
action='store_true',
default=False,
help='summarize locked-machine counts by owner',
)
parser.add_argument(
'-a', '--all',
action='store_true',
default=False,
help='list all machines, not just those owned by you',
)
parser.add_argument(
'--owner',
default=None,
help='owner of the lock(s) (must match to unlock a machine)',
)
parser.add_argument(
'-f',
action='store_true',
default=False,
help='don\'t exit after the first error, continue locking or unlocking other machines',
)
parser.add_argument(
'--desc',
default=None,
help='update description',
)
parser.add_argument(
'--status',
default=None,
choices=['up', 'down'],
help='whether a machine is usable for testing',
)
parser.add_argument(
'--locked',
default=None,
choices=['true', 'false'],
help='whether a machine is locked',
)
parser.add_argument(
'--brief',
action='store_true',
default=False,
help='Shorten information reported from --list',
)
parser.add_argument(
'-t', '--targets',
dest='targets',
default=None,
help='input yaml containing targets',
)
parser.add_argument(
'machines',
metavar='MACHINE',
default=[],
nargs='*',
help='machines to operate on',
)
ctx = parser.parse_args()
loglevel = logging.ERROR
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
)
teuthology.read_config(ctx)
ret = 0
user = ctx.owner
machines = [canonicalize_hostname(m) for m in ctx.machines]
machines_to_update = []
if ctx.targets:
try:
with file(ctx.targets) as f:
g = yaml.safe_load_all(f)
for new in g:
if 'targets' in new:
for t in new['targets'].iterkeys():
machines.append(t)
except IOError, e:
raise argparse.ArgumentTypeError(str(e))
def analyze():
parser = argparse.ArgumentParser(description="""
Analyze the coverage of a suite of test runs, generating html output with lcov.
""")
parser.add_argument(
'-o',
'--lcov-output',
help='the directory in which to store results',
required=True,
)
parser.add_argument(
'--html-output',
help='the directory in which to store html output',
)
parser.add_argument(
'--cov-tools-dir',
help='the location of coverage scripts (cov-init and cov-analyze)',
default='../../coverage',
)
parser.add_argument(
'--skip-init',
help='skip initialization (useful if a run stopped partway through)',
action='store_true',
default=False,
)
parser.add_argument(
'-v',
'--verbose',
help='be more verbose',
action='store_true',
default=False,
)
parser.add_argument(
'test_dir',
help='the location of the test results',
)
args = parser.parse_args()
loglevel = logging.INFO
if args.verbose:
loglevel = logging.DEBUG
logging.basicConfig(level=loglevel, )
teuthology.read_config(args)
tests = [
f for f in sorted(os.listdir(args.test_dir)) if not f.startswith('.')
and os.path.exists(os.path.join(args.test_dir, f, 'summary.yaml'))
and os.path.exists(os.path.join(args.test_dir, f, 'ceph-sha1'))
]
test_summaries = {}
for test in tests:
summary = {}
with file(os.path.join(args.test_dir, test, 'summary.yaml')) as f:
g = yaml.safe_load_all(f)
for new in g:
summary.update(new)
if summary['flavor'] != 'gcov':
log.debug('Skipping %s, since it does not include coverage', test)
continue
test_summaries[test] = summary
assert len(test_summaries) > 0
suite = os.path.basename(args.test_dir)
# only run cov-init once.
# this only works if all tests were run against the same version.
if not args.skip_init:
log.info('initializing coverage data...')
subprocess.check_call(args=[
os.path.join(args.cov_tools_dir, 'cov-init.sh'),
os.path.join(args.test_dir, tests[0]),
args.lcov_output,
os.path.join(
args.teuthology_config['ceph_build_output_dir'],
'{suite}.tgz'.format(suite=suite),
),
])
shutil.copy(os.path.join(args.lcov_output, 'base.lcov'),
os.path.join(args.lcov_output, 'total.lcov'))
test_coverage = {}
for test, summary in test_summaries.iteritems():
lcov_file = '{name}.lcov'.format(name=test)
log.info('analyzing coverage for %s', test)
proc = subprocess.Popen(
args=[
os.path.join(args.cov_tools_dir, 'cov-analyze.sh'),
'-t',
os.path.join(args.test_dir, test),
'-d',
args.lcov_output,
'-o',
test,
],
stdout=subprocess.PIPE,
)
output, _ = proc.communicate()
desc = summary.get('description', test)
test_coverage[desc] = read_coverage(output)
log.info('adding %s data to total', test)
proc = subprocess.Popen(
args=[
'lcov',
'-a',
os.path.join(args.lcov_output, lcov_file),
'-a',
os.path.join(args.lcov_output, 'total.lcov'),
'-o',
os.path.join(args.lcov_output, 'total_tmp.lcov'),
],
stdout=subprocess.PIPE,
)
output, _ = proc.communicate()
os.rename(os.path.join(args.lcov_output, 'total_tmp.lcov'),
os.path.join(args.lcov_output, 'total.lcov'))
coverage = read_coverage(output)
test_coverage['total for {suite}'.format(suite=suite)] = coverage
log.debug('total coverage is %s', str(coverage))
if args.html_output:
subprocess.check_call(args=[
'genhtml',
'-s',
'-o',
os.path.join(args.html_output, 'total'),
'-t',
'Total for {suite}'.format(suite=suite),
'--',
os.path.join(args.lcov_output, 'total.lcov'),
])
store_coverage(args, test_coverage, summary['ceph-sha1'], suite)
def worker():
parser = argparse.ArgumentParser(description="""
Grab jobs from a beanstalk queue and run the teuthology tests they
describe. One job is run at a time.
""")
parser.add_argument(
'-v', '--verbose',
action='store_true', default=None,
help='be more verbose',
)
parser.add_argument(
'--archive-dir',
metavar='DIR',
help='path under which to archive results',
required=True,
)
parser.add_argument(
'-l', '--log-dir',
help='path in which to store logs',
required=True,
)
parser.add_argument(
'-t', '--tube',
help='which beanstalk tube to read jobs from',
required=True,
)
ctx = parser.parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
filename=os.path.join(ctx.log_dir, 'worker.{pid}'.format(pid=os.getpid())),
format='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
)
if not os.path.isdir(ctx.archive_dir):
sys.exit("{prog}: archive directory must exist: {path}".format(
prog=os.path.basename(sys.argv[0]),
path=ctx.archive_dir,
))
from teuthology.misc import read_config
read_config(ctx)
beanstalk = connect(ctx)
beanstalk.watch(ctx.tube)
beanstalk.ignore('default')
while True:
job = beanstalk.reserve(timeout=60)
if job is None:
continue
# bury the job so it won't be re-run if it fails
job.bury()
log.debug('Reserved job %d', job.jid)
log.debug('Config is: %s', job.body)
job_config = yaml.safe_load(job.body)
safe_archive = safepath.munge(job_config['name'])
teuthology_branch = job_config.get('config', {}).get('teuthology_branch', 'master')
teuth_path = os.path.join(os.getenv("HOME"), 'teuthology-' + teuthology_branch, 'virtualenv', 'bin')
if not os.path.isdir(teuth_path):
raise Exception('Teuthology branch ' + teuthology_branch + ' not found at ' + teuth_path)
if job_config.get('last_in_suite'):
log.debug('Generating coverage for %s', job_config['name'])
args = [
os.path.join(teuth_path, 'teuthology-results'),
'--timeout',
str(job_config.get('results_timeout', 21600)),
'--email',
job_config['email'],
'--archive-dir',
os.path.join(ctx.archive_dir, safe_archive),
'--name',
job_config['name'],
]
subprocess.Popen(args=args)
else:
log.debug('Creating archive dir...')
safepath.makedirs(ctx.archive_dir, safe_archive)
archive_path = os.path.join(ctx.archive_dir, safe_archive, str(job.jid))
log.info('Running job %d', job.jid)
run_job(job_config, archive_path, teuth_path)
job.delete()
def schedule():
parser = argparse.ArgumentParser(description='Schedule ceph integration tests')
parser.add_argument(
'config',
metavar='CONFFILE',
nargs='*',
type=config_file,
action=MergeConfig,
default={},
help='config file to read',
)
parser.add_argument(
'--name',
help='name of suite run the job is part of',
)
parser.add_argument(
'--last-in-suite',
action='store_true',
default=False,
help='mark the last job in a suite so suite post-processing can be run',
)
parser.add_argument(
'--email',
help='where to send the results of a suite (only applies to the last job in a suite)',
)
parser.add_argument(
'--timeout',
help='how many seconds to wait for jobs to finish before emailing results (only applies to the last job in a suite',
type=int,
)
parser.add_argument(
'--description',
help='job description',
)
parser.add_argument(
'--owner',
help='job owner',
)
parser.add_argument(
'--delete',
metavar='JOBID',
type=int,
nargs='*',
help='list of jobs to remove from the queue',
)
parser.add_argument(
'-n', '--num',
default=1,
type=int,
help='number of times to run/queue the job'
)
parser.add_argument(
'-p', '--priority',
default=1000,
type=int,
help='beanstalk priority (lower is sooner)'
)
parser.add_argument(
'-v', '--verbose',
action='store_true',
default=False,
help='be more verbose',
)
parser.add_argument(
'-w', '--worker',
default='plana',
help='which worker to use (type of machine)',
)
parser.add_argument(
'-s', '--show',
metavar='JOBID',
type=int,
nargs='*',
help='output the contents of specified jobs in the queue',
)
ctx = parser.parse_args()
if not ctx.last_in_suite:
assert not ctx.email, '--email is only applicable to the last job in a suite'
assert not ctx.timeout, '--timeout is only applicable to the last job in a suite'
from teuthology.misc import read_config, get_user
if ctx.owner is None:
ctx.owner = 'scheduled_{user}'.format(user=get_user())
read_config(ctx)
import teuthology.queue
beanstalk = teuthology.queue.connect(ctx)
tube=ctx.worker
beanstalk.use(tube)
if ctx.show:
for job_id in ctx.show:
job = beanstalk.peek(job_id)
if job is None and ctx.verbose:
print 'job {jid} is not in the queue'.format(jid=job_id)
else:
print '--- job {jid} priority {prio} ---\n'.format(
jid=job_id,
prio=job.stats()['pri']), job.body
return
if ctx.delete:
for job_id in ctx.delete:
job = beanstalk.peek(job_id)
if job is None:
print 'job {jid} is not in the queue'.format(jid=job_id)
else:
job.delete()
return
# strip out targets; the worker will allocate new ones when we run
# the job with --lock.
if ctx.config.get('targets'):
del ctx.config['targets']
job_config = dict(
name=ctx.name,
last_in_suite=ctx.last_in_suite,
email=ctx.email,
description=ctx.description,
owner=ctx.owner,
verbose=ctx.verbose,
)
# Merge job_config and ctx.config
job_config.update(ctx.config)
if ctx.timeout is not None:
job_config['results_timeout'] = ctx.timeout
job = yaml.safe_dump(job_config)
num = ctx.num
while num > 0:
jid = beanstalk.put(
job,
ttr=60*60*24,
priority=ctx.priority,
)
print 'Job scheduled with ID {jid}'.format(jid=jid)
num -= 1
