def pull_directory(remote, remotedir, localdir):
"""
Copy a remote directory to a local directory.
"""
log.debug('Transferring archived files from %s:%s to %s',
remote.shortname, remotedir, localdir)
if not os.path.exists(localdir):
os.mkdir(localdir)
r = remote.get_tar_stream(remotedir, sudo=True)
tar = tarfile.open(mode='r|gz', fileobj=r.stdout)
while True:
ti = tar.next()
if ti is None:
break
if ti.isdir():
# ignore silently; easier to just create leading dirs below
# XXX this mean empty dirs are not transferred
pass
elif ti.isfile():
sub = safepath.munge(ti.name)
safepath.makedirs(root=localdir, path=os.path.dirname(sub))
tar.makefile(ti, targetpath=os.path.join(localdir, sub))
else:
if ti.isdev():
type_ = 'device'
elif ti.issym():
type_ = 'symlink'
elif ti.islnk():
type_ = 'hard link'
else:
type_ = 'unknown'
log.info('Ignoring tar entry: %r type %r', ti.name, type_)
def pull_directory(remote, remotedir, localdir):
"""
Copy a remote directory to a local directory.
"""
log.debug('Transferring archived files from %s:%s to %s', remote.shortname,
remotedir, localdir)
if not os.path.exists(localdir):
os.mkdir(localdir)
r = remote.get_tar_stream(remotedir, sudo=True)
tar = tarfile.open(mode='r|gz', fileobj=r.stdout)
while True:
ti = tar.next()
if ti is None:
break
if ti.isdir():
# ignore silently; easier to just create leading dirs below
# XXX this mean empty dirs are not transferred
pass
elif ti.isfile():
sub = safepath.munge(ti.name)
safepath.makedirs(root=localdir, path=os.path.dirname(sub))
tar.makefile(ti, targetpath=os.path.join(localdir, sub))
else:
if ti.isdev():
type_ = 'device'
elif ti.issym():
type_ = 'symlink'
elif ti.islnk():
type_ = 'hard link'
else:
type_ = 'unknown'
log.info('Ignoring tar entry: %r type %r', ti.name, type_)
def create_job_archive(job_name, job_archive_path, archive_dir):
log.info('Creating job\'s archive dir %s', job_archive_path)
safe_archive = safepath.munge(job_name)
run_archive = os.path.join(archive_dir, safe_archive)
if not os.path.exists(run_archive):
safepath.makedirs('/', run_archive)
safepath.makedirs('/', job_archive_path)
def pull_directory(remote, remotedir, localdir):
"""
Copy a remote directory to a local directory.
"""
log.debug('Transferring archived files from %s:%s to %s', remote.shortname,
remotedir, localdir)
if not os.path.exists(localdir):
os.mkdir(localdir)
proc = remote.run(
args=[
'sudo',
'tar',
'c',
'-f',
'-',
'-C',
remotedir,
'--',
'.',
],
stdout=run.PIPE,
wait=False,
)
tar = tarfile.open(mode='r|', fileobj=proc.stdout)
while True:
ti = tar.next()
if ti is None:
break
if ti.isdir():
# ignore silently; easier to just create leading dirs below
pass
elif ti.isfile():
sub = safepath.munge(ti.name)
safepath.makedirs(root=localdir, path=os.path.dirname(sub))
tar.makefile(ti, targetpath=os.path.join(localdir, sub))
else:
if ti.isdev():
type_ = 'device'
elif ti.issym():
type_ = 'symlink'
elif ti.islnk():
type_ = 'hard link'
else:
type_ = 'unknown'
log.info('Ignoring tar entry: %r type %r', ti.name, type_)
continue
proc.exitstatus.get()
def pull_directory(remote, remotedir, localdir):
"""
Copy a remote directory to a local directory.
"""
log.debug('Transferring archived files from %s:%s to %s',
remote.shortname, remotedir, localdir)
if not os.path.exists(localdir):
os.mkdir(localdir)
proc = remote.run(
args=[
'sudo',
'tar',
'c',
'-f', '-',
'-C', remotedir,
'--',
'.',
],
stdout=run.PIPE,
wait=False,
)
tar = tarfile.open(mode='r|', fileobj=proc.stdout)
while True:
ti = tar.next()
if ti is None:
break
if ti.isdir():
# ignore silently; easier to just create leading dirs below
pass
elif ti.isfile():
sub = safepath.munge(ti.name)
safepath.makedirs(root=localdir, path=os.path.dirname(sub))
tar.makefile(ti, targetpath=os.path.join(localdir, sub))
else:
if ti.isdev():
type_ = 'device'
elif ti.issym():
type_ = 'symlink'
elif ti.islnk():
type_ = 'hard link'
else:
type_ = 'unknown'
log.info('Ignoring tar entry: %r type %r', ti.name, type_)
continue
proc.exitstatus.get()
def pull_directory(remote, remotedir, localdir, write_to=copy_fileobj):
"""
Copy a remote directory to a local directory.
:param remote: the remote object representing the remote host from where
the specified directory is pulled
:param remotedir: the source directory on remote host
:param localdir: the destination directory on localhost
:param write_to: optional function to write the file to localdir.
its signature should be:
func(src: fileobj,
tarinfo: tarfile.TarInfo,
local_path: str)
"""
log.debug('Transferring archived files from %s:%s to %s',
remote.shortname, remotedir, localdir)
if not os.path.exists(localdir):
os.mkdir(localdir)
r = remote.get_tar_stream(remotedir, sudo=True)
tar = tarfile.open(mode='r|gz', fileobj=r.stdout)
while True:
ti = tar.next()
if ti is None:
break
if ti.isdir():
# ignore silently; easier to just create leading dirs below
# XXX this mean empty dirs are not transferred
pass
elif ti.isfile():
sub = safepath.munge(ti.name)
safepath.makedirs(root=localdir, path=os.path.dirname(sub))
with tar.extractfile(ti) as src:
write_to(src, ti, os.path.join(localdir, sub))
else:
if ti.isdev():
type_ = 'device'
elif ti.issym():
type_ = 'symlink'
elif ti.islnk():
type_ = 'hard link'
else:
type_ = 'unknown'
log.info('Ignoring tar entry: %r type %r', ti.name, type_)
def pull_directory(remote, remotedir, localdir):
"""
Copy a remote directory to a local directory.
"""
log.debug('Transferring archived files from %s:%s to %s', remote.shortname,
remotedir, localdir)
if not os.path.exists(localdir):
os.mkdir(localdir)
_, local_tarfile = tempfile.mkstemp(dir=localdir)
remote.get_tar(remotedir, local_tarfile, sudo=True)
with open(local_tarfile, 'r+') as fb1:
tar = tarfile.open(mode='r|gz', fileobj=fb1)
while True:
ti = tar.next()
if ti is None:
break
if ti.isdir():
# ignore silently; easier to just create leading dirs below
pass
elif ti.isfile():
sub = safepath.munge(ti.name)
safepath.makedirs(root=localdir, path=os.path.dirname(sub))
tar.makefile(ti, targetpath=os.path.join(localdir, sub))
else:
if ti.isdev():
type_ = 'device'
elif ti.issym():
type_ = 'symlink'
elif ti.islnk():
type_ = 'hard link'
else:
type_ = 'unknown'
log.info('Ignoring tar entry: %r type %r', ti.name, type_)
continue
os.remove(local_tarfile)
def pull_directory(remote, remotedir, localdir):
"""
Copy a remote directory to a local directory.
"""
log.debug('Transferring archived files from %s:%s to %s',
remote.shortname, remotedir, localdir)
if not os.path.exists(localdir):
os.mkdir(localdir)
_, local_tarfile = tempfile.mkstemp(dir=localdir)
remote.get_tar(remotedir, local_tarfile, sudo=True)
with open(local_tarfile, 'r+') as fb1:
tar = tarfile.open(mode='r|gz', fileobj=fb1)
while True:
ti = tar.next()
if ti is None:
break
if ti.isdir():
# ignore silently; easier to just create leading dirs below
pass
elif ti.isfile():
sub = safepath.munge(ti.name)
safepath.makedirs(root=localdir, path=os.path.dirname(sub))
tar.makefile(ti, targetpath=os.path.join(localdir, sub))
else:
if ti.isdev():
type_ = 'device'
elif ti.issym():
type_ = 'symlink'
elif ti.islnk():
type_ = 'hard link'
else:
type_ = 'unknown'
log.info('Ignoring tar entry: %r type %r', ti.name, type_)
continue
os.remove(local_tarfile)
def run_job(job_config, teuth_bin_path, archive_dir, verbose):
safe_archive = safepath.munge(job_config['name'])
if job_config.get('first_in_suite') or job_config.get('last_in_suite'):
if teuth_config.results_server:
try:
report.try_delete_jobs(job_config['name'],
job_config['job_id'])
except Exception as e:
log.warning("Unable to delete job %s, exception occurred: %s",
job_config['job_id'], e)
suite_archive_dir = os.path.join(archive_dir, safe_archive)
safepath.makedirs('/', suite_archive_dir)
args = [
os.path.join(teuth_bin_path, 'teuthology-results'),
'--archive-dir',
suite_archive_dir,
'--name',
job_config['name'],
]
if job_config.get('first_in_suite'):
log.info('Generating memo for %s', job_config['name'])
if job_config.get('seed'):
args.extend(['--seed', job_config['seed']])
if job_config.get('subset'):
args.extend(['--subset', job_config['subset']])
else:
log.info('Generating results for %s', job_config['name'])
timeout = job_config.get('results_timeout',
teuth_config.results_timeout)
args.extend(['--timeout', str(timeout)])
if job_config.get('email'):
args.extend(['--email', job_config['email']])
# Execute teuthology-results, passing 'preexec_fn=os.setpgrp' to
# make sure that it will continue to run if this worker process
# dies (e.g. because of a restart)
result_proc = subprocess.Popen(args=args, preexec_fn=os.setpgrp)
log.info("teuthology-results PID: %s", result_proc.pid)
return
log.info('Creating archive dir %s', job_config['archive_path'])
safepath.makedirs('/', job_config['archive_path'])
log.info('Running job %s', job_config['job_id'])
suite_path = job_config['suite_path']
arg = [
os.path.join(teuth_bin_path, 'teuthology'),
]
# The following is for compatibility with older schedulers, from before we
# started merging the contents of job_config['config'] into job_config
# itself.
if 'config' in job_config:
inner_config = job_config.pop('config')
if not isinstance(inner_config, dict):
log.warn("run_job: job_config['config'] isn't a dict, it's a %s",
str(type(inner_config)))
else:
job_config.update(inner_config)
if verbose or job_config['verbose']:
arg.append('-v')
arg.extend([
'--lock',
'--block',
'--owner',
job_config['owner'],
'--archive',
job_config['archive_path'],
'--name',
job_config['name'],
])
if job_config['description'] is not None:
arg.extend(['--description', job_config['description']])
arg.append('--')
with tempfile.NamedTemporaryFile(prefix='teuthology-worker.',
suffix='.tmp',
mode='w+t') as tmp:
yaml.safe_dump(data=job_config, stream=tmp)
tmp.flush()
arg.append(tmp.name)
env = os.environ.copy()
python_path = env.get('PYTHONPATH', '')
python_path = ':'.join([suite_path, python_path]).strip(':')
env['PYTHONPATH'] = python_path
log.debug("Running: %s" % ' '.join(arg))
p = subprocess.Popen(args=arg, env=env)
log.info("Job archive: %s", job_config['archive_path'])
log.info("Job PID: %s", str(p.pid))
if teuth_config.results_server:
log.info("Running with watchdog")
try:
run_with_watchdog(p, job_config)
except Exception:
log.exception("run_with_watchdog had an unhandled exception")
raise
else:
log.info("Running without watchdog")
# This sleep() is to give the child time to start up and create the
# archive dir.
time.sleep(5)
symlink_worker_log(job_config['worker_log'],
job_config['archive_path'])
p.wait()
if p.returncode != 0:
log.error('Child exited with code %d', p.returncode)
else:
log.info('Success!')
def worker():
parser = argparse.ArgumentParser(
description="""
Grab jobs from a beanstalk queue and run the teuthology tests they
describe. One job is run at a time.
"""
)
parser.add_argument("-v", "--verbose", action="store_true", default=None, help="be more verbose")
parser.add_argument("--archive-dir", metavar="DIR", help="path under which to archive results", required=True)
parser.add_argument("-l", "--log-dir", help="path in which to store logs", required=True)
parser.add_argument("-t", "--tube", help="which beanstalk tube to read jobs from", required=True)
ctx = parser.parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
filename=os.path.join(ctx.log_dir, "worker.{pid}".format(pid=os.getpid())),
format="%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s",
datefmt="%Y-%m-%dT%H:%M:%S",
)
if not os.path.isdir(ctx.archive_dir):
sys.exit(
"{prog}: archive directory must exist: {path}".format(
prog=os.path.basename(sys.argv[0]), path=ctx.archive_dir
)
)
from teuthology.misc import read_config
read_config(ctx)
beanstalk = connect(ctx)
beanstalk.watch(ctx.tube)
beanstalk.ignore("default")
while True:
job = beanstalk.reserve(timeout=60)
if job is None:
continue
# bury the job so it won't be re-run if it fails
job.bury()
log.debug("Reserved job %d", job.jid)
log.debug("Config is: %s", job.body)
job_config = yaml.safe_load(job.body)
safe_archive = safepath.munge(job_config["name"])
if job_config.get("last_in_suite", False):
log.debug("Generating coverage for %s", job_config["name"])
args = [
os.path.join(os.path.dirname(sys.argv[0]), "teuthology-results"),
"--timeout",
str(job_config.get("results_timeout", 21600)),
"--email",
job_config["email"],
"--archive-dir",
os.path.join(ctx.archive_dir, safe_archive),
"--name",
job_config["name"],
]
subprocess.Popen(args=args)
else:
log.debug("Creating archive dir...")
safepath.makedirs(ctx.archive_dir, safe_archive)
archive_path = os.path.join(ctx.archive_dir, safe_archive, str(job.jid))
log.info("Running job %d", job.jid)
run_job(job_config, archive_path)
job.delete()
def archive(ctx, config):
log.info('Creating archive directory...')
run.wait(
ctx.cluster.run(
args=[
'install', '-d', '-m0755', '--',
'/tmp/cephtest/archive',
],
wait=False,
)
)
try:
yield
finally:
if ctx.archive is not None:
log.info('Transferring archived files...')
logdir = os.path.join(ctx.archive, 'remote')
os.mkdir(logdir)
for remote in ctx.cluster.remotes.iterkeys():
path = os.path.join(logdir, remote.shortname)
os.mkdir(path)
log.debug('Transferring archived files from %s to %s', remote.shortname, path)
proc = remote.run(
args=[
'tar',
'c',
'-f', '-',
'-C', '/tmp/cephtest/archive',
'--',
'.',
],
stdout=run.PIPE,
wait=False,
)
tar = tarfile.open(mode='r|', fileobj=proc.stdout)
while True:
ti = tar.next()
if ti is None:
break
if ti.isdir():
# ignore silently; easier to just create leading dirs below
pass
elif ti.isfile():
sub = safepath.munge(ti.name)
safepath.makedirs(root=path, path=os.path.dirname(sub))
tar.makefile(ti, targetpath=os.path.join(path, sub))
else:
if ti.isdev():
type_ = 'device'
elif ti.issym():
type_ = 'symlink'
elif ti.islnk():
type_ = 'hard link'
else:
type_ = 'unknown'
log.info('Ignoring tar entry: %r type %r', ti.name, type_)
continue
proc.exitstatus.get()
log.info('Removing archive directory...')
run.wait(
ctx.cluster.run(
args=[
'rm',
'-rf',
'--',
'/tmp/cephtest/archive',
],
wait=False,
),
)
def worker():
parser = argparse.ArgumentParser(description="""
Grab jobs from a beanstalk queue and run the teuthology tests they
describe. One job is run at a time.
""")
parser.add_argument(
'-v', '--verbose',
action='store_true', default=None,
help='be more verbose',
)
parser.add_argument(
'--archive-dir',
metavar='DIR',
help='path under which to archive results',
required=True,
)
parser.add_argument(
'-l', '--log-dir',
help='path in which to store logs',
required=True,
)
parser.add_argument(
'-t', '--tube',
help='which beanstalk tube to read jobs from',
required=True,
)
ctx = parser.parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
filename=os.path.join(ctx.log_dir, 'worker.{pid}'.format(pid=os.getpid())),
format='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
)
if not os.path.isdir(ctx.archive_dir):
sys.exit("{prog}: archive directory must exist: {path}".format(
prog=os.path.basename(sys.argv[0]),
path=ctx.archive_dir,
))
from teuthology.misc import read_config
read_config(ctx)
beanstalk = connect(ctx)
beanstalk.watch(ctx.tube)
beanstalk.ignore('default')
while True:
job = beanstalk.reserve(timeout=60)
if job is None:
continue
# bury the job so it won't be re-run if it fails
job.bury()
log.debug('Reserved job %d', job.jid)
log.debug('Config is: %s', job.body)
job_config = yaml.safe_load(job.body)
safe_archive = safepath.munge(job_config['name'])
teuthology_branch = job_config.get('config', {}).get('teuthology_branch', 'master')
teuth_path = os.path.join(os.getenv("HOME"), 'teuthology-' + teuthology_branch, 'virtualenv', 'bin')
if not os.path.isdir(teuth_path):
raise Exception('Teuthology branch ' + teuthology_branch + ' not found at ' + teuth_path)
if job_config.get('last_in_suite'):
log.debug('Generating coverage for %s', job_config['name'])
args = [
os.path.join(teuth_path, 'teuthology-results'),
'--timeout',
str(job_config.get('results_timeout', 21600)),
'--email',
job_config['email'],
'--archive-dir',
os.path.join(ctx.archive_dir, safe_archive),
'--name',
job_config['name'],
]
subprocess.Popen(args=args)
else:
log.debug('Creating archive dir...')
safepath.makedirs(ctx.archive_dir, safe_archive)
archive_path = os.path.join(ctx.archive_dir, safe_archive, str(job.jid))
log.info('Running job %d', job.jid)
run_job(job_config, archive_path, teuth_path)
job.delete()
def worker():
parser = argparse.ArgumentParser(description="""
Grab jobs from a beanstalk queue and run the teuthology tests they
describe. One job is run at a time.
""")
parser.add_argument(
'-v',
'--verbose',
action='store_true',
default=None,
help='be more verbose',
)
parser.add_argument(
'--archive-dir',
metavar='DIR',
help='path under which to archive results',
required=True,
)
parser.add_argument(
'-l',
'--log-dir',
help='path in which to store logs',
required=True,
)
ctx = parser.parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
filename=os.path.join(ctx.log_dir,
'worker.{pid}'.format(pid=os.getpid())),
format='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
)
if not os.path.isdir(ctx.archive_dir):
sys.exit("{prog}: archive directory must exist: {path}".format(
prog=os.path.basename(sys.argv[0]),
path=ctx.archive_dir,
))
from teuthology.misc import read_config
read_config(ctx)
beanstalk = connect(ctx)
beanstalk.watch('teuthology')
beanstalk.ignore('default')
while True:
job = beanstalk.reserve(timeout=60)
if job is None:
continue
# bury the job so it won't be re-run if it fails
job.bury()
log.debug('Reserved job %d', job.jid)
log.debug('Config is: %s', job.body)
job_config = yaml.safe_load(job.body)
safe_archive = safepath.munge(job_config['name'])
if job_config.get('last_in_suite', False):
log.debug('Generating coverage for %s', job_config['name'])
args = [
os.path.join(os.path.dirname(sys.argv[0]),
'teuthology-results'),
'--timeout',
str(job_config.get('results_timeout', 21600)),
'--email',
job_config['email'],
'--archive-dir',
os.path.join(ctx.archive_dir, safe_archive),
'--name',
job_config['name'],
]
subprocess.Popen(args=args)
else:
log.debug('Creating archive dir...')
safepath.makedirs(ctx.archive_dir, safe_archive)
archive_path = os.path.join(ctx.archive_dir, safe_archive,
str(job.jid))
log.info('Running job %d', job.jid)
run_job(job_config, archive_path)
job.delete()
def archive(ctx, config):
log.info('Creating archive directory...')
run.wait(
ctx.cluster.run(
args=[
'install',
'-d',
'-m0755',
'--',
'/tmp/cephtest/archive',
],
wait=False,
))
try:
yield
finally:
if ctx.archive is not None:
log.info('Transferring archived files...')
logdir = os.path.join(ctx.archive, 'remote')
os.mkdir(logdir)
for remote in ctx.cluster.remotes.iterkeys():
path = os.path.join(logdir, remote.shortname)
os.mkdir(path)
log.debug('Transferring archived files from %s to %s',
remote.shortname, path)
proc = remote.run(
args=[
'tar',
'c',
'-f',
'-',
'-C',
'/tmp/cephtest/archive',
'--',
'.',
],
stdout=run.PIPE,
wait=False,
)
tar = tarfile.open(mode='r|', fileobj=proc.stdout)
while True:
ti = tar.next()
if ti is None:
break
if ti.isdir():
# ignore silently; easier to just create leading dirs below
pass
elif ti.isfile():
sub = safepath.munge(ti.name)
safepath.makedirs(root=path, path=os.path.dirname(sub))
tar.makefile(ti, targetpath=os.path.join(path, sub))
else:
if ti.isdev():
type_ = 'device'
elif ti.issym():
type_ = 'symlink'
elif ti.islnk():
type_ = 'hard link'
else:
type_ = 'unknown'
log.info('Ignoring tar entry: %r type %r', ti.name,
type_)
continue
proc.exitstatus.get()
log.info('Removing archive directory...')
run.wait(
ctx.cluster.run(
args=[
'rm',
'-rf',
'--',
'/tmp/cephtest/archive',
],
wait=False,
), )
