def exec_test():
# Parse arguments
interactive_on_error = False
create_cluster = False
args = sys.argv[1:]
flags = [a for a in args if a.startswith("-")]
modules = [a for a in args if not a.startswith("-")]
for f in flags:
if f == "--interactive":
interactive_on_error = True
elif f == "--create":
create_cluster = True
else:
log.error("Unknown option '{0}'".format(f))
sys.exit(-1)
# Help developers by stopping up-front if their tree isn't built enough for all the
# tools that the tests might want to use (add more here if needed)
require_binaries = [
"ceph-dencoder", "cephfs-journal-tool", "cephfs-data-scan",
"cephfs-table-tool", "ceph-fuse", "rados"
]
missing_binaries = [
b for b in require_binaries
if not os.path.exists(os.path.join(BIN_PREFIX, b))
]
if missing_binaries:
log.error("Some ceph binaries missing, please build them: {0}".format(
" ".join(missing_binaries)))
sys.exit(-1)
max_required_mds, max_required_clients, max_required_mgr = scan_tests(
modules)
remote = LocalRemote()
# Tolerate no MDSs or clients running at start
ps_txt = remote.run(args=["ps", "-u" +
str(os.getuid())]).stdout.getvalue().strip()
lines = ps_txt.split("\n")[1:]
for line in lines:
if 'ceph-fuse' in line or 'ceph-mds' in line:
pid = int(line.split()[0])
log.warn("Killing stray process {0}".format(line))
os.kill(pid, signal.SIGKILL)
# Fire up the Ceph cluster if the user requested it
if create_cluster:
log.info(
"Creating cluster with {0} MDS daemons".format(max_required_mds))
remote.run([os.path.join(SRC_PREFIX, "stop.sh")], check_status=False)
remote.run(["rm", "-rf", "./out"])
remote.run(["rm", "-rf", "./dev"])
vstart_env = os.environ.copy()
vstart_env["FS"] = "0"
vstart_env["MDS"] = max_required_mds.__str__()
vstart_env["OSD"] = "1"
vstart_env["MGR"] = max(max_required_mgr, 1).__str__()
remote.run(
[os.path.join(SRC_PREFIX, "vstart.sh"), "-n", "-d", "--nolockdep"],
env=vstart_env)
# Wait for OSD to come up so that subsequent injectargs etc will
# definitely succeed
LocalCephCluster(
LocalContext()).mon_manager.wait_for_all_osds_up(timeout=30)
# List of client mounts, sufficient to run the selected tests
clients = [i.__str__() for i in range(0, max_required_clients)]
test_dir = tempfile.mkdtemp()
teuth_config['test_path'] = test_dir
# Construct Mount classes
mounts = []
for client_id in clients:
# Populate client keyring (it sucks to use client.admin for test clients
# because it's awkward to find the logs later)
client_name = "client.{0}".format(client_id)
if client_name not in open("./keyring").read():
p = remote.run(args=[
os.path.join(BIN_PREFIX,
"ceph"), "auth", "get-or-create", client_name,
"osd", "allow rw", "mds", "allow", "mon", "allow r"
])
open("./keyring", "a").write(p.stdout.getvalue())
mount = LocalFuseMount(test_dir, client_id)
mounts.append(mount)
if mount.is_mounted():
log.warn("unmounting {0}".format(mount.mountpoint))
mount.umount_wait()
else:
if os.path.exists(mount.mountpoint):
os.rmdir(mount.mountpoint)
ctx = LocalContext()
ceph_cluster = LocalCephCluster(ctx)
mds_cluster = LocalMDSCluster(ctx)
mgr_cluster = LocalMgrCluster(ctx)
from tasks.cephfs_test_runner import DecoratingLoader
class LogStream(object):
def __init__(self):
self.buffer = ""
def write(self, data):
self.buffer += data
if "\n" in self.buffer:
lines = self.buffer.split("\n")
for line in lines[:-1]:
pass
# sys.stderr.write(line + "\n")
log.info(line)
self.buffer = lines[-1]
def flush(self):
pass
decorating_loader = DecoratingLoader({
"ctx": ctx,
"mounts": mounts,
"ceph_cluster": ceph_cluster,
"mds_cluster": mds_cluster,
"mgr_cluster": mgr_cluster,
})
# For the benefit of polling tests like test_full -- in teuthology land we set this
# in a .yaml, here it's just a hardcoded thing for the developer's pleasure.
remote.run(args=[
os.path.join(BIN_PREFIX, "ceph"), "tell", "osd.*", "injectargs",
"--osd-mon-report-interval-max", "5"
])
ceph_cluster.set_ceph_conf("osd", "osd_mon_report_interval_max", "5")
# Vstart defaults to two segments, which very easily gets a "behind on trimming" health warning
# from normal IO latency. Increase it for running teests.
ceph_cluster.set_ceph_conf("mds", "mds log max segments", "10")
# Make sure the filesystem created in tests has uid/gid that will let us talk to
# it after mounting it (without having to go root). Set in 'global' not just 'mds'
# so that cephfs-data-scan will pick it up too.
ceph_cluster.set_ceph_conf("global", "mds root ino uid",
"%s" % os.getuid())
ceph_cluster.set_ceph_conf("global", "mds root ino gid",
"%s" % os.getgid())
# Monkeypatch get_package_version to avoid having to work out what kind of distro we're on
def _get_package_version(remote, pkg_name):
# Used in cephfs tests to find fuse version. Your development workstation *does* have >=2.9, right?
return "2.9"
import teuthology.packaging
teuthology.packaging.get_package_version = _get_package_version
overall_suite = load_tests(modules, decorating_loader)
# Filter out tests that don't lend themselves to interactive running,
victims = []
for case, method in enumerate_methods(overall_suite):
fn = getattr(method, method._testMethodName)
drop_test = False
if hasattr(fn, 'is_for_teuthology') and getattr(
fn, 'is_for_teuthology') is True:
drop_test = True
log.warn("Dropping test because long running: ".format(
method.id()))
if getattr(fn, "needs_trimming", False) is True:
drop_test = (os.getuid() != 0)
log.warn("Dropping test because client trim unavailable: ".format(
method.id()))
if drop_test:
# Don't drop the test if it was explicitly requested in arguments
is_named = False
for named in modules:
if named.endswith(method.id()):
is_named = True
break
if not is_named:
victims.append((case, method))
log.info(
"Disabling {0} tests because of is_for_teuthology or needs_trimming".
format(len(victims)))
for s, method in victims:
s._tests.remove(method)
if interactive_on_error:
result_class = InteractiveFailureResult
else:
result_class = unittest.TextTestResult
fail_on_skip = False
class LoggingResult(result_class):
def startTest(self, test):
log.info("Starting test: {0}".format(self.getDescription(test)))
test.started_at = datetime.datetime.utcnow()
return super(LoggingResult, self).startTest(test)
def stopTest(self, test):
log.info("Stopped test: {0} in {1}s".format(
self.getDescription(test), (datetime.datetime.utcnow() -
test.started_at).total_seconds()))
def addSkip(self, test, reason):
if fail_on_skip:
# Don't just call addFailure because that requires a traceback
self.failures.append((test, reason))
else:
super(LoggingResult, self).addSkip(test, reason)
# Execute!
result = unittest.TextTestRunner(stream=LogStream(),
resultclass=LoggingResult,
verbosity=2,
failfast=True).run(overall_suite)
if not result.wasSuccessful():
result.printErrors() # duplicate output at end for convenience
bad_tests = []
for test, error in result.errors:
bad_tests.append(str(test))
for test, failure in result.failures:
bad_tests.append(str(test))
sys.exit(-1)
else:
sys.exit(0)
def exec_test():
# Help developers by stopping up-front if their tree isn't built enough for all the
# tools that the tests might want to use (add more here if needed)
require_binaries = ["ceph-dencoder", "cephfs-journal-tool", "cephfs-data-scan",
"cephfs-table-tool", "ceph-fuse", "rados"]
missing_binaries = [b for b in require_binaries if not os.path.exists(os.path.join(BIN_PREFIX, b))]
if missing_binaries:
log.error("Some ceph binaries missing, please build them: {0}".format(" ".join(missing_binaries)))
sys.exit(-1)
test_dir = tempfile.mkdtemp()
# Create as many of these as the biggest test requires
clients = ["0", "1", "2"]
remote = LocalRemote()
# Tolerate no MDSs or clients running at start
ps_txt = remote.run(
args=["ps", "aux"]
).stdout.getvalue().strip()
lines = ps_txt.split("\n")[1:]
for line in lines:
if 'ceph-fuse' in line or 'ceph-mds' in line:
pid = int(line.split()[1])
log.warn("Killing stray process {0}".format(line))
os.kill(pid, signal.SIGKILL)
class LocalCluster(object):
def __init__(self, rolename="placeholder"):
self.remotes = {
remote: [rolename]
}
def only(self, requested):
return self.__class__(rolename=requested)
teuth_config['test_path'] = test_dir
class LocalContext(object):
def __init__(self):
self.config = {}
self.teuthology_config = teuth_config
self.cluster = LocalCluster()
self.daemons = DaemonGroup()
# Shove some LocalDaemons into the ctx.daemons DaemonGroup instance so that any
# tests that want to look these up via ctx can do so.
# Inspect ceph.conf to see what roles exist
for conf_line in open("ceph.conf").readlines():
for svc_type in ["mon", "osd", "mds"]:
if svc_type not in self.daemons.daemons:
self.daemons.daemons[svc_type] = {}
match = re.match("^\[{0}\.(.+)\]$".format(svc_type), conf_line)
if match:
svc_id = match.group(1)
self.daemons.daemons[svc_type][svc_id] = LocalDaemon(svc_type, svc_id)
def __del__(self):
shutil.rmtree(self.teuthology_config['test_path'])
ctx = LocalContext()
mounts = []
for client_id in clients:
# Populate client keyring (it sucks to use client.admin for test clients
# because it's awkward to find the logs later)
client_name = "client.{0}".format(client_id)
if client_name not in open("./keyring").read():
p = remote.run(args=[os.path.join(BIN_PREFIX, "ceph"), "auth", "get-or-create", client_name,
"osd", "allow rw",
"mds", "allow",
"mon", "allow r"])
open("./keyring", "a").write(p.stdout.getvalue())
mount = LocalFuseMount(test_dir, client_id)
mounts.append(mount)
if mount.is_mounted():
log.warn("unmounting {0}".format(mount.mountpoint))
mount.umount_wait()
else:
if os.path.exists(mount.mountpoint):
os.rmdir(mount.mountpoint)
filesystem = LocalFilesystem(ctx)
mds_cluster = LocalMDSCluster(ctx)
from tasks.cephfs_test_runner import DecoratingLoader
class LogStream(object):
def __init__(self):
self.buffer = ""
def write(self, data):
self.buffer += data
if "\n" in self.buffer:
lines = self.buffer.split("\n")
for line in lines[:-1]:
pass
# sys.stderr.write(line + "\n")
log.info(line)
self.buffer = lines[-1]
def flush(self):
pass
decorating_loader = DecoratingLoader({
"ctx": ctx,
"mounts": mounts,
"fs": filesystem,
"mds_cluster": mds_cluster
})
# For the benefit of polling tests like test_full -- in teuthology land we set this
# in a .yaml, here it's just a hardcoded thing for the developer's pleasure.
remote.run(args=[os.path.join(BIN_PREFIX, "ceph"), "tell", "osd.*", "injectargs", "--osd-mon-report-interval-max", "5"])
filesystem.set_ceph_conf("osd", "osd_mon_report_interval_max", "5")
# Vstart defaults to two segments, which very easily gets a "behind on trimming" health warning
# from normal IO latency. Increase it for running teests.
filesystem.set_ceph_conf("mds", "mds log max segments", "10")
# Make sure the filesystem created in tests has uid/gid that will let us talk to
# it after mounting it (without having to go root). Set in 'global' not just 'mds'
# so that cephfs-data-scan will pick it up too.
filesystem.set_ceph_conf("global", "mds root ino uid", "%s" % os.getuid())
filesystem.set_ceph_conf("global", "mds root ino gid", "%s" % os.getgid())
# Monkeypatch get_package_version to avoid having to work out what kind of distro we're on
def _get_package_version(remote, pkg_name):
# Used in cephfs tests to find fuse version. Your development workstation *does* have >=2.9, right?
return "2.9"
import teuthology.packaging
teuthology.packaging.get_package_version = _get_package_version
def enumerate_methods(s):
for t in s._tests:
if isinstance(t, suite.BaseTestSuite):
for sub in enumerate_methods(t):
yield sub
else:
yield s, t
interactive_on_error = False
args = sys.argv[1:]
flags = [a for a in args if a.startswith("-")]
modules = [a for a in args if not a.startswith("-")]
for f in flags:
if f == "--interactive":
interactive_on_error = True
else:
log.error("Unknown option '{0}'".format(f))
sys.exit(-1)
if modules:
log.info("Executing modules: {0}".format(modules))
module_suites = []
for mod_name in modules:
# Test names like cephfs.test_auto_repair
log.info("Loaded: {0}".format(list(module_suites)))
module_suites.append(decorating_loader.loadTestsFromName(mod_name))
overall_suite = suite.TestSuite(module_suites)
else:
log.info("Excuting all tests")
overall_suite = decorating_loader.discover(
os.path.dirname(os.path.abspath(__file__))
)
# Filter out tests that don't lend themselves to interactive running,
victims = []
for case, method in enumerate_methods(overall_suite):
fn = getattr(method, method._testMethodName)
drop_test = False
if hasattr(fn, 'is_long_running') and getattr(fn, 'is_long_running') is True:
drop_test = True
log.warn("Dropping test because long running: ".format(method.id()))
if getattr(fn, "needs_trimming", False) is True:
drop_test = (os.getuid() != 0)
log.warn("Dropping test because client trim unavailable: ".format(method.id()))
if drop_test:
# Don't drop the test if it was explicitly requested in arguments
is_named = False
for named in modules:
if named.endswith(method.id()):
is_named = True
break
if not is_named:
victims.append((case, method))
log.info("Disabling {0} tests because of is_long_running or needs_trimming".format(len(victims)))
for s, method in victims:
s._tests.remove(method)
if interactive_on_error:
result_class = InteractiveFailureResult
else:
result_class = unittest.TextTestResult
fail_on_skip = False
class LoggingResult(result_class):
def startTest(self, test):
log.info("Starting test: {0}".format(self.getDescription(test)))
test.started_at = datetime.datetime.utcnow()
return super(LoggingResult, self).startTest(test)
def stopTest(self, test):
log.info("Stopped test: {0} in {1}s".format(
self.getDescription(test),
(datetime.datetime.utcnow() - test.started_at).total_seconds()
))
def addSkip(self, test, reason):
if fail_on_skip:
# Don't just call addFailure because that requires a traceback
self.failures.append((test, reason))
else:
super(LoggingResult, self).addSkip(test, reason)
# Execute!
result = unittest.TextTestRunner(
stream=LogStream(),
resultclass=LoggingResult,
verbosity=2,
failfast=True).run(overall_suite)
if not result.wasSuccessful():
result.printErrors() # duplicate output at end for convenience
bad_tests = []
for test, error in result.errors:
bad_tests.append(str(test))
for test, failure in result.failures:
bad_tests.append(str(test))
sys.exit(-1)
else:
sys.exit(0)
def exec_test():
# Help developers by stopping up-front if their tree isn't built enough for all the
# tools that the tests might want to use (add more here if needed)
require_binaries = ["ceph-dencoder", "cephfs-journal-tool", "cephfs-data-scan",
"cephfs-table-tool", "ceph-fuse", "rados"]
missing_binaries = [b for b in require_binaries if not os.path.exists(os.path.join(BIN_PREFIX, b))]
if missing_binaries:
log.error("Some ceph binaries missing, please build them: {0}".format(" ".join(missing_binaries)))
sys.exit(-1)
test_dir = tempfile.mkdtemp()
# Create as many of these as the biggest test requires
clients = ["0", "1", "2", "3"]
remote = LocalRemote()
# Tolerate no MDSs or clients running at start
ps_txt = remote.run(
args=["ps", "-u"+str(os.getuid())]
).stdout.getvalue().strip()
lines = ps_txt.split("\n")[1:]
for line in lines:
if 'ceph-fuse' in line or 'ceph-mds' in line:
pid = int(line.split()[0])
log.warn("Killing stray process {0}".format(line))
os.kill(pid, signal.SIGKILL)
class LocalCluster(object):
def __init__(self, rolename="placeholder"):
self.remotes = {
remote: [rolename]
}
def only(self, requested):
return self.__class__(rolename=requested)
teuth_config['test_path'] = test_dir
class LocalContext(object):
def __init__(self):
self.config = {}
self.teuthology_config = teuth_config
self.cluster = LocalCluster()
self.daemons = DaemonGroup()
# Shove some LocalDaemons into the ctx.daemons DaemonGroup instance so that any
# tests that want to look these up via ctx can do so.
# Inspect ceph.conf to see what roles exist
for conf_line in open("ceph.conf").readlines():
for svc_type in ["mon", "osd", "mds", "mgr"]:
if svc_type not in self.daemons.daemons:
self.daemons.daemons[svc_type] = {}
match = re.match("^\[{0}\.(.+)\]$".format(svc_type), conf_line)
if match:
svc_id = match.group(1)
self.daemons.daemons[svc_type][svc_id] = LocalDaemon(svc_type, svc_id)
def __del__(self):
shutil.rmtree(self.teuthology_config['test_path'])
ctx = LocalContext()
mounts = []
for client_id in clients:
# Populate client keyring (it sucks to use client.admin for test clients
# because it's awkward to find the logs later)
client_name = "client.{0}".format(client_id)
if client_name not in open("./keyring").read():
p = remote.run(args=[os.path.join(BIN_PREFIX, "ceph"), "auth", "get-or-create", client_name,
"osd", "allow rw",
"mds", "allow",
"mon", "allow r"])
open("./keyring", "a").write(p.stdout.getvalue())
mount = LocalFuseMount(test_dir, client_id)
mounts.append(mount)
if mount.is_mounted():
log.warn("unmounting {0}".format(mount.mountpoint))
mount.umount_wait()
else:
if os.path.exists(mount.mountpoint):
os.rmdir(mount.mountpoint)
ceph_cluster = LocalCephCluster(ctx)
mds_cluster = LocalMDSCluster(ctx)
mgr_cluster = LocalMgrCluster(ctx)
from tasks.cephfs_test_runner import DecoratingLoader
class LogStream(object):
def __init__(self):
self.buffer = ""
def write(self, data):
self.buffer += data
if "\n" in self.buffer:
lines = self.buffer.split("\n")
for line in lines[:-1]:
pass
# sys.stderr.write(line + "\n")
log.info(line)
self.buffer = lines[-1]
def flush(self):
pass
decorating_loader = DecoratingLoader({
"ctx": ctx,
"mounts": mounts,
"ceph_cluster": ceph_cluster,
"mds_cluster": mds_cluster,
"mgr_cluster": mgr_cluster,
})
# For the benefit of polling tests like test_full -- in teuthology land we set this
# in a .yaml, here it's just a hardcoded thing for the developer's pleasure.
remote.run(args=[os.path.join(BIN_PREFIX, "ceph"), "tell", "osd.*", "injectargs", "--osd-mon-report-interval-max", "5"])
ceph_cluster.set_ceph_conf("osd", "osd_mon_report_interval_max", "5")
# Vstart defaults to two segments, which very easily gets a "behind on trimming" health warning
# from normal IO latency. Increase it for running teests.
ceph_cluster.set_ceph_conf("mds", "mds log max segments", "10")
# Make sure the filesystem created in tests has uid/gid that will let us talk to
# it after mounting it (without having to go root). Set in 'global' not just 'mds'
# so that cephfs-data-scan will pick it up too.
ceph_cluster.set_ceph_conf("global", "mds root ino uid", "%s" % os.getuid())
ceph_cluster.set_ceph_conf("global", "mds root ino gid", "%s" % os.getgid())
# Monkeypatch get_package_version to avoid having to work out what kind of distro we're on
def _get_package_version(remote, pkg_name):
# Used in cephfs tests to find fuse version. Your development workstation *does* have >=2.9, right?
return "2.9"
import teuthology.packaging
teuthology.packaging.get_package_version = _get_package_version
def enumerate_methods(s):
for t in s._tests:
if isinstance(t, suite.BaseTestSuite):
for sub in enumerate_methods(t):
yield sub
else:
yield s, t
interactive_on_error = False
args = sys.argv[1:]
flags = [a for a in args if a.startswith("-")]
modules = [a for a in args if not a.startswith("-")]
for f in flags:
if f == "--interactive":
interactive_on_error = True
else:
log.error("Unknown option '{0}'".format(f))
sys.exit(-1)
if modules:
log.info("Executing modules: {0}".format(modules))
module_suites = []
for mod_name in modules:
# Test names like cephfs.test_auto_repair
module_suites.append(decorating_loader.loadTestsFromName(mod_name))
log.info("Loaded: {0}".format(list(module_suites)))
overall_suite = suite.TestSuite(module_suites)
else:
log.info("Executing all cephfs tests")
overall_suite = decorating_loader.discover(
os.path.join(os.path.dirname(os.path.abspath(__file__)), "cephfs")
)
# Filter out tests that don't lend themselves to interactive running,
victims = []
for case, method in enumerate_methods(overall_suite):
fn = getattr(method, method._testMethodName)
drop_test = False
if hasattr(fn, 'is_for_teuthology') and getattr(fn, 'is_for_teuthology') is True:
drop_test = True
log.warn("Dropping test because long running: ".format(method.id()))
if getattr(fn, "needs_trimming", False) is True:
drop_test = (os.getuid() != 0)
log.warn("Dropping test because client trim unavailable: ".format(method.id()))
if drop_test:
# Don't drop the test if it was explicitly requested in arguments
is_named = False
for named in modules:
if named.endswith(method.id()):
is_named = True
break
if not is_named:
victims.append((case, method))
log.info("Disabling {0} tests because of is_for_teuthology or needs_trimming".format(len(victims)))
for s, method in victims:
s._tests.remove(method)
if interactive_on_error:
result_class = InteractiveFailureResult
else:
result_class = unittest.TextTestResult
fail_on_skip = False
class LoggingResult(result_class):
def startTest(self, test):
log.info("Starting test: {0}".format(self.getDescription(test)))
test.started_at = datetime.datetime.utcnow()
return super(LoggingResult, self).startTest(test)
def stopTest(self, test):
log.info("Stopped test: {0} in {1}s".format(
self.getDescription(test),
(datetime.datetime.utcnow() - test.started_at).total_seconds()
))
def addSkip(self, test, reason):
if fail_on_skip:
# Don't just call addFailure because that requires a traceback
self.failures.append((test, reason))
else:
super(LoggingResult, self).addSkip(test, reason)
# Execute!
result = unittest.TextTestRunner(
stream=LogStream(),
resultclass=LoggingResult,
verbosity=2,
failfast=True).run(overall_suite)
if not result.wasSuccessful():
result.printErrors() # duplicate output at end for convenience
bad_tests = []
for test, error in result.errors:
bad_tests.append(str(test))
for test, failure in result.failures:
bad_tests.append(str(test))
sys.exit(-1)
else:
sys.exit(0)
