def exec_test(): # Parse arguments interactive_on_error = False create_cluster = False args = sys.argv[1:] flags = [a for a in args if a.startswith("-")] modules = [a for a in args if not a.startswith("-")] for f in flags: if f == "--interactive": interactive_on_error = True elif f == "--create": create_cluster = True else: log.error("Unknown option '{0}'".format(f)) sys.exit(-1) # Help developers by stopping up-front if their tree isn't built enough for all the # tools that the tests might want to use (add more here if needed) require_binaries = [ "ceph-dencoder", "cephfs-journal-tool", "cephfs-data-scan", "cephfs-table-tool", "ceph-fuse", "rados" ] missing_binaries = [ b for b in require_binaries if not os.path.exists(os.path.join(BIN_PREFIX, b)) ] if missing_binaries: log.error("Some ceph binaries missing, please build them: {0}".format( " ".join(missing_binaries))) sys.exit(-1) max_required_mds, max_required_clients, max_required_mgr = scan_tests( modules) remote = LocalRemote() # Tolerate no MDSs or clients running at start ps_txt = remote.run(args=["ps", "-u" + str(os.getuid())]).stdout.getvalue().strip() lines = ps_txt.split("\n")[1:] for line in lines: if 'ceph-fuse' in line or 'ceph-mds' in line: pid = int(line.split()[0]) log.warn("Killing stray process {0}".format(line)) os.kill(pid, signal.SIGKILL) # Fire up the Ceph cluster if the user requested it if create_cluster: log.info( "Creating cluster with {0} MDS daemons".format(max_required_mds)) remote.run([os.path.join(SRC_PREFIX, "stop.sh")], check_status=False) remote.run(["rm", "-rf", "./out"]) remote.run(["rm", "-rf", "./dev"]) vstart_env = os.environ.copy() vstart_env["FS"] = "0" vstart_env["MDS"] = max_required_mds.__str__() vstart_env["OSD"] = "1" vstart_env["MGR"] = max(max_required_mgr, 1).__str__() remote.run( [os.path.join(SRC_PREFIX, "vstart.sh"), "-n", "-d", "--nolockdep"], env=vstart_env) # Wait for OSD to come up so that subsequent injectargs etc will # definitely succeed LocalCephCluster( LocalContext()).mon_manager.wait_for_all_osds_up(timeout=30) # List of client mounts, sufficient to run the selected tests clients = [i.__str__() for i in range(0, max_required_clients)] test_dir = tempfile.mkdtemp() teuth_config['test_path'] = test_dir # Construct Mount classes mounts = [] for client_id in clients: # Populate client keyring (it sucks to use client.admin for test clients # because it's awkward to find the logs later) client_name = "client.{0}".format(client_id) if client_name not in open("./keyring").read(): p = remote.run(args=[ os.path.join(BIN_PREFIX, "ceph"), "auth", "get-or-create", client_name, "osd", "allow rw", "mds", "allow", "mon", "allow r" ]) open("./keyring", "a").write(p.stdout.getvalue()) mount = LocalFuseMount(test_dir, client_id) mounts.append(mount) if mount.is_mounted(): log.warn("unmounting {0}".format(mount.mountpoint)) mount.umount_wait() else: if os.path.exists(mount.mountpoint): os.rmdir(mount.mountpoint) ctx = LocalContext() ceph_cluster = LocalCephCluster(ctx) mds_cluster = LocalMDSCluster(ctx) mgr_cluster = LocalMgrCluster(ctx) from tasks.cephfs_test_runner import DecoratingLoader class LogStream(object): def __init__(self): self.buffer = "" def write(self, data): self.buffer += data if "\n" in self.buffer: lines = self.buffer.split("\n") for line in lines[:-1]: pass # sys.stderr.write(line + "\n") log.info(line) self.buffer = lines[-1] def flush(self): pass decorating_loader = DecoratingLoader({ "ctx": ctx, "mounts": mounts, "ceph_cluster": ceph_cluster, "mds_cluster": mds_cluster, "mgr_cluster": mgr_cluster, }) # For the benefit of polling tests like test_full -- in teuthology land we set this # in a .yaml, here it's just a hardcoded thing for the developer's pleasure. remote.run(args=[ os.path.join(BIN_PREFIX, "ceph"), "tell", "osd.*", "injectargs", "--osd-mon-report-interval-max", "5" ]) ceph_cluster.set_ceph_conf("osd", "osd_mon_report_interval_max", "5") # Vstart defaults to two segments, which very easily gets a "behind on trimming" health warning # from normal IO latency. Increase it for running teests. ceph_cluster.set_ceph_conf("mds", "mds log max segments", "10") # Make sure the filesystem created in tests has uid/gid that will let us talk to # it after mounting it (without having to go root). Set in 'global' not just 'mds' # so that cephfs-data-scan will pick it up too. ceph_cluster.set_ceph_conf("global", "mds root ino uid", "%s" % os.getuid()) ceph_cluster.set_ceph_conf("global", "mds root ino gid", "%s" % os.getgid()) # Monkeypatch get_package_version to avoid having to work out what kind of distro we're on def _get_package_version(remote, pkg_name): # Used in cephfs tests to find fuse version. Your development workstation *does* have >=2.9, right? return "2.9" import teuthology.packaging teuthology.packaging.get_package_version = _get_package_version overall_suite = load_tests(modules, decorating_loader) # Filter out tests that don't lend themselves to interactive running, victims = [] for case, method in enumerate_methods(overall_suite): fn = getattr(method, method._testMethodName) drop_test = False if hasattr(fn, 'is_for_teuthology') and getattr( fn, 'is_for_teuthology') is True: drop_test = True log.warn("Dropping test because long running: ".format( method.id())) if getattr(fn, "needs_trimming", False) is True: drop_test = (os.getuid() != 0) log.warn("Dropping test because client trim unavailable: ".format( method.id())) if drop_test: # Don't drop the test if it was explicitly requested in arguments is_named = False for named in modules: if named.endswith(method.id()): is_named = True break if not is_named: victims.append((case, method)) log.info( "Disabling {0} tests because of is_for_teuthology or needs_trimming". format(len(victims))) for s, method in victims: s._tests.remove(method) if interactive_on_error: result_class = InteractiveFailureResult else: result_class = unittest.TextTestResult fail_on_skip = False class LoggingResult(result_class): def startTest(self, test): log.info("Starting test: {0}".format(self.getDescription(test))) test.started_at = datetime.datetime.utcnow() return super(LoggingResult, self).startTest(test) def stopTest(self, test): log.info("Stopped test: {0} in {1}s".format( self.getDescription(test), (datetime.datetime.utcnow() - test.started_at).total_seconds())) def addSkip(self, test, reason): if fail_on_skip: # Don't just call addFailure because that requires a traceback self.failures.append((test, reason)) else: super(LoggingResult, self).addSkip(test, reason) # Execute! result = unittest.TextTestRunner(stream=LogStream(), resultclass=LoggingResult, verbosity=2, failfast=True).run(overall_suite) if not result.wasSuccessful(): result.printErrors() # duplicate output at end for convenience bad_tests = [] for test, error in result.errors: bad_tests.append(str(test)) for test, failure in result.failures: bad_tests.append(str(test)) sys.exit(-1) else: sys.exit(0)
def exec_test(): # Help developers by stopping up-front if their tree isn't built enough for all the # tools that the tests might want to use (add more here if needed) require_binaries = ["ceph-dencoder", "cephfs-journal-tool", "cephfs-data-scan", "cephfs-table-tool", "ceph-fuse", "rados"] missing_binaries = [b for b in require_binaries if not os.path.exists(os.path.join(BIN_PREFIX, b))] if missing_binaries: log.error("Some ceph binaries missing, please build them: {0}".format(" ".join(missing_binaries))) sys.exit(-1) test_dir = tempfile.mkdtemp() # Create as many of these as the biggest test requires clients = ["0", "1", "2"] remote = LocalRemote() # Tolerate no MDSs or clients running at start ps_txt = remote.run( args=["ps", "aux"] ).stdout.getvalue().strip() lines = ps_txt.split("\n")[1:] for line in lines: if 'ceph-fuse' in line or 'ceph-mds' in line: pid = int(line.split()[1]) log.warn("Killing stray process {0}".format(line)) os.kill(pid, signal.SIGKILL) class LocalCluster(object): def __init__(self, rolename="placeholder"): self.remotes = { remote: [rolename] } def only(self, requested): return self.__class__(rolename=requested) teuth_config['test_path'] = test_dir class LocalContext(object): def __init__(self): self.config = {} self.teuthology_config = teuth_config self.cluster = LocalCluster() self.daemons = DaemonGroup() # Shove some LocalDaemons into the ctx.daemons DaemonGroup instance so that any # tests that want to look these up via ctx can do so. # Inspect ceph.conf to see what roles exist for conf_line in open("ceph.conf").readlines(): for svc_type in ["mon", "osd", "mds"]: if svc_type not in self.daemons.daemons: self.daemons.daemons[svc_type] = {} match = re.match("^\[{0}\.(.+)\]$".format(svc_type), conf_line) if match: svc_id = match.group(1) self.daemons.daemons[svc_type][svc_id] = LocalDaemon(svc_type, svc_id) def __del__(self): shutil.rmtree(self.teuthology_config['test_path']) ctx = LocalContext() mounts = [] for client_id in clients: # Populate client keyring (it sucks to use client.admin for test clients # because it's awkward to find the logs later) client_name = "client.{0}".format(client_id) if client_name not in open("./keyring").read(): p = remote.run(args=[os.path.join(BIN_PREFIX, "ceph"), "auth", "get-or-create", client_name, "osd", "allow rw", "mds", "allow", "mon", "allow r"]) open("./keyring", "a").write(p.stdout.getvalue()) mount = LocalFuseMount(test_dir, client_id) mounts.append(mount) if mount.is_mounted(): log.warn("unmounting {0}".format(mount.mountpoint)) mount.umount_wait() else: if os.path.exists(mount.mountpoint): os.rmdir(mount.mountpoint) filesystem = LocalFilesystem(ctx) mds_cluster = LocalMDSCluster(ctx) from tasks.cephfs_test_runner import DecoratingLoader class LogStream(object): def __init__(self): self.buffer = "" def write(self, data): self.buffer += data if "\n" in self.buffer: lines = self.buffer.split("\n") for line in lines[:-1]: pass # sys.stderr.write(line + "\n") log.info(line) self.buffer = lines[-1] def flush(self): pass decorating_loader = DecoratingLoader({ "ctx": ctx, "mounts": mounts, "fs": filesystem, "mds_cluster": mds_cluster }) # For the benefit of polling tests like test_full -- in teuthology land we set this # in a .yaml, here it's just a hardcoded thing for the developer's pleasure. remote.run(args=[os.path.join(BIN_PREFIX, "ceph"), "tell", "osd.*", "injectargs", "--osd-mon-report-interval-max", "5"]) filesystem.set_ceph_conf("osd", "osd_mon_report_interval_max", "5") # Vstart defaults to two segments, which very easily gets a "behind on trimming" health warning # from normal IO latency. Increase it for running teests. filesystem.set_ceph_conf("mds", "mds log max segments", "10") # Make sure the filesystem created in tests has uid/gid that will let us talk to # it after mounting it (without having to go root). Set in 'global' not just 'mds' # so that cephfs-data-scan will pick it up too. filesystem.set_ceph_conf("global", "mds root ino uid", "%s" % os.getuid()) filesystem.set_ceph_conf("global", "mds root ino gid", "%s" % os.getgid()) # Monkeypatch get_package_version to avoid having to work out what kind of distro we're on def _get_package_version(remote, pkg_name): # Used in cephfs tests to find fuse version. Your development workstation *does* have >=2.9, right? return "2.9" import teuthology.packaging teuthology.packaging.get_package_version = _get_package_version def enumerate_methods(s): for t in s._tests: if isinstance(t, suite.BaseTestSuite): for sub in enumerate_methods(t): yield sub else: yield s, t interactive_on_error = False args = sys.argv[1:] flags = [a for a in args if a.startswith("-")] modules = [a for a in args if not a.startswith("-")] for f in flags: if f == "--interactive": interactive_on_error = True else: log.error("Unknown option '{0}'".format(f)) sys.exit(-1) if modules: log.info("Executing modules: {0}".format(modules)) module_suites = [] for mod_name in modules: # Test names like cephfs.test_auto_repair log.info("Loaded: {0}".format(list(module_suites))) module_suites.append(decorating_loader.loadTestsFromName(mod_name)) overall_suite = suite.TestSuite(module_suites) else: log.info("Excuting all tests") overall_suite = decorating_loader.discover( os.path.dirname(os.path.abspath(__file__)) ) # Filter out tests that don't lend themselves to interactive running, victims = [] for case, method in enumerate_methods(overall_suite): fn = getattr(method, method._testMethodName) drop_test = False if hasattr(fn, 'is_long_running') and getattr(fn, 'is_long_running') is True: drop_test = True log.warn("Dropping test because long running: ".format(method.id())) if getattr(fn, "needs_trimming", False) is True: drop_test = (os.getuid() != 0) log.warn("Dropping test because client trim unavailable: ".format(method.id())) if drop_test: # Don't drop the test if it was explicitly requested in arguments is_named = False for named in modules: if named.endswith(method.id()): is_named = True break if not is_named: victims.append((case, method)) log.info("Disabling {0} tests because of is_long_running or needs_trimming".format(len(victims))) for s, method in victims: s._tests.remove(method) if interactive_on_error: result_class = InteractiveFailureResult else: result_class = unittest.TextTestResult fail_on_skip = False class LoggingResult(result_class): def startTest(self, test): log.info("Starting test: {0}".format(self.getDescription(test))) test.started_at = datetime.datetime.utcnow() return super(LoggingResult, self).startTest(test) def stopTest(self, test): log.info("Stopped test: {0} in {1}s".format( self.getDescription(test), (datetime.datetime.utcnow() - test.started_at).total_seconds() )) def addSkip(self, test, reason): if fail_on_skip: # Don't just call addFailure because that requires a traceback self.failures.append((test, reason)) else: super(LoggingResult, self).addSkip(test, reason) # Execute! result = unittest.TextTestRunner( stream=LogStream(), resultclass=LoggingResult, verbosity=2, failfast=True).run(overall_suite) if not result.wasSuccessful(): result.printErrors() # duplicate output at end for convenience bad_tests = [] for test, error in result.errors: bad_tests.append(str(test)) for test, failure in result.failures: bad_tests.append(str(test)) sys.exit(-1) else: sys.exit(0)
def exec_test(): # Help developers by stopping up-front if their tree isn't built enough for all the # tools that the tests might want to use (add more here if needed) require_binaries = ["ceph-dencoder", "cephfs-journal-tool", "cephfs-data-scan", "cephfs-table-tool", "ceph-fuse", "rados"] missing_binaries = [b for b in require_binaries if not os.path.exists(os.path.join(BIN_PREFIX, b))] if missing_binaries: log.error("Some ceph binaries missing, please build them: {0}".format(" ".join(missing_binaries))) sys.exit(-1) test_dir = tempfile.mkdtemp() # Create as many of these as the biggest test requires clients = ["0", "1", "2", "3"] remote = LocalRemote() # Tolerate no MDSs or clients running at start ps_txt = remote.run( args=["ps", "-u"+str(os.getuid())] ).stdout.getvalue().strip() lines = ps_txt.split("\n")[1:] for line in lines: if 'ceph-fuse' in line or 'ceph-mds' in line: pid = int(line.split()[0]) log.warn("Killing stray process {0}".format(line)) os.kill(pid, signal.SIGKILL) class LocalCluster(object): def __init__(self, rolename="placeholder"): self.remotes = { remote: [rolename] } def only(self, requested): return self.__class__(rolename=requested) teuth_config['test_path'] = test_dir class LocalContext(object): def __init__(self): self.config = {} self.teuthology_config = teuth_config self.cluster = LocalCluster() self.daemons = DaemonGroup() # Shove some LocalDaemons into the ctx.daemons DaemonGroup instance so that any # tests that want to look these up via ctx can do so. # Inspect ceph.conf to see what roles exist for conf_line in open("ceph.conf").readlines(): for svc_type in ["mon", "osd", "mds", "mgr"]: if svc_type not in self.daemons.daemons: self.daemons.daemons[svc_type] = {} match = re.match("^\[{0}\.(.+)\]$".format(svc_type), conf_line) if match: svc_id = match.group(1) self.daemons.daemons[svc_type][svc_id] = LocalDaemon(svc_type, svc_id) def __del__(self): shutil.rmtree(self.teuthology_config['test_path']) ctx = LocalContext() mounts = [] for client_id in clients: # Populate client keyring (it sucks to use client.admin for test clients # because it's awkward to find the logs later) client_name = "client.{0}".format(client_id) if client_name not in open("./keyring").read(): p = remote.run(args=[os.path.join(BIN_PREFIX, "ceph"), "auth", "get-or-create", client_name, "osd", "allow rw", "mds", "allow", "mon", "allow r"]) open("./keyring", "a").write(p.stdout.getvalue()) mount = LocalFuseMount(test_dir, client_id) mounts.append(mount) if mount.is_mounted(): log.warn("unmounting {0}".format(mount.mountpoint)) mount.umount_wait() else: if os.path.exists(mount.mountpoint): os.rmdir(mount.mountpoint) ceph_cluster = LocalCephCluster(ctx) mds_cluster = LocalMDSCluster(ctx) mgr_cluster = LocalMgrCluster(ctx) from tasks.cephfs_test_runner import DecoratingLoader class LogStream(object): def __init__(self): self.buffer = "" def write(self, data): self.buffer += data if "\n" in self.buffer: lines = self.buffer.split("\n") for line in lines[:-1]: pass # sys.stderr.write(line + "\n") log.info(line) self.buffer = lines[-1] def flush(self): pass decorating_loader = DecoratingLoader({ "ctx": ctx, "mounts": mounts, "ceph_cluster": ceph_cluster, "mds_cluster": mds_cluster, "mgr_cluster": mgr_cluster, }) # For the benefit of polling tests like test_full -- in teuthology land we set this # in a .yaml, here it's just a hardcoded thing for the developer's pleasure. remote.run(args=[os.path.join(BIN_PREFIX, "ceph"), "tell", "osd.*", "injectargs", "--osd-mon-report-interval-max", "5"]) ceph_cluster.set_ceph_conf("osd", "osd_mon_report_interval_max", "5") # Vstart defaults to two segments, which very easily gets a "behind on trimming" health warning # from normal IO latency. Increase it for running teests. ceph_cluster.set_ceph_conf("mds", "mds log max segments", "10") # Make sure the filesystem created in tests has uid/gid that will let us talk to # it after mounting it (without having to go root). Set in 'global' not just 'mds' # so that cephfs-data-scan will pick it up too. ceph_cluster.set_ceph_conf("global", "mds root ino uid", "%s" % os.getuid()) ceph_cluster.set_ceph_conf("global", "mds root ino gid", "%s" % os.getgid()) # Monkeypatch get_package_version to avoid having to work out what kind of distro we're on def _get_package_version(remote, pkg_name): # Used in cephfs tests to find fuse version. Your development workstation *does* have >=2.9, right? return "2.9" import teuthology.packaging teuthology.packaging.get_package_version = _get_package_version def enumerate_methods(s): for t in s._tests: if isinstance(t, suite.BaseTestSuite): for sub in enumerate_methods(t): yield sub else: yield s, t interactive_on_error = False args = sys.argv[1:] flags = [a for a in args if a.startswith("-")] modules = [a for a in args if not a.startswith("-")] for f in flags: if f == "--interactive": interactive_on_error = True else: log.error("Unknown option '{0}'".format(f)) sys.exit(-1) if modules: log.info("Executing modules: {0}".format(modules)) module_suites = [] for mod_name in modules: # Test names like cephfs.test_auto_repair module_suites.append(decorating_loader.loadTestsFromName(mod_name)) log.info("Loaded: {0}".format(list(module_suites))) overall_suite = suite.TestSuite(module_suites) else: log.info("Executing all cephfs tests") overall_suite = decorating_loader.discover( os.path.join(os.path.dirname(os.path.abspath(__file__)), "cephfs") ) # Filter out tests that don't lend themselves to interactive running, victims = [] for case, method in enumerate_methods(overall_suite): fn = getattr(method, method._testMethodName) drop_test = False if hasattr(fn, 'is_for_teuthology') and getattr(fn, 'is_for_teuthology') is True: drop_test = True log.warn("Dropping test because long running: ".format(method.id())) if getattr(fn, "needs_trimming", False) is True: drop_test = (os.getuid() != 0) log.warn("Dropping test because client trim unavailable: ".format(method.id())) if drop_test: # Don't drop the test if it was explicitly requested in arguments is_named = False for named in modules: if named.endswith(method.id()): is_named = True break if not is_named: victims.append((case, method)) log.info("Disabling {0} tests because of is_for_teuthology or needs_trimming".format(len(victims))) for s, method in victims: s._tests.remove(method) if interactive_on_error: result_class = InteractiveFailureResult else: result_class = unittest.TextTestResult fail_on_skip = False class LoggingResult(result_class): def startTest(self, test): log.info("Starting test: {0}".format(self.getDescription(test))) test.started_at = datetime.datetime.utcnow() return super(LoggingResult, self).startTest(test) def stopTest(self, test): log.info("Stopped test: {0} in {1}s".format( self.getDescription(test), (datetime.datetime.utcnow() - test.started_at).total_seconds() )) def addSkip(self, test, reason): if fail_on_skip: # Don't just call addFailure because that requires a traceback self.failures.append((test, reason)) else: super(LoggingResult, self).addSkip(test, reason) # Execute! result = unittest.TextTestRunner( stream=LogStream(), resultclass=LoggingResult, verbosity=2, failfast=True).run(overall_suite) if not result.wasSuccessful(): result.printErrors() # duplicate output at end for convenience bad_tests = [] for test, error in result.errors: bad_tests.append(str(test)) for test, failure in result.failures: bad_tests.append(str(test)) sys.exit(-1) else: sys.exit(0)