def ordered_module_files(): ordered_modules = [ cmk_path() + "/modules/check_mk_base.py", cmk_path() + "/modules/check_mk.py", cmk_path() + "/modules/config.py", cmk_path() + "/modules/discovery.py", cmk_path() + "/modules/snmp.py", cmk_path() + "/modules/notify.py", cmk_path() + "/modules/events.py", cmk_path() + "/modules/nagios.py", cmk_path() + "/modules/automation.py", cmk_path() + "/modules/inventory.py", cmc_path() + "/modules/real_time_checks.py", cmc_path() + "/modules/alert_handling.py", cmc_path() + "/modules/keepalive.py", cmc_path() + "/modules/cmc.py", cmc_path() + "/modules/inline_snmp.py", cmc_path() + "/modules/agent_bakery.py", cmc_path() + "/modules/rrd.py", cme_path() + "/modules/managed.py", ] modules = ordered_modules # Add modules which are not specified above for path in module_files(): if path not in modules: modules.append(path) return modules
def test_pylint(pylint_test_dir): # Only specify the path to python packages or modules here modules_or_packages = [ # OMD "omd/packages/omd/omdlib", "livestatus/api/python/livestatus.py", # Check_MK base "cmk_base", # TODO: Check if this kind of "overlay" really works. # TODO: Why do we have e.g. a symlink cmk_base/cee -> enterprise/cmk_base/cee? "enterprise/cmk_base/automations/cee.py", "enterprise/cmk_base/cee", "enterprise/cmk_base/default_config/cee.py", "enterprise/cmk_base/modes/cee.py", "managed/cmk_base/default_config/cme.py", # cmk module level # TODO: This checks the whole cmk hierarchy, including things like # cmk.gui.plugins.cron etc. Do we really want that here? # TODO: Funny links there, see above. "cmk", "enterprise/cmk/cee", # GUI specific "web/app/index.wsgi", "enterprise/cmk/gui/cee", "managed/cmk/gui/cme", ] # Add the compiled files for things that are no modules yet open(pylint_test_dir + "/__init__.py", "w") _compile_check_and_inventory_plugins(pylint_test_dir) _compile_bakery_plugins(pylint_test_dir) modules_or_packages += [ pylint_test_dir, ] # We use our own search logic to find scripts without python extension search_paths = [ "omd/packages/omd.bin", "bin", "notifications", "agents/plugins", "agents/special", "active_checks", "enterprise/agents/plugins", "enterprise/bin", "enterprise/misc", ] for path in search_paths: abs_path = cmk_path() + "/" + path for fname in pylint_cmk.get_pylint_files(abs_path, "*"): modules_or_packages.append(path + "/" + fname) exit_code = pylint_cmk.run_pylint(cmk_path(), modules_or_packages) assert exit_code == 0, "PyLint found an error"
def test_manpage_files(): os.chdir(cmk_path()) checkend_manpages = 0 for f in os.listdir("%s/checkman" % cmk_path()): if f[0] == ".": continue _check_manpage(f) checkend_manpages += 1 assert checkend_manpages > 0
def _compile_bakery_plugins(pylint_test_dir): with open(pylint_test_dir + "/cmk_bakery_plugins.py", "w") as f: pylint_cmk.add_file( f, os.path.realpath( os.path.join(cmk_path(), "enterprise/cmk_base/cee/agent_bakery_plugins.py"))) # This pylint warning is incompatible with our "concatenation technology". f.write("# pylint: disable=reimported,wrong-import-order,wrong-import-position\n") # Also add bakery plugins for path in pylint_cmk.check_files(os.path.join(cmk_path(), "enterprise/agents/bakery")): pylint_cmk.add_file(f, path)
def test_write_precompiled_werks(tmp_path, monkeypatch): tmp_dir = str(tmp_path) all_werks = cmk.utils.werks.load_raw_files(Path(testlib.cmk_path()) / ".werks") cre_werks = {w["id"]: w for w in all_werks.values() if w["edition"] == "cre"} cee_werks = {w["id"]: w for w in all_werks.values() if w["edition"] == "cee"} cme_werks = {w["id"]: w for w in all_werks.values() if w["edition"] == "cme"} assert len(cre_werks) > 1000 assert [w for w in cre_werks.keys() if 9000 <= w < 10000] == [] cmk.utils.werks.write_precompiled_werks(Path(tmp_dir) / "werks", cre_werks) assert len(cee_werks) > 700 cmk.utils.werks.write_precompiled_werks(Path(tmp_dir) / "werks-enterprise", cee_werks) assert len(cme_werks) > 5 cmk.utils.werks.write_precompiled_werks(Path(tmp_dir) / "werks-managed", cme_werks) monkeypatch.setattr(cmk.utils.werks, "_compiled_werks_dir", lambda: Path(tmp_dir)) werks_loaded = cmk.utils.werks.load() merged_werks = cre_werks merged_werks.update(cee_werks) merged_werks.update(cme_werks) assert merged_werks == werks_loaded
def test_find_debug_code(path): scanned = 0 for dirpath, _, filenames in os.walk(path): scanned += 1 for filename in filenames: file_path = "%s/%s" % (dirpath, filename) if [folder for folder in exclude_folders if folder in file_path]: continue if file_path.endswith((".pyc", ".whl", ".tar.gz", ".swp")): continue if os.path.relpath(file_path, cmk_path()) in exclude_files: continue LOGGER.info("Checking file %s", file_path) try: with open(file_path) as file: for nr, line in enumerate(file.readlines()): if nr == 0 and ("bash" in line or "php" in line): break # skip non python files assert not find_debugs(line), ("Found \"print(...)\" call in %s:%d" % (file_path, nr + 1)) except UnicodeDecodeError: LOGGER.warning("Could not read %r due to UnicodeDecodeError", file_path) assert scanned > 0
def _get_python_plugins(): return [ "agents/plugins/%s" % p.name for p in Path(testlib.cmk_path(), "agents", "plugins").iterdir() if testlib.pylint_cmk.is_python_file(str(p), "python") or testlib.pylint_cmk.is_python_file(str(p), "python3") ]
def test_find_debug_code(path): scanned = 0 for dirpath, _, filenames in os.walk(path): scanned += 1 for filename in filenames: file_path = "%s/%s" % (dirpath, filename) if [folder for folder in exclude_folders if folder in file_path]: continue if file_path.endswith((".pyc", ".whl", ".tar.gz")): continue if os.path.relpath(file_path, cmk_path()) in exclude_files: continue LOGGER.info("Checking file %s", file_path) for nr, line in enumerate(open(file_path)): if nr == 0 and ("bash" in line or "php" in line): break # skip non python files assert not find_debugs(line), "Found \"print(...)\" call in %s:%d" % (file_path, nr + 1) assert scanned > 0
def _git_tag_exists(tag): return subprocess.Popen( ["git", "rev-list", tag], stdout=open(os.devnull, "w"), stderr=subprocess.STDOUT, cwd=testlib.cmk_path(), ).wait() == 0
def test_write_precompiled_werks(tmpdir, site, monkeypatch): tmp_dir = "%s" % tmpdir all_werks = cmk.werks.load_raw_files( os.path.join(testlib.cmk_path(), ".werks")) cmk_werks = dict([(w["id"], w) for w in all_werks.values() if w["edition"] == "cre"]) cmc_werks = dict([(w["id"], w) for w in all_werks.values() if w["edition"] == "cee"]) assert len(cmk_werks) > 1000 assert [w for w in cmk_werks.keys() if w >= 9000] == [] cmk.werks.write_precompiled_werks(os.path.join(tmp_dir, "werks"), cmk_werks) if site.version.edition() == "raw": assert len(cmc_werks) > 1000 assert [w for w in cmc_werks.keys() if w < 8000] == [] cmk.werks.write_precompiled_werks( os.path.join(tmp_dir, "werks-enterprise"), cmc_werks) monkeypatch.setattr(cmk.werks, "_compiled_werks_dir", lambda: tmp_dir) werks_loaded = cmk.werks.load() merged_werks = cmk_werks if site.version.edition() == "raw": merged_werks.update(cmc_werks) assert merged_werks == werks_loaded
def run_all_generated_tests(file_ending = ".testgen"): genpath = cmk_path() + "/tests/web/unittest_generation/" onlyfiles = [f for f in listdir(genpath) if f.endswith(".testgen") and isfile(join(genpath, f))] for test_name in onlyfiles: tests = load_html_test(test_name.rstrip(file_ending), test_files_dir = "%s/tests/web/unittest_files" % cmk_path()) for test in tests: test.run()
def test_all_agents_tested(): agents = { os.path.basename(os.path.splitext(agent_file)[0]) for agent_file in glob("%s/cmk/special_agents/agent_*.py" % cmk_path()) } untested = agents - set(REQUIRED_ARGUMENTS) - NOT_TESTED_YET assert not untested, "Please add test cases for special agents %s" % untested
def _check_manpage(name): found_catalog = False for line in open("%s/checkman/%s" % (cmk_path(), name)): if line.startswith("catalog: "): found_catalog = True assert found_catalog, "Did not find \"catalog:\" header in manpage \"%s\"" % name
def test_permissions(): for pattern, check_func, excludes in permissions: for f in glob.glob("%s/%s" % (cmk_path(), pattern)): if f.split('/')[-1] in excludes: continue assert check_func(f), "%s has wrong permissions (%r)" % \ (f, check_func)
def test_find_debug_code(): scanned = 0 for base_path in [ cmk_path(), cmc_path() ]: for dir_path in check_paths: path = "%s/%s" % (base_path, dir_path) if not os.path.exists(path): continue for dirpath, dirnames, filenames in os.walk(path): scanned += 1 for filename in filenames: file_path = "%s/%s" % (dirpath, filename) for nr, line in enumerate(open(file_path)): if nr == 0 and ("bash" in line or "php" in line): break # skip non python files l = line.lstrip() assert not l.startswith("print("), \ "Found \"print(...)\" call in %s:%d" % \ (file_path, nr+1) assert not l.startswith("print "), \ "Found \"print ...\" call in %s:%d" % \ (file_path, nr+1) assert scanned > 0
def load_html_test(test_name, test_files_dir = "%s/tests/web/unittest_files" % cmk_path()): try: with open("%s/%s.unittest" % (test_files_dir.rstrip('/'), test_name), "r") as tfile: tests = ast.literal_eval(tfile.read()) except Exception, e: print tools.bcolors.WARNING + "\nERROR: No test file for test '%s'.\n" % test_name \ + "Generate a test file first, e.g. by calling 'py.test -sv -m html_gentest -k test_generate_integration'.\n" raise e
def test_precompiled_file_ages(): newest_source_file, newest_source_time = find_newest_source_file() for filename in precompiled_files: path = "%s/agents/windows/%s" % (cmk_path(), filename) commit_time = last_commit_time(path) assert commit_time >= newest_source_time, \ "%s is older than source code file %s" % (path, newest_source_file)
def add_file(f, path): relpath = os.path.relpath(os.path.realpath(path), cmk_path()) f.write("# -*- encoding: utf-8 -*-") f.write("#\n") f.write("# ORIG-FILE: " + relpath + "\n") f.write("#\n") f.write("\n") f.write(open(path).read())
def module_files(): modules = [] for base_path in [cmk_path() + "/modules", cmc_path() + "/modules"]: modules += [ base_path + "/" + f for f in os.listdir(base_path) if not f.startswith(".") ] return sorted(modules)
def mk_mongodb(request): agent_path = os.path.abspath(os.path.join(cmk_path(), 'agents', 'plugins', 'mk_mongodb')) for action in ('setup', 'teardown'): try: os.remove(agent_path + "c") except OSError: pass if action == 'setup': yield imp.load_source("mk_mongodb", agent_path)
def get_plugin_files(plugin_dir): files = [] for path in [ cmk_path() + "/web/plugins/" + plugin_dir, cmc_path() + "/web/plugins/" + plugin_dir ]: if os.path.exists(path): files += [ (f, path) for f in os.listdir(path) ] return sorted(files)
def test_permissions() -> None: for pattern, check_func, excludes in _PERMISSIONS: git_dir = Path(cmk_path()) for f in git_dir.glob(pattern): if not f.is_file(): continue if f.name in excludes or f.name in _GLOBAL_EXCLUDES: continue assert check_func( f), "%s has wrong permissions (%r)" % (f, check_func)
def save_html_test(test_name, test, test_files_dir = "%s/tests/web/unittest_files" % cmk_path()): if not isinstance(test, list): test = [test] assert all(isinstance(t, HtmlTest) for t in test), test if not os.path.exists(test_files_dir): os.makedirs(test_files_dir) file_location = "%s/%s.unittest" % (test_files_dir.rstrip('/'), test_name) with open(file_location, "w+") as tfile: tfile.write(pprint.pformat([t.to_dict() for t in test])) return file_location
def get_plugin_files(plugin_dir): files = [] for path in [ cmk_path() + "/web/plugins/" + plugin_dir, cmc_path() + "/web/plugins/" + plugin_dir ]: if os.path.exists(path): files += [(f, path) for f in os.listdir(path)] return sorted(files)
def get_web_plugin_dirs(): plugin_dirs = sorted(list(set(os.listdir(cmk_path() + "/web/plugins") + os.listdir(cmc_path() + "/web/plugins")))) # icons are included from a plugin of views module. Move to the end to # make them be imported after the views plugins. Same for perfometers. plugin_dirs.remove("icons") plugin_dirs.append("icons") plugin_dirs.remove("perfometer") plugin_dirs.append("perfometer") return plugin_dirs
def test_agent_plugin_syntax_compatibility(python_container, plugin_path, python_version): _exit_code, output = python_container.exec_run( [ "python%s" % python_version, "%s/%s" % (testlib.cmk_path(), plugin_path) ], workdir="/cmk", ) assert "SyntaxError: " not in output
def test_exclude_section_options(): actual = sorted( skip_section for skip_section, _section_title in agent_config_mk_agent_sections()) agent_skip_functions = os.popen( # nosec "grep \"\\$MK_SKIP_[A-Z]*[_A-Z]*\" %s/agents/check_mk_agent.linux -o" % cmk_path()).read() expected = sorted( skip_section.replace("$MK_SKIP_", "").lower() for skip_section in agent_skip_functions.split("\n") if "$MK_SKIP_" in skip_section) assert expected == actual
def load_gentest_file(test_name, cmk_version = "running"): assert test_name, "Specify a test file using the '--testfile $name' option!" genpath = cmk_path() + "/tests/web/unittest_generation/" ending = ".testgen" filename = "%s/%s.%s" % (genpath.rstrip('/'), test_name, ending.lstrip('.')) try: with open(filename, "r") as tfile: test = ast.literal_eval(tfile.read()) except Exception, e: print tools.bcolors.WARNING + "\nERROR: No gentest file for test '%s'.\n" % test_name \ + "Generate a test file first (see README for more details!)\n" raise e
def read_dataset(filename): """ reads pre-recorded mongodb server output ('serverStatus') from dataset directory. the dataset is in extended JSON format. (https://docs.mongodb.com/manual/reference/mongodb-extended-json/). :param filename: filename of the dataset :return: dataset as extended JSON """ dataset_file = os.path.abspath( os.path.join(cmk_path(), 'tests', 'unit', 'agents', 'plugins', 'datasets', 'mk_mongodb', filename)) with open(dataset_file) as f: return loads(f.read())
def test_all_man_pages(tmp_path): (tmp_path / ".asd").write_text(u"", encoding="utf-8") (tmp_path / "asd~").write_text(u"", encoding="utf-8") (tmp_path / "if").write_text(u"", encoding="utf-8") pages = man_pages.all_man_pages() assert len(pages) > 1241 assert ".asd" not in pages assert "asd~" not in pages assert pages["if"] == str(tmp_path / "if") assert pages["if64"] == "%s/checkman/if64" % cmk_path()
def test_all_man_pages(tmpdir): f1 = tmpdir.join(".asd").write("") f2 = tmpdir.join("asd~").write("") f3 = tmpdir.join("if").write("") pages = man_pages.all_man_pages() assert len(pages) > 1241 assert ".asd" not in pages assert "asd~" not in pages assert pages["if"] == "%s/if" % tmpdir assert pages["if64"] == "%s/checkman/if64" % cmk_path()
def bakelet(request): """ Fixture to inject bakelet as module """ path = os.path.join(cmk_path(), 'enterprise', 'agents', 'bakery', 'mk_logwatch') with open(path, "r") as handle: source = handle.read() extended_source = 'from cmk_base.cee.agent_bakery_plugins import bakery_info' + '\n' + source bakelet = imp.new_module('bakelet') exec extended_source in bakelet.__dict__ yield bakelet
def test_all_man_pages(tmpdir): tmpdir.join(".asd").write("") tmpdir.join("asd~").write("") tmpdir.join("if").write("") pages = man_pages.all_man_pages() assert len(pages) > 1241 assert ".asd" not in pages assert "asd~" not in pages assert pages["if"] == "%s/if" % tmpdir assert pages["if64"] == "%s/checkman/if64" % cmk_path()
def test_find_debug_code(): scanned = 0 for base_path in [ cmk_path(), cmc_path() ]: for dirpath, dirnames, filenames in os.walk("%s/web" % base_path): scanned += 1 for filename in filenames: path = "%s/%s" % (dirpath, filename) for line in open(path): l = line.lstrip() assert not l.startswith("html.debug("), \ "Found \"html.debug(...)\" call" assert scanned > 0
def find_newest_source_file(): newest_path, newest_commit_time = None, 0 for dirpath, dirnames, filenames in os.walk("%s/agents/windows" % cmk_path()): for filename in filenames: path = "%s/%s" % (dirpath, filename) if not filename.endswith(".cc") and not filename.endswith(".h"): continue if "/msibuild/" in path: continue commit_time = last_commit_time(path) if commit_time > newest_commit_time: newest_path, newest_commit_time = path, commit_time assert newest_path != None assert newest_commit_time != 0 return newest_path, newest_commit_time
def test_pylint_web(): base_path = pylint_cmk.get_test_dir() # Make compiled files import eachother by default sys.path.insert(0, base_path) modules = glob.glob(cmk_path() + "/web/htdocs/*.py") \ + glob.glob(cmc_path() + "/web/htdocs/*.py") for module in modules: print("Copy %s to test directory" % module) f = open(base_path + "/" + os.path.basename(module), "w") pylint_cmk.add_file(f, module) f.close() # Move the whole plugins code to their modules, then # run pylint only on the modules for plugin_dir in get_web_plugin_dirs(): files = get_plugin_files(plugin_dir) for plugin_file, plugin_base in files: plugin_path = plugin_base +"/"+plugin_file if plugin_file.startswith('.'): continue elif plugin_dir in ["icons","perfometer"]: module_name = "views" elif plugin_dir == "pages": module_name = "modules" else: module_name = plugin_dir print("[%s] add %s" % (module_name, plugin_path)) module = file(base_path + "/" + module_name + ".py", "a") pylint_cmk.add_file(module, plugin_path) module.close() exit_code = pylint_cmk.run_pylint(base_path, cleanup_test_dir=True) assert exit_code == 0, "PyLint found an error in the web code"
def test_man_page_path_only_shipped(): assert man_pages.man_page_path("if64") == "%s/checkman/if64" % cmk_path() assert man_pages.man_page_path("not_existant") == None
#!/usr/bin/python # call using # > py.test -s -k test_html_generator.py # enable imports from web directory from testlib import cmk_path import sys sys.path.insert(0, "%s/web/htdocs" % cmk_path()) # external imports import re # internal imports from htmllib import html from htmllib import HTMLGenerator, HTMLCheck_MK # .--Deprecated Renderer-------------------------------------------------. # | ____ _ _ | # | | _ \ ___ _ __ _ __ ___ ___ __ _| |_ ___ __| | | # | | | | |/ _ \ '_ \| '__/ _ \/ __/ _` | __/ _ \/ _` | | # | | |_| | __/ |_) | | | __/ (_| (_| | || __/ (_| | | # | |____/ \___| .__/|_| \___|\___\__,_|\__\___|\__,_| | # | |_| | # | ____ _ | # | | _ \ ___ _ __ __| | ___ _ __ ___ _ __ | # | | |_) / _ \ '_ \ / _` |/ _ \ '__/ _ \ '__| | # | | _ < __/ | | | (_| | __/ | | __/ | | # | |_| \_\___|_| |_|\__,_|\___|_| \___|_| | # | | # +----------------------------------------------------------------------+ # | Rendering methods which are to be replaced by the new HTMLGenerator. |
def patch_cmk_paths(monkeypatch, tmpdir): monkeypatch.setattr("cmk.paths.check_manpages_dir", "%s/checkman" % cmk_path()) monkeypatch.setattr("cmk.paths.local_check_manpages_dir", "%s" % tmpdir)
def test_precompiled_files_present(): for filename in precompiled_files: path = "%s/agents/windows/%s" % (cmk_path(), filename) assert os.path.exists(path)