def __init__(self, num, crawler):
super(Worker, self).__init__()
self.name = "worker-%d" % num
self.crawler = crawler
self.daemon = True
self.terminate = False
self.idle = True
self.client = CMKWebSession(self.crawler.site)
self.client.login()
self.client.set_language("en")
def __init__(self, num, crawler):
super(Worker, self).__init__()
self.name = "worker-%d" % num
self.crawler = crawler
self.daemon = True
self.terminate = False
self.idle = True
self.client = CMKWebSession(self.crawler.site)
self.client.login()
self.client.enforce_non_localized_gui()
def init_wato(self):
if not self._missing_but_required_wato_files():
logger.info("WATO is already initialized -> Skipping initializiation")
return
logger.debug("Initializing WATO...")
web = CMKWebSession(self)
web.login()
web.set_language("en")
# Call WATO once for creating the default WATO configuration
logger.debug("Requesting wato.py (which creates the WATO factory settings)...")
response = web.get("wato.py?mode=sites").text
#logger.debug("Debug: %r" % response)
assert "<title>Distributed Monitoring</title>" in response
assert "replication_status_%s" % web.site.id in response, \
"WATO does not seem to be initialized: %r" % response
logger.debug("Waiting for WATO files to be created...")
wait_time = 20.0
while self._missing_but_required_wato_files() and wait_time >= 0:
time.sleep(0.5)
wait_time -= 0.5
missing_files = self._missing_but_required_wato_files()
assert not missing_files, \
"Failed to initialize WATO data structures " \
"(Still missing: %s)" % missing_files
self._add_wato_test_config(web)
def init_wato(self):
if not self._missing_but_required_wato_files():
print("WATO is already initialized -> Skipping initializiation")
return
web = CMKWebSession(self)
web.login()
web.set_language("en")
# Call WATO once for creating the default WATO configuration
response = web.get("wato.py").text
assert "<title>WATO" in response
assert "<div class=\"title\">Manual Checks</div>" in response, \
"WATO does not seem to be initialized: %r" % response
wait_time = 20
while self._missing_but_required_wato_files() and wait_time >= 0:
time.sleep(0.5)
wait_time -= 0.5
missing_files = self._missing_but_required_wato_files()
assert not missing_files, \
"Failed to initialize WATO data structures " \
"(Still missing: %s)" % missing_files
self._add_wato_test_config(web)
def init_wato(self):
if not self._missing_but_required_wato_files():
logger.info(
"WATO is already initialized -> Skipping initializiation")
return
logger.debug("Initializing WATO...")
web = CMKWebSession(self)
web.login()
# Call WATO once for creating the default WATO configuration
logger.debug(
"Requesting wato.py (which creates the WATO factory settings)...")
response = web.get("wato.py?mode=sites").text
#logger.debug("Debug: %r" % response)
assert "site=%s" % web.site.id in response
logger.debug("Waiting for WATO files to be created...")
wait_time = 20.0
while self._missing_but_required_wato_files() and wait_time >= 0:
time.sleep(0.5)
wait_time -= 0.5
missing_files = self._missing_but_required_wato_files()
assert not missing_files, \
"Failed to initialize WATO data structures " \
"(Still missing: %s)" % missing_files
web.enforce_non_localized_gui()
self._add_wato_test_config(web)
def _new_site(self, name):
site = self._site_obj(name)
site.create()
site.open_livestatus_tcp(encrypted=False)
site.start()
site.prepare_for_tests()
# There seem to be still some changes that want to be activated
CMKWebSession(site).activate_changes()
logger.debug("Created site %s", site.id)
self._sites[site.id] = site
return site
def _new_site(self, name):
site_id = "%s%s" % (self._base_ident, name)
site = Site(site_id=site_id,
reuse=False,
version=self._version,
edition=self._edition,
branch=self._branch)
site.create()
site.open_livestatus_tcp()
# No TLS for testing
site.set_config("LIVESTATUS_TCP_TLS", "off")
site.start()
site.prepare_for_tests()
# There seem to be still some changes that want to be activated
CMKWebSession(site).activate_changes()
print("Created site %s" % site_id)
self._sites[site_id] = site
return site
def __init__(self, site):
super(CMKEventConsole, self).__init__()
self.site = site
self.status = CMKEventConsoleStatus("%s/tmp/run/mkeventd/status" %
site.root)
self.web_session = CMKWebSession(site)
class CMKEventConsole(object): # pylint: disable=useless-object-inheritance
def __init__(self, site):
super(CMKEventConsole, self).__init__()
self.site = site
self.status = CMKEventConsoleStatus("%s/tmp/run/mkeventd/status" %
site.root)
self.web_session = CMKWebSession(site)
def _config(self):
cfg = {} # type: Dict[str, Any]
content = self.site.read_file("etc/check_mk/mkeventd.d/wato/global.mk")
exec(content, {}, cfg)
return cfg
def _gather_status_port(self):
config = self._config()
if self.site.reuse and self.site.exists(
) and "remote_status" in config:
port = config["remote_status"][0]
else:
port = self.site.get_free_port_from(self.site.livestatus_port + 1)
self.status_port = port
def enable_remote_status_port(self, web):
html = web.get("wato.py?mode=mkeventd_config").text
assert "mode=mkeventd_edit_configvar&site=&varname=remote_status" in html
html = web.get(
"wato.py?folder=&mode=mkeventd_edit_configvar&site=&varname=remote_status"
).text
assert "Save" in html
html = web.post("wato.py",
data={
"filled_in": "value_editor",
"ve_use": "on",
"ve_value_0": self.status_port,
"ve_value_2_use": "on",
"ve_value_2_value_0": "127.0.0.1",
"save": "Save",
"varname": "remote_status",
"mode": "mkeventd_edit_configvar",
},
add_transid=True).text
assert "%d, no commands, 127.0.0.1" % self.status_port in html
def activate_changes(self, web):
old_t = web.site.live.query_value(
"GET eventconsolestatus\nColumns: status_config_load_time\n")
assert old_t > time.time() - 86400
self.web_session.activate_changes(allow_foreign_changes=True)
def config_reloaded():
new_t = web.site.live.query_value(
"GET eventconsolestatus\nColumns: status_config_load_time\n")
return new_t > old_t
reload_time, timeout = time.time(), 10
while not config_reloaded():
if time.time() > reload_time + timeout:
raise Exception("Config did not update within %d seconds" %
timeout)
time.sleep(0.2)
assert config_reloaded()
@classmethod
def new_event(cls, attrs):
now = time.time()
default_event = {
"rule_id": 815,
"text": "",
"phase": "open",
"count": 1,
"time": now,
"first": now,
"last": now,
"comment": "",
"host": "test-host",
"ipaddress": "127.0.0.1",
"application": "",
"pid": 0,
"priority": 3,
"facility": 1, # user
"match_groups": (),
}
event = default_event.copy()
event.update(attrs)
return event
class Worker(threading.Thread):
def __init__(self, num, crawler):
super(Worker, self).__init__()
self.name = "worker-%d" % num
self.crawler = crawler
self.daemon = True
self.terminate = False
self.idle = True
self.client = CMKWebSession(self.crawler.site)
self.client.login()
self.client.set_language("en")
def run(self):
while not self.terminate:
try:
while not self.terminate:
url = self.crawler.todo.get(block=False)
self.idle = False
try:
self.visit_url(url)
except Exception as e:
self.error(
url, "Failed to visit: %s\n%s" %
(e, traceback.format_exc()))
self.crawler.todo.task_done()
except queue.Empty:
self.idle = True
time.sleep(0.5)
def stop(self):
self.terminate = True
def visit_url(self, url):
if url.url in self.crawler.visited:
logger.info("Already visited: %s", url.url)
return
self.crawler.visited.append(url.url)
#print("%s - Visiting #%d (todo %d): %s" %
# (self.name, len(self.crawler.visited), self.crawler.todo.qsize(), url.url))
started = time.time()
try:
#print "FETCH", url.url_without_host()
response = self.client.get(url.url_without_host())
except AssertionError as e:
if "This view can only be used in mobile mode" in "%s" % e:
logger.info("Skipping mobile mode view checking")
return
raise
duration = time.time() - started
self.update_stats(url, duration, len(response.content))
content_type = response.headers.get('content-type')
#print self.name, content_type, len(response.text)
if content_type.startswith("text/html"):
self.check_response(url, response)
elif content_type.startswith("text/plain"):
pass # no specific test
elif content_type.startswith("text/csv"):
pass # no specific test
elif content_type in ["image/png", "image/gif"]:
pass # no specific test
elif content_type in ["application/pdf"]:
pass # no specific test
elif content_type in [
"application/x-rpm",
"application/x-deb",
"application/x-debian-package",
"application/x-gzip",
"application/x-msdos-program",
"application/x-msi",
"application/x-tgz",
"application/x-redhat-package-manager",
"application/x-pkg",
"application/x-tar",
"application/json",
"text/x-chdr",
"text/x-c++src",
"text/x-sh",
]:
pass # no specific test
else:
self.error(url, "Unknown content type: %s" % (content_type))
return
def update_stats(self, url, duration, content_size):
stats = self.crawler.stats.setdefault(
url.neutral_url(), {
"first_duration": duration,
"first_content_size": content_size,
})
avg_duration = (duration + stats.get("avg_duration", duration)) / 2.0
avg_content_size = (content_size +
stats.get("avg_content_size", content_size)) / 2.0
stats.update({
"orig_url": url.orig_url,
"referer_url": url.referer_url,
"num_visited": stats.get("num_visited", 0) + 1,
"last_duration": duration,
"last_content_size": content_size,
"avg_duration": avg_duration,
"avg_content_size": avg_content_size,
})
def error(self, url, s):
s = "[%s - found on %s] %s" % (url.url, url.referer_url, s)
self.crawler.error(s)
def check_response(self, url, response):
soup = BeautifulSoup(response.text, "lxml")
# The referenced resources (images, stylesheets, javascript files) are checked by
# the generic web client handler. This only needs to reaslize the crawling.
self.check_content(url, response, soup)
self.check_links(url, soup)
self.check_frames(url, soup)
self.check_iframes(url, soup)
def check_content(self, url, response, soup):
ignore_texts = [
"This view can only be used in mobile mode.",
# Some single context views are accessed without their context information, which
# results in a helpful error message since 1.7. These are not failures that this test
# should report.
"Missing context information",
# Same for dashlets that are related to a specific context
"There are no metrics meeting your context filters",
]
for element in soup.select("div.error"):
inner_html = "%s" % element
skip = False
for ignore_text in ignore_texts:
if ignore_text in inner_html:
skip = True
break
if not skip:
self.error(url, "Found error: %s" % (element))
def check_frames(self, url, soup):
self.check_referenced(url, soup, "frame", "src")
def check_iframes(self, url, soup):
self.check_referenced(url, soup, "iframe", "src")
def check_links(self, url, soup):
self.check_referenced(url, soup, "a", "href")
def check_referenced(self, referer_url, soup, tag, attr):
elements = soup.find_all(tag)
for element in elements:
orig_url = element.get(attr)
if orig_url is None:
continue # Skip elements that don't have the attribute in question
url = self.normalize_url(self.crawler.site.internal_url, orig_url)
if url is None:
continue
try:
self.verify_is_valid_url(url)
except InvalidUrl:
#print self.name, "skip invalid", url, e
self.crawler.skipped.add(url)
continue
# Ensure that this url has not been crawled yet
crawl_it = False
with self.crawler.handled_lock:
if url not in self.crawler.handled:
crawl_it = True
self.crawler.handled.add(url)
if crawl_it:
#open("/tmp/todo", "a").write("%s (%s)\n" % (url, referer_url.url))
self.crawler.todo.put(
Url(url, orig_url=orig_url, referer_url=referer_url.url))
def verify_is_valid_url(self, url):
parsed = urlsplit(url)
if parsed.scheme != "http":
raise InvalidUrl("invalid scheme: %r" % (parsed, ))
# skip external urls
if url.startswith("http://") and not url.startswith(
self.crawler.site.internal_url):
raise InvalidUrl("Skipping external URL: %s" % url)
# skip non check_mk urls
if not parsed.path.startswith("/%s/check_mk" % self.crawler.site.id) \
or "../pnp4nagios/" in parsed.path \
or "../nagvis/" in parsed.path \
or "check_mk/plugin-api" in parsed.path \
or "../nagios/" in parsed.path:
raise InvalidUrl("Skipping non Check_MK URL: %s %s" %
(url, parsed))
# skip current url with link to index
if "index.py?start_url=" in url:
raise InvalidUrl("Skipping link to index with current URL: %s" %
url)
if "logout.py" in url:
raise InvalidUrl("Skipping logout URL: %s" % url)
if "_transid=" in url:
raise InvalidUrl("Skipping action URL: %s" % url)
if "selection=" in url:
raise InvalidUrl("Skipping selection URL: %s" % url)
# TODO: Remove this exclude when ModeCheckManPage works without an
# automation call. Currently we have to use such a call to enrich the
# man page with some additional info from config.check_info, see
# AutomationGetCheckManPage.
if "mode=check_manpage" in url and "wato.py" in url:
raise InvalidUrl("Skipping man page URL: %s" % url)
# Don't follow filled in filter form views
if "view.py" in url and "filled_in=filter" in url:
raise InvalidUrl("Skipping filled in filter URL: %s" % url)
# Don't follow the view editor
if "edit_view.py" in url:
raise InvalidUrl("Skipping view editor URL: %s" % url)
# Skip agent download files
if parsed.path.startswith("/%s/check_mk/agents/" %
self.crawler.site.id):
raise InvalidUrl("Skipping agent download file: %s" % url)
def normalize_url(self, base_url, url):
url = urljoin(base_url, url.rstrip("#"))
parsed = list(urlsplit(url))
parsed[3] = urlencode(
sorted(parse_qsl(parsed[3], keep_blank_values=True)))
return urlunsplit(parsed)
def web(site):
web = CMKWebSession(site)
web.login()
web.set_language("en")
return web
def web(site):
web = CMKWebSession(site)
web.login()
web.enforce_non_localized_gui()
return web
