def evaluate_all(self, match=None, ignore="spec"):
"""
Evaluate all components that match.
Args:
match (str, optional): regular expression for matching against
the fully qualified name of components to keep.
ignore (str, optional): regular expression for searching against
the fully qualified name of components to ignore.
"""
match, ignore = self._desugar_match_ignore(match, ignore)
tasks = []
for c in self.values():
name = dr.get_name(c)
if match.test(name) and not ignore.test(name):
if not any([
c in self._broker.instances,
c in self._broker.exceptions,
c in self._broker.missing_requirements,
]):
tasks.append(c)
if not tasks:
return
dr.run(tasks, broker=self._broker)
self.show_timings(match, ignore)
def make_broker(ctx):
broker = dr.Broker()
broker[ctx.__class__] = ctx
dr.run(datasources, broker=broker)
del broker[ctx.__class__]
return broker
def test_syslog_format():
broker = dr.Broker()
output = StringIO()
with SysLogFormat(broker, stream=output):
dr.run(report, broker=broker)
output.seek(0)
data = output.read()
assert SL_MSG in data
def _evaluate(self, broker, path):
ctx = create_context(path)
broker[ctx.__class__] = ctx
output = StringIO()
with self._Format(broker, stream=output):
dr.run(self._target_components, broker=broker)
output.seek(0)
return output.read().encode("utf-8")
def test_json_format():
broker = dr.Broker()
output = StringIO()
with JsonFormat(broker, stream=output):
dr.run(report, broker=broker)
output.seek(0)
data = output.read()
assert "foo" in data
assert "bar" in data
def test_simple_html_format():
broker = dr.Broker()
output = StringIO()
with SimpleHtmlFormat(broker, stream=output):
dr.run(report, broker=broker)
output.seek(0)
data = output.read()
assert "foo" in data
assert "bar" in data
def test_disabled_run():
assert dr.ENABLED[one]
broker = dr.run(dr.COMPONENTS[dr.GROUPS.single])
assert one in broker
dr.set_enabled(one, False)
assert not dr.ENABLED[one]
broker = dr.run(dr.COMPONENTS[dr.GROUPS.single])
assert one not in broker
def test_human_readable():
broker = dr.Broker()
output = StringIO()
with HumanReadableFormat(broker, stream=output):
dr.run(report, broker=broker)
output.seek(0)
data = output.read()
assert "foo" in data
assert "bar" in data
def test_insights_evaluator_attrs_serial():
broker = dr.Broker()
broker[Specs.hostname] = context_wrap("www.example.com")
broker[Specs.machine_id] = context_wrap("12345")
broker[Specs.redhat_release] = context_wrap("Red Hat Enterprise Linux Server release 7.4 (Maipo)")
with InsightsEvaluator(broker) as e:
dr.run(components, broker=broker)
result = e.get_response()
assert result["system"]["hostname"] == "www.example.com"
assert result["system"]["system_id"] == "12345"
assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)"
def test_insights_evaluator():
broker = dr.Broker()
e = InsightsEvaluator(broker)
graph = dr.get_dependency_graph(report)
result1 = e.process(graph)
broker = dr.Broker()
with InsightsEvaluator(broker) as e:
dr.run(report, broker=broker)
result2 = e.get_response()
assert result1 == result2
def test_syslog_format_archive():
broker = dr.Broker()
output = StringIO()
with SysLogFormat(broker, archive="../../insights/core", stream=output):
dr.run(report, broker=broker)
output.seek(0)
data = output.read()
assert SL_MSG in data
assert SL_CMD in data
assert SL_ARCHIVE in data
assert SL_PATH in data
def make_broker(ctx):
broker = dr.Broker()
broker[ctx.__class__] = ctx
if isinstance(ctx, SerializedArchiveContext):
h = Hydration(ctx.root)
broker = h.hydrate(broker=broker)
dr.run(datasources, broker=broker)
del broker[ctx.__class__]
return broker
def test_insights_evaluator():
broker = dr.Broker()
e = InsightsEvaluator(broker)
graph = dr.get_dependency_graph(report)
result1 = e.process(graph)
broker = dr.Broker()
with InsightsEvaluator(broker) as e:
dr.run(report, broker=broker)
result2 = e.get_response()
assert result1["reports"] == result2["reports"]
for k in ["start", "finish", "execution_context", "plugin_sets"]:
assert k in result1["analysis_metadata"]
assert k in result2["analysis_metadata"]
def evaluate(self, name):
"""
Evaluate a component and return its result. Prints diagnostic
information in the case of failure. This function is useful when a
component's name contains characters that aren't valid for python
identifiers so you can't access it with ``models.<name>``.
Args:
name (str): the name of the component as shown by :func:`Models.find()`.
"""
comp = self.get(name) or dr.get_component(name)
if not comp:
return
if not plugins.is_rule(comp):
self._requested.add((name, comp))
if comp in self._broker:
return self._broker[comp]
if comp in self._broker.exceptions or comp in self._broker.missing_requirements:
self._dump_diagnostics(comp)
return
val = dr.run(comp, broker=self._broker).get(comp)
if comp not in self._broker:
if comp in self._broker.exceptions or comp in self._broker.missing_requirements:
self._dump_diagnostics(comp)
else:
print("{} chose to skip.".format(dr.get_name(comp)))
return val
def run(spec, archive=None, quiet=False):
with create_broker(archive) as broker:
value = dr.run(spec, broker=broker).get(spec)
if value:
dump_spec(value, quiet=quiet)
else:
dump_error(spec, broker)
def get_rule_hit_info(archive, rule_name, timeout=None, tmp_dir=None):
# We have to load everything again for multiprocessing or clustering to
# work. Even though imports are cached internally after the first call, we
# can still optimize a bit with this LOADED hack.
rule_func = dr.get_component(rule_name)
if not LOADED[0]:
load_default_plugins()
LOADED[0] = True
# this is also cached behind get_deps after the first call.
graph, bool_deps = get_deps(rule_func)
with extract(archive, timeout=timeout, extract_dir=tmp_dir) as arc:
ctx = create_context(arc.tmp_dir, None)
broker = dr.Broker()
broker[ctx.__class__] = ctx
results = dr.run(graph, broker=broker)
rule_result = results.get(rule_func)
rhr = results.get(RedHatRelease)
result = extract_hits(bool_deps, results)
result["archive"] = archive
result["key"] = rule_result.get_key() if rule_result else None
result["type"] = rule_result.__class__.__name__ if rule_result else None
result["make_fail"] = True if rule_result and isinstance(rule_result, make_response) else False
result["major"] = rhr.major if rhr else -1
result["minor"] = rhr.minor if rhr else -1
return result
def run_stages(graph=dr.COMPONENTS["stage"], broker=None):
"""
Runs the component graph with an optional broker. If any ``Stage``
instances are present in the results, start them and wait until they're all
done.
"""
def handler(signum, frame):
sys.exit(0)
signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGTERM, handler)
broker = dr.run(graph, broker=broker)
if broker.exceptions:
print(broker.tracebacks)
sys.exit(0)
stages = broker.get_by_type(stage)
dependencies = {}
dependents = {}
instances = []
stop_queue = mp.Queue()
for s, v in stages.items():
dependencies[s] = set(d for d in graph[s] if issubclass(d, Stage))
dependents[s] = set(d for d in dr.get_dependents(s)
if issubclass(d, Stage))
for i in v:
i.stop_queue = stop_queue
i.start()
instances.append(i)
done_counter = Counter()
finished = set()
def handle_finished():
try:
done = stop_queue.get(0.5)
except Empty:
return
else:
done_counter[done] += 1
if done_counter[done] == done.num_instances:
finished.add(done)
for d in dependents[done]:
if dependencies[d].issubset(finished):
for i in stages[d]:
i.queue.put(StopProcessing)
if not d.shared_queue:
i.queue.close()
i.queue.join_thread()
if d.shared_queue:
d.queue.close()
d.queue.join_thread()
for i in stages[d]:
i.join()
while any(s.is_alive() for s in instances):
handle_finished()
def test_spec_factory():
hn = HostContext()
broker = dr.Broker()
broker[HostContext] = hn
broker = dr.run(dr.get_dependency_graph(dostuff), broker)
assert dostuff in broker, broker.tracebacks
assert broker[Stuff.smpl_file].content == file_content
assert not any(l.endswith("\n") for l in broker[Stuff.smpl_file].content)
def run(spec, archive=None, quiet=False, no_header=False):
with create_broker(archive) as broker:
value = dr.run(spec, broker=broker).get(spec)
if value:
dump_spec(value, quiet=quiet, no_header=no_header)
else:
dump_error(spec, broker)
return sys.exit(1)
def test_line_terminators():
add_filter(Stuff.smpl_file, "def test")
hn = HostContext()
broker = dr.Broker()
broker[HostContext] = hn
broker = dr.run(dr.get_dependency_graph(dostuff), broker)
content = broker[Stuff.smpl_file].content
assert all("def test" in l for l in content), content
assert not any(l.endswith("\n") for l in content)
def test_spec_factory():
add_filter(Stuff.smpl_cmd_list_of_lists, " hello ")
hn = HostContext()
broker = dr.Broker()
broker[HostContext] = hn
broker = dr.run(dr.get_dependency_graph(dostuff), broker)
assert dostuff in broker, broker.tracebacks
assert broker[Stuff.smpl_file].content == file_content
assert not any(l.endswith("\n") for l in broker[Stuff.smpl_file].content)
assert "hello" in broker[Stuff.smpl_cmd_list_of_lists].content[0]
assert len(broker[Stuff.smpl_cmd_list_of_lists].content) == 1
def insights_call(data_str):
decoded_str = base64.b64decode(data_str)
data_dict = json.loads(decoded_str)
try:
if "INSIGHTS_RULES" in os.environ:
rules_list = os.environ["INSIGHTS_RULES"].split(":")
for rule_path in rules_list:
if len(rule_path) == 0:
continue
if rule_path.endswith("/"):
rule_path = rule_path[:-1]
if rule_path not in sys.path:
sys.path.append(rule_path)
dr.load_components(os.path.basename(rule_path))
except Exception as e:
return str(e)
try:
dr.load_components("insights.specs.default")
broker = dr.Broker()
broker[Specs.hostname] = context_wrap(data_dict["hostname"])
broker[Specs.uname] = context_wrap(data_dict["uname"])
broker[Specs.dmesg] = context_wrap(data_dict["dmesg"])
broker[Specs.messages] = context_wrap(data_dict["dmesg"])
broker[Specs.sysctl] = context_wrap(data_dict["sysctl"])
broker[Specs.meminfo] = context_wrap(data_dict["meminfo"])
broker[Specs.ps_aux] = context_wrap(data_dict["ps_aux"])
broker[Specs.ps_auxcww] = context_wrap(data_dict["ps_auxcww"])
broker[Specs.ps_auxww] = context_wrap(data_dict["ps_auxww"])
broker[Specs.ps_ef] = context_wrap(data_dict["ps_ef"])
output = six.StringIO()
with Formatter(broker, stream=output):
dr.run(broker=broker)
output.seek(0)
data = output.read()
return data
except Exception as e:
return str(e)
def run(component, archive=None):
with create_broker(archive) as broker:
value = dr.run(component, broker=broker).get(component)
name = dr.get_name(component).rsplit(".", 1)[1]
vars()[name] = value
if value:
print(C.Fore.BLUE + "\nIPython Console Usage Info:\n" + C.Style.RESET_ALL, file=sys.stderr)
print("Enter '{}.' and tab to get a list of properties ".format(name))
print("Example:")
print("In [1]: {}.<property_name>".format(name))
print("Out[1]: <property value>\n")
print("To exit ipython enter 'exit' and hit enter or use 'CTL D'\n\n")
embed()
else:
dump_error(component, broker)
return sys.exit(1)
def run_streams(graph=dr.COMPONENTS["stream"], broker=None):
"""
Runs the component graph with an optional broker. If any ``Stream``
instances are present in the results, start them and wait until they're all
done.
"""
def handler(signum, frame):
sys.exit(0)
signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGTERM, handler)
broker = dr.run(graph, broker=broker)
streams = broker.get_by_type(stream).values()
for s in streams:
s.start()
# hang out while anything is alive so we can catch signals
while any(s.is_alive() for s in streams):
time.sleep(.5)
def invoke(self, broker):
result = super(prerequisite, self).invoke(broker)
if result is None:
raise dr.SkipComponent()
return result
@needs()
def one():
return 1
@needs()
def two():
return 2
@prerequisite(one, two)
def three(a, b):
pass # implied return None
# `report` will not attempt to execute since `three` returns `None`.
@needs(one, two, three)
def report(a, b, c):
return a + b + c
if __name__ == "__main__":
print(dr.run().instances)
[<ffffffff812a4641>] ? radix_tree_gang_lookup_slot+0x81/0xf0
[<ffffffff81130c0b>] ? find_get_pages+0x3b/0x150
[<ffffffff81147432>] ? pagevec_lookup+0x22/0x30
[<ffffffff81148f44>] ? invalidate_mapping_pages+0x84/0x1e0
[<ffffffff811cc2ee>] ? drop_caches_sysctl_handler+0x12e/0x1d0
[<ffffffff81213e9c>] ? proc_sys_call_handler+0x9c/0xd0
[<ffffffff81213ee4>] ? proc_sys_write+0x14/0x20
[<ffffffff8119cb5a>] ? vfs_write+0xba/0x1a0
[<ffffffff8119e056>] ? fget_light_pos+0x16/0x50
[<ffffffff8119d691>] ? sys_write+0x51/0xb0
[<ffffffffa03eea23>] ? symev_write+0x53/0xa0 [symev_rh_ES_6_2_6_32_431_el6_x86_64]
[<ffffffff8155e351>] ? system_call_fastpath+0x2f/0x34
""".strip()
dr.load_components("rules")
dr.load_components("insights.specs.default")
# Below 3 components are not working as it's based on python2
#dr.load_components("telemetry")
#dr.load_components("diag_insights_rules")
#dr.load_components("prodsec")
dr.load_components("support-rules")
broker = dr.Broker()
broker[Specs.hostname] = context_wrap("www.example.com")
#broker[Specs.kernel] = context_wrap("2.6.32-696.23.1.el6.x86_64")
broker[Specs.uname] = context_wrap(
"Linux a03a5df6f247 2.6.32-696.23.1.el6.x86_64 #1 SMP Wed Jun 6 16:55:56 UTC 2018 x86_64 GNU/Linux"
)
broker[Specs.dmesg] = context_wrap(MSGINFO)
with Formatter(broker):
dr.run(broker=broker)
def test_continue_on_error():
Lax.count = 0
broker = dr.run(Lax)
assert len(broker[Lax]) == 4
def test_find():
broker = dr.run(report)
results = broker[report]
assert "num_all_foos" in results
assert "num_direct_foos" in results
def test_dont_continue_on_error():
Boom.count = 0
broker = dr.run(Boom)
assert Boom not in broker
#!/usr/bin/env python36
import sys
import yaml
import json
from insights import dr
from insights.specs.openshift import OpenshiftContext
from insights.specs.openshift.default import OpenshiftSpecs
broker = dr.Broker()
ctx = OpenshiftContext()
broker[OpenshiftContext] = ctx
b = dr.run(OpenshiftSpecs.openshift_namespaces, broker)
doc = yaml.safe_load(b[OpenshiftSpecs.openshift_namespaces].content)
fact_dict = {
"insights_id":
next(o["metadata"]["uid"] for o in doc["items"]
if o["metadata"]["name"] == "kube-system")
}
print(json.dumps(fact_dict))