def inner(func):
name = dr.get_name(_type)
if name in SERIALIZERS:
msg = "%s already has a serializer registered: %s"
raise Exception(msg % (name, dr.get_name(SERIALIZERS[name])))
SERIALIZERS[name] = func
return func
def inner(func):
name = dr.get_name(_type)
if name in DESERIALIZERS:
msg = "%s already has a deserializer registered: %s"
raise Exception(msg % (dr.get_name(name), dr.get_name(DESERIALIZERS[name])))
DESERIALIZERS[name] = (_type, func)
return func
def component_info(component):
dependents = dr.get_dependents(component)
dependencies = dr.get_dependencies(component)
return {
'name': dr.get_name(component),
'dependents': [dr.get_name(c) for c in dependents],
'dependencies': [dr.get_name(c) for c in dependencies]
}
def rule_executor(component, broker, requires, optional, executor=dr.default_executor):
try:
r = executor(component, broker, requires, optional)
if r is None:
name = dr.get_name(component)
log.debug("Rule %s returned None" % name)
raise dr.SkipComponent()
except dr.MissingRequirements as mr:
details = dr.stringify_requirements(mr.requirements)
r = make_skip(dr.get_name(component),
reason="MISSING_REQUIREMENTS", details=details)
validate_response(r)
return r
def get_info(spec):
return {
"name": dr.get_simple_name(spec),
"description": get_description(spec),
"rules": ul(sorted(dr.get_name(r) for r in get_rules(spec))),
"filters": ul(sorted(filters.get_filters(spec))),
}
def add_filter(ds, patterns):
"""
Add a filter or list of filters to a datasource. A filter is a simple
string, and it matches if it is contained anywhere within a line.
Args:
ds (@datasource component): The datasource to filter
patterns (str, [str]): A string, list of strings, or set of strings to
add to the datasource's filters.
"""
if not plugins.is_datasource(ds):
raise Exception("Filters are applicable only to datasources.")
delegate = dr.get_delegate(ds)
if delegate.raw:
raise Exception("Filters aren't applicable to raw datasources.")
if not delegate.filterable:
raise Exception("Filters aren't applicable to %s." % dr.get_name(ds))
if ds in _CACHE:
del _CACHE[ds]
if isinstance(patterns, six.string_types):
FILTERS[ds].add(patterns)
elif isinstance(patterns, list):
FILTERS[ds] |= set(patterns)
elif isinstance(patterns, set):
FILTERS[ds] |= patterns
else:
raise TypeError("patterns must be string, list, or set.")
def invoke(self, broker):
dependency = self.requires[0]
if dependency not in broker:
raise dr.MissingRequirements(([dependency], []))
dep_value = broker[dependency]
if not isinstance(dep_value, list):
return self.component(dep_value)
results = []
for d in dep_value:
try:
r = self.component(d)
if r is not None:
results.append(r)
except dr.SkipComponent:
pass
except Exception as ex:
tb = traceback.format_exc()
log.warn(tb)
broker.add_exception(self.component, ex, tb)
if not results:
log.debug("All failed: %s" % dr.get_name(self.component))
raise dr.SkipComponent()
return results
def parser_executor(component, broker, requires, optional):
dependency = requires[0]
if dependency not in broker:
raise dr.MissingRequirements(([dependency], []))
dep_value = broker[dependency]
if not isinstance(dep_value, list):
return component(dep_value)
results = []
for d in dep_value:
try:
r = component(d)
if r is not None:
results.append(r)
except dr.SkipComponent:
pass
except Exception as ex:
log.exception(ex)
broker.add_exception(component, ex, traceback.format_exc())
if not results:
log.debug("All failed: %s" % dr.get_name(component))
raise dr.SkipComponent()
return results
def generate_tests(metafunc, test_func, package_names, pattern=None):
"""
This function hooks in to pytest's test collection framework and provides a
test for every (input_data, expected) tuple that is generated from all
@archive_provider-decorated functions.
"""
if metafunc.function is test_func:
if type(package_names) not in (list, tuple):
package_names = [package_names]
for package_name in package_names:
load_components(package_name,
include=pattern or ".*",
exclude=None)
args = []
ids = []
slow_mode = metafunc.config.getoption("--runslow")
fast_mode = metafunc.config.getoption("--smokey")
for f in tests.ARCHIVE_GENERATORS:
ts = f(stride=1 if slow_mode else f.stride)
if fast_mode:
ts = islice(ts, 0, 1)
for t in ts:
args.append(t)
input_data_name = t[2].name if not isinstance(
t[2], list) else "multi-node"
ids.append("#".join([get_name(f), input_data_name]))
metafunc.parametrize("component,compare_func,input_data,expected",
args,
ids=ids)
def get_serializer(obj):
""" Get a registered serializer for the given object.
This function walks the mro of obj looking for serializers.
Returns None if no valid serializer is found.
"""
return SERIALIZERS.get(dr.get_name(type(obj)))
def main(filename):
dr.load_components("insights.specs.default")
dr.load_components("insights.specs.insights_archive")
dr.load_components("insights.specs.sos_archive")
dr.load_components("insights.specs.jdr_archive")
dr.load_components("insights.parsers")
dr.load_components("insights.combiners")
parsers = sorted([c for c in dr.DELEGATES if is_parser(c)],
key=dr.get_name)
combiners = sorted([c for c in dr.DELEGATES if is_combiner(c)],
key=dr.get_name)
specs = sorted([
c for c in dr.DELEGATES
if is_datasource(c) and dr.get_module_name(c) == 'insights.specs'
],
key=dr.get_name)
with open(filename, "w") as fh:
fh.write("Components Cross-Reference\n")
fh.write("==========================\n")
fh.write("Specs Dependents\n")
fh.write("----------------\n")
for spec in specs:
info = dict(name=dr.get_name(spec))
info['dependents'] = []
for d in dr.get_dependents(spec):
info['dependents'].append({
'name':
dr.get_name(d),
'dependents':
[dr.get_name(sd) for sd in dr.get_dependents(d)]
})
print_spec_info(info, fh)
blank_line(fh)
fh.write("Parser Dependents/Dependencies\n")
fh.write("------------------------------\n")
for pars in parsers:
print_info(component_info(pars), fh)
blank_line(fh)
fh.write("Combiner Dependents/Dependencies\n")
fh.write("--------------------------------\n")
for comb in combiners:
print_info(component_info(comb), fh)
def invoke(self, broker):
dep_value = broker[self.requires[0]]
exception = False
if not isinstance(dep_value, list):
try:
return self.component(dep_value)
except ContentException as ce:
log.debug(ce)
broker.add_exception(self.component, ce,
traceback.format_exc())
exception = True
except CalledProcessError as cpe:
log.debug(cpe)
broker.add_exception(self.component, cpe,
traceback.format_exc())
exception = True
if exception:
raise dr.SkipComponent()
results = []
for d in dep_value:
try:
r = self.component(d)
if r is not None:
results.append(r)
except dr.SkipComponent:
pass
except ContentException as ce:
log.debug(ce)
broker.add_exception(self.component, ce,
traceback.format_exc())
if not self.continue_on_error:
exception = True
break
except CalledProcessError as cpe:
log.debug(cpe)
broker.add_exception(self.component, cpe,
traceback.format_exc())
if not self.continue_on_error:
exception = True
break
except Exception as ex:
tb = traceback.format_exc()
log.warn(tb)
broker.add_exception(self.component, ex, tb)
if not self.continue_on_error:
exception = True
break
if exception:
raise dr.SkipComponent()
if not results:
log.debug("All failed: %s" % dr.get_name(self.component))
raise dr.SkipComponent()
return results
def inner(comp, broker):
ct = dr.get_delegate(comp).__class__.__name__
res = StatResponse(name=dr.get_name(comp), component=ct)
if comp in broker:
stat_filler(broker[comp], res)
elif comp in broker.exceptions:
for e in broker.exceptions[comp]:
res.errors.append(broker.tracebacks[e])
self.results.append(attr.asdict(res))
def __str__(self):
required = self.missing[0]
at_least_one = self.missing[1]
buf = StringIO()
print("Missing Dependencies:", file=buf)
if required:
print(" Requires:", file=buf)
for d in required:
print(" %s" % dr.get_name(d), file=buf)
if at_least_one:
for alo in at_least_one:
print(" At Least One Of:", file=buf)
for d in alo:
print(" %s" % dr.get_name(d), file=buf)
buf.seek(0)
return buf.read()
def add_filter(component, patterns):
"""
Add a filter or list of filters to a component. When the component is
a datasource, the filter will be directly added to that datasouce.
In cases when the component is a parser or combiner, the filter will be
added to underlying filterable datasources by traversing dependency graph.
A filter is a simple string, and it matches if it is contained anywhere
within a line.
Args:
component (component): The component to filter, can be datasource,
parser or combiner.
patterns (str, [str]): A string, list of strings, or set of strings to
add to the datasource's filters.
"""
def inner(component, patterns):
if component in _CACHE:
del _CACHE[component]
types = six.string_types + (list, set)
if not isinstance(patterns, types):
raise TypeError(
"Filter patterns must be of type string, list, or set.")
if isinstance(patterns, six.string_types):
patterns = set([patterns])
elif isinstance(patterns, list):
patterns = set(patterns)
for pat in patterns:
if not pat:
raise Exception("Filter patterns must not be empy.")
FILTERS[component] |= patterns
if not plugins.is_datasource(component):
for dep in dr.run_order(dr.get_dependency_graph(component)):
if plugins.is_datasource(dep):
d = dr.get_delegate(dep)
if d.filterable:
inner(dep, patterns)
else:
delegate = dr.get_delegate(component)
if delegate.raw:
raise Exception("Filters aren't applicable to raw datasources.")
if not delegate.filterable:
raise Exception("Filters aren't applicable to %s." %
dr.get_name(component))
inner(component, patterns)
def invoke(self, broker):
try:
r = super(rule, self).invoke(broker)
if r is None:
raise dr.SkipComponent()
except dr.MissingRequirements as mr:
details = dr.stringify_requirements(mr.requirements)
r = _make_skip(dr.get_name(self.component),
reason="MISSING_REQUIREMENTS",
details=details)
if not isinstance(r, Response):
raise Exception("rules must return Response objects.")
return r
def __init__(self, spec, pattern):
if getattr(spec, "raw", False):
name = dr.get_name(spec)
raise ValueError("{}: Cannot filter raw files.".format(name))
self.spec = spec
self.pattern = pattern if isinstance(pattern, list) else [pattern]
self.__name__ = self.__class__.__name__
self.__module__ = self.__class__.__module__
if getattr(spec, "filterable", False):
_add_filter(spec, pattern)
component(spec)(self)
def dehydrate(self, comp, broker):
"""
Saves a component in the given broker to the file system.
"""
if not self.meta_data:
raise Exception("Hydration meta_path not set. Can't dehydrate.")
if not self.created:
fs.ensure_path(self.meta_data, mode=0o770)
if self.data:
fs.ensure_path(self.data, mode=0o770)
self.created = True
c = comp
doc = None
try:
name = dr.get_name(c)
value = broker.get(c)
errors = [
t for e in broker.exceptions.get(c, [])
for t in broker.tracebacks[e]
]
doc = {
"name": name,
"exec_time": broker.exec_times.get(c),
"errors": errors
}
try:
start = time.time()
doc["results"] = marshal(value, root=self.data, pool=self.pool)
except Exception:
errors.append(traceback.format_exc())
doc["results"] = None
finally:
doc["ser_time"] = time.time() - start
except Exception as ex:
log.exception(ex)
else:
if doc is not None and (doc["results"] or doc["errors"]):
try:
path = os.path.join(self.meta_data,
name + "." + self.ser_name)
with open(path, "w") as f:
ser.dump(doc, f)
except Exception as boom:
log.error("Could not serialize %s to %s: %r" %
(name, self.ser_name, boom))
if path:
fs.remove(path)
def process(self, broker):
"""
Ensures dependencies have been met before delegating to `self.invoke`.
"""
if any(i in broker for i in dr.IGNORE.get(self.component, [])):
raise dr.SkipComponent()
missing = self.get_missing_dependencies(broker)
if missing:
return _make_skip(dr.get_name(self.component), missing)
r = self.invoke(broker)
if r is None:
raise dr.SkipComponent()
if not isinstance(r, Response):
raise Exception("rules must return Response objects.")
return r
def observer(c, broker):
if ignore_hidden and dr.is_hidden(c):
return
if c not in broker and c not in broker.exceptions:
return
ser_name = dr.get_base_module_name(ser)
name = dr.get_name(c)
c_type = dr.get_component_type(c)
doc = {}
doc["name"] = name
doc["dr_type"] = dr.get_name(c_type) if c_type else None
doc["is_rule"] = plugins.is_rule(c)
doc["time"] = broker.exec_times.get(c)
doc["results"] = marshal(broker.get(c))
doc["errors"] = marshal(broker.exceptions.get(c))
path = os.path.join(output_dir, name + "." + ser_name)
try:
with open(path, "wb") as f:
ser.dump(doc, f)
except Exception as boom:
log.error("Could not serialize %s to %s: %s" % (name, ser_name, boom))
fs.remove(path)
def __call__(self, ds):
# /usr/bin/grep level filtering is applied behind .content or
# .stream(), but we still need to ensure we get only what *this* find
# instance wants. This can be inefficient on files where many lines
# match.
results = defaultdict(list)
ds = ds if isinstance(ds, list) else [ds]
for d in ds:
origin = d.cmd or d.path or dr.get_name(self.spec)
stream = d.content if d.loaded else d.stream()
lines = []
for line in stream:
if any(p in line for p in self.pattern):
lines.append(line)
results[origin].append(line)
return dict(results)
def run_test(component, input_data, expected=None):
if filters.ENABLED:
mod = component.__module__
sup_mod = '.'.join(mod.split('.')[:-1])
rps = _get_registry_points(component)
filterable = set(d for d in rps if dr.get_delegate(d).filterable)
missing_filters = filterable - ADDED_FILTERS.get(mod, set()) - ADDED_FILTERS.get(sup_mod, set())
if missing_filters:
names = [dr.get_name(m) for m in missing_filters]
msg = "%s must add filters to %s"
raise Exception(msg % (mod, ", ".join(names)))
broker = run_input_data(component, input_data)
if expected:
deep_compare(broker.get(component), expected)
return broker.get(component)
def process(self, broker):
"""
Ensures dependencies have been met before delegating to `self.invoke`.
"""
if any(i in broker for i in dr.IGNORE.get(self.component, [])):
raise dr.SkipComponent()
missing = self.get_missing_dependencies(broker)
if missing:
details = dr.stringify_requirements(missing)
return _make_skip(dr.get_name(self.component),
reason="MISSING_REQUIREMENTS",
details=details)
r = self.invoke(broker)
if r is None:
raise dr.SkipComponent()
if not isinstance(r, Response):
raise Exception("rules must return Response objects.")
return r
def run_test(component, input_data, expected=None, return_make_none=False):
if filters.ENABLED:
mod = component.__module__
sup_mod = '.'.join(mod.split('.')[:-1])
rps = _get_registry_points(component)
filterable = set(d for d in rps if dr.get_delegate(d).filterable)
missing_filters = filterable - ADDED_FILTERS.get(
mod, set()) - ADDED_FILTERS.get(sup_mod, set())
if missing_filters:
names = [dr.get_name(m) for m in missing_filters]
msg = "%s must add filters to %s"
raise Exception(msg % (mod, ", ".join(names)))
broker = run_input_data(component, input_data)
result = broker.get(component)
if expected:
deep_compare(result, expected)
elif result == MAKE_NONE_RESULT and not return_make_none:
# Convert make_none() result to None as default unless
# make_none explicitly requested
return None
return result
def __call__(self, ds):
# /usr/bin/grep level filtering is applied behind .content or
# .stream(), but we still need to ensure we get only what *this* find
# instance wants. This can be inefficient on files where many lines
# match.
results = {}
ds = ds if isinstance(ds, list) else [ds]
for d in ds:
if d.relative_path:
origin = os.path.join("/", d.relative_path.lstrip("/"))
elif d.cmd:
origin = d.cmd
else:
origin = dr.get_name(self.spec)
stream = d.content if d.loaded else d.stream()
lines = []
for line in stream:
if any(p in line for p in self.pattern):
lines.append(line)
if lines:
results[origin] = lines
if not results:
raise dr.SkipComponent()
return dict(results)
def inner(func):
if _type in DESERIALIZERS:
msg = "%s already has a deserializer registered: %s"
raise Exception(msg % (dr.get_name(_type), dr.get_name(DESERIALIZERS[_type])))
DESERIALIZERS[_type] = func
return func
def __repr__(self):
return dr.get_name(self)
def main():
# config = get_config()
dr.load_components("insights.specs.default")
dr.load_components("insights.parsers")
dr.load_components("insights.combiners")
dr.load_components("telemetry.rules.plugins")
dr.load_components("prodsec")
ds = dr.COMPONENTS_BY_TYPE[datasource]
specs = []
for c in ds:
if not is_datasource(c):
continue
if not any(is_datasource(d) for d in dr.get_dependents(c)):
specs.append(c)
deps = defaultdict(dict)
pspec = ''
for spec in sorted(specs, key=dr.get_name):
info = dict(name=dr.get_simple_name(spec))
f = filters.get_filters(spec)
info['dependents'] = []
spds = None
d = [d for d in dr.get_dependencies(spec) if is_datasource(d)]
for dp in d:
c = dr.get_dependencies(dp)
for cdeps in c:
if is_datasource(cdeps) and '__qualname__' in cdeps.func_dict and 'DefaultSpecs' in cdeps.func_dict['__qualname__']:
spds = cdeps
for d in dr.get_dependencies(spec):
cp = ''
lines = []
if d.__doc__ and "Returns the first" in d.__doc__:
lines = d.__doc__.replace(',', '\n')
lines = lines.splitlines()
head = [lines[0]]
top = ["<ul>"]
bottom = ["</ul>"]
if spds:
lines = [l.replace('Command:', '') for l in lines]
lines = [l.replace('Path:', '') for l in lines]
lines = ["<li>" + l + "</li>" for l in lines[1:]]
# lines = ["<li>" + spds.func_doc + ',' + l + "</li>" for l in lines[1:]]
else:
lines = ["<li>" + l + "</li>" for l in lines[1:]]
cp = "\n".join(head + top + lines + bottom)
else:
if spds:
d.__doc__ = d.__doc__.replace('Command:', '')
d.__doc__ = d.__doc__.replace('Path:', '')
d.__doc__ = spds.func_doc + ', ' + d.__doc__
cp = d.__doc__
for d in dr.get_dependents(spec):
if dr.get_simple_name(pspec) == dr.get_simple_name(d):
continue
pspec = d
p = [dr.get_name(sd) for sd in dr.get_dependents(d)]
rules = sorted([x.rsplit('.', 2)[1] for x in p])
deps[info['name']][info['name'] + "_spec-def"] = cp
deps[info['name']][info['name'] + "_rules"] = ", ".join(rules)
deps[info['name']][info['name'] + "_filters"] = f
report = Environment().from_string(REPORT).render(
report_date=datetime.date.today().strftime("%B %d, %Y"), specs=deps)
print(report)
def dumps():
"""Returns a string representation of the FILTERS dictionary."""
d = {}
for k, v in FILTERS.items():
d[dr.get_name(k)] = list(v)
return _dumps(d)
def to_dict(x):
return {"type": dr.get_name(type(obj)), "object": the_ser(x)}