def get_info(spec):
return {
"name": dr.get_simple_name(spec),
"description": get_description(spec),
"rules": ul(sorted(dr.get_name(r) for r in get_rules(spec))),
"filters": ul(sorted(filters.get_filters(spec))),
}
def process_dir(self, path):
broker = create_broker(path)
broker = dr.run(broker=broker)
archive_meta = meta(hostname=get_hostname(broker),
uname=get_uname(broker),
release=get_release(broker),
version=get_version(broker),
**self.kwargs)
datasources = get_datasources(broker)
for d in datasources:
name = dr.get_simple_name(d)
large = is_large(name)
reader = line_reader if large else file_reader
providers = broker[d]
if not isinstance(providers, list):
providers = [providers]
for p in providers:
file_meta = meta(path=p.path, target=name)
transform = compose(archive_meta, file_meta, to_dict)
if large:
transform = compose(line_counter(), transform)
stream_transform = liftI(transform)
yield (name, p.path, compose(stream_transform, reader))
def run_graph(seed_broker, g, output_dir):
to_save = [
plugins.datasource, plugins.parser, plugins.combiner, plugins.rule
]
broker = dr.Broker(seed_broker)
for _type in to_save:
path = os.path.join(output_dir, dr.get_simple_name(_type))
fs.ensure_path(path)
broker.add_observer(persister(path), _type)
dr.run(g, broker)
def main():
# config = get_config()
dr.load_components("insights.specs.default")
dr.load_components("insights.parsers")
dr.load_components("insights.combiners")
dr.load_components("telemetry.rules.plugins")
dr.load_components("prodsec")
ds = dr.COMPONENTS_BY_TYPE[datasource]
specs = []
for c in ds:
if not is_datasource(c):
continue
if not any(is_datasource(d) for d in dr.get_dependents(c)):
specs.append(c)
deps = defaultdict(dict)
pspec = ''
for spec in sorted(specs, key=dr.get_name):
info = dict(name=dr.get_simple_name(spec))
f = filters.get_filters(spec)
info['dependents'] = []
spds = None
d = [d for d in dr.get_dependencies(spec) if is_datasource(d)]
for dp in d:
c = dr.get_dependencies(dp)
for cdeps in c:
if is_datasource(cdeps) and '__qualname__' in cdeps.func_dict and 'DefaultSpecs' in cdeps.func_dict['__qualname__']:
spds = cdeps
for d in dr.get_dependencies(spec):
cp = ''
lines = []
if d.__doc__ and "Returns the first" in d.__doc__:
lines = d.__doc__.replace(',', '\n')
lines = lines.splitlines()
head = [lines[0]]
top = ["<ul>"]
bottom = ["</ul>"]
if spds:
lines = [l.replace('Command:', '') for l in lines]
lines = [l.replace('Path:', '') for l in lines]
lines = ["<li>" + l + "</li>" for l in lines[1:]]
# lines = ["<li>" + spds.func_doc + ',' + l + "</li>" for l in lines[1:]]
else:
lines = ["<li>" + l + "</li>" for l in lines[1:]]
cp = "\n".join(head + top + lines + bottom)
else:
if spds:
d.__doc__ = d.__doc__.replace('Command:', '')
d.__doc__ = d.__doc__.replace('Path:', '')
d.__doc__ = spds.func_doc + ', ' + d.__doc__
cp = d.__doc__
for d in dr.get_dependents(spec):
if dr.get_simple_name(pspec) == dr.get_simple_name(d):
continue
pspec = d
p = [dr.get_name(sd) for sd in dr.get_dependents(d)]
rules = sorted([x.rsplit('.', 2)[1] for x in p])
deps[info['name']][info['name'] + "_spec-def"] = cp
deps[info['name']][info['name'] + "_rules"] = ", ".join(rules)
deps[info['name']][info['name'] + "_filters"] = f
report = Environment().from_string(REPORT).render(
report_date=datetime.date.today().strftime("%B %d, %Y"), specs=deps)
print(report)
def get_name(o):
return "_".join([dr.get_base_module_name(o), dr.get_simple_name(o)])