def get_info(spec): return { "name": dr.get_simple_name(spec), "description": get_description(spec), "rules": ul(sorted(dr.get_name(r) for r in get_rules(spec))), "filters": ul(sorted(filters.get_filters(spec))), }
def process_dir(self, path): broker = create_broker(path) broker = dr.run(broker=broker) archive_meta = meta(hostname=get_hostname(broker), uname=get_uname(broker), release=get_release(broker), version=get_version(broker), **self.kwargs) datasources = get_datasources(broker) for d in datasources: name = dr.get_simple_name(d) large = is_large(name) reader = line_reader if large else file_reader providers = broker[d] if not isinstance(providers, list): providers = [providers] for p in providers: file_meta = meta(path=p.path, target=name) transform = compose(archive_meta, file_meta, to_dict) if large: transform = compose(line_counter(), transform) stream_transform = liftI(transform) yield (name, p.path, compose(stream_transform, reader))
def run_graph(seed_broker, g, output_dir): to_save = [ plugins.datasource, plugins.parser, plugins.combiner, plugins.rule ] broker = dr.Broker(seed_broker) for _type in to_save: path = os.path.join(output_dir, dr.get_simple_name(_type)) fs.ensure_path(path) broker.add_observer(persister(path), _type) dr.run(g, broker)
def main(): # config = get_config() dr.load_components("insights.specs.default") dr.load_components("insights.parsers") dr.load_components("insights.combiners") dr.load_components("telemetry.rules.plugins") dr.load_components("prodsec") ds = dr.COMPONENTS_BY_TYPE[datasource] specs = [] for c in ds: if not is_datasource(c): continue if not any(is_datasource(d) for d in dr.get_dependents(c)): specs.append(c) deps = defaultdict(dict) pspec = '' for spec in sorted(specs, key=dr.get_name): info = dict(name=dr.get_simple_name(spec)) f = filters.get_filters(spec) info['dependents'] = [] spds = None d = [d for d in dr.get_dependencies(spec) if is_datasource(d)] for dp in d: c = dr.get_dependencies(dp) for cdeps in c: if is_datasource(cdeps) and '__qualname__' in cdeps.func_dict and 'DefaultSpecs' in cdeps.func_dict['__qualname__']: spds = cdeps for d in dr.get_dependencies(spec): cp = '' lines = [] if d.__doc__ and "Returns the first" in d.__doc__: lines = d.__doc__.replace(',', '\n') lines = lines.splitlines() head = [lines[0]] top = ["<ul>"] bottom = ["</ul>"] if spds: lines = [l.replace('Command:', '') for l in lines] lines = [l.replace('Path:', '') for l in lines] lines = ["<li>" + l + "</li>" for l in lines[1:]] # lines = ["<li>" + spds.func_doc + ',' + l + "</li>" for l in lines[1:]] else: lines = ["<li>" + l + "</li>" for l in lines[1:]] cp = "\n".join(head + top + lines + bottom) else: if spds: d.__doc__ = d.__doc__.replace('Command:', '') d.__doc__ = d.__doc__.replace('Path:', '') d.__doc__ = spds.func_doc + ', ' + d.__doc__ cp = d.__doc__ for d in dr.get_dependents(spec): if dr.get_simple_name(pspec) == dr.get_simple_name(d): continue pspec = d p = [dr.get_name(sd) for sd in dr.get_dependents(d)] rules = sorted([x.rsplit('.', 2)[1] for x in p]) deps[info['name']][info['name'] + "_spec-def"] = cp deps[info['name']][info['name'] + "_rules"] = ", ".join(rules) deps[info['name']][info['name'] + "_filters"] = f report = Environment().from_string(REPORT).render( report_date=datetime.date.today().strftime("%B %d, %Y"), specs=deps) print(report)
def get_name(o): return "_".join([dr.get_base_module_name(o), dr.get_simple_name(o)])