def test_get_filter_registry_point(): s = set(["COMMAND", "MEM"]) f = filters.get_filters(DefaultSpecs.ps_aux) assert f & s == s f = filters.get_filters(Specs.ps_aux) assert "COMMAND" in f assert "MEM" not in f
def test_add_filter_to_parser(): filter_string = "bash" filters.add_filter(PsAux, filter_string) spec_filters = filters.get_filters(Specs.ps_aux) assert filter_string in spec_filters parser_filters = filters.get_filters(PsAux) assert not parser_filters
def test_add_filter_to_parser_patterns_list(): filters_list = ["bash", "systemd", "Network"] filters.add_filter(PsAux, filters_list) spec_filters = filters.get_filters(Specs.ps_aux) assert all(f in spec_filters for f in filters_list) parser_filters = filters.get_filters(PsAux) assert not parser_filters
def test_add_filter_to_parser_non_filterable(): filter_string = "bash" filters.add_filter(PsAuxcww, filter_string) spec_filters = filters.get_filters(Specs.ps_auxcww) assert not spec_filters parser_filters = filters.get_filters(PsAuxcww) assert not parser_filters
def test_get_filter(): filters.add_filter(Specs.fstab, "COMMAND") f = filters.get_filters(Specs.fstab) assert "COMMAND" in f f = filters.get_filters(DefaultSpecs.fstab) assert "COMMAND" in f lines = ["COMMAND", "DISCARD"] lines = filters.apply_filters(DefaultSpecs.fstab, lines) assert "COMMAND" in lines assert "DISCARD" not in lines
def inner(broker): ctx = broker[context] rc = None if split: filters = "\n".join(get_filters(inner)) if filters: command = [shlex.split(cmd)] + [["grep", "-F", filters]] raw = ctx.shell_out(command, split=split, keep_rc=keep_rc, timeout=timeout) else: command = [shlex.split(cmd)] raw = ctx.shell_out(command, split=split, keep_rc=keep_rc, timeout=timeout) if keep_rc: rc, result = raw else: result = raw return CommandOutputProvider(cmd, ctx, split=split, content=result, rc=rc, keep_rc=keep_rc)
def _stream(self): """ Returns a generator of lines instead of a list of lines. """ if self._exception: raise self._exception try: if self._content: yield self._content raise StopIteration filters = get_filters(self.ds) if self.ds else None if filters: grep = ["grep", "-F", "\n".join(filters)] with self.ctx.connect(self.cmd, grep, env=SAFE_ENV, timeout=self.timeout) as s: yield s else: with self.ctx.stream(self.cmd, env=SAFE_ENV, timeout=self.timeout) as s: yield s except StopIteration: raise except Exception as ex: self._exception = ex raise ContentException(str(ex))
def cmd_and_pkg(broker): """ Collect a list of running commands and the associated RPM package providing those commands. The commands are based on filters so rules must add the desired commands as filters to enable collection. If a command is not provided by an RPM then it will not be included in the output. In order for the full command line to be present in the Ps combiner a filter must be added to the spec ``ps_auxww``. A filter must also be added to ``package_provides_command`` so this datasource will look for the command in Ps. Arguments: broker: the broker object for the current session Returns: DatasourceProvider: Returns the collected information as a file with 1 line per command Raises: SkipComponent: Raised if no data is collected """ commands = get_filters(Specs.package_provides_command) """ list: List of commands to search for, added as filters for the spec """ if commands: pkg_cmd = list() for cmd in get_running_commands(broker[Ps], broker[HostContext], list(commands)): pkg = get_package(broker[HostContext], cmd) if pkg is not None: pkg_cmd.append("{0} {1}".format(cmd, pkg)) if pkg_cmd: return DatasourceProvider( '\n'.join(pkg_cmd), relative_path='insights_commands/package_provides_command') raise SkipComponent
def create_args(self): args = [] if not self.filter: return args filters = "\n".join(get_filters(self.ds)) if self.ds else None if filters: args.append(["grep", "-F", filters, self.path]) patterns = "\n".join(blacklist.get_disallowed_patterns()) if patterns: grep = ["grep", "-v" "-F", patterns] if not args: grep.append(self.path) args.append(grep) keywords = blacklist.get_disallowed_keywords() if keywords: sed = ["sed"] for kw in keywords: sed.extend(["-e", "s/%s/keyword/g" % kw.replace("/", "\\/")]) if not args: sed.append(self.path) args.append(sed) return args
def get_info(spec): return { "name": dr.get_simple_name(spec), "description": get_description(spec), "rules": ul(sorted(dr.get_name(r) for r in get_rules(spec))), "filters": ul(sorted(filters.get_filters(spec))), }
def awx_manage_check_license_data_datasource(broker): """ This datasource provides the not-sensitive information collected from ``/usr/bin/awx-manage check_license --data``. Typical content of ``/usr/bin/awx-manage check_license --data`` file is:: {"contact_email": "*****@*****.**", "company_name": "test Inc", "instance_count": 100, "license_date": 1655092799, "license_type": "enterprise", "subscription_name": "Red Hat Ansible Automation, Standard (100 Managed Nodes)", "sku": "MCT3691", "support_level": "Standard", "product_name": "Red Hat Ansible Automation Platform", "valid_key": true, "satellite": null, "pool_id": "2c92808179803e530179ea5989a157a4", "current_instances": 1, "available_instances": 100, "free_instances": 99, "time_remaining": 29885220, "trial": false, "grace_period_remaining": 32477220, "compliant": true, "date_warning": false, "date_expired": false} Returns: str: JSON string containing non-sensitive information. Raises: SkipComponent: When the filter/path does not exist or any exception occurs. """ try: filters = get_filters(Specs.awx_manage_check_license_data) content = broker[LocalSpecs.awx_manage_check_license_data_raw].content if content and filters: json_data = json.loads(content[0]) filter_result = {} for item in filters: filter_result[item] = json_data.get(item) if filter_result: return DatasourceProvider( content=json.dumps( collections.OrderedDict(sorted( filter_result.items()))), relative_path= 'insights_commands/awx-manage_check_license_--data') except Exception as e: raise SkipComponent("Unexpected exception:{e}".format(e=str(e))) raise SkipComponent
def __call__(self, broker): ctx = broker[self.context] rc = None if self.split: filters = "\n".join(get_filters(self)) if filters: command = [shlex.split(self.cmd)] + [["grep", "-F", filters]] raw = ctx.shell_out(command, split=self.split, keep_rc=self.keep_rc, timeout=self.timeout) else: command = [shlex.split(self.cmd)] raw = ctx.shell_out(command, split=self.split, keep_rc=self.keep_rc, timeout=self.timeout) if self.keep_rc: rc, result = raw else: result = raw return CommandOutputProvider(self.cmd, ctx, split=self.split, content=result, rc=rc, keep_rc=self.keep_rc)
def inner(broker): result = [] source = broker[provider] ctx = broker[context] if isinstance(source, ContentProvider): source = source.content if not isinstance(source, (list, set)): source = [source] for e in source: try: the_cmd = cmd % e rc = None if split: filters = "\n".join(get_filters(inner)) if filters: command = "{0} | grep -F '{1}'".format(the_cmd, filters) raw = ctx.shell_out(command, split=split, keep_rc=keep_rc, timeout=timeout) else: raw = ctx.shell_out(the_cmd, split=split, keep_rc=keep_rc, timeout=timeout) if keep_rc: rc, output = raw else: output = raw result.append(CommandOutputProvider(the_cmd, ctx, args=e, content=output, rc=rc, split=split, keep_rc=keep_rc)) except: log.debug(traceback.format_exc()) if result: return result raise ContentException("No results found for [%s]" % cmd)
def load(self): filters = False if self.ds: filters = "\n".join(get_filters(self.ds)) if filters: cmd = [["grep", "-F", filters, self.path]] rc, out = self.ctx.shell_out(cmd, keep_rc=True, env=SAFE_ENV) self.rc = rc return out else: with open(self.path, "rU") as f: # universal newlines results = [l.rstrip("\n") for l in f] return results
def inner(broker): ctx = broker[context] rc = None if split: filters = "\n".join(get_filters(inner)) if filters: command = "{0} | grep -F '{1}'".format(cmd, filters) raw = ctx.shell_out(command, split=split, keep_rc=keep_rc, timeout=timeout) else: raw = ctx.shell_out(cmd, split=split, keep_rc=keep_rc, timeout=timeout) if keep_rc: rc, result = raw else: result = raw return CommandOutputProvider(cmd, ctx, split=split, content=result, rc=rc, keep_rc=keep_rc)
def load(self): if self.split: filters = "\n".join(get_filters(self.ds)) if filters: command = [shlex.split(self.cmd)] + [["grep", "-F", filters]] raw = self.ctx.shell_out(command, split=self.split, keep_rc=self.keep_rc, timeout=self.timeout, env=SAFE_ENV) else: command = [shlex.split(self.cmd)] raw = self.ctx.shell_out(command, split=self.split, keep_rc=self.keep_rc, timeout=self.timeout, env=SAFE_ENV) if self.keep_rc: self.rc, output = raw else: output = raw return output
def __call__(self, broker): result = [] source = broker[self.provider] ctx = broker[self.context] if isinstance(source, ContentProvider): source = source.content if not isinstance(source, (list, set)): source = [source] for e in source: try: the_cmd = self.cmd % e rc = None if self.split: filters = "\n".join(get_filters(self)) if filters: command = [shlex.split(the_cmd) ] + [["grep", "-F", filters]] raw = ctx.shell_out(command, split=self.split, keep_rc=self.keep_rc, timeout=self.timeout) else: command = [shlex.split(the_cmd)] raw = ctx.shell_out(command, split=self.split, keep_rc=self.keep_rc, timeout=self.timeout) if self.keep_rc: rc, output = raw else: output = raw result.append( CommandOutputProvider(the_cmd, ctx, args=e, content=output, rc=rc, split=self.split, keep_rc=self.keep_rc)) except: log.debug(traceback.format_exc()) if result: return result raise ContentException("No results found for [%s]" % self.cmd)
def create_args(self): command = [shlex.split(self.cmd)] if self.split: filters = "\n".join(get_filters(self.ds)) if filters: command.append(["grep", "-F", filters]) patterns = "\n".join(blacklist.get_disallowed_patterns()) if patterns: command.append(["grep", "-v", "-F", patterns]) keywords = blacklist.get_disallowed_keywords() if keywords: sed = ["sed"] for kw in keywords: sed.extend(["-e", "s/%s/keyword/g" % kw.replace("/", "\\/")]) command.append(sed) return command
def load(self): filters = False if self.ds: filters = "\n".join(get_filters(self.ds)) if filters: cmd = [["grep", "-F", filters, self.path]] rc, out = subproc.call(cmd, shell=False, keep_rc=True, env=SAFE_ENV) if rc == 0 and out != '': results = out.splitlines() else: return [] else: with open(self.path, "rU") as f: results = [l.rstrip("\n") for l in f] return results
def load(self): filters = False if self.ds: filters = "\n".join(get_filters(self.ds)) if filters: cmd = "/bin/grep -F '{0}' {1}".format(filters, self.path) rc, out = subproc.call(cmd.encode("utf-8"), shell=False, keep_rc=True) if rc == 0 and out != '': results = out.splitlines() else: return [] else: with open(self.path, "rU") as f: results = [l.rstrip("\n") for l in f] if not self.validate_lines(results): first = results[0] if results else "<no content>" raise ContentException(self.relative_path + ": " + first) return results
def _stream(self): """ Returns a generator of lines instead of a list of lines. """ if self._exception: raise self._exception try: if self._content: yield self._content raise StopIteration filters = get_filters(self.ds) if self.ds else None if filters: cmd = ["grep", "-F", "\n".join(filters), self.path] with streams.stream(cmd, env=SAFE_ENV) as s: yield s else: with open(self.path, "rU") as f: # universal newlines yield f except StopIteration: raise except Exception as ex: self._exception = ex raise ContentException(str(ex))
def cloud_cfg(broker): """ This datasource provides configuration collected from ``/etc/cloud/cloud.cfg``. Typical content of ``/etc/cloud/cloud.cfg`` file is:: #cloud-config users: - name: demo ssh-authorized-keys: - key_one - key_two passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ ssh_deletekeys: 1 network: version: 1 config: - type: physical name: eth0 subnets: - type: dhcp - type: dhcp6 system_info: default_user: name: user2 plain_text_passwd: 'someP@assword' home: /home/user2 debug: output: /var/log/cloud-init-debug.log verbose: true Note: This datasource may be executed using the following command: ``insights cat --no-header cloud_cfg`` Sample output in JSON format:: { "ssh_deletekeys": 1, "network": { "version": 1, "config": [ { "type": "physical", "name": "eth0", "subnets": [ { "type": "dhcp" }, { "type": "dhcp6" } ] } ] }, "debug": { "output": "/var/log/cloud-init-debug.log", "verbose": true } } Returns: str: JSON string after removing the sensitive information. Raises: SkipComponent: When the path does not exist or any exception occurs. """ relative_path = '/etc/cloud/cloud.cfg' try: filters = get_filters(Specs.cloud_cfg) content = broker[LocalSpecs.cloud_cfg_input].content if content and filters: result = dict() content = yaml.load('\n'.join(content), Loader=yaml.SafeLoader) if isinstance(content, dict): # remove sensitive data content.pop('users', None) content.pop('system_info', None) # apply filters for item in filters: if item in content: result[item] = content[item] if result: result = dict(sorted(result.items(), key=lambda x: x[0])) return DatasourceProvider(content=json.dumps(result), relative_path=relative_path) raise SkipComponent("Invalid YAML format") except Exception as e: raise SkipComponent("Unexpected exception:{e}".format(e=str(e))) raise SkipComponent
def test_get_filter(): f = filters.get_filters(Specs.ps_aux) assert "COMMAND" in f f = filters.get_filters(DefaultSpecs.ps_aux) assert "COMMAND" in f
def main(): # config = get_config() dr.load_components("insights.specs.default") dr.load_components("insights.parsers") dr.load_components("insights.combiners") dr.load_components("telemetry.rules.plugins") dr.load_components("prodsec") ds = dr.COMPONENTS_BY_TYPE[datasource] specs = [] for c in ds: if not is_datasource(c): continue if not any(is_datasource(d) for d in dr.get_dependents(c)): specs.append(c) deps = defaultdict(dict) pspec = '' for spec in sorted(specs, key=dr.get_name): info = dict(name=dr.get_simple_name(spec)) f = filters.get_filters(spec) info['dependents'] = [] spds = None d = [d for d in dr.get_dependencies(spec) if is_datasource(d)] for dp in d: c = dr.get_dependencies(dp) for cdeps in c: if is_datasource(cdeps) and '__qualname__' in cdeps.func_dict and 'DefaultSpecs' in cdeps.func_dict['__qualname__']: spds = cdeps for d in dr.get_dependencies(spec): cp = '' lines = [] if d.__doc__ and "Returns the first" in d.__doc__: lines = d.__doc__.replace(',', '\n') lines = lines.splitlines() head = [lines[0]] top = ["<ul>"] bottom = ["</ul>"] if spds: lines = [l.replace('Command:', '') for l in lines] lines = [l.replace('Path:', '') for l in lines] lines = ["<li>" + l + "</li>" for l in lines[1:]] # lines = ["<li>" + spds.func_doc + ',' + l + "</li>" for l in lines[1:]] else: lines = ["<li>" + l + "</li>" for l in lines[1:]] cp = "\n".join(head + top + lines + bottom) else: if spds: d.__doc__ = d.__doc__.replace('Command:', '') d.__doc__ = d.__doc__.replace('Path:', '') d.__doc__ = spds.func_doc + ', ' + d.__doc__ cp = d.__doc__ for d in dr.get_dependents(spec): if dr.get_simple_name(pspec) == dr.get_simple_name(d): continue pspec = d p = [dr.get_name(sd) for sd in dr.get_dependents(d)] rules = sorted([x.rsplit('.', 2)[1] for x in p]) deps[info['name']][info['name'] + "_spec-def"] = cp deps[info['name']][info['name'] + "_rules"] = ", ".join(rules) deps[info['name']][info['name'] + "_filters"] = f report = Environment().from_string(REPORT).render( report_date=datetime.date.today().strftime("%B %d, %Y"), specs=deps) print(report)