def test_get_filter_registry_point():
s = set(["COMMAND", "MEM"])
f = filters.get_filters(DefaultSpecs.ps_aux)
assert f & s == s
f = filters.get_filters(Specs.ps_aux)
assert "COMMAND" in f
assert "MEM" not in f
def test_add_filter_to_parser():
filter_string = "bash"
filters.add_filter(PsAux, filter_string)
spec_filters = filters.get_filters(Specs.ps_aux)
assert filter_string in spec_filters
parser_filters = filters.get_filters(PsAux)
assert not parser_filters
def test_add_filter_to_parser_patterns_list():
filters_list = ["bash", "systemd", "Network"]
filters.add_filter(PsAux, filters_list)
spec_filters = filters.get_filters(Specs.ps_aux)
assert all(f in spec_filters for f in filters_list)
parser_filters = filters.get_filters(PsAux)
assert not parser_filters
def test_add_filter_to_parser_non_filterable():
filter_string = "bash"
filters.add_filter(PsAuxcww, filter_string)
spec_filters = filters.get_filters(Specs.ps_auxcww)
assert not spec_filters
parser_filters = filters.get_filters(PsAuxcww)
assert not parser_filters
def test_get_filter():
filters.add_filter(Specs.fstab, "COMMAND")
f = filters.get_filters(Specs.fstab)
assert "COMMAND" in f
f = filters.get_filters(DefaultSpecs.fstab)
assert "COMMAND" in f
lines = ["COMMAND", "DISCARD"]
lines = filters.apply_filters(DefaultSpecs.fstab, lines)
assert "COMMAND" in lines
assert "DISCARD" not in lines
def inner(broker):
ctx = broker[context]
rc = None
if split:
filters = "\n".join(get_filters(inner))
if filters:
command = [shlex.split(cmd)] + [["grep", "-F", filters]]
raw = ctx.shell_out(command,
split=split,
keep_rc=keep_rc,
timeout=timeout)
else:
command = [shlex.split(cmd)]
raw = ctx.shell_out(command,
split=split,
keep_rc=keep_rc,
timeout=timeout)
if keep_rc:
rc, result = raw
else:
result = raw
return CommandOutputProvider(cmd,
ctx,
split=split,
content=result,
rc=rc,
keep_rc=keep_rc)
def _stream(self):
"""
Returns a generator of lines instead of a list of lines.
"""
if self._exception:
raise self._exception
try:
if self._content:
yield self._content
raise StopIteration
filters = get_filters(self.ds) if self.ds else None
if filters:
grep = ["grep", "-F", "\n".join(filters)]
with self.ctx.connect(self.cmd,
grep,
env=SAFE_ENV,
timeout=self.timeout) as s:
yield s
else:
with self.ctx.stream(self.cmd,
env=SAFE_ENV,
timeout=self.timeout) as s:
yield s
except StopIteration:
raise
except Exception as ex:
self._exception = ex
raise ContentException(str(ex))
def cmd_and_pkg(broker):
"""
Collect a list of running commands and the associated RPM package providing those commands.
The commands are based on filters so rules must add the desired commands as filters to
enable collection. If a command is not provided by an RPM then it will not be included
in the output.
In order for the full command line to be present in the Ps combiner a filter must be added
to the spec ``ps_auxww``. A filter must also be added to ``package_provides_command`` so
this datasource will look for the command in Ps.
Arguments:
broker: the broker object for the current session
Returns:
DatasourceProvider: Returns the collected information as a file with 1 line per command
Raises:
SkipComponent: Raised if no data is collected
"""
commands = get_filters(Specs.package_provides_command)
""" list: List of commands to search for, added as filters for the spec """
if commands:
pkg_cmd = list()
for cmd in get_running_commands(broker[Ps], broker[HostContext],
list(commands)):
pkg = get_package(broker[HostContext], cmd)
if pkg is not None:
pkg_cmd.append("{0} {1}".format(cmd, pkg))
if pkg_cmd:
return DatasourceProvider(
'\n'.join(pkg_cmd),
relative_path='insights_commands/package_provides_command')
raise SkipComponent
def create_args(self):
args = []
if not self.filter:
return args
filters = "\n".join(get_filters(self.ds)) if self.ds else None
if filters:
args.append(["grep", "-F", filters, self.path])
patterns = "\n".join(blacklist.get_disallowed_patterns())
if patterns:
grep = ["grep", "-v" "-F", patterns]
if not args:
grep.append(self.path)
args.append(grep)
keywords = blacklist.get_disallowed_keywords()
if keywords:
sed = ["sed"]
for kw in keywords:
sed.extend(["-e", "s/%s/keyword/g" % kw.replace("/", "\\/")])
if not args:
sed.append(self.path)
args.append(sed)
return args
def get_info(spec):
return {
"name": dr.get_simple_name(spec),
"description": get_description(spec),
"rules": ul(sorted(dr.get_name(r) for r in get_rules(spec))),
"filters": ul(sorted(filters.get_filters(spec))),
}
def awx_manage_check_license_data_datasource(broker):
"""
This datasource provides the not-sensitive information collected
from ``/usr/bin/awx-manage check_license --data``.
Typical content of ``/usr/bin/awx-manage check_license --data`` file is::
{"contact_email": "*****@*****.**", "company_name": "test Inc", "instance_count": 100, "license_date": 1655092799, "license_type": "enterprise", "subscription_name": "Red Hat Ansible Automation, Standard (100 Managed Nodes)", "sku": "MCT3691", "support_level": "Standard", "product_name": "Red Hat Ansible Automation Platform", "valid_key": true, "satellite": null, "pool_id": "2c92808179803e530179ea5989a157a4", "current_instances": 1, "available_instances": 100, "free_instances": 99, "time_remaining": 29885220, "trial": false, "grace_period_remaining": 32477220, "compliant": true, "date_warning": false, "date_expired": false}
Returns:
str: JSON string containing non-sensitive information.
Raises:
SkipComponent: When the filter/path does not exist or any exception occurs.
"""
try:
filters = get_filters(Specs.awx_manage_check_license_data)
content = broker[LocalSpecs.awx_manage_check_license_data_raw].content
if content and filters:
json_data = json.loads(content[0])
filter_result = {}
for item in filters:
filter_result[item] = json_data.get(item)
if filter_result:
return DatasourceProvider(
content=json.dumps(
collections.OrderedDict(sorted(
filter_result.items()))),
relative_path=
'insights_commands/awx-manage_check_license_--data')
except Exception as e:
raise SkipComponent("Unexpected exception:{e}".format(e=str(e)))
raise SkipComponent
def __call__(self, broker):
ctx = broker[self.context]
rc = None
if self.split:
filters = "\n".join(get_filters(self))
if filters:
command = [shlex.split(self.cmd)] + [["grep", "-F", filters]]
raw = ctx.shell_out(command,
split=self.split,
keep_rc=self.keep_rc,
timeout=self.timeout)
else:
command = [shlex.split(self.cmd)]
raw = ctx.shell_out(command,
split=self.split,
keep_rc=self.keep_rc,
timeout=self.timeout)
if self.keep_rc:
rc, result = raw
else:
result = raw
return CommandOutputProvider(self.cmd,
ctx,
split=self.split,
content=result,
rc=rc,
keep_rc=self.keep_rc)
def inner(broker):
result = []
source = broker[provider]
ctx = broker[context]
if isinstance(source, ContentProvider):
source = source.content
if not isinstance(source, (list, set)):
source = [source]
for e in source:
try:
the_cmd = cmd % e
rc = None
if split:
filters = "\n".join(get_filters(inner))
if filters:
command = "{0} | grep -F '{1}'".format(the_cmd, filters)
raw = ctx.shell_out(command, split=split, keep_rc=keep_rc, timeout=timeout)
else:
raw = ctx.shell_out(the_cmd, split=split, keep_rc=keep_rc, timeout=timeout)
if keep_rc:
rc, output = raw
else:
output = raw
result.append(CommandOutputProvider(the_cmd, ctx, args=e, content=output, rc=rc, split=split,
keep_rc=keep_rc))
except:
log.debug(traceback.format_exc())
if result:
return result
raise ContentException("No results found for [%s]" % cmd)
def load(self):
filters = False
if self.ds:
filters = "\n".join(get_filters(self.ds))
if filters:
cmd = [["grep", "-F", filters, self.path]]
rc, out = self.ctx.shell_out(cmd, keep_rc=True, env=SAFE_ENV)
self.rc = rc
return out
else:
with open(self.path, "rU") as f: # universal newlines
results = [l.rstrip("\n") for l in f]
return results
def inner(broker):
ctx = broker[context]
rc = None
if split:
filters = "\n".join(get_filters(inner))
if filters:
command = "{0} | grep -F '{1}'".format(cmd, filters)
raw = ctx.shell_out(command, split=split, keep_rc=keep_rc, timeout=timeout)
else:
raw = ctx.shell_out(cmd, split=split, keep_rc=keep_rc, timeout=timeout)
if keep_rc:
rc, result = raw
else:
result = raw
return CommandOutputProvider(cmd, ctx, split=split, content=result, rc=rc, keep_rc=keep_rc)
def load(self):
if self.split:
filters = "\n".join(get_filters(self.ds))
if filters:
command = [shlex.split(self.cmd)] + [["grep", "-F", filters]]
raw = self.ctx.shell_out(command, split=self.split, keep_rc=self.keep_rc,
timeout=self.timeout, env=SAFE_ENV)
else:
command = [shlex.split(self.cmd)]
raw = self.ctx.shell_out(command, split=self.split, keep_rc=self.keep_rc,
timeout=self.timeout, env=SAFE_ENV)
if self.keep_rc:
self.rc, output = raw
else:
output = raw
return output
def __call__(self, broker):
result = []
source = broker[self.provider]
ctx = broker[self.context]
if isinstance(source, ContentProvider):
source = source.content
if not isinstance(source, (list, set)):
source = [source]
for e in source:
try:
the_cmd = self.cmd % e
rc = None
if self.split:
filters = "\n".join(get_filters(self))
if filters:
command = [shlex.split(the_cmd)
] + [["grep", "-F", filters]]
raw = ctx.shell_out(command,
split=self.split,
keep_rc=self.keep_rc,
timeout=self.timeout)
else:
command = [shlex.split(the_cmd)]
raw = ctx.shell_out(command,
split=self.split,
keep_rc=self.keep_rc,
timeout=self.timeout)
if self.keep_rc:
rc, output = raw
else:
output = raw
result.append(
CommandOutputProvider(the_cmd,
ctx,
args=e,
content=output,
rc=rc,
split=self.split,
keep_rc=self.keep_rc))
except:
log.debug(traceback.format_exc())
if result:
return result
raise ContentException("No results found for [%s]" % self.cmd)
def create_args(self):
command = [shlex.split(self.cmd)]
if self.split:
filters = "\n".join(get_filters(self.ds))
if filters:
command.append(["grep", "-F", filters])
patterns = "\n".join(blacklist.get_disallowed_patterns())
if patterns:
command.append(["grep", "-v", "-F", patterns])
keywords = blacklist.get_disallowed_keywords()
if keywords:
sed = ["sed"]
for kw in keywords:
sed.extend(["-e", "s/%s/keyword/g" % kw.replace("/", "\\/")])
command.append(sed)
return command
def load(self):
filters = False
if self.ds:
filters = "\n".join(get_filters(self.ds))
if filters:
cmd = [["grep", "-F", filters, self.path]]
rc, out = subproc.call(cmd,
shell=False,
keep_rc=True,
env=SAFE_ENV)
if rc == 0 and out != '':
results = out.splitlines()
else:
return []
else:
with open(self.path, "rU") as f:
results = [l.rstrip("\n") for l in f]
return results
def load(self):
filters = False
if self.ds:
filters = "\n".join(get_filters(self.ds))
if filters:
cmd = "/bin/grep -F '{0}' {1}".format(filters, self.path)
rc, out = subproc.call(cmd.encode("utf-8"), shell=False, keep_rc=True)
if rc == 0 and out != '':
results = out.splitlines()
else:
return []
else:
with open(self.path, "rU") as f:
results = [l.rstrip("\n") for l in f]
if not self.validate_lines(results):
first = results[0] if results else "<no content>"
raise ContentException(self.relative_path + ": " + first)
return results
def _stream(self):
"""
Returns a generator of lines instead of a list of lines.
"""
if self._exception:
raise self._exception
try:
if self._content:
yield self._content
raise StopIteration
filters = get_filters(self.ds) if self.ds else None
if filters:
cmd = ["grep", "-F", "\n".join(filters), self.path]
with streams.stream(cmd, env=SAFE_ENV) as s:
yield s
else:
with open(self.path, "rU") as f: # universal newlines
yield f
except StopIteration:
raise
except Exception as ex:
self._exception = ex
raise ContentException(str(ex))
def cloud_cfg(broker):
"""
This datasource provides configuration collected from ``/etc/cloud/cloud.cfg``.
Typical content of ``/etc/cloud/cloud.cfg`` file is::
#cloud-config
users:
- name: demo
ssh-authorized-keys:
- key_one
- key_two
passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
ssh_deletekeys: 1
network:
version: 1
config:
- type: physical
name: eth0
subnets:
- type: dhcp
- type: dhcp6
system_info:
default_user:
name: user2
plain_text_passwd: 'someP@assword'
home: /home/user2
debug:
output: /var/log/cloud-init-debug.log
verbose: true
Note:
This datasource may be executed using the following command:
``insights cat --no-header cloud_cfg``
Sample output in JSON format::
{
"ssh_deletekeys": 1,
"network": {
"version": 1,
"config": [
{
"type": "physical",
"name": "eth0",
"subnets": [
{
"type": "dhcp"
},
{
"type": "dhcp6"
}
]
}
]
},
"debug": {
"output": "/var/log/cloud-init-debug.log",
"verbose": true
}
}
Returns:
str: JSON string after removing the sensitive information.
Raises:
SkipComponent: When the path does not exist or any exception occurs.
"""
relative_path = '/etc/cloud/cloud.cfg'
try:
filters = get_filters(Specs.cloud_cfg)
content = broker[LocalSpecs.cloud_cfg_input].content
if content and filters:
result = dict()
content = yaml.load('\n'.join(content), Loader=yaml.SafeLoader)
if isinstance(content, dict):
# remove sensitive data
content.pop('users', None)
content.pop('system_info', None)
# apply filters
for item in filters:
if item in content:
result[item] = content[item]
if result:
result = dict(sorted(result.items(), key=lambda x: x[0]))
return DatasourceProvider(content=json.dumps(result),
relative_path=relative_path)
raise SkipComponent("Invalid YAML format")
except Exception as e:
raise SkipComponent("Unexpected exception:{e}".format(e=str(e)))
raise SkipComponent
def test_get_filter():
f = filters.get_filters(Specs.ps_aux)
assert "COMMAND" in f
f = filters.get_filters(DefaultSpecs.ps_aux)
assert "COMMAND" in f
def main():
# config = get_config()
dr.load_components("insights.specs.default")
dr.load_components("insights.parsers")
dr.load_components("insights.combiners")
dr.load_components("telemetry.rules.plugins")
dr.load_components("prodsec")
ds = dr.COMPONENTS_BY_TYPE[datasource]
specs = []
for c in ds:
if not is_datasource(c):
continue
if not any(is_datasource(d) for d in dr.get_dependents(c)):
specs.append(c)
deps = defaultdict(dict)
pspec = ''
for spec in sorted(specs, key=dr.get_name):
info = dict(name=dr.get_simple_name(spec))
f = filters.get_filters(spec)
info['dependents'] = []
spds = None
d = [d for d in dr.get_dependencies(spec) if is_datasource(d)]
for dp in d:
c = dr.get_dependencies(dp)
for cdeps in c:
if is_datasource(cdeps) and '__qualname__' in cdeps.func_dict and 'DefaultSpecs' in cdeps.func_dict['__qualname__']:
spds = cdeps
for d in dr.get_dependencies(spec):
cp = ''
lines = []
if d.__doc__ and "Returns the first" in d.__doc__:
lines = d.__doc__.replace(',', '\n')
lines = lines.splitlines()
head = [lines[0]]
top = ["<ul>"]
bottom = ["</ul>"]
if spds:
lines = [l.replace('Command:', '') for l in lines]
lines = [l.replace('Path:', '') for l in lines]
lines = ["<li>" + l + "</li>" for l in lines[1:]]
# lines = ["<li>" + spds.func_doc + ',' + l + "</li>" for l in lines[1:]]
else:
lines = ["<li>" + l + "</li>" for l in lines[1:]]
cp = "\n".join(head + top + lines + bottom)
else:
if spds:
d.__doc__ = d.__doc__.replace('Command:', '')
d.__doc__ = d.__doc__.replace('Path:', '')
d.__doc__ = spds.func_doc + ', ' + d.__doc__
cp = d.__doc__
for d in dr.get_dependents(spec):
if dr.get_simple_name(pspec) == dr.get_simple_name(d):
continue
pspec = d
p = [dr.get_name(sd) for sd in dr.get_dependents(d)]
rules = sorted([x.rsplit('.', 2)[1] for x in p])
deps[info['name']][info['name'] + "_spec-def"] = cp
deps[info['name']][info['name'] + "_rules"] = ", ".join(rules)
deps[info['name']][info['name'] + "_filters"] = f
report = Environment().from_string(REPORT).render(
report_date=datetime.date.today().strftime("%B %d, %Y"), specs=deps)
print(report)
