def testAssertCheckUndetected(self):
"""Tests for the asertCheckUndetected() method."""
anomaly = {
"finding": ["Adware 2.1.1 is installed"],
"symptom": "Found: Malicious software.",
"type": "ANALYSIS_ANOMALY"
}
# Simple no anomaly case.
no_anomaly = {"SW-CHECK": checks.CheckResult(check_id="SW-CHECK")}
self.assertCheckUndetected("SW-CHECK", no_anomaly)
# The case were there is an anomaly in the results, just not the check
# we are looking for.
other_anomaly = {
"SW-CHECK":
checks.CheckResult(check_id="SW-CHECK"),
"OTHER":
checks.CheckResult(check_id="OTHER",
anomaly=rdf_anomaly.Anomaly(**anomaly))
}
self.assertCheckUndetected("SW-CHECK", other_anomaly)
# Check the simple failure case works.
has_anomaly = {
"SW-CHECK":
checks.CheckResult(check_id="SW-CHECK",
anomaly=rdf_anomaly.Anomaly(**anomaly))
}
self.assertRaises(AssertionError, self.assertCheckUndetected,
"SW-CHECK", has_anomaly)
def testSystemGroupParserAnomaly(self):
"""Detect anomalies in group/gshadow files."""
group = [
"root:x:0:root,usr1", "adm:x:1:syslog,usr1",
"users:x:1000:usr1,usr2,usr3,usr4"
]
gshadow = ["root::usr4:root", "users:{DES}:usr1:usr2,usr3,usr4"]
stats, files = self._GenFiles(None, None, group, gshadow)
# Set up expected anomalies.
member = {
"symptom":
"Group/gshadow members differ in group: root",
"finding": [
"Present in group, missing in gshadow: usr1",
"Present in gshadow, missing in group: usr4"
],
"type":
"PARSER_ANOMALY"
}
group = {
"symptom": "Mismatched group and gshadow files.",
"finding": ["Present in group, missing in gshadow: adm"],
"type": "PARSER_ANOMALY"
}
expected = [rdf_anomaly.Anomaly(**member), rdf_anomaly.Anomaly(**group)]
parser = linux_file_parser.LinuxSystemGroupParser()
rdfs = parser.ParseMultiple(stats, files, None)
results = [r for r in rdfs if isinstance(r, rdf_anomaly.Anomaly)]
self.assertEqual(len(expected), len(results))
for expect, result in zip(expected, results):
self.assertRDFValuesEqual(expect, result)
def testSystemAccountAnomaly(self):
passwd = [
"root:x:0:0::/root:/bin/sash",
"miss:x:1000:100:Missing:/home/miss:/bin/bash",
"bad1:x:0:1001:Bad 1:/home/bad1:/bin/bash",
"bad2:x:1002:0:Bad 2:/home/bad2:/bin/bash"
]
shadow = [
"root:{UNSET}:16000:0:99999:7:::", "ok:{SHA512}:16000:0:99999:7:::",
"bad1::16333:0:99999:7:::", "bad2:{DES}:16333:0:99999:7:::"
]
group = [
"root:x:0:root", "miss:x:1000:miss", "bad1:x:1001:bad1",
"bad2:x:1002:bad2"
]
gshadow = ["root:::root", "miss:::miss", "bad1:::bad1", "bad2:::bad2"]
stats, files = self._GenFiles(passwd, shadow, group, gshadow)
no_grp = {
"symptom": "Accounts with invalid gid.",
"finding": ["gid 100 assigned without /etc/groups entry: miss"],
"type": "PARSER_ANOMALY"
}
uid = {
"symptom": "Accounts with shared uid.",
"finding": ["uid 0 assigned to multiple accounts: bad1,root"],
"type": "PARSER_ANOMALY"
}
gid = {
"symptom": "Privileged group with unusual members.",
"finding": ["Accounts in 'root' group: bad2"],
"type": "PARSER_ANOMALY"
}
no_match = {
"symptom":
"Mismatched passwd and shadow files.",
"finding": [
"Present in passwd, missing in shadow: miss",
"Present in shadow, missing in passwd: ok"
],
"type":
"PARSER_ANOMALY"
}
expected = [
rdf_anomaly.Anomaly(**no_grp), rdf_anomaly.Anomaly(**uid),
rdf_anomaly.Anomaly(**gid), rdf_anomaly.Anomaly(**no_match)
]
parser = linux_file_parser.LinuxSystemPasswdParser()
rdfs = parser.ParseMultiple(stats, files, None)
results = [r for r in rdfs if isinstance(r, rdf_anomaly.Anomaly)]
self.assertEqual(len(expected), len(results))
for expect, result in zip(expected, results):
self.assertEqual(expect.symptom, result.symptom)
# Expand out repeated field helper.
self.assertItemsEqual(list(expect.finding), list(result.finding))
self.assertEqual(expect.type, result.type)
def Issue(self, state, results):
"""Collect anomalous findings into a CheckResult.
Comparisons with anomalous conditions collect anomalies into a single
CheckResult message. The contents of the result varies depending on whether
the method making the comparison is a Check, Method or Probe.
- Probes evaluate raw host data and generate Anomalies. These are condensed
into a new CheckResult.
- Checks and Methods evaluate the results of probes (i.e. CheckResults). If
there are multiple probe results, all probe anomalies are aggregated into
a single new CheckResult for the Check or Method.
Args:
state: A text description of what combination of results were anomalous
(e.g. some condition was missing or present.)
results: Anomalies or CheckResult messages.
Returns:
A CheckResult message.
"""
result = CheckResult()
# If there are CheckResults we're aggregating methods or probes.
# Merge all current results into one CheckResult.
# Otherwise, the results are raw host data.
# Generate a new CheckResult and add the specific findings.
if results and all(isinstance(r, CheckResult) for r in results):
result.ExtendAnomalies(results)
else:
result.anomaly = rdf_anomaly.Anomaly(
type=anomaly_pb2.Anomaly.AnomalyType.Name(
anomaly_pb2.Anomaly.ANALYSIS_ANOMALY),
symptom=self.hint.Problem(state),
finding=self.hint.Render(results),
explanation=self.hint.Fix())
return result
def ParseMultiple(self, stats, file_objects, knowledge_base):
"""Parse the found release files."""
_ = knowledge_base
# Collate files into path: contents dictionary.
found_files = self._Combine(stats, file_objects)
# Determine collected files and apply weighting.
weights = [w for w in self.WEIGHTS if w.path in found_files]
weights = sorted(weights, key=lambda x: x.weight)
for _, path, handler in weights:
contents = found_files[path]
obj = handler(contents)
complete, result = obj.Parse()
if result is None:
continue
elif complete:
yield rdf_protodict.Dict({
'os_release': result.release,
'os_major_version': result.major,
'os_minor_version': result.minor
})
break
else:
# No successful parse.
yield rdf_anomaly.Anomaly(
type='PARSER_ANOMALY', symptom='Unable to determine distribution.')
def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
knowledge_base):
_ = cmd, args, stdout, stderr, return_val, time_taken, knowledge_base
installed = rdf_client.SoftwarePackage.InstallState.INSTALLED
soft = rdf_client.SoftwarePackage(
name="Package1",
description="Desc1",
version="1",
architecture="amd64",
install_state=installed)
yield soft
soft = rdf_client.SoftwarePackage(
name="Package2",
description="Desc2",
version="1",
architecture="i386",
install_state=installed)
yield soft
# Also yield something random so we can test return type filtering.
yield rdf_client.StatEntry()
# Also yield an anomaly to test that.
yield rdf_anomaly.Anomaly(
type="PARSER_ANOMALY", symptom="could not parse gremlins.")
def _CheckMultipleSymPerCheck(self, check_id, results, sym_list, found_list):
"""Ensure results for a check containing multiple symptoms match."""
anom = []
for sym, found in zip(sym_list, found_list):
anom.append(
rdf_anomaly.Anomaly(
symptom=sym, finding=found, type="ANALYSIS_ANOMALY"))
expected = checks.CheckResult(check_id=check_id, anomaly=anom)
self.assertResultEqual(expected, results[check_id])
def testExtendAnomalies(self):
anomaly1 = {
"finding": ["Adware 2.1.1 is installed"],
"explanation": "Found: Malicious software.",
"type": "ANALYSIS_ANOMALY"
}
anomaly2 = {
"finding": ["Java 6.0.240 is installed"],
"explanation": "Found: Old Java installation.",
"type": "ANALYSIS_ANOMALY"
}
result = checks.CheckResult(check_id="SW-CHECK",
anomaly=anomaly_rdf.Anomaly(**anomaly1))
other = checks.CheckResult(check_id="SW-CHECK",
anomaly=anomaly_rdf.Anomaly(**anomaly2))
result.ExtendAnomalies(other)
expect = {"check_id": "SW-CHECK", "anomaly": [anomaly1, anomaly2]}
self.assertDictEqual(expect, result.ToPrimitiveDict())
def testParse(self):
filt = filters.RDFFilter()
cfg = rdf_protodict.AttributedDict()
anom = anomaly.Anomaly()
objs = [cfg, anom]
results = filt.Parse(objs, "KnowledgeBase")
self.assertFalse(results)
results = filt.Parse(objs, "AttributedDict,KnowledgeBase")
self.assertItemsEqual([cfg], results)
results = filt.Parse(objs, "Anomaly,AttributedDict,KnowledgeBase")
self.assertItemsEqual(objs, results)
def setUp(self):
super(ProcessHostDataTests, self).setUp()
registered = checks.CheckRegistry.checks.keys()
if "SW-CHECK" not in registered:
checks.LoadChecksFromFiles([os.path.join(CHECKS_DIR, "sw.yaml")])
if "SSHD-CHECK" not in registered:
checks.LoadChecksFromFiles([os.path.join(CHECKS_DIR, "sshd.yaml")])
self.netcat = checks.CheckResult(
check_id="SW-CHECK",
anomaly=[
rdf_anomaly.Anomaly(
finding=["netcat-traditional 1.10-40 is installed"],
symptom="Found: l337 software installed",
type="ANALYSIS_ANOMALY")
])
self.sshd = checks.CheckResult(
check_id="SSHD-CHECK",
anomaly=[
rdf_anomaly.Anomaly(
finding=["Configured protocols: 2,1"],
symptom="Found: Sshd allows protocol 1.",
type="ANALYSIS_ANOMALY")
])
self.windows = checks.CheckResult(
check_id="SW-CHECK",
anomaly=[
rdf_anomaly.Anomaly(
finding=["Java 6.0.240 is installed"],
symptom="Found: Old Java installation.",
type="ANALYSIS_ANOMALY"),
rdf_anomaly.Anomaly(
finding=["Adware 2.1.1 is installed"],
symptom="Found: Malicious software.",
type="ANALYSIS_ANOMALY")
])
self.data = {
"WMIInstalledSoftware": self.SetArtifactData(parsed=GetWMIData()),
"DebianPackagesStatus": self.SetArtifactData(parsed=GetDPKGData()),
"SshdConfigFile": self.SetArtifactData(parsed=GetSSHDConfig())
}
def setUp(self):
super(ProcessHostDataTests, self).setUp()
registered = checks.CheckRegistry.checks.keys()
if "SW-CHECK" not in registered:
checks.LoadChecksFromFiles([os.path.join(CHECKS_DIR, "sw.yaml")])
if "SSHD-CHECK" not in registered:
checks.LoadChecksFromFiles([os.path.join(CHECKS_DIR, "sshd.yaml")])
self.netcat = rdfvalue.CheckResult(
check_id="SW-CHECK",
anomaly=[
anomaly_rdf.Anomaly(
finding=["netcat-traditional 1.10-40 is installed"],
explanation="Found: l337 software installed",
type="ANALYSIS_ANOMALY")
])
self.sshd = rdfvalue.CheckResult(
check_id="SSHD-CHECK",
anomaly=[
anomaly_rdf.Anomaly(
finding=["Configured protocols: 2,1"],
explanation="Found: Sshd allows protocol 1.",
type="ANALYSIS_ANOMALY")
])
self.windows = rdfvalue.CheckResult(
check_id="SW-CHECK",
anomaly=[
anomaly_rdf.Anomaly(
finding=["Java 6.0.240 is installed"],
explanation="Found: Old Java installation.",
type="ANALYSIS_ANOMALY"),
anomaly_rdf.Anomaly(finding=["Adware 2.1.1 is installed"],
explanation="Found: Malicious software.",
type="ANALYSIS_ANOMALY")
])
self.host_data = {
"WMIInstalledSoftware": WMI_SW,
"DebianPackagesStatus": DPKG_SW,
"SshdConfigFile": SSHD_CFG
}
def Parse(self, stat, file_object, knowledge_base):
"""Parse the status file."""
_, _ = stat, knowledge_base
try:
for pkg in deb822.Packages.iter_paragraphs(file_object):
if self.installed_re.match(pkg["Status"]):
soft = rdf_client.SoftwarePackage(
name=pkg["Package"],
description=pkg["Description"],
version=pkg["Version"],
architecture=pkg["Architecture"],
publisher=pkg["Maintainer"],
install_state="INSTALLED")
yield soft
except SystemError:
yield rdf_anomaly.Anomaly(type="PARSER_ANOMALY",
symptom="Invalid dpkg status file")
def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
knowledge_base):
"""Parse the rpm -qa output."""
_ = time_taken, args, knowledge_base # Unused.
rpm_re = re.compile(r"^(\w[-\w\+]+?)-(\d.*)$")
self.CheckReturn(cmd, return_val)
for line in stdout.splitlines():
pkg_match = rpm_re.match(line.strip())
if pkg_match:
name, version = pkg_match.groups()
status = rdf_client.SoftwarePackage.InstallState.INSTALLED
yield rdf_client.SoftwarePackage(name=name, version=version,
install_state=status)
for line in stderr.splitlines():
if "error: rpmdbNextIterator: skipping h#" in line:
yield rdf_anomaly.Anomaly(
type="PARSER_ANOMALY", symptom="Broken rpm database.")
break
def Parse(self, query, result, knowledge_base):
"""Parse a WMI Event Consumer."""
_ = query, knowledge_base
wmi_dict = result.ToDict()
try:
wmi_dict["CreatorSID"] = BinarySIDtoStringSID("".join(
[chr(i) for i in wmi_dict["CreatorSID"]]))
except (ValueError, TypeError) as e:
# We recover from corrupt SIDs by outputting it raw as a string
wmi_dict["CreatorSID"] = str(wmi_dict["CreatorSID"])
except KeyError as e:
pass
for output_type in self.output_types:
anomalies = []
output = rdfvalue.RDFValue.classes[output_type]()
for k, v in wmi_dict.iteritems():
try:
output.Set(k, v)
except AttributeError as e:
# Skip any attribute we don't know about
anomalies.append("Unknown field %s, with value %s" %
(k, v))
except ValueError as e:
anomalies.append("Invalid value %s for field %s: %s" %
(v, k, e))
# Yield anomalies first to help with debugging
if anomalies:
yield rdf_anomaly.Anomaly(type="PARSER_ANOMALY",
generated_by=self.__class__.__name__,
finding=anomalies)
# Raise if the parser generated no output but there were fields.
if wmi_dict and not output:
raise ValueError("Non-empty dict %s returned empty output.",
wmi_dict)
yield output
def Parse(self, stat, file_obj, unused_knowledge_base):
lines = set([l.strip() for l in file_obj.read(100000).splitlines()])
users = []
bad_lines = []
for line in lines:
if " " in line: # behaviour of At/Cron is undefined for lines
bad_lines.append(line) # with whitespace separated fields/usernames
elif line: # drop empty lines
users.append(line)
filename = stat.pathspec.path
cfg = {"filename": filename, "users": users}
yield rdf_protodict.AttributedDict(**cfg)
if bad_lines:
yield rdf_anomaly.Anomaly(type="PARSER_ANOMALY",
symptom="Dodgy entries in %s." % (filename),
reference_pathspec=stat.pathspec,
finding=bad_lines)
def ParseLines(cls, lines):
users = set()
filter_regexes = [
re.compile(x)
for x in config_lib.CONFIG["Artifacts.netgroup_filter_regexes"]
]
username_regex = re.compile(cls.USERNAME_REGEX)
blacklist = config_lib.CONFIG["Artifacts.netgroup_user_blacklist"]
for index, line in enumerate(lines):
if line.startswith("#"):
continue
splitline = line.split(" ")
group_name = splitline[0]
if filter_regexes:
filter_match = False
for regex in filter_regexes:
if regex.search(group_name):
filter_match = True
break
if not filter_match:
continue
for member in splitline[1:]:
if member.startswith("("):
try:
_, user, _ = member.split(",")
if user not in users and user not in blacklist:
if not username_regex.match(user):
yield anomaly.Anomaly(
type="PARSER_ANOMALY",
symptom="Invalid username: %s" % user)
else:
users.add(user)
yield client.KnowledgeBaseUser(
username=utils.SmartUnicode(user))
except ValueError:
raise parsers.ParseError(
"Invalid netgroup file at line %d: %s" %
(index + 1, line))
def ParseMultiple(self, stats, unused_file_obj, unused_kb):
"""Identify the init scripts and the start/stop scripts at each runlevel.
Evaluate all the stat entries collected from the system.
If the path name matches a runlevel spec, and if the filename matches a
sysv init symlink process the link as a service.
Args:
stats: An iterator of StatEntry rdfs.
unused_file_obj: An iterator of file contents. Not needed as the parser
only evaluates link attributes.
unused_kb: Unused KnowledgeBase rdf.
Yields:
rdf_anomaly.Anomaly if the startup link seems wierd.
rdf_client.LinuxServiceInformation for each detected service.
"""
services = {}
for stat_entry in stats:
path = stat_entry.pathspec.path
runlevel = self.runlevel_re.match(os.path.dirname(path))
runscript = self.runscript_re.match(os.path.basename(path))
if runlevel and runscript:
svc = runscript.groupdict()
service = services.setdefault(
svc["name"],
rdf_client.LinuxServiceInformation(name=svc["name"],
start_mode="INIT"))
runlvl = GetRunlevelNums(runlevel.group(1))
if svc["action"] == "S" and runlvl:
service.start_on.append(runlvl[0])
service.starts = True
elif runlvl:
service.stop_on.append(runlvl[0])
if not stat.S_ISLNK(int(stat_entry.st_mode)):
yield rdf_anomaly.Anomaly(
type="PARSER_ANOMALY",
finding=[path],
explanation="Startup script is not a symlink.")
for svc in services.itervalues():
yield svc
def _Anomaly(self, msg, found):
return anomaly.Anomaly(type="PARSER_ANOMALY",
symptom=msg,
finding=found)
def testAssertCheckDetectedAnom(self):
"""Tests for the assertCheckDetectedAnom() method."""
# Check we fail when our checkid isn't in the results.
no_checks = {}
self.assertRaises(AssertionError,
self.assertCheckDetectedAnom,
"UNICORN",
no_checks,
sym=None,
findings=None)
# Check we fail when our checkid is in the results but hasn't
# produced an anomaly.
passing_checks = {"EXISTS": checks.CheckResult(check_id="EXISTS")}
self.assertRaises(AssertionError,
self.assertCheckDetectedAnom,
"EXISTS",
passing_checks,
sym=None,
findings=None)
# On to a 'successful' cases.
anomaly = {
"finding": ["Finding"],
"symptom": "Found: An issue.",
"type": "ANALYSIS_ANOMALY"
}
failing_checks = {
"EXISTS":
checks.CheckResult(check_id="EXISTS",
anomaly=rdf_anomaly.Anomaly(**anomaly))
}
# Check we pass when our check produces an anomaly and we don't care
# about the details.
self.assertCheckDetectedAnom("EXISTS",
failing_checks,
sym=None,
findings=None)
# When we do care only about the 'symptom'.
self.assertCheckDetectedAnom("EXISTS",
failing_checks,
sym="Found: An issue.",
findings=None)
# And when we also care about the findings.
self.assertCheckDetectedAnom("EXISTS",
failing_checks,
sym="Found: An issue.",
findings=["Finding"])
# And check we match substrings of a 'finding'.
self.assertCheckDetectedAnom("EXISTS",
failing_checks,
sym="Found: An issue.",
findings=["Fin"])
# Check we complain when the symptom doesn't match.
self.assertRaises(AssertionError,
self.assertCheckDetectedAnom,
"EXISTS",
failing_checks,
sym="wrong symptom",
findings=None)
# Check we complain when the symptom matches but the findings don't.
self.assertRaises(AssertionError,
self.assertCheckDetectedAnom,
"EXISTS",
failing_checks,
sym="Found: An issue.",
findings=["Not found"])
# Lastly, if there is a finding in the anomaly we didn't expect, we consider
# that a problem.
self.assertRaises(AssertionError,
self.assertCheckDetectedAnom,
"EXISTS",
failing_checks,
sym="Found: An issue.",
findings=[])
