def testUserMergeWindows(self):
"""Check Windows users are accurately merged."""
kb = rdfvalue.KnowledgeBase()
self.assertEqual(len(kb.users), 0)
kb.MergeOrAddUser(rdfvalue.KnowledgeBaseUser(sid="1234"))
self.assertEqual(len(kb.users), 1)
kb.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(sid="5678", username="******"))
self.assertEqual(len(kb.users), 2)
_, conflicts = kb.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(sid="5678", username="******"))
self.assertEqual(len(kb.users), 2)
self.assertEqual(conflicts[0], ("username", "test1", "test2"))
self.assertEqual(kb.GetUser(sid="5678").username, "test2")
# This should merge on user name as we have no other data.
kb.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******", homedir="a"))
self.assertEqual(len(kb.users), 2)
# This should create a new user since the sid is different.
new_attrs, conflicts = kb.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******",
sid="12345",
temp="/blah"))
self.assertEqual(len(kb.users), 3)
self.assertItemsEqual(new_attrs,
["users.username", "users.temp", "users.sid"])
self.assertEqual(conflicts, [])
def testInterpolateArgs(self):
collect_flow = collectors.ArtifactCollectorFlow(None, token=self.token)
collect_flow.state.Register("knowledge_base", rdfvalue.KnowledgeBase())
collect_flow.current_artifact_name = "blah"
collect_flow.state.knowledge_base.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******"))
collect_flow.state.knowledge_base.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******"))
test_rdf = rdfvalue.KnowledgeBase()
action_args = {"usernames": ["%%users.username%%", "%%users.username%%"],
"nointerp": "asdfsdf", "notastring": test_rdf}
kwargs = collect_flow.InterpolateDict(action_args)
self.assertItemsEqual(kwargs["usernames"],
["test1", "test2", "test1", "test2"])
self.assertEqual(kwargs["nointerp"], "asdfsdf")
self.assertEqual(kwargs["notastring"], test_rdf)
# We should be using an array since users.username will expand to multiple
# values.
self.assertRaises(ValueError, collect_flow.InterpolateDict,
{"bad": "%%users.username%%"})
list_args = collect_flow.InterpolateList(["%%users.username%%",
"%%users.username%%aa"])
self.assertItemsEqual(list_args, ["test1", "test2", "test1aa", "test2aa"])
list_args = collect_flow.InterpolateList(["one"])
self.assertEqual(list_args, ["one"])
def testInterpolation(self):
"""Check we can interpolate values from the knowledge base."""
kb = rdfvalue.KnowledgeBase()
self.assertRaises(
artifact_lib.KnowledgeBaseInterpolationError, list,
artifact_lib.InterpolateKbAttributes("test%%users.username%%test",
kb))
kb.users.Append(rdfvalue.KnowledgeBaseUser(username="******", uid=1))
kb.users.Append(rdfvalue.KnowledgeBaseUser(username="******", uid=2))
kb.Set("environ_allusersprofile", "c:\\programdata")
paths = artifact_lib.InterpolateKbAttributes(
"test%%users.username%%test", kb)
paths = list(paths)
self.assertEqual(len(paths), 2)
self.assertItemsEqual(paths, ["testjoetest", "testjimtest"])
paths = artifact_lib.InterpolateKbAttributes(
"%%environ_allusersprofile%%\\a", kb)
self.assertEqual(list(paths), ["c:\\programdata\\a"])
self.assertRaises(
artifact_lib.KnowledgeBaseInterpolationError, list,
artifact_lib.InterpolateKbAttributes("%%nonexistent%%\\a", kb))
kb.Set("environ_allusersprofile", "")
self.assertRaises(
artifact_lib.KnowledgeBaseInterpolationError, list,
artifact_lib.InterpolateKbAttributes(
"%%environ_allusersprofile%%\\a", kb))
def testGrep(self):
class MockCallFlow(object):
def CallFlow(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
mock_call_flow = MockCallFlow()
with utils.Stubber(collectors.ArtifactCollectorFlow, "CallFlow",
mock_call_flow.CallFlow):
collect_flow = collectors.ArtifactCollectorFlow(None, token=self.token)
collect_flow.state.Register("knowledge_base", rdfvalue.KnowledgeBase())
collect_flow.current_artifact_name = "blah"
collect_flow.state.knowledge_base.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******"))
collect_flow.state.knowledge_base.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******"))
collector = rdfvalue.Collector(
collector_type=rdfvalue.Collector.CollectorType.GREP,
args={"path_list": ["/etc/passwd"],
"content_regex_list": [r"^a%%users.username%%b$"]})
collect_flow.Grep(collector, rdfvalue.PathSpec.PathType.TSK)
conditions = mock_call_flow.kwargs["conditions"]
self.assertEqual(len(conditions), 1)
regexes = conditions[0].contents_regex_match.regex.SerializeToString()
self.assertItemsEqual(regexes.split("|"), ["(^atest1b$)", "(^atest2b$)"])
self.assertEqual(mock_call_flow.kwargs["paths"], ["/etc/passwd"])
def testGrep(self):
class MockCallFlow(object):
def CallFlow(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
mock_call_flow = MockCallFlow()
with test_lib.Stubber(collectors.ArtifactCollectorFlow, "CallFlow",
mock_call_flow.CallFlow):
collect_flow = collectors.ArtifactCollectorFlow(None,
token=self.token)
collect_flow.state.Register("knowledge_base",
rdfvalue.KnowledgeBase())
collect_flow.current_artifact_name = "blah"
collect_flow.state.knowledge_base.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******"))
collect_flow.state.knowledge_base.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******"))
collector = rdfvalue.Collector(action="Grep",
args={
"path_list": ["/etc/passwd"],
"content_regex_list":
[r"^a%%users.username%%b$"]
})
collect_flow.Grep(collector, rdfvalue.PathSpec.PathType.TSK)
filters = mock_call_flow.kwargs["filters"]
regexes = [
f.contents_regex_match.regex.SerializeToString() for f in filters
]
self.assertItemsEqual(regexes, [r"^atest1b$", r"^atest2b$"])
self.assertEqual(mock_call_flow.kwargs["paths"], ["/etc/passwd"])
def testUserMergeLinux(self):
"""Check Linux users are accurately merged."""
kb = rdfvalue.KnowledgeBase()
self.assertEqual(len(kb.users), 0)
kb.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******", last_logon=1111))
self.assertEqual(len(kb.users), 1)
# This should merge since the username is the same.
kb.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(uid="12", username="******"))
self.assertEqual(len(kb.users), 1)
# This should create a new record because the uid is different
kb.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******",
uid="13",
desktop="/home/blake/Desktop"))
self.assertEqual(len(kb.users), 2)
kb.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******",
uid="14",
desktop="/home/blake/Desktop"))
self.assertEqual(len(kb.users), 3)
# Check merging where we don't specify uid works
new_attrs, conflicts = kb.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******",
desktop="/home/blakey/Desktop"))
self.assertEqual(len(kb.users), 3)
self.assertItemsEqual(new_attrs, ["users.username", "users.desktop"])
self.assertItemsEqual(
conflicts,
[("desktop", u"/home/blake/Desktop", u"/home/blakey/Desktop")])
def testInterpolation(self):
"""Check we can interpolate values from the knowledge base."""
kb = rdfvalue.KnowledgeBase()
# No users yet, this should raise
self.assertRaises(
artifact_lib.KnowledgeBaseInterpolationError, list,
artifact_lib.InterpolateKbAttributes("test%%users.username%%test",
kb))
# Now we have two users
kb.users.Append(rdfvalue.KnowledgeBaseUser(username="******", uid=1))
kb.users.Append(rdfvalue.KnowledgeBaseUser(username="******", uid=2))
kb.Set("environ_allusersprofile", "c:\\programdata")
paths = artifact_lib.InterpolateKbAttributes(
"test%%users.username%%test", kb)
paths = list(paths)
self.assertEqual(len(paths), 2)
self.assertItemsEqual(paths, ["testjoetest", "testjimtest"])
paths = artifact_lib.InterpolateKbAttributes(
"%%environ_allusersprofile%%\\a", kb)
self.assertEqual(list(paths), ["c:\\programdata\\a"])
# Check a bad attribute raises
self.assertRaises(
artifact_lib.KnowledgeBaseInterpolationError, list,
artifact_lib.InterpolateKbAttributes("%%nonexistent%%\\a", kb))
# Empty values should also raise
kb.Set("environ_allusersprofile", "")
self.assertRaises(
artifact_lib.KnowledgeBaseInterpolationError, list,
artifact_lib.InterpolateKbAttributes(
"%%environ_allusersprofile%%\\a", kb))
# No users have temp defined, so this should raise
self.assertRaises(
artifact_lib.KnowledgeBaseInterpolationError, list,
artifact_lib.InterpolateKbAttributes("%%users.temp%%\\a", kb))
# One user has users.temp defined, the others do not. This is common on
# windows where users have been created but have never logged in. We should
# get just one value back.
kb.users.Append(
rdfvalue.KnowledgeBaseUser(
username="******",
uid=1,
temp="C:\\Users\\jason\\AppData\\Local\\Temp"))
paths = artifact_lib.InterpolateKbAttributes(r"%%users.temp%%\abcd",
kb)
self.assertItemsEqual(paths,
["C:\\Users\\jason\\AppData\\Local\\Temp\\abcd"])
def setUp(self):
"""Make sure things are initialized."""
super(ArtifactFlowTest, self).setUp()
fd = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw")
fd.Set(fd.Schema.SYSTEM("Linux"))
kb = fd.Schema.KNOWLEDGE_BASE()
artifact.SetCoreGRRKnowledgeBaseValues(kb, fd)
kb.MergeOrAddUser(rdfvalue.KnowledgeBaseUser(username="******"))
kb.MergeOrAddUser(rdfvalue.KnowledgeBaseUser(username="******"))
kb.MergeOrAddUser(rdfvalue.KnowledgeBaseUser(username="******"))
fd.Set(kb)
fd.Flush()
self.LoadTestArtifacts()
def ParseMultiple(self, stats, knowledge_base):
"""Parse each returned registry value."""
user_dict = {}
for stat in stats:
sid_str = stat.pathspec.path.split("/", 3)[2]
if SID_RE.match(sid_str):
if sid_str not in user_dict:
user_dict[sid_str] = rdfvalue.KnowledgeBaseUser(
sid=sid_str)
if stat.registry_data.GetValue():
# Look up in the mapping if we can use this entry to populate a user
# attribute, and if so, set it.
reg_key_name = stat.pathspec.Dirname().Basename()
if reg_key_name in self.key_var_mapping:
map_dict = self.key_var_mapping[reg_key_name]
reg_key = stat.pathspec.Basename()
kb_attr = map_dict.get(reg_key)
if kb_attr:
value = artifact_lib.ExpandWindowsEnvironmentVariables(
stat.registry_data.GetValue(), knowledge_base)
value = artifact_lib.ExpandWindowsUserEnvironmentVariables(
value, knowledge_base, sid=sid_str)
user_dict[sid_str].Set(kb_attr, value)
# Now yield each user we found.
return user_dict.itervalues()
def Parse(self, stat, knowledge_base):
"""Parse each returned registry value."""
_ = knowledge_base # Unused.
sid_str = stat.pathspec.Dirname().Basename()
if SID_RE.match(sid_str):
kb_user = rdfvalue.KnowledgeBaseUser()
kb_user.sid = sid_str
if stat.pathspec.Basename() == "ProfileImagePath":
if stat.resident:
# Support old clients.
kb_user.homedir = utils.SmartUnicode(stat.resident)
else:
kb_user.homedir = stat.registry_data.GetValue()
kb_user.userprofile = kb_user.homedir
try:
# Assume username is the last component of the path. This is not
# robust, but other user artifacts will override it if there is a
# better match.
kb_user.username = kb_user.homedir.rsplit("\\", 1)[-1]
except IndexError:
pass
yield kb_user
def testFindsKeyWithInterpolatedGlobWithoutConditions(self):
# Initialize client's knowledge base in order for the interpolation
# to work.
user = rdfvalue.KnowledgeBaseUser(
sid="S-1-5-21-2911950750-476812067-1487428992-1001")
kb = rdfvalue.KnowledgeBase(users=[user])
with aff4.FACTORY.Open(self.client_id, mode="rw",
token=self.token) as client:
client.Set(client.Schema.KNOWLEDGE_BASE, kb)
self.RunFlow([
"HKEY_USERS/%%users.sid%%/Software/Microsoft/Windows/"
"CurrentVersion/*"
])
results = self.GetResults()
self.assertEqual(len(results), 1)
key = ("/HKEY_USERS/S-1-5-21-2911950750-476812067-1487428992-1001/"
"Software/Microsoft/Windows/CurrentVersion/Explorer")
self.assertEqual(results[0].stat_entry.aff4path,
"aff4:/C.1000000000000000/registry" + key)
self.assertEqual(results[0].stat_entry.pathspec.path, key)
self.assertEqual(results[0].stat_entry.pathspec.pathtype,
rdfvalue.PathSpec.PathType.REGISTRY)
def Parse(self, stat, file_object, knowledge_base):
"""Parse the wtmp file."""
_, _ = stat, knowledge_base
users = {}
wtmp = file_object.read(10000000)
while wtmp:
try:
record = UtmpStruct(wtmp)
except RuntimeError:
break
wtmp = wtmp[record.size:]
# Users only appear for USER_PROCESS events, others are system.
if record.ut_type != 7:
continue
# Lose the null termination
record.user = record.user.split("\x00", 1)[0]
# Store the latest login time.
# TODO(user): remove the 0 here once RDFDatetime can support times
# pre-epoch properly.
try:
users[record.user] = max(users[record.user], record.sec, 0)
except KeyError:
users[record.user] = record.sec
for user, last_login in users.iteritems():
yield rdfvalue.KnowledgeBaseUser(username=utils.SmartUnicode(user),
last_logon=last_login * 1000000)
def GetExpectedUser(self, algo, user_store, group_store):
user = rdfvalue.KnowledgeBaseUser(username="******", full_name="User",
uid="1001", gid="1001",
homedir="/home/user", shell="/bin/bash")
user.pw_entry = rdfvalue.PwEntry(store=user_store, hash_type=algo)
user.gids = [1001]
grp = rdfvalue.Group(gid=1001, members=["user"], name="user")
grp.pw_entry = rdfvalue.PwEntry(store=group_store, hash_type=algo)
return user, grp
def Parse(self, query, result, knowledge_base):
"""Parse the wmi Win32_UserAccount output."""
_ = query, knowledge_base
kb_user = rdfvalue.KnowledgeBaseUser()
for wmi_key, kb_key in self.account_mapping.items():
try:
kb_user.Set(kb_key, result[wmi_key])
except KeyError:
pass
yield kb_user
def testRdfFormatterFanOut(self):
rdf = rdfvalue.Dict()
user1 = rdfvalue.KnowledgeBaseUser(username="******")
user2 = rdfvalue.KnowledgeBaseUser(username="******")
rdf["cataclysm"] = "GreyGoo"
rdf["thinkers"] = [user1, user2]
rdf["reference"] = {
"ecophage": ["bots", ["nanobots", ["picobots"]]],
"doomsday": {
"books": ["cats cradle", "prey"]
}
}
template = ("{cataclysm}; {thinkers.username}; {reference.ecophage}; "
"{reference.doomsday}")
hinter = hints.Hinter(template=template)
expected = ("GreyGoo; drexler,joy; bots,nanobots,picobots; "
"books:cats cradle,prey")
result = hinter.Render(rdf)
self.assertEqual(expected, result)
def Parse(self, stat_entries, knowledge_base, path_type):
"""Parse the StatEntry objects."""
_, _ = knowledge_base, path_type
for stat_entry in stat_entries:
if stat.S_ISDIR(stat_entry.st_mode):
homedir = stat_entry.pathspec.path
username = os.path.basename(homedir)
if username not in self.blacklist:
yield rdfvalue.KnowledgeBaseUser(username=username,
homedir=homedir)
def testUserMerge(self):
"""Check users are accurately merged."""
kb = rdfvalue.KnowledgeBase()
self.assertEquals(len(kb.users), 0)
kb.MergeOrAddUser(rdfvalue.KnowledgeBaseUser(sid="1234"))
self.assertEquals(len(kb.users), 1)
kb.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(sid="5678", username="******"))
self.assertEquals(len(kb.users), 2)
_, conflicts = kb.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(sid="5678", username="******"))
self.assertEquals(len(kb.users), 2)
self.assertEquals(conflicts[0], ("username", "test1", "test2"))
self.assertEquals(kb.GetUser(sid="5678").username, "test2")
# This should merge on user name as we have no other data.
kb.MergeOrAddUser(
rdfvalue.KnowledgeBaseUser(username="******", homedir="a"))
self.assertEquals(len(kb.users), 2)
def ToKnowledgeBaseUser(self):
"""Convert a User value into a KnowledgeBaseUser value."""
kb_user = rdfvalue.KnowledgeBaseUser()
for old_pb_name, new_pb_name in self.kb_user_mapping.items():
if len(old_pb_name.split(".")) > 1:
attr, old_pb_name = old_pb_name.split(".", 1)
val = getattr(getattr(self, attr), old_pb_name)
else:
val = getattr(self, old_pb_name)
if val:
kb_user.Set(new_pb_name, val)
return kb_user
def testConvertFromKnowledgeBaseUser(self):
kbuser = rdfvalue.KnowledgeBaseUser(username="******", userdomain="test.com",
homedir="/usr/local/test",
desktop="/usr/local/test/Desktop",
localappdata="/usr/local/test/AppData")
user = rdfvalue.User().FromKnowledgeBaseUser(kbuser)
self.assertEqual(user.username, "test")
self.assertEqual(user.domain, "test.com")
self.assertEqual(user.homedir, "/usr/local/test")
self.assertEqual(user.special_folders.desktop, "/usr/local/test/Desktop")
self.assertEqual(user.special_folders.local_app_data,
"/usr/local/test/AppData")
def Parse(self, query, result, knowledge_base):
"""Parse the wmi Win32_UserAccount output."""
_ = query, knowledge_base
kb_user = rdfvalue.KnowledgeBaseUser()
for wmi_key, kb_key in self.account_mapping.items():
try:
kb_user.Set(kb_key, result[wmi_key])
except KeyError:
pass
# We need at least a sid or a username. If these are missing its likely we
# retrieved just the userdomain for an AD account that has a name collision
# with a local account that is correctly populated. We drop the bogus
# domain account.
if kb_user.sid or kb_user.username:
yield kb_user
def ParseLine(cls, index, line):
fields = "username,password,uid,gid,fullname,homedir,shell".split(",")
try:
if not line: return
dat = dict(zip(fields, line.split(":")))
user = rdfvalue.KnowledgeBaseUser(username=dat["username"],
uid=int(dat["uid"]),
homedir=dat["homedir"],
shell=dat["shell"],
gid=int(dat["gid"]),
full_name=dat["fullname"])
return user
except (IndexError, KeyError):
raise parsers.ParseError("Invalid passwd file at line %d. %s" %
((index + 1), line))
def setUp(self):
super(TestWebHistoryWithArtifacts, self).setUp()
self.SetLinuxClient()
fd = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw")
self.kb = fd.Schema.KNOWLEDGE_BASE()
self.kb.users.Append(
rdfvalue.KnowledgeBaseUser(username="******",
full_name="test user",
homedir="/home/test/",
last_logon=250))
self.kb.os = "Linux"
fd.AddAttribute(fd.Schema.KNOWLEDGE_BASE, self.kb)
fd.Flush()
self.client_mock = action_mocks.ActionMock(
"ReadBuffer", "FingerprintFile", "HashBuffer", "TransferBuffer",
"StatFile", "Find", "ListDirectory")
def ParseLines(cls, lines):
users = set()
filter_regexes = [
re.compile(x)
for x in config_lib.CONFIG["Artifacts.netgroup_filter_regexes"]
]
username_regex = re.compile(cls.USERNAME_REGEX)
blacklist = config_lib.CONFIG["Artifacts.netgroup_user_blacklist"]
for index, line in enumerate(lines):
if line.startswith("#"):
continue
splitline = line.split(" ")
group_name = splitline[0]
if filter_regexes:
filter_match = False
for regex in filter_regexes:
if regex.search(group_name):
filter_match = True
break
if not filter_match:
continue
for member in splitline[1:]:
if member.startswith("("):
try:
_, user, _ = member.split(",")
if user not in users and user not in blacklist:
if not username_regex.match(user):
yield rdfvalue.Anomaly(
type="PARSER_ANOMALY",
symptom="Invalid username: %s" % user)
else:
users.add(user)
yield rdfvalue.KnowledgeBaseUser(
username=utils.SmartUnicode(user))
except ValueError:
raise parsers.ParseError(
"Invalid netgroup file at line %d: %s" %
(index + 1, line))