def testUserMergeWindows(self): """Check Windows users are accurately merged.""" kb = rdfvalue.KnowledgeBase() self.assertEqual(len(kb.users), 0) kb.MergeOrAddUser(rdfvalue.KnowledgeBaseUser(sid="1234")) self.assertEqual(len(kb.users), 1) kb.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(sid="5678", username="******")) self.assertEqual(len(kb.users), 2) _, conflicts = kb.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(sid="5678", username="******")) self.assertEqual(len(kb.users), 2) self.assertEqual(conflicts[0], ("username", "test1", "test2")) self.assertEqual(kb.GetUser(sid="5678").username, "test2") # This should merge on user name as we have no other data. kb.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(username="******", homedir="a")) self.assertEqual(len(kb.users), 2) # This should create a new user since the sid is different. new_attrs, conflicts = kb.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(username="******", sid="12345", temp="/blah")) self.assertEqual(len(kb.users), 3) self.assertItemsEqual(new_attrs, ["users.username", "users.temp", "users.sid"]) self.assertEqual(conflicts, [])
def testInterpolateArgs(self): collect_flow = collectors.ArtifactCollectorFlow(None, token=self.token) collect_flow.state.Register("knowledge_base", rdfvalue.KnowledgeBase()) collect_flow.current_artifact_name = "blah" collect_flow.state.knowledge_base.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(username="******")) collect_flow.state.knowledge_base.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(username="******")) test_rdf = rdfvalue.KnowledgeBase() action_args = {"usernames": ["%%users.username%%", "%%users.username%%"], "nointerp": "asdfsdf", "notastring": test_rdf} kwargs = collect_flow.InterpolateDict(action_args) self.assertItemsEqual(kwargs["usernames"], ["test1", "test2", "test1", "test2"]) self.assertEqual(kwargs["nointerp"], "asdfsdf") self.assertEqual(kwargs["notastring"], test_rdf) # We should be using an array since users.username will expand to multiple # values. self.assertRaises(ValueError, collect_flow.InterpolateDict, {"bad": "%%users.username%%"}) list_args = collect_flow.InterpolateList(["%%users.username%%", "%%users.username%%aa"]) self.assertItemsEqual(list_args, ["test1", "test2", "test1aa", "test2aa"]) list_args = collect_flow.InterpolateList(["one"]) self.assertEqual(list_args, ["one"])
def testInterpolation(self): """Check we can interpolate values from the knowledge base.""" kb = rdfvalue.KnowledgeBase() self.assertRaises( artifact_lib.KnowledgeBaseInterpolationError, list, artifact_lib.InterpolateKbAttributes("test%%users.username%%test", kb)) kb.users.Append(rdfvalue.KnowledgeBaseUser(username="******", uid=1)) kb.users.Append(rdfvalue.KnowledgeBaseUser(username="******", uid=2)) kb.Set("environ_allusersprofile", "c:\\programdata") paths = artifact_lib.InterpolateKbAttributes( "test%%users.username%%test", kb) paths = list(paths) self.assertEqual(len(paths), 2) self.assertItemsEqual(paths, ["testjoetest", "testjimtest"]) paths = artifact_lib.InterpolateKbAttributes( "%%environ_allusersprofile%%\\a", kb) self.assertEqual(list(paths), ["c:\\programdata\\a"]) self.assertRaises( artifact_lib.KnowledgeBaseInterpolationError, list, artifact_lib.InterpolateKbAttributes("%%nonexistent%%\\a", kb)) kb.Set("environ_allusersprofile", "") self.assertRaises( artifact_lib.KnowledgeBaseInterpolationError, list, artifact_lib.InterpolateKbAttributes( "%%environ_allusersprofile%%\\a", kb))
def testGrep(self): class MockCallFlow(object): def CallFlow(self, *args, **kwargs): self.args = args self.kwargs = kwargs mock_call_flow = MockCallFlow() with utils.Stubber(collectors.ArtifactCollectorFlow, "CallFlow", mock_call_flow.CallFlow): collect_flow = collectors.ArtifactCollectorFlow(None, token=self.token) collect_flow.state.Register("knowledge_base", rdfvalue.KnowledgeBase()) collect_flow.current_artifact_name = "blah" collect_flow.state.knowledge_base.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(username="******")) collect_flow.state.knowledge_base.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(username="******")) collector = rdfvalue.Collector( collector_type=rdfvalue.Collector.CollectorType.GREP, args={"path_list": ["/etc/passwd"], "content_regex_list": [r"^a%%users.username%%b$"]}) collect_flow.Grep(collector, rdfvalue.PathSpec.PathType.TSK) conditions = mock_call_flow.kwargs["conditions"] self.assertEqual(len(conditions), 1) regexes = conditions[0].contents_regex_match.regex.SerializeToString() self.assertItemsEqual(regexes.split("|"), ["(^atest1b$)", "(^atest2b$)"]) self.assertEqual(mock_call_flow.kwargs["paths"], ["/etc/passwd"])
def testGrep(self): class MockCallFlow(object): def CallFlow(self, *args, **kwargs): self.args = args self.kwargs = kwargs mock_call_flow = MockCallFlow() with test_lib.Stubber(collectors.ArtifactCollectorFlow, "CallFlow", mock_call_flow.CallFlow): collect_flow = collectors.ArtifactCollectorFlow(None, token=self.token) collect_flow.state.Register("knowledge_base", rdfvalue.KnowledgeBase()) collect_flow.current_artifact_name = "blah" collect_flow.state.knowledge_base.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(username="******")) collect_flow.state.knowledge_base.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(username="******")) collector = rdfvalue.Collector(action="Grep", args={ "path_list": ["/etc/passwd"], "content_regex_list": [r"^a%%users.username%%b$"] }) collect_flow.Grep(collector, rdfvalue.PathSpec.PathType.TSK) filters = mock_call_flow.kwargs["filters"] regexes = [ f.contents_regex_match.regex.SerializeToString() for f in filters ] self.assertItemsEqual(regexes, [r"^atest1b$", r"^atest2b$"]) self.assertEqual(mock_call_flow.kwargs["paths"], ["/etc/passwd"])
def testUserMergeLinux(self): """Check Linux users are accurately merged.""" kb = rdfvalue.KnowledgeBase() self.assertEqual(len(kb.users), 0) kb.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(username="******", last_logon=1111)) self.assertEqual(len(kb.users), 1) # This should merge since the username is the same. kb.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(uid="12", username="******")) self.assertEqual(len(kb.users), 1) # This should create a new record because the uid is different kb.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(username="******", uid="13", desktop="/home/blake/Desktop")) self.assertEqual(len(kb.users), 2) kb.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(username="******", uid="14", desktop="/home/blake/Desktop")) self.assertEqual(len(kb.users), 3) # Check merging where we don't specify uid works new_attrs, conflicts = kb.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(username="******", desktop="/home/blakey/Desktop")) self.assertEqual(len(kb.users), 3) self.assertItemsEqual(new_attrs, ["users.username", "users.desktop"]) self.assertItemsEqual( conflicts, [("desktop", u"/home/blake/Desktop", u"/home/blakey/Desktop")])
def testInterpolation(self): """Check we can interpolate values from the knowledge base.""" kb = rdfvalue.KnowledgeBase() # No users yet, this should raise self.assertRaises( artifact_lib.KnowledgeBaseInterpolationError, list, artifact_lib.InterpolateKbAttributes("test%%users.username%%test", kb)) # Now we have two users kb.users.Append(rdfvalue.KnowledgeBaseUser(username="******", uid=1)) kb.users.Append(rdfvalue.KnowledgeBaseUser(username="******", uid=2)) kb.Set("environ_allusersprofile", "c:\\programdata") paths = artifact_lib.InterpolateKbAttributes( "test%%users.username%%test", kb) paths = list(paths) self.assertEqual(len(paths), 2) self.assertItemsEqual(paths, ["testjoetest", "testjimtest"]) paths = artifact_lib.InterpolateKbAttributes( "%%environ_allusersprofile%%\\a", kb) self.assertEqual(list(paths), ["c:\\programdata\\a"]) # Check a bad attribute raises self.assertRaises( artifact_lib.KnowledgeBaseInterpolationError, list, artifact_lib.InterpolateKbAttributes("%%nonexistent%%\\a", kb)) # Empty values should also raise kb.Set("environ_allusersprofile", "") self.assertRaises( artifact_lib.KnowledgeBaseInterpolationError, list, artifact_lib.InterpolateKbAttributes( "%%environ_allusersprofile%%\\a", kb)) # No users have temp defined, so this should raise self.assertRaises( artifact_lib.KnowledgeBaseInterpolationError, list, artifact_lib.InterpolateKbAttributes("%%users.temp%%\\a", kb)) # One user has users.temp defined, the others do not. This is common on # windows where users have been created but have never logged in. We should # get just one value back. kb.users.Append( rdfvalue.KnowledgeBaseUser( username="******", uid=1, temp="C:\\Users\\jason\\AppData\\Local\\Temp")) paths = artifact_lib.InterpolateKbAttributes(r"%%users.temp%%\abcd", kb) self.assertItemsEqual(paths, ["C:\\Users\\jason\\AppData\\Local\\Temp\\abcd"])
def setUp(self): """Make sure things are initialized.""" super(ArtifactFlowTest, self).setUp() fd = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw") fd.Set(fd.Schema.SYSTEM("Linux")) kb = fd.Schema.KNOWLEDGE_BASE() artifact.SetCoreGRRKnowledgeBaseValues(kb, fd) kb.MergeOrAddUser(rdfvalue.KnowledgeBaseUser(username="******")) kb.MergeOrAddUser(rdfvalue.KnowledgeBaseUser(username="******")) kb.MergeOrAddUser(rdfvalue.KnowledgeBaseUser(username="******")) fd.Set(kb) fd.Flush() self.LoadTestArtifacts()
def ParseMultiple(self, stats, knowledge_base): """Parse each returned registry value.""" user_dict = {} for stat in stats: sid_str = stat.pathspec.path.split("/", 3)[2] if SID_RE.match(sid_str): if sid_str not in user_dict: user_dict[sid_str] = rdfvalue.KnowledgeBaseUser( sid=sid_str) if stat.registry_data.GetValue(): # Look up in the mapping if we can use this entry to populate a user # attribute, and if so, set it. reg_key_name = stat.pathspec.Dirname().Basename() if reg_key_name in self.key_var_mapping: map_dict = self.key_var_mapping[reg_key_name] reg_key = stat.pathspec.Basename() kb_attr = map_dict.get(reg_key) if kb_attr: value = artifact_lib.ExpandWindowsEnvironmentVariables( stat.registry_data.GetValue(), knowledge_base) value = artifact_lib.ExpandWindowsUserEnvironmentVariables( value, knowledge_base, sid=sid_str) user_dict[sid_str].Set(kb_attr, value) # Now yield each user we found. return user_dict.itervalues()
def Parse(self, stat, knowledge_base): """Parse each returned registry value.""" _ = knowledge_base # Unused. sid_str = stat.pathspec.Dirname().Basename() if SID_RE.match(sid_str): kb_user = rdfvalue.KnowledgeBaseUser() kb_user.sid = sid_str if stat.pathspec.Basename() == "ProfileImagePath": if stat.resident: # Support old clients. kb_user.homedir = utils.SmartUnicode(stat.resident) else: kb_user.homedir = stat.registry_data.GetValue() kb_user.userprofile = kb_user.homedir try: # Assume username is the last component of the path. This is not # robust, but other user artifacts will override it if there is a # better match. kb_user.username = kb_user.homedir.rsplit("\\", 1)[-1] except IndexError: pass yield kb_user
def testFindsKeyWithInterpolatedGlobWithoutConditions(self): # Initialize client's knowledge base in order for the interpolation # to work. user = rdfvalue.KnowledgeBaseUser( sid="S-1-5-21-2911950750-476812067-1487428992-1001") kb = rdfvalue.KnowledgeBase(users=[user]) with aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token) as client: client.Set(client.Schema.KNOWLEDGE_BASE, kb) self.RunFlow([ "HKEY_USERS/%%users.sid%%/Software/Microsoft/Windows/" "CurrentVersion/*" ]) results = self.GetResults() self.assertEqual(len(results), 1) key = ("/HKEY_USERS/S-1-5-21-2911950750-476812067-1487428992-1001/" "Software/Microsoft/Windows/CurrentVersion/Explorer") self.assertEqual(results[0].stat_entry.aff4path, "aff4:/C.1000000000000000/registry" + key) self.assertEqual(results[0].stat_entry.pathspec.path, key) self.assertEqual(results[0].stat_entry.pathspec.pathtype, rdfvalue.PathSpec.PathType.REGISTRY)
def Parse(self, stat, file_object, knowledge_base): """Parse the wtmp file.""" _, _ = stat, knowledge_base users = {} wtmp = file_object.read(10000000) while wtmp: try: record = UtmpStruct(wtmp) except RuntimeError: break wtmp = wtmp[record.size:] # Users only appear for USER_PROCESS events, others are system. if record.ut_type != 7: continue # Lose the null termination record.user = record.user.split("\x00", 1)[0] # Store the latest login time. # TODO(user): remove the 0 here once RDFDatetime can support times # pre-epoch properly. try: users[record.user] = max(users[record.user], record.sec, 0) except KeyError: users[record.user] = record.sec for user, last_login in users.iteritems(): yield rdfvalue.KnowledgeBaseUser(username=utils.SmartUnicode(user), last_logon=last_login * 1000000)
def GetExpectedUser(self, algo, user_store, group_store): user = rdfvalue.KnowledgeBaseUser(username="******", full_name="User", uid="1001", gid="1001", homedir="/home/user", shell="/bin/bash") user.pw_entry = rdfvalue.PwEntry(store=user_store, hash_type=algo) user.gids = [1001] grp = rdfvalue.Group(gid=1001, members=["user"], name="user") grp.pw_entry = rdfvalue.PwEntry(store=group_store, hash_type=algo) return user, grp
def Parse(self, query, result, knowledge_base): """Parse the wmi Win32_UserAccount output.""" _ = query, knowledge_base kb_user = rdfvalue.KnowledgeBaseUser() for wmi_key, kb_key in self.account_mapping.items(): try: kb_user.Set(kb_key, result[wmi_key]) except KeyError: pass yield kb_user
def testRdfFormatterFanOut(self): rdf = rdfvalue.Dict() user1 = rdfvalue.KnowledgeBaseUser(username="******") user2 = rdfvalue.KnowledgeBaseUser(username="******") rdf["cataclysm"] = "GreyGoo" rdf["thinkers"] = [user1, user2] rdf["reference"] = { "ecophage": ["bots", ["nanobots", ["picobots"]]], "doomsday": { "books": ["cats cradle", "prey"] } } template = ("{cataclysm}; {thinkers.username}; {reference.ecophage}; " "{reference.doomsday}") hinter = hints.Hinter(template=template) expected = ("GreyGoo; drexler,joy; bots,nanobots,picobots; " "books:cats cradle,prey") result = hinter.Render(rdf) self.assertEqual(expected, result)
def Parse(self, stat_entries, knowledge_base, path_type): """Parse the StatEntry objects.""" _, _ = knowledge_base, path_type for stat_entry in stat_entries: if stat.S_ISDIR(stat_entry.st_mode): homedir = stat_entry.pathspec.path username = os.path.basename(homedir) if username not in self.blacklist: yield rdfvalue.KnowledgeBaseUser(username=username, homedir=homedir)
def testUserMerge(self): """Check users are accurately merged.""" kb = rdfvalue.KnowledgeBase() self.assertEquals(len(kb.users), 0) kb.MergeOrAddUser(rdfvalue.KnowledgeBaseUser(sid="1234")) self.assertEquals(len(kb.users), 1) kb.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(sid="5678", username="******")) self.assertEquals(len(kb.users), 2) _, conflicts = kb.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(sid="5678", username="******")) self.assertEquals(len(kb.users), 2) self.assertEquals(conflicts[0], ("username", "test1", "test2")) self.assertEquals(kb.GetUser(sid="5678").username, "test2") # This should merge on user name as we have no other data. kb.MergeOrAddUser( rdfvalue.KnowledgeBaseUser(username="******", homedir="a")) self.assertEquals(len(kb.users), 2)
def ToKnowledgeBaseUser(self): """Convert a User value into a KnowledgeBaseUser value.""" kb_user = rdfvalue.KnowledgeBaseUser() for old_pb_name, new_pb_name in self.kb_user_mapping.items(): if len(old_pb_name.split(".")) > 1: attr, old_pb_name = old_pb_name.split(".", 1) val = getattr(getattr(self, attr), old_pb_name) else: val = getattr(self, old_pb_name) if val: kb_user.Set(new_pb_name, val) return kb_user
def testConvertFromKnowledgeBaseUser(self): kbuser = rdfvalue.KnowledgeBaseUser(username="******", userdomain="test.com", homedir="/usr/local/test", desktop="/usr/local/test/Desktop", localappdata="/usr/local/test/AppData") user = rdfvalue.User().FromKnowledgeBaseUser(kbuser) self.assertEqual(user.username, "test") self.assertEqual(user.domain, "test.com") self.assertEqual(user.homedir, "/usr/local/test") self.assertEqual(user.special_folders.desktop, "/usr/local/test/Desktop") self.assertEqual(user.special_folders.local_app_data, "/usr/local/test/AppData")
def Parse(self, query, result, knowledge_base): """Parse the wmi Win32_UserAccount output.""" _ = query, knowledge_base kb_user = rdfvalue.KnowledgeBaseUser() for wmi_key, kb_key in self.account_mapping.items(): try: kb_user.Set(kb_key, result[wmi_key]) except KeyError: pass # We need at least a sid or a username. If these are missing its likely we # retrieved just the userdomain for an AD account that has a name collision # with a local account that is correctly populated. We drop the bogus # domain account. if kb_user.sid or kb_user.username: yield kb_user
def ParseLine(cls, index, line): fields = "username,password,uid,gid,fullname,homedir,shell".split(",") try: if not line: return dat = dict(zip(fields, line.split(":"))) user = rdfvalue.KnowledgeBaseUser(username=dat["username"], uid=int(dat["uid"]), homedir=dat["homedir"], shell=dat["shell"], gid=int(dat["gid"]), full_name=dat["fullname"]) return user except (IndexError, KeyError): raise parsers.ParseError("Invalid passwd file at line %d. %s" % ((index + 1), line))
def setUp(self): super(TestWebHistoryWithArtifacts, self).setUp() self.SetLinuxClient() fd = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw") self.kb = fd.Schema.KNOWLEDGE_BASE() self.kb.users.Append( rdfvalue.KnowledgeBaseUser(username="******", full_name="test user", homedir="/home/test/", last_logon=250)) self.kb.os = "Linux" fd.AddAttribute(fd.Schema.KNOWLEDGE_BASE, self.kb) fd.Flush() self.client_mock = action_mocks.ActionMock( "ReadBuffer", "FingerprintFile", "HashBuffer", "TransferBuffer", "StatFile", "Find", "ListDirectory")
def ParseLines(cls, lines): users = set() filter_regexes = [ re.compile(x) for x in config_lib.CONFIG["Artifacts.netgroup_filter_regexes"] ] username_regex = re.compile(cls.USERNAME_REGEX) blacklist = config_lib.CONFIG["Artifacts.netgroup_user_blacklist"] for index, line in enumerate(lines): if line.startswith("#"): continue splitline = line.split(" ") group_name = splitline[0] if filter_regexes: filter_match = False for regex in filter_regexes: if regex.search(group_name): filter_match = True break if not filter_match: continue for member in splitline[1:]: if member.startswith("("): try: _, user, _ = member.split(",") if user not in users and user not in blacklist: if not username_regex.match(user): yield rdfvalue.Anomaly( type="PARSER_ANOMALY", symptom="Invalid username: %s" % user) else: users.add(user) yield rdfvalue.KnowledgeBaseUser( username=utils.SmartUnicode(user)) except ValueError: raise parsers.ParseError( "Invalid netgroup file at line %d: %s" % (index + 1, line))