def testGrep(self): class MockCallFlow(object): def CallFlow(self, *args, **kwargs): self.args = args self.kwargs = kwargs mock_call_flow = MockCallFlow() with utils.Stubber(collectors.ArtifactCollectorFlow, "CallFlow", mock_call_flow.CallFlow): collect_flow = collectors.ArtifactCollectorFlow(None, token=self.token) collect_flow.state.Register("knowledge_base", rdf_client.KnowledgeBase()) collect_flow.current_artifact_name = "blah" collect_flow.state.knowledge_base.MergeOrAddUser( rdf_client.KnowledgeBaseUser(username="******")) collect_flow.state.knowledge_base.MergeOrAddUser( rdf_client.KnowledgeBaseUser(username="******")) collector = artifact_registry.ArtifactSource( type=artifact_registry.ArtifactSource.SourceType.GREP, attributes={ "paths": ["/etc/passwd"], "content_regex_list": [r"^a%%users.username%%b$"] }) collect_flow.Grep(collector, rdf_paths.PathSpec.PathType.TSK) conditions = mock_call_flow.kwargs["conditions"] self.assertEqual(len(conditions), 1) regexes = conditions[0].contents_regex_match.regex.SerializeToString() self.assertItemsEqual(regexes.split("|"), ["(^atest1b$)", "(^atest2b$)"]) self.assertEqual(mock_call_flow.kwargs["paths"], ["/etc/passwd"])
def testUserMergeWindows(self): """Check Windows users are accurately merged.""" kb = rdf_client.KnowledgeBase() self.assertEqual(len(kb.users), 0) kb.MergeOrAddUser(rdf_client.KnowledgeBaseUser(sid="1234")) self.assertEqual(len(kb.users), 1) kb.MergeOrAddUser( rdf_client.KnowledgeBaseUser(sid="5678", username="******")) self.assertEqual(len(kb.users), 2) _, conflicts = kb.MergeOrAddUser( rdf_client.KnowledgeBaseUser(sid="5678", username="******")) self.assertEqual(len(kb.users), 2) self.assertEqual(conflicts[0], ("username", "test1", "test2")) self.assertEqual(kb.GetUser(sid="5678").username, "test2") # This should merge on user name as we have no other data. kb.MergeOrAddUser( rdf_client.KnowledgeBaseUser(username="******", homedir="a")) self.assertEqual(len(kb.users), 2) # This should create a new user since the sid is different. new_attrs, conflicts = kb.MergeOrAddUser( rdf_client.KnowledgeBaseUser(username="******", sid="12345", temp="/blah")) self.assertEqual(len(kb.users), 3) self.assertItemsEqual(new_attrs, ["users.username", "users.temp", "users.sid"]) self.assertEqual(conflicts, [])
def testInterpolateArgs(self): collect_flow = collectors.ArtifactCollectorFlow(None, token=self.token) collect_flow.state.Register("knowledge_base", rdf_client.KnowledgeBase()) collect_flow.current_artifact_name = "blah" collect_flow.state.knowledge_base.MergeOrAddUser( rdf_client.KnowledgeBaseUser(username="******")) collect_flow.state.knowledge_base.MergeOrAddUser( rdf_client.KnowledgeBaseUser(username="******")) test_rdf = rdf_client.KnowledgeBase() action_args = { "usernames": ["%%users.username%%", "%%users.username%%"], "nointerp": "asdfsdf", "notastring": test_rdf } kwargs = collect_flow.InterpolateDict(action_args) self.assertItemsEqual(kwargs["usernames"], ["test1", "test2", "test1", "test2"]) self.assertEqual(kwargs["nointerp"], "asdfsdf") self.assertEqual(kwargs["notastring"], test_rdf) # We should be using an array since users.username will expand to multiple # values. self.assertRaises(ValueError, collect_flow.InterpolateDict, {"bad": "%%users.username%%"}) list_args = collect_flow.InterpolateList( ["%%users.username%%", r"%%users.username%%\aa"]) self.assertItemsEqual(list_args, ["test1", "test2", r"test1\aa", r"test2\aa"]) list_args = collect_flow.InterpolateList(["one"]) self.assertEqual(list_args, ["one"])
def testUserMergeLinux(self): """Check Linux users are accurately merged.""" kb = rdf_client.KnowledgeBase() self.assertEqual(len(kb.users), 0) kb.MergeOrAddUser( rdf_client.KnowledgeBaseUser(username="******", last_logon=1111)) self.assertEqual(len(kb.users), 1) # This should merge since the username is the same. kb.MergeOrAddUser( rdf_client.KnowledgeBaseUser(uid="12", username="******")) self.assertEqual(len(kb.users), 1) # This should create a new record because the uid is different kb.MergeOrAddUser( rdf_client.KnowledgeBaseUser(username="******", uid="13", desktop="/home/blake/Desktop")) self.assertEqual(len(kb.users), 2) kb.MergeOrAddUser( rdf_client.KnowledgeBaseUser(username="******", uid="14", desktop="/home/blake/Desktop")) self.assertEqual(len(kb.users), 3) # Check merging where we don't specify uid works new_attrs, conflicts = kb.MergeOrAddUser( rdf_client.KnowledgeBaseUser(username="******", desktop="/home/blakey/Desktop")) self.assertEqual(len(kb.users), 3) self.assertItemsEqual(new_attrs, ["users.username", "users.desktop"]) self.assertItemsEqual( conflicts, [("desktop", u"/home/blake/Desktop", u"/home/blakey/Desktop")])
def testInterpolation(self): """Check we can interpolate values from the knowledge base.""" kb = rdf_client.KnowledgeBase() # No users yet, this should raise self.assertRaises( artifact_lib.KnowledgeBaseInterpolationError, list, artifact_lib.InterpolateKbAttributes("test%%users.username%%test", kb)) # Now we have two users kb.users.Append(rdf_client.KnowledgeBaseUser(username="******", uid=1)) kb.users.Append(rdf_client.KnowledgeBaseUser(username="******", uid=2)) kb.Set("environ_allusersprofile", "c:\\programdata") paths = artifact_lib.InterpolateKbAttributes( "test%%users.username%%test", kb) paths = list(paths) self.assertEqual(len(paths), 2) self.assertItemsEqual(paths, ["testjoetest", "testjimtest"]) paths = artifact_lib.InterpolateKbAttributes( "%%environ_allusersprofile%%\\a", kb) self.assertEqual(list(paths), ["c:\\programdata\\a"]) # Check a bad attribute raises self.assertRaises( artifact_lib.KnowledgeBaseInterpolationError, list, artifact_lib.InterpolateKbAttributes("%%nonexistent%%\\a", kb)) # Empty values should also raise kb.Set("environ_allusersprofile", "") self.assertRaises( artifact_lib.KnowledgeBaseInterpolationError, list, artifact_lib.InterpolateKbAttributes( "%%environ_allusersprofile%%\\a", kb)) # No users have temp defined, so this should raise self.assertRaises( artifact_lib.KnowledgeBaseInterpolationError, list, artifact_lib.InterpolateKbAttributes("%%users.temp%%\\a", kb)) # One user has users.temp defined, the others do not. This is common on # windows where users have been created but have never logged in. We should # get just one value back. kb.users.Append( rdf_client.KnowledgeBaseUser( username="******", uid=1, temp="C:\\Users\\jason\\AppData\\Local\\Temp")) paths = artifact_lib.InterpolateKbAttributes(r"%%users.temp%%\abcd", kb) self.assertItemsEqual(paths, ["C:\\Users\\jason\\AppData\\Local\\Temp\\abcd"])
def testGlob(self): """Test that glob works properly.""" # Add some usernames we can interpolate later. client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token) kb = client.Get(client.Schema.KNOWLEDGE_BASE) kb.MergeOrAddUser(rdf_client.KnowledgeBaseUser(username="******")) kb.MergeOrAddUser(rdf_client.KnowledgeBaseUser(username="******")) client.Set(kb) client.Close() client_mock = action_mocks.ActionMock("Find", "StatFile") # This glob selects all files which start with the username on this system. paths = [os.path.join(self.base_path, "%%Users.username%%*"), os.path.join(self.base_path, "VFSFixture/var/*/wtmp")] # Set iterator really low to force iteration. with utils.Stubber(filesystem.Glob, "FILE_MAX_PER_DIR", 2): for _ in test_lib.TestFlowHelper( "Glob", client_mock, client_id=self.client_id, paths=paths, pathtype=rdf_paths.PathSpec.PathType.OS, token=self.token, sync=False, check_flow_errors=False): pass output_path = self.client_id.Add("fs/os").Add( self.base_path.replace("\\", "/")) children = [] fd = aff4.FACTORY.Open(output_path, token=self.token) for child in fd.ListChildren(): filename = child.Basename() if filename != "VFSFixture": children.append(filename) expected = [filename for filename in os.listdir(self.base_path) if filename.startswith("test") or filename.startswith("syslog")] self.assertTrue([x for x in expected if x.startswith("test")], "Need a file starting with 'test'" " in test_data for this test!") self.assertTrue([x for x in expected if x.startswith("syslog")], "Need a file starting with 'syslog'" " in test_data for this test!") self.assertItemsEqual(expected, children) children = [] fd = aff4.FACTORY.Open(output_path.Add("VFSFixture/var/log"), token=self.token) for child in fd.ListChildren(): children.append(child.Basename()) self.assertItemsEqual(children, ["wtmp"])
def setUp(self): """Make sure things are initialized.""" super(ArtifactFlowTest, self).setUp() fd = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw") fd.Set(fd.Schema.SYSTEM("Linux")) kb = fd.Schema.KNOWLEDGE_BASE() artifact.SetCoreGRRKnowledgeBaseValues(kb, fd) kb.MergeOrAddUser(rdf_client.KnowledgeBaseUser(username="******")) kb.MergeOrAddUser(rdf_client.KnowledgeBaseUser(username="******")) kb.MergeOrAddUser(rdf_client.KnowledgeBaseUser(username="******")) fd.Set(kb) fd.Flush() self.LoadTestArtifacts()
def ParseMultiple(self, stats, knowledge_base): """Parse each returned registry value.""" user_dict = {} for stat in stats: sid_str = stat.pathspec.path.split("/", 3)[2] if SID_RE.match(sid_str): if sid_str not in user_dict: user_dict[sid_str] = rdf_client.KnowledgeBaseUser( sid=sid_str) if stat.registry_data.GetValue(): # Look up in the mapping if we can use this entry to populate a user # attribute, and if so, set it. reg_key_name = stat.pathspec.Dirname().Basename() if reg_key_name in self.key_var_mapping: map_dict = self.key_var_mapping[reg_key_name] reg_key = stat.pathspec.Basename() kb_attr = map_dict.get(reg_key) if kb_attr: value = artifact_lib.ExpandWindowsEnvironmentVariables( stat.registry_data.GetValue(), knowledge_base) value = artifact_lib.ExpandWindowsUserEnvironmentVariables( value, knowledge_base, sid=sid_str) user_dict[sid_str].Set(kb_attr, value) # Now yield each user we found. return user_dict.itervalues()
def setUp(self): """Make sure things are initialized.""" super(ArtifactFlowLinuxTest, self).setUp() with aff4.FACTORY.Open( self.SetupClients(1, system="Linux", os_version="12.04")[0], mode="rw", token=self.token) as fd: # Add some users kb = fd.Get(fd.Schema.KNOWLEDGE_BASE) kb.MergeOrAddUser(rdf_client.KnowledgeBaseUser(username="******")) kb.MergeOrAddUser(rdf_client.KnowledgeBaseUser(username="******")) kb.MergeOrAddUser(rdf_client.KnowledgeBaseUser(username="******")) fd.Set(kb) self.LoadTestArtifacts()
def setUp(self): super(ListVADBinariesTest, self).setUp() self.SetupClients(1, system="Windows", os_version="6.2", arch="AMD64") self.os_overrider = test_lib.VFSOverrider( rdf_paths.PathSpec.PathType.OS, test_lib.ClientVFSHandlerFixture) self.reg_overrider = test_lib.VFSOverrider( rdf_paths.PathSpec.PathType.REGISTRY, test_lib.FakeRegistryVFSHandler) self.os_overrider.Start() self.reg_overrider.Start() # Add some user accounts to this client. fd = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token) kb = fd.Get(fd.Schema.KNOWLEDGE_BASE) kb.environ_systemdrive = "C:" kb.MergeOrAddUser( rdf_client.KnowledgeBaseUser(username="******", userdomain="testing-PC", homedir=r"C:\Users\localservice", sid="S-1-5-20")) fd.Set(kb) fd.Close() self.old_driver_flow = flow.GRRFlow.classes["LoadMemoryDriver"] flow.GRRFlow.classes["LoadMemoryDriver"] = DummyLoadMemoryDriverFlow
def testFindsKeyWithInterpolatedGlobWithoutConditions(self): # Initialize client's knowledge base in order for the interpolation # to work. user = rdf_client.KnowledgeBaseUser( sid="S-1-5-21-2911950750-476812067-1487428992-1001") kb = rdf_client.KnowledgeBase(users=[user]) with aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token) as client: client.Set(client.Schema.KNOWLEDGE_BASE, kb) self.RunFlow([ "HKEY_USERS/%%users.sid%%/Software/Microsoft/Windows/" "CurrentVersion/*" ]) results = self.GetResults() self.assertEqual(len(results), 1) key = ("/HKEY_USERS/S-1-5-21-2911950750-476812067-1487428992-1001/" "Software/Microsoft/Windows/CurrentVersion/Explorer") self.assertEqual(results[0].stat_entry.aff4path, "aff4:/C.1000000000000000/registry" + key) self.assertEqual(results[0].stat_entry.pathspec.path, key) self.assertEqual(results[0].stat_entry.pathspec.pathtype, rdf_paths.PathSpec.PathType.REGISTRY)
def Parse(self, stat, file_object, knowledge_base): """Parse the wtmp file.""" _, _ = stat, knowledge_base users = {} wtmp = file_object.read(10000000) while wtmp: try: record = UtmpStruct(wtmp) except RuntimeError: break wtmp = wtmp[record.size:] # Users only appear for USER_PROCESS events, others are system. if record.ut_type != 7: continue # Lose the null termination record.user = record.user.split("\x00", 1)[0] # Store the latest login time. # TODO(user): remove the 0 here once RDFDatetime can support times # pre-epoch properly. try: users[record.user] = max(users[record.user], record.sec, 0) except KeyError: users[record.user] = record.sec for user, last_login in users.iteritems(): yield client.KnowledgeBaseUser(username=utils.SmartUnicode(user), last_logon=last_login * 1000000)
def Parse(self, stat, knowledge_base): """Parse each returned registry value.""" _ = knowledge_base # Unused. sid_str = stat.pathspec.Dirname().Basename() if SID_RE.match(sid_str): kb_user = rdf_client.KnowledgeBaseUser() kb_user.sid = sid_str if stat.pathspec.Basename() == "ProfileImagePath": if stat.resident: # Support old clients. kb_user.homedir = utils.SmartUnicode(stat.resident) else: kb_user.homedir = stat.registry_data.GetValue() kb_user.userprofile = kb_user.homedir try: # Assume username is the last component of the path. This is not # robust, but other user artifacts will override it if there is a # better match. kb_user.username = kb_user.homedir.rsplit("\\", 1)[-1] except IndexError: pass yield kb_user
def ParsePasswdEntry(self, line): """Process the passwd entry fields and primary group memberships.""" fields = ("uname", "passwd", "uid", "gid", "fullname", "homedir", "shell") if line: rslt = dict(zip(fields, line.split(":"))) user = self.entry.setdefault(rslt["uname"], client.KnowledgeBaseUser()) user.username = rslt["uname"] user.pw_entry.store = self.GetPwStore(rslt["passwd"]) if user.pw_entry.store == self.base_store: user.pw_entry.hash_type = self.GetHashType(rslt["passwd"]) # If the passwd file contains NIS entries they may not have uid/gid set. if rslt["uid"]: user.uid = int(rslt["uid"]) if rslt["gid"]: user.gid = int(rslt["gid"]) user.homedir = rslt["homedir"] user.shell = rslt["shell"] user.full_name = rslt["fullname"] # Map uid numbers to detect duplicates. uids = self.uids.setdefault(user.uid, set()) uids.add(user.username) # Map primary group memberships to populate memberships. gid = self.gids.setdefault(user.gid, set()) gid.add(user.username)
def GetExpectedUser(self, algo, user_store, group_store): user = rdf_client.KnowledgeBaseUser(username="******", full_name="User", uid="1001", gid="1001", homedir="/home/user", shell="/bin/bash") user.pw_entry = rdf_client.PwEntry(store=user_store, hash_type=algo) user.gids = [1001] grp = rdf_client.Group(gid=1001, members=["user"], name="user") grp.pw_entry = rdf_client.PwEntry(store=group_store, hash_type=algo) return user, grp
def testRdfFormatterFanOut(self): rdf = rdf_protodict.Dict() user1 = rdf_client.KnowledgeBaseUser(username="******") user2 = rdf_client.KnowledgeBaseUser(username="******") rdf["cataclysm"] = "GreyGoo" rdf["thinkers"] = [user1, user2] rdf["reference"] = { "ecophage": ["bots", ["nanobots", ["picobots"]]], "doomsday": { "books": ["cats cradle", "prey"] } } template = ("{cataclysm}; {thinkers.username}; {reference.ecophage}; " "{reference.doomsday}\n") hinter = hints.Hinter(template=template) expected = ("GreyGoo; drexler,joy; bots,nanobots,picobots; " "books:cats cradle,prey") result = hinter.Render(rdf) self.assertEqual(expected, result)
def testKBUserBackwardsCompatibility(self): """Check User can be created from deprecated KBUser.""" kbuser = rdf_client.KnowledgeBaseUser() kbuser.username = "******" kbuser.desktop = "User Desktop 1" user = rdf_client.User(kbuser) self.assertEqual(user.username, "user1") self.assertEqual(user.desktop, "User Desktop 1")
def SetLinuxKB(self): client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw") kb = client.Schema.KNOWLEDGE_BASE() kb.os = "Linux" user = rdf_client.KnowledgeBaseUser(username="******", homedir="/home/user1") kb.users = [user] client.Set(client.Schema.KNOWLEDGE_BASE, kb) client.Set(client.Schema.SYSTEM("Linux")) client.Set(client.Schema.OS_VERSION("12.04")) client.Flush()
def Parse(self, stat_entries, knowledge_base, path_type): """Parse the StatEntry objects.""" _, _ = knowledge_base, path_type for stat_entry in stat_entries: if stat.S_ISDIR(stat_entry.st_mode): homedir = stat_entry.pathspec.path username = os.path.basename(homedir) if username not in self.blacklist: yield rdf_client.KnowledgeBaseUser(username=username, homedir=homedir)
def testGlobDirectory(self): """Test that glob expands directories.""" # Add some usernames we can interpolate later. client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token) kb = client.Get(client.Schema.KNOWLEDGE_BASE) kb.MergeOrAddUser(rdf_client.KnowledgeBaseUser( username="******", appdata="test_data/index.dat")) kb.MergeOrAddUser(rdf_client.KnowledgeBaseUser( username="******", appdata="test_data/History")) # This is a record which means something to the interpolation system. We # should not process this especially. kb.MergeOrAddUser(rdf_client.KnowledgeBaseUser( username="******", appdata="%%PATH%%")) client.Set(kb) client.Close() client_mock = action_mocks.ActionMock("Find", "StatFile") # This glob selects all files which start with the username on this system. path = os.path.join(os.path.dirname(self.base_path), "%%users.appdata%%") # Run the flow. for _ in test_lib.TestFlowHelper( "Glob", client_mock, client_id=self.client_id, paths=[path], token=self.token): pass path = self.client_id.Add("fs/os").Add(self.base_path).Add("index.dat") aff4.FACTORY.Open(path, aff4_type="VFSFile", token=self.token) path = self.client_id.Add("fs/os").Add(self.base_path).Add("index.dat") aff4.FACTORY.Open(path, aff4_type="VFSFile", token=self.token)
def testConvertFromKnowledgeBaseUser(self): kbuser = rdf_client.KnowledgeBaseUser( username="******", userdomain="test.com", homedir="/usr/local/test", desktop="/usr/local/test/Desktop", localappdata="/usr/local/test/AppData") user = rdf_client.User().FromKnowledgeBaseUser(kbuser) self.assertEqual(user.username, "test") self.assertEqual(user.domain, "test.com") self.assertEqual(user.homedir, "/usr/local/test") self.assertEqual(user.special_folders.desktop, "/usr/local/test/Desktop") self.assertEqual(user.special_folders.local_app_data, "/usr/local/test/AppData")
def Parse(self, query, result, knowledge_base): """Parse the wmi Win32_UserAccount output.""" _ = query, knowledge_base kb_user = rdf_client.KnowledgeBaseUser() for wmi_key, kb_key in self.account_mapping.items(): try: kb_user.Set(kb_key, result[wmi_key]) except KeyError: pass # We need at least a sid or a username. If these are missing its likely we # retrieved just the userdomain for an AD account that has a name collision # with a local account that is correctly populated. We drop the bogus # domain account. if kb_user.sid or kb_user.username: yield kb_user
def setUp(self): super(TestWebHistoryWithArtifacts, self).setUp() self.SetupClients(1, system="Linux", os_version="12.04") fd = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw") self.kb = fd.Get(fd.Schema.KNOWLEDGE_BASE) self.kb.users.Append( rdf_client.KnowledgeBaseUser(username="******", full_name="test user", homedir="/home/test/", last_logon=250)) fd.AddAttribute(fd.Schema.KNOWLEDGE_BASE, self.kb) fd.Flush() self.client_mock = action_mocks.ActionMock( "ReadBuffer", "FingerprintFile", "HashBuffer", "HashFile", "TransferBuffer", "StatFile", "Find", "ListDirectory")
def ParseLine(cls, index, line): fields = "username,password,uid,gid,fullname,homedir,shell".split(",") try: if not line: return dat = dict(zip(fields, line.split(":"))) user = client.KnowledgeBaseUser(username=dat["username"], uid=int(dat["uid"]), homedir=dat["homedir"], shell=dat["shell"], gid=int(dat["gid"]), full_name=dat["fullname"]) return user except (IndexError, KeyError): raise parsers.ParseError("Invalid passwd file at line %d. %s" % ((index + 1), line))
def ParseLines(cls, lines): users = set() filter_regexes = [ re.compile(x) for x in config_lib.CONFIG["Artifacts.netgroup_filter_regexes"] ] username_regex = re.compile(cls.USERNAME_REGEX) blacklist = config_lib.CONFIG["Artifacts.netgroup_user_blacklist"] for index, line in enumerate(lines): if line.startswith("#"): continue splitline = line.split(" ") group_name = splitline[0] if filter_regexes: filter_match = False for regex in filter_regexes: if regex.search(group_name): filter_match = True break if not filter_match: continue for member in splitline[1:]: if member.startswith("("): try: _, user, _ = member.split(",") if user not in users and user not in blacklist: if not username_regex.match(user): yield anomaly.Anomaly( type="PARSER_ANOMALY", symptom="Invalid username: %s" % user) else: users.add(user) yield client.KnowledgeBaseUser( username=utils.SmartUnicode(user)) except ValueError: raise parsers.ParseError( "Invalid netgroup file at line %d: %s" % (index + 1, line))