def testTwoRegexMatchConditionsWithDifferentActions2(self): expected_files = ["auth.log"] non_expected_files = ["dpkg.log", "dpkg_false.log"] regex_condition1 = rdf_file_finder.FileFinderCondition( condition_type=( rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH), contents_regex_match=( rdf_file_finder.FileFinderContentsRegexMatchCondition( mode="ALL_HITS", bytes_before=10, bytes_after=10, regex=b"session opened for user .*?john"))) regex_condition2 = rdf_file_finder.FileFinderCondition( condition_type=( rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH), contents_regex_match=( rdf_file_finder.FileFinderContentsRegexMatchCondition( mode="FIRST_HIT", bytes_before=10, bytes_after=10, regex=b".*"))) for action in self.CONDITION_TESTS_ACTIONS: results = self.RunFlowAndCheckResults( action=action, conditions=[regex_condition1, regex_condition2], expected_files=expected_files, non_expected_files=non_expected_files) self.assertLen(results, 1) self.assertLen(results[0].matches, 2) self.assertEqual(results[0].matches[0].offset, 350) self.assertEqual(results[0].matches[0].data, b"session): session opened for user dearjohn by (uid=0") self.assertEqual(results[0].matches[1].offset, 0) self.assertEqual(results[0].matches[1].length, 770)
def _ConditionsToFileFinderConditions(conditions): """Converts FileFinderSizeConditions to RegistryFinderConditions.""" ff_condition_type_cls = rdf_file_finder.FileFinderCondition.Type result = [] for c in conditions: if c.condition_type == RegistryFinderCondition.Type.MODIFICATION_TIME: result.append( rdf_file_finder.FileFinderCondition( condition_type=ff_condition_type_cls.MODIFICATION_TIME, modification_time=c.modification_time)) elif c.condition_type == RegistryFinderCondition.Type.VALUE_LITERAL_MATCH: result.append( rdf_file_finder.FileFinderCondition( condition_type=ff_condition_type_cls. CONTENTS_LITERAL_MATCH, contents_literal_match=c.value_literal_match)) elif c.condition_type == RegistryFinderCondition.Type.VALUE_REGEX_MATCH: result.append( rdf_file_finder.FileFinderCondition( condition_type=ff_condition_type_cls.CONTENTS_REGEX_MATCH, contents_regex_match=c.value_regex_match)) elif c.condition_type == RegistryFinderCondition.Type.SIZE: result.append( rdf_file_finder.FileFinderCondition( condition_type=ff_condition_type_cls.SIZE, size=c.size)) else: raise ValueError("Unknown condition type: %s" % c.condition_type) return result
def testRegexMatchConditionWithDifferentActions(self): expected_files = ["auth.log"] non_expected_files = ["dpkg.log", "dpkg_false.log"] regex_condition = rdf_file_finder.FileFinderCondition( condition_type=( rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH), contents_regex_match=( rdf_file_finder.FileFinderContentsRegexMatchCondition( mode="ALL_HITS", bytes_before=10, bytes_after=10, regex="session opened for user .*?john"))) for action in self.CONDITION_TESTS_ACTIONS: self.RunFlowAndCheckResults(action=action, conditions=[regex_condition], expected_files=expected_files, non_expected_files=non_expected_files) fd = flow.GRRFlow.ResultCollectionForFID(self.last_session_id) self.assertEqual(len(fd), 1) self.assertEqual(len(fd[0].matches), 1) self.assertEqual(fd[0].matches[0].offset, 350) self.assertEqual( fd[0].matches[0].data, "session): session opened for user dearjohn by (uid=0")
def testLiteralMatchConditionWithHexEncodedValue(self): match = rdf_file_finder.FileFinderContentsLiteralMatchCondition( mode=rdf_file_finder.FileFinderContentsLiteralMatchCondition.Mode. FIRST_HIT, bytes_before=10, bytes_after=10, literal="\x4D\x5A\x90") literal_condition = rdf_file_finder.FileFinderCondition( condition_type=rdf_file_finder.FileFinderCondition.Type. CONTENTS_LITERAL_MATCH, contents_literal_match=match) paths = [os.path.join(os.path.dirname(self.fixture_path), "hello.exe")] session_id = flow_test_lib.TestFlowHelper( file_finder.FileFinder.__name__, self.client_mock, client_id=self.client_id, paths=paths, pathtype=rdf_paths.PathSpec.PathType.OS, conditions=[literal_condition], token=self.token) # Check that the results' matches fields are correctly filled. Expecting a # match from hello.exe fd = flow.GRRFlow.ResultCollectionForFID(session_id) self.assertEqual(len(fd[0].matches), 1) self.assertEqual(fd[0].matches[0].offset, 0) self.assertEqual(fd[0].matches[0].data, "MZ\x90\x00\x03\x00\x00\x00\x04\x00\x00\x00\xff")
def testLiteralMatchConditionWithDifferentActions(self): expected_files = ["auth.log"] non_expected_files = ["dpkg.log", "dpkg_false.log"] match = rdf_file_finder.FileFinderContentsLiteralMatchCondition( mode=rdf_file_finder.FileFinderContentsLiteralMatchCondition.Mode. ALL_HITS, bytes_before=10, bytes_after=10, literal="session opened for user dearjohn") literal_condition = rdf_file_finder.FileFinderCondition( condition_type=rdf_file_finder.FileFinderCondition.Type. CONTENTS_LITERAL_MATCH, contents_literal_match=match) for action in self.CONDITION_TESTS_ACTIONS: results = self.RunFlowAndCheckResults( action=action, conditions=[literal_condition], expected_files=expected_files, non_expected_files=non_expected_files) # Check that the results' matches fields are correctly filled. self.assertEqual(len(results), 1) self.assertEqual(len(results[0].matches), 1) self.assertEqual(results[0].matches[0].offset, 350) self.assertEqual( results[0].matches[0].data, "session): session opened for user dearjohn by (uid=0")
def testLiteralMatchConditionWithHexEncodedValue(self): match = rdf_file_finder.FileFinderContentsLiteralMatchCondition( mode=rdf_file_finder.FileFinderContentsLiteralMatchCondition.Mode. FIRST_HIT, bytes_before=10, bytes_after=10, literal=b"\x4D\x5A\x90") literal_condition = rdf_file_finder.FileFinderCondition( condition_type=rdf_file_finder.FileFinderCondition.Type. CONTENTS_LITERAL_MATCH, contents_literal_match=match) paths = [ os.path.join(os.path.dirname(self.fixture_path), "win_hello.exe") ] results = self.RunFlow(paths=paths, conditions=[literal_condition]) # Check that the results' matches fields are correctly filled. Expecting a # match from win_hello.exe self.assertLen(results, 1) self.assertLen(results[0].matches, 1) self.assertEqual(results[0].matches[0].offset, 0) self.assertEqual(results[0].matches[0].data, b"MZ\x90\x00\x03\x00\x00\x00\x04\x00\x00\x00\xff")
def StartRequests(self): """Generate and send the Find requests.""" client = aff4.FACTORY.Open(self.client_id, token=self.token) usernames = [ "%s\\%s" % (u.userdomain, u.username) for u in self.state.users ] usernames = [u.lstrip("\\") for u in usernames] # Strip \\ if no domain. condition = rdf_file_finder.FileFinderCondition( condition_type=( rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH), contents_regex_match=rdf_file_finder. FileFinderContentsRegexMatchCondition( regex=self.args.data_regex, mode=rdf_file_finder.FileFinderContentsRegexMatchCondition. Mode.FIRST_HIT)) for path in self.state.all_paths: full_paths = flow_utils.InterpolatePath(path, client, users=usernames) for full_path in full_paths: self.CallFlow( file_finder.FileFinder.__name__, paths=[os.path.join(full_path, "**5")], pathtype=self.args.pathtype, conditions=[condition], action=rdf_file_finder.FileFinderAction.Download(), next_state="HandleResults")
def testRange(self): params = rdf_file_finder.FileFinderCondition() params.size.min_file_size = 2 params.size.max_file_size = 6 condition = conditions.SizeCondition(params) with io.open(self.temp_filepath, "wb") as fd: fd.write(b"1") self.assertFalse(condition.Check(self.Stat())) with io.open(self.temp_filepath, "wb") as fd: fd.write(b"12") self.assertTrue(condition.Check(self.Stat())) with io.open(self.temp_filepath, "wb") as fd: fd.write(b"1234") self.assertTrue(condition.Check(self.Stat())) with io.open(self.temp_filepath, "wb") as fd: fd.write(b"123456") self.assertTrue(condition.Check(self.Stat())) with io.open(self.temp_filepath, "wb") as fd: fd.write(b"1234567") self.assertFalse(condition.Check(self.Stat()))
def _CreateHuntFromFlow(self): self.client_id = self.SetupClient(0) flow_args = rdf_file_finder.FileFinderArgs( paths=["a/*", "b/*"], action=rdf_file_finder.FileFinderAction(action_type="STAT")) flow_runner_args = rdf_flow_runner.FlowRunnerArgs( flow_name=file_finder.FileFinder.__name__) flow_urn = flow.StartAFF4Flow( client_id=self.client_id, args=flow_args, runner_args=flow_runner_args, token=self.token) ref = rdf_hunts.FlowLikeObjectReference.FromFlowIdAndClientId( flow_urn.Basename(), self.client_id.Basename()) # Modify flow_args so that there are differences. flow_args.paths = ["b/*", "c/*"] flow_args.action.action_type = "DOWNLOAD" flow_args.conditions = [ rdf_file_finder.FileFinderCondition( condition_type="SIZE", size=rdf_file_finder.FileFinderSizeCondition(min_file_size=42)) ] return self.CreateHunt( flow_args=flow_args, flow_runner_args=flow_runner_args, original_object=ref), flow_urn
def Grep(self, source, pathtype): """Grep files in paths for any matches to content_regex_list. Args: source: artifact source pathtype: pathspec path type When multiple regexes are supplied, combine them into a single regex as an OR match so that we check all regexes at once. """ path_list = self.InterpolateList(source.attributes.get("paths", [])) content_regex_list = self.InterpolateList( source.attributes.get("content_regex_list", [])) regex_condition = rdf_file_finder.FileFinderContentsRegexMatchCondition( regex=self._CombineRegex(content_regex_list), bytes_before=0, bytes_after=0, mode="ALL_HITS") file_finder_condition = rdf_file_finder.FileFinderCondition( condition_type=( rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH), contents_regex_match=regex_condition) self.CallFlow(file_finder.FileFinder.__name__, paths=path_list, conditions=[file_finder_condition], action=rdf_file_finder.FileFinderAction(), pathtype=pathtype, request_data={ "artifact_name": self.current_artifact_name, "source": source.ToPrimitiveDict() }, next_state="ProcessCollected")
def testNoMatchOsxBitsUnset(self): params = rdf_file_finder.FileFinderCondition() params.ext_flags.osx_bits_unset = self.UF_NODUMP | self.UF_HIDDEN condition = conditions.ExtFlagsCondition(params) self._Chflags(["hidden"]) self.assertFalse(condition.Check(self.Stat()))
def testNoMatchOsxBitsSet(self): params = rdf_file_finder.FileFinderCondition() params.ext_flags.osx_bits_set = self.UF_IMMUTABLE | self.UF_NODUMP condition = conditions.ExtFlagsCondition(params) self._Chflags(["nodump"]) self.assertFalse(condition.Check(self.Stat()))
def testMatchLinuxBitsUnset(self): params = rdf_file_finder.FileFinderCondition() params.ext_flags.linux_bits_unset = self.FS_IMMUTABLE_FL condition = conditions.ExtFlagsCondition(params) self._Chattr(["+c", "+d"]) self.assertTrue(condition.Check(self.Stat()))
def testDefault(self): params = rdf_file_finder.FileFinderCondition() condition = conditions.AccessTimeCondition(params) self.Touch("-a", "241007151200") # 2410-07-15 12:00 self.assertTrue(condition.Check(self.Stat())) self.Touch("-a", "201005160745") # 2010-05-16 7:45 self.assertTrue(condition.Check(self.Stat()))
def testDefault(self): params = rdf_file_finder.FileFinderCondition() condition = conditions.ModificationTimeCondition(params) self.Touch("-m", "198309121200") # 1983-09-12 12:00 self.assertTrue(condition.Check(self.Stat())) self.Touch("-m", "201710020815") # 2017-10-02 8:15 self.assertTrue(condition.Check(self.Stat()))
def Start(self): """Redirect to start on the workers and not in the UI.""" # Figure out which paths we are going to check. if data_store.RelationalDBReadEnabled(): client = data_store.REL_DB.ReadClientSnapshot(self.client_id) kb = client.knowledge_base system = kb.os else: client = aff4.FACTORY.Open(self.client_id, token=self.token) system = client.Get(client.Schema.SYSTEM) kb = client.Get(client.Schema.KNOWLEDGE_BASE) paths = BROWSER_PATHS.get(system) self.state.all_paths = [] if self.args.check_chrome: self.state.all_paths += paths.get("Chrome", []) if self.args.check_ie: self.state.all_paths += paths.get("IE", []) if self.args.check_firefox: self.state.all_paths += paths.get("Firefox", []) if not self.state.all_paths: raise flow.FlowError("Unsupported system %s for CacheGrep" % system) self.state.users = [] for user in self.args.grep_users: user_info = flow_utils.GetUserInfo(kb, user) if not user_info: raise flow.FlowError("No such user %s" % user) self.state.users.append(user_info) usernames = [ "%s\\%s" % (u.userdomain, u.username) for u in self.state.users ] usernames = [u.lstrip("\\") for u in usernames] # Strip \\ if no domain. condition = rdf_file_finder.FileFinderCondition( condition_type=( rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH), contents_regex_match=rdf_file_finder. FileFinderContentsRegexMatchCondition( regex=self.args.data_regex, mode=rdf_file_finder.FileFinderContentsRegexMatchCondition. Mode.FIRST_HIT)) for path in self.state.all_paths: full_paths = flow_utils.InterpolatePath(path, kb, users=usernames) for full_path in full_paths: self.CallFlow( file_finder.FileFinder.__name__, paths=[os.path.join(full_path, "**5")], pathtype=self.args.pathtype, conditions=[condition], action=rdf_file_finder.FileFinderAction.Download(), next_state="HandleResults")
def testMatchLinuxBitsMixed(self): params = rdf_file_finder.FileFinderCondition() params.ext_flags.linux_bits_set = self.FS_NODUMP_FL params.ext_flags.linux_bits_unset = self.FS_COMPR_FL params.ext_flags.osx_bits_unset = self.UF_IMMUTABLE condition = conditions.ExtFlagsCondition(params) self._Chattr(["+d"]) self.assertTrue(condition.Check(self.Stat()))
def testNoHits(self): with open(self.temp_filepath, "wb") as fd: fd.write("foo bar quux") params = rdf_file_finder.FileFinderCondition() params.contents_literal_match.literal = b"baz" params.contents_literal_match.mode = "ALL_HITS" condition = conditions.LiteralMatchCondition(params) results = list(condition.Search(self.temp_filepath)) self.assertFalse(results)
def testNoHits(self): with open(self.temp_filepath, "wb") as fd: fd.write("foo bar quux") params = rdf_file_finder.FileFinderCondition() params.contents_regex_match.regex = "\\d+" params.contents_regex_match.mode = "FIRST_HIT" condition = conditions.RegexMatchCondition(params) results = list(condition.Search(self.temp_filepath)) self.assertFalse(results)
def testDefault(self): params = rdf_file_finder.FileFinderCondition() condition = conditions.SizeCondition(params) with io.open(self.temp_filepath, "wb") as fd: fd.write(b"1234567") self.assertTrue(condition.Check(self.Stat())) with io.open(self.temp_filepath, "wb") as fd: fd.write(b"") self.assertTrue(condition.Check(self.Stat()))
def testSizeAndRegexConditionsWithDifferentActions(self): files_over_size_limit = ["auth.log"] filtered_files = ["dpkg.log", "dpkg_false.log"] expected_files = [] non_expected_files = files_over_size_limit + filtered_files sizes = [ os.stat(os.path.join(self.fixture_path, f)).st_size for f in files_over_size_limit ] size_condition = rdf_file_finder.FileFinderCondition( condition_type=rdf_file_finder.FileFinderCondition.Type.SIZE, size=rdf_file_finder.FileFinderSizeCondition( max_file_size=min(sizes) - 1)) regex_condition = rdf_file_finder.FileFinderCondition( condition_type=( rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH), contents_regex_match=rdf_file_finder. FileFinderContentsRegexMatchCondition( mode=(rdf_file_finder.FileFinderContentsRegexMatchCondition. Mode.ALL_HITS), bytes_before=10, bytes_after=10, regex="session opened for user .*?john")) for action in self.CONDITION_TESTS_ACTIONS: self.RunFlowAndCheckResults( action=action, conditions=[size_condition, regex_condition], expected_files=expected_files, non_expected_files=non_expected_files) # Check that order of conditions doesn't influence results for action in self.CONDITION_TESTS_ACTIONS: self.RunFlowAndCheckResults( action=action, conditions=[regex_condition, size_condition], expected_files=expected_files, non_expected_files=non_expected_files)
def testMinTime(self): time = rdfvalue.RDFDatetime.FromHumanReadable("2017-12-24 19:00:00") params = rdf_file_finder.FileFinderCondition() params.modification_time.min_last_modified_time = time condition = conditions.ModificationTimeCondition(params) self.Touch("-m", "201712240100") # 2017-12-24 1:30 self.assertFalse(condition.Check(self.Stat())) self.Touch("-m", "201806141700") # 2018-06-14 17:00 self.assertTrue(condition.Check(self.Stat()))
def testMaxTime(self): time = rdfvalue.RDFDatetime.FromHumanReadable("2125-12-28 18:45") params = rdf_file_finder.FileFinderCondition() params.modification_time.max_last_modified_time = time condition = conditions.ModificationTimeCondition(params) self.Touch("-m", "211811111200") # 2118-11-11 12:00 self.assertTrue(condition.Check(self.Stat())) self.Touch("-m", "222510201500") # 2225-10-20 15:00 self.assertFalse(condition.Check(self.Stat()))
def testMatchOsxBitsUnset(self): params = rdf_file_finder.FileFinderCondition() params.ext_flags.osx_bits_unset = self.UF_NODUMP | self.UF_IMMUTABLE condition = conditions.ExtFlagsCondition(params) self._Chflags(["hidden", "uappend"]) try: self.assertTrue(condition.Check(self.Stat())) finally: # Make the test file deletable. self._Chflags(["nouappend"])
def testFirstHit(self): with open(self.temp_filepath, "wb") as fd: fd.write("bar foo baz foo") params = rdf_file_finder.FileFinderCondition() params.contents_literal_match.literal = b"foo" params.contents_literal_match.mode = "FIRST_HIT" condition = conditions.LiteralMatchCondition(params) results = list(condition.Search(self.temp_filepath)) self.assertEqual(len(results), 1) self.assertEqual(results[0].data, "foo") self.assertEqual(results[0].offset, 4) self.assertEqual(results[0].length, 3)
def testFirstHit(self): with open(self.temp_filepath, "wb") as fd: fd.write("4 8 15 16 23 42 foo 108 bar") params = rdf_file_finder.FileFinderCondition() params.contents_regex_match.regex = "[a-z]+" params.contents_regex_match.mode = "FIRST_HIT" condition = conditions.RegexMatchCondition(params) results = list(condition.Search(self.temp_filepath)) self.assertEqual(len(results), 1) self.assertEqual(results[0].data, "foo") self.assertEqual(results[0].offset, 16) self.assertEqual(results[0].length, 3)
def testMatchOsxBitsMixed(self): params = rdf_file_finder.FileFinderCondition() params.ext_flags.osx_bits_set = self.UF_NODUMP params.ext_flags.osx_bits_unset = self.UF_HIDDEN params.ext_flags.linux_bits_unset = self.FS_NODUMP_FL condition = conditions.ExtFlagsCondition(params) self._Chflags(["nodump", "uappend"]) try: self.assertTrue(condition.Check(self.Stat())) finally: # Make the test file deletable. self._Chflags(["nouappend"])
def Grep(self, source, pathtype, implementation_type): """Grep files in paths for any matches to content_regex_list. When multiple regexes are supplied, combine them into a single regex as an OR match so that we check all regexes at once. Args: source: artifact source pathtype: pathspec path typed implementation_type: Pathspec implementation type to use. """ path_list = self.InterpolateList(source.attributes.get("paths", [])) # `content_regex_list` elements should be binary strings, but forcing # artifact creators to use verbose YAML syntax for binary literals would # be cruel. Therefore, we allow both kind of strings and we convert to bytes # if required. content_regex_list = [] for content_regex in source.attributes.get("content_regex_list", []): if isinstance(content_regex, Text): content_regex = content_regex.encode("utf-8") content_regex_list.append(content_regex) content_regex_list = self.InterpolateList(content_regex_list) regex_condition = rdf_file_finder.FileFinderContentsRegexMatchCondition( regex=self._CombineRegex(content_regex_list), bytes_before=0, bytes_after=0, mode="ALL_HITS") file_finder_condition = rdf_file_finder.FileFinderCondition( condition_type=( rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH), contents_regex_match=regex_condition) self.CallFlow( file_finder.FileFinder.__name__, paths=path_list, conditions=[file_finder_condition], action=rdf_file_finder.FileFinderAction(), pathtype=pathtype, implementation_type=implementation_type, request_data={ "artifact_name": self.current_artifact_name, "source": source.ToPrimitiveDict() }, next_state=compatibility.GetName(self.ProcessCollected))
def testStartOffset(self): with open(self.temp_filepath, "wb") as fd: fd.write("ooooooo") params = rdf_file_finder.FileFinderCondition() params.contents_regex_match.regex = "o+" params.contents_regex_match.mode = "FIRST_HIT" params.contents_regex_match.start_offset = 3 condition = conditions.RegexMatchCondition(params) results = list(condition.Search(self.temp_filepath)) self.assertEqual(len(results), 1) self.assertEqual(results[0].data, "oooo") self.assertEqual(results[0].offset, 3) self.assertEqual(results[0].length, 4)
def testAccessTimeConditionWithDifferentActions(self): expected_files = ["dpkg.log", "dpkg_false.log"] non_expected_files = ["auth.log"] change_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1444444440) access_time_condition = rdf_file_finder.FileFinderCondition( condition_type=rdf_file_finder.FileFinderCondition.Type. ACCESS_TIME, access_time=rdf_file_finder.FileFinderAccessTimeCondition( min_last_access_time=change_time)) for action in self.CONDITION_TESTS_ACTIONS: self.RunFlowAndCheckResults(action=action, conditions=[access_time_condition], expected_files=expected_files, non_expected_files=non_expected_files)