def testApplyMultipleParsersError(self): class QuuxParseError(parsers.ParseError): pass class QuuxParser(parsers.MultiResponseParser): supported_artifacts = ["Quux"] def ParseResponses( self, knowledge_base: rdf_client.KnowledgeBase, responses: Collection[rdf_client_fs.StatEntry], ) -> Iterable[rdfvalue.RDFInteger]: del knowledge_base, responses # Unused. raise QuuxParseError("Lorem ipsum.") with parser_test_lib._ParserContext("Quux", QuuxParser): factory = parsers.ArtifactParserFactory("Quux") client_id = self.client_id knowledge_base = rdf_client.KnowledgeBase() applicator = artifact.ParserApplicator(factory, client_id, knowledge_base) applicator.Apply([rdf_client_fs.StatEntry()]) errors = list(applicator.Errors()) self.assertLen(errors, 1) self.assertIsInstance(errors[0], QuuxParseError) responses = list(applicator.Responses()) self.assertEmpty(responses)
def testSingleFileResponse(self): class NorfParser(parsers.SingleFileParser): supported_artifacts = ["Norf"] def ParseFile( self, knowledge_base: rdf_client.KnowledgeBase, pathspec: rdf_paths.PathSpec, filedesc: file_store.BlobStream, ) -> Iterable[rdfvalue.RDFBytes]: del knowledge_base, pathspec # Unused. return [rdfvalue.RDFBytes(filedesc.Read())] with parser_test_lib._ParserContext("Norf", NorfParser): factory = parsers.ArtifactParserFactory("Norf") client_id = self.client_id knowledge_base = rdf_client.KnowledgeBase() stat_entry = rdf_client_fs.StatEntry() stat_entry.pathspec.path = "foo/bar/baz" stat_entry.pathspec.pathtype = rdf_paths.PathSpec.PathType.OS self._WriteFile(stat_entry.pathspec.path, b"4815162342") applicator = artifact.ParserApplicator(factory, client_id, knowledge_base) applicator.Apply([stat_entry]) errors = list(applicator.Errors()) self.assertEmpty(errors) responses = list(applicator.Responses()) self.assertLen(responses, 1) self.assertEqual(responses[0], b"4815162342")
def testApplySingleResponseError(self): class FooParseError(parsers.ParseError): pass class FooParser(parsers.SingleResponseParser): supported_artifacts = ["Foo"] def ParseResponse( self, knowledge_base: rdf_client.KnowledgeBase, response: rdf_client_fs.StatEntry, ) -> Iterable[rdfvalue.RDFString]: del knowledge_base, response # Unused. raise FooParseError("Lorem ipsum.") with parser_test_lib._ParserContext("Foo", FooParser): factory = parsers.ArtifactParserFactory("Foo") client_id = self.client_id knowledge_base = rdf_client.KnowledgeBase() applicator = artifact.ParserApplicator(factory, client_id, knowledge_base) applicator.Apply([rdf_client_fs.StatEntry()]) errors = list(applicator.Errors()) self.assertLen(errors, 1) self.assertIsInstance(errors[0], FooParseError) responses = list(applicator.Responses()) self.assertEmpty(responses)
def testApplyMultiResponseSuccess(self): class QuuxParser(parsers.MultiResponseParser): supported_artifacts = ["Quux"] def ParseResponses( self, knowledge_base: rdf_client.KnowledgeBase, responses: Collection[rdf_client_fs.StatEntry], ) -> Iterable[rdfvalue.RDFInteger]: return [stat_entry.st_dev for stat_entry in responses] with parser_test_lib._ParserContext("Quux", QuuxParser): factory = parsers.ArtifactParserFactory("Quux") client_id = self.client_id knowledge_base = rdf_client.KnowledgeBase() applicator = artifact.ParserApplicator(factory, client_id, knowledge_base) applicator.Apply([ rdf_client_fs.StatEntry(st_dev=42), rdf_client_fs.StatEntry(st_dev=1337), ]) errors = list(applicator.Errors()) self.assertEmpty(errors) responses = list(applicator.Responses()) self.assertCountEqual(responses, [42, 1337])
def testApplySingleResponseSuccessful(self): class FooParser(parsers.SingleResponseParser): supported_artifacts = ["Foo"] def ParseResponse( self, knowledge_base: rdf_client.KnowledgeBase, response: rdf_client_fs.StatEntry, ) -> Iterable[rdfvalue.RDFString]: return [ rdfvalue.RDFString( f"{knowledge_base.os}:{response.st_dev}") ] with parser_test_lib._ParserContext("Foo", FooParser): factory = parsers.ArtifactParserFactory("Foo") client_id = self.client_id knowledge_base = rdf_client.KnowledgeBase(os="Redox") applicator = artifact.ParserApplicator(factory, client_id, knowledge_base) applicator.Apply([rdf_client_fs.StatEntry(st_dev=1337)]) errors = list(applicator.Errors()) self.assertEmpty(errors) responses = list(applicator.Responses()) self.assertEqual(responses, ["Redox:1337"])
def testMultiFileSuccess(self): class ThudParser(parsers.MultiFileParser): supported_artifacts = ["Thud"] def ParseFiles( self, knowledge_base: rdf_client.KnowledgeBase, pathspecs: Collection[rdf_paths.PathSpec], filedescs: Collection[file_store.BlobStream], ) -> Iterable[rdf_protodict.Dict]: results = [] for pathspec, filedesc in zip(pathspecs, filedescs): result = rdf_protodict.Dict() result["path"] = pathspec.path result["content"] = filedesc.Read() results.append(result) return results with parser_test_lib._ParserContext("Thud", ThudParser): factory = parsers.ArtifactParserFactory("Thud") client_id = self.client_id knowledge_base = rdf_client.KnowledgeBase() stat_entry_foo = rdf_client_fs.StatEntry() stat_entry_foo.pathspec.path = "quux/foo" stat_entry_foo.pathspec.pathtype = rdf_paths.PathSpec.PathType.OS self._WriteFile(stat_entry_foo.pathspec.path, b"FOO") stat_entry_bar = rdf_client_fs.StatEntry() stat_entry_bar.pathspec.path = "quux/bar" stat_entry_bar.pathspec.pathtype = rdf_paths.PathSpec.PathType.OS self._WriteFile(stat_entry_bar.pathspec.path, b"BAR") applicator = artifact.ParserApplicator(factory, client_id, knowledge_base) applicator.Apply([stat_entry_foo, stat_entry_bar]) errors = list(applicator.Errors()) self.assertEmpty(errors) responses = list(applicator.Responses()) self.assertLen(responses, 2) self.assertEqual(responses[0], { "path": "quux/foo", "content": b"FOO" }) self.assertEqual(responses[1], { "path": "quux/bar", "content": b"BAR" })
def testSingleResponseAndSingleFileParser(self): class FooParser(parsers.SingleResponseParser[rdfvalue.RDFString]): supported_artifacts = ["Quux"] def ParseResponse( self, knowledge_base: rdf_client.KnowledgeBase, response: rdfvalue.RDFValue, ) -> Iterator[rdfvalue.RDFString]: del knowledge_base # Unused. if not isinstance(response, rdfvalue.RDFString): raise TypeError(f"Unexpected response type: {type(response)}") yield rdfvalue.RDFString(f"FOO-{response}") class BarParser(parsers.SingleFileParser[rdfvalue.RDFString]): supported_artifacts = ["Quux"] def ParseFile( self, knowledge_base: rdf_client.KnowledgeBase, pathspec: rdf_paths.PathSpec, filedesc: IO[bytes], ) -> Iterator[rdfvalue.RDFString]: del knowledge_base, pathspec, filedesc # Unused. yield rdfvalue.RDFString("BAR") with parser_test_lib._ParserContext("Foo", FooParser): with parser_test_lib._ParserContext("Bar", BarParser): factory = parsers.ArtifactParserFactory("Quux") knowledge_base = rdf_client.KnowledgeBase() applicator = artifact.ParserApplicator( factory, client_id=self.client_id, knowledge_base=knowledge_base) applicator.Apply([ rdfvalue.RDFString("THUD"), rdfvalue.RDFString("BLARGH"), ]) responses = list(applicator.Responses()) self.assertLen(responses, 2) self.assertEqual(responses[0], rdfvalue.RDFString("FOO-THUD")) self.assertEqual(responses[1], rdfvalue.RDFString("FOO-BLARGH"))
def testSingleResponseAndSingleFileParserWithStatResponse(self): class FooParser(parsers.SingleResponseParser[rdfvalue.RDFString]): supported_artifacts = ["Quux"] def ParseResponse( self, knowledge_base: rdf_client.KnowledgeBase, response: rdfvalue.RDFValue, ) -> Iterator[rdfvalue.RDFString]: del knowledge_base # Unused. if not isinstance(response, rdf_client_fs.StatEntry): raise TypeError(f"Unexpected response type: {type(response)}") yield rdfvalue.RDFString(f"PATH('{response.pathspec.path}')") class BarParser(parsers.SingleFileParser[rdfvalue.RDFString]): supported_artifacts = ["Quux"] def ParseFile( self, knowledge_base: rdf_client.KnowledgeBase, pathspec: rdf_paths.PathSpec, filedesc: IO[bytes], ) -> Iterator[rdfvalue.RDFString]: raise NotImplementedError() with parser_test_lib._ParserContext("Foo", FooParser): with parser_test_lib._ParserContext("Bar", BarParser): factory = parsers.ArtifactParserFactory("Quux") knowledge_base = rdf_client.KnowledgeBase() stat_entry = rdf_client_fs.StatEntry() stat_entry.pathspec.path = "foo/bar/baz" stat_entry.pathspec.pathtype = rdf_paths.PathSpec.PathType.OS applicator = artifact.ParserApplicator( factory, client_id=self.client_id, knowledge_base=knowledge_base) applicator.Apply([stat_entry]) responses = list(applicator.Responses()) self.assertLen(responses, 1) self.assertEqual(responses[0], "PATH('foo/bar/baz')")
def testMultiFileError(self): class ThudParseError(parsers.ParseError): pass class ThudParser(parsers.MultiFileParser): supported_artifacts = ["Thud"] def ParseFiles( self, knowledge_base: rdf_client.KnowledgeBase, pathspecs: Collection[rdf_paths.PathSpec], filedescs: Collection[file_store.BlobStream], ) -> Iterable[rdf_protodict.Dict]: del knowledge_base, pathspecs, filedescs # Unused. raise ThudParseError("Lorem ipsum.") with parser_test_lib._ParserContext("Thud", ThudParser): factory = parsers.ArtifactParserFactory("Thud") client_id = self.client_id knowledge_base = rdf_client.KnowledgeBase() stat_entry = rdf_client_fs.StatEntry() stat_entry.pathspec.path = "foo/bar/baz" stat_entry.pathspec.pathtype = rdf_paths.PathSpec.PathType.OS self._WriteFile(stat_entry.pathspec.path, b"\xff\x00\xff") applicator = artifact.ParserApplicator(factory, client_id, knowledge_base) applicator.Apply([stat_entry]) errors = list(applicator.Errors()) self.assertLen(errors, 1) self.assertIsInstance(errors[0], ThudParseError) responses = list(applicator.Responses()) self.assertEmpty(responses)
def Handle( self, args: ApiListParsedFlowResultsArgs, token: access_control.ACLToken, ) -> ApiListParsedFlowResultsResult: client_id = str(args.client_id) flow_id = str(args.flow_id) flow_obj = data_store.REL_DB.ReadFlowObject(client_id, flow_id) if flow_obj.flow_class_name != collectors.ArtifactCollectorFlow.__name__: message = "Not an artifact-collector flow: {}" raise ValueError(message.format(flow_obj.flow_class_name)) if flow_obj.args.apply_parsers: message = "Flow already parsed its results" raise ValueError(message) flow_results = data_store.REL_DB.ReadFlowResults(client_id=client_id, flow_id=flow_id, offset=args.offset, count=args.count) # We determine results collection timestamp as the maximum of timestamps of # individual flow results. We cannot use just the flow timestamp for this, # because flows can be modified, affecting the timestamp. We also don't want # to use flow start time, because it can be to "early" to do parsing. if flow_results: flow_results_timestamp = max([_.timestamp for _ in flow_results]) else: flow_results_timestamp = None flow_results_by_artifact = {} for flow_result in flow_results: artifact_name_match = self._TAG_ARTIFACT_NAME.match( flow_result.tag) if artifact_name_match is None: continue artifact_name = artifact_name_match["name"] artifact_results = flow_results_by_artifact.setdefault( artifact_name, []) artifact_results.append(flow_result) knowledge_base = flow_obj.persistent_data["knowledge_base"] result = ApiListParsedFlowResultsResult() for artifact_name, flow_results in flow_results_by_artifact.items(): factory = parsers.ArtifactParserFactory(artifact_name) applicator = artifact.ParserApplicator( factory, client_id=client_id, knowledge_base=knowledge_base, timestamp=flow_results_timestamp) applicator.Apply( [flow_result.payload for flow_result in flow_results]) for response in applicator.Responses(): item = ApiFlowResult() item.payload_type = response.__class__.__name__ item.payload = response item.tag = f"artifact:{artifact_name}" result.items.Append(item) result.errors.Extend(map(str, applicator.Errors())) return result