def testApplyMultipleParsersError(self):
class QuuxParseError(parsers.ParseError):
pass
class QuuxParser(parsers.MultiResponseParser):
supported_artifacts = ["Quux"]
def ParseResponses(
self,
knowledge_base: rdf_client.KnowledgeBase,
responses: Collection[rdf_client_fs.StatEntry],
) -> Iterable[rdfvalue.RDFInteger]:
del knowledge_base, responses # Unused.
raise QuuxParseError("Lorem ipsum.")
with parser_test_lib._ParserContext("Quux", QuuxParser):
factory = parsers.ArtifactParserFactory("Quux")
client_id = self.client_id
knowledge_base = rdf_client.KnowledgeBase()
applicator = artifact.ParserApplicator(factory, client_id,
knowledge_base)
applicator.Apply([rdf_client_fs.StatEntry()])
errors = list(applicator.Errors())
self.assertLen(errors, 1)
self.assertIsInstance(errors[0], QuuxParseError)
responses = list(applicator.Responses())
self.assertEmpty(responses)
def testSingleFileResponse(self):
class NorfParser(parsers.SingleFileParser):
supported_artifacts = ["Norf"]
def ParseFile(
self,
knowledge_base: rdf_client.KnowledgeBase,
pathspec: rdf_paths.PathSpec,
filedesc: file_store.BlobStream,
) -> Iterable[rdfvalue.RDFBytes]:
del knowledge_base, pathspec # Unused.
return [rdfvalue.RDFBytes(filedesc.Read())]
with parser_test_lib._ParserContext("Norf", NorfParser):
factory = parsers.ArtifactParserFactory("Norf")
client_id = self.client_id
knowledge_base = rdf_client.KnowledgeBase()
stat_entry = rdf_client_fs.StatEntry()
stat_entry.pathspec.path = "foo/bar/baz"
stat_entry.pathspec.pathtype = rdf_paths.PathSpec.PathType.OS
self._WriteFile(stat_entry.pathspec.path, b"4815162342")
applicator = artifact.ParserApplicator(factory, client_id,
knowledge_base)
applicator.Apply([stat_entry])
errors = list(applicator.Errors())
self.assertEmpty(errors)
responses = list(applicator.Responses())
self.assertLen(responses, 1)
self.assertEqual(responses[0], b"4815162342")
def testApplySingleResponseError(self):
class FooParseError(parsers.ParseError):
pass
class FooParser(parsers.SingleResponseParser):
supported_artifacts = ["Foo"]
def ParseResponse(
self,
knowledge_base: rdf_client.KnowledgeBase,
response: rdf_client_fs.StatEntry,
) -> Iterable[rdfvalue.RDFString]:
del knowledge_base, response # Unused.
raise FooParseError("Lorem ipsum.")
with parser_test_lib._ParserContext("Foo", FooParser):
factory = parsers.ArtifactParserFactory("Foo")
client_id = self.client_id
knowledge_base = rdf_client.KnowledgeBase()
applicator = artifact.ParserApplicator(factory, client_id,
knowledge_base)
applicator.Apply([rdf_client_fs.StatEntry()])
errors = list(applicator.Errors())
self.assertLen(errors, 1)
self.assertIsInstance(errors[0], FooParseError)
responses = list(applicator.Responses())
self.assertEmpty(responses)
def testApplyMultiResponseSuccess(self):
class QuuxParser(parsers.MultiResponseParser):
supported_artifacts = ["Quux"]
def ParseResponses(
self,
knowledge_base: rdf_client.KnowledgeBase,
responses: Collection[rdf_client_fs.StatEntry],
) -> Iterable[rdfvalue.RDFInteger]:
return [stat_entry.st_dev for stat_entry in responses]
with parser_test_lib._ParserContext("Quux", QuuxParser):
factory = parsers.ArtifactParserFactory("Quux")
client_id = self.client_id
knowledge_base = rdf_client.KnowledgeBase()
applicator = artifact.ParserApplicator(factory, client_id,
knowledge_base)
applicator.Apply([
rdf_client_fs.StatEntry(st_dev=42),
rdf_client_fs.StatEntry(st_dev=1337),
])
errors = list(applicator.Errors())
self.assertEmpty(errors)
responses = list(applicator.Responses())
self.assertCountEqual(responses, [42, 1337])
def testApplySingleResponseSuccessful(self):
class FooParser(parsers.SingleResponseParser):
supported_artifacts = ["Foo"]
def ParseResponse(
self,
knowledge_base: rdf_client.KnowledgeBase,
response: rdf_client_fs.StatEntry,
) -> Iterable[rdfvalue.RDFString]:
return [
rdfvalue.RDFString(
f"{knowledge_base.os}:{response.st_dev}")
]
with parser_test_lib._ParserContext("Foo", FooParser):
factory = parsers.ArtifactParserFactory("Foo")
client_id = self.client_id
knowledge_base = rdf_client.KnowledgeBase(os="Redox")
applicator = artifact.ParserApplicator(factory, client_id,
knowledge_base)
applicator.Apply([rdf_client_fs.StatEntry(st_dev=1337)])
errors = list(applicator.Errors())
self.assertEmpty(errors)
responses = list(applicator.Responses())
self.assertEqual(responses, ["Redox:1337"])
def testMultiFileSuccess(self):
class ThudParser(parsers.MultiFileParser):
supported_artifacts = ["Thud"]
def ParseFiles(
self,
knowledge_base: rdf_client.KnowledgeBase,
pathspecs: Collection[rdf_paths.PathSpec],
filedescs: Collection[file_store.BlobStream],
) -> Iterable[rdf_protodict.Dict]:
results = []
for pathspec, filedesc in zip(pathspecs, filedescs):
result = rdf_protodict.Dict()
result["path"] = pathspec.path
result["content"] = filedesc.Read()
results.append(result)
return results
with parser_test_lib._ParserContext("Thud", ThudParser):
factory = parsers.ArtifactParserFactory("Thud")
client_id = self.client_id
knowledge_base = rdf_client.KnowledgeBase()
stat_entry_foo = rdf_client_fs.StatEntry()
stat_entry_foo.pathspec.path = "quux/foo"
stat_entry_foo.pathspec.pathtype = rdf_paths.PathSpec.PathType.OS
self._WriteFile(stat_entry_foo.pathspec.path, b"FOO")
stat_entry_bar = rdf_client_fs.StatEntry()
stat_entry_bar.pathspec.path = "quux/bar"
stat_entry_bar.pathspec.pathtype = rdf_paths.PathSpec.PathType.OS
self._WriteFile(stat_entry_bar.pathspec.path, b"BAR")
applicator = artifact.ParserApplicator(factory, client_id,
knowledge_base)
applicator.Apply([stat_entry_foo, stat_entry_bar])
errors = list(applicator.Errors())
self.assertEmpty(errors)
responses = list(applicator.Responses())
self.assertLen(responses, 2)
self.assertEqual(responses[0], {
"path": "quux/foo",
"content": b"FOO"
})
self.assertEqual(responses[1], {
"path": "quux/bar",
"content": b"BAR"
})
def testSingleResponseAndSingleFileParser(self):
class FooParser(parsers.SingleResponseParser[rdfvalue.RDFString]):
supported_artifacts = ["Quux"]
def ParseResponse(
self,
knowledge_base: rdf_client.KnowledgeBase,
response: rdfvalue.RDFValue,
) -> Iterator[rdfvalue.RDFString]:
del knowledge_base # Unused.
if not isinstance(response, rdfvalue.RDFString):
raise TypeError(f"Unexpected response type: {type(response)}")
yield rdfvalue.RDFString(f"FOO-{response}")
class BarParser(parsers.SingleFileParser[rdfvalue.RDFString]):
supported_artifacts = ["Quux"]
def ParseFile(
self,
knowledge_base: rdf_client.KnowledgeBase,
pathspec: rdf_paths.PathSpec,
filedesc: IO[bytes],
) -> Iterator[rdfvalue.RDFString]:
del knowledge_base, pathspec, filedesc # Unused.
yield rdfvalue.RDFString("BAR")
with parser_test_lib._ParserContext("Foo", FooParser):
with parser_test_lib._ParserContext("Bar", BarParser):
factory = parsers.ArtifactParserFactory("Quux")
knowledge_base = rdf_client.KnowledgeBase()
applicator = artifact.ParserApplicator(
factory, client_id=self.client_id, knowledge_base=knowledge_base)
applicator.Apply([
rdfvalue.RDFString("THUD"),
rdfvalue.RDFString("BLARGH"),
])
responses = list(applicator.Responses())
self.assertLen(responses, 2)
self.assertEqual(responses[0], rdfvalue.RDFString("FOO-THUD"))
self.assertEqual(responses[1], rdfvalue.RDFString("FOO-BLARGH"))
def testSingleResponseAndSingleFileParserWithStatResponse(self):
class FooParser(parsers.SingleResponseParser[rdfvalue.RDFString]):
supported_artifacts = ["Quux"]
def ParseResponse(
self,
knowledge_base: rdf_client.KnowledgeBase,
response: rdfvalue.RDFValue,
) -> Iterator[rdfvalue.RDFString]:
del knowledge_base # Unused.
if not isinstance(response, rdf_client_fs.StatEntry):
raise TypeError(f"Unexpected response type: {type(response)}")
yield rdfvalue.RDFString(f"PATH('{response.pathspec.path}')")
class BarParser(parsers.SingleFileParser[rdfvalue.RDFString]):
supported_artifacts = ["Quux"]
def ParseFile(
self,
knowledge_base: rdf_client.KnowledgeBase,
pathspec: rdf_paths.PathSpec,
filedesc: IO[bytes],
) -> Iterator[rdfvalue.RDFString]:
raise NotImplementedError()
with parser_test_lib._ParserContext("Foo", FooParser):
with parser_test_lib._ParserContext("Bar", BarParser):
factory = parsers.ArtifactParserFactory("Quux")
knowledge_base = rdf_client.KnowledgeBase()
stat_entry = rdf_client_fs.StatEntry()
stat_entry.pathspec.path = "foo/bar/baz"
stat_entry.pathspec.pathtype = rdf_paths.PathSpec.PathType.OS
applicator = artifact.ParserApplicator(
factory, client_id=self.client_id, knowledge_base=knowledge_base)
applicator.Apply([stat_entry])
responses = list(applicator.Responses())
self.assertLen(responses, 1)
self.assertEqual(responses[0], "PATH('foo/bar/baz')")
def testMultiFileError(self):
class ThudParseError(parsers.ParseError):
pass
class ThudParser(parsers.MultiFileParser):
supported_artifacts = ["Thud"]
def ParseFiles(
self,
knowledge_base: rdf_client.KnowledgeBase,
pathspecs: Collection[rdf_paths.PathSpec],
filedescs: Collection[file_store.BlobStream],
) -> Iterable[rdf_protodict.Dict]:
del knowledge_base, pathspecs, filedescs # Unused.
raise ThudParseError("Lorem ipsum.")
with parser_test_lib._ParserContext("Thud", ThudParser):
factory = parsers.ArtifactParserFactory("Thud")
client_id = self.client_id
knowledge_base = rdf_client.KnowledgeBase()
stat_entry = rdf_client_fs.StatEntry()
stat_entry.pathspec.path = "foo/bar/baz"
stat_entry.pathspec.pathtype = rdf_paths.PathSpec.PathType.OS
self._WriteFile(stat_entry.pathspec.path, b"\xff\x00\xff")
applicator = artifact.ParserApplicator(factory, client_id,
knowledge_base)
applicator.Apply([stat_entry])
errors = list(applicator.Errors())
self.assertLen(errors, 1)
self.assertIsInstance(errors[0], ThudParseError)
responses = list(applicator.Responses())
self.assertEmpty(responses)
def Handle(
self,
args: ApiListParsedFlowResultsArgs,
token: access_control.ACLToken,
) -> ApiListParsedFlowResultsResult:
client_id = str(args.client_id)
flow_id = str(args.flow_id)
flow_obj = data_store.REL_DB.ReadFlowObject(client_id, flow_id)
if flow_obj.flow_class_name != collectors.ArtifactCollectorFlow.__name__:
message = "Not an artifact-collector flow: {}"
raise ValueError(message.format(flow_obj.flow_class_name))
if flow_obj.args.apply_parsers:
message = "Flow already parsed its results"
raise ValueError(message)
flow_results = data_store.REL_DB.ReadFlowResults(client_id=client_id,
flow_id=flow_id,
offset=args.offset,
count=args.count)
# We determine results collection timestamp as the maximum of timestamps of
# individual flow results. We cannot use just the flow timestamp for this,
# because flows can be modified, affecting the timestamp. We also don't want
# to use flow start time, because it can be to "early" to do parsing.
if flow_results:
flow_results_timestamp = max([_.timestamp for _ in flow_results])
else:
flow_results_timestamp = None
flow_results_by_artifact = {}
for flow_result in flow_results:
artifact_name_match = self._TAG_ARTIFACT_NAME.match(
flow_result.tag)
if artifact_name_match is None:
continue
artifact_name = artifact_name_match["name"]
artifact_results = flow_results_by_artifact.setdefault(
artifact_name, [])
artifact_results.append(flow_result)
knowledge_base = flow_obj.persistent_data["knowledge_base"]
result = ApiListParsedFlowResultsResult()
for artifact_name, flow_results in flow_results_by_artifact.items():
factory = parsers.ArtifactParserFactory(artifact_name)
applicator = artifact.ParserApplicator(
factory,
client_id=client_id,
knowledge_base=knowledge_base,
timestamp=flow_results_timestamp)
applicator.Apply(
[flow_result.payload for flow_result in flow_results])
for response in applicator.Responses():
item = ApiFlowResult()
item.payload_type = response.__class__.__name__
item.payload = response
item.tag = f"artifact:{artifact_name}"
result.items.Append(item)
result.errors.Extend(map(str, applicator.Errors()))
return result
