def testLegacyDataMigration(self):
res = rdf_memory.YaraProcessDumpResponse(dumped_processes=[
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="C:\\Foo\\Bar\\%s_%d_%x_%x.tmp" %
("my_proc", 123, 111, 222),
pathtype="TMPFILE"),
rdf_paths.PathSpec(path="/foo/bar/%s_%d_%x_%x.tmp" %
("my_proc", 123, 456, 789),
pathtype="TMPFILE")
])
])
memory._MigrateLegacyDumpFilesToMemoryAreas(res)
self.assertEqual(
res,
rdf_memory.YaraProcessDumpResponse(dumped_processes=[
rdf_memory.YaraProcessDumpInformation(memory_regions=[
rdf_memory.ProcessMemoryRegion(
start=111,
size=111,
file=rdf_paths.PathSpec(
path="/C:/Foo/Bar/%s_%d_%x_%x.tmp" %
("my_proc", 123, 111, 222),
pathtype="TMPFILE")),
rdf_memory.ProcessMemoryRegion(
start=456,
size=333,
file=rdf_paths.PathSpec(
path="/foo/bar/%s_%d_%x_%x.tmp" %
("my_proc", 123, 456, 789),
pathtype="TMPFILE"))
])
]))
def testMultipleClientsWithIdenticalPathsYieldDifferentPMIs(self):
client1 = self.SetupClient(1)
client2 = self.SetupClient(2)
yara_dump1 = rdf_memory.YaraProcessDumpResponse(dumped_processes=[
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE),
])
])
yara_dump2 = rdf_memory.YaraProcessDumpResponse(dumped_processes=[
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE),
])
])
stat_entry1 = rdf_client_fs.StatEntry(
pathspec=rdf_paths.PathSpec(
path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE))
stat_entry2 = rdf_client_fs.StatEntry(
pathspec=rdf_paths.PathSpec(
path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE))
m0 = rdf_flows.GrrMessage(source=client1, payload=yara_dump1)
m1 = rdf_flows.GrrMessage(source=client2, payload=yara_dump2)
m2 = rdf_flows.GrrMessage(source=client1, payload=stat_entry1)
m3 = rdf_flows.GrrMessage(source=client2, payload=stat_entry2)
self._ProcessValuesWithPlugin([m0, m1, m2, m3])
self.assertEqual(self.plugin.OutputMemoryDump.call_count, 2)
self.plugin.OutputMemoryDump.assert_any_call(
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE),
]), client1)
self.plugin.OutputMemoryDump.assert_any_call(
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE),
]), client2)
def testPathSpecCasingIsCorrected(self):
flow = memory.DumpProcessMemory(rdf_flow_objects.Flow())
flow.SendReply = mock.Mock(spec=flow.SendReply)
request = rdf_flow_objects.FlowRequest(
request_data={
"YaraProcessDumpResponse":
rdf_memory.YaraProcessDumpResponse(dumped_processes=[
rdf_memory.YaraProcessDumpInformation(memory_regions=[
rdf_memory.ProcessMemoryRegion(
start=1,
size=1,
file=rdf_paths.PathSpec.Temp(
path="/C:/grr/x_1_0_1.tmp")),
rdf_memory.ProcessMemoryRegion(
start=1,
size=1,
file=rdf_paths.PathSpec.Temp(
path="/C:/GRR/x_1_1_2.tmp"))
])
])
})
pathspecs = [
rdf_paths.PathSpec.Temp(path="/C:/Grr/x_1_0_1.tmp"),
rdf_paths.PathSpec.Temp(path="/C:/Grr/x_1_1_2.tmp")
]
responses = flow_responses.Responses.FromResponses(
request, [
rdf_flow_objects.FlowResponse(payload=rdf_client_fs.StatEntry(
pathspec=pathspec)) for pathspec in pathspecs
])
flow.ProcessMemoryRegions(responses)
flow.SendReply.assert_any_call(
rdf_memory.YaraProcessDumpResponse(dumped_processes=[
rdf_memory.YaraProcessDumpInformation(memory_regions=[
rdf_memory.ProcessMemoryRegion(
start=1,
size=1,
file=rdf_paths.PathSpec.Temp(
path="/C:/Grr/x_1_0_1.tmp")),
rdf_memory.ProcessMemoryRegion(
start=1,
size=1,
file=rdf_paths.PathSpec.Temp(
path="/C:/Grr/x_1_1_2.tmp"))
])
]))
def DumpProcess(self, psutil_process, args):
response = rdf_memory.YaraProcessDumpInformation()
response.process = rdf_client.Process.FromPsutilProcess(psutil_process)
streamer = streaming.Streamer(chunk_size=args.chunk_size)
with client_utils.OpenProcessForMemoryAccess(
psutil_process.pid) as process:
regions = list(client_utils.MemoryRegions(process, args))
if args.prioritize_offsets:
regions = _PrioritizeRegions(regions, args.prioritize_offsets)
if args.size_limit:
total_regions = len(regions)
regions = _ApplySizeLimit(regions, args.size_limit)
if len(regions) < total_regions:
response.error = ("Byte limit exceeded. Writing {} of {} "
"regions.").format(
len(regions), total_regions)
else:
for region in regions:
region.dumped_size = region.size
regions = sorted(regions, key=lambda r: r.start)
with tempfiles.TemporaryDirectory(cleanup=False) as tmp_dir:
for region in regions:
self.Progress()
pathspec = self._SaveRegionToDirectory(
psutil_process, process, region, tmp_dir, streamer)
if pathspec is not None:
region.file = pathspec
response.memory_regions.Append(region)
return response
def testOutputPluginPersistsStateCorrectly(self):
yara_dump_0 = rdf_memory.YaraProcessDumpResponse(dumped_processes=[
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE),
rdf_paths.PathSpec(path="my_proc_123_fa_104.tmp", pathtype=TMPFILE)
])
])
yara_dump_1 = rdf_memory.YaraProcessDumpResponse(dumped_processes=[
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="foobar_456_f0_fa.tmp", pathtype=TMPFILE),
])
])
stat_entry_00 = rdf_client_fs.StatEntry(
pathspec=rdf_paths.PathSpec(
path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE))
stat_entry_01 = rdf_client_fs.StatEntry(
pathspec=rdf_paths.PathSpec(
path="my_proc_123_fa_104.tmp", pathtype=TMPFILE))
stat_entry_10 = rdf_client_fs.StatEntry(
pathspec=rdf_paths.PathSpec(
path="foobar_456_f0_fa.tmp", pathtype=TMPFILE))
self._ProcessValuesWithPlugin(
[rdf_flows.GrrMessage(source=self.client_id, payload=yara_dump_0)])
self.plugin = None
self._ProcessValuesWithPlugin(
[rdf_flows.GrrMessage(source=self.client_id, payload=yara_dump_1)])
self.plugin = None
self._ProcessValuesWithPlugin(
[rdf_flows.GrrMessage(source=self.client_id, payload=stat_entry_00)])
self.plugin = None
self._ProcessValuesWithPlugin(
[rdf_flows.GrrMessage(source=self.client_id, payload=stat_entry_01)])
self.plugin.OutputMemoryDump.assert_called_once_with(
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE),
rdf_paths.PathSpec(path="my_proc_123_fa_104.tmp", pathtype=TMPFILE)
]), self.client_id)
self.plugin = None
self._ProcessValuesWithPlugin(
[rdf_flows.GrrMessage(source=self.client_id, payload=stat_entry_10)])
self.plugin.OutputMemoryDump.assert_called_once_with(
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="foobar_456_f0_fa.tmp", pathtype=TMPFILE),
]), self.client_id)
def testCallsOutputMemoryDumpWithSingleBlob(self):
yara_dump = rdf_memory.YaraProcessDumpResponse(dumped_processes=[
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE),
])
])
stat_entry = rdf_client_fs.StatEntry(
pathspec=rdf_paths.PathSpec(
path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE))
m0 = rdf_flows.GrrMessage(source=self.client_id, payload=yara_dump)
m1 = rdf_flows.GrrMessage(source=self.client_id, payload=stat_entry)
self._ProcessValuesWithPlugin([m0, m1])
self.plugin.OutputMemoryDump.assert_called_once_with(
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE),
]), self.client_id)
def DumpProcess(self, psutil_process, args):
response = rdf_memory.YaraProcessDumpInformation()
response.process = rdf_client.Process.FromPsutilProcess(psutil_process)
process = client_utils.OpenProcessForMemoryAccess(
pid=psutil_process.pid)
bytes_limit = args.size_limit
with process:
streamer = streaming.Streamer(chunk_size=args.chunk_size)
with tempfiles.TemporaryDirectory(cleanup=False) as tmp_dir:
for region in client_utils.MemoryRegions(process, args):
if bytes_limit and self.bytes_written + region.size > bytes_limit:
response.error = (
"Byte limit exceeded. Wrote %d bytes, "
"next block is %d bytes, limit is %d." %
(self.bytes_written, region.size, bytes_limit))
return response
end = region.start + region.size
# _ReplaceDumpPathspecsWithMultiGetFilePathspec in DumpProcessMemory
# flow asserts that MemoryRegions can be uniquely identified by their
# file's basename.
filename = "%s_%d_%x_%x.tmp" % (psutil_process.name(),
psutil_process.pid,
region.start, end)
filepath = os.path.join(tmp_dir.path, filename)
chunks = streamer.StreamMemory(process,
offset=region.start,
amount=region.size)
bytes_written = self._SaveMemDumpToFilePath(
filepath, chunks)
if not bytes_written:
continue
self.bytes_written += bytes_written
# TODO: Remove workaround after client_utils are fixed.
canonical_path = client_utils.LocalPathToCanonicalPath(
filepath)
if not canonical_path.startswith("/"):
canonical_path = "/" + canonical_path
region.file = rdf_paths.PathSpec(
path=canonical_path,
pathtype=rdf_paths.PathSpec.PathType.TMPFILE)
response.memory_regions.Append(region)
return response
def testProcessesMultipleYaraProcessDumpResponsesCorrectly(self):
yara_dump_0 = rdf_memory.YaraProcessDumpResponse(dumped_processes=[
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE),
rdf_paths.PathSpec(path="my_proc_123_fa_104.tmp", pathtype=TMPFILE)
])
])
yara_dump_1 = rdf_memory.YaraProcessDumpResponse(dumped_processes=[
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="foobar_456_f0_fa.tmp", pathtype=TMPFILE),
])
])
stat_entry_00 = rdf_client_fs.StatEntry(
pathspec=rdf_paths.PathSpec(
path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE))
stat_entry_01 = rdf_client_fs.StatEntry(
pathspec=rdf_paths.PathSpec(
path="my_proc_123_fa_104.tmp", pathtype=TMPFILE))
stat_entry_10 = rdf_client_fs.StatEntry(
pathspec=rdf_paths.PathSpec(
path="foobar_456_f0_fa.tmp", pathtype=TMPFILE))
m0 = rdf_flows.GrrMessage(source=self.client_id, payload=yara_dump_0)
m1 = rdf_flows.GrrMessage(source=self.client_id, payload=yara_dump_1)
m2 = rdf_flows.GrrMessage(source=self.client_id, payload=stat_entry_00)
m3 = rdf_flows.GrrMessage(source=self.client_id, payload=stat_entry_01)
m4 = rdf_flows.GrrMessage(source=self.client_id, payload=stat_entry_10)
self._ProcessValuesWithPlugin([m0, m1, m2, m3, m4])
self.assertEqual(self.plugin.OutputMemoryDump.call_count, 2)
self.plugin.OutputMemoryDump.assert_any_call(
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="my_proc_123_f0_fa.tmp", pathtype=TMPFILE),
rdf_paths.PathSpec(path="my_proc_123_fa_104.tmp", pathtype=TMPFILE)
]), self.client_id)
self.plugin.OutputMemoryDump.assert_any_call(
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(path="foobar_456_f0_fa.tmp", pathtype=TMPFILE),
]), self.client_id)
def testIsResilientToTemporaryPathSpecRegression(self):
yara_dump = rdf_memory.YaraProcessDumpResponse(dumped_processes=[
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(
path="C:\\Foo\\my_proc_123_f0_fa.tmp", pathtype=TMPFILE),
])
])
stat_entry = rdf_client_fs.StatEntry(
pathspec=rdf_paths.PathSpec(
path="/C:/Foo/my_proc_123_f0_fa.tmp",
pathtype=TMPFILE,
path_options=rdf_paths.PathSpec.Options.CASE_LITERAL))
m0 = rdf_flows.GrrMessage(source=self.client_id, payload=yara_dump)
m1 = rdf_flows.GrrMessage(source=self.client_id, payload=stat_entry)
self._ProcessValuesWithPlugin([m0, m1])
self.plugin.OutputMemoryDump.assert_called_once_with(
rdf_memory.YaraProcessDumpInformation(dump_files=[
rdf_paths.PathSpec(
path="/C:/Foo/my_proc_123_f0_fa.tmp",
pathtype=TMPFILE,
path_options=rdf_paths.PathSpec.Options.CASE_LITERAL),
]), self.client_id)
def DumpProcess(self, psutil_process, args):
response = rdf_memory.YaraProcessDumpInformation()
response.process = rdf_client.Process.FromPsutilProcess(psutil_process)
process = client_utils.OpenProcessForMemoryAccess(
pid=psutil_process.pid)
bytes_limit = args.size_limit
with process:
streamer = streaming.Streamer(chunk_size=args.chunk_size)
with tempfiles.TemporaryDirectory(cleanup=False) as tmp_dir:
for start, length in client_utils.MemoryRegions(process, args):
if bytes_limit and self.bytes_written + length > bytes_limit:
response.error = (
"Byte limit exceeded. Wrote %d bytes, "
"next block is %d bytes, limit is %d." %
(self.bytes_written, length, bytes_limit))
return response
end = start + length
# TODO: The filename is parsed on the server side to
# extract the memory address again. This should be changed by
# saving the `start` and `end` in YaraProcessDumpInformation.
filename = "%s_%d_%x_%x.tmp" % (
psutil_process.name(), psutil_process.pid, start, end)
filepath = os.path.join(tmp_dir.path, filename)
chunks = streamer.StreamMemory(process,
offset=start,
amount=length)
bytes_written = self._SaveMemDumpToFilePath(
filepath, chunks)
if not bytes_written:
continue
self.bytes_written += bytes_written
response.dump_files.Append(
rdf_paths.PathSpec(
path=filepath,
pathtype=rdf_paths.PathSpec.PathType.TMPFILE))
return response
def ProcessResponses(self, state, responses):
"""Processes Yara memory dumps and collected memory areas.
Args:
state: persistent plugin state
responses: GrrMessages, containing YaraProcessDumpResponse, StatEntry, and
other RDFValues
"""
# This Plugin keeps local state, grouped by the client id that sent the
# Yara dump:
# - `dumps` is a list of all seen YaraProcessDumpInformation
# - `paths` is a list of all collected memory areas
# - `completed` is a list of all exported YaraProcessDumpInformation
self._ReInitializeLocalState(state)
for response in responses:
client_id = response.source.Basename()
client_state = self._GetOrInitClientState(client_id)
# First, add the data that came in to the Plugin's state.
if isinstance(response.payload,
rdf_memory.YaraProcessDumpResponse):
client_state.dumps.Extend(response.payload.dumped_processes)
elif isinstance(response.payload, rdf_client_fs.StatEntry):
client_state.paths.Append(response.payload.pathspec)
# Second, iterate through the state and process all Yara dumps whose
# memory areas have been fully collected.
for process_dump in self._IterateReadyProcessDumpsOnce(
client_state):
# TODO: Fix Windows PathSpec inconsistency.
paths = _NormalizePaths(client_state.paths,
process_dump.dump_files)
fixed_pd = rdf_memory.YaraProcessDumpInformation(process_dump)
fixed_pd.dump_files = paths
self.OutputMemoryDump(fixed_pd, client_id)
