def _SaveRegionToDirectory(self, psutil_process, process, region, tmp_dir,
streamer):
end = region.start + region.size
# _ReplaceDumpPathspecsWithMultiGetFilePathspec in DumpProcessMemory
# flow asserts that MemoryRegions can be uniquely identified by their
# file's basename.
filename = "%s_%d_%x_%x.tmp" % (psutil_process.name(),
psutil_process.pid, region.start, end)
filepath = os.path.join(tmp_dir.path, filename)
chunks = streamer.StreamMemory(process,
offset=region.start,
amount=region.dumped_size)
bytes_written = self._SaveMemDumpToFilePath(filepath, chunks)
if not bytes_written:
return None
# TODO: Remove workaround after client_utils are fixed.
canonical_path = client_utils.LocalPathToCanonicalPath(filepath)
if not canonical_path.startswith("/"):
canonical_path = "/" + canonical_path
return rdf_paths.PathSpec(path=canonical_path,
pathtype=rdf_paths.PathSpec.PathType.TMPFILE)
def Open(cls, fd, component, pathspec=None, progress_callback=None):
# A Pathspec which starts with TSK means we need to resolve the mount point
# at runtime.
if fd is None and component.pathtype == rdf_paths.PathSpec.PathType.TSK:
# We are the top level handler. This means we need to check the system
# mounts to work out the exact mount point and device we need to
# open. We then modify the pathspec so we get nested in the raw
# pathspec.
raw_pathspec, corrected_path = client_utils.GetRawDevice(component.path)
# Insert the raw device before the component in the pathspec and correct
# the path
component.path = corrected_path
pathspec.Insert(0, component)
pathspec.Insert(0, raw_pathspec)
# Allow incoming pathspec to be given in the local system path
# conventions.
for component in pathspec:
if component.path:
component.path = client_utils.LocalPathToCanonicalPath(component.path)
# We have not actually opened anything in this iteration, but modified the
# pathspec. Next time we should be able to open it properly.
return fd
# If an inode is specified, just use it directly.
elif component.HasField("inode"):
return TSKFile(fd, component, progress_callback=progress_callback)
# Otherwise do the usual case folding.
else:
return vfs.VFSHandler.Open(
fd, component, pathspec=pathspec, progress_callback=progress_callback)
def Open(cls, fd, component, handlers, pathspec=None, progress_callback=None):
# A Pathspec which starts with NTFS means we need to resolve the mount
# point at runtime.
if fd is None and component.pathtype == rdf_paths.PathSpec.PathType.NTFS:
# We are the top level handler. This means we need to check the system
# mounts to work out the exact mount point and device we need to
# open. We then modify the pathspec so we get nested in the raw
# pathspec.
raw_pathspec, corrected_path = client_utils.GetRawDevice(component.path)
# Insert the raw device before the component in the pathspec and correct
# the path
component.path = corrected_path
pathspec.Insert(0, component)
pathspec.Insert(0, raw_pathspec)
# Allow incoming pathspec to be given in the local system path
# conventions.
for component in pathspec:
if component.path:
component.path = client_utils.LocalPathToCanonicalPath(component.path)
# We have not actually opened anything in this iteration, but modified the
# pathspec. Next time we should be able to open it properly.
return fd
else:
return super(NTFSFile, cls).Open(
fd=fd,
component=component,
handlers=handlers,
pathspec=pathspec,
progress_callback=progress_callback)
def DumpProcess(self, psutil_process, args):
response = rdf_memory.YaraProcessDumpInformation()
response.process = rdf_client.Process.FromPsutilProcess(psutil_process)
process = client_utils.OpenProcessForMemoryAccess(
pid=psutil_process.pid)
bytes_limit = args.size_limit
with process:
streamer = streaming.Streamer(chunk_size=args.chunk_size)
with tempfiles.TemporaryDirectory(cleanup=False) as tmp_dir:
for region in client_utils.MemoryRegions(process, args):
if bytes_limit and self.bytes_written + region.size > bytes_limit:
response.error = (
"Byte limit exceeded. Wrote %d bytes, "
"next block is %d bytes, limit is %d." %
(self.bytes_written, region.size, bytes_limit))
return response
end = region.start + region.size
# _ReplaceDumpPathspecsWithMultiGetFilePathspec in DumpProcessMemory
# flow asserts that MemoryRegions can be uniquely identified by their
# file's basename.
filename = "%s_%d_%x_%x.tmp" % (psutil_process.name(),
psutil_process.pid,
region.start, end)
filepath = os.path.join(tmp_dir.path, filename)
chunks = streamer.StreamMemory(process,
offset=region.start,
amount=region.size)
bytes_written = self._SaveMemDumpToFilePath(
filepath, chunks)
if not bytes_written:
continue
self.bytes_written += bytes_written
# TODO: Remove workaround after client_utils are fixed.
canonical_path = client_utils.LocalPathToCanonicalPath(
filepath)
if not canonical_path.startswith("/"):
canonical_path = "/" + canonical_path
region.file = rdf_paths.PathSpec(
path=canonical_path,
pathtype=rdf_paths.PathSpec.PathType.TMPFILE)
response.memory_regions.Append(region)
return response
def Open(
cls,
fd: Optional[vfs_base.VFSHandler],
component: rdf_paths.PathSpec,
handlers: Dict[Any, Type[vfs_base.VFSHandler]],
pathspec: Optional[rdf_paths.PathSpec] = None,
progress_callback: Optional[Callable[[], None]] = None
) -> Optional[vfs_base.VFSHandler]:
# A Pathspec which starts with NTFS means we need to resolve the mount
# point at runtime.
if (fd is None
and component.pathtype == rdf_paths.PathSpec.PathType.NTFS
and pathspec is not None):
# We are the top level handler. This means we need to check the system
# mounts to work out the exact mount point and device we need to
# open. We then modify the pathspec so we get nested in the raw
# pathspec.
raw_pathspec, corrected_path = client_utils.GetRawDevice(
component.path)
# Insert the raw device before the component in the pathspec and correct
# the path
component.path = corrected_path
pathspec.Insert(0, component)
pathspec.Insert(0, raw_pathspec)
# Allow incoming pathspec to be given in the local system path
# conventions.
for component in pathspec:
if component.path:
component.path = client_utils.LocalPathToCanonicalPath(
component.path)
# We have not actually opened anything in this iteration, but modified the
# pathspec. Next time we should be able to open it properly.
return fd
# If an inode is specified, just use it directly.
# This is necessary so that component.path is ignored.
elif component.HasField("inode"):
return NTFSFile(fd,
handlers,
component,
progress_callback=progress_callback)
else:
return super(NTFSFile,
cls).Open(fd=fd,
component=component,
handlers=handlers,
pathspec=pathspec,
progress_callback=progress_callback)
def testFileArtifactParser(self):
"""Test parsing a fake file artifact with a file parser."""
processor = config_file.CronAtAllowDenyParser()
source = rdf_artifact.ArtifactSource(
type=rdf_artifact.ArtifactSource.SourceType.FILE,
attributes={
"paths": ["VFSFixture/etc/passwd", "numbers.txt"],
})
paths = []
for path in source.attributes["paths"]:
paths.append(os.path.join(self.base_path, path))
stat_cache = utils.StatCache()
expanded_paths = []
opts = globbing.PathOpts(follow_links=True)
for path in paths:
for expanded_path in globbing.ExpandPath(path, opts):
expanded_paths.append(expanded_path)
path_type = rdf_paths.PathSpec.PathType.OS
results = []
for path in expanded_paths:
stat = stat_cache.Get(path, follow_symlink=True)
pathspec = rdf_paths.PathSpec(
pathtype=path_type,
path=client_utils.LocalPathToCanonicalPath(stat.GetPath()),
path_options=rdf_paths.PathSpec.Options.CASE_LITERAL)
response = rdf_client_fs.FindSpec(pathspec=pathspec)
for res in artifact_collector.ParseSingleResponse(
processor, response, {}, path_type):
results.append(res)
self.assertEqual(len(results), 3)
self.assertTrue(
results[0]["filename"].endswith("test_data/VFSFixture/etc/passwd"))
self.assertIsInstance(results[0], rdf_protodict.AttributedDict)
self.assertEqual(len(results[0]["users"]), 3)
self.assertIsInstance(results[1], rdf_anomaly.Anomaly)
self.assertEqual(len(results[2]["users"]), 1000)
def Open(cls,
fd,
component,
handlers,
pathspec=None,
progress_callback=None):
"""Try to correct the casing of component.
This method is called when we failed to open the component directly. We try
to transform the component into something which is likely to work.
In this implementation, we correct the case of the component until we can
not open the path any more.
Args:
fd: The base fd we will use.
component: The component we should open.
handlers: A mapping from rdf_paths.PathSpec.PathType to classes
implementing VFSHandler.
pathspec: The rest of the pathspec object.
progress_callback: A callback to indicate that the open call is still
working but needs more time.
Returns:
A file object.
Raises:
IOError: If nothing could be opened still.
"""
# The handler for this component
try:
handler = handlers[component.pathtype]
except KeyError:
raise UnsupportedHandlerError(component.pathtype)
# We will not do any case folding unless requested.
if component.path_options == rdf_paths.PathSpec.Options.CASE_LITERAL:
return handler(base_fd=fd, pathspec=component, handlers=handlers)
path_components = client_utils.LocalPathToCanonicalPath(component.path)
path_components = ["/"] + list(filter(None,
path_components.split("/")))
for i, path_component in enumerate(path_components):
try:
if fd:
new_pathspec = fd.MatchBestComponentName(
path_component, component.pathtype)
else:
new_pathspec = component.Copy()
new_pathspec.path = path_component
# If the component has a stream_name (NTFS alternate data stream) set,
# set it on the last path_component.
if i == len(path_components) - 1 and component.HasField(
"stream_name"):
new_pathspec.stream_name = component.stream_name
# The handler for this component
try:
handler = handlers[new_pathspec.pathtype]
except KeyError:
raise UnsupportedHandlerError(new_pathspec.pathtype)
fd = handler(base_fd=fd,
handlers=handlers,
pathspec=new_pathspec,
progress_callback=progress_callback)
except IOError as e:
# Can not open the first component, we must raise here.
if i <= 1:
raise IOError("File not found: {}".format(component))
# Do not try to use TSK to open a not-found registry entry, fail
# instead. Using TSK would lead to confusing error messages, hiding
# the fact that the Registry entry is simply not there.
if component.pathtype == rdf_paths.PathSpec.PathType.REGISTRY:
raise IOError("Registry entry not found: {}".format(e))
# Insert the remaining path at the front of the pathspec.
pathspec.Insert(0,
path=utils.JoinPath(*path_components[i:]),
pathtype=rdf_paths.PathSpec.PathType.TSK)
break
return fd
def _StatEntry(stat, ext_attrs):
pathspec = rdf_paths.PathSpec(
pathtype=rdf_paths.PathSpec.PathType.OS,
path=client_utils.LocalPathToCanonicalPath(stat.GetPath()),
path_options=rdf_paths.PathSpec.Options.CASE_LITERAL)
return client_utils.StatEntryFromStat(stat, pathspec, ext_attrs=ext_attrs)
def Open(cls,
fd,
component,
pathspec=None,
progress_callback=None,
full_pathspec=None):
"""Try to correct the casing of component.
This method is called when we failed to open the component directly. We try
to transform the component into something which is likely to work.
In this implementation, we correct the case of the component until we can
not open the path any more.
Args:
fd: The base fd we will use.
component: The component we should open.
pathspec: The rest of the pathspec object.
progress_callback: A callback to indicate that the open call is still
working but needs more time.
full_pathspec: The full pathspec we are trying to open.
Returns:
A file object.
Raises:
IOError: If nothing could be opened still.
"""
# The handler for this component
try:
handler = VFS_HANDLERS[component.pathtype]
except KeyError:
raise IOError("VFS handler %d not supported." % component.pathtype)
# We will not do any case folding unless requested.
if component.path_options == rdf_paths.PathSpec.Options.CASE_LITERAL:
return handler(base_fd=fd, pathspec=component)
path_components = client_utils.LocalPathToCanonicalPath(component.path)
path_components = ["/"] + filter(None, path_components.split("/"))
for i, path_component in enumerate(path_components):
try:
if fd:
new_pathspec = fd.MatchBestComponentName(path_component)
else:
new_pathspec = component
new_pathspec.path = path_component
# The handler for this component
try:
handler = VFS_HANDLERS[new_pathspec.pathtype]
except KeyError:
raise IOError("VFS handler %d not supported." %
new_pathspec.pathtype)
fd = handler(base_fd=fd,
pathspec=new_pathspec,
full_pathspec=full_pathspec,
progress_callback=progress_callback)
except IOError:
# Can not open the first component, we must raise here.
if i <= 1:
raise IOError("File not found")
# Insert the remaining path at the front of the pathspec.
pathspec.Insert(0,
path=utils.JoinPath(*path_components[i:]),
pathtype=rdf_paths.PathSpec.PathType.TSK)
break
return fd
