def GenerateDirectory(self,
input_dir=None,
output_dir=None,
replacements=None):
input_dir = utils.NormalizePath(input_dir)
output_dir = utils.NormalizePath(output_dir)
replacements = replacements or []
for (root, _, files) in os.walk(input_dir):
for filename in files:
in_file = utils.JoinPath(root, filename)
out_file = in_file.replace(input_dir, output_dir)
for (s, replacement) in replacements:
out_file = out_file.replace(s, replacement)
self.EnsureDirExists(os.path.dirname(out_file))
self.GenerateFile(in_file, out_file)
def GetCommitInformation(transid):
"""Read transaction information from stored file."""
loc = data_store.DB.Location()
if not os.path.exists(loc):
return False
if not os.path.isdir(loc):
return False
tempdir = _GetTransactionDirectory(loc, transid)
tempfile = utils.JoinPath(tempdir, constants.TRANSACTION_FILENAME)
if not os.path.exists(tempfile):
return None
if not os.path.isfile(tempfile):
return None
with open(tempfile, "rb") as fp:
return rdf_data_server.DataServerRebalance.FromSerializedString(
fp.read())
def testGlobWithMultiplePaths(self):
self._MakeTestDirs()
paths = [
os.path.join(self.temp_dir, "*/*/foo*"),
os.path.join(self.temp_dir, "notthere"),
os.path.join(self.temp_dir, "*/notthere"),
os.path.join(self.temp_dir, "*/foo*")
]
# Handle filesystem case sensitivity
expected_results = ["1/foo1", "/1/2/foo2"]
if platform.system() == "Linux":
expected_results = ["1/foo1", "1/fOo1", "/1/2/fOo2", "/1/2/foo2"]
self._RunGlob(paths)
self.assertItemsEqual(
self.flow_replies,
[utils.JoinPath(self.temp_dir, x) for x in expected_results])
def setUp(self):
super(DeleteGRRTempFiles, self).setUp()
filename = "%s_blah" % config_lib.CONFIG["Client.tempfile_prefix"]
self.tempfile = utils.JoinPath(self.temp_dir, "delete_test", filename)
self.dirname = os.path.dirname(self.tempfile)
os.makedirs(self.dirname)
self.not_tempfile = os.path.join(self.dirname, "notatempfile")
open(self.not_tempfile, "w").write("something")
self.temp_fd = tempfiles.CreateGRRTempFile(self.dirname)
self.temp_fd2 = tempfiles.CreateGRRTempFile(self.dirname)
self.assertTrue(os.path.exists(self.not_tempfile))
self.assertTrue(os.path.exists(self.temp_fd.name))
self.assertTrue(os.path.exists(self.temp_fd2.name))
self.pathspec = rdfvalue.PathSpec(
path=self.dirname, pathtype=rdfvalue.PathSpec.PathType.OS)
def GetPrefix(self, prefix):
"""Yields all databases which could contain records begining with prefix."""
components = common.Components(prefix)
components = [common.ConvertStringToFilename(x) for x in components]
path_prefix = utils.JoinPath(*components)
if path_prefix == "/":
path_prefix = ""
for regex in self.path_regexes:
result = common.EvaluatePrefix(path_prefix, regex)
if result == "MATCH":
yield self.Get(prefix)
return
if result == "POSSIBLE":
for data_server in self.AllDatabases():
yield data_server
return
yield self.Get(prefix)
def GetPrefix(self, subject_prefix):
"""Return list of databases matching subject_prefix."""
components = common.Components(subject_prefix)
components = [common.ConvertStringToFilename(x) for x in components]
path_prefix = utils.JoinPath(*components)
if path_prefix == "/":
path_prefix = ""
for regex in self.path_regexes:
result = common.EvaluatePrefix(path_prefix, regex)
if result == "MATCH":
yield self.Get(subject_prefix)
return
if result == "POSSIBLE":
for conn in self.DatabasesByPath(path_prefix):
yield conn
return
yield self.Get(subject_prefix)
def Start(self):
"""Determine the Chrome directory."""
self.state.Register("storage", {})
if self.args.path:
paths = [self.args.path]
elif self.args.username:
paths = self.GuessExtensionPaths(self.args.username)
if not paths:
raise flow.FlowError("No valid extension paths found.")
for path in paths:
rel_path = utils.JoinPath(path, "Extensions")
pathspec = rdfvalue.PathSpec(path=rel_path,
pathtype=self.args.pathtype)
self.CallClient("ListDirectory",
next_state="EnumerateExtensionDirs",
pathspec=pathspec)
def _Stat(self, name, value, value_type):
response = rdfvalue.StatEntry()
response_pathspec = self.pathspec.Copy()
# No matter how we got here, there is no need to do case folding from now on
# since this is the exact filename casing.
response_pathspec.path_options = rdfvalue.PathSpec.Options.CASE_LITERAL
response_pathspec.last.path = utils.JoinPath(
response_pathspec.last.path, name)
response.pathspec = response_pathspec
if self.IsDirectory():
response.st_mode = stat.S_IFDIR
else:
response.st_mode = stat.S_IFREG
response.st_mtime = self.last_modified
response.st_size = len(utils.SmartStr(value))
response.registry_type = self.registry_map.get(value_type, 0)
response.registry_data = rdfvalue.DataBlob().SetValue(value)
return response
def Add(self, path, age=None):
"""Add a relative stem to the current value and return a new RDFURN.
If urn is a fully qualified URN, replace the current value with it.
Args:
path: A string containing a relative path.
age: The age of the object. If None set to current time.
Returns:
A new RDFURN that can be chained.
Raises:
ValueError: if the path component is not a string.
"""
if not isinstance(path, basestring):
raise ValueError("Only strings should be added to a URN.")
result = self.Copy(age)
result.Update(path=utils.JoinPath(self._urn.path, path))
return result
def Add(self, path, age=None):
"""Add a relative stem to the current value and return a new RDFURN.
Note that this returns an RDFURN, not a ClientURN since the resulting object
would not pass validation.
Args:
path: A string containing a relative path.
age: The age of the object. If None set to current time.
Returns:
A new RDFURN that can be chained.
Raises:
ValueError: if the path component is not a string.
"""
if not isinstance(path, basestring):
raise ValueError("Only strings should be added to a URN.")
result = rdfvalue.RDFURN(self.Copy(age))
result.Update(path=utils.JoinPath(self._string_urn, path))
return result
def setUp(self):
super(DeleteGRRTempFiles, self).setUp()
filename = "%s_blah" % config_lib.CONFIG["Client.tempfile_prefix"]
self.tempfile = utils.JoinPath(self.temp_dir,
"delete_test", filename)
self.dirname = os.path.dirname(self.tempfile)
os.makedirs(self.dirname)
self.tempdir_overrider = test_lib.ConfigOverrider({
"Client.tempdir_roots": [os.path.dirname(self.dirname)],
"Client.grr_tempdir": os.path.basename(self.dirname)})
self.tempdir_overrider.Start()
self.not_tempfile = os.path.join(self.temp_dir, "notatempfile")
open(self.not_tempfile, "w").write("something")
self.temp_fd = tempfiles.CreateGRRTempFile(self.dirname)
self.temp_fd2 = tempfiles.CreateGRRTempFile(self.dirname)
self.assertTrue(os.path.exists(self.not_tempfile))
self.assertTrue(os.path.exists(self.temp_fd.name))
self.assertTrue(os.path.exists(self.temp_fd2.name))
self.pathspec = rdf_paths.PathSpec(
path=self.dirname, pathtype=rdf_paths.PathSpec.PathType.OS)
def DownloadDir(aff4_path, output_dir, bufsize=8192, preserve_path=True):
"""Take an aff4 path and download all files in it to output_dir.
Args:
aff4_path: Any aff4 path as a string
output_dir: A local directory to write to, will be created if not there.
bufsize: Buffer size to use.
preserve_path: If set all paths will be created.
Note that this works for collections as well. It will download all
files in the collection.
This only downloads files that are already in the datastore, it doesn't
queue anything on the client.
"""
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
fd = aff4.FACTORY.Open(aff4_path)
for child in fd.OpenChildren():
if preserve_path:
# Get a full path without the aff4:
full_dir = utils.JoinPath(output_dir, child.urn.Path())
full_dir = os.path.dirname(full_dir)
if not os.path.isdir(full_dir):
os.makedirs(full_dir)
outfile = os.path.join(full_dir, child.urn.Basename())
else:
outfile = os.path.join(output_dir, child.urn.Basename())
logging.info(u"Downloading %s to %s", child.urn, outfile)
with open(outfile, "wb") as out_fd:
try:
buf = child.Read(bufsize)
while buf:
out_fd.write(buf)
buf = child.Read(bufsize)
except IOError as e:
logging.error("Failed to read %s. Err: %s", child.urn, e)
def _FileWithRemoveList(database_dir, rebalance):
tempdir = _CreateDirectory(database_dir, rebalance.id)
return utils.JoinPath(tempdir, constants.REMOVE_FILENAME)
def CollapsePath(self):
return utils.JoinPath(*[x.path for x in self])
def MakeStatResponse(self,
tsk_file,
tsk_attribute=None,
append_name=False):
"""Given a TSK info object make a StatResponse.
Note that tsk uses two things to uniquely identify a data stream - the inode
object given in tsk_file and the attribute object which may correspond to an
ADS of this file for filesystems which support ADS. We store both of these
in the stat response.
Args:
tsk_file: A TSK File object for the specified inode.
tsk_attribute: A TSK Attribute object for the ADS. If None we use the main
stream.
append_name: If specified we append this name to the last element of the
pathspec.
Returns:
A StatResponse protobuf which can be used to re-open this exact VFS node.
"""
info = tsk_file.info
response = rdfvalue.StatEntry()
meta = info.meta
if meta:
response.st_ino = meta.addr
for attribute in "mode nlink uid gid size atime mtime ctime".split(
):
try:
value = int(getattr(meta, attribute))
if value < 0: value &= 0xFFFFFFFF
setattr(response, "st_%s" % attribute, value)
except AttributeError:
pass
name = info.name
child_pathspec = self.pathspec.Copy()
if append_name:
# Append the name to the most inner pathspec
child_pathspec.last.path = utils.JoinPath(
child_pathspec.last.path, utils.SmartUnicode(append_name))
child_pathspec.last.inode = meta.addr
if tsk_attribute is not None:
child_pathspec.last.ntfs_type = int(tsk_attribute.info.type)
child_pathspec.last.ntfs_id = int(tsk_attribute.info.id)
child_pathspec.last.stream_name = tsk_attribute.info.name
# Update the size with the attribute size.
response.st_size = tsk_attribute.info.size
if name:
# Encode the type onto the st_mode response
response.st_mode |= self.FILE_TYPE_LOOKUP.get(int(name.type), 0)
if meta:
# What if the types are different? What to do here?
response.st_mode |= self.META_TYPE_LOOKUP.get(int(meta.type), 0)
# Write the pathspec on the response.
response.pathspec = child_pathspec
return response
def _GetTransactionDirectory(database_dir, rebalance_id):
dirname = common.ConvertStringToFilename(rebalance_id)
tempdir = utils.JoinPath(database_dir, constants.REBALANCE_DIRECTORY,
dirname)
return tempdir
def __init__(self,
base_fd,
pathspec=None,
progress_callback=None,
full_pathspec=None):
super(File, self).__init__(base_fd,
pathspec=pathspec,
full_pathspec=full_pathspec,
progress_callback=progress_callback)
if base_fd is None:
self.pathspec.Append(pathspec)
# We can stack on another directory, which means we concatenate their
# directory with ours.
elif base_fd.IsDirectory():
self.pathspec.last.path = utils.JoinPath(self.pathspec.last.path,
pathspec.path)
else:
raise IOError(
"File handler can not be stacked on another handler.")
self.path = self.pathspec.last.path
# We can optionally apply a global offset to the file.
if self.pathspec[0].HasField("offset"):
self.file_offset = self.pathspec[0].offset
self.pathspec.last.path_options = paths.PathSpec.Options.CASE_LITERAL
self.FileHacks()
self.filename = client_utils.CanonicalPathToLocalPath(self.path)
error = None
# Pythonic way - duck typing. Is the handle a directory?
try:
if not self.files:
# Note that the encoding of local path is system specific
local_path = client_utils.CanonicalPathToLocalPath(self.path +
"/")
self.files = [
utils.SmartUnicode(entry)
for entry in os.listdir(local_path)
]
# Some filesystems do not support unicode properly
except UnicodeEncodeError as e:
raise IOError(str(e))
except (IOError, OSError) as e:
self.files = []
error = e
# Ok, it's not. Is it a file then?
try:
with FileHandleManager(self.filename) as fd:
if pathspec.last.HasField("file_size_override"):
self.size = pathspec.last.file_size_override - self.file_offset
else:
# Work out how large the file is.
if self.size is None:
fd.Seek(0, 2)
end = fd.Tell()
if end == 0:
# This file is not seekable, we just use the default.
end = pathspec.last.file_size_override
self.size = end - self.file_offset
error = None
# Some filesystems do not support unicode properly
except UnicodeEncodeError as e:
raise IOError(str(e))
except IOError as e:
if error:
error = e
if error is not None:
raise error # pylint: disable=raising-bad-type
def Run(self, request):
"""Load the component requested.
The component defines a set of python imports which should be imported into
the running program. The purpose of this client action is to ensure that the
imports are available and of the correct version. We ensure this by:
1) Attempt to import the relevant modules.
2) If that fails checks for the presence of a component installed at the
require path. Attempt to import the modules again.
3) If no component is installed, we fetch and install the component from the
server. We then attempt to use it.
If all imports succeed we return a success status, otherwise we raise an
exception.
Args:
request: The LoadComponent request.
Raises:
RuntimeError: If the component is invalid.
"""
summary = request.summary
# Just try to load the required modules.
try:
self.LoadComponent(summary)
# If we succeed we just report this component is done.
self.SendReply(request)
return
except ImportError:
pass
# Try to add an existing component path.
component_path = utils.JoinPath(
config_lib.CONFIG.Get("Client.component_path"), summary.name,
summary.version)
# Add the component path to the site packages:
site = Site()
site.AddSiteDir(component_path)
LOADED_COMPONENTS[summary.name] = summary.version
try:
self.LoadComponent(summary)
logging.info("Component %s already present.", summary.name)
self.SendReply(request)
return
except ImportError:
pass
# Could not import component - will have to fetch it.
logging.info("Unable to import component %s.", summary.name)
# Derive the name of the component that we need depending on the current
# architecture. The client build system should have burned its environment
# into the client config file. This is the best choice because it will
# choose the same component that was built together with the client
# itself (on the same build environment).
build_environment = config_lib.CONFIG.Get("Client.build_environment")
if not build_environment:
# Failing this we try to get something similar to the running system.
build_environment = rdf_client.Uname.FromCurrentSystem().signature(
)
url = "%s/%s" % (summary.url, build_environment)
logging.info("Fetching component from %s", url)
crypted_data = self.grr_worker.http_manager.OpenServerEndpoint(
url).data
# Decrypt and check signature. The cipher is created when the component is
# uploaded and contains the key to decrypt it.
signed_blob = rdf_crypto.SignedBlob(
summary.cipher.Decrypt(crypted_data))
# Ensure the blob is signed with the correct key.
signed_blob.Verify(
config_lib.CONFIG["Client.executable_signing_public_key"])
component = rdf_client.ClientComponent(signed_blob.data)
# Make sure its the component we actually want.
if (component.summary.name != summary.name
or component.summary.version != summary.version):
raise RuntimeError(
"Downloaded component is not the correct version")
# Make intermediate directories.
try:
os.makedirs(component_path)
except (OSError, IOError):
pass
# Unzip the component into the path.
logging.info("Installing component to %s", component_path)
component_zip = zipfile.ZipFile(StringIO.StringIO(component.raw_data))
component_zip.extractall(component_path)
# Add the component to the site packages:
site.AddSiteDir(component_path)
LOADED_COMPONENTS[component.summary.name] = component.summary.version
# If this does not work now, we just fail.
self.LoadComponent(summary)
# If we succeed we just report this component is done.
self.SendReply(request)
def _Touch(self, relative_path): open(utils.JoinPath(self.temp_dir, relative_path), "wb").close()
def MakeDestinationKey(directory, filename):
"""Creates a name that identifies a database file."""
return utils.SmartStr(utils.JoinPath(directory, filename)).lstrip("/")
def MakeStatResponse(self,
tsk_file,
tsk_attribute=None,
append_name=False):
"""Given a TSK info object make a StatEntry.
Note that tsk uses two things to uniquely identify a data stream - the inode
object given in tsk_file and the attribute object which may correspond to an
ADS of this file for filesystems which support ADS. We store both of these
in the stat response.
Args:
tsk_file: A TSK File object for the specified inode.
tsk_attribute: A TSK Attribute object for the ADS. If None we use the main
stream.
append_name: If specified we append this name to the last element of the
pathspec.
Returns:
A StatEntry which can be used to re-open this exact VFS node.
"""
info = tsk_file.info
response = rdf_client.StatEntry()
meta = info.meta
if meta:
response.st_ino = meta.addr
for attribute in [
"mode", "nlink", "uid", "gid", "size", "atime", "mtime",
"ctime", "crtime"
]:
try:
value = int(getattr(meta, attribute))
if value < 0:
value &= 0xFFFFFFFF
setattr(response, "st_%s" % attribute, value)
except AttributeError:
pass
name = info.name
child_pathspec = self.pathspec.Copy()
if append_name:
# Append the name to the most inner pathspec
child_pathspec.last.path = utils.JoinPath(
child_pathspec.last.path, utils.SmartUnicode(append_name))
child_pathspec.last.inode = meta.addr
if tsk_attribute is not None:
child_pathspec.last.ntfs_type = int(tsk_attribute.info.type)
child_pathspec.last.ntfs_id = int(tsk_attribute.info.id)
child_pathspec.last.stream_name = tsk_attribute.info.name
# Update the size with the attribute size.
response.st_size = tsk_attribute.info.size
default = rdf_paths.PathSpec.tsk_fs_attr_type.TSK_FS_ATTR_TYPE_DEFAULT
last = child_pathspec.last
if last.ntfs_type != default or last.ntfs_id:
# This is an ads and should be treated as a file.
# Clear all file type bits.
response.st_mode &= ~self.stat_type_mask
response.st_mode |= stat.S_IFREG
else:
child_pathspec.last.ntfs_type = None
child_pathspec.last.ntfs_id = None
child_pathspec.last.stream_name = None
if name:
# Encode the type onto the st_mode response
response.st_mode |= self.FILE_TYPE_LOOKUP.get(int(name.type), 0)
if meta:
# What if the types are different? What to do here?
response.st_mode |= self.META_TYPE_LOOKUP.get(int(meta.type), 0)
# Write the pathspec on the response.
response.pathspec = child_pathspec
return response
def __init__(self,
base_fd,
pathspec=None,
progress_callback=None,
full_pathspec=None):
"""Use TSK to read the pathspec.
Args:
base_fd: The file like object we read this component from.
pathspec: An optional pathspec to open directly.
progress_callback: A callback to indicate that the open call is still
working but needs more time.
full_pathspec: The full pathspec we are trying to open.
Raises:
IOError: If the file can not be opened.
"""
super(TSKFile, self).__init__(base_fd,
pathspec=pathspec,
progress_callback=progress_callback,
full_pathspec=full_pathspec)
if self.base_fd is None:
raise IOError("TSK driver must have a file base.")
# If our base is another tsk driver - borrow the reference to the raw
# device, and replace the last pathspec component with this one after
# extending its path.
elif isinstance(base_fd, TSKFile) and self.base_fd.IsDirectory():
self.tsk_raw_device = self.base_fd.tsk_raw_device
last_path = utils.JoinPath(self.pathspec.last.path, pathspec.path)
# Replace the last component with this one.
self.pathspec.Pop(-1)
self.pathspec.Append(pathspec)
self.pathspec.last.path = last_path
# Use the base fd as a base to parse the filesystem only if its file like.
elif not self.base_fd.IsDirectory():
self.tsk_raw_device = self.base_fd
self.pathspec.Append(pathspec)
else:
# If we get here we have a directory from a non sleuthkit driver - dont
# know what to do with it.
raise IOError("Unable to parse base using Sleuthkit.")
# If we are successful in opening this path below the path casing is
# correct.
self.pathspec.last.path_options = rdf_paths.PathSpec.Options.CASE_LITERAL
fd_hash = self.tsk_raw_device.pathspec.SerializeToString()
# Cache the filesystem using the path of the raw device
try:
self.filesystem = vfs.DEVICE_CACHE.Get(fd_hash)
self.fs = self.filesystem.fs
except KeyError:
self.img = MyImgInfo(fd=self.tsk_raw_device,
progress_callback=progress_callback)
self.fs = pytsk3.FS_Info(self.img, 0)
self.filesystem = CachedFilesystem(self.fs, self.img)
vfs.DEVICE_CACHE.Put(fd_hash, self.filesystem)
# We prefer to open the file based on the inode because that is more
# efficient.
if pathspec.HasField("inode"):
self.fd = self.fs.open_meta(pathspec.inode)
self.tsk_attribute = self.GetAttribute(pathspec.ntfs_type,
pathspec.ntfs_id)
if self.tsk_attribute:
self.size = self.tsk_attribute.info.size
else:
self.size = self.fd.info.meta.size
else:
# Does the filename exist in the image?
self.fd = self.fs.open(utils.SmartStr(self.pathspec.last.path))
self.size = self.fd.info.meta.size
self.pathspec.last.inode = self.fd.info.meta.addr
def Open(cls,
fd,
component,
pathspec=None,
progress_callback=None,
full_pathspec=None):
"""Try to correct the casing of component.
This method is called when we failed to open the component directly. We try
to transform the component into something which is likely to work.
In this implementation, we correct the case of the component until we can
not open the path any more.
Args:
fd: The base fd we will use.
component: The component we should open.
pathspec: The rest of the pathspec object.
progress_callback: A callback to indicate that the open call is still
working but needs more time.
full_pathspec: The full pathspec we are trying to open.
Returns:
A file object.
Raises:
IOError: If nothing could be opened still.
"""
# The handler for this component
try:
handler = VFS_HANDLERS[component.pathtype]
except KeyError:
raise IOError("VFS handler %d not supported." % component.pathtype)
# We will not do any case folding unless requested.
if component.path_options == rdf_paths.PathSpec.Options.CASE_LITERAL:
return handler(base_fd=fd, pathspec=component)
path_components = client_utils.LocalPathToCanonicalPath(component.path)
path_components = ["/"] + filter(None, path_components.split("/"))
for i, path_component in enumerate(path_components):
try:
if fd:
new_pathspec = fd.MatchBestComponentName(path_component)
else:
new_pathspec = component
new_pathspec.path = path_component
# The handler for this component
try:
handler = VFS_HANDLERS[new_pathspec.pathtype]
except KeyError:
raise IOError("VFS handler %d not supported." % new_pathspec.pathtype)
fd = handler(
base_fd=fd,
pathspec=new_pathspec,
full_pathspec=full_pathspec,
progress_callback=progress_callback)
except IOError:
# Can not open the first component, we must raise here.
if i <= 1:
raise IOError("File not found")
# Insert the remaining path at the front of the pathspec.
pathspec.Insert(
0,
path=utils.JoinPath(*path_components[i:]),
pathtype=rdf_paths.PathSpec.PathType.TSK)
break
return fd
