def _SetUpTempDirStructure(self, grr_tempdir="grr_temp"): temproot1 = utils.JoinPath(self.temp_dir, "del_test1") temproot2 = utils.JoinPath(self.temp_dir, "del_test2") temproot3 = utils.JoinPath(self.temp_dir, "del_test3") tempdir1 = utils.JoinPath(temproot1, grr_tempdir) tempdir2 = utils.JoinPath(temproot2, grr_tempdir) tempdir3 = utils.JoinPath(temproot3, grr_tempdir) os.makedirs(tempdir1) os.makedirs(tempdir2) # Omit tempdir3. file1 = utils.JoinPath(tempdir1, "file1") file2 = utils.JoinPath(tempdir2, "file2") with io.open(file1, "wb") as fd: fd.write(b"something") with io.open(file2, "wb") as fd: fd.write(b"something") # Unrelated file in the tempdir_roots should be left alone. not_a_grr_file1 = utils.JoinPath(temproot1, "file1") not_a_grr_file2 = utils.JoinPath(temproot1, "file2") with io.open(not_a_grr_file1, "wb") as fd: fd.write(b"something") with io.open(not_a_grr_file2, "wb") as fd: fd.write(b"something") self.assertTrue(os.path.exists(file1)) self.assertTrue(os.path.exists(file2)) self.assertTrue(os.path.exists(not_a_grr_file1)) self.assertTrue(os.path.exists(not_a_grr_file2)) return ([temproot1, temproot2, temproot3], [tempdir1, tempdir2], [tempdir3], [file1, file2], [not_a_grr_file1, not_a_grr_file1])
def DatabasesByPath(self, path_prefix): """Yields connections which might contain data prefixed by path_prefix.""" # We are looking for database files which start with this prefix, or # which could be extended to match this prefix. dir_prefix = utils.JoinPath(self.root_path, path_prefix) # Shortened path_prefix - we will shorten it one component at a time # checking directories for databases of interest as we go. shortened_path_prefix = path_prefix databases_found = set() while True: shortened_path = utils.JoinPath(self.root_path, shortened_path_prefix) if os.path.isdir(shortened_path): for db in self.DatabasesInDir(shortened_path): if db in databases_found: continue mod_db = db if mod_db == utils.JoinPath(self.root_path, "aff4"): mod_db = self.root_path if mod_db.startswith(dir_prefix) or dir_prefix.startswith(mod_db): databases_found.add(db) yield SqliteConnection(db + SQLITE_EXTENSION) if not shortened_path_prefix: break components = shortened_path_prefix.split(os.path.sep) shortened_path_prefix = utils.JoinPath(*(components[:-1])) if shortened_path_prefix == "/": shortened_path_prefix = ""
def Get(self, subject): """This will create the connection if needed so should not fail.""" filename, directory = common.ResolveSubjectDestination( subject, self.path_regexes) key = common.MakeDestinationKey(directory, filename) try: return super(SqliteConnectionCache, self).Get(key) except KeyError: dirname = utils.JoinPath(self.root_path, directory) path = utils.JoinPath(dirname, filename) + SQLITE_EXTENSION dirname = utils.SmartStr(dirname) path = utils.SmartStr(path) # Make sure directory exists. if not os.path.isdir(dirname): try: os.makedirs(dirname) except OSError: pass self._EnsureDatabaseExists(path) connection = SqliteConnection(path) super(SqliteConnectionCache, self).Put(key, connection) return connection
def ListFiles(self, ext_attrs=None): """A generator of all keys and values.""" del ext_attrs # Unused. if not self.IsDirectory(): return if self.hive is None: for name in dir(winreg): if name.startswith("HKEY_"): response = rdf_client_fs.StatEntry(st_mode=stat.S_IFDIR) response_pathspec = self.pathspec.Copy() response_pathspec.last.path = utils.JoinPath( response_pathspec.last.path, name) response.pathspec = response_pathspec yield response return try: with OpenKey(self.hive, self.local_path) as key: (self.number_of_keys, self.number_of_values, self.last_modified) = QueryInfoKey(key) # First keys - These will look like directories. for i in range(self.number_of_keys): try: name = EnumKey(key, i) key_name = utils.JoinPath(self.local_path, name) try: # Store the default value in the stat response for values. with OpenKey(self.hive, key_name) as subkey: value, value_type = QueryValueEx(subkey, "") except OSError: value, value_type = None, None response = self._Stat(name, value, value_type) # Keys look like Directories in the VFS. response.st_mode = stat.S_IFDIR yield response except OSError: pass # Now Values - These will look like files. for i in range(self.number_of_values): try: name, value, value_type = EnumValue(key, i) response = self._Stat(name, value, value_type) # Values look like files in the VFS. response.st_mode = stat.S_IFREG yield response except OSError: pass except OSError as e: raise IOError("Unable to list key %s: %s" % (self.key_name, e))
def testGlobWithStarStar(self): """Test that ** expressions mean recursion.""" self._MakeTestDirs() # Test filename and directory with spaces os.makedirs(utils.JoinPath(self.temp_dir, "1/2 space")) path_spaces = utils.JoinPath(self.temp_dir, "1/2 space/foo something") io.open(path_spaces, "wb").close() self.assertTrue(os.path.exists(path_spaces)) # Get the foos using default of 3 directory levels. paths = [os.path.join(self.temp_dir, "1/**/foo*")] # Handle filesystem case insensitivity expected_results = [ "1/2/3/4/foo4", "/1/2/3/foo3", "1/2/foo2", "1/2 space/foo something" ] if platform.system() == "Linux": expected_results = [ "1/2/3/4/fOo4", "1/2/3/4/foo4", "/1/2/3/fOo3", "/1/2/3/foo3", "1/2/fOo2", "1/2/foo2", "1/2 space/foo something" ] expected_results = [ utils.JoinPath(self.temp_dir, x) for x in expected_results ] results = self._RunGlob(paths) self.assertCountEqual(results, expected_results) # Get the files 2 levels down only. paths = [os.path.join(self.temp_dir, "1/", "**2/foo*")] # Handle filesystem case insensitivity expected_results = ["1/2/3/foo3", "/1/2/foo2", "1/2 space/foo something"] if platform.system() == "Linux": expected_results = [ "1/2/3/foo3", "1/2/3/fOo3", "/1/2/fOo2", "/1/2/foo2", "1/2 space/foo something" ] expected_results = [ utils.JoinPath(self.temp_dir, x) for x in expected_results ] results = self._RunGlob(paths) self.assertCountEqual(results, expected_results) # Get all of the bars. paths = [os.path.join(self.temp_dir, "**10bar*")] expected_results = [ "bar", "1/bar1", "/1/2/bar2", "/1/2/3/bar3", "/1/2/3/4/bar4" ] expected_results = [ utils.JoinPath(self.temp_dir, x) for x in expected_results ] results = self._RunGlob(paths) self.assertCountEqual(results, expected_results)
def _MakeTestDirs(self): fourth_level_dir = utils.JoinPath(self.temp_dir, "1/2/3/4") os.makedirs(fourth_level_dir) top_level_path = self.temp_dir open(utils.JoinPath(top_level_path, "bar"), "wb").close() for level in range(1, 5): top_level_path = utils.JoinPath(top_level_path, level) for filename in ("foo", "fOo", "bar"): file_path = utils.JoinPath(top_level_path, filename + str(level)) open(file_path, "wb").close() self.assertTrue(os.path.exists(file_path))
def testGlobBackslashHandlingNoRegex(self): self._Touch("foo.txt") self._Touch("foo.txt~") paths = [ utils.JoinPath(self.temp_dir, "foo.txt"), # The backslash in the path will be replaced by a forward-slash when # building a tree representing all the paths (this behavior isn't # particularly correct). Note that the tilde does not need to be # escaped. utils.JoinPath(self.temp_dir, r"foo.txt\~"), ] expected_paths = [utils.JoinPath(self.temp_dir, "foo.txt")] self._RunGlob(paths) self.assertItemsEqual(expected_paths, self.flow_replies)
def DownloadDir(aff4_path, output_dir, bufsize=8192, preserve_path=True): """Take an aff4 path and download all files in it to output_dir. Args: aff4_path: Any aff4 path as a string output_dir: A local directory to write to, will be created if not there. bufsize: Buffer size to use. preserve_path: If set all paths will be created. Note that this works for collections as well. It will download all files in the collection. This only downloads files that are already in the datastore, it doesn't queue anything on the client. """ if not os.path.isdir(output_dir): os.makedirs(output_dir) fd = aff4.FACTORY.Open(aff4_path) for child in fd.OpenChildren(): if preserve_path: # Get a full path without the aff4: full_dir = utils.JoinPath(output_dir, child.urn.Path()) full_dir = os.path.dirname(full_dir) if not os.path.isdir(full_dir): os.makedirs(full_dir) outfile = os.path.join(full_dir, child.urn.Basename()) else: outfile = os.path.join(output_dir, child.urn.Basename()) logging.info(u"Downloading %s to %s", child.urn, outfile) with open(outfile, "wb") as out_fd: try: buf = child.Read(bufsize) while buf: out_fd.write(buf) buf = child.Read(bufsize) except IOError as e: logging.error("Failed to read %s. Err: %s", child.urn, e)
def _Stat(self, name, value, value_type, mtime=None): response = rdf_client_fs.StatEntry() response_pathspec = self.pathspec.Copy() # No matter how we got here, there is no need to do case folding from now on # since this is the exact filename casing. response_pathspec.path_options = rdf_paths.PathSpec.Options.CASE_LITERAL response_pathspec.last.path = utils.JoinPath( response_pathspec.last.path, name) response.pathspec = response_pathspec if self.IsDirectory(): response.st_mode = stat.S_IFDIR else: response.st_mode = stat.S_IFREG if mtime: response.st_mtime = mtime if value is None: response.st_size = 0 elif isinstance(value, bytes): response.st_size = len(value) else: response.st_size = len(str(value).encode("utf-8")) if value_type is not None: response.registry_type = self.registry_map.get(value_type, 0) response.registry_data = rdf_protodict.DataBlob().SetValue(value) return response
def ResolveSubjectDestination(subject, regexes): """Returns the directory/filename where the subject will be stored. Args: subject: The subject. regexes: The list of regular expressions by priority. Returns: File name and directory. """ components = Components(subject) if not components: # No components to work with. return "aff4", "" # Make all the components safe to use. path = utils.JoinPath(*[ConvertStringToFilename(x) for x in components]) for route in regexes: m = route.match(path) if m: value = m.group("path") if value: base = os.path.basename(value) dirname = os.path.dirname(value) return base, dirname # Default value if nothing else matches. return "aff4", ""
def testGlobBackslashHandlingWithRegex(self): os.mkdir(utils.JoinPath(self.temp_dir, "1")) self._Touch("1/foo.txt") self._Touch("1/foo.txt~") paths = [ utils.JoinPath(self.temp_dir, "*/foo.txt"), # The backslash in the path will be replaced by a forward-slash when # building a tree representing all the paths (this behavior isn't # particularly correct). Note that the tilde does not need to be # escaped. utils.JoinPath(self.temp_dir, r"*/foo.txt\~"), ] expected_paths = [utils.JoinPath(self.temp_dir, "1/foo.txt")] results = self._RunGlob(paths) self.assertCountEqual(expected_paths, results)
def ListFiles(self, ext_attrs=False): """List all files in the dir.""" if not self.IsDirectory(): raise IOError("%s is not a directory." % self.path) for path in self.files: try: filepath = utils.JoinPath(self.path, path) response = self._Stat(filepath, ext_attrs=ext_attrs) pathspec = self.pathspec.Copy() pathspec.last.path = utils.JoinPath(pathspec.last.path, path) response.pathspec = pathspec yield response except OSError: pass
def setUp(self): super().setUp() filename = "%s_blah" % config.CONFIG["Client.tempfile_prefix"] self.tempfile = utils.JoinPath(self.temp_dir, "delete_test", filename) self.dirname = os.path.dirname(self.tempfile) os.makedirs(self.dirname) tempdir_overrider = test_lib.ConfigOverrider({ "Client.tempdir_roots": [os.path.dirname(self.dirname)], "Client.grr_tempdir": os.path.basename(self.dirname) }) tempdir_overrider.Start() self.addCleanup(tempdir_overrider.Stop) self.not_tempfile = os.path.join(self.temp_dir, "notatempfile") with io.open(self.not_tempfile, "wb") as fd: fd.write(b"something") self.temp_fd = tempfiles.CreateGRRTempFile(filename="file1") self.temp_fd2 = tempfiles.CreateGRRTempFile(filename="file2") self.assertTrue(os.path.exists(self.not_tempfile)) self.assertTrue(os.path.exists(self.temp_fd.name)) self.assertTrue(os.path.exists(self.temp_fd2.name)) self.pathspec = rdf_paths.PathSpec( path=self.dirname, pathtype=rdf_paths.PathSpec.PathType.OS)
def __init__(self, base_fd, pathspec=None, progress_callback=None): super(RegistryFile, self).__init__( base_fd, pathspec=pathspec, progress_callback=progress_callback) self.value = None self.value_type = _winreg.REG_NONE self.hive = None self.hive_name = None self.local_path = None self.last_modified = 0 self.is_directory = True self.fd = None if base_fd is None: self.pathspec.Append(pathspec) elif base_fd.IsDirectory(): self.pathspec.last.path = utils.JoinPath(self.pathspec.last.path, pathspec.path) else: raise IOError("Registry handler can not be stacked on another handler.") path_components = list(filter(None, self.pathspec.last.path.split("/"))) try: # The first component MUST be a hive self.hive_name = path_components[0] self.hive = KeyHandle(getattr(_winreg, self.hive_name)) except AttributeError: raise IOError("Unknown hive name %s" % self.hive_name) except IndexError: # A hive is not specified, we just list all the hives. return # Normalize the path casing if needed self.key_name = "/".join(path_components[1:]) self.local_path = CanonicalPathToLocalPath(self.key_name) try: # Maybe its a value key_name, value_name = os.path.split(self.local_path) with OpenKey(self.hive, key_name) as key: self.value, self.value_type = QueryValueEx(key, value_name) # We are a value and therefore not a directory. self.is_directory = False except OSError: try: # Try to get the default value for this key with OpenKey(self.hive, self.local_path) as key: # Check for default value. try: self.value, self.value_type = QueryValueEx(key, "") except OSError: # Empty default value self.value = "" self.value_type = _winreg.REG_NONE except OSError: raise IOError("Unable to open key %s" % self.key_name)
def DatabasesInDir(self, directory): """Returns a list of the database files in directory.""" for (path, dirs, files) in os.walk(directory, topdown=True): dirs.sort() # controls os.walk recurse order! files.sort() for f in files: if f.endswith(SQLITE_EXTENSION): f = f[:-len(SQLITE_EXTENSION)] yield utils.JoinPath(path, f)
def testGlobWithTwoStars(self): self._MakeTestDirs() paths = [os.path.join(self.temp_dir, "1/", "*/*/foo*")] # Handle filesystem case insensitivity results = ["1/2/3/foo3"] if platform.system() == "Linux": results = ["1/2/3/foo3", "1/2/3/fOo3"] self._RunGlob(paths) self.assertItemsEqual(self.flow_replies, [utils.JoinPath(self.temp_dir, x) for x in results])
def ListFiles(self, ext_attrs = False): del ext_attrs # Unused. self._CheckIsDirectory() for entry in self.fd.sub_file_entries: pathspec = self.pathspec.Copy() pathspec.last.path = utils.JoinPath(pathspec.last.path, entry.name) pathspec.last.inode = entry.file_reference pathspec.last.options = rdf_paths.PathSpec.Options.CASE_LITERAL yield self._Stat(entry, pathspec)
def ListFiles(self, ext_attrs: bool = False) -> Iterator[rdf_client_fs.StatEntry]: del ext_attrs # Unused. self._CheckIsDirectory() for entry in self.fd.ListFiles(): pathspec = self.pathspec.Copy() pathspec.last.path = utils.JoinPath(pathspec.last.path, entry.name) pathspec.last.inode = entry.st_ino pathspec.last.options = rdf_paths.PathSpec.Options.CASE_LITERAL if entry.HasField("stream_name"): pathspec.last.stream_name = entry.stream_name yield _ConvertStatEntry(entry, pathspec)
def testGlobWithTwoStars(self): self._MakeTestDirs() paths = [os.path.join(self.temp_dir, "1/", "*/*/foo*")] # Handle filesystem case insensitivity expected_results = ["1/2/3/foo3"] if platform.system() == "Linux": expected_results = ["1/2/3/foo3", "1/2/3/fOo3"] expected_results = [ utils.JoinPath(self.temp_dir, x) for x in expected_results ] results = self._RunGlob(paths) self.assertCountEqual(results, expected_results)
def LocalPathToCanonicalPath(path): """Converts path from the local system's convention to the canonical.""" path_components = path.split("/") result = [] for component in path_components: # Devices must maintain their \\ so they do not get broken up. m = re.match(r"\\\\.\\", component) # The component is not special and can be converted as normal if not m: component = component.replace("\\", "/") result.append(component) return utils.JoinPath(*result)
def testGlobWithMultiplePaths(self): self._MakeTestDirs() paths = [ os.path.join(self.temp_dir, "*/*/foo*"), os.path.join(self.temp_dir, "notthere"), os.path.join(self.temp_dir, "*/notthere"), os.path.join(self.temp_dir, "*/foo*") ] # Handle filesystem case sensitivity expected_results = ["1/foo1", "/1/2/foo2"] if platform.system() == "Linux": expected_results = ["1/foo1", "1/fOo1", "/1/2/fOo2", "/1/2/foo2"] results = self._RunGlob(paths) self.assertCountEqual( results, [utils.JoinPath(self.temp_dir, x) for x in expected_results])
def GenerateDirectory(self, input_dir=None, output_dir=None, replacements=None): input_dir = utils.NormalizePath(input_dir) output_dir = utils.NormalizePath(output_dir) replacements = replacements or [] for (root, _, files) in os.walk(input_dir): for filename in files: in_file = utils.JoinPath(root, filename) out_file = in_file.replace(input_dir, output_dir) for (s, replacement) in replacements: out_file = out_file.replace(s, replacement) utils.EnsureDirExists(os.path.dirname(out_file)) self.GenerateFile(in_file, out_file)
def _SetupTestDir(self, directory): base = utils.JoinPath(self.temp_dir, directory) os.makedirs(base) with io.open(utils.JoinPath(base, "a.txt"), "wb") as fd: fd.write("Hello World!\n") with io.open(utils.JoinPath(base, "b.txt"), "wb") as fd: pass with io.open(utils.JoinPath(base, "c.txt"), "wb") as fd: pass with io.open(utils.JoinPath(base, "d.txt"), "wb") as fd: pass sub = utils.JoinPath(base, "sub1") os.makedirs(sub) with io.open(utils.JoinPath(sub, "a.txt"), "wb") as fd: fd.write("Hello World!\n") with io.open(utils.JoinPath(sub, "b.txt"), "wb") as fd: pass with io.open(utils.JoinPath(sub, "c.txt"), "wb") as fd: pass return base
def _FindPathspec(self, args): path_type, components = rdf_objects.ParseCategorizedPath( args.file_path.rstrip("/")) components_copy = components[:] all_components = [] while components_copy: all_components.append(components_copy) components_copy = components_copy[:-1] res = data_store.REL_DB.ReadPathInfos(str(args.client_id), path_type, all_components) for k in sorted(res, key=len, reverse=True): path_info = res[k] if path_info is None: raise FileNotFoundError(args.client_id, path_type, components) if path_info.stat_entry and path_info.stat_entry.pathspec: ps = path_info.stat_entry.pathspec if len(k) < len(components): new_path = utils.JoinPath(*components[len(k):]) ps.Append( rdf_paths.PathSpec(path=new_path, pathtype=ps.last.pathtype)) return ps # We don't have any pathspec in the database so we just send the path we # have with the correct path type and hope for the best. pathspec = rdf_paths.PathSpec(path="/" + "/".join(components)) if path_type == rdf_objects.PathInfo.PathType.TSK: pathspec.pathtype = pathspec.PathType.TSK elif path_type == rdf_objects.PathInfo.PathType.NTFS: pathspec.pathtype = pathspec.PathType.NTFS elif path_type == rdf_objects.PathInfo.PathType.OS: pathspec.pathtype = pathspec.PathType.OS elif path_type == rdf_objects.PathInfo.PathType.REGISTRY: pathspec.pathtype = pathspec.PathType.REGISTRY elif path_type == rdf_objects.PathInfo.PathType.TEMP: pathspec.pathtype = pathspec.PathType.TMPFILE else: raise ValueError("Invalid path_type: %r" % path_type) return pathspec
def ListFiles(self, ext_attrs: bool = False) -> Iterable[rdf_client_fs.StatEntry]: del ext_attrs # Unused. self._CheckIsDirectory() for entry in self.fd.sub_file_entries: pathspec = self.pathspec.Copy() pathspec.last.path = utils.JoinPath(pathspec.last.path, entry.name) pathspec.last.inode = entry.file_reference pathspec.last.options = rdf_paths.PathSpec.Options.CASE_LITERAL data_stream = entry if entry.has_default_data_stream() else None yield self._Stat(entry, data_stream, pathspec.Copy()) # Create extra entries for alternate data streams for data_stream in entry.alternate_data_streams: pathspec.last.stream_name = data_stream.name yield self._Stat(entry, data_stream, pathspec.Copy())
def Add(self, path): """Add a relative stem to the current value and return a new RDFURN. If urn is a fully qualified URN, replace the current value with it. Args: path: A string containing a relative path. Returns: A new RDFURN that can be chained. Raises: ValueError: if the path component is not a string. """ if not isinstance(path, string_types): raise ValueError("Only strings should be added to a URN, not %s" % path.__class__) return self.__class__(utils.JoinPath(self._value, path))
def GetPrefix(self, subject_prefix): """Return list of databases matching subject_prefix.""" components = common.Components(subject_prefix) components = [common.ConvertStringToFilename(x) for x in components] path_prefix = utils.JoinPath(*components) if path_prefix == "/": path_prefix = "" for regex in self.path_regexes: result = common.EvaluatePrefix(path_prefix, regex) if result == "MATCH": yield self.Get(subject_prefix) return if result == "POSSIBLE": for conn in self.DatabasesByPath(path_prefix): yield conn return yield self.Get(subject_prefix)
def Add(self, path): """Add a relative stem to the current value and return a new RDFURN. Note that this returns an RDFURN, not a ClientURN since the resulting object would not pass validation. Args: path: A string containing a relative path. Returns: A new RDFURN that can be chained. Raises: ValueError: if the path component is not a string. """ if not isinstance(path, str): raise ValueError("Only strings should be added to a URN.") return rdfvalue.RDFURN(utils.JoinPath(self._value, path))
def GenerateDirectory(input_dir = None, output_dir = None, replacements = None, context = None): """Copies an a directory rewriting file names according to spec.""" if context is None: raise ValueError("context must be provided") input_dir = utils.NormalizePath(input_dir) output_dir = utils.NormalizePath(output_dir) replacements = replacements or [] for (root, _, files) in os.walk(input_dir): for filename in files: in_file = utils.JoinPath(root, filename) out_file = in_file.replace(input_dir, output_dir) for (s, replacement) in replacements: out_file = out_file.replace(s, replacement) utils.EnsureDirExists(os.path.dirname(out_file)) GenerateFile(in_file, out_file, context=context)
def Add(self, path, age=None): """Add a relative stem to the current value and return a new RDFURN. If urn is a fully qualified URN, replace the current value with it. Args: path: A string containing a relative path. age: The age of the object. If None set to current time. Returns: A new RDFURN that can be chained. Raises: ValueError: if the path component is not a string. """ if not isinstance(path, basestring): raise ValueError("Only strings should be added to a URN.") result = self.Copy(age) result.Update(path=utils.JoinPath(self._string_urn, path)) return result