def Stat(self, responses): """Save stat information on the directory.""" # Did it work? if not responses.success: self.Error("Could not stat directory: %s" % responses.status) else: # Keep the stat response for later. stat_entry = rdf_client.StatEntry(responses.First()) self.state.stat = stat_entry # The full path of the object is the combination of the client_id and the # path. self.state.urn = stat_entry.pathspec.AFF4Path(self.client_id)
def testHuntAuthorizationIsRequiredToGenerateResultsArchive(self): stat_entry = rdf_client.StatEntry(aff4path="aff4:/foo/bar") values = [file_finder.FileFinderResult(stat_entry=stat_entry)] with self.ACLChecksDisabled(): self.CreateGenericHuntWithCollection(values=values) self.Open("/") self.Click("css=a[grrtarget=ManageHunts]") self.Click("css=td:contains('GenericHunt')") self.Click("css=li[heading=Results]") self.Click("css=button.DownloadButton") self.WaitUntil(self.IsElementPresent, "acl_dialog")
def testFileViewHasExportTabWhenCollectionHasFileFinderResults(self): collection_urn = "aff4:/C.0000000000000001/analysis/SomeFlow/results" with self.ACLChecksDisabled(): with aff4.FACTORY.Create(collection_urn, "RDFValueCollection", token=self.token) as fd: fd.Add( file_finder.FileFinderResult( stat_entry=rdf_client.StatEntry( aff4path="aff4:/some/unique/path"))) self.GrantClientApproval("C.0000000000000001") self.CheckExportTabIsPresent()
def Handle(self, args, token=None): ValidateVfsPath(args.file_path) if args.timestamp: age = args.timestamp else: age = aff4.ALL_TIMES file_obj = aff4.FACTORY.Open(args.client_id.ToClientURN().Add( args.file_path), mode="r", age=age, token=token) if data_store.RelationalDBReadEnabled(category="vfs"): # These are not really "files" so they cannot be stored in the database # but they still can be queried so we need to return something. Sometimes # they contain a trailing slash so we need to take care of that. # # TODO(hanuszczak): Require VFS paths to be normalized so that trailing # slash is either forbidden or mandatory. if args.file_path.endswith("/"): args.file_path = args.file_path[:-1] if args.file_path in ["fs", "registry", "temp", "fs/os", "fs/tsk"]: api_file = ApiFile() api_file.name = api_file.path = args.file_path api_file.is_directory = True return ApiGetFileDetailsResult(file=api_file) path_type, components = rdf_objects.ParseCategorizedPath( args.file_path) # TODO(hanuszczak): The tests passed even without support for timestamp # filtering. The test suite should be probably improved in that regard. path_id = rdf_objects.PathID(components) path_info = data_store.REL_DB.FindPathInfoByPathID( str(args.client_id), path_type, path_id, timestamp=args.timestamp) if path_info: stat_entry = path_info.stat_entry else: stat_entry = rdf_client.StatEntry() else: stat_entry = None return ApiGetFileDetailsResult(file=ApiFile().InitFromAff4Object( file_obj, stat_entry=stat_entry, with_details=True))
def testHuntAuthorizationIsRequiredToGenerateResultsArchive(self): stat_entry = rdf_client.StatEntry(pathspec=rdf_paths.PathSpec( path="/foo/bar", pathtype=rdf_paths.PathSpec.PathType.OS)) values = [rdf_file_finder.FileFinderResult(stat_entry=stat_entry)] self.CreateGenericHuntWithCollection(values=values) self.Open("/") self.Click("css=a[grrtarget=hunts]") self.Click("css=td:contains('GenericHunt')") self.Click("css=li[heading=Results]") self.Click("css=button.DownloadButton") self.WaitUntil(self.IsTextPresent, "Create a new approval request")
def _GenFiles(self, passwd, shadow, group, gshadow): stats = [] files = [] for path in [ "/etc/passwd", "/etc/shadow", "/etc/group", "/etc/gshadow" ]: p = rdf_paths.PathSpec(path=path) stats.append(rdf_client.StatEntry(pathspec=p)) for data in passwd, shadow, group, gshadow: if data is None: data = [] lines = "\n".join(data).format(**self.crypt) files.append(StringIO.StringIO(lines)) return stats, files
def testShowsGenerateArchiveButtonForFileFinderHunt(self): stat_entry = rdf_client.StatEntry(pathspec=rdf_paths.PathSpec( path="/foo/bar", pathtype=rdf_paths.PathSpec.PathType.OS)) values = [rdf_file_finder.FileFinderResult(stat_entry=stat_entry)] self.CreateGenericHuntWithCollection(values=values) self.Open("/") self.Click("css=a[grrtarget=hunts]") self.Click("css=td:contains('GenericHunt')") self.Click("css=li[heading=Results]") self.WaitUntil(self.IsTextPresent, "Files referenced in this collection can be downloaded")
def GetTimelineItems(folder_urn, token=None): """Retrieves the timeline items for a given folder. The timeline consists of items indicating a state change of a file. To construct the timeline, MAC times are used. Whenever a timestamp on a file changes, a corresponding timeline item is created. Args: folder_urn: The urn of the target folder. token: The user token. Returns: A list of timeline items, each consisting of a file path, a timestamp and an action describing the nature of the file change. """ child_urns = [] for _, children in aff4.FACTORY.RecursiveMultiListChildren( folder_urn, token=token): child_urns.extend(children) # Get the stats attributes for all clients. attribute = aff4.Attribute.GetAttributeByName("stat") items = [] for subject, values in data_store.DB.MultiResolvePrefix( child_urns, attribute.predicate, token=token): for _, serialized, _ in values: stat = rdf_client.StatEntry(serialized) # Add a new event for each MAC time if it exists. for c in "mac": timestamp = getattr(stat, "st_%stime" % c) if timestamp is not None: item = ApiVfsTimelineItem() item.timestamp = timestamp * 1000000 # Remove aff4:/<client_id> to have a more concise path to the # subject. item.file_path = "/".join(subject.split("/")[2:]) if c == "m": item.action = ApiVfsTimelineItem.FileActionType.MODIFICATION elif c == "a": item.action = ApiVfsTimelineItem.FileActionType.ACCESS elif c == "c": item.action = ApiVfsTimelineItem.FileActionType.METADATA_CHANGED items.append(item) return sorted(items, key=lambda x: x.timestamp, reverse=True)
def testPackageSourceData(self): test_data = r""" # comment 1 [centosdvdiso] name=CentOS DVD ISO baseurl=file:///mnt http://mirror1.centos.org/CentOS/6/os/i386/ baseurl =ssh://mirror2.centos.org/CentOS/6/os/i386/ enabled=1 gpgcheck=1 gpgkey=file:///mnt/RPM-GPG-KEY-CentOS-6 # comment2 [examplerepo] name=Example Repository baseurl = https://mirror3.centos.org/CentOS/6/os/i386/ enabled=1 gpgcheck=1 gpgkey=http://mirror.centos.org/CentOS/6/os/i386/RPM-GPG-KEY-CentOS-6 """ file_obj = StringIO.StringIO(test_data) pathspec = rdf_paths.PathSpec(path="/etc/yum.repos.d/test1.repo") stat = rdf_client.StatEntry(pathspec=pathspec) parser = config_file.YumPackageSourceParser() results = list(parser.Parse(stat, file_obj, None)) result = [ d for d in results if isinstance(d, rdf_protodict.AttributedDict) ][0] self.assertEqual("/etc/yum.repos.d/test1.repo", result.filename) self.assertEqual(4, len(result.uris)) self.assertEqual("file", result.uris[0].transport) self.assertEqual("", result.uris[0].host) self.assertEqual("/mnt", result.uris[0].path) self.assertEqual("http", result.uris[1].transport) self.assertEqual("mirror1.centos.org", result.uris[1].host) self.assertEqual("/CentOS/6/os/i386/", result.uris[1].path) self.assertEqual("ssh", result.uris[2].transport) self.assertEqual("mirror2.centos.org", result.uris[2].host) self.assertEqual("/CentOS/6/os/i386/", result.uris[2].path) self.assertEqual("https", result.uris[3].transport) self.assertEqual("mirror3.centos.org", result.uris[3].host) self.assertEqual("/CentOS/6/os/i386/", result.uris[3].path)
def Stat(self, responses): """Save stat information on the directory.""" # Did it work? if not responses.success: self.Error("Could not stat directory: %s" % responses.status) else: # Keep the stat response for later. self.state.Register("stat", rdf_client.StatEntry(responses.First())) self.state.Register("directory_pathspec", self.state.stat.pathspec) # The full path of the object is the combination of the client_id and the # path. self.state.Register("urn", aff4.AFF4Object.VFSGRRClient.PathspecToURN( self.state.directory_pathspec, self.client_id))
def testOSXUsersParser(self): """Ensure we can extract users from a passwd file.""" paths = ["/Users/user1", "/Users/user2", "/Users/Shared"] statentries = [] client = "C.1000000000000000" for path in paths: statentries.append(rdf_client.StatEntry( aff4path=rdf_client.ClientURN(client).Add("fs/os").Add(path), pathspec=rdf_paths.PathSpec(path=path, pathtype=rdf_paths.PathSpec.PathType.OS), st_mode=16877)) statentries.append(rdf_client.StatEntry( aff4path=rdf_client.ClientURN(client).Add( "fs/os").Add("/Users/.localized"), pathspec=rdf_paths.PathSpec(path="/Users/.localized", pathtype=rdf_paths.PathSpec.PathType.OS), st_mode=33261)) parser = osx_file_parser.OSXUsersParser() out = list(parser.Parse(statentries, None, None)) self.assertItemsEqual([x.username for x in out], ["user1", "user2"]) self.assertItemsEqual([x.homedir for x in out], ["/Users/user1", "/Users/user2"])
def testDownloadCollectionWithFlattenOption(self): """Check we can download files references in RDFValueCollection.""" # Create a collection with URNs to some files. fd = aff4.FACTORY.Create("aff4:/testcoll", "RDFValueCollection", token=self.token) fd.Add(rdfvalue.RDFURN(self.out.Add("testfile1"))) fd.Add(rdf_client.StatEntry(aff4path=self.out.Add("testfile2"))) fd.Add( file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry( aff4path=self.out.Add("testfile5")))) fd.Close() with utils.TempDirectory() as tmpdir: export_utils.DownloadCollection("aff4:/testcoll", tmpdir, overwrite=True, dump_client_info=True, flatten=True, token=self.token, max_threads=2) # Check that "files" folder is filled with symlinks to downloaded files. symlinks = os.listdir(os.path.join(tmpdir, "files")) self.assertEqual(len(symlinks), 3) self.assertListEqual(sorted(symlinks), [ "C.1000000000000000_fs_os_testfile1", "C.1000000000000000_fs_os_testfile2", "C.1000000000000000_fs_os_testfile5" ]) self.assertEqual( os.readlink( os.path.join(tmpdir, "files", "C.1000000000000000_fs_os_testfile1")), os.path.join(tmpdir, "C.1000000000000000", "fs", "os", "testfile1"))
def testDownloadHuntResultCollection(self): """Check we can download files references in HuntResultCollection.""" # Create a collection with URNs to some files. fd = aff4.FACTORY.Create(self.collection_urn, results.HuntResultCollection, token=self.token) fd.AddAsMessage(rdfvalue.RDFURN(self.out.Add("testfile1")), self.client_id) fd.AddAsMessage( rdf_client.StatEntry( pathspec=rdf_paths.PathSpec(path="testfile2", pathtype="OS")), self.client_id) fd.AddAsMessage( rdf_file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry( pathspec=rdf_paths.PathSpec(path="testfile5", pathtype="OS"))), self.client_id) fd.AddAsMessage( collectors.ArtifactFilesDownloaderResult( downloaded_file=rdf_client.StatEntry( pathspec=rdf_paths.PathSpec(path="testfile6", pathtype="OS"))), self.client_id) fd.Close() self._VerifyDownload()
def testDownloadCollectionWithFlattenOption(self): """Check we can download files references in a collection.""" # Create a collection with URNs to some files. fd = sequential_collection.GeneralIndexedCollection( self.collection_urn, token=self.token) fd.Add(rdfvalue.RDFURN(self.out.Add("testfile1"))) fd.Add( rdf_client.StatEntry( pathspec=rdf_paths.PathSpec(path="testfile2", pathtype="OS"))) fd.Add( rdf_file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry( pathspec=rdf_paths.PathSpec(path="testfile5", pathtype="OS")))) with utils.TempDirectory() as tmpdir: export_utils.DownloadCollection(self.collection_urn, tmpdir, overwrite=True, dump_client_info=True, flatten=True, token=self.token, max_threads=2) # Check that "files" folder is filled with symlinks to downloaded files. symlinks = os.listdir(os.path.join(tmpdir, "files")) self.assertEqual(len(symlinks), 3) self.assertListEqual(sorted(symlinks), [ "C.1000000000000000_fs_os_testfile1", "C.1000000000000000_fs_os_testfile2", "C.1000000000000000_fs_os_testfile5" ]) self.assertEqual( os.readlink( os.path.join(tmpdir, "files", "C.1000000000000000_fs_os_testfile1")), os.path.join(tmpdir, "C.1000000000000000", "fs", "os", "testfile1"))
def testDownloadCollectionWithFoldersEntries(self): """Check we can download RDFValueCollection that also references folders.""" fd = aff4.FACTORY.Create("aff4:/testcoll", collects.RDFValueCollection, token=self.token) fd.Add(file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry( aff4path=self.out.Add("testfile5")))) fd.Add(file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry( aff4path=self.out.Add("testdir1"), st_mode=stat.S_IFDIR))) fd.Close() with utils.TempDirectory() as tmpdir: export_utils.DownloadCollection("aff4:/testcoll", tmpdir, overwrite=True, dump_client_info=True, token=self.token, max_threads=2) expected_outdir = os.path.join(tmpdir, self.out.Path()[1:]) # Check we found both files. self.assertTrue("testfile5" in os.listdir(expected_outdir)) self.assertTrue("testdir1" in os.listdir(expected_outdir))
def testDownloadCollectionWithFoldersEntries(self): """Check we can download a collection that also references folders.""" fd = sequential_collection.GeneralIndexedCollection( self.collection_urn, token=self.token) fd.Add( rdf_file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry( pathspec=rdf_paths.PathSpec(path="testfile5", pathtype="OS")))) fd.Add( rdf_file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry( pathspec=rdf_paths.PathSpec(path="testdir1", pathtype="OS"), st_mode=stat.S_IFDIR))) with utils.TempDirectory() as tmpdir: export_utils.DownloadCollection(self.collection_urn, tmpdir, overwrite=True, dump_client_info=True, token=self.token, max_threads=2) expected_outdir = os.path.join(tmpdir, self.out.Path()[1:]) # Check we found both files. self.assertTrue("testfile5" in os.listdir(expected_outdir)) self.assertTrue("testdir1" in os.listdir(expected_outdir))
def testShowsPerFileDownloadButtonForFileFinderHunt(self): stat_entry = rdf_client.StatEntry(pathspec=rdf_paths.PathSpec( path="/foo/bar", pathtype=rdf_paths.PathSpec.PathType.OS)) values = [rdf_file_finder.FileFinderResult(stat_entry=stat_entry)] self.CreateGenericHuntWithCollection(values=values) self.Open("/") self.Click("css=a[grrtarget=hunts]") self.Click("css=td:contains('GenericHunt')") self.Click("css=li[heading=Results]") self.WaitUntil( self.IsElementPresent, "css=grr-results-collection button:has(span.glyphicon-download)")
def testExportCommandIsShownForStatEntryResults(self): stat_entry = rdf_client.StatEntry(pathspec=rdf_paths.PathSpec( path="/foo/bar", pathtype=rdf_paths.PathSpec.PathType.OS)) values = [rdf_file_finder.FileFinderResult(stat_entry=stat_entry)] hunt_urn = self.CreateGenericHuntWithCollection(values=values) self.Open("/#/hunts/%s/results" % hunt_urn.Basename()) self.Click("link=Show export command") self.WaitUntil(self.IsTextPresent, "/usr/bin/grr_api_shell 'http://localhost:8000/' " "--exec_code 'grrapi.Hunt(\"%s\").GetFilesArchive()." "WriteToFile(\"./hunt_results_%s.zip\")'" % (hunt_urn.Basename(), hunt_urn.Basename().replace(":", "_")))
def testCSVPluginWritesUnicodeValuesCorrectly(self): streams = self.ProcessResponses( plugin_args=csv_plugin.CSVOutputPluginArgs(), responses=[ rdf_client.StatEntry(pathspec=rdf_paths.PathSpec( path="/中国新闻网新闻中", pathtype="OS")) ], process_responses_separately=True) contents = StringIO.StringIO(streams["ExportedFile.csv"].Read(16384)) parsed_output = list(csv.DictReader(contents)) self.assertEqual(len(parsed_output), 1) self.assertEqual(parsed_output[0]["urn"], self.client_id.Add("/fs/os/中国新闻网新闻中"))
def CreateFileWithTimeline(file_path, token): """Add a file with timeline.""" # Add a version of the file at gui_test_lib.TIME_0. Since we write all MAC # times, this will result in three timeline items. with test_lib.FakeTime(gui_test_lib.TIME_0): with aff4.FACTORY.Create(file_path, aff4_grr.VFSAnalysisFile, mode="w", token=token) as fd: stats = rdf_client.StatEntry( st_atime=gui_test_lib.TIME_0.AsSecondsFromEpoch() + 1000, st_mtime=gui_test_lib.TIME_0.AsSecondsFromEpoch(), st_ctime=gui_test_lib.TIME_0.AsSecondsFromEpoch() - 1000) fd.Set(fd.Schema.STAT, stats) # Add a version with a stat entry, but without timestamps. with test_lib.FakeTime(gui_test_lib.TIME_1): with aff4.FACTORY.Create(file_path, aff4_grr.VFSAnalysisFile, mode="w", token=token) as fd: stats = rdf_client.StatEntry(st_ino=99) fd.Set(fd.Schema.STAT, stats)
def testShowsGenerateArchiveButtonForFileFinderHunt(self): stat_entry = rdf_client.StatEntry(aff4path="aff4:/foo/bar") values = [file_finder.FileFinderResult(stat_entry=stat_entry)] with self.ACLChecksDisabled(): self.CreateGenericHuntWithCollection(values=values) self.Open("/") self.Click("css=a[grrtarget=hunts]") self.Click("css=td:contains('GenericHunt')") self.Click("css=li[heading=Results]") self.WaitUntil( self.IsTextPresent, "Files referenced in this collection can be downloaded")
def testDownloadHuntResultCollection(self): """Check we can download files references in HuntResultCollection.""" # Create a collection with URNs to some files. fd = results.HuntResultCollection(self.collection_urn) with data_store.DB.GetMutationPool() as pool: fd.AddAsMessage(rdfvalue.RDFURN(self.out.Add("testfile1")), self.client_id, mutation_pool=pool) fd.AddAsMessage(rdf_client.StatEntry( pathspec=rdf_paths.PathSpec(path="testfile2", pathtype="OS")), self.client_id, mutation_pool=pool) fd.AddAsMessage(rdf_file_finder.FileFinderResult( stat_entry=rdf_client.StatEntry(pathspec=rdf_paths.PathSpec( path="testfile5", pathtype="OS"))), self.client_id, mutation_pool=pool) fd.AddAsMessage(collectors.ArtifactFilesDownloaderResult( downloaded_file=rdf_client.StatEntry( pathspec=rdf_paths.PathSpec(path="testfile6", pathtype="OS"))), self.client_id, mutation_pool=pool) self._VerifyDownload()
def _FakeDirStat(self, vfs_type=None): # We return some fake data, this makes writing tests easier for some # things but we give an error to the tester as it is often not what you # want. logging.warn("Fake value for %s under %s", self.path, self.prefix) for path in self.pathspec: path.path = self._NormalizeCaseForPath(self.path, vfs_type=vfs_type) return rdf_client.StatEntry( pathspec=self.pathspec, st_mode=16877, st_size=12288, st_atime=1319796280, st_dev=1)
def testBigQueryPluginWithValuesOfSameType(self): responses = [] for i in range(10): responses.append( rdf_client.StatEntry( aff4path=self.client_id.Add("/fs/os/foo/bar").Add(str(i)), pathspec=rdf_paths.PathSpec(path="/foo/bar"), st_mode=33184, # octal = 100640 => u=rw,g=r,o= => -rw-r----- st_ino=1063090, st_dev=64512L, st_nlink=1 + i, st_uid=139592, st_gid=5000, st_size=0, st_atime=1336469177, st_mtime=1336129892, st_ctime=1336129892)) output = self.ProcessResponses( plugin_args=bigquery_plugin.BigQueryOutputPluginArgs(), responses=responses) self.assertEqual(len(output), 1) _, stream, schema, job_id = output[0] self.assertEqual(job_id, "C-1000000000000000_Results_ExportedFile_1445995873") self.CompareSchemaToKnownGood(schema) actual_fd = gzip.GzipFile( None, "r", bigquery_plugin.BigQueryOutputPlugin.GZIP_COMPRESSION_LEVEL, stream) # Compare to our stored data. expected_fd = open( os.path.join(config_lib.CONFIG["Test.data_dir"], "bigquery", "ExportedFile.json")) # Bigquery expects a newline separarted list of JSON dicts, but this isn't # valid JSON so we can't just load the whole thing and compare. counter = 0 for actual, expected in zip(actual_fd, expected_fd): self.assertEqual(json.loads(actual), json.loads(expected)) counter += 1 self.assertEqual(counter, 10)
def ProcessListDirectory(self, responses): """Processes the results of the ListDirectory client action. Args: responses: a flow Responses object. """ if not responses.success: raise flow.FlowError("Unable to list directory.") for response in responses: stat_entry = rdf_client.StatEntry(response) filesystem.CreateAFF4Object( stat_entry, self.client_id, self.token, sync=False) self.SendReply(stat_entry) aff4.FACTORY.Flush()
def testGenerateZipButtonGetsDisabledAfterClick(self): stat_entry = rdf_client.StatEntry(pathspec=rdf_paths.PathSpec( path="/foo/bar", pathtype=rdf_paths.PathSpec.PathType.OS)) values = [rdf_file_finder.FileFinderResult(stat_entry=stat_entry)] hunt_urn = self.CreateGenericHuntWithCollection(values=values) self.GrantHuntApproval(hunt_urn) self.Open("/") self.Click("css=a[grrtarget=hunts]") self.Click("css=td:contains('GenericHunt')") self.Click("css=li[heading=Results]") self.Click("css=button.DownloadButton") self.WaitUntil(self.IsElementPresent, "css=button.DownloadButton[disabled]") self.WaitUntil(self.IsTextPresent, "Generation has started")
def _MakeRegStat(self, path, value, registry_type): options = rdf_paths.PathSpec.Options.CASE_LITERAL pathspec = rdf_paths.PathSpec( path=path, path_options=options, pathtype=rdf_paths.PathSpec.PathType.REGISTRY) if registry_type == rdf_client.StatEntry.RegistryType.REG_MULTI_SZ: reg_data = rdf_protodict.DataBlob(list=rdf_protodict.BlobArray( content=rdf_protodict.DataBlob(string=value))) else: reg_data = rdf_protodict.DataBlob().SetValue(value) return rdf_client.StatEntry(pathspec=pathspec, registry_data=reg_data, registry_type=registry_type)
def testShowsPerFileDownloadButtonForFileFinderHunt(self): with self.ACLChecksDisabled(): client_id = self.SetupClients(1)[0] stat_entry = rdf_client.StatEntry(aff4path=client_id.Add("fs/os/foo/bar")) values = [file_finder.FileFinderResult(stat_entry=stat_entry)] with self.ACLChecksDisabled(): self.CreateGenericHuntWithCollection(values=values) self.Open("/") self.Click("css=a[grrtarget=hunts]") self.Click("css=td:contains('GenericHunt')") self.Click("css=li[heading=Results]") self.WaitUntil(self.IsElementPresent, "css=grr-results-collection grr-downloadable-urn button")
def SetupTestTimeline(self): self.client_id = self.SetupClient(0) fixture_test_lib.ClientFixture(self.client_id, token=self.token) # Choose some directory with pathspec in the ClientFixture. self.folder_path = "fs/os/Users/中国新闻网新闻中/Shared" self.file_path = self.folder_path + "/a.txt" file_urn = self.client_id.Add(self.file_path) for i in range(0, 5): with test_lib.FakeTime(i): with aff4.FACTORY.Create( file_urn, aff4_grr.VFSFile, mode="w", token=self.token) as fd: stats = rdf_client.StatEntry( st_mtime=rdfvalue.RDFDatetimeSeconds().Now()) fd.Set(fd.Schema.STAT, stats)
def testBigQueryPluginWithEarlyFlush(self): responses = [] for i in range(10): responses.append( rdf_client.StatEntry( pathspec=rdf_paths.PathSpec( path="/foo/bar/%d" % i, pathtype="OS"), st_mode=33184, # octal = 100640 => u=rw,g=r,o= => -rw-r----- st_ino=1063090, st_dev=64512L, st_nlink=1 + i, st_uid=139592, st_gid=5000, st_size=0, st_atime=1336469177, st_mtime=1336129892, st_ctime=1336129892)) # Force an early flush. This max file size value has been chosen to force a # flush at least once, but not for all 10 records. with test_lib.ConfigOverrider({"BigQuery.max_file_post_size": 800}): output = self.ProcessResponses( plugin_args=bigquery_plugin.BigQueryOutputPluginArgs(), responses=responses) self.assertEqual(len(output), 2) # Check that the output is still consistent actual_fds = [] for _, stream, _, _ in output: actual_fds.append(gzip.GzipFile(None, "r", 9, stream)) # Compare to our stored data. # TODO(user): there needs to be a better way to generate these files on # change than breaking into the debugger. expected_fd = open( os.path.join(config.CONFIG["Test.data_dir"], "bigquery", "ExportedFile.json"), "rb") # Check that the same entries we expect are spread across the two files. counter = 0 for actual_fd in actual_fds: for actual, expected in zip(actual_fd, expected_fd): self.assertEqual(json.loads(actual), json.loads(expected)) counter += 1 self.assertEqual(counter, 10)