def Run(self, args): """Downloads files referenced in the collection.""" export_utils.DownloadCollection(args.path, args.output, overwrite=args.overwrite, dump_client_info=args.dump_client_info, max_threads=args.threads)
def testDownloadCollectionWithFlattenOption(self): """Check we can download files references in RDFValueCollection.""" # Create a collection with URNs to some files. fd = aff4.FACTORY.Create("aff4:/testcoll", collects.RDFValueCollection, token=self.token) fd.Add(rdfvalue.RDFURN(self.out.Add("testfile1"))) fd.Add(rdf_client.StatEntry(aff4path=self.out.Add("testfile2"))) fd.Add(file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry( aff4path=self.out.Add("testfile5")))) fd.Close() with utils.TempDirectory() as tmpdir: export_utils.DownloadCollection("aff4:/testcoll", tmpdir, overwrite=True, dump_client_info=True, flatten=True, token=self.token, max_threads=2) # Check that "files" folder is filled with symlinks to downloaded files. symlinks = os.listdir(os.path.join(tmpdir, "files")) self.assertEqual(len(symlinks), 3) self.assertListEqual( sorted(symlinks), ["C.1000000000000000_fs_os_testfile1", "C.1000000000000000_fs_os_testfile2", "C.1000000000000000_fs_os_testfile5"]) self.assertEqual( os.readlink(os.path.join(tmpdir, "files", "C.1000000000000000_fs_os_testfile1")), os.path.join(tmpdir, "C.1000000000000000", "fs", "os", "testfile1"))
def testDownloadCollection(self): """Check we can download files references in RDFValueCollection.""" # Create a collection with URNs to some files. fd = aff4.FACTORY.Create("aff4:/testcoll", "RDFValueCollection", token=self.token) fd.Add(rdfvalue.RDFURN(self.out.Add("testfile1"))) fd.Add(rdfvalue.StatEntry(aff4path=self.out.Add("testfile2"))) fd.Add( file_finder.FileFinderResult(stat_entry=rdfvalue.StatEntry( aff4path=self.out.Add("testfile5")))) fd.Close() with utils.TempDirectory() as tmpdir: export_utils.DownloadCollection("aff4:/testcoll", tmpdir, overwrite=True, dump_client_info=True, token=self.token, max_threads=2) expected_outdir = os.path.join(tmpdir, self.out.Path()[1:]) # Check we found both files. self.assertTrue("testfile1" in os.listdir(expected_outdir)) self.assertTrue("testfile2" in os.listdir(expected_outdir)) self.assertTrue("testfile5" in os.listdir(expected_outdir)) # Check we dumped a YAML file to the root of the client. expected_rootdir = os.path.join(tmpdir, self.client_id.Basename()) self.assertTrue("client_info.yaml" in os.listdir(expected_rootdir))
def testDownloadCollectionWithFoldersEntries(self): """Check we can download RDFValueCollection that also references folders.""" fd = aff4.FACTORY.Create("aff4:/testcoll", "RDFValueCollection", token=self.token) fd.Add( file_finder.FileFinderResult(stat_entry=rdfvalue.StatEntry( aff4path=self.out.Add("testfile5")))) fd.Add( file_finder.FileFinderResult(stat_entry=rdfvalue.StatEntry( aff4path=self.out.Add("testdir1"), st_mode=stat.S_IFDIR))) fd.Close() with utils.TempDirectory() as tmpdir: export_utils.DownloadCollection("aff4:/testcoll", tmpdir, overwrite=True, dump_client_info=True, token=self.token, max_threads=2) expected_outdir = os.path.join(tmpdir, self.out.Path()[1:]) # Check we found both files. self.assertTrue("testfile5" in os.listdir(expected_outdir)) self.assertTrue("testdir1" in os.listdir(expected_outdir))
def Run(self, args): """Downloads files referenced in the collection.""" base_url = config_lib.CONFIG.Get("AdminUI.url", context=["AdminUI Context"]) source_path = rdfvalue.RDFURN(args.path) components = source_path.Split() if len(components) >= 2 and components[0] == "hunts": url = "%s/#/hunts/%s/results" % (base_url, components[1]) elif len(components) >= 3: url = "%s/#/clients/%s/flows/%s/results" % ( base_url, components[0], components[2]) else: url = "[AdminUI url unknown]" print "=============================================================" print( "WARNING: Command line export tool is DEPRECATED and will be " "removed soon.") print print "Please use the 'Generate ZIP' button on the results page instead." print "(the archive will be generated instantly on the fly):" print url print print "=============================================================" print if not args.no_legacy_warning_pause: raw_input("Press Enter if you still want to continue...") export_utils.DownloadCollection(args.path, args.output, overwrite=args.overwrite, dump_client_info=args.dump_client_info, flatten=args.flatten, max_threads=args.threads)
def testDownloadCollectionIgnoresArtifactResultsWithoutFiles(self): # Create a collection with URNs to some files. fd = sequential_collection.GeneralIndexedCollection( self.collection_urn, token=self.token) fd.Add(collectors.ArtifactFilesDownloaderResult()) with utils.TempDirectory() as tmpdir: export_utils.DownloadCollection(self.collection_urn, tmpdir, overwrite=True, dump_client_info=True, token=self.token, max_threads=2) expected_outdir = os.path.join(tmpdir, self.out.Path()[1:]) self.assertFalse(os.path.exists(expected_outdir))
def testDownloadCollectionIgnoresArtifactResultsWithoutFiles(self): # Create a collection with URNs to some files. fd = aff4.FACTORY.Create("aff4:/testcoll", collects.RDFValueCollection, token=self.token) fd.Add(collectors.ArtifactFilesDownloaderResult()) fd.Close() with utils.TempDirectory() as tmpdir: export_utils.DownloadCollection("aff4:/testcoll", tmpdir, overwrite=True, dump_client_info=True, token=self.token, max_threads=2) expected_outdir = os.path.join(tmpdir, self.out.Path()[1:]) self.assertFalse(os.path.exists(expected_outdir))
def _VerifyDownload(self): with utils.TempDirectory() as tmpdir: export_utils.DownloadCollection("aff4:/testcoll", tmpdir, overwrite=True, dump_client_info=True, token=self.token, max_threads=2) expected_outdir = os.path.join(tmpdir, self.out.Path()[1:]) # Check we found both files. self.assertTrue("testfile1" in os.listdir(expected_outdir)) self.assertTrue("testfile2" in os.listdir(expected_outdir)) self.assertTrue("testfile5" in os.listdir(expected_outdir)) self.assertTrue("testfile6" in os.listdir(expected_outdir)) # Check we dumped a YAML file to the root of the client. expected_rootdir = os.path.join(tmpdir, self.client_id.Basename()) self.assertTrue("client_info.yaml" in os.listdir(expected_rootdir))
def testDownloadCollectionWithFoldersEntries(self): """Check we can download a collection that also references folders.""" fd = sequential_collection.GeneralIndexedCollection( self.collection_urn, token=self.token) fd.Add( rdf_file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry( pathspec=rdf_paths.PathSpec(path="testfile5", pathtype="OS")))) fd.Add( rdf_file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry( pathspec=rdf_paths.PathSpec(path="testdir1", pathtype="OS"), st_mode=stat.S_IFDIR))) with utils.TempDirectory() as tmpdir: export_utils.DownloadCollection(self.collection_urn, tmpdir, overwrite=True, dump_client_info=True, token=self.token, max_threads=2) expected_outdir = os.path.join(tmpdir, self.out.Path()[1:]) # Check we found both files. self.assertTrue("testfile5" in os.listdir(expected_outdir)) self.assertTrue("testdir1" in os.listdir(expected_outdir))
def testDownloadCollectionWithFlattenOption(self): """Check we can download files references in a collection.""" # Create a collection with URNs to some files. fd = sequential_collection.GeneralIndexedCollection( self.collection_urn, token=self.token) fd.Add(rdfvalue.RDFURN(self.out.Add("testfile1"))) fd.Add( rdf_client.StatEntry( pathspec=rdf_paths.PathSpec(path="testfile2", pathtype="OS"))) fd.Add( rdf_file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry( pathspec=rdf_paths.PathSpec(path="testfile5", pathtype="OS")))) with utils.TempDirectory() as tmpdir: export_utils.DownloadCollection(self.collection_urn, tmpdir, overwrite=True, dump_client_info=True, flatten=True, token=self.token, max_threads=2) # Check that "files" folder is filled with symlinks to downloaded files. symlinks = os.listdir(os.path.join(tmpdir, "files")) self.assertEqual(len(symlinks), 3) self.assertListEqual(sorted(symlinks), [ "C.1000000000000000_fs_os_testfile1", "C.1000000000000000_fs_os_testfile2", "C.1000000000000000_fs_os_testfile5" ]) self.assertEqual( os.readlink( os.path.join(tmpdir, "files", "C.1000000000000000_fs_os_testfile1")), os.path.join(tmpdir, "C.1000000000000000", "fs", "os", "testfile1"))