def Run(self, args):
"""Downloads files referenced in the collection."""
export_utils.DownloadCollection(args.path,
args.output,
overwrite=args.overwrite,
dump_client_info=args.dump_client_info,
max_threads=args.threads)
def testDownloadCollectionWithFlattenOption(self):
"""Check we can download files references in RDFValueCollection."""
# Create a collection with URNs to some files.
fd = aff4.FACTORY.Create("aff4:/testcoll",
collects.RDFValueCollection,
token=self.token)
fd.Add(rdfvalue.RDFURN(self.out.Add("testfile1")))
fd.Add(rdf_client.StatEntry(aff4path=self.out.Add("testfile2")))
fd.Add(file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry(
aff4path=self.out.Add("testfile5"))))
fd.Close()
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection("aff4:/testcoll",
tmpdir,
overwrite=True,
dump_client_info=True,
flatten=True,
token=self.token,
max_threads=2)
# Check that "files" folder is filled with symlinks to downloaded files.
symlinks = os.listdir(os.path.join(tmpdir, "files"))
self.assertEqual(len(symlinks), 3)
self.assertListEqual(
sorted(symlinks), ["C.1000000000000000_fs_os_testfile1",
"C.1000000000000000_fs_os_testfile2",
"C.1000000000000000_fs_os_testfile5"])
self.assertEqual(
os.readlink(os.path.join(tmpdir, "files",
"C.1000000000000000_fs_os_testfile1")),
os.path.join(tmpdir, "C.1000000000000000", "fs", "os", "testfile1"))
def testDownloadCollection(self):
"""Check we can download files references in RDFValueCollection."""
# Create a collection with URNs to some files.
fd = aff4.FACTORY.Create("aff4:/testcoll",
"RDFValueCollection",
token=self.token)
fd.Add(rdfvalue.RDFURN(self.out.Add("testfile1")))
fd.Add(rdfvalue.StatEntry(aff4path=self.out.Add("testfile2")))
fd.Add(
file_finder.FileFinderResult(stat_entry=rdfvalue.StatEntry(
aff4path=self.out.Add("testfile5"))))
fd.Close()
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection("aff4:/testcoll",
tmpdir,
overwrite=True,
dump_client_info=True,
token=self.token,
max_threads=2)
expected_outdir = os.path.join(tmpdir, self.out.Path()[1:])
# Check we found both files.
self.assertTrue("testfile1" in os.listdir(expected_outdir))
self.assertTrue("testfile2" in os.listdir(expected_outdir))
self.assertTrue("testfile5" in os.listdir(expected_outdir))
# Check we dumped a YAML file to the root of the client.
expected_rootdir = os.path.join(tmpdir, self.client_id.Basename())
self.assertTrue("client_info.yaml" in os.listdir(expected_rootdir))
def testDownloadCollectionWithFoldersEntries(self):
"""Check we can download RDFValueCollection that also references folders."""
fd = aff4.FACTORY.Create("aff4:/testcoll",
"RDFValueCollection",
token=self.token)
fd.Add(
file_finder.FileFinderResult(stat_entry=rdfvalue.StatEntry(
aff4path=self.out.Add("testfile5"))))
fd.Add(
file_finder.FileFinderResult(stat_entry=rdfvalue.StatEntry(
aff4path=self.out.Add("testdir1"), st_mode=stat.S_IFDIR)))
fd.Close()
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection("aff4:/testcoll",
tmpdir,
overwrite=True,
dump_client_info=True,
token=self.token,
max_threads=2)
expected_outdir = os.path.join(tmpdir, self.out.Path()[1:])
# Check we found both files.
self.assertTrue("testfile5" in os.listdir(expected_outdir))
self.assertTrue("testdir1" in os.listdir(expected_outdir))
def Run(self, args):
"""Downloads files referenced in the collection."""
base_url = config_lib.CONFIG.Get("AdminUI.url",
context=["AdminUI Context"])
source_path = rdfvalue.RDFURN(args.path)
components = source_path.Split()
if len(components) >= 2 and components[0] == "hunts":
url = "%s/#/hunts/%s/results" % (base_url, components[1])
elif len(components) >= 3:
url = "%s/#/clients/%s/flows/%s/results" % (
base_url, components[0], components[2])
else:
url = "[AdminUI url unknown]"
print "============================================================="
print(
"WARNING: Command line export tool is DEPRECATED and will be "
"removed soon.")
print
print "Please use the 'Generate ZIP' button on the results page instead."
print "(the archive will be generated instantly on the fly):"
print url
print
print "============================================================="
print
if not args.no_legacy_warning_pause:
raw_input("Press Enter if you still want to continue...")
export_utils.DownloadCollection(args.path,
args.output,
overwrite=args.overwrite,
dump_client_info=args.dump_client_info,
flatten=args.flatten,
max_threads=args.threads)
def testDownloadCollectionIgnoresArtifactResultsWithoutFiles(self):
# Create a collection with URNs to some files.
fd = sequential_collection.GeneralIndexedCollection(
self.collection_urn, token=self.token)
fd.Add(collectors.ArtifactFilesDownloaderResult())
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection(self.collection_urn,
tmpdir,
overwrite=True,
dump_client_info=True,
token=self.token,
max_threads=2)
expected_outdir = os.path.join(tmpdir, self.out.Path()[1:])
self.assertFalse(os.path.exists(expected_outdir))
def testDownloadCollectionIgnoresArtifactResultsWithoutFiles(self):
# Create a collection with URNs to some files.
fd = aff4.FACTORY.Create("aff4:/testcoll",
collects.RDFValueCollection,
token=self.token)
fd.Add(collectors.ArtifactFilesDownloaderResult())
fd.Close()
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection("aff4:/testcoll",
tmpdir,
overwrite=True,
dump_client_info=True,
token=self.token,
max_threads=2)
expected_outdir = os.path.join(tmpdir, self.out.Path()[1:])
self.assertFalse(os.path.exists(expected_outdir))
def _VerifyDownload(self):
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection("aff4:/testcoll",
tmpdir,
overwrite=True,
dump_client_info=True,
token=self.token,
max_threads=2)
expected_outdir = os.path.join(tmpdir, self.out.Path()[1:])
# Check we found both files.
self.assertTrue("testfile1" in os.listdir(expected_outdir))
self.assertTrue("testfile2" in os.listdir(expected_outdir))
self.assertTrue("testfile5" in os.listdir(expected_outdir))
self.assertTrue("testfile6" in os.listdir(expected_outdir))
# Check we dumped a YAML file to the root of the client.
expected_rootdir = os.path.join(tmpdir, self.client_id.Basename())
self.assertTrue("client_info.yaml" in os.listdir(expected_rootdir))
def testDownloadCollectionWithFoldersEntries(self):
"""Check we can download a collection that also references folders."""
fd = sequential_collection.GeneralIndexedCollection(
self.collection_urn, token=self.token)
fd.Add(
rdf_file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry(
pathspec=rdf_paths.PathSpec(path="testfile5", pathtype="OS"))))
fd.Add(
rdf_file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry(
pathspec=rdf_paths.PathSpec(path="testdir1", pathtype="OS"),
st_mode=stat.S_IFDIR)))
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection(self.collection_urn,
tmpdir,
overwrite=True,
dump_client_info=True,
token=self.token,
max_threads=2)
expected_outdir = os.path.join(tmpdir, self.out.Path()[1:])
# Check we found both files.
self.assertTrue("testfile5" in os.listdir(expected_outdir))
self.assertTrue("testdir1" in os.listdir(expected_outdir))
def testDownloadCollectionWithFlattenOption(self):
"""Check we can download files references in a collection."""
# Create a collection with URNs to some files.
fd = sequential_collection.GeneralIndexedCollection(
self.collection_urn, token=self.token)
fd.Add(rdfvalue.RDFURN(self.out.Add("testfile1")))
fd.Add(
rdf_client.StatEntry(
pathspec=rdf_paths.PathSpec(path="testfile2", pathtype="OS")))
fd.Add(
rdf_file_finder.FileFinderResult(stat_entry=rdf_client.StatEntry(
pathspec=rdf_paths.PathSpec(path="testfile5", pathtype="OS"))))
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection(self.collection_urn,
tmpdir,
overwrite=True,
dump_client_info=True,
flatten=True,
token=self.token,
max_threads=2)
# Check that "files" folder is filled with symlinks to downloaded files.
symlinks = os.listdir(os.path.join(tmpdir, "files"))
self.assertEqual(len(symlinks), 3)
self.assertListEqual(sorted(symlinks), [
"C.1000000000000000_fs_os_testfile1",
"C.1000000000000000_fs_os_testfile2",
"C.1000000000000000_fs_os_testfile5"
])
self.assertEqual(
os.readlink(
os.path.join(tmpdir, "files",
"C.1000000000000000_fs_os_testfile1")),
os.path.join(tmpdir, "C.1000000000000000", "fs", "os",
"testfile1"))
