def test_open_repository_verification(self):
self.mock_repo.make_valid_whitelist()
self.mock_repo.serve_via_http()
repo1 = cvmfs.open_repository(self.mock_repo.url,
self.mock_repo.public_key)
self.assertTrue(isinstance(repo1, cvmfs.RemoteRepository))
self.assertTrue(repo1.verify(self.mock_repo.public_key))
self.assertEqual(self.mock_repo.repo_name,
repo1.manifest.repository_name)
repo2 = cvmfs.open_repository(self.mock_repo.dir,
self.mock_repo.public_key)
self.assertTrue(isinstance(repo2, cvmfs.LocalRepository))
self.assertTrue(repo2.verify(self.mock_repo.public_key))
self.assertEqual(self.mock_repo.repo_name,
repo2.manifest.repository_name)
repo3 = cvmfs.open_repository(self.mock_repo.url)
self.assertTrue(isinstance(repo3, cvmfs.RemoteRepository))
self.assertTrue(repo3.verify(self.mock_repo.public_key))
self.assertEqual(self.mock_repo.repo_name,
repo3.manifest.repository_name)
repo4 = cvmfs.open_repository(self.mock_repo.dir)
self.assertTrue(isinstance(repo4, cvmfs.LocalRepository))
self.assertTrue(repo4.verify(self.mock_repo.public_key))
self.assertEqual(self.mock_repo.repo_name,
repo4.manifest.repository_name)
def test_open_repository_http(self):
self.mock_repo.serve_via_http()
repo = cvmfs.open_repository(self.mock_repo.url)
self.assertTrue(isinstance(repo, cvmfs.RemoteRepository))
self.assertEqual(self.mock_repo.repo_name,
repo.manifest.repository_name)
self.assertEqual(self.mock_repo.url, repo.endpoint)
def test_read_custom_data(self):
repo = cvmfs.open_repository(self.mock_repo.dir)
self.assertTrue(isinstance(repo, cvmfs.Repository))
repoinfo = repo.retrieve_repoinfo()
self.assertTrue(isinstance(repoinfo, cvmfs.RepoInfo))
self.assertEqual("This is", repoinfo.json_data["custom"]["foo"])
self.assertEqual("arbitrary data", repoinfo.json_data["custom"]["bar"])
def test_download_non_existent_file(self):
self.mock_repo.make_valid_whitelist()
self.mock_repo.serve_via_http()
repo = cvmfs.open_repository(self.mock_repo.url,
self.mock_repo.public_key)
self.assertRaises(cvmfs.FileNotFoundInRepository, repo.retrieve_file,
"unobtainium.txt")
def test_read_basic_repoinfo(self):
repo = cvmfs.open_repository(self.mock_repo.dir)
self.assertTrue(isinstance(repo, cvmfs.Repository))
repoinfo = repo.retrieve_repoinfo()
self.assertTrue(isinstance(repoinfo, cvmfs.RepoInfo))
self.assertEqual("Rene Meusel", repoinfo.administrator)
self.assertEqual("*****@*****.**", repoinfo.email)
self.assertEqual("CERN", repoinfo.organisation)
self.assertEqual("This is a test repository", repoinfo.description)
self.assertEqual(3, len(repoinfo.stratum1s))
def test_catalog_lookup(self):
self.mock_repo.make_valid_whitelist()
self.mock_repo.serve_via_http()
repo = cvmfs.open_repository(self.mock_repo.url,
public_key=self.mock_repo.public_key)
rev = repo.get_current_revision()
for catalog in rev.catalogs():
if catalog.root_prefix == '/bar/4':
self.assertIsNone(
catalog.find_nested_for_path('/bar/4/foobar'))
self.assertIsNone(catalog.find_nested_for_path('/bar/4/foo'))
break
def test_open_repository_verification(self):
self.mock_repo.make_valid_whitelist()
self.mock_repo.serve_via_http()
repo1 = cvmfs.open_repository(self.mock_repo.url, self.mock_repo.public_key)
self.assertTrue(isinstance(repo1, cvmfs.RemoteRepository))
self.assertTrue(repo1.verify(self.mock_repo.public_key))
self.assertEqual(self.mock_repo.repo_name, repo1.manifest.repository_name)
repo2 = cvmfs.open_repository(self.mock_repo.dir, self.mock_repo.public_key)
self.assertTrue(isinstance(repo2, cvmfs.LocalRepository))
self.assertTrue(repo2.verify(self.mock_repo.public_key))
self.assertEqual(self.mock_repo.repo_name, repo2.manifest.repository_name)
repo3 = cvmfs.open_repository(self.mock_repo.url)
self.assertTrue(isinstance(repo3, cvmfs.RemoteRepository))
self.assertTrue(repo3.verify(self.mock_repo.public_key))
self.assertEqual(self.mock_repo.repo_name, repo3.manifest.repository_name)
repo4 = cvmfs.open_repository(self.mock_repo.dir)
self.assertTrue(isinstance(repo4, cvmfs.LocalRepository))
self.assertTrue(repo4.verify(self.mock_repo.public_key))
self.assertEqual(self.mock_repo.repo_name, repo4.manifest.repository_name)
def test_read_stratum1_list(self):
repo = cvmfs.open_repository(self.mock_repo.dir)
self.assertTrue(isinstance(repo, cvmfs.Repository))
repoinfo = repo.retrieve_repoinfo()
self.assertTrue(isinstance(repoinfo, cvmfs.RepoInfo))
self.assertEqual(3, len(repoinfo.stratum1s))
stratum1s = [
"http://cvmfs-stratum-one.cern.ch/cvmfs/test.cern.ch",
"http://cernvmfs.gridpp.rl.ac.uk/cvmfs/test.cern.ch",
"http://cvmfs.racf.bnl.gov/cvmfs/test.cern.ch"
]
for s1 in stratum1s:
self.assertTrue(s1 in repoinfo.stratum1s)
def test_revision(self):
self.mock_repo.make_valid_whitelist()
self.mock_repo.serve_via_http()
repo = cvmfs.open_repository(self.mock_repo.url,
public_key=self.mock_repo.public_key)
rev3 = repo.get_current_revision()
self.assertEqual(3, rev3.revision_number)
dirent = rev3.lookup('/bar/3')
self.assertIsNotNone(dirent)
self.assertTrue(dirent.is_directory())
rev1 = repo.get_revision(1)
self.assertEqual(1, rev1.revision_number)
dirent = rev1.lookup('/bar/3')
self.assertIsNone(dirent)
def test_lookup(self):
self.mock_repo.make_valid_whitelist()
self.mock_repo.serve_via_http()
repo = cvmfs.open_repository(self.mock_repo.url,
public_key=self.mock_repo.public_key)
rev = repo.get_current_revision()
dirent = rev.lookup('/.cvmfsdirtab')
self.assertIsNotNone(dirent)
dirent = rev.lookup('/bar/4/foo')
self.assertIsNotNone(dirent)
dirent = rev.lookup('/bar/4/foobar')
self.assertIsNone(dirent)
# with trailing slash this time
dirent1 = rev.lookup('/bar/4/foo/')
self.assertIsNotNone(dirent1)
dirent2 = rev.lookup('/bar/4/../4/foo/')
self.assertIsNotNone(dirent2)
self.assertEquals(dirent1.name, dirent2.name)
def test_list(self):
self.mock_repo.make_valid_whitelist()
self.mock_repo.serve_via_http()
repo = cvmfs.open_repository(self.mock_repo.url,
public_key=self.mock_repo.public_key)
rev = repo.get_current_revision()
dirents = rev.list_directory('/')
self.assertIsNotNone(dirents)
self.assertEqual(3, len(dirents))
dirents = rev.list_directory('/bar/3')
self.assertIsNotNone(dirents)
self.assertEqual(4, len(dirents))
self.assertEquals('.cvmfscatalog', dirents[0].name)
self.assertEquals('1', dirents[1].name)
self.assertEquals('2', dirents[2].name)
self.assertEquals('3', dirents[3].name)
dirents = rev.list_directory('/bar/4/foo')
self.assertIsNone(dirents)
dirents = rev.list_directory('/fakedir')
self.assertIsNone(dirents)
# with trailing slash this time
dirents = rev.list_directory('/bar/3/')
self.assertIsNotNone(dirents)
self.assertEqual(4, len(dirents))
def test_open_repository_local(self):
repo = cvmfs.open_repository(self.mock_repo.dir)
self.assertTrue(isinstance(repo, cvmfs.LocalRepository))
self.assertEqual(self.mock_repo.repo_name, repo.manifest.repository_name)
self.assertEqual(self.mock_repo.dir, repo.endpoint)
def test_open_repository_http(self):
self.mock_repo.serve_via_http()
repo = cvmfs.open_repository(self.mock_repo.url)
self.assertTrue(isinstance(repo, cvmfs.RemoteRepository))
self.assertEqual(self.mock_repo.repo_name, repo.manifest.repository_name)
self.assertEqual(self.mock_repo.url, repo.endpoint)
def get_hashes_for_catalog_tree(repo, root_catalog):
hashes = {root_catalog.hash + "C"}
for catalog in repo.catalogs(root_catalog):
hashes = hashes | get_hashes_for_catalog(catalog)
if not catalog.has_nested():
repo.close_catalog(catalog)
return hashes
def get_hashes_for_revision(repo, root_hash=None):
root_catalog = repo.retrieve_catalog(
root_hash) if root_hash else repo.retrieve_root_catalog()
return get_hashes_for_catalog_tree(repo, root_catalog)
# check input values
if len(sys.argv) != 2 and len(sys.argv) != 3:
usage()
sys.exit(1)
# get input parameters
repo_identifier = sys.argv[1]
root_catalog_hash = sys.argv[2] if len(sys.argv) == 3 else None
repo = cvmfs.open_repository(repo_identifier)
hashes = get_hashes_for_revision(repo, root_catalog_hash)
print '\n'.join(hashes)
def usage():
print sys.argv[0] + "<repo url/path> <download destination> [<history depth>]"
print "Downloads the whole catalog graph of a given repository."
print "The optional <history depth> puts a threshold on how many historic"
print "catalog tree revisions should be downloaded (default: all)"
if len(sys.argv) < 3 or len(sys.argv) > 4:
usage()
sys.exit(1)
main_folder = sys.argv[2]
dest = main_folder + "/data"
url = sys.argv[1]
repo = cvmfs.open_repository(url)
depth = sys.argv[3] if len(sys.argv) == 4 else 0
try:
depth = int(depth)
except ValueError, e:
usage()
print
print "<history depth> needs to be an integer"
sys.exit(1)
# download the .cvmfspublished file first
try:
urllib.URLopener().retrieve(url + "/.cvmfspublished", main_folder + "/.cvmfspublished")
except ValueError, e:
usage()
def test_has_repoinfo(self):
repo = cvmfs.open_repository(self.mock_repo.dir)
self.assertTrue(isinstance(repo, cvmfs.Repository))
self.assertEqual(self.mock_repo.repo_name, repo.manifest.repository_name)
self.assertTrue(repo.manifest.has_repoinfo())
def test_open_repository_local(self):
repo = cvmfs.open_repository(self.mock_repo.dir)
self.assertTrue(isinstance(repo, cvmfs.LocalRepository))
self.assertEqual(self.mock_repo.repo_name,
repo.manifest.repository_name)
self.assertEqual(self.mock_repo.dir, repo.endpoint)
def test_retrieve_repoinfo(self):
repo = cvmfs.open_repository(self.mock_repo.dir)
self.assertTrue(isinstance(repo, cvmfs.Repository))
repoinfo = repo.retrieve_repoinfo()
self.assertTrue(isinstance(repoinfo, cvmfs.RepoInfo))
self.assertEqual(repo.manifest.repository_name, repoinfo.fqrn)
def _push_catalog_wrapper(self, catalog):
if not catalog.catalog_reference or \
catalog.catalog_reference.hash not in self.visited_hashes:
cvmfs.CatalogTreeIterator._push_catalog_wrapper(self, catalog)
def usage():
print sys.argv[0] + "<repo url/path> <download destination>"
print "Downloads the whole catalog graph of a given repository"
if len(sys.argv) != 3:
usage()
sys.exit(1)
dest = sys.argv[2]
repo = cvmfs.open_repository(sys.argv[1])
print "Downloading catalog tree from " + repo.manifest.repository_name
root_clg = repo.retrieve_root_catalog()
visited_hashes = set()
while True:
next_root_clg = root_clg.get_predecessor()
for catalog in MerkleCatalogTreeIterator(repo, root_clg, visited_hashes):
if catalog.is_root():
print "Downloading revision" , catalog.revision , "..."
shutil.copyfile(catalog.get_compressed_file().name, dest + "/" + catalog.hash)
repo.close_catalog(catalog)
if next_root_clg != None:
def test_download_non_existent_file(self):
self.mock_repo.make_valid_whitelist()
self.mock_repo.serve_via_http()
repo = cvmfs.open_repository(self.mock_repo.url, self.mock_repo.public_key)
self.assertRaises(cvmfs.FileNotFoundInRepository, repo.retrieve_file, "unobtainium.txt")
#!/usr/bin/env python
import sys
import cvmfs
def usage():
print sys.argv[0] + " <local repo name | remote repo url>"
print "This script opens the root catalog of the provided CVMFS repository."
if len(sys.argv) != 2:
usage();
sys.exit(1)
repo_identifier = sys.argv[1]
repo = cvmfs.open_repository(repo_identifier)
root_catalog = repo.retrieve_root_catalog()
root_catalog.open_interactive()
def connect_to(self, fqrn):
return cvmfs.open_repository(self.make_endpoint(fqrn))
