def test_s3_prefixes(self):
# Requires these environment variables set
#
# AWS_ACCESS_KEY_ID
# AWS_SECRET_ACCESS_KEY
# S3TS_BUCKET
#
# NB: **this will only work if the bucket is empty
s3c = boto.connect_s3()
bucket = s3c.get_bucket(os.environ['S3TS_BUCKET'])
with EmptyS3Bucket(bucket):
localCache = LocalFileStore(
makeEmptyDir(os.path.join(self.workdir, 'cache')))
treestore1 = TreeStore.create(S3FileStore(bucket,
"prefix1"), localCache,
TreeStoreConfig(100, True))
treestore2 = TreeStore.create(S3FileStore(bucket,
"prefix2"), localCache,
TreeStoreConfig(100, True))
# Confirm we can write the different values to the same path in both treestores,
# and the different prefix keeps them separate independent
creationTime = datetimeFromIso('2015-01-01T00:00:00.0')
treestore1.upload('release', '', creationTime, self.srcTree,
CaptureUploadProgress())
treestore2.upload('release', '', creationTime, self.srcTree2,
CaptureUploadProgress())
pkg1 = treestore1.findPackage('release')
pkg2 = treestore2.findPackage('release')
self.assertEquals(len(pkg1.files), 3)
self.assertEquals(len(pkg2.files), 4)
def createTreeStore(chunksize):
localCacheDir = getEnv('S3TS_LOCALCACHE',
'the local directory used for caching')
bucket, s3PathPrefix = connectToBucket()
config = TreeStoreConfig(chunksize, True)
return TreeStore.create(S3FileStore(bucket, s3PathPrefix),
LocalFileStore(localCacheDir), config)
def test_s3_many_treestore(self):
# Create an s3 backed treestore
# Requires these environment variables set
#
# AWS_ACCESS_KEY_ID
# AWS_SECRET_ACCESS_KEY
# S3TS_BUCKET
#
# NB: **this will only work if the bucket is empty
s3c = boto.connect_s3()
bucket = s3c.get_bucket(os.environ['S3TS_BUCKET'])
with EmptyS3Bucket(bucket):
fileStore = S3FileStore(bucket)
localCache = LocalFileStore(
makeEmptyDir(os.path.join(self.workdir, 'cache')))
treestore = TreeStore.create(fileStore, localCache,
TreeStoreConfig(100, True))
# Upload it as a tree
creationTime = datetimeFromIso('2015-01-01T00:00:00.0')
treestore.uploadMany('v1.0', '', creationTime, self.srcTree,
self.srcVariant, CaptureUploadProgress())
pkg = treestore.findPackage('v1.0:kiosk-01')
# Confirm it's in the index
self.assertEquals(treestore.listPackages(),
['v1.0:kiosk-01', 'v1.0:kiosk-02'])
# Verify it
treestore.verify(pkg)
# Download it, checking we get expected progress callbacks
cb = CaptureDownloadProgress()
treestore.download(pkg, cb)
self.assertEquals(sorted(cb.recorded), [29, 30, 45, 47, 100, 100])
# Verify it locally
treestore.verifyLocal(pkg)
# Install it
destTree = os.path.join(self.workdir, 'dest-1')
treestore.install(pkg, destTree, CaptureInstallProgress())
# Check that the installed tree is the same as the source tree
self.assertEquals(
subprocess.call('diff -r -x {0} {1} {2}'.format(
S3TS_PROPERTIES, self.srcTree + '/assets',
destTree + '/assets'),
shell=True), 0)
self.assertEquals(
subprocess.call('diff -r -x {0} {1} {2}'.format(
S3TS_PROPERTIES, self.srcTree + '/code',
destTree + '/code'),
shell=True), 0)
self.assertEquals(
readInstallProperties(destTree).treeName, 'v1.0:kiosk-01')
def openTreeStore(dryRun=False, verbose=False):
localCacheDir = getEnv('S3TS_LOCALCACHE',
'the local directory used for caching')
bucket, s3PathPrefix = connectToBucket()
treeStore = TreeStore.open(S3FileStore(bucket, s3PathPrefix),
LocalFileStore(localCacheDir))
treeStore.setDryRun(dryRun)
if verbose:
treeStore.setOutVerbose(outVerbose)
return treeStore
def runtest(testContent):
print "----------------------------------------------------------------------"
print "* path", testContent
dir = tempfile.mkdtemp()
try:
store = LocalFileStore(os.path.join(dir, 'store'))
cache = LocalFileStore(os.path.join(dir, 'cache'))
config = TreeStoreConfig(1000000, True)
ts = TreeStore.create(store, cache, config)
creationTime = datetime.datetime.now()
print "* Initial upload"
stats = TransferStats()
ts.upload( "test", "", creationTime, testContent, stats.progress)
stats.done()
print
print "* Repeat upload"
stats = TransferStats()
ts.upload( "test", "", creationTime, testContent, stats.progress)
stats.done()
print
print "* Download"
stats = TransferStats()
pkg = ts.find("test")
ts.download(pkg, stats.progress)
stats.done()
print
print "* Clean installation"
installDir = os.path.join(dir,"install")
os.makedirs(installDir)
stats = InstallStats()
pkg = ts.find("test")
ts.install(pkg, installDir, stats.progress)
stats.done()
finally:
shutil.rmtree(dir)
def test_metapackages(self):
# Create a file system backed treestore
fileStore = LocalFileStore(
makeEmptyDir(os.path.join(self.workdir, 'fs')))
localCache = LocalFileStore(
makeEmptyDir(os.path.join(self.workdir, 'cache')))
treestore = TreeStore.create(fileStore, localCache,
TreeStoreConfig(10, True))
creationTime = datetimeFromIso('2015-01-01T00:00:00.0')
treestore.upload('v1.0', '', creationTime, self.srcTree,
CaptureUploadProgress())
treestore.upload('v1.3', '', creationTime, self.srcTree3,
CaptureUploadProgress())
treestore.upload('v1.4', '', creationTime, self.srcTree4,
CaptureUploadProgress())
meta1 = MetaPackage(name='meta1',
description='',
creationTime=creationTime,
components=[
SubPackage('dir-1', 'v1.0'),
SubPackage('dir-2', 'v1.3'),
])
meta1.verify(treestore, {})
treestore.uploadMetaPackage(meta1)
meta1p = treestore.find('meta1', {})
treestore.download(meta1p, CaptureDownloadProgress())
# Install it
destTree = os.path.join(self.workdir, 'dest-1')
treestore.install(meta1p, destTree, CaptureInstallProgress())
def assertContains(path, text):
self.assertEquals(open(os.path.join(destTree, path)).read(), text)
assertContains("dir-1/code/file1.py", self.FILE1)
assertContains("dir-2/text/text", self.FILE5)
def test_s3_merged_package(self):
# Test the creation and subsequent installation of merged packages
# Requires these environment variables set
#
# AWS_ACCESS_KEY_ID
# AWS_SECRET_ACCESS_KEY
# S3TS_BUCKET
#
# NB: **this will only work if the bucket is empty
s3c = boto.connect_s3()
bucket = s3c.get_bucket(os.environ['S3TS_BUCKET'])
with EmptyS3Bucket(bucket):
localCache = LocalFileStore(
makeEmptyDir(os.path.join(self.workdir, 'cache')))
treestore = TreeStore.create(S3FileStore(bucket), localCache,
TreeStoreConfig(100, True))
creationTime = datetimeFromIso('2015-01-01T00:00:00.0')
treestore.upload('src1', '', creationTime, self.srcTree,
CaptureUploadProgress())
treestore.upload('src2', '', creationTime, self.srcTree2,
CaptureUploadProgress())
treestore.upload('src3', '', creationTime, self.srcTree3,
CaptureUploadProgress())
treestore.createMerged('merged', creationTime, {
'.': 'src1',
'subdir-a': 'src2',
'subdir-b': 'src3'
})
pkg = treestore.findPackage('merged')
treestore.download(pkg, CaptureDownloadProgress())
destTree = os.path.join(self.workdir, 'merged')
treestore.install(pkg, destTree, CaptureInstallProgress())
def assertSameContent(path1, path2):
with open(path1) as f1:
with open(path2) as f2:
self.assertEquals(f1.read(), f2.read())
assertSameContent(os.path.join(destTree, "code/file1.py"),
os.path.join(self.srcTree, "code/file1.py"))
assertSameContent(os.path.join(destTree, "subdir-a/code/file4.py"),
os.path.join(self.srcTree2, "code/file4.py"))
assertSameContent(os.path.join(destTree, "subdir-b/text/text"),
os.path.join(self.srcTree3, "text/text"))
def setUp(self):
self.workdir = tempfile.mkdtemp()
if os.path.exists(self.workdir):
shutil.rmtree(self.workdir)
os.makedirs(self.workdir)
self.FILE1 = '#!/bin/env python\n def main(): print "hello"\n'
self.FILE2 = '#!/bin/env python\n def main(): print "goodbye"\n'
self.FILE2_A = '#!/bin/env python\n def main(): print "goodbye foreever"\n'
self.FILE3 = '#!/bin/env python\n def main(): print "goodbye foreever"\n'
self.FILE4 = '#!/bin/env python\n def main(): print "what now"\n'
self.FILE5 = 'Just text'
self.CAR01 = (
'Some big and complicated data structure goes here, hopefully big enough that it requires chunking and compression.\n'
'sydney london paris port moresby okinawa st petersburg salt lake city new york whitehorse mawson woy woy st louis\n'
)
# Create some test input data
self.srcTree = makeEmptyDir(os.path.join(self.workdir, 'src-1'))
fs = LocalFileStore(self.srcTree)
fs.put('code/file1.py', self.FILE1)
fs.put('code/file2.py', self.FILE2)
fs.put('assets/car-01.db', self.CAR01)
self.srcTree2 = makeEmptyDir(os.path.join(self.workdir, 'src-2'))
fs = LocalFileStore(self.srcTree2)
fs.put('code/file1.py', self.FILE1)
fs.put('code/file3.py', self.FILE3)
fs.put('code/file4.py', self.FILE4)
fs.put('assets/car-01.db', self.CAR01)
self.srcTree3 = makeEmptyDir(os.path.join(self.workdir, 'src-3'))
fs = LocalFileStore(self.srcTree3)
fs.put('code/file1.py', self.FILE1)
fs.put('code/file2.py', self.FILE2_A)
fs.put('code/file4.py', self.FILE4)
fs.put('text/text', self.FILE5)
self.srcTree4 = makeEmptyDir(os.path.join(self.workdir, 'src-4'))
fs = LocalFileStore(self.srcTree4)
fs.put('file1.py', self.FILE1)
fs.put('code/file2.py', self.FILE2_A)
fs.put('code/file4.py', self.FILE4)
fs.put('text', self.FILE5)
self.srcVariant = makeEmptyDir(os.path.join(self.workdir,
'src1-kiosk'))
fs = LocalFileStore(self.srcVariant)
fs.put('kiosk-01/key', 'this is the key src1:kiosk-01')
fs.put('kiosk-02/key', 'this is the key src1:kiosk-02')
def test_s3_treestore(self):
# Create an s3 backed treestore
# Requires these environment variables set
#
# AWS_ACCESS_KEY_ID
# AWS_SECRET_ACCESS_KEY
# S3TS_BUCKET
#
# NB: **this will only work if the bucket is empty
s3c = boto.connect_s3()
bucket = s3c.get_bucket(os.environ['S3TS_BUCKET'])
with EmptyS3Bucket(bucket):
fileStore = S3FileStore(bucket)
localCache = LocalFileStore(
makeEmptyDir(os.path.join(self.workdir, 'cache')))
treestore = TreeStore.create(fileStore, localCache,
TreeStoreConfig(100, True))
# Upload it as a tree
creationTime = datetimeFromIso('2015-01-01T00:00:00.0')
treestore.upload('v1.0', '', creationTime, self.srcTree,
CaptureUploadProgress())
pkg = treestore.findPackage('v1.0')
# Confirm it's in the index
self.assertEquals(treestore.listPackages(), ['v1.0'])
# Verify it
treestore.verify(pkg)
# Download it, checking we get expected progress callbacks
cb = CaptureDownloadProgress()
treestore.download(pkg, cb)
self.assertEquals(sorted(cb.recorded), [30, 45, 47, 100, 100])
# Verify it locally
treestore.verifyLocal(pkg)
# Install it
destTree = os.path.join(self.workdir, 'dest-1')
treestore.install(pkg, destTree, CaptureInstallProgress())
# Check that the installed tree is the same as the source tree
self.assertEquals(
subprocess.call('diff -r -x {0} {1} {2}'.format(
S3TS_PROPERTIES, self.srcTree, destTree),
shell=True), 0)
self.assertEquals(readInstallProperties(destTree).treeName, 'v1.0')
# Use the compareInstall function to confirm the installed package is ok, and
# then check that modifying the files show up in the comparison
result = treestore.compareInstall(pkg, destTree)
self.assertEquals(len(result.missing), 0)
self.assertEquals(len(result.extra), 0)
self.assertEquals(len(result.diffs), 0)
with open(os.path.join(destTree, "code/file1.py"), "w") as f:
f.write("x")
with open(os.path.join(destTree, "code/file3.py"), "w") as f:
f.write("y")
os.unlink(os.path.join(destTree, 'assets/car-01.db'))
result = treestore.compareInstall(pkg, destTree)
self.assertEquals(result.missing, set(['assets/car-01.db']))
self.assertEquals(result.extra, set(['code/file3.py']))
self.assertEquals(result.diffs, set(['code/file1.py']))
# Reinstall to fix directory content
shutil.rmtree(destTree)
treestore.install(pkg, destTree, CaptureInstallProgress())
result = treestore.compareInstall(pkg, destTree)
self.assertEquals(len(result.missing), 0)
self.assertEquals(len(result.extra), 0)
self.assertEquals(len(result.diffs), 0)
# Now create a pre-signed version of the package
pkg = treestore.findPackage('v1.0')
treestore.addUrls(pkg, 3600)
self.assertEquals(len(result.missing), 0)
self.assertEquals(len(result.extra), 0)
self.assertEquals(len(result.diffs), 0)
# And download it directly via http. Create a new local cache
# to ensure that we actually redownload each chunk
localCache = LocalFileStore(
makeEmptyDir(os.path.join(self.workdir, 'cache')))
treestore2 = TreeStore.forHttpOnly(localCache)
cb = CaptureDownloadProgress()
treestore2.downloadHttp(pkg, cb)
self.assertEquals(sorted(cb.recorded), [30, 45, 47, 100, 100])
# Install it
destTree2 = os.path.join(self.workdir, 'dest-2')
treestore2.install(pkg, destTree2, CaptureInstallProgress())
# Check that the new installed tree is the same as the source tree
self.assertEquals(
subprocess.call('diff -r -x {0} {1} {2}'.format(
S3TS_PROPERTIES, self.srcTree, destTree2),
shell=True), 0)
# Rename the tree, and check that installing that is the same
treestore.rename('v1.0', 'v1.0x')
pkg = treestore.findPackage('v1.0x')
treestore.download(pkg, CaptureDownloadProgress())
destTree = os.path.join(self.workdir, 'dest-3')
treestore.install(pkg, destTree, CaptureInstallProgress())
self.assertEquals(
subprocess.call('diff -r -x {0} {1} {2}'.format(
S3TS_PROPERTIES, self.srcTree, destTree),
shell=True), 0)
# Remove the tree
treestore.remove('v1.0x')
def test_sync(self):
# Create a file system backed treestore
fileStore = LocalFileStore(
makeEmptyDir(os.path.join(self.workdir, 'fs')))
localCache = LocalFileStore(
makeEmptyDir(os.path.join(self.workdir, 'cache')))
treestore = TreeStore.create(fileStore, localCache,
TreeStoreConfig(10, True))
creationTime = datetimeFromIso('2015-01-01T00:00:00.0')
treestore.upload('v1.0', '', creationTime, self.srcTree,
CaptureUploadProgress())
treestore.upload('v1.3', '', creationTime, self.srcTree3,
CaptureUploadProgress())
treestore.upload('v1.4', '', creationTime, self.srcTree4,
CaptureUploadProgress())
testdir = makeEmptyDir(os.path.join(self.workdir, 'test'))
def assertExists(path):
self.assertTrue(os.path.exists(os.path.join(testdir, path)))
def assertContains(path, text):
self.assertEquals(open(os.path.join(testdir, path)).read(), text)
def assertDoesntExist(path):
self.assertFalse(os.path.exists(os.path.join(testdir, path)))
def assertInstalled(pkg, testdir):
result = treestore.compareInstall(pkg, testdir)
self.assertEquals(result.missing, set())
self.assertEquals(result.extra, set())
self.assertEquals(result.diffs, set())
# sync a package to an empty directory
pkg = treestore.findPackage('v1.0')
treestore.download(pkg, CaptureDownloadProgress())
treestore.sync(pkg, testdir, CaptureInstallProgress())
assertContains("code/file1.py", self.FILE1)
assertContains("code/file2.py", self.FILE2)
assertContains("assets/car-01.db", self.CAR01)
assertExists(S3TS_PACKAGEFILE)
assertInstalled(pkg, testdir)
# Re-sync the same package
pkg = treestore.findPackage('v1.0')
treestore.download(pkg, CaptureDownloadProgress())
treestore.sync(pkg, testdir, CaptureInstallProgress())
assertContains("code/file1.py", self.FILE1)
assertContains("code/file2.py", self.FILE2)
assertContains("assets/car-01.db", self.CAR01)
assertExists(S3TS_PACKAGEFILE)
assertInstalled(pkg, testdir)
# Sync to a different package
pkg = treestore.findPackage('v1.3')
treestore.download(pkg, CaptureDownloadProgress())
treestore.sync(pkg, testdir, CaptureInstallProgress())
assertContains("code/file1.py", self.FILE1)
assertContains("code/file2.py", self.FILE2_A)
assertDoesntExist("assets/car-01.db")
assertContains("code/file4.py", self.FILE4)
assertContains("text/text", self.FILE5)
assertExists(S3TS_PACKAGEFILE)
assertInstalled(pkg, testdir)
# Sync back to the first package
pkg = treestore.findPackage('v1.0')
treestore.download(pkg, CaptureDownloadProgress())
treestore.sync(pkg, testdir, CaptureInstallProgress())
assertContains("code/file1.py", self.FILE1)
assertContains("code/file2.py", self.FILE2)
assertContains("assets/car-01.db", self.CAR01)
assertDoesntExist("code/file4.py")
assertExists(S3TS_PACKAGEFILE)
assertInstalled(pkg, testdir)
# Remove the package file, and sync the second package again
os.unlink(os.path.join(testdir, S3TS_PACKAGEFILE))
pkg = treestore.findPackage('v1.3')
treestore.download(pkg, CaptureDownloadProgress())
treestore.sync(pkg, testdir, CaptureInstallProgress())
assertContains("code/file1.py", self.FILE1)
assertContains("code/file2.py", self.FILE2_A)
assertDoesntExist("assets/car-01.db")
assertContains("code/file4.py", self.FILE4)
assertExists(S3TS_PACKAGEFILE)
assertInstalled(pkg, testdir)
# Add an extra file not in the package, and ensure
# that syncing deletes it
with open(os.path.join(testdir, "debug.log"), 'w') as f:
f.write("something")
pkg = treestore.findPackage('v1.3')
treestore.sync(pkg, testdir, CaptureInstallProgress())
assertInstalled(pkg, testdir)
# Sync to test replacing a directory with a file
pkg = treestore.findPackage('v1.4')
treestore.download(pkg, CaptureDownloadProgress())
treestore.sync(pkg, testdir, CaptureInstallProgress())
assertContains("text", self.FILE5)
assertInstalled(pkg, testdir)
def test_fs_treestore(self):
# Create a file system backed treestore
fileStore = LocalFileStore(
makeEmptyDir(os.path.join(self.workdir, 'fs')))
localCache = LocalFileStore(
makeEmptyDir(os.path.join(self.workdir, 'cache')))
treestore = TreeStore.create(fileStore, localCache,
TreeStoreConfig(100, True))
# Upload 2 trees
creationTime = datetimeFromIso('2015-01-01T00:00:00.0')
treestore.upload('v1.0', '', creationTime, self.srcTree,
CaptureUploadProgress())
pkg = treestore.findPackage('v1.0')
# Confirm it's in the index
self.assertEquals(treestore.listPackages(), ['v1.0'])
# Verify it
treestore.verify(pkg)
# Test the cache priming function
treestore.prime(self.srcTree2, CaptureUploadProgress())
# Test whether the verifyCache works
corruptedFiles = treestore.validateLocalCache()
self.assertEquals(len(corruptedFiles), 0)
# Download it, checking we get expected progress callbacks
# The order of the callbacks will depend on the order of the
# chunks in the package definition, which will depend on the
# iteration order of the file system when the package was created.
# So check independently of ordering.
cb = CaptureDownloadProgress()
treestore.download(pkg, cb)
self.assertEquals(sorted(cb.recorded), [30, 45, 47, 100, 100])
# Verify it locally
treestore.verifyLocal(pkg)
# Install it
destTree = os.path.join(self.workdir, 'dest-1')
treestore.install(pkg, destTree, CaptureInstallProgress())
# Check that the installed tree is the same as the source tree
self.assertEquals(
subprocess.call('diff -r -x {0} {1} {2}'.format(
S3TS_PROPERTIES, self.srcTree, destTree),
shell=True), 0)
# Rename the tree, and check that installing that is the same
treestore.rename('v1.0', 'v1.0x')
pkg = treestore.findPackage('v1.0x')
treestore.download(pkg, CaptureDownloadProgress())
destTree = os.path.join(self.workdir, 'dest-2')
treestore.install(pkg, destTree, CaptureInstallProgress())
self.assertEquals(
subprocess.call('diff -r -x {0} {1} {2}'.format(
S3TS_PROPERTIES, self.srcTree, destTree),
shell=True), 0)
# Test the flushStore function has nothing to remove)
treestore.upload('extra', '', creationTime, self.srcTree2,
CaptureUploadProgress())
removed = treestore.flushStore()
self.assertEquals(len(removed), 0)
# Remove a tree
treestore.remove('v1.0x')
# Test the store now has dangling chunks when can be removed
removed = treestore.flushStore()
self.assertTrue(len(removed) > 0)
treestore.upload('v1.0', '', creationTime, self.srcTree,
CaptureUploadProgress())
# Initially the local cache should contain chunks for v1.0 and extra. Empty
# the local cache by successive flush operations
removed = treestore.flushLocalCache(['extra'])
self.assertTrue(len(removed) > 0)
removed = treestore.flushLocalCache(['v1.0'])
self.assertTrue(len(removed) > 0)
# Confirm that removing everything from the local cache is refused
with self.assertRaises(RuntimeError):
treestore.flushLocalCache([])
def nonS3TreeStore():
# Don't use or require S3 - some operations won't be available
localCacheDir = getEnv('S3TS_LOCALCACHE',
'the local directory used for caching')
return TreeStore(FileStore(), LocalFileStore(localCacheDir), None)