def test_compression(self):
from databundles.run import get_runconfig
from databundles.cache import new_cache
from databundles.util import temp_file_name, md5_for_file, copy_file_or_flo
rc = get_runconfig((os.path.join(self.bundle_dir,'test-run-config.yaml'),RunConfig.USER_CONFIG))
comp_cache = new_cache(rc.filesystem('compressioncache'))
test_file_name = 'test_file'
fn = temp_file_name()
print 'orig file ', fn
with open(fn,'wb') as f:
for i in range(1000):
f.write("{:03d}:".format(i))
cf = comp_cache.put(fn, test_file_name)
with open(cf) as stream:
from databundles.util.sgzip import GzipFile
stream = GzipFile(stream)
uncomp_cache = new_cache(rc.filesystem('fscache'))
uncomp_stream = uncomp_cache.put_stream('decomp')
copy_file_or_flo(stream, uncomp_stream)
uncomp_stream.close()
dcf = uncomp_cache.get('decomp')
self.assertEquals(md5_for_file(fn), md5_for_file(dcf))
def test_caches(self):
'''Basic test of put(), get() and has() for all cache types'''
from functools import partial
from databundles.run import get_runconfig, RunConfig
from databundles.filesystem import Filesystem
from databundles.cache import new_cache
from databundles.util import md5_for_file
from databundles.bundle import DbBundle
self.start_server() # For the rest-cache
#fn = '/tmp/1mbfile'
#with open(fn, 'wb') as f:
# f.write('.'*(1024))
fn = self.bundle.database.path
# Opening the file might run the database updates in
# database.sqlite._on_connect_update_schema, which can affect the md5.
b = DbBundle(fn)
md5 = md5_for_file(fn)
print "MD5 {} = {}".format(fn, md5)
rc = get_runconfig((os.path.join(self.bundle_dir,'test-run-config.yaml'),RunConfig.USER_CONFIG))
for i, fsname in enumerate(['fscache', 'limitedcache', 'compressioncache','cached-s3', 'cached-compressed-s3', 'rest-cache']): #'compressioncache',
config = rc.filesystem(fsname)
cache = new_cache(config)
print '---', fsname, cache
identity = self.bundle.identity
relpath = identity.cache_key
r = cache.put(fn, relpath,identity.to_meta(md5=md5))
r = cache.get(relpath)
if not r.startswith('http'):
self.assertTrue(os.path.exists(r), str(cache))
self.assertTrue(cache.has(relpath, md5=md5))
cache.remove(relpath, propagate=True)
self.assertFalse(os.path.exists(r), str(cache))
self.assertFalse(cache.has(relpath))
cache = new_cache(rc.filesystem('s3cache-noupstream'))
r = cache.put(fn, 'a')
def test_basic(self):
from databundles.cache import new_cache
c = new_cache(self.rc.filesystem('google'))
print c
print c.list()
def test_s3(self):
#databundles.util.get_logger('databundles.filesystem').setLevel(logging.DEBUG)
# Set up the test directory and make some test files.
from databundles.cache import new_cache
root = self.rc.group('filesystem').root_dir
os.makedirs(root)
testfile = os.path.join(root,'testfile')
with open(testfile,'w+') as f:
for i in range(1024):
f.write('.'*1023)
f.write('\n')
#fs = self.bundle.filesystem
#local = fs.get_cache('downloads')
cache = new_cache(self.rc.filesystem('s3'))
repo_dir = cache.cache_dir
print "Repo Dir: {}".format(repo_dir)
for i in range(0,10):
logger.info("Putting "+str(i))
cache.put(testfile,'many'+str(i))
self.assertFalse(os.path.exists(os.path.join(repo_dir, 'many1')))
self.assertFalse(os.path.exists(os.path.join(repo_dir, 'many2')))
self.assertFalse(os.path.exists(os.path.join(repo_dir, 'many3')))
p = cache.get('many1')
self.assertTrue(p is not None)
self.assertTrue(os.path.exists(os.path.join(repo_dir, 'many1')))
self.assertFalse(os.path.exists(os.path.join(repo_dir, 'many2')))
self.assertFalse(os.path.exists(os.path.join(repo_dir, 'many3')))
p = cache.get('many2')
self.assertTrue(p is not None)
self.assertFalse(os.path.exists(os.path.join(repo_dir, 'many3')))
self.assertTrue(os.path.exists(os.path.join(repo_dir, 'many7')))
p = cache.get('many3')
self.assertTrue(p is not None)
self.assertTrue(os.path.exists(os.path.join(repo_dir, 'many3')))
self.assertFalse(os.path.exists(os.path.join(repo_dir, 'many7')))
def get_cache(fsname):
rc = get_runconfig((os.path.join(self.bundle_dir,'test-run-config.yaml'),RunConfig.USER_CONFIG))
config = rc.filesystem(fsname)
cache = new_cache(config)
return cache
