def test_compression(self): from databundles.run import get_runconfig from databundles.cache import new_cache from databundles.util import temp_file_name, md5_for_file, copy_file_or_flo rc = get_runconfig((os.path.join(self.bundle_dir,'test-run-config.yaml'),RunConfig.USER_CONFIG)) comp_cache = new_cache(rc.filesystem('compressioncache')) test_file_name = 'test_file' fn = temp_file_name() print 'orig file ', fn with open(fn,'wb') as f: for i in range(1000): f.write("{:03d}:".format(i)) cf = comp_cache.put(fn, test_file_name) with open(cf) as stream: from databundles.util.sgzip import GzipFile stream = GzipFile(stream) uncomp_cache = new_cache(rc.filesystem('fscache')) uncomp_stream = uncomp_cache.put_stream('decomp') copy_file_or_flo(stream, uncomp_stream) uncomp_stream.close() dcf = uncomp_cache.get('decomp') self.assertEquals(md5_for_file(fn), md5_for_file(dcf))
def test_caches(self): '''Basic test of put(), get() and has() for all cache types''' from functools import partial from databundles.run import get_runconfig, RunConfig from databundles.filesystem import Filesystem from databundles.cache import new_cache from databundles.util import md5_for_file from databundles.bundle import DbBundle self.start_server() # For the rest-cache #fn = '/tmp/1mbfile' #with open(fn, 'wb') as f: # f.write('.'*(1024)) fn = self.bundle.database.path # Opening the file might run the database updates in # database.sqlite._on_connect_update_schema, which can affect the md5. b = DbBundle(fn) md5 = md5_for_file(fn) print "MD5 {} = {}".format(fn, md5) rc = get_runconfig((os.path.join(self.bundle_dir,'test-run-config.yaml'),RunConfig.USER_CONFIG)) for i, fsname in enumerate(['fscache', 'limitedcache', 'compressioncache','cached-s3', 'cached-compressed-s3', 'rest-cache']): #'compressioncache', config = rc.filesystem(fsname) cache = new_cache(config) print '---', fsname, cache identity = self.bundle.identity relpath = identity.cache_key r = cache.put(fn, relpath,identity.to_meta(md5=md5)) r = cache.get(relpath) if not r.startswith('http'): self.assertTrue(os.path.exists(r), str(cache)) self.assertTrue(cache.has(relpath, md5=md5)) cache.remove(relpath, propagate=True) self.assertFalse(os.path.exists(r), str(cache)) self.assertFalse(cache.has(relpath)) cache = new_cache(rc.filesystem('s3cache-noupstream')) r = cache.put(fn, 'a')
def test_basic(self): from databundles.cache import new_cache c = new_cache(self.rc.filesystem('google')) print c print c.list()
def test_s3(self): #databundles.util.get_logger('databundles.filesystem').setLevel(logging.DEBUG) # Set up the test directory and make some test files. from databundles.cache import new_cache root = self.rc.group('filesystem').root_dir os.makedirs(root) testfile = os.path.join(root,'testfile') with open(testfile,'w+') as f: for i in range(1024): f.write('.'*1023) f.write('\n') #fs = self.bundle.filesystem #local = fs.get_cache('downloads') cache = new_cache(self.rc.filesystem('s3')) repo_dir = cache.cache_dir print "Repo Dir: {}".format(repo_dir) for i in range(0,10): logger.info("Putting "+str(i)) cache.put(testfile,'many'+str(i)) self.assertFalse(os.path.exists(os.path.join(repo_dir, 'many1'))) self.assertFalse(os.path.exists(os.path.join(repo_dir, 'many2'))) self.assertFalse(os.path.exists(os.path.join(repo_dir, 'many3'))) p = cache.get('many1') self.assertTrue(p is not None) self.assertTrue(os.path.exists(os.path.join(repo_dir, 'many1'))) self.assertFalse(os.path.exists(os.path.join(repo_dir, 'many2'))) self.assertFalse(os.path.exists(os.path.join(repo_dir, 'many3'))) p = cache.get('many2') self.assertTrue(p is not None) self.assertFalse(os.path.exists(os.path.join(repo_dir, 'many3'))) self.assertTrue(os.path.exists(os.path.join(repo_dir, 'many7'))) p = cache.get('many3') self.assertTrue(p is not None) self.assertTrue(os.path.exists(os.path.join(repo_dir, 'many3'))) self.assertFalse(os.path.exists(os.path.join(repo_dir, 'many7')))
def get_cache(fsname): rc = get_runconfig((os.path.join(self.bundle_dir,'test-run-config.yaml'),RunConfig.USER_CONFIG)) config = rc.filesystem(fsname) cache = new_cache(config) return cache