def put(self, source, rel_path, metadata=None): from ambry.util import bundle_file_type import gzip # Pass through if the file is already compressed if not metadata: metadata = {} metadata['Content-Encoding'] = 'gzip' bft = bundle_file_type(source) sink = self.upstream.put_stream(self._rename(rel_path), metadata = metadata) if bft == 'gzip': copy_file_or_flo(source, sink) else: copy_file_or_flo(source, gzip.GzipFile(fileobj=sink, mode='wb')) sink.close() #self.put_metadata(rel_path, metadata) return self.path(self._rename(rel_path))
def get_stream(self, rel_path, cb=None): from ..util import bundle_file_type from ..util.flo import MetadataFlo import gzip source = self.upstream.get_stream(self._rename(rel_path)) if not source: return None if bundle_file_type(source) == 'gzip': logger.debug("CC returning {} with decompression".format(rel_path)) return MetadataFlo(gzip.GzipFile(fileobj=source), source.meta) else: logger.debug("CC returning {} with passthrough".format(rel_path)) return source
def x_test_put_redirect(self): from ambry.bundle import DbBundle from ambry.library.query import QueryCommand from ambry.util import md5_for_file, rm_rf, bundle_file_type # # Simple out and retrieve # cache = self.bundle.filesystem._get_cache(self.server_rc.filesystem, 'direct-remote') cache2 = self.bundle.filesystem._get_cache(self.server_rc.filesystem, 'direct-remote-2') rm_rf(os.path.dirname(cache.cache_dir)) rm_rf(os.path.dirname(cache2.cache_dir)) cache.put( self.bundle.database.path, 'direct') path = cache2.get('direct') self.assertEquals('sqlite',bundle_file_type(path)) cache.remove('direct', propagate = True) # # Connect through server. # rm_rf('/tmp/server') self.start_server(name='default-remote') api = None # Rest(self.server_url, self.rc.accounts) # Upload directly, then download via the cache. cache.remove(self.bundle.identity.cache_key, propagate = True) r = api.upload_file(self.bundle.identity, self.bundle.database.path, force=True ) path = cache.get(self.bundle.identity.cache_key) b = DbBundle(path) self.assertEquals("source-dataset-subset-variation-ca0d",b.identity.name ) # # Full service # p = self.bundle.partitions.all[0] cache.remove(self.bundle.identity.cache_key, propagate = True) cache.remove(p.identity.cache_key, propagate = True) r = api.put( self.bundle.database.path, self.bundle.identity ) print "Put {}".format(r.object) r = api.put(p.database.path, p.identity ) print "Put {}".format(r.object) r = api.put(p.database.path, p.identity ) r = api.get(p.identity,'/tmp/foo.db') print "Get {}".format(r) b = DbBundle(r) self.assertEquals("source-dataset-subset-variation-ca0d",b.identity.name )