def put(self, source, rel_path, metadata=None):
from ambry.util import bundle_file_type
import gzip
# Pass through if the file is already compressed
if not metadata:
metadata = {}
metadata['Content-Encoding'] = 'gzip'
bft = bundle_file_type(source)
sink = self.upstream.put_stream(self._rename(rel_path), metadata = metadata)
if bft == 'gzip':
copy_file_or_flo(source, sink)
else:
copy_file_or_flo(source, gzip.GzipFile(fileobj=sink, mode='wb'))
sink.close()
#self.put_metadata(rel_path, metadata)
return self.path(self._rename(rel_path))
def get_stream(self, rel_path, cb=None):
from ..util import bundle_file_type
from ..util.flo import MetadataFlo
import gzip
source = self.upstream.get_stream(self._rename(rel_path))
if not source:
return None
if bundle_file_type(source) == 'gzip':
logger.debug("CC returning {} with decompression".format(rel_path))
return MetadataFlo(gzip.GzipFile(fileobj=source), source.meta)
else:
logger.debug("CC returning {} with passthrough".format(rel_path))
return source
def x_test_put_redirect(self):
from ambry.bundle import DbBundle
from ambry.library.query import QueryCommand
from ambry.util import md5_for_file, rm_rf, bundle_file_type
#
# Simple out and retrieve
#
cache = self.bundle.filesystem._get_cache(self.server_rc.filesystem, 'direct-remote')
cache2 = self.bundle.filesystem._get_cache(self.server_rc.filesystem, 'direct-remote-2')
rm_rf(os.path.dirname(cache.cache_dir))
rm_rf(os.path.dirname(cache2.cache_dir))
cache.put( self.bundle.database.path, 'direct')
path = cache2.get('direct')
self.assertEquals('sqlite',bundle_file_type(path))
cache.remove('direct', propagate = True)
#
# Connect through server.
#
rm_rf('/tmp/server')
self.start_server(name='default-remote')
api = None # Rest(self.server_url, self.rc.accounts)
# Upload directly, then download via the cache.
cache.remove(self.bundle.identity.cache_key, propagate = True)
r = api.upload_file(self.bundle.identity, self.bundle.database.path, force=True )
path = cache.get(self.bundle.identity.cache_key)
b = DbBundle(path)
self.assertEquals("source-dataset-subset-variation-ca0d",b.identity.name )
#
# Full service
#
p = self.bundle.partitions.all[0]
cache.remove(self.bundle.identity.cache_key, propagate = True)
cache.remove(p.identity.cache_key, propagate = True)
r = api.put( self.bundle.database.path, self.bundle.identity )
print "Put {}".format(r.object)
r = api.put(p.database.path, p.identity )
print "Put {}".format(r.object)
r = api.put(p.database.path, p.identity )
r = api.get(p.identity,'/tmp/foo.db')
print "Get {}".format(r)
b = DbBundle(r)
self.assertEquals("source-dataset-subset-variation-ca0d",b.identity.name )
