def add_stream(self, name, stream):
key, version = self._parse_name(name)
existing_verion = self.versions[key]
if version is not None and existing_verion > version:
return versioned_name(key, existing_verion)
if version is None:
version = max(self.versions[key] + 1, int(time.time()))
data = b''
while True:
record = stream.read()
if not record:
break
data += record
self.data[key] = data
self.versions[key] = version
return versioned_name(key, version)
def add_stream(self, name, stream):
key, version = self._parse_name(name)
existing_verion = self.versions[key]
if version is not None and existing_verion > version:
return versioned_name(key, existing_verion)
if version is None:
version = max(self.versions[key] + 1, int(time.time()))
data = b''
while True:
record = stream.read()
if not record:
break
data += record
self.data[key] = data
self.versions[key] = version
return versioned_name(key, version)
def get_stream(self, name):
url, version = self._parse_name(name)
response = requests.get(url, stream=True)
response.raise_for_status()
remote_version = self._parse_last_modified(response)
if version is not None and remote_version is not None \
and version != remote_version:
raise FiletrackerError("Version %s not available. Server has %s" \
% (name, remote_version))
name, version = split_name(name)
stream = _FileLikeFromResponse(response)
return stream, versioned_name(name, remote_version)
def get_stream(self, name):
url, version = self._parse_name(name)
response = requests.get(url, stream=True)
response.raise_for_status()
remote_version = self._parse_last_modified(response)
if version is not None and remote_version is not None \
and version != remote_version:
raise FiletrackerError("Version %s not available. Server has %s" \
% (name, remote_version))
name, version = split_name(name)
stream = _FileLikeFromResponse(response)
return stream, versioned_name(name, remote_version)
def list_files(self):
result = []
for root, _dirs, files in os.walk(self.dir):
for basename in files:
relative_dir = os.path.relpath(root, self.dir)
store_dir = os.path.normpath(os.path.join('/', relative_dir))
name = os.path.join(store_dir, basename)
path, version = self._parse_name(name)
file_stat = os.lstat(path)
vname = versioned_name(name, self.file_version(name))
result.append(
DataStore.FileInfoEntry(name=vname,
mtime=file_stat.st_mtime,
size=file_stat.st_size))
return result
def list_files(self):
result = []
for root, _dirs, files in os.walk(self.dir):
for basename in files:
relative_dir = os.path.relpath(root, self.dir)
store_dir = os.path.normpath(
os.path.join('/', relative_dir))
name = os.path.join(store_dir, basename)
path, version = self._parse_name(name)
file_stat = os.lstat(path)
vname = versioned_name(name, self.file_version(name))
result.append(
DataStore.FileInfoEntry(name=vname,
mtime=file_stat.st_mtime,
size=file_stat.st_size))
return result
def get_stream(self, name, force_refresh=False, serve_from_cache=False):
"""Retrieves file identified by ``name`` in streaming mode.
Works like :meth:`get_file`, except that returns a tuple
(file-like object, versioned name).
When both remote_store and local_store are present, serve_from_cache
can be used to ensure that the file will be downloaded and served
from a local cache. If a full version is specified and the file
exists in the cache a file will be always served locally.
"""
uname, version = split_name(name)
lock = None
if self.local_store:
lock = self.lock_manager.lock_for(uname)
lock.lock_shared()
try:
if not self.remote_store or (version is not None
and not force_refresh):
try:
if self.local_store and self.local_store.exists(name):
return self.local_store.get_stream(name)
except Exception:
if self.remote_store:
logger.warning("Error getting '%s' from local store",
name,
exc_info=True)
else:
raise
if self.remote_store:
if self.local_store and serve_from_cache:
if version is None:
version = self.remote_store.file_version(name)
if version:
name = versioned_name(uname, version)
if force_refresh or not self.local_store.exists(name):
(stream, vname) = self.remote_store.get_stream(name)
name = self.local_store.add_stream(vname, stream)
return self.local_store.get_stream(name)
return self.remote_store.get_stream(name)
raise FiletrackerError("File not available: %s" % name)
finally:
if lock:
lock.close()
def get_stream(self, name, force_refresh=False, serve_from_cache=False):
"""Retrieves file identified by ``name`` in streaming mode.
Works like :meth:`get_file`, except that returns a tuple
(file-like object, versioned name).
When both remote_store and local_store are present, serve_from_cache
can be used to ensure that the file will be downloaded and served
from a local cache. If a full version is specified and the file
exists in the cache a file will be always served locally.
"""
uname, version = split_name(name)
lock = None
if self.local_store:
lock = self.lock_manager.lock_for(uname)
lock.lock_shared()
try:
if not self.remote_store or (version is not None
and not force_refresh):
try:
if self.local_store and self.local_store.exists(name):
return self.local_store.get_stream(name)
except Exception:
if self.remote_store:
logger.warning("Error getting '%s' from local store",
name, exc_info=True)
else:
raise
if self.remote_store:
if self.local_store and serve_from_cache:
if version is None:
version = self.remote_store.file_version(name)
if version:
name = versioned_name(uname, version)
if force_refresh or not self.local_store.exists(name):
(stream, vname) = self.remote_store.get_stream(name)
name = self.local_store.add_stream(vname, stream)
return self.local_store.get_stream(name)
return self.remote_store.get_stream(name)
raise FiletrackerError("File not available: %s" % name)
finally:
if lock:
lock.close()
def add_file(self, name, filename, compress_hint=True):
url, version = self._parse_name(name)
if self._has_capability(SERVER_ACCEPTS_SHA256_DIGEST):
sha = file_digest(filename)
else:
sha = ''
headers = {'SHA256-Checksum': sha}
# Important detail: this upload is streaming.
# http://docs.python-requests.org/en/latest/user/advanced/#streaming-uploads
with open(filename, 'rb') as f:
if (compress_hint and self._has_capability(SERVER_ACCEPTS_GZIP)):
# Unfortunately it seems a temporary file is required here.
# Our server requires Content-Length to be present, because
# some WSGI implementations (among others the one used in
# our tests) are not required to support EOF (instead the
# user is required to not read beyond content length,
# but that cannot be done if we don't know the content
# length). As content length is required for the tests to
# work, we need to send it, and to be able to compute it we
# need to temporarily store the compressed data before
# sending. It can be stored in memory or in a temporary file
# and a temporary file seems to be a more suitable choice.
with tempfile.TemporaryFile() as tmp:
with gzip.GzipFile(fileobj=tmp, mode='wb') as gz:
shutil.copyfileobj(f, gz)
tmp.seek(0)
headers['Content-Encoding'] = 'gzip'
headers['Logical-Size'] = str(os.stat(filename).st_size)
response = self._put_file(url, version, tmp, headers)
else:
response = self._put_file(url, version, f, headers)
name, version = split_name(name)
return versioned_name(name, self._parse_last_modified(response))
def add_file(self, name, filename, compress_hint=True):
url, version = self._parse_name(name)
headers = {}
if (compress_hint
and self._has_capability(SERVER_ACCEPTS_SHA256_DIGEST)):
headers['SHA256-Checksum'] = file_digest(filename)
# Important detail: this upload is streaming.
# http://docs.python-requests.org/en/latest/user/advanced/#streaming-uploads
with open(filename, 'rb') as f:
if (compress_hint
and self._has_capability(SERVER_ACCEPTS_GZIP)):
# Unfortunately it seems a temporary file is required here.
# Our server requires Content-Length to be present, because
# some WSGI implementations (among others the one used in
# our tests) are not required to support EOF (instead the
# user is required to not read beyond content length,
# but that cannot be done if we don't know the content
# length). As content length is required for the tests to
# work, we need to send it, and to be able to compute it we
# need to temporarily store the compressed data before
# sending. It can be stored in memory or in a temporary file
# and a temporary file seems to be a more suitable choice.
with tempfile.TemporaryFile() as tmp:
with gzip.GzipFile(fileobj=tmp, mode='wb') as gz:
shutil.copyfileobj(f, gz)
tmp.seek(0)
headers['Content-Encoding'] = 'gzip'
headers['Logical-Size'] = str(os.stat(filename).st_size)
response = self._put_file(url, version, tmp, headers)
else:
response = self._put_file(url, version, f, headers)
name, version = split_name(name)
return versioned_name(name, self._parse_last_modified(response))
def get_stream(self, name):
path, version = self._parse_name(name)
if not os.path.exists(path):
raise FiletrackerError("File not found: " + path)
return open(path, 'rb'), versioned_name(name, _file_version(path))
def add_stream(self, name, stream):
path, version = self._parse_name(name)
return versioned_name(
split_name(name)[0], _save_stream(path, stream, version))
def get_stream(self, name):
key, version = self._parse_name(name)
if version is not None and self.versions[key] != version:
raise KeyError("Version %s of %s not found" % (version, key))
return BytesIO(self.data[key]), versioned_name(key, self.versions[key])
def get_stream(self, name):
key, version = self._parse_name(name)
if version is not None and self.versions[key] != version:
raise KeyError("Version %s of %s not found" % (version, key))
return BytesIO(self.data[key]), \
versioned_name(key, self.versions[key])
def get_stream(self, name):
path, version = self._parse_name(name)
if not os.path.exists(path):
raise FiletrackerError("File not found: " + path)
return open(path, 'rb'), versioned_name(name, _file_version(path))
def add_stream(self, name, stream):
path, version = self._parse_name(name)
return versioned_name(
split_name(name)[0], _save_stream(path, stream, version))