def _protocol_version(self):
"""Returns the protocol version that should be used.
If the version wasn't established yet, asks the server what
versions it supports and picks the highest one.
"""
if hasattr(self, '_protocol_ver'):
return self._protocol_ver
response = requests.get(self.base_url + '/version/')
if response.status_code == 404:
server_versions = {1}
elif response.status_code == 200:
server_versions = set(response.json()['protocol_versions'])
if not server_versions:
raise FiletrackerError(
'Server hasn\'t reported any supported protocols'
)
else:
response.raise_for_status()
common_versions = _SUPPORTED_VERSIONS.intersection(server_versions)
if not common_versions:
raise FiletrackerError(
'Couldn\'t agree on protocol version: client supports '
'{}, server supports {}.'.format(
_PROTOCOL_CAPABILITIES, server_versions
)
)
self._protocol_ver = max(common_versions)
print('Settled for protocol version {}'.format(self._protocol_ver))
return self._protocol_ver
def wrapped(*args, **kwargs):
try:
return fn(*args, **kwargs)
except requests.exceptions.RequestException as e:
if e.response is None:
raise FiletrackerError('Error making HTTP request: %s' % e)
code = e.response.status_code
message = e.response.headers.get('x-exception', str(e))
stacktrace = e.response.text
raise FiletrackerError('HTTP/%d: %s\n%s' % (code, message, stacktrace))
def file_size(self, name, force_refresh=False):
"""Returns the size of the file.
For efficiency this operation does not use locking, so may return
inconsistent data. Use it for informational purposes.
"""
uname, version = split_name(name)
t = time.time()
logger.debug(' querying size of %s', name)
try:
if not self.remote_store or (version is not None and not force_refresh):
try:
if self.local_store and self.local_store.exists(name):
return self.local_store.file_size(name)
except Exception:
if self.remote_store:
logger.warning(
"Error getting '%s' from local store", name, exc_info=True
)
else:
raise
if self.remote_store:
return self.remote_store.file_size(name)
raise FiletrackerError("File not available: %s" % name)
finally:
logger.debug(' processed %s in %.2fs', name, time.time() - t)
def get_stream(self, name):
url, version = self._parse_name(name)
response = requests.get(url, stream=True)
response.raise_for_status()
remote_version = self._parse_last_modified(response)
if version is not None and remote_version is not None \
and version != remote_version:
raise FiletrackerError("Version %s not available. Server has %s" \
% (name, remote_version))
name, version = split_name(name)
stream = _FileLikeFromResponse(response)
return stream, versioned_name(name, remote_version)
def get_stream(self, name, force_refresh=False, serve_from_cache=False):
"""Retrieves file identified by ``name`` in streaming mode.
Works like :meth:`get_file`, except that returns a tuple
(file-like object, versioned name).
When both remote_store and local_store are present, serve_from_cache
can be used to ensure that the file will be downloaded and served
from a local cache. If a full version is specified and the file
exists in the cache a file will be always served locally.
"""
uname, version = split_name(name)
lock = None
if self.local_store:
lock = self.lock_manager.lock_for(uname)
lock.lock_shared()
try:
if not self.remote_store or (version is not None
and not force_refresh):
try:
if self.local_store and self.local_store.exists(name):
return self.local_store.get_stream(name)
except Exception:
if self.remote_store:
logger.warning("Error getting '%s' from local store",
name,
exc_info=True)
else:
raise
if self.remote_store:
if self.local_store and serve_from_cache:
if version is None:
version = self.remote_store.file_version(name)
if version:
name = versioned_name(uname, version)
if force_refresh or not self.local_store.exists(name):
(stream, vname) = self.remote_store.get_stream(name)
name = self.local_store.add_stream(vname, stream)
return self.local_store.get_stream(name)
return self.remote_store.get_stream(name)
raise FiletrackerError("File not available: %s" % name)
finally:
if lock:
lock.close()
def file_size(self, name):
path, version = self._parse_name(name)
size = _file_size(path)
if version is not None and _file_version(path) != version:
raise FiletrackerError("Version not found: " + name)
return size
def get_stream(self, name):
path, version = self._parse_name(name)
if not os.path.exists(path):
raise FiletrackerError("File not found: " + path)
return open(path, 'rb'), versioned_name(name, _file_version(path))
def get_file(
self,
name,
save_to,
add_to_cache=True,
force_refresh=False,
_lock_exclusive=False,
):
"""Retrieves file identified by ``name``.
The file is saved as ``save_to``. If ``add_to_cache`` is ``True``,
the file is added to the local store. If ``force_refresh`` is
``True``, local cache is not examined if a remote store is
configured.
If a remote store is configured, but ``name`` does not contain a
version, the local data store is not used, as we cannot guarantee
that the version there is fresh.
Local data store implemented in :class:`LocalDataStore` tries to not
copy the entire file to ``save_to`` if possible, but instead uses
hardlinking. Therefore you should not modify the file if you don't
want to totally blow something.
This method returns the full versioned name of the retrieved file.
"""
uname, version = split_name(name)
lock = None
if self.local_store:
lock = self.lock_manager.lock_for(uname)
if _lock_exclusive:
lock.lock_exclusive()
else:
lock.lock_shared()
else:
add_to_cache = False
t = time.time()
logger.debug(' downloading %s', name)
try:
if not self.remote_store or (version is not None and not force_refresh):
try:
if self.local_store and self.local_store.exists(name):
return self.local_store.get_file(name, save_to)
except Exception:
if self.remote_store:
logger.warning(
"Error getting '%s' from local store", name, exc_info=True
)
else:
raise
if self.remote_store:
if not _lock_exclusive and add_to_cache:
if lock:
lock.unlock()
return self.get_file(
name, save_to, add_to_cache, _lock_exclusive=True
)
vname = self.remote_store.get_file(name, save_to)
if add_to_cache:
self._add_to_cache(vname, save_to)
return vname
raise FiletrackerError("File not available: %s" % name)
finally:
if lock:
lock.close()
logger.debug(' processed %s in %.2fs', name, time.time() - t)