def lock_for(self, name):
check_name(name)
name, version = split_name(name)
path = self.dir + name
dir = os.path.dirname(path)
mkdir(dir)
return self.FcntlLock(path)
def file_size(self, name, force_refresh=False):
"""Returns the size of the file.
For efficiency this operation does not use locking, so may return
inconsistent data. Use it for informational purposes.
"""
uname, version = split_name(name)
t = time.time()
logger.debug(' querying size of %s', name)
try:
if not self.remote_store or (version is not None
and not force_refresh):
try:
if self.local_store and self.local_store.exists(name):
return self.local_store.file_size(name)
except Exception:
if self.remote_store:
logger.warning("Error getting '%s' from local store",
name, exc_info=True)
else:
raise
if self.remote_store:
return self.remote_store.file_size(name)
raise FiletrackerError("File not available: %s" % name)
finally:
logger.debug(' processed %s in %.2fs', name, time.time() - t)
def file_size(self, name, force_refresh=False):
"""Returns the size of the file.
For efficiency this operation does not use locking, so may return
inconsistent data. Use it for informational purposes.
"""
uname, version = split_name(name)
t = time.time()
logger.debug(' querying size of %s', name)
try:
if not self.remote_store or (version is not None and not force_refresh):
try:
if self.local_store and self.local_store.exists(name):
return self.local_store.file_size(name)
except Exception:
if self.remote_store:
logger.warning(
"Error getting '%s' from local store", name, exc_info=True
)
else:
raise
if self.remote_store:
return self.remote_store.file_size(name)
raise FiletrackerError("File not available: %s" % name)
finally:
logger.debug(' processed %s in %.2fs', name, time.time() - t)
def __new__(cls, versioned_name):
# http://stackoverflow.com/questions/14783698/how-to-or-why-not-call-unicode-init-from-subclass
if isinstance(versioned_name, FiletrackerFilename):
versioned_name = versioned_name.versioned_name
versioned_name = six.text_type(versioned_name)
name, _version = split_name(versioned_name)
self = six.text_type.__new__(cls, name)
self.versioned_name = versioned_name
return self
def _get_needed_files(self):
result = []
for app in cache.get_apps():
model_list = cache.get_models(app)
for model in model_list:
file_fields = [field.name for field in model._meta.fields
if field.get_internal_type() == 'FileField']
if len(file_fields) > 0:
files = model.objects.all().values_list(*file_fields)
result.extend([split_name(file)[0] for file in itertools.
chain.from_iterable(files) if file])
return result
def get_stream(self, name):
url, version = self._parse_name(name)
response = requests.get(url, stream=True)
response.raise_for_status()
remote_version = self._parse_last_modified(response)
if version is not None and remote_version is not None \
and version != remote_version:
raise FiletrackerError("Version %s not available. Server has %s" \
% (name, remote_version))
name, version = split_name(name)
stream = _FileLikeFromResponse(response)
return stream, versioned_name(name, remote_version)
def get_stream(self, name):
url, version = self._parse_name(name)
response = requests.get(url, stream=True)
response.raise_for_status()
remote_version = self._parse_last_modified(response)
if version is not None and remote_version is not None \
and version != remote_version:
raise FiletrackerError("Version %s not available. Server has %s" \
% (name, remote_version))
name, version = split_name(name)
stream = _FileLikeFromResponse(response)
return stream, versioned_name(name, remote_version)
def get_stream(self, name, force_refresh=False, serve_from_cache=False):
"""Retrieves file identified by ``name`` in streaming mode.
Works like :meth:`get_file`, except that returns a tuple
(file-like object, versioned name).
When both remote_store and local_store are present, serve_from_cache
can be used to ensure that the file will be downloaded and served
from a local cache. If a full version is specified and the file
exists in the cache a file will be always served locally.
"""
uname, version = split_name(name)
lock = None
if self.local_store:
lock = self.lock_manager.lock_for(uname)
lock.lock_shared()
try:
if not self.remote_store or (version is not None
and not force_refresh):
try:
if self.local_store and self.local_store.exists(name):
return self.local_store.get_stream(name)
except Exception:
if self.remote_store:
logger.warning("Error getting '%s' from local store",
name,
exc_info=True)
else:
raise
if self.remote_store:
if self.local_store and serve_from_cache:
if version is None:
version = self.remote_store.file_version(name)
if version:
name = versioned_name(uname, version)
if force_refresh or not self.local_store.exists(name):
(stream, vname) = self.remote_store.get_stream(name)
name = self.local_store.add_stream(vname, stream)
return self.local_store.get_stream(name)
return self.remote_store.get_stream(name)
raise FiletrackerError("File not available: %s" % name)
finally:
if lock:
lock.close()
def get_stream(self, name, force_refresh=False, serve_from_cache=False):
"""Retrieves file identified by ``name`` in streaming mode.
Works like :meth:`get_file`, except that returns a tuple
(file-like object, versioned name).
When both remote_store and local_store are present, serve_from_cache
can be used to ensure that the file will be downloaded and served
from a local cache. If a full version is specified and the file
exists in the cache a file will be always served locally.
"""
uname, version = split_name(name)
lock = None
if self.local_store:
lock = self.lock_manager.lock_for(uname)
lock.lock_shared()
try:
if not self.remote_store or (version is not None
and not force_refresh):
try:
if self.local_store and self.local_store.exists(name):
return self.local_store.get_stream(name)
except Exception:
if self.remote_store:
logger.warning("Error getting '%s' from local store",
name, exc_info=True)
else:
raise
if self.remote_store:
if self.local_store and serve_from_cache:
if version is None:
version = self.remote_store.file_version(name)
if version:
name = versioned_name(uname, version)
if force_refresh or not self.local_store.exists(name):
(stream, vname) = self.remote_store.get_stream(name)
name = self.local_store.add_stream(vname, stream)
return self.local_store.get_stream(name)
return self.remote_store.get_stream(name)
raise FiletrackerError("File not available: %s" % name)
finally:
if lock:
lock.close()
def _get_needed_files(self):
result = []
for app in cache.get_apps():
model_list = cache.get_models(app)
for model in model_list:
file_fields = [
field.name for field in model._meta.fields
if field.get_internal_type() == 'FileField'
]
if len(file_fields) > 0:
files = model.objects.all().values_list(*file_fields)
result.extend([
split_name(file)[0]
for file in itertools.chain.from_iterable(files)
if file
])
return result
def get_file(self, name, filename):
"""Saves the content of file named ``name`` to ``filename``.
Works like :meth:`get_stream`, but ``filename`` is the name of
a file which will be created (or overwritten).
Returns the full versioned name of the retrieved file.
"""
stream, vname = self.get_stream(name)
path, version = split_name(vname)
dir_path = os.path.dirname(filename)
if dir_path:
mkdir(dir_path)
with open(filename, 'wb') as f:
shutil.copyfileobj(stream, f)
return vname
def add_file(self, name, filename, compress_hint=True):
url, version = self._parse_name(name)
if self._has_capability(SERVER_ACCEPTS_SHA256_DIGEST):
sha = file_digest(filename)
else:
sha = ''
headers = {'SHA256-Checksum': sha}
# Important detail: this upload is streaming.
# http://docs.python-requests.org/en/latest/user/advanced/#streaming-uploads
with open(filename, 'rb') as f:
if (compress_hint and self._has_capability(SERVER_ACCEPTS_GZIP)):
# Unfortunately it seems a temporary file is required here.
# Our server requires Content-Length to be present, because
# some WSGI implementations (among others the one used in
# our tests) are not required to support EOF (instead the
# user is required to not read beyond content length,
# but that cannot be done if we don't know the content
# length). As content length is required for the tests to
# work, we need to send it, and to be able to compute it we
# need to temporarily store the compressed data before
# sending. It can be stored in memory or in a temporary file
# and a temporary file seems to be a more suitable choice.
with tempfile.TemporaryFile() as tmp:
with gzip.GzipFile(fileobj=tmp, mode='wb') as gz:
shutil.copyfileobj(f, gz)
tmp.seek(0)
headers['Content-Encoding'] = 'gzip'
headers['Logical-Size'] = str(os.stat(filename).st_size)
response = self._put_file(url, version, tmp, headers)
else:
response = self._put_file(url, version, f, headers)
name, version = split_name(name)
return versioned_name(name, self._parse_last_modified(response))
def add_file(self, name, filename, compress_hint=True):
url, version = self._parse_name(name)
headers = {}
if (compress_hint
and self._has_capability(SERVER_ACCEPTS_SHA256_DIGEST)):
headers['SHA256-Checksum'] = file_digest(filename)
# Important detail: this upload is streaming.
# http://docs.python-requests.org/en/latest/user/advanced/#streaming-uploads
with open(filename, 'rb') as f:
if (compress_hint
and self._has_capability(SERVER_ACCEPTS_GZIP)):
# Unfortunately it seems a temporary file is required here.
# Our server requires Content-Length to be present, because
# some WSGI implementations (among others the one used in
# our tests) are not required to support EOF (instead the
# user is required to not read beyond content length,
# but that cannot be done if we don't know the content
# length). As content length is required for the tests to
# work, we need to send it, and to be able to compute it we
# need to temporarily store the compressed data before
# sending. It can be stored in memory or in a temporary file
# and a temporary file seems to be a more suitable choice.
with tempfile.TemporaryFile() as tmp:
with gzip.GzipFile(fileobj=tmp, mode='wb') as gz:
shutil.copyfileobj(f, gz)
tmp.seek(0)
headers['Content-Encoding'] = 'gzip'
headers['Logical-Size'] = str(os.stat(filename).st_size)
response = self._put_file(url, version, tmp, headers)
else:
response = self._put_file(url, version, f, headers)
name, version = split_name(name)
return versioned_name(name, self._parse_last_modified(response))
def _parse_name(self, name):
check_name(name)
name, version = split_name(name)
url = self.base_url + '/files' + pathname2url(name)
return url, version
def lock_for(self, name):
check_name(name)
name, version = split_name(name)
path = self.dir + name
return self.FcntlLock(self, path)
def get_file(self, name, save_to, add_to_cache=True,
force_refresh=False, _lock_exclusive=False):
"""Retrieves file identified by ``name``.
The file is saved as ``save_to``. If ``add_to_cache`` is ``True``,
the file is added to the local store. If ``force_refresh`` is
``True``, local cache is not examined if a remote store is
configured.
If a remote store is configured, but ``name`` does not contain a
version, the local data store is not used, as we cannot guarantee
that the version there is fresh.
Local data store implemented in :class:`LocalDataStore` tries to not
copy the entire file to ``save_to`` if possible, but instead uses
hardlinking. Therefore you should not modify the file if you don't
want to totally blow something.
This method returns the full versioned name of the retrieved file.
"""
uname, version = split_name(name)
lock = None
if self.local_store:
lock = self.lock_manager.lock_for(uname)
if _lock_exclusive:
lock.lock_exclusive()
else:
lock.lock_shared()
else:
add_to_cache = False
t = time.time()
logger.debug(' downloading %s', name)
try:
if not self.remote_store or (version is not None
and not force_refresh):
try:
if self.local_store and self.local_store.exists(name):
return self.local_store.get_file(name, save_to)
except Exception:
if self.remote_store:
logger.warning("Error getting '%s' from local store",
name, exc_info=True)
else:
raise
if self.remote_store:
if not _lock_exclusive and add_to_cache:
if lock:
lock.unlock()
return self.get_file(name, save_to, add_to_cache,
_lock_exclusive=True)
vname = self.remote_store.get_file(name, save_to)
if add_to_cache:
self._add_to_cache(vname, save_to)
return vname
raise FiletrackerError("File not available: %s" % name)
finally:
if lock:
lock.close()
logger.debug(' processed %s in %.2fs', name, time.time() - t)
def get_file(self, name, filename):
# Use hardlinks to avoid unnecessary copying.
stream, vname = self.get_stream(name)
path, version = split_name(vname)
_save_stream(filename, stream, version)
return vname
def _parse_name(self, name):
check_name(name)
name, version = split_name(name)
path = self.dir + name
return path, version
def add_stream(self, name, stream):
path, version = self._parse_name(name)
return versioned_name(
split_name(name)[0], _save_stream(path, stream, version))
def _parse_name(self, name):
check_name(name)
key, version = split_name(name)
return key, version
def _parse_name(self, name):
check_name(name)
name, version = split_name(name)
url = self.base_url + '/files' + pathname2url(name)
return url, version
def get_file(
self,
name,
save_to,
add_to_cache=True,
force_refresh=False,
_lock_exclusive=False,
):
"""Retrieves file identified by ``name``.
The file is saved as ``save_to``. If ``add_to_cache`` is ``True``,
the file is added to the local store. If ``force_refresh`` is
``True``, local cache is not examined if a remote store is
configured.
If a remote store is configured, but ``name`` does not contain a
version, the local data store is not used, as we cannot guarantee
that the version there is fresh.
Local data store implemented in :class:`LocalDataStore` tries to not
copy the entire file to ``save_to`` if possible, but instead uses
hardlinking. Therefore you should not modify the file if you don't
want to totally blow something.
This method returns the full versioned name of the retrieved file.
"""
uname, version = split_name(name)
lock = None
if self.local_store:
lock = self.lock_manager.lock_for(uname)
if _lock_exclusive:
lock.lock_exclusive()
else:
lock.lock_shared()
else:
add_to_cache = False
t = time.time()
logger.debug(' downloading %s', name)
try:
if not self.remote_store or (version is not None and not force_refresh):
try:
if self.local_store and self.local_store.exists(name):
return self.local_store.get_file(name, save_to)
except Exception:
if self.remote_store:
logger.warning(
"Error getting '%s' from local store", name, exc_info=True
)
else:
raise
if self.remote_store:
if not _lock_exclusive and add_to_cache:
if lock:
lock.unlock()
return self.get_file(
name, save_to, add_to_cache, _lock_exclusive=True
)
vname = self.remote_store.get_file(name, save_to)
if add_to_cache:
self._add_to_cache(vname, save_to)
return vname
raise FiletrackerError("File not available: %s" % name)
finally:
if lock:
lock.close()
logger.debug(' processed %s in %.2fs', name, time.time() - t)
def _parse_name(self, name):
check_name(name)
key, version = split_name(name)
return key, version
def _parse_name(self, name):
check_name(name)
name, version = split_name(name)
path = self.dir + name
return path, version
def get_file(self, name, filename):
# Use hardlinks to avoid unnecessary copying.
stream, vname = self.get_stream(name)
path, version = split_name(vname)
_save_stream(filename, stream, version)
return vname
def add_stream(self, name, stream):
path, version = self._parse_name(name)
return versioned_name(
split_name(name)[0], _save_stream(path, stream, version))
