def load(self, digest):
"""Load the file with the given digest into the cache.
Ask the backend to provide the file and, if it's available,
copy its content into the file-system cache.
digest (unicode): the digest of the file to load.
raise (KeyError): if the backend cannot find the file.
"""
ftmp_handle, temp_file_path = tempfile.mkstemp(dir=self.temp_dir,
text=False)
ftmp = os.fdopen(ftmp_handle, 'w')
cache_file_path = os.path.join(self.file_dir, digest)
fobj = self.backend.get_file(digest)
# Copy the file to a temporary position
try:
copyfileobj(fobj, ftmp, self.CHUNK_SIZE)
finally:
ftmp.close()
fobj.close()
# Then move it to its real location (this operation is atomic
# by POSIX requirement)
os.rename(temp_file_path, cache_file_path)
def save(self, digest, desc=""):
"""Save the file with the given digest into the backend.
Use to local copy, available in the file-system cache, to store
the file in the backend, if it's not already there.
digest (unicode): the digest of the file to load.
desc (unicode): the (optional) description to associate to the
file.
"""
cache_file_path = os.path.join(self.file_dir, digest)
fobj = self.backend.put_file(digest, desc)
if fobj is None:
return
try:
with io.open(cache_file_path, 'rb') as src:
copyfileobj(src, fobj, self.CHUNK_SIZE)
finally:
fobj.close()
if not isinstance(self.backend, NullBackend):
os.unlink(cache_file_path)
def save(self, digest, desc="", file_path=None):
"""Save the file with the given digest into the backend.
Store the file in the backend, if it's not already there, using
either the file given in the file_path argument or, if that argument
is None, the copy of the file present in the cache.
digest (unicode): the digest of the file to load.
desc (unicode): the (optional) description to associate to the
file.
file_path (unicode): if it is None, use the copy of the file present
in the cache. Otherwise, use the file pointed to by this path.
"""
if file_path is None:
file_path = os.path.join(self.file_dir, digest)
fobj = self.backend.put_file(digest, desc)
if fobj is None:
return
try:
with io.open(file_path, 'rb') as src:
copyfileobj(src, fobj, self.CHUNK_SIZE)
finally:
fobj.close()
def save(self, digest, desc="", file_path=None):
"""Save the file with the given digest into the backend.
Store the file in the backend, if it's not already there, using
either the file given in the file_path argument or, if that argument
is None, the copy of the file present in the cache.
digest (unicode): the digest of the file to load.
desc (unicode): the (optional) description to associate to the
file.
file_path (unicode): if it is None, use the copy of the file present
in the cache. Otherwise, use the file pointed to by this path.
"""
if file_path is None:
file_path = os.path.join(self.file_dir, digest)
fobj = self.backend.put_file(digest, desc)
if fobj is None:
return
try:
with io.open(file_path, 'rb') as src:
copyfileobj(src, fobj, self.CHUNK_SIZE)
finally:
fobj.close()
def save(self, digest, desc=""):
"""Save the file with the given digest into the backend.
Use to local copy, available in the file-system cache, to store
the file in the backend, if it's not already there.
digest (unicode): the digest of the file to load.
desc (unicode): the (optional) description to associate to the
file.
raise (TombstoneError): if the digest is the tombstone
"""
if digest == FileCacher.TOMBSTONE_DIGEST:
raise TombstoneError()
cache_file_path = os.path.join(self.file_dir, digest)
fobj = self.backend.put_file(digest, desc)
if fobj is None:
return
try:
with io.open(cache_file_path, 'rb') as src:
copyfileobj(src, fobj, self.CHUNK_SIZE)
finally:
fobj.close()
def save(self, digest, desc=""):
"""Save the file with the given digest into the backend.
Use to local copy, available in the file-system cache, to store
the file in the backend, if it's not already there.
digest (unicode): the digest of the file to load.
desc (unicode): the (optional) description to associate to the
file.
raise (TombstoneError): if the digest is the tombstone
"""
if digest == FileCacher.TOMBSTONE_DIGEST:
raise TombstoneError()
cache_file_path = os.path.join(self.file_dir, digest)
fobj = self.backend.create_file(digest)
if fobj is None:
return
with io.open(cache_file_path, 'rb') as src:
copyfileobj(src, fobj, self.CHUNK_SIZE)
self.backend.commit_file(fobj, digest, desc)
def load(self, digest, if_needed=False):
"""Load the file with the given digest into the cache.
Ask the backend to provide the file and, if it's available,
copy its content into the file-system cache.
digest (unicode): the digest of the file to load.
if_needed (bool): only load the file if it is not present in
the local cache.
raise (KeyError): if the backend cannot find the file.
"""
cache_file_path = os.path.join(self.file_dir, digest)
if if_needed and os.path.exists(cache_file_path):
return
ftmp_handle, temp_file_path = tempfile.mkstemp(dir=self.temp_dir,
text=False)
ftmp = os.fdopen(ftmp_handle, 'w')
fobj = self.backend.get_file(digest)
# Copy the file to a temporary position
try:
copyfileobj(fobj, ftmp, self.CHUNK_SIZE)
finally:
ftmp.close()
fobj.close()
# Then move it to its real location (this operation is atomic
# by POSIX requirement)
os.rename(temp_file_path, cache_file_path)
def put_file(self, description="", binary_data=None,
file_obj=None, path=None):
"""Put a file in the storage, and keep a copy locally. The
caller has to provide exactly one among binary_data, file_obj
and path.
description (string): a human-readable description of the
content.
binary_data (string): the content of the file to send.
file_obj (file): the file-like object to send.
path (string): the file to send.
"""
temp_fd, temp_path = tempfile.mkstemp(dir=self.tmp_dir)
os.close(temp_fd)
# Input checking
if [binary_data, file_obj, path].count(None) != 2:
error_string = "No content (or too many) specified in put_file."
logger.error(error_string)
raise ValueError(error_string)
logger.debug("Reading input file to store on the database.")
# Copy the file content, whatever forms it arrives, into the
# temporary file
if path is not None:
copy(path, temp_path)
elif binary_data is not None:
with open(temp_path, 'wb') as temp_file:
temp_file.write(binary_data)
else: # file_obj is not None.
with open(temp_path, 'wb') as temp_file:
copyfileobj(file_obj, temp_file)
hasher = hashlib.sha1()
# Calculate the file SHA1 digest
with open(temp_path, 'rb') as temp_file:
buf = temp_file.read(self.CHUNK_SIZE)
while buf != '':
hasher.update(buf)
buf = temp_file.read(self.CHUNK_SIZE)
digest = hasher.hexdigest()
logger.debug("File has digest %s." % digest)
self.backend.put_file(digest, temp_path, description=description)
# Move the temporary file in the cache
move(temp_path,
os.path.join(self.obj_dir, digest))
return digest
def create_file_from_fileobj(self, path, file_obj, executable=False):
"""Write a file in the sandbox copying the content of an open
file-like object.
path (string): relative path of the file inside the sandbox.
file_obj (file): where from read the file content.
executable (bool): to set permissions.
"""
dest = self.create_file(path, executable)
copyfileobj(file_obj, dest)
dest.close()
def create_file_from_fileobj(self, path, file_obj, executable=False):
"""Write a file in the sandbox copying the content of an open
file-like object.
path (string): relative path of the file inside the sandbox.
file_obj (file): where from read the file content.
executable (bool): to set permissions.
"""
dest = self.create_file(path, executable)
copyfileobj(file_obj, dest)
dest.close()
def get_file_to_fobj(self, digest, dst):
"""Retrieve a file from the storage.
See `get_file'. This method will write the content of the file
to the given file-object.
digest (unicode): the digest of the file to get.
dst (fileobj): a writable binary file-like object on which to
write the contents of the file.
raise (KeyError): if the file cannot be found.
"""
with self.get_file(digest) as src:
copyfileobj(src, dst, self.CHUNK_SIZE)
def get_file_to_fobj(self, digest, dst):
"""Retrieve a file from the storage.
See `get_file'. This method will write the content of the file
to the given file-object.
digest (unicode): the digest of the file to get.
dst (fileobj): a writable binary file-like object on which to
write the contents of the file.
raise (KeyError): if the file cannot be found.
"""
with self.get_file(digest) as src:
copyfileobj(src, dst, self.CHUNK_SIZE)
def get_file_to_path(self, digest, dst_path):
"""Retrieve a file from the storage.
See `get_file'. This method will write the content of a file
to the given file-system location.
digest (unicode): the digest of the file to get.
dst_path (string): an accessible location on the file-system on
which to write the contents of the file.
raise (KeyError): if the file cannot be found.
"""
with self.get_file(digest) as src:
with io.open(dst_path, 'wb') as dst:
copyfileobj(src, dst, self.CHUNK_SIZE)
def get_file_to_path(self, digest, dst_path):
"""Retrieve a file from the storage.
See `get_file'. This method will write the content of a file
to the given file-system location.
digest (unicode): the digest of the file to get.
dst_path (string): an accessible location on the file-system on
which to write the contents of the file.
raise (KeyError): if the file cannot be found.
"""
with self.get_file(digest) as src:
with io.open(dst_path, 'wb') as dst:
copyfileobj(src, dst, self.CHUNK_SIZE)
def load(self, digest):
"""Load the file with the given digest into the cache.
Ask the backend to provide the file and, if it's available,
copy its content into the file-system cache.
digest (unicode): the digest of the file to load.
raise (KeyError): if the backend cannot find the file.
"""
cache_file_path = os.path.join(self.file_dir, digest)
fobj = self.backend.get_file(digest)
try:
with io.open(cache_file_path, 'wb') as dst:
copyfileobj(fobj, dst, self.CHUNK_SIZE)
finally:
fobj.close()
def load(self, digest):
"""Load the file with the given digest into the cache.
Ask the backend to provide the file and, if it's available,
copy its content into the file-system cache.
digest (unicode): the digest of the file to load.
raise (KeyError): if the backend cannot find the file.
"""
cache_file_path = os.path.join(self.file_dir, digest)
fobj = self.backend.get_file(digest)
try:
with io.open(cache_file_path, 'wb') as dst:
copyfileobj(fobj, dst, self.CHUNK_SIZE)
finally:
fobj.close()
def save(self, digest, desc=""):
"""Save the file with the given digest into the backend.
Use to local copy, available in the file-system cache, to store
the file in the backend, if it's not already there.
digest (unicode): the digest of the file to load.
desc (unicode): the (optional) description to associate to the
file.
"""
cache_file_path = os.path.join(self.file_dir, digest)
fobj = self.backend.put_file(digest, desc)
if fobj is None:
return
try:
with io.open(cache_file_path, 'rb') as src:
copyfileobj(src, fobj, self.CHUNK_SIZE)
finally:
fobj.close()
def load(self, digest, if_needed=False):
"""Load the file with the given digest into the cache.
Ask the backend to provide the file and, if it's available,
copy its content into the file-system cache.
digest (unicode): the digest of the file to load.
if_needed (bool): only load the file if it is not present in
the local cache.
raise (KeyError): if the backend cannot find the file.
raise (TombstoneError): if the digest is the tombstone
"""
if digest == FileCacher.TOMBSTONE_DIGEST:
raise TombstoneError()
cache_file_path = os.path.join(self.file_dir, digest)
if if_needed and os.path.exists(cache_file_path):
return
ftmp_handle, temp_file_path = tempfile.mkstemp(dir=self.temp_dir,
text=False)
ftmp = os.fdopen(ftmp_handle, 'w')
fobj = self.backend.get_file(digest)
# Copy the file to a temporary position
try:
copyfileobj(fobj, ftmp, self.CHUNK_SIZE)
finally:
ftmp.close()
fobj.close()
# Then move it to its real location (this operation is atomic
# by POSIX requirement)
os.rename(temp_file_path, cache_file_path)
def get_file(self, digest, path=None, file_obj=None,
string=False, temp_path=False, temp_file_obj=False):
"""Get a file from the storage, possibly using the cache if
the file is available there.
digest (string): the sha1 sum of the file.
path (string): a path where to save the file.
file_obj (file): a handler where to save the file (that is not
closed at return).
string (bool): True to return content as a string.
temp_path (bool): True to return path of a temporary file with
that content. The file is reserved to the
caller, who has the duty to unlink it.
temp_file-obj (bool): True to return a file object opened to a
temporary file with that content. The
file is reserved to the caller. Use this
method only for debugging purpose, as it
leave a file lying in the temporary
directory of FileCacher.
"""
if [string, temp_path, temp_file_obj].count(True) > 1:
raise ValueError("Ask for at most one amongst content, "
"temp path and temp file obj.")
cache_path = os.path.join(self.obj_dir, digest)
cache_exists = os.path.exists(cache_path)
logger.debug("Getting file %s." % (digest))
if not cache_exists:
logger.debug("File %s not in cache, downloading "
"from database." % digest)
# Receives the file from the database
temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir)
temp_file = os.fdopen(temp_file, "wb")
self.backend.get_file(digest, temp_filename)
# And move it in the cache. Warning: this is not atomic if
# the temp and the cache dir are on different filesystems.
move(temp_filename, cache_path)
logger.debug("File %s downloaded." % digest)
# Saving to path
if path is not None:
copy(cache_path, path)
# Saving to file object
if file_obj is not None:
with open(cache_path, "rb") as file_:
copyfileobj(file_, file_obj)
# Returning string?
if string:
with open(cache_path, "rb") as cache_file:
return cache_file.read()
# Returning temporary file?
elif temp_path:
temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir)
os.close(temp_file)
copy(cache_path, temp_filename)
return temp_filename
# Returning temporary file object?
elif temp_file_obj:
temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir)
os.close(temp_file)
copy(cache_path, temp_filename)
temp_file = open(temp_filename, "rb")
return temp_file