def dump_files(cls, debug=True, verbose=False):
"""
Writes all files to the filesystem.
"""
if debug:
tmp_debug = settings.DEBUG
settings.DEBUG = False
try:
q = cls.objects.only('id', 'name', '_content_hash')\
.values_list('id', 'name', '_content_hash')
total = q.count()
if verbose:
print('Checking %i total files...' % (total, ))
i = 0
for (file_id, name, content_hash) in q.iterator():
i += 1
if verbose and not i % 100:
print('%i of %i' % (i, total))
if not is_fresh(name=name, content_hash=content_hash):
if verbose:
print('File %i-%s is stale. Writing to local file '
'system...' % (file_id, name))
f = File.objects.get(id=file_id)
write_file(f.name, f.content, overwrite=True)
f._content_hash = None
f.save()
finally:
if debug:
settings.DEBUG = tmp_debug
def dump_files(cls, debug=True, verbose=False):
"""
Writes all files to the filesystem.
"""
if debug:
tmp_debug = settings.DEBUG
settings.DEBUG = False
try:
q = cls.objects.only('id', 'name', '_content_hash')\
.values_list('id', 'name', '_content_hash')
total = q.count()
if verbose:
print('Checking %i total files...' % (total,))
i = 0
for (file_id, name, content_hash) in q.iterator():
i += 1
if verbose and not i % 100:
print('%i of %i' % (i, total))
if not is_fresh(name=name, content_hash=content_hash):
if verbose:
print(('File %i-%s is stale. Writing to local file '
'system...') % (file_id, name))
file = File.objects.get(id=file_id)
write_file(
file.name,
file.content,
overwrite=True)
file._content_hash = None
file.save()
finally:
if debug:
settings.DEBUG = tmp_debug
def dump(self, check_hash=False):
"""
Writes the file content to the filesystem.
If check_hash is true, clears the stored file hash and recalculates.
"""
if is_fresh(self.name, self._content_hash):
return
write_file(self.name, self.content, overwrite=True)
if check_hash:
self._content_hash = None
self.save()
def dump(self, check_hash=False):
"""
Writes the file content to the filesystem.
If check_hash is true, clears the stored file hash and recalculates.
"""
if is_fresh(self.name, self._content_hash):
return
write_file(
self.name,
self.content,
overwrite=True)
if check_hash:
self._content_hash = None
self.save()
def _save(self, name, content):
"""
Save file with filename `name` and given content to the database.
"""
full_path = self.path(name)
try:
size = content.size
except AttributeError:
size = os.path.getsize(full_path)
content.seek(0)
content = content.read()
File.objects.create(content=content, size=size, name=name)
# Automatically write the change to the local file system.
if _settings.DB_FILES_AUTO_EXPORT_DB_TO_FS:
utils.write_file(name, content, overwrite=True)
return name
def _save(self, name, content):
"""
Save file with filename `name` and given content to the database.
"""
full_path = self.path(name)
try:
size = content.size
except AttributeError:
size = os.path.getsize(full_path)
content.seek(0)
content = content.read()
f = models.File.objects.create(
content=content,
size=size,
name=name,
)
# Automatically write the change to the local file system.
if settings.DB_FILES_AUTO_EXPORT_DB_TO_FS:
utils.write_file(name, content, overwrite=True)
#TODO:add callback to handle custom save behavior?
return self._generate_name(name, f.pk)
def _open(self, name, mode='rb'):
"""
Open file with filename `name` from the database.
"""
try:
# Load file from database.
f = models.File.objects.get_from_name(name)
content = f.content
size = f.size
if settings.DB_FILES_AUTO_EXPORT_DB_TO_FS \
and not utils.is_fresh(f.name, f.content_hash):
# Automatically write the file to the filesystem
# if it's missing and exists in the database.
# This happens if we're using multiple web servers connected
# to a common databaes behind a load balancer.
# One user might upload a file from one web server, and then
# another might access if from another server.
utils.write_file(f.name, f.content)
except models.File.DoesNotExist:
# If not yet in the database, check the local file system
# and load it into the database if present.
fqfn = self.path(name)
if os.path.isfile(fqfn):
#print('Loading file into database.')
self._save(name, open(fqfn, mode))
fh = super(DatabaseStorage, self)._open(name, mode)
content = fh.read()
size = fh.size
else:
# Otherwise we don't know where the file is.
return
# Normalize the content to a new file object.
#fh = StringIO(content)
fh = six.BytesIO(content)
fh.name = name
fh.mode = mode
fh.size = size
o = files.File(fh)
return o
def _open(self, name, mode='rb'):
"""
Open file with filename `name` from the database.
"""
try:
# Load file from database.
f = models.File.objects.get_from_name(name)
content = f.content
size = f.size
if _settings.DB_FILES_AUTO_EXPORT_DB_TO_FS and not utils.is_fresh(
f.name, f.content_hash):
# Automatically write the file to the filesystem
# if it's missing and exists in the database.
# This happens if we're using multiple web servers connected
# to a common databaes behind a load balancer.
# One user might upload a file from one web server, and then
# another might access if from another server.
utils.write_file(f.name, f.content)
except models.File.DoesNotExist:
# If not yet in the database, check the local file system
# and load it into the database if present.
fqfn = self.path(name)
if os.path.isfile(fqfn):
#print('Loading file into database.')
self._save(name, open(fqfn, mode))
fh = super(DatabaseStorage, self)._open(name, mode)
content = fh.read()
size = fh.size
else:
# Otherwise we don't know where the file is.
return
# Normalize the content to a new file object.
#fh = StringIO(content)
fh = six.BytesIO(content)
fh.name = name
fh.mode = mode
fh.size = size
o = files.File(fh)
return o
