def fix(self):
if self.fixed:
return
# sys.stderr.write("fix: starting " + str(self.section) + '/' + str(self.file_number) + "\n")
if cloud is not None:
self.modtimes = dict()
self.keydict = dict()
if not os.path.isdir(self.directory):
os.makedirs(self.directory)
#self.directory = tempfile.mkdtemp(prefix='SavedFile')
#docassemble.base.functions.this_thread.temporary_resources.add(self.directory)
prefix = str(self.section) + '/' + str(self.file_number) + '/'
#sys.stderr.write("fix: prefix is " + prefix + "\n")
for key in cloud.list_keys(prefix):
filename = re.sub(r'.*/', '', key.name)
fullpath = os.path.join(self.directory, filename)
server_time = key.get_epoch_modtime()
if not (os.path.isfile(fullpath) and os.path.getmtime(fullpath) == server_time):
key.get_contents_to_filename(fullpath)
self.modtimes[filename] = server_time
#logmessage("cloud modtime for file " + filename + " is " + str(key.last_modified))
self.keydict[filename] = key
self.path = os.path.join(self.directory, self.filename)
to_delete = list()
for filename in os.listdir(self.directory):
if filename not in self.modtimes:
to_delete.append(filename)
for filename in to_delete:
os.remove(os.path.join(self.directory, filename))
else:
if not os.path.isdir(self.directory):
os.makedirs(self.directory)
self.fixed = True
def fix(self):
if self.fixed:
return
# sys.stderr.write("fix: starting " + str(self.section) + '/' + str(self.file_number) + "\n")
if cloud is not None:
self.modtimes = dict()
self.keydict = dict()
if not os.path.isdir(self.directory):
os.makedirs(self.directory)
#self.directory = tempfile.mkdtemp(prefix='SavedFile')
#docassemble.base.functions.this_thread.temporary_resources.add(self.directory)
prefix = str(self.section) + '/' + str(self.file_number) + '/'
#sys.stderr.write("fix: prefix is " + prefix + "\n")
for key in cloud.list_keys(prefix):
filename = re.sub(r'.*/', '', key.name)
fullpath = os.path.join(self.directory, filename)
server_time = key.get_epoch_modtime()
if not (os.path.isfile(fullpath) and os.path.getmtime(fullpath) == server_time):
key.get_contents_to_filename(fullpath)
self.modtimes[filename] = server_time
#logmessage("cloud modtime for file " + filename + " is " + str(key.last_modified))
self.keydict[filename] = key
self.path = os.path.join(self.directory, self.filename)
to_delete = list()
for filename in os.listdir(self.directory):
if filename not in self.modtimes:
to_delete.append(filename)
for filename in to_delete:
os.remove(os.path.join(self.directory, filename))
else:
if not os.path.isdir(self.directory):
os.makedirs(self.directory)
self.fixed = True
def delete(self):
if cloud is not None:
prefix = str(self.section) + '/' + str(self.file_number) + '/'
for key in cloud.list_keys(prefix):
key.delete()
if hasattr(self, 'directory') and os.path.isdir(self.directory):
shutil.rmtree(self.directory)
def __init__(self, file_number, extension=None, fix=False, section='files', filename='file', subdir=None, should_not_exist=False):
file_number = int(file_number)
section = str(section)
if section not in docassemble.base.functions.this_thread.saved_files:
docassemble.base.functions.this_thread.saved_files[section] = {}
if file_number in docassemble.base.functions.this_thread.saved_files[section]:
# sys.stderr.write("SavedFile: using cache for " + section + '/' + str(file_number) + "\n")
sf = docassemble.base.functions.this_thread.saved_files[section][file_number]
for attribute in ['file_number', 'fixed', 'section', 'filename', 'extension', 'directory', 'path', 'modtimes', 'keydict', 'subdir']:
if hasattr(sf, attribute):
setattr(self, attribute, getattr(sf, attribute))
self.extension = extension
self.filename = filename
self.subdir = subdir
else:
# sys.stderr.write("SavedFile: not using cache for " + section + '/' + str(file_number) + "\n")
self.fixed = False
self.file_number = file_number
self.section = section
self.extension = extension
self.filename = filename
self.subdir = subdir
if cloud is None:
if self.section == 'files':
parts = re.sub(r'(...)', r'\1/', '{0:012x}'.format(int(file_number))).split('/')
self.directory = os.path.join(UPLOAD_DIRECTORY, *parts)
else:
self.directory = os.path.join(UPLOAD_DIRECTORY, str(self.section), str(file_number))
else:
self.directory = os.path.join(tempfile.gettempdir(), str(self.section), str(self.file_number))
docassemble.base.functions.this_thread.saved_files[section][file_number] = self
if self.subdir and self.subdir != '' and self.subdir != 'default':
self.path = os.path.join(self.directory, self.subdir, self.filename)
else:
self.path = os.path.join(self.directory, self.filename)
if fix:
self.fix()
if should_not_exist and os.path.isdir(self.directory):
found_error = False
for root, dirs, files in os.walk(self.directory):
if len(files) > 0 or len(dirs) > 0:
found_error = True
break
if found_error:
logmessage("WARNING! Possible database corruption due to an unsafe shutdown. Your database indicated that the next file number is " + str(file_number) + ", but there is already a file in the file storage for that number. It is recommended that you restart your system. If that does not make this error go away, you should investigate why there are existing files in the file system.")
if cloud is not None:
prefix = str(self.section) + '/' + str(self.file_number) + '/'
for key in list(cloud.list_keys(prefix)):
try:
key.delete()
except:
pass
if hasattr(self, 'directory') and os.path.isdir(self.directory):
shutil.rmtree(self.directory)
if not os.path.isdir(self.directory):
os.makedirs(self.directory)
def delete_file(self, filename):
if cloud is not None:
prefix = str(self.section) + '/' + str(
self.file_number) + '/' + str(filename)
for key in cloud.list_keys(prefix):
key.delete()
the_path = os.path.join(self.directory, filename)
if hasattr(self, 'directory') and os.path.isdir(
self.directory) and os.path.isfile(the_path):
os.remove(the_path)
def list_of_files(self):
output = list()
if cloud is not None and not self.fixed:
prefix = str(self.section) + '/' + str(self.file_number) + '/'
for key in cloud.list_keys(prefix):
output.append(os.path.join(*key.name[len(prefix):].split('/')))
else:
if os.path.isdir(self.directory):
for filename in listfiles(self.directory):
output.append(filename)
return sorted(output)
def delete(self):
if cloud is not None:
prefix = str(self.section) + '/' + str(self.file_number) + '/'
for key in list(cloud.list_keys(prefix)):
try:
key.delete()
except:
pass
if hasattr(self, 'directory') and os.path.isdir(self.directory):
shutil.rmtree(self.directory)
del docassemble.base.functions.this_thread.saved_files[str(self.section)][int(self.file_number)]
def list_of_files(self):
output = list()
if cloud is not None and not self.fixed:
prefix = str(self.section) + '/' + str(self.file_number) + '/'
for key in cloud.list_keys(prefix):
output.append(re.sub(r'.*/', '', key.name))
else:
if os.path.isdir(self.directory):
for filename in os.listdir(self.directory):
output.append(filename)
return sorted(output)
def list_of_files(self):
output = list()
if cloud is not None and not self.fixed:
prefix = str(self.section) + '/' + str(self.file_number) + '/'
for key in cloud.list_keys(prefix):
output.append(re.sub(r'.*/', '', key.name))
else:
if os.path.isdir(self.directory):
for filename in os.listdir(self.directory):
output.append(filename)
return sorted(output)
def delete(self):
if cloud is not None:
prefix = str(self.section) + '/' + str(self.file_number) + '/'
for key in [x for x in cloud.list_keys(prefix)]:
try:
key.delete()
except:
pass
if hasattr(self, 'directory') and os.path.isdir(self.directory):
shutil.rmtree(self.directory)
del docassemble.base.functions.this_thread.saved_files[str(self.section)][int(self.file_number)]
def fix(self):
if self.fixed:
return
# sys.stderr.write("fix: starting " + str(self.section) + '/' + str(self.file_number) + "\n")
if cloud is not None:
dirs_in_use = set()
self.modtimes = dict()
self.keydict = dict()
if not os.path.isdir(self.directory):
os.makedirs(self.directory)
#self.directory = tempfile.mkdtemp(prefix='SavedFile')
#docassemble.base.functions.this_thread.temporary_resources.add(self.directory)
prefix = str(self.section) + '/' + str(self.file_number) + '/'
#sys.stderr.write("fix: prefix is " + prefix + "\n")
for key in cloud.list_keys(prefix):
filename = os.path.join(*key.name[len(prefix):].split('/'))
fullpath = os.path.join(self.directory, filename)
fulldir = os.path.dirname(fullpath)
dirs_in_use.add(fulldir)
if not os.path.isdir(fulldir):
os.makedirs(fulldir)
server_time = key.get_epoch_modtime()
if not os.path.isfile(fullpath):
key.get_contents_to_filename(fullpath)
else:
local_time = os.path.getmtime(fullpath)
access_time = os.path.getatime(fullpath)
if self.section == 'files':
if not (local_time == server_time
and time.time() - access_time < 7000):
key.get_contents_to_filename(fullpath)
else:
if not (local_time == server_time):
key.get_contents_to_filename(fullpath)
self.modtimes[filename] = server_time
#logmessage("cloud modtime for file " + filename + " is " + str(key.last_modified))
self.keydict[filename] = key
if self.subdir and self.subdir != '' and self.subdir != 'default':
self.path = os.path.join(self.directory, self.subdir,
self.filename)
else:
self.path = os.path.join(self.directory, self.filename)
for filename in listfiles(self.directory):
if filename not in self.modtimes:
os.remove(os.path.join(self.directory, filename))
for subdir in listdirs(self.directory):
if subdir not in dirs_in_use and os.path.isdir(subdir):
shutil.rmtree(subdir)
else:
if not os.path.isdir(self.directory):
os.makedirs(self.directory)
self.fixed = True
def delete_file(self, filename):
if cloud is not None:
prefix = str(self.section) + '/' + str(self.file_number) + '/' + str(filename)
to_delete = list()
for key in cloud.list_keys(prefix):
to_delete.append(key)
for key in to_delete:
try:
key.delete()
except:
pass
the_path = os.path.join(self.directory, filename)
if hasattr(self, 'directory') and os.path.isdir(self.directory) and os.path.isfile(the_path):
os.remove(the_path)
def delete_file(self, filename):
if cloud is not None:
prefix = str(self.section) + '/' + str(self.file_number) + '/' + str(filename)
to_delete = list()
for key in cloud.list_keys(prefix):
to_delete.append(key)
for key in to_delete:
try:
key.delete()
except:
pass
the_path = os.path.join(self.directory, filename)
if hasattr(self, 'directory') and os.path.isdir(self.directory) and os.path.isfile(the_path):
os.remove(the_path)
def fix(self):
if self.fixed:
return
# sys.stderr.write("fix: starting " + str(self.section) + '/' + str(self.file_number) + "\n")
if cloud is not None:
dirs_in_use = set()
self.modtimes = {}
self.keydict = {}
if not os.path.isdir(self.directory):
os.makedirs(self.directory)
prefix = str(self.section) + '/' + str(self.file_number) + '/'
# sys.stderr.write("fix: prefix is " + prefix + "\n")
for key in cloud.list_keys(prefix):
filename = os.path.join(*key.name[len(prefix):].split('/'))
fullpath = os.path.join(self.directory, filename)
fulldir = os.path.dirname(fullpath)
dirs_in_use.add(fulldir)
if not os.path.isdir(fulldir):
os.makedirs(fulldir)
server_time = key.get_epoch_modtime()
if not os.path.isfile(fullpath):
key.get_contents_to_filename(fullpath)
else:
local_time = os.path.getmtime(fullpath)
if self.section == 'files':
if local_time != server_time:
key.get_contents_to_filename(fullpath)
update_access_time(fullpath)
else:
if local_time != server_time:
key.get_contents_to_filename(fullpath)
self.modtimes[filename] = server_time
# sys.stderr.write("cloud modtime for file " + filename + " is " + str(key.last_modified) + "\n")
self.keydict[filename] = key
if self.subdir and self.subdir != '' and self.subdir != 'default':
self.path = os.path.join(self.directory, self.subdir,
self.filename)
else:
self.path = os.path.join(self.directory, self.filename)
for filename in listfiles(self.directory):
if filename not in self.modtimes:
os.remove(os.path.join(self.directory, filename))
for subdir in listdirs(self.directory):
if subdir not in dirs_in_use and os.path.isdir(subdir):
shutil.rmtree(subdir)
else:
if not os.path.isdir(self.directory):
os.makedirs(self.directory)
self.fixed = True
def delete_directory(self, directory):
if cloud is not None:
prefix = str(self.section) + '/' + str(self.file_number) + '/' + path_to_key(directory) + '/'
to_delete = []
for key in cloud.list_keys(prefix):
to_delete.append(key)
for key in to_delete:
try:
key.delete()
except:
pass
if hasattr(self, 'directory') and os.path.isdir(self.directory):
the_path = os.path.join(self.directory, directory)
if os.path.isdir(the_path):
shutil.rmtree(the_path)
def fix(self):
if self.fixed:
return
if cloud is not None:
self.modtimes = dict()
self.keydict = dict()
self.directory = tempfile.mkdtemp(prefix='SavedFile')
prefix = str(self.section) + '/' + str(self.file_number) + '/'
#logmessage("fix: prefix is " + prefix)
for key in cloud.list_keys(prefix):
filename = re.sub(r'.*/', '', key.name)
fullpath = os.path.join(self.directory, filename)
#logmessage("fix: saving to " + fullpath)
key.get_contents_to_filename(fullpath)
self.modtimes[filename] = os.path.getmtime(fullpath)
#logmessage("cloud modtime for file " + filename + " is " + str(key.last_modified))
self.keydict[filename] = key
self.path = os.path.join(self.directory, self.filename)
else:
if not os.path.isdir(self.directory):
os.makedirs(self.directory)
self.fixed = True
