def fix(self): if self.fixed: return # sys.stderr.write("fix: starting " + str(self.section) + '/' + str(self.file_number) + "\n") if cloud is not None: self.modtimes = dict() self.keydict = dict() if not os.path.isdir(self.directory): os.makedirs(self.directory) #self.directory = tempfile.mkdtemp(prefix='SavedFile') #docassemble.base.functions.this_thread.temporary_resources.add(self.directory) prefix = str(self.section) + '/' + str(self.file_number) + '/' #sys.stderr.write("fix: prefix is " + prefix + "\n") for key in cloud.list_keys(prefix): filename = re.sub(r'.*/', '', key.name) fullpath = os.path.join(self.directory, filename) server_time = key.get_epoch_modtime() if not (os.path.isfile(fullpath) and os.path.getmtime(fullpath) == server_time): key.get_contents_to_filename(fullpath) self.modtimes[filename] = server_time #logmessage("cloud modtime for file " + filename + " is " + str(key.last_modified)) self.keydict[filename] = key self.path = os.path.join(self.directory, self.filename) to_delete = list() for filename in os.listdir(self.directory): if filename not in self.modtimes: to_delete.append(filename) for filename in to_delete: os.remove(os.path.join(self.directory, filename)) else: if not os.path.isdir(self.directory): os.makedirs(self.directory) self.fixed = True
def delete(self): if cloud is not None: prefix = str(self.section) + '/' + str(self.file_number) + '/' for key in cloud.list_keys(prefix): key.delete() if hasattr(self, 'directory') and os.path.isdir(self.directory): shutil.rmtree(self.directory)
def __init__(self, file_number, extension=None, fix=False, section='files', filename='file', subdir=None, should_not_exist=False): file_number = int(file_number) section = str(section) if section not in docassemble.base.functions.this_thread.saved_files: docassemble.base.functions.this_thread.saved_files[section] = {} if file_number in docassemble.base.functions.this_thread.saved_files[section]: # sys.stderr.write("SavedFile: using cache for " + section + '/' + str(file_number) + "\n") sf = docassemble.base.functions.this_thread.saved_files[section][file_number] for attribute in ['file_number', 'fixed', 'section', 'filename', 'extension', 'directory', 'path', 'modtimes', 'keydict', 'subdir']: if hasattr(sf, attribute): setattr(self, attribute, getattr(sf, attribute)) self.extension = extension self.filename = filename self.subdir = subdir else: # sys.stderr.write("SavedFile: not using cache for " + section + '/' + str(file_number) + "\n") self.fixed = False self.file_number = file_number self.section = section self.extension = extension self.filename = filename self.subdir = subdir if cloud is None: if self.section == 'files': parts = re.sub(r'(...)', r'\1/', '{0:012x}'.format(int(file_number))).split('/') self.directory = os.path.join(UPLOAD_DIRECTORY, *parts) else: self.directory = os.path.join(UPLOAD_DIRECTORY, str(self.section), str(file_number)) else: self.directory = os.path.join(tempfile.gettempdir(), str(self.section), str(self.file_number)) docassemble.base.functions.this_thread.saved_files[section][file_number] = self if self.subdir and self.subdir != '' and self.subdir != 'default': self.path = os.path.join(self.directory, self.subdir, self.filename) else: self.path = os.path.join(self.directory, self.filename) if fix: self.fix() if should_not_exist and os.path.isdir(self.directory): found_error = False for root, dirs, files in os.walk(self.directory): if len(files) > 0 or len(dirs) > 0: found_error = True break if found_error: logmessage("WARNING! Possible database corruption due to an unsafe shutdown. Your database indicated that the next file number is " + str(file_number) + ", but there is already a file in the file storage for that number. It is recommended that you restart your system. If that does not make this error go away, you should investigate why there are existing files in the file system.") if cloud is not None: prefix = str(self.section) + '/' + str(self.file_number) + '/' for key in list(cloud.list_keys(prefix)): try: key.delete() except: pass if hasattr(self, 'directory') and os.path.isdir(self.directory): shutil.rmtree(self.directory) if not os.path.isdir(self.directory): os.makedirs(self.directory)
def delete_file(self, filename): if cloud is not None: prefix = str(self.section) + '/' + str( self.file_number) + '/' + str(filename) for key in cloud.list_keys(prefix): key.delete() the_path = os.path.join(self.directory, filename) if hasattr(self, 'directory') and os.path.isdir( self.directory) and os.path.isfile(the_path): os.remove(the_path)
def list_of_files(self): output = list() if cloud is not None and not self.fixed: prefix = str(self.section) + '/' + str(self.file_number) + '/' for key in cloud.list_keys(prefix): output.append(os.path.join(*key.name[len(prefix):].split('/'))) else: if os.path.isdir(self.directory): for filename in listfiles(self.directory): output.append(filename) return sorted(output)
def delete(self): if cloud is not None: prefix = str(self.section) + '/' + str(self.file_number) + '/' for key in list(cloud.list_keys(prefix)): try: key.delete() except: pass if hasattr(self, 'directory') and os.path.isdir(self.directory): shutil.rmtree(self.directory) del docassemble.base.functions.this_thread.saved_files[str(self.section)][int(self.file_number)]
def list_of_files(self): output = list() if cloud is not None and not self.fixed: prefix = str(self.section) + '/' + str(self.file_number) + '/' for key in cloud.list_keys(prefix): output.append(re.sub(r'.*/', '', key.name)) else: if os.path.isdir(self.directory): for filename in os.listdir(self.directory): output.append(filename) return sorted(output)
def delete(self): if cloud is not None: prefix = str(self.section) + '/' + str(self.file_number) + '/' for key in [x for x in cloud.list_keys(prefix)]: try: key.delete() except: pass if hasattr(self, 'directory') and os.path.isdir(self.directory): shutil.rmtree(self.directory) del docassemble.base.functions.this_thread.saved_files[str(self.section)][int(self.file_number)]
def fix(self): if self.fixed: return # sys.stderr.write("fix: starting " + str(self.section) + '/' + str(self.file_number) + "\n") if cloud is not None: dirs_in_use = set() self.modtimes = dict() self.keydict = dict() if not os.path.isdir(self.directory): os.makedirs(self.directory) #self.directory = tempfile.mkdtemp(prefix='SavedFile') #docassemble.base.functions.this_thread.temporary_resources.add(self.directory) prefix = str(self.section) + '/' + str(self.file_number) + '/' #sys.stderr.write("fix: prefix is " + prefix + "\n") for key in cloud.list_keys(prefix): filename = os.path.join(*key.name[len(prefix):].split('/')) fullpath = os.path.join(self.directory, filename) fulldir = os.path.dirname(fullpath) dirs_in_use.add(fulldir) if not os.path.isdir(fulldir): os.makedirs(fulldir) server_time = key.get_epoch_modtime() if not os.path.isfile(fullpath): key.get_contents_to_filename(fullpath) else: local_time = os.path.getmtime(fullpath) access_time = os.path.getatime(fullpath) if self.section == 'files': if not (local_time == server_time and time.time() - access_time < 7000): key.get_contents_to_filename(fullpath) else: if not (local_time == server_time): key.get_contents_to_filename(fullpath) self.modtimes[filename] = server_time #logmessage("cloud modtime for file " + filename + " is " + str(key.last_modified)) self.keydict[filename] = key if self.subdir and self.subdir != '' and self.subdir != 'default': self.path = os.path.join(self.directory, self.subdir, self.filename) else: self.path = os.path.join(self.directory, self.filename) for filename in listfiles(self.directory): if filename not in self.modtimes: os.remove(os.path.join(self.directory, filename)) for subdir in listdirs(self.directory): if subdir not in dirs_in_use and os.path.isdir(subdir): shutil.rmtree(subdir) else: if not os.path.isdir(self.directory): os.makedirs(self.directory) self.fixed = True
def delete_file(self, filename): if cloud is not None: prefix = str(self.section) + '/' + str(self.file_number) + '/' + str(filename) to_delete = list() for key in cloud.list_keys(prefix): to_delete.append(key) for key in to_delete: try: key.delete() except: pass the_path = os.path.join(self.directory, filename) if hasattr(self, 'directory') and os.path.isdir(self.directory) and os.path.isfile(the_path): os.remove(the_path)
def fix(self): if self.fixed: return # sys.stderr.write("fix: starting " + str(self.section) + '/' + str(self.file_number) + "\n") if cloud is not None: dirs_in_use = set() self.modtimes = {} self.keydict = {} if not os.path.isdir(self.directory): os.makedirs(self.directory) prefix = str(self.section) + '/' + str(self.file_number) + '/' # sys.stderr.write("fix: prefix is " + prefix + "\n") for key in cloud.list_keys(prefix): filename = os.path.join(*key.name[len(prefix):].split('/')) fullpath = os.path.join(self.directory, filename) fulldir = os.path.dirname(fullpath) dirs_in_use.add(fulldir) if not os.path.isdir(fulldir): os.makedirs(fulldir) server_time = key.get_epoch_modtime() if not os.path.isfile(fullpath): key.get_contents_to_filename(fullpath) else: local_time = os.path.getmtime(fullpath) if self.section == 'files': if local_time != server_time: key.get_contents_to_filename(fullpath) update_access_time(fullpath) else: if local_time != server_time: key.get_contents_to_filename(fullpath) self.modtimes[filename] = server_time # sys.stderr.write("cloud modtime for file " + filename + " is " + str(key.last_modified) + "\n") self.keydict[filename] = key if self.subdir and self.subdir != '' and self.subdir != 'default': self.path = os.path.join(self.directory, self.subdir, self.filename) else: self.path = os.path.join(self.directory, self.filename) for filename in listfiles(self.directory): if filename not in self.modtimes: os.remove(os.path.join(self.directory, filename)) for subdir in listdirs(self.directory): if subdir not in dirs_in_use and os.path.isdir(subdir): shutil.rmtree(subdir) else: if not os.path.isdir(self.directory): os.makedirs(self.directory) self.fixed = True
def delete_directory(self, directory): if cloud is not None: prefix = str(self.section) + '/' + str(self.file_number) + '/' + path_to_key(directory) + '/' to_delete = [] for key in cloud.list_keys(prefix): to_delete.append(key) for key in to_delete: try: key.delete() except: pass if hasattr(self, 'directory') and os.path.isdir(self.directory): the_path = os.path.join(self.directory, directory) if os.path.isdir(the_path): shutil.rmtree(the_path)
def fix(self): if self.fixed: return if cloud is not None: self.modtimes = dict() self.keydict = dict() self.directory = tempfile.mkdtemp(prefix='SavedFile') prefix = str(self.section) + '/' + str(self.file_number) + '/' #logmessage("fix: prefix is " + prefix) for key in cloud.list_keys(prefix): filename = re.sub(r'.*/', '', key.name) fullpath = os.path.join(self.directory, filename) #logmessage("fix: saving to " + fullpath) key.get_contents_to_filename(fullpath) self.modtimes[filename] = os.path.getmtime(fullpath) #logmessage("cloud modtime for file " + filename + " is " + str(key.last_modified)) self.keydict[filename] = key self.path = os.path.join(self.directory, self.filename) else: if not os.path.isdir(self.directory): os.makedirs(self.directory) self.fixed = True