def cleanup_extracted_file(): log.info('Removing extracted files for file viewer.') root = os.path.join(settings.TMP_PATH, 'file_viewer') # Local storage uses local time for file modification. S3 uses UTC time. now = datetime.utcnow if storage_is_remote() else datetime.now for path in private_storage.listdir(root)[0]: full = os.path.join(root, path) age = now() - private_storage.modified_time( os.path.join(full, 'manifest.webapp')) if age.total_seconds() > (60 * 60): log.debug('Removing extracted files: %s, %dsecs old.' % (full, age.total_seconds())) for subroot, dirs, files in walk_storage(full): for f in files: private_storage.delete(os.path.join(subroot, f)) # Nuke out the file and diff caches when the file gets removed. id = os.path.basename(path) try: int(id) except ValueError: continue key = hashlib.md5() key.update(str(id)) cache.delete('%s:memoize:%s:%s' % (settings.CACHE_PREFIX, 'file-viewer', key.hexdigest()))
def rmtree(prefix): dirs, files = private_storage.listdir(prefix) for fname in files: private_storage.delete(os.path.join(prefix, fname)) for d in dirs: rmtree(os.path.join(prefix, d)) private_storage.delete(prefix)
def cleanup_extracted_file(): log.info('Removing extracted files for file viewer.') root = os.path.join(settings.TMP_PATH, 'file_viewer') # Local storage uses local time for file modification. S3 uses UTC time. now = datetime.utcnow if storage_is_remote() else datetime.now for path in private_storage.listdir(root)[0]: full = os.path.join(root, path) age = now() - private_storage.modified_time( os.path.join(full, 'manifest.webapp')) if age.total_seconds() > (60 * 60): log.debug('Removing extracted files: %s, %dsecs old.' % (full, age.total_seconds())) for subroot, dirs, files in walk_storage(full): for f in files: private_storage.delete(os.path.join(subroot, f)) # Nuke out the file and diff caches when the file gets removed. id = os.path.basename(path) try: int(id) except ValueError: continue key = hashlib.md5() key.update(str(id)) cache.delete( '%s:memoize:%s:%s' % (settings.CACHE_PREFIX, 'file-viewer', key.hexdigest()))
def iterate(path): path_dirs, path_files = private_storage.listdir(path) for dirname in sorted(path_dirs): full = os.path.join(path, dirname) all_files.append(full) iterate(full) for filename in sorted(path_files): full = os.path.join(path, filename) all_files.append(full)
def _get_files(self): all_files, res = [], OrderedDict() # Not using os.path.walk so we get just the right order. def iterate(path): path_dirs, path_files = private_storage.listdir(path) for dirname in sorted(path_dirs): full = os.path.join(path, dirname) all_files.append(full) iterate(full) for filename in sorted(path_files): full = os.path.join(path, filename) all_files.append(full) iterate(self.dest) for path in all_files: filename = smart_unicode(os.path.basename(path), errors='replace') short = smart_unicode(path[len(self.dest) + 1:], errors='replace') mime, encoding = mimetypes.guess_type(filename) if not mime and filename == 'manifest.webapp': mime = 'application/x-web-app-manifest+json' if storage_is_remote(): # S3 doesn't have directories, so we check for names with this # prefix and call it a directory if there are some. subdirs, subfiles = private_storage.listdir(path) directory = bool(subdirs or subfiles) else: directory = os.path.isdir(path) res[short] = { 'binary': self._is_binary(mime, path), 'depth': short.count(os.sep), 'directory': directory, 'filename': filename, 'full': path, 'md5': get_md5(path) if not directory else '', 'mimetype': mime or 'application/octet-stream', 'syntax': self.get_syntax(filename), 'modified': ( time.mktime( private_storage.modified_time(path).timetuple()) if not directory else 0), 'short': short, 'size': private_storage.size(path) if not directory else 0, 'truncated': self.truncate(filename), 'url': reverse('mkt.files.list', args=[self.file.id, 'file', short]), 'url_serve': reverse('mkt.files.redirect', args=[self.file.id, short]), 'version': self.file.version.version, } return res
def _get_files(self): all_files, res = [], OrderedDict() # Not using os.path.walk so we get just the right order. def iterate(path): path_dirs, path_files = private_storage.listdir(path) for dirname in sorted(path_dirs): full = os.path.join(path, dirname) all_files.append(full) iterate(full) for filename in sorted(path_files): full = os.path.join(path, filename) all_files.append(full) iterate(self.dest) for path in all_files: filename = smart_unicode(os.path.basename(path), errors='replace') short = smart_unicode(path[len(self.dest) + 1:], errors='replace') mime, encoding = mimetypes.guess_type(filename) if not mime and filename == 'manifest.webapp': mime = 'application/x-web-app-manifest+json' if storage_is_remote(): # S3 doesn't have directories, so we check for names with this # prefix and call it a directory if there are some. subdirs, subfiles = private_storage.listdir(path) directory = bool(subdirs or subfiles) else: directory = os.path.isdir(path) res[short] = { 'binary': self._is_binary(mime, path), 'depth': short.count(os.sep), 'directory': directory, 'filename': filename, 'full': path, 'md5': get_md5(path) if not directory else '', 'mimetype': mime or 'application/octet-stream', 'syntax': self.get_syntax(filename), 'modified': (time.mktime(private_storage.modified_time(path).timetuple()) if not directory else 0), 'short': short, 'size': private_storage.size(path) if not directory else 0, 'truncated': self.truncate(filename), 'url': reverse('mkt.files.list', args=[self.file.id, 'file', short]), 'url_serve': reverse('mkt.files.redirect', args=[self.file.id, short]), 'version': self.file.version.version, } return res