def remove_old_files(self): """ remove oldest registered files. """ if self.size_limit <= 0 or self.file_limit <= 0: return # sort by creation time, oldest first files = sorted(self.files, key=lambda f: f.stat.st_atime) while len(files) > self.file_limit and \ self._files_size >= self.size_limit: f = files.pop(0) fs.remove(f.path) self._files_size -= f.stat.st_size self.files = files
def remove_old_files(self): from dlmanager import fs files = sorted(self.files, key=lambda f: f.stat.st_atime) kept = [] while len(files) > self.file_limit and \ self._files_size >= self.size_limit: f = files.pop(0) if f.path in self._downloaded_now: kept.append(f) continue fs.remove(f.path) self.log(logging.INFO, 'artifact', {'filename': f.path}, 'Purged artifact {filename}') self._files_size -= f.stat.st_size self.files = files + kept
def _download(self, url, dest, finished_callback, chunk_size, session): # save the file under a temporary name # this allow to not use a broken file in case things went really bad # while downloading the file (ie the python interpreter is killed # abruptly) temp = None bytes_so_far = 0 try: with closing(session.get(url, stream=True)) as response: total_size = response.headers.get('Content-length', '').strip() total_size = int(total_size) if total_size else None self._update_progress(bytes_so_far, total_size) # we use NamedTemporaryFile as raw open() call was causing # issues on windows - see: # https://bugzilla.mozilla.org/show_bug.cgi?id=1185756 with tempfile.NamedTemporaryFile( delete=False, suffix='.tmp', dir=os.path.dirname(dest)) as temp: for chunk in response.iter_content(chunk_size): if self.is_canceled(): break if chunk: temp.write(chunk) bytes_so_far += len(chunk) self._update_progress(bytes_so_far, total_size) response.raise_for_status() except: self.__error = sys.exc_info() try: if temp is None: pass # not even opened the temp file, nothing to do elif self.is_canceled() or self.__error: fs.remove(temp.name) else: # if all goes well, then rename the file to the real dest fs.remove(dest) # just in case it already existed fs.move(temp.name, dest) finally: if finished_callback: finished_callback(self)
def remove_old_files(self): from dlmanager import fs files = sorted(self.files, key=lambda f: f.stat.st_atime) kept = [] while len(files) > self.file_limit and \ self._files_size >= self.size_limit: f = files.pop(0) if f.path in self._downloaded_now: kept.append(f) continue try: fs.remove(f.path) except WindowsError: # For some reason, on automation, we can't remove those files. # So for now, ignore the error. kept.append(f) continue self.log(logging.INFO, 'artifact', {'filename': f.path}, 'Purged artifact {filename}') self._files_size -= f.stat.st_size self.files = files + kept
def temp(): tmp = TempCreator() yield tmp fs.remove(tmp.tempdir)
def remove_all(self): from dlmanager import fs for f in self.files: fs.remove(f.path) self._files_size = 0 self.files = []