def remove_old_files(self):
"""
remove oldest registered files.
"""
if self.size_limit <= 0 or self.file_limit <= 0:
return
# sort by creation time, oldest first
files = sorted(self.files, key=lambda f: f.stat.st_atime)
while len(files) > self.file_limit and \
self._files_size >= self.size_limit:
f = files.pop(0)
fs.remove(f.path)
self._files_size -= f.stat.st_size
self.files = files
def remove_old_files(self):
from dlmanager import fs
files = sorted(self.files, key=lambda f: f.stat.st_atime)
kept = []
while len(files) > self.file_limit and \
self._files_size >= self.size_limit:
f = files.pop(0)
if f.path in self._downloaded_now:
kept.append(f)
continue
fs.remove(f.path)
self.log(logging.INFO, 'artifact',
{'filename': f.path},
'Purged artifact {filename}')
self._files_size -= f.stat.st_size
self.files = files + kept
def remove_old_files(self):
from dlmanager import fs
files = sorted(self.files, key=lambda f: f.stat.st_atime)
kept = []
while len(files) > self.file_limit and \
self._files_size >= self.size_limit:
f = files.pop(0)
if f.path in self._downloaded_now:
kept.append(f)
continue
fs.remove(f.path)
self.log(logging.INFO, 'artifact',
{'filename': f.path},
'Purged artifact {filename}')
self._files_size -= f.stat.st_size
self.files = files + kept
def _download(self, url, dest, finished_callback, chunk_size, session):
# save the file under a temporary name
# this allow to not use a broken file in case things went really bad
# while downloading the file (ie the python interpreter is killed
# abruptly)
temp = None
bytes_so_far = 0
try:
with closing(session.get(url, stream=True)) as response:
total_size = response.headers.get('Content-length', '').strip()
total_size = int(total_size) if total_size else None
self._update_progress(bytes_so_far, total_size)
# we use NamedTemporaryFile as raw open() call was causing
# issues on windows - see:
# https://bugzilla.mozilla.org/show_bug.cgi?id=1185756
with tempfile.NamedTemporaryFile(
delete=False,
suffix='.tmp',
dir=os.path.dirname(dest)) as temp:
for chunk in response.iter_content(chunk_size):
if self.is_canceled():
break
if chunk:
temp.write(chunk)
bytes_so_far += len(chunk)
self._update_progress(bytes_so_far, total_size)
response.raise_for_status()
except:
self.__error = sys.exc_info()
try:
if temp is None:
pass # not even opened the temp file, nothing to do
elif self.is_canceled() or self.__error:
fs.remove(temp.name)
else:
# if all goes well, then rename the file to the real dest
fs.remove(dest) # just in case it already existed
fs.move(temp.name, dest)
finally:
if finished_callback:
finished_callback(self)
def remove_old_files(self):
from dlmanager import fs
files = sorted(self.files, key=lambda f: f.stat.st_atime)
kept = []
while len(files) > self.file_limit and \
self._files_size >= self.size_limit:
f = files.pop(0)
if f.path in self._downloaded_now:
kept.append(f)
continue
try:
fs.remove(f.path)
except WindowsError:
# For some reason, on automation, we can't remove those files.
# So for now, ignore the error.
kept.append(f)
continue
self.log(logging.INFO, 'artifact', {'filename': f.path},
'Purged artifact {filename}')
self._files_size -= f.stat.st_size
self.files = files + kept
def _download(self, url, dest, finished_callback, chunk_size, session):
# save the file under a temporary name
# this allow to not use a broken file in case things went really bad
# while downloading the file (ie the python interpreter is killed
# abruptly)
temp = None
bytes_so_far = 0
try:
with closing(session.get(url, stream=True)) as response:
total_size = response.headers.get('Content-length', '').strip()
total_size = int(total_size) if total_size else None
self._update_progress(bytes_so_far, total_size)
# we use NamedTemporaryFile as raw open() call was causing
# issues on windows - see:
# https://bugzilla.mozilla.org/show_bug.cgi?id=1185756
with tempfile.NamedTemporaryFile(
delete=False, suffix='.tmp',
dir=os.path.dirname(dest)) as temp:
for chunk in response.iter_content(chunk_size):
if self.is_canceled():
break
if chunk:
temp.write(chunk)
bytes_so_far += len(chunk)
self._update_progress(bytes_so_far, total_size)
response.raise_for_status()
except:
self.__error = sys.exc_info()
try:
if temp is None:
pass # not even opened the temp file, nothing to do
elif self.is_canceled() or self.__error:
fs.remove(temp.name)
else:
# if all goes well, then rename the file to the real dest
fs.remove(dest) # just in case it already existed
fs.move(temp.name, dest)
finally:
if finished_callback:
finished_callback(self)
def remove_old_files(self):
from dlmanager import fs
files = sorted(self.files, key=lambda f: f.stat.st_atime)
kept = []
while len(files) > self.file_limit and \
self._files_size >= self.size_limit:
f = files.pop(0)
if f.path in self._downloaded_now:
kept.append(f)
continue
try:
fs.remove(f.path)
except WindowsError:
# For some reason, on automation, we can't remove those files.
# So for now, ignore the error.
kept.append(f)
continue
self.log(logging.INFO, 'artifact',
{'filename': f.path},
'Purged artifact {filename}')
self._files_size -= f.stat.st_size
self.files = files + kept
def temp():
tmp = TempCreator()
yield tmp
fs.remove(tmp.tempdir)
def temp():
tmp = TempCreator()
yield tmp
fs.remove(tmp.tempdir)
def remove_all(self):
from dlmanager import fs
for f in self.files:
fs.remove(f.path)
self._files_size = 0
self.files = []
def remove_all(self):
from dlmanager import fs
for f in self.files:
fs.remove(f.path)
self._files_size = 0
self.files = []
