def test_safe_rmtree_directory_scrub_directory_failure(self, mock_os_system, mock_exists, mock_islink, mock_isfile):
""" Tests that the safe_rmtree function returns a GeneralException when the path parameter is a directory
and the scrub of the directories in the directory fails.
"""
try:
safe_rmtree('/mnt/folder', False, True)
raise Exception('A GenericError was not raised during the test')
except koji.GenericError as e:
self.assertEquals(e.message, 'dir removal failed (code 1) for /mnt/folder')
def test_safe_rmtree_directory_scrub_directory_failure(self, mock_os_system, mock_exists, mock_islink, mock_isfile):
""" Tests that the safe_rmtree function returns a GeneralException when the path parameter is a directory
and the scrub of the directories in the directory fails.
"""
try:
safe_rmtree('/mnt/folder', False, True)
raise Exception('A GenericError was not raised during the test')
except koji.GenericError as e:
self.assertEquals(e.message, 'dir removal failed (code 1) for /mnt/folder')
def test_rmtree_directory_scrub_file_failure(self, rmtree, remove, islink, isfile, exists):
""" Tests that the koji.util.rmtree function returns a GeneralException when the path parameter is a directory
and the scrub of the files in the directory fails.
"""
rmtree.side_effect = koji.GenericError('xyz')
path = '/mnt/folder'
try:
safe_rmtree(path, False, 1)
raise Exception('A GenericError was not raised during the test')
except koji.GenericError as e:
self.assertEquals(e.args[0], 'xyz')
isfile.assert_called_once_with(path)
islink.assert_called_once_with(path)
exists.assert_called_once_with(path)
remove.assert_not_called()
rmtree.assert_called_once_with(path)
def test_rmtree_directory_scrub_file_failure(self, rmtree, remove, islink, isfile, exists):
""" Tests that the koji.util.rmtree function returns a GeneralException when the path parameter is a directory
and the scrub of the files in the directory fails.
"""
rmtree.side_effect = koji.GenericError('xyz')
path = '/mnt/folder'
try:
safe_rmtree(path, False, 1)
raise Exception('A GenericError was not raised during the test')
except koji.GenericError as e:
self.assertEquals(e.args[0], 'xyz')
isfile.assert_called_once_with(path)
islink.assert_called_once_with(path)
exists.assert_called_once_with(path)
remove.assert_not_called()
rmtree.assert_called_once_with(path)
def test_rmtree_directory(self, rmtree, remove, islink, isfile, exists):
""" Tests that the koji.util.rmtree function returns nothing when the path is a directory.
"""
path = '/mnt/folder'
self.assertEquals(safe_rmtree(path, False, True), 0)
isfile.assert_called_once_with(path)
islink.assert_called_once_with(path)
exists.assert_called_once_with(path)
remove.assert_not_called()
rmtree.assert_called_once_with(path)
def test_rmtree_does_not_exist(self, rmtree, remove, islink, isfile, exists):
""" Tests that the koji.util.rmtree function returns nothing if the path does not exist.
"""
path = '/mnt/folder/some_file'
self.assertEquals(safe_rmtree(path, False, True), 0)
isfile.assert_called_once_with(path)
islink.assert_called_once_with(path)
exists.assert_called_once_with(path)
remove.assert_not_called()
rmtree.assert_not_called()
def test_rmtree_does_not_exist(self, rmtree, remove, islink, isfile, exists):
""" Tests that the koji.util.rmtree function returns nothing if the path does not exist.
"""
path = '/mnt/folder/some_file'
self.assertEquals(safe_rmtree(path, False, True), 0)
isfile.assert_called_once_with(path)
islink.assert_called_once_with(path)
exists.assert_called_once_with(path)
remove.assert_not_called()
rmtree.assert_not_called()
def test_rmtree_directory(self, rmtree, remove, islink, isfile, exists):
""" Tests that the koji.util.rmtree function returns nothing when the path is a directory.
"""
path = '/mnt/folder'
self.assertEquals(safe_rmtree(path, False, True), 0)
isfile.assert_called_once_with(path)
islink.assert_called_once_with(path)
exists.assert_called_once_with(path)
remove.assert_not_called()
rmtree.assert_called_once_with(path)
def test_safe_rmtree_file(self, rmtree, remove, islink, isfile, exists):
""" Tests that the koji.util.rmtree function returns nothing when
the path parameter is a file.
"""
path = '/mnt/folder/some_file'
self.assertEqual(safe_rmtree(path, False, True), 0)
isfile.assert_called_once_with(path)
islink.assert_not_called()
exists.assert_not_called()
remove.assert_called_once_with(path)
rmtree.assert_not_called()
def test_safe_rmtree_directory(self, mock_os_system, mock_exists, mock_islink, mock_isfile):
""" Tests that the safe_rmtree function returns nothing when the path is a directory.
"""
self.assertEquals(safe_rmtree('/mnt/folder', False, True), 0)
def updateBuildroots(self):
"""Handle buildroot cleanup/maintenance
- examine current buildroots on system
- compare with db
- clean up as needed
- /var/lib/mock
- /etc/mock/koji
"""
local_br = self._scanLocalBuildroots()
#query buildroots in db that are not expired
states = [ koji.BR_STATES[x] for x in ('INIT','WAITING','BUILDING') ]
db_br = self.session.listBuildroots(hostID=self.host_id,state=tuple(states))
# index by id
db_br = dict([(row['id'],row) for row in db_br])
st_expired = koji.BR_STATES['EXPIRED']
for id, br in db_br.items():
task_id = br['task_id']
if task_id is None:
# not associated with a task
# this makes no sense now, but may in the future
self.logger.warn("Expiring taskless buildroot: %(id)i/%(tag_name)s/%(arch)s" % br)
self.session.host.setBuildRootState(id,st_expired)
elif not self.tasks.has_key(task_id):
#task not running - expire the buildroot
#TODO - consider recycling hooks here (with strong sanity checks)
self.logger.info("Expiring buildroot: %(id)i/%(tag_name)s/%(arch)s" % br)
self.logger.debug("Buildroot task: %r, Current tasks: %r" % (task_id,self.tasks.keys()))
self.session.host.setBuildRootState(id,st_expired)
continue
# get info on local_only buildroots (most likely expired)
local_only = [id for id in local_br.iterkeys() if not db_br.has_key(id)]
if local_only:
missed_br = self.session.listBuildroots(buildrootID=tuple(local_only))
#get all the task info in one call
tasks = []
for br in missed_br:
task_id = br['task_id']
if task_id:
tasks.append(task_id)
#index
missed_br = dict([(row['id'],row) for row in missed_br])
tasks = dict([(row['id'],row) for row in self.session.getTaskInfo(tasks)])
for id in local_only:
# Cleaning options
# - wait til later
# - "soft" clean (leaving empty root/ dir)
# - full removal
data = local_br[id]
br = missed_br.get(id)
if not br:
self.logger.warn("%(name)s: not in db" % data)
continue
desc = "%(id)i/%(tag_name)s/%(arch)s" % br
if not br['retire_ts']:
self.logger.warn("%s: no retire timestamp" % desc)
continue
age = time.time() - br['retire_ts']
self.logger.debug("Expired/stray buildroot: %s" % desc)
if br and br['task_id']:
task = tasks.get(br['task_id'])
if not task:
self.logger.warn("%s: invalid task %s" % (desc, br['task_id']))
continue
if (task['state'] == koji.TASK_STATES['FAILED'] and age < 3600 * 4):
#XXX - this could be smarter
# keep buildroots for failed tasks around for a little while
self.logger.debug("Keeping failed buildroot: %s" % desc)
continue
topdir = data['dir']
rootdir = None
if topdir:
rootdir = "%s/root" % topdir
try:
st = os.lstat(rootdir)
except OSError, e:
if e.errno == errno.ENOENT:
rootdir = None
else:
self.logger.warn("%s: %s" % (desc, e))
continue
else:
age = min(age, time.time() - st.st_mtime)
#note: https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=192153)
#If rpmlib is installing in this chroot, removing it entirely
#can lead to a world of hurt.
#We remove the rootdir contents but leave the rootdir unless it
#is really old
if age > 3600*24:
#dir untouched for a day
self.logger.info("Removing buildroot: %s" % desc)
if topdir and safe_rmtree(topdir, unmount=True, strict=False) != 0:
continue
#also remove the config
try:
os.unlink(data['cfg'])
except OSError, e:
self.logger.warn("%s: can't remove config: %s" % (desc, e))
elif age > 120:
if rootdir:
try:
flist = os.listdir(rootdir)
except OSError, e:
self.logger.warn("%s: can't list rootdir: %s" % (desc, e))
continue
if flist:
self.logger.info("%s: clearing rootdir" % desc)
for fn in flist:
safe_rmtree("%s/%s" % (rootdir,fn), unmount=True, strict=False)
resultdir = "%s/result" % topdir
if os.path.isdir(resultdir):
self.logger.info("%s: clearing resultdir" % desc)
safe_rmtree(resultdir, unmount=True, strict=False)
def test_safe_rmtree_directory(self, mock_os_system, mock_exists,
mock_islink, mock_isfile):
""" Tests that the safe_rmtree function returns nothing when the path is a directory.
"""
self.assertEquals(safe_rmtree('/mnt/folder', False, True), 0)
def test_safe_rmtree_does_not_exist(self, mock_exists, mock_islink,
mock_isfile):
""" Tests that the safe_rmtree function returns nothing if the path does not exist.
"""
self.assertEquals(safe_rmtree('/mnt/folder/some_file', False, True),
None)
def test_safe_rmtree_link(self, mock_remove, mock_islink, mock_isfile):
""" Tests that the safe_rmtree function returns nothing when the path parameter is a link.
"""
self.assertEquals(safe_rmtree('/mnt/folder/some_link', False, True),
None)
def test_safe_rmtree_does_not_exist(self, mock_exists, mock_islink, mock_isfile):
""" Tests that the safe_rmtree function returns nothing if the path does not exist.
"""
self.assertEquals(safe_rmtree('/mnt/folder/some_file', False, True), None)
def test_safe_rmtree_link(self, mock_remove, mock_islink, mock_isfile):
""" Tests that the safe_rmtree function returns nothing when the path parameter is a link.
"""
self.assertEquals(safe_rmtree('/mnt/folder/some_link', False, True), None)
def updateBuildroots(self, nolocal=False):
"""Handle buildroot cleanup/maintenance
- examine current buildroots on system
- compare with db
- clean up as needed
- /var/lib/mock
- /etc/mock/koji
If nolocal is True, do not try to scan local buildroots.
"""
#query buildroots in db that are not expired
states = [koji.BR_STATES[x] for x in ('INIT', 'WAITING', 'BUILDING')]
db_br = self.session.listBuildroots(hostID=self.host_id, state=tuple(states))
# index by id
db_br = dict([(row['id'], row) for row in db_br])
st_expired = koji.BR_STATES['EXPIRED']
for id, br in db_br.items():
task_id = br['task_id']
if task_id is None:
# not associated with a task
# this makes no sense now, but may in the future
self.logger.warn("Expiring taskless buildroot: %(id)i/%(tag_name)s/%(arch)s" % br)
self.session.host.setBuildRootState(id, st_expired)
elif task_id not in self.tasks:
#task not running - expire the buildroot
#TODO - consider recycling hooks here (with strong sanity checks)
self.logger.info("Expiring buildroot: %(id)i/%(tag_name)s/%(arch)s" % br)
self.logger.debug("Buildroot task: %r, Current tasks: %r" % (task_id, self.tasks.keys()))
self.session.host.setBuildRootState(id, st_expired)
continue
if nolocal:
return
local_br = self._scanLocalBuildroots()
# get info on local_only buildroots (most likely expired)
local_only = [id for id in local_br.iterkeys() if id not in db_br]
if local_only:
missed_br = self.session.listBuildroots(buildrootID=tuple(local_only))
#get all the task info in one call
tasks = []
for br in missed_br:
task_id = br['task_id']
if task_id:
tasks.append(task_id)
#index
missed_br = dict([(row['id'], row) for row in missed_br])
tasks = dict([(row['id'], row) for row in self.session.getTaskInfo(tasks)])
for id in local_only:
# Cleaning options
# - wait til later
# - "soft" clean (leaving empty root/ dir)
# - full removal
data = local_br[id]
br = missed_br.get(id)
if not br:
self.logger.warn("%(name)s: not in db" % data)
continue
desc = "%(id)i/%(tag_name)s/%(arch)s" % br
if not br['retire_ts']:
self.logger.warn("%s: no retire timestamp" % desc)
continue
age = time.time() - br['retire_ts']
self.logger.debug("Expired/stray buildroot: %s" % desc)
if br and br['task_id']:
task = tasks.get(br['task_id'])
if not task:
self.logger.warn("%s: invalid task %s" % (desc, br['task_id']))
continue
if (task['state'] == koji.TASK_STATES['FAILED'] and age < self.options.failed_buildroot_lifetime):
#XXX - this could be smarter
# keep buildroots for failed tasks around for a little while
self.logger.debug("Keeping failed buildroot: %s" % desc)
continue
topdir = data['dir']
rootdir = None
if topdir:
rootdir = "%s/root" % topdir
try:
st = os.lstat(rootdir)
except OSError, e:
if e.errno == errno.ENOENT:
rootdir = None
else:
self.logger.warn("%s: %s" % (desc, e))
continue
else:
age = min(age, time.time() - st.st_mtime)
#note: https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=192153)
#If rpmlib is installing in this chroot, removing it entirely
#can lead to a world of hurt.
#We remove the rootdir contents but leave the rootdir unless it
#is really old
if age > 3600*24:
#dir untouched for a day
self.logger.info("Removing buildroot: %s" % desc)
if topdir and safe_rmtree(topdir, unmount=True, strict=False) != 0:
continue
#also remove the config
try:
os.unlink(data['cfg'])
except OSError, e:
self.logger.warn("%s: can't remove config: %s" % (desc, e))
elif age > 120:
if rootdir:
try:
flist = os.listdir(rootdir)
except OSError, e:
self.logger.warn("%s: can't list rootdir: %s" % (desc, e))
continue
if flist:
self.logger.info("%s: clearing rootdir" % desc)
for fn in flist:
safe_rmtree("%s/%s" % (rootdir, fn), unmount=True, strict=False)
resultdir = "%s/result" % topdir
if os.path.isdir(resultdir):
self.logger.info("%s: clearing resultdir" % desc)
safe_rmtree(resultdir, unmount=True, strict=False)