def test_cleanupOrphanedPackages2(self):
from agent.lib.utils import rchown
from agent.controllers.service import ServiceController
from agent.controllers.manifest import ManifestController
from agent.tests.unit.test_util import createManifest
if os.name == 'nt':
pass
else:
serviceName = 'service2'
manifestName = 'manifestA'
serviceName1 = 'service3'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"
],
service=serviceName1,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
installedPkgPath = os.path.realpath(
os.path.join(ServiceController.serviceRootPath(), serviceName,
'installed-packages'))
installedPkgPath1 = os.path.realpath(
os.path.join(ServiceController.serviceRootPath(), serviceName1,
'installed-packages'))
if (not os.name == 'nt'):
import pwd
uname = pylons.config['agent_user_account']
uid = pwd.getpwnam(uname).pw_uid
gid = pwd.getpwnam(uname).pw_gid
rchown(installedPkgPath, uid, gid)
rchown(installedPkgPath1, uid, gid)
self.assertEquals(len(os.listdir(installedPkgPath)), 1)
self.assertEquals(len(os.listdir(installedPkgPath1)), 1)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName,
manifest=manifestName),
expect_errors=True)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName1,
manifest=manifestName),
expect_errors=True)
pylons.config['packageMgr_install_package_age'] = 0
pylons.config['packageMgr_install_package_min_age'] = 0.0
PackageUtil.cleanupOrphanedPackages()
self.assertEquals(len(os.listdir(installedPkgPath)), 0)
self.assertEquals(len(os.listdir(installedPkgPath1)), 0)
def test_cleanupOrphanedPackages(self):
from agent.lib.utils import rchown
from agent.controllers.service import ServiceController
from agent.controllers.manifest import ManifestController
from agent.tests.unit.test_util import createManifest
if os.name == 'nt':
pass
else:
serviceName = 'service2'
manifestName = 'manifestA'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
installedPkgPath = os.path.realpath(
os.path.join(ServiceController.serviceRootPath(), serviceName,
'installed-packages'))
if (not os.name == 'nt'):
import pwd
uname = pylons.config['agent_user_account']
uid = pwd.getpwnam(uname).pw_uid
gid = pwd.getpwnam(uname).pw_gid
rchown(installedPkgPath, uid, gid)
#installedPkgPath = os.path.realpath(os.path.join('/ebay/srengarajan/e2e/local-deploy/service_nodes/' , '.sbe.appService.SI1', 'installed-packages' ))
#pkgs = os.listdir('C:\\srini\\projects\\gc\\agent\\trunk')
self.assertEquals(len(os.listdir(installedPkgPath)), 1)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName,
manifest=manifestName),
expect_errors=True)
pylons.config['packageMgr_install_package_age'] = 0
pylons.config['packageMgr_install_package_min_age'] = 0.0
time.sleep(1)
PackageUtil.cleanupOrphanedPackages()
self.assertEquals(len(os.listdir(installedPkgPath)), 0)
def run(self):
"""
the garbage collection thread body
go through the list of packages and delete packages over a certain threshold
two variables affect operation
packageMgr_gc_freq = how log GC sleeps for between checks
pacageMgr_package_age = how old a package can get to before it's deleted
go through the list of inprogress threads and remove all threads that are done
"""
#pylint:disable=R0914, R0912, R0915
while (self.__stop == False):
time.sleep(self.__garbageFreq)
try:
# GC in progress hash
self.__inProgressLock.acquire()
try:
for key in self.__inProgressPackages.keys():
if (not self.__inProgressPackages[key].isAlive()):
del self.__inProgressPackages[key]
finally:
self.__inProgressLock.release()
if not self.__stop:
# go through all the packages and see if they are older than the threshold
for filename in glob.glob(
os.path.join(PackageMgr.packagePath(),
'*.cronus.inprogress')):
if (time.time() > os.path.getatime(filename) + float(
pylons.config['packageMgr_package_age'])):
try:
GCLOG.info('Garbage collecting package(%s)' %
filename)
LOG.info('Garbage collecting package(%s)' %
filename)
os.remove(filename)
except OSError, osErr:
GCLOG.error(
'Unable to garbage collect %s - %s' %
(filename, osErr))
LOG.error('Unable to garbage collect %s - %s' %
(filename, osErr))
for filename in glob.glob(
os.path.join(PackageMgr.packagePath(),
'*.cronus')):
LOG.debug(
'garbage collection check in progress for filename %s'
% filename)
try:
if (time.time(
) > os.path.getatime(filename) + float(
pylons.config['packageMgr_package_age'])):
LOG.info('Garbage collecting package(%s)' %
filename)
os.remove(filename)
if os.path.exists(filename + '.prop'):
os.remove(filename + '.prop')
if os.path.exists(filename + '.torrent'):
os.remove(filename + '.torrent')
if os.path.exists(filename + '.inprogress'):
os.remove(filename + '.inprogress')
except OSError, osErr:
GCLOG.error('Unable to garbage collect %s - %s' %
(filename, osErr))
LOG.error('Unable to garbage collect %s - %s' %
(filename, osErr))
#appGlobal = pylons.config['pylons.app_globals']
#import pdb;pdb.set_trace();
if not self.__stop:
PackageUtil.cleanupOrphanedPackages(False)
packageMount = pylons.config['agent_root']
if (not self.__stop
and agenthealth.needAggressiveGC(packageMount)):
PackageUtil.forceCleanUpDownloadedPkgs()
# validate all services and remove rogue services
PackageMgr.cleanRogueService()
def run(self):
"""
the garbage collection thread body
go through the list of packages and delete packages over a certain threshold
two variables affect operation
packageMgr_gc_freq = how log GC sleeps for between checks
pacageMgr_package_age = how old a package can get to before it's deleted
go through the list of inprogress threads and remove all threads that are done
"""
#pylint:disable=R0914, R0912, R0915
while (self.__stop == False):
time.sleep(self.__garbageFreq)
try:
# GC in progress hash
self.__inProgressLock.acquire()
try:
for key in self.__inProgressPackages.keys():
if (not self.__inProgressPackages[key].isAlive()):
del self.__inProgressPackages[key]
finally:
self.__inProgressLock.release()
if not self.__stop:
# go through all the packages and see if they are older than the threshold
for filename in glob.glob(os.path.join(PackageMgr.packagePath(), '*.cronus.inprogress')):
if (time.time() > os.path.getatime(filename) + float(pylons.config['packageMgr_package_age'])):
try:
GCLOG.info('Garbage collecting package(%s)' % filename)
LOG.info('Garbage collecting package(%s)' % filename)
os.remove(filename)
except OSError, osErr:
GCLOG.error('Unable to garbage collect %s - %s' % (filename, osErr))
LOG.error('Unable to garbage collect %s - %s' % (filename, osErr))
for filename in glob.glob(os.path.join(PackageMgr.packagePath(), '*.cronus')):
LOG.debug('garbage collection check in progress for filename %s' % filename)
try:
if (time.time() > os.path.getatime(filename) + float(pylons.config['packageMgr_package_age'])):
LOG.info('Garbage collecting package(%s)' % filename)
os.remove(filename)
if os.path.exists(filename + '.prop'):
os.remove(filename + '.prop')
if os.path.exists(filename + '.torrent'):
os.remove(filename + '.torrent')
if os.path.exists(filename + '.inprogress'):
os.remove(filename + '.inprogress')
except OSError, osErr:
GCLOG.error('Unable to garbage collect %s - %s' % (filename, osErr))
LOG.error('Unable to garbage collect %s - %s' % (filename, osErr))
#appGlobal = pylons.config['pylons.app_globals']
#import pdb;pdb.set_trace();
if not self.__stop:
PackageUtil.cleanupOrphanedPackages(False)
packageMount = pylons.config['agent_root']
if(not self.__stop and agenthealth.needAggressiveGC(packageMount)):
PackageUtil.forceCleanUpDownloadedPkgs()
# validate all services and remove rogue services
PackageMgr.cleanRogueService()
