def __init__(self, threadMgr, uuid):
""" Constructor """
AgentThread.__init__(self,
threadMgr,
cat=[uuid],
name='cancel_agentthread')
self.__uuid = uuid
def __init__(self, threadMgr, service, manifest, name = 'agent_thread', parentId = None):
""" Constructor """
AgentThread.__init__(self, threadMgr, cat = [manifestutil.serviceCat(service)], name = name, parentId = parentId)
self.__manifest = manifest
self.__service = service
contextutils.injectcontext(self, {'service':service})
self._LOG = manifestutil.getServiceLogger(self, logging.getLogger(__name__))
def __init__(self,
threadMgr,
packageUri,
packageloc,
path=None,
category=None,
skipProp=True):
# cat = 'DC_DowloadThread' + str(datetime.now())
AgentThread.__init__(self,
threadMgr,
cat=category,
name='download_thread')
self.__path = pylons.config['repo_root']
if path is not None:
self.__path = path
# check to see if the package path exists
if (not os.path.isdir(self.__path)):
msg = 'Package path(%s) does not exist' % self.__path
LOG.error(msg)
raise AgentException(Errors.PACKAGE_PATH_ERROR, msg)
self.__uriDict = PackageUtil.parseUri(packageUri, self.__path,
packageloc)
self.__prop = {}
self.__error = None
self.__progress = 0.0
self.__timeouts = None
self.__skipProp = skipProp
def __init__(self, threadMgr, service, manifest):
""" Constructor """
from agent.controllers.service import ServiceController
AgentThread.__init__(self, threadMgr, cat = [ServiceController.serviceCat(service)], name = 'delete_manifest')
self.service = service
self.manifest = manifest
def __init__(self, threadMgr, wisbVersion, wisbSource = None):
""" Constructor """
AgentThread.__init__(self, threadMgr, cat = [AgentUpdate.CAT], name = AgentUpdate.THREAD_NAME)
self.__wisbVersion = wisbVersion
self.__wisbSource = wisbSource
self.__packages = []
self.__service = 'agent'
self.__manifest = ('agent_selfupdate-%s' % self.__wisbVersion)
def __init__(self, threadMgr, service, path):
""" Constructor """
AgentThread.__init__(self,
threadMgr,
cat=[manifestutil.serviceCat(service)],
name='delete_service')
self.__path = path
self.service = service
def __init__(self, threadMgr, module):
""" Constructor """
AgentThread.__init__(self,
threadMgr,
cat=[manifestutil.serviceCat('agent')],
name='delete_module')
self.__module = module
self.__service = 'agent'
def __init__(self, threadMgr, service, manifest, name = 'agent_thread'):
""" Constructor """
AgentThread.__init__(self, threadMgr, cat = [manifestutil.serviceCat(service)], name = name)
self.__manifest = manifest
self.__service = service
injectcontext(self, {'service':service})
# maintain a stack that has all the recovery script paths
# this stack is protected
self._stack = []
def __init__(self, threadMgr, stopOnly=False):
""" Constructor """
AgentThread.__init__(self,
threadMgr,
cat=[ServicesCleanup.CAT],
name='cleanup_service')
self.__killTimeout = configutil.getConfigAsInt(
'agent_thread_timeout') #kill -9 should take no time
self.__stopOnly = stopOnly
def __init__(self, threadMgr, service, manifest):
""" Constructor """
from agent.controllers.service import ServiceController
AgentThread.__init__(self,
threadMgr,
cat=[ServiceController.serviceCat(service)],
name='delete_manifest')
self.service = service
self.manifest = manifest
def _updateHttpProgress(self, amount_read):
""" custom progress computation """
if not self.__skipProp:
progress = 2 + ((float(amount_read) / float(self.__prop['size'])) * (97))
AgentThread._updateProgress(self, progress)
else:
progress = min(self.getProgress(), 97) + 1
AgentThread._updateProgress(self, progress)
self._checkStop()
def _updateHttpProgress(self, amount_read):
""" custom progress computation """
if not self.__skipProp:
progress = 2 + ((float(amount_read) / float(self.__prop['size'])) *
(97))
AgentThread._updateProgress(self, progress)
else:
progress = min(self.getProgress(), 97) + 1
AgentThread._updateProgress(self, progress)
self._checkStop()
def __init__(self, threadMgr, service, manifest, name='agent_thread'):
""" Constructor """
AgentThread.__init__(self,
threadMgr,
cat=[manifestutil.serviceCat(service)],
name=name)
self.__manifest = manifest
self.__service = service
injectcontext(self, {'service': service})
# maintain a stack that has all the recovery script paths
# this stack is protected
self._stack = []
def __init__(self, threadMgr, wisbVersion, wisbSource=None, skipProp=True):
""" Constructor """
AgentThread.__init__(self,
threadMgr,
cat=[AgentUpdate.CAT],
name=AgentUpdate.THREAD_NAME)
self.__wisbVersion = wisbVersion
self.__wisbSource = wisbSource
self.__packages = []
self.__service = 'agent'
self.__manifest = ('agent_selfupdate-%s' % self.__wisbVersion)
self.__skipProp = skipProp
def _runExecThreads(self, execThreads, minProgress, maxProgress):
""" run exec threads """
if (len(execThreads) == 0):
self._updateProgress(maxProgress)
return
# start with min progress
self._updateProgress(minProgress)
# now wait for the threads to complete and update progress
while (True):
self._checkStop()
progress = 0
running = False
for execThread in execThreads:
self.extendTimeout(execThread.getTimeout())
status = AgentThread._runExeThread(self, execThread)
progress += status['progress']
if (not running):
break
# update the progress
percent = progress / len(execThreads)
self._updateProgress(calcProgress(minProgress, maxProgress, float(percent) / 100))
time.sleep(float(pylons.config['exec_thread_sleep_time']))
self._updateProgress(maxProgress)
def __init__(self, threadMgr, packageUri, packageloc, path = None, category = None, skipProp = True):
# cat = 'DC_DowloadThread' + str(datetime.now())
AgentThread.__init__(self, threadMgr, cat = category, name = 'download_thread')
self.__path = pylons.config['repo_root']
if path is not None:
self.__path = path
# check to see if the package path exists
if (not os.path.isdir(self.__path)):
msg = 'Package path(%s) does not exist' % self.__path
LOG.error(msg)
raise AgentException(Errors.PACKAGE_PATH_ERROR, msg)
self.__uriDict = PackageUtil.parseUri(packageUri, self.__path, packageloc)
self.__prop = {}
self.__error = None
self.__progress = 0.0
self.__timeouts = None
self.__skipProp = skipProp
def __init__(self, threadMgr, packageUri, packageloc, path = None, category = None, parentId = None):
AgentThread.__init__(self, threadMgr, cat = category, name = 'download_thread', parentId = parentId)
self._mergeOnFound = True
self.__path = pylons.config['repo_root']
if path is not None:
self.__path = path
# check to see if the package path exists
if (not os.path.isdir(self.__path)):
msg = 'Package path(%s) does not exist' % self.__path
LOG.error(msg)
raise AgentException(Errors.PACKAGE_PATH_ERROR, msg)
self.__uriDict = PackageUtil.parseUri(packageUri, self.__path, packageloc)
self.__prop = {}
self.__error = None
self.__progress = 0.0
self.__timeouts = None
self.__skipProp = configutil.getConfigAsBool('download_skip_prop')
def __init__(self, threadMgr, cmd, cat = None, parentId = None):
""" Constructor.
@param threadMgr - the thread Mgr object
@param cmd - cmd list
@param timeout - amount of time we give to this thread
"""
AgentThread.__init__(self, threadMgr, cat, name = 'exec_thread', parentId = parentId)
self.__cmd = cmd
# default timeouts configuration
self._timeout = int(config['exec_thread_timeout'])
self._progressTimeout = int(config['exec_thread_progress_timeout'])
self.__service = None
self.__cmdProcess = None
self.__response = {}
self.__partial_response = None
self.__partial = ''
self.__outputlines = []
self.__injectctx = True
self.__logLevel = 'info'
self.__LOG = None
def __init__(self, threadMgr, cmd, cat = None):
""" Constructor.
@param threadMgr - the thread Mgr object
@param cmd - cmd list
@param timeout - amount of time we give to this thread
"""
if threadMgr is None:
threadMgr = NullThreadMgr()
AgentThread.__init__(self, threadMgr, cat, name = 'exec_thread')
self.__cmd = cmd
# default timeouts configuration
self._timeout = int(config['exec_thread_timeout'])
self._progressTimeout = int(config['exec_thread_progress_timeout'])
self.__cmdProcess = None
self.__response = {}
self.__partial_response = None
self.__partial = ''
self.__logLevel = 'info'
service = 'agent'
self.__LOG = manifestutil.getServiceLogger(self, logging.getLogger(__name__), service)
def _runExecThreads(self, execThreads, minProgress, maxProgress):
""" run exec threads """
if (len(execThreads) == 0):
self._updateProgress(maxProgress)
return
# start with min progress
self._updateProgress(minProgress)
# now wait for the threads to complete and update progress
firstTime = True
while (True):
self._checkStop()
progress = 0
running = False
for (execThread, dummy) in execThreads:
#first time, push recovery stack
if firstTime:
self._pushRecStack(execThread.getCmd())
if dummy:
continue
self.extendTimeout(execThread.getTimeout())
status = AgentThread._runExeThread(self, (execThread, dummy))
progress += status['progress']
firstTime = False
if (not running):
break
# update the progress
percent = progress / len(execThreads)
self._updateProgress(
calcProgress(minProgress, maxProgress,
float(percent) / 100))
time.sleep(float(pylons.config['exec_thread_sleep_time']))
self._updateProgress(maxProgress)
def _runExecThreads(self, execThreads, minProgress, maxProgress):
""" run exec threads """
if (len(execThreads) == 0):
self._updateProgress(maxProgress)
return
# start with min progress
self._updateProgress(minProgress)
# now wait for the threads to complete and update progress
firstTime = True
while (True):
self._checkStop()
progress = 0
running = False
for (execThread, dummy) in execThreads:
#first time, push recovery stack
if firstTime:
self._pushRecStack(execThread.getCmd())
if dummy:
continue
self.extendTimeout(execThread.getTimeout())
status = AgentThread._runExeThread(self, (execThread, dummy))
progress += status['progress']
firstTime = False
if (not running):
break
# update the progress
percent = progress / len(execThreads)
self._updateProgress(calcProgress(minProgress, maxProgress, float(percent) / 100))
time.sleep(float(pylons.config['exec_thread_sleep_time']))
self._updateProgress(maxProgress)
def __init__(self, threadMgr, cat=None):
AgentThread.__init__(self, threadMgr, cat)
def __init__(self, threadMgr, uuid):
""" Constructor """
AgentThread.__init__(self, threadMgr, cat = [uuid], name = 'cancel_agentthread')
self.__uuid = uuid
def stop(self):
""" stopping client before calling the super method """
LOG.info('STOP download thread is called stopping')
AgentThread.stop(self)
def __startDownload(self):
""" actual download logic """
try:
LOG.info("Starting package download for package %s" %
self.__uriDict['package'])
# check to see if there's an in progress file,
# since PackageMgr guarantees that duplicate threads will not be spawned
# for same pkg, assume an existing thread was killed.
# attempt to clean up package n move
if (os.path.exists(self.__uriDict['inProgressPackagePath'])):
LOG.debug(
'In progress file (%s) already exists. Will validate and reattempt download if necessary'
% self.__uriDict['inProgressPackagePath'])
if os.path.exists(self.__uriDict['packagePath']):
if (os.path.exists(self.__uriDict['propPath']) and
PackageUtil.validateProp(self.__uriDict['propPath'])
and PackageUtil.validatePackage(
self.__uriDict['packagePath'],
self.__uriDict['propPath'])):
msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict[
'packagePath']
LOG.info(msg)
os.utime(self.__uriDict['packagePath'], None)
os.utime(self.__uriDict['propPath'], None)
self._updateStatus(progress=100)
# NOTE: this is a normal exit not an error!
return
LOG.warning(
'The package already exists. However package prop (%s) failed validation. Downloading package.'
% self.__uriDict['propPath'])
# Delete all traces of package before beginning download
LOG.info('Cleaning up all packages for %s ' %
self.__uriDict['packagePath'])
PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'],
self.__uriDict['packagePath'],
self.__uriDict['propPath'])
AgentThread._updateProgress(self, 0)
if not self.__skipProp:
# First, download .prop file
LOG.info(
'Starting download of prop file %s - %s' %
(self.__uriDict['propUri'], self.__uriDict['propPath']))
self.__download_prop_file()
try:
self.__prop = loadPropFile(self.__uriDict['propPath'])
except FileNotFoundError:
raise AgentException(
Errors.DC_MISSING_PROP_FILE,
'Prop file (%s) unable to read or did not parse' %
(self.__uriDict['propPath']))
AgentThread._updateProgress(self, 2)
self.__setProgressTimeouts()
if self.__uriDict['scheme'] == 'http':
# try download 3 times, with random sleep
for _ in range(3):
try:
sotimeout = float(
pylons.config['download_thread_sotimeout'])
proxies = json.loads(
pylons.config['urlgrabber_proxies'])
urlgrabber.urlgrab(
self.__uriDict['uri'],
self.__uriDict['inProgressPackagePath'],
checkfunc=None if self.__skipProp else
(PackageUtil.validateDownload, (), {}),
progress_obj=DownloadProgress(self),
throttle=float(pylons.config['package_throttle']),
bandwidth=int(pylons.config['package_bandwidth']),
keepalive=0,
timeout=sotimeout,
proxies=proxies)
break
except Exception as exc:
msg = 'Download error %s - %s' % (
str(exc), traceback.format_exc(3))
LOG.warning(msg)
randsleep = randint(30, 60)
time.sleep(randsleep)
else:
# oops! only http and bittorrent supported now
raise AgentException(
Errors.DC_UNSUPPORTED_PROTOCOL,
'Only http protocols is supported at the moment')
self._checkStop()
if not self.__skipProp:
if (not PackageUtil.validatePackage(
self.__uriDict['inProgressPackagePath'],
self.__uriDict['propPath'])):
raise AgentException(
Errors.DC_FAILED_VALIDATE, 'Package ' +
self.__uriDict['packagePath'] + ' failed validation')
os.utime(self.__uriDict['propPath'], None)
utils.rchmod(self.__uriDict['propPath'], "777", 'no')
LOG.info(
'Download complete, will now rename and do validation on this file %s'
% self.__uriDict['packagePath'])
os.rename(self.__uriDict['inProgressPackagePath'],
self.__uriDict['packagePath'])
os.utime(self.__uriDict['packagePath'], None)
utils.rchmod(self.__uriDict['packagePath'], "777", 'no')
LOG.info(
"Download complete, Validation completed, updating progress to 100"
)
self._updateStatus(progress=100)
except AgentException, exc:
self._updateStatus(httpStatus=500,
progress=0,
error=exc.getCode(),
errorMsg=exc.getMsg())
msg = 'Download error %s - %s' % (str(exc),
traceback.format_exc(3))
LOG.error(msg)
raise exc
def __init__(self, threadMgr, package):
""" Constructor """
AgentThread.__init__(self, threadMgr, name='validate_package')
self.__package = package
def __init__(self, threadMgr, cat = None, name = 'agent_thread', parentId = None):
""" Constructor """
AgentThread.__init__(self, threadMgr, cat = cat, name = name, parentId = parentId)
def __init__(self, threadMgr, package, parentId = None):
""" Constructor """
AgentThread.__init__(self, threadMgr, name = 'validate_package', parentId = parentId)
self.__package = package
def __init__(self, threadMgr, module, parentId = None):
""" Constructor """
AgentThread.__init__(self, threadMgr, cat = [manifestutil.serviceCat('agent')], name = 'delete_module', parentId = parentId)
self.__module = module
self.__service = 'agent'
def __init__(self, threadMgr, cat = None):
AgentThread.__init__(self, threadMgr, cat)
def __init__(self, threadMgr, progress_queue):
AgentThread.__init__(self, threadMgr, cat=['Service/test_status'])
self.progress_queue = progress_queue
def stop(self):
""" stopping client before calling the super method """
LOG.info('STOP download thread is called stopping')
AgentThread.stop(self)
def __init__(self, threadMgr, service, manifest, parentId = None):
""" Constructor """
AgentThread.__init__(self, threadMgr, cat = [manifestutil.serviceCat(service)], name = 'delete_manifest', parentId = parentId)
self.service = service
self.manifest = manifest
def __init__(self, threadMgr, stopOnly=False):
""" Constructor """
AgentThread.__init__(self, threadMgr, cat = [ServicesCleanup.CAT], name = 'cleanup_service')
self.__killTimeout = configutil.getConfigAsInt('agent_thread_timeout') #kill -9 should take no time
self.__stopOnly = stopOnly
def __init__(self, threadMgr, cat = ['service/test']):
AgentThread.__init__(self, threadMgr, cat = cat)
def __init__(self, threadMgr, cat=['service/test']):
AgentThread.__init__(self, threadMgr, cat=cat)
def __startDownload(self):
""" actual download logic """
try:
LOG.info("Starting package download for package %s" % self.__uriDict['package'])
# check to see if there's an in progress file,
# since PackageMgr guarantees that duplicate threads will not be spawned
# for same pkg, assume an existing thread was killed.
# attempt to clean up package n move
if (os.path.exists(self.__uriDict['inProgressPackagePath'])):
LOG.debug('In progress file (%s) already exists. Cleanup and reattempt download'
% self.__uriDict['inProgressPackagePath'])
if os.path.exists(self.__uriDict['packagePath']):
if self.__skipProp or ((os.path.exists(self.__uriDict['propPath']) and
PackageUtil.validateProp(self.__uriDict['propPath']) and
PackageUtil.validatePackage(self.__uriDict['packagePath'],
self.__uriDict['propPath']))):
msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict['packagePath']
LOG.info(msg)
os.utime(self.__uriDict['packagePath'], None)
if os.path.exists(self.__uriDict['propPath']):
os.utime(self.__uriDict['propPath'], None)
self._updateStatus(progress = 100)
# NOTE: this is a normal exit not an error!
return
else:
LOG.warning('The package already exists. However package prop (%s) failed validation.'
% self.__uriDict['propPath'])
# Delete all traces of package before beginning download
LOG.debug('Cleaning up all packages for %s ' % self.__uriDict['packagePath'])
PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'],
self.__uriDict['packagePath'],
self.__uriDict['propPath'])
AgentThread._updateProgress(self, 0)
if self.__skipProp:
LOG.info('Skip download of prop file')
else:
# First, download .prop file
LOG.info('Starting download of prop file %s - %s' % (self.__uriDict['propUri'], self.__uriDict['propPath']))
self.__download_prop_file()
try:
self.__prop = loadPropFile(self.__uriDict['propPath'])
except FileNotFoundError:
raise AgentException(Errors.DC_MISSING_PROP_FILE,
'Prop file (%s) unable to read or did not parse' % (self.__uriDict['propPath']))
AgentThread._updateProgress(self, 2)
self.__setProgressTimeouts()
if self.__uriDict['scheme'] == 'http':
# try download 3 times, with random sleep
attempt = configutil.getConfigAsInt('download_thread_attempt')
for _ in range(attempt):
try:
sotimeout = float(pylons.config['download_thread_sotimeout'])
proxies = json.loads(pylons.config['urlgrabber_proxies'])
urlgrabber.urlgrab(self.__uriDict['uri'],
self.__uriDict['inProgressPackagePath'],
# checkfunc = None if self.__skipProp else (PackageUtil.validateDownload, (), {}),
progress_obj = DownloadProgress(self),
throttle = float(pylons.config['package_throttle']),
bandwidth = int(pylons.config['package_bandwidth']),
keepalive = 0,
timeout = sotimeout,
proxies = proxies)
break
except Exception as exc:
msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3))
LOG.warning(msg)
if _ == attempt-1:
raise exc
randsleep = randint(5, 10)
time.sleep(randsleep)
else:
# oops! only http supported now
raise AgentException(Errors.DC_UNSUPPORTED_PROTOCOL, 'Only http protocols is supported at the moment')
self._checkStop()
if self.__skipProp:
LOG.info('Skip validating against prop file')
else:
if (not PackageUtil.validatePackage(self.__uriDict['inProgressPackagePath'],
self.__uriDict['propPath'])):
raise AgentException(Errors.DC_FAILED_VALIDATE, 'Package ' +
self.__uriDict['packagePath'] + ' failed validation')
os.utime(self.__uriDict['propPath'], None)
utils.rchmod(self.__uriDict['propPath'], "777", 'no')
LOG.info('Download complete, rename this file %s' % self.__uriDict['packagePath'])
os.rename(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath'])
os.utime(self.__uriDict['packagePath'], None)
utils.rchmod(self.__uriDict['packagePath'], "777", 'no')
LOG.info("Download complete, updating progress to 100")
self._updateStatus(progress = 100)
except AgentException, exc:
self._updateStatus(httpStatus = 500, progress = 0, error = exc.getCode(), errorMsg = exc.getMsg())
msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3))
LOG.error(msg)
raise exc
def __init__(self, threadMgr, cat = None, name = 'agent_thread'):
""" Constructor """
AgentThread.__init__(self, threadMgr, cat = cat, name = name)
def __init__(self, threadMgr):
""" Constructor """
AgentThread.__init__(self, threadMgr, cat = [SafeShutdown.CAT], name = SafeShutdown.THREAD_NAME)
def __init__(self, threadMgr):
""" Constructor """
AgentThread.__init__(self,
threadMgr,
cat=[SafeShutdown.CAT],
name=SafeShutdown.THREAD_NAME)
def __init__(self, threadMgr, service, path):
""" Constructor """
AgentThread.__init__(self, threadMgr, cat = [manifestutil.serviceCat(service)], name = 'delete_service')
self.__path = path
self.service = service