def __download_prop_file(self):
""" download prop file and validate """
# retry 3 times download prop file
for _ in range(3):
try:
sotimeout = float(pylons.config['download_thread_sotimeout'])
proxies = json.loads(pylons.config['urlgrabber_proxies'])
urlgrabber.urlgrab(self.__uriDict['propUri'],
self.__uriDict['propPath'],
keepalive=0,
timeout=sotimeout,
proxies=proxies)
break
except Exception:
randsleep = randint(30, 60)
time.sleep(randsleep)
if (not os.path.exists(self.__uriDict['propPath'])):
raise AgentException(
Errors.DC_MISSING_PROP_FILE,
'Prop file (%s) does not exist' % (self.__uriDict['propPath']))
if not PackageUtil.validateProp(self.__uriDict['propPath']):
raise AgentException(
Errors.DC_MISSING_PROP_FILE,
'Prop file (%s) failed validation' %
(self.__uriDict['propPath']))
def __activateManifest(self):
""" activate a manifest
"""
LOG.info("Activate Manifest %s - %s" %
(self.__service, self.__manifest))
# make sure manifest to be activated exist
manifestPath = manifestutil.manifestPath(self.__service,
self.__manifest)
if (not os.path.exists(manifestPath)):
LOG.error('Manifest %s does not exist, fail activation' %
self.__manifest)
raise AgentException(
Errors.MANIFEST_NOT_FOUND,
'Manifest %s does not exist' % self.__manifest)
# check to see if the manifest already active
activeManifest = manifestutil.activeManifestPath(self.__service)
if activeManifest == self.__manifest:
LOG.info('Manifest %s already active, skip activation' %
self.__manifest)
return
from agent.lib.agent_thread.activate_manifest import ActivateManifest
activateThread = ActivateManifest(self._threadMgr, self.__service,
self.__manifest)
contextutils.copycontexts(self, activateThread, contextutils.CTX_NAMES)
activateThread.run()
status = activateThread.getStatus()
if (status.has_key('error') and status['error']):
raise AgentException(status['error'], status['errorMsg'])
def getPackageByName(service, manifest, pkgnamePrefix):
""" finding an existing package that matches a package name prefix
@param __service: __service
@param manifest: if None, try active manifest first, then non active manifests
@param pkgnamePrefix: package name pattern to match against
"""
if not service:
raise AgentException(Errors.MANIFEST_MISSING_SERVICE, 'missing __service')
if not pkgnamePrefix:
raise AgentException(Errors.PACKAGE_PATH_ERROR, 'package name prefix is missing for reuse of package')
if manifest is None:
manifest = ACTIVE_MANIFEST if hasActiveManifest(service) else getManifests(service)[0]
if manifest is None or not os.path.exists(manifestContentPath(service, manifest)):
raise AgentException(Errors.MANIFEST_NOT_FOUND, 'cannot find .manifest for manifest %s ' % manifest)
fullPkgLoc = None
with open(manifestContentPath(service, manifest)) as mfile:
lines = mfile.readlines()
for line in lines:
tokens = line.split('/')
if tokens[-1].startswith(pkgnamePrefix):
# found
fullPkgLoc = line.rstrip()
break
return fullPkgLoc
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
failed = False
activeManifest = None
try:
activePath = manifestutil.manifestPath(self._service, 'active')
# make sure that the active path exists and it is a link
# Should we check this again since we already have a check in action controller
if not os.path.exists(activePath) or not islink(activePath):
raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING, errorMsg = 'No active manifest - cannot restart service')
activeManifest = os.path.basename(readlink(activePath))
if self.__action == StartStopService.ACTION_SHUTDOWN:
self.__shutdownManifest(self._service, activeManifest)
elif self.__action == StartStopService.ACTION_STARTUP:
self.__startupManifest(self._service, activeManifest)
elif self.__action == StartStopService.ACTION_RESTART:
self.__restartManifest(self._service, activeManifest)
elif self.__action == StartStopService.ACTION_REBOOT:
self.__rebootManifest(self._service, activeManifest)
else:
raise AgentException(error = Errors.INVALID_LIFECYCLE_ACTION, errorMsg = 'Invalid life cycle action - %s' % self.__action)
self.__LOG.info('Done: %s service for (%s/%s)' % (self.__action, self._service, activeManifest))
self._updateStatus(progress = 100)
except AgentException as exc:
failed = True
errorMsg = '%s Service - Agent Exception - %s' % (self.__action, exc.getMsg())
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = '%s Service - Unknown error - (%s/%s) - %s - %s' \
% (self.__action, self._service, self._manifest, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self.__LOG.error(errorMsg)
if not self._skipCleanupOnFailure() and self.__action != StartStopService.ACTION_SHUTDOWN and self._service and activeManifest:
try:
self.__LOG.info('%s Service %s failed, shutdown to cleanup' % (self.__action, self._service))
self.__shutdownManifest(self._service, activeManifest)
except BaseException as excep:
self.__LOG.error('Cleanup failed - %s' % str(excep))
self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
def addThread(self, thread):
""" add thread to the thread list
this may fail as only 1 active thread for a category is allowed
returns None if success
return (errorCode, errorMessage)
"""
uuid = thread.getUuid()
cat_list = thread.getCat()
# disallow threads with of the same categories
if self.__rejectRequest:
raise AgentException(Errors.AGENT_STOPPING, 'Agent is stopping')
with self.__lock:
if (uuid in self.__threads):
errMsg = 'Warning: adding thread(%s) which already exists. Ignoring add' % uuid
LOG.warning(errMsg)
return (errMsg, Errors.THREAD_ALREADY_ADDED)
if cat_list:
catThreads = self.getLiveThreadByCatsAndName(cat_list, None, haslock = True)
for catThread in catThreads:
errMsg = ('Concurrent activity is detected (%s / %s) on resource (%s). Another thread (%s / %s) already running '
% (thread.getName(), uuid, cat_list, catThread.getName(), catThread.getUuid()))
LOG.warning(errMsg)
# add this thread anyway, but raise an exception
# the catcher of this exception should set the correct errors
# and kill itself
# Bin: if thread has merge on found flag, do not add it to the map
# thread will merge with existing by change its uuid
if not thread.isMergeOnFound():
self.__threads[uuid] = (thread, None)
raise ConcurrentActivityException(errMsg, catThread.getUuid())
#check thread counter
maxCounter = getattr(thread, "MAX_INSTANCE", 0)
if maxCounter:
nameThreads = self.getLiveThreadByCatsAndName(None, thread.getName(), haslock = True)
if maxCounter <= len(nameThreads):
errMsg = ('Too many instances are detected (%s / %s). Other threads uuids %s'
% (thread.getName(), uuid, [t.getUuid() for t in nameThreads]))
LOG.warning(errMsg)
# add this thread anyway, but raise an exception
# the catcher of this exception should set the correct errors
# and kill itself
self.__threads[uuid] = (thread, None)
raise AgentException(Errors.THREAD_CAT_ALREADY_EXISTS, errMsg)
LOG.info('adding thread: %s with cat %s' % (uuid, cat_list))
self.__threads[uuid] = (thread, None)
def runExtMonitor(self, service, monitorstream, resSec, result, monitorgroup='default'):
'''
Report metrics from external
'''
LOG.debug('%s, %s, %s, %s' % (service, monitorstream, resSec, result))
LOG.debug('%s' % self.__monitorTags)
if resSec not in METRIC_RESOLUTION_SECS_STR:
raise AgentException(Errors.MONITOR_INVALID_RESOLUTION,
'Invalid resolution %s, allowable values %s' % (resSec, ','.join(METRIC_RESOLUTION_SECS)))
metricoptions = (resSec,) + (self.__monitorTags[service][monitorgroup]
if (service in self.__monitorTags and monitorgroup in self.__monitorTags[service])
else (None, None, None, None, None))
if (service, monitorgroup) not in self.__monitorMessages:
self.__monitorMessages[monitorgroup] = {}
if monitorstream not in self.__monitorMessages[(service, monitorgroup)]:
self.__monitorMessages[(service, monitorgroup)][monitorstream] = []
try:
if type(result) == dict:
self.processScriptOuput(service, monitorgroup, monitorstream, [result])
elif type(result) == list:
self.processScriptOuput(service, monitorgroup, monitorstream, result)
else:
err_msg = 'monitor %s result error, format not accepted: %s' % (monitorstream, result)
LOG.warning(err_msg)
return
self.runReport(service, monitorgroup, monitorstream, metricoptions)
except Exception as excep:
err_msg = 'Cannot run monitor %s error (%s) - (%s)' % (monitorstream, excep, traceback.format_exc(2))
LOG.error(err_msg)
def selfupdate(self):
""" agent selfupdate through api
"""
LOG.info('selfupdate agent with body: %s', request.body)
try:
appGlobal = config['pylons.app_globals']
wisbVersion = None
wisbSource = None
if request.body:
requestjson = json.loads(request.body)
if 'version' not in requestjson:
raise AgentException(Errors.INVALID_REQUEST, 'version is required')
wisbVersion = requestjson['version']
wisbSource = requestjson['wisbSource'] if 'wisbSource' in requestjson else configutil.getConfig('selfupdate_source')
skipProp = asbool(requestjson['skipProp']) if 'skipProp' in requestjson else True
updateThread = AgentUpdate(appGlobal.threadMgr, wisbVersion, wisbSource, skipProp = skipProp)
self.injectJobCtx(updateThread)
updateThread.start()
updateThread.threadMgrEvent.wait()
return statusResult(request, response, updateThread, controller = self)
except AgentException as aexcep:
return errorResult(request, response, error = aexcep.getCode(),
errorMsg = aexcep.getMsg(), controller = self)
except Exception as excep:
msg = 'Unknown error for agent update(%s) - %s - %s' % (wisbVersion, str(excep), traceback.format_exc(2))
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = msg, controller = self)
def __init__(self,
threadMgr,
packageUri,
packageloc,
path=None,
category=None,
skipProp=True):
# cat = 'DC_DowloadThread' + str(datetime.now())
AgentThread.__init__(self,
threadMgr,
cat=category,
name='download_thread')
self.__path = pylons.config['repo_root']
if path is not None:
self.__path = path
# check to see if the package path exists
if (not os.path.isdir(self.__path)):
msg = 'Package path(%s) does not exist' % self.__path
LOG.error(msg)
raise AgentException(Errors.PACKAGE_PATH_ERROR, msg)
self.__uriDict = PackageUtil.parseUri(packageUri, self.__path,
packageloc)
self.__prop = {}
self.__error = None
self.__progress = 0.0
self.__timeouts = None
self.__skipProp = skipProp
def sysuntar(tarfilepath, destdir):
''' use system untar to untar a package '''
cmd = 'tar -C %s -xz -f %s' % (destdir, tarfilepath)
ret = os.system(cmd)
if (ret != 0):
error = 'untar cmd (%s) returned error code (%d)' % (cmd, ret)
LOG.error(error)
raise AgentException(Errors.PACKAGE_UNTAR_FAILURE, error)
def readlink(link):
"""read sym link"""
if (_is_unix_link(link)):
return os.readlink(link) #@UndefinedVariable
else:
raise AgentException(
'Running platform seems to be neither win32 nor *nix with any (sym)link support (or) provided link is wrong. Can\'t proceed with link read'
)
def __execChmod(cmd, errcode):
'''
Executes given chmod command and throws AgentException with give errCode in case if exec fails.
'''
ret = os.system(cmd)
if (ret != 0):
error = 'chmod cmd (%s) returned error code (%d)' % (cmd, ret)
LOG.error(error)
raise AgentException(errcode, error)
def __removeSymlink(self, activePath):
""" remove symlink """
if os.path.exists(activePath):
if (os.path.islink(activePath)): # *nix
os.remove(activePath)
else:
raise AgentException(
'Running platform seems to be neither win32 nor *nix with any (sym)link support. Can\'t proceed with link deletion'
)
def runsyscmd(cmd):
''' run system command
@param cmd: system command to run
'''
ret = os.system(cmd)
if (ret != 0):
error = 'system cmd (%s) returned error code (%d)' % (cmd, ret)
LOG.error(error)
raise AgentException(Errors.UTIL_SYSCMD_ERROR, error)
def rmlink(link):
""" remove symlink """
LOG.info("removing link %s" % link)
if (_is_unix_link(link)):
return os.remove(link)
else:
raise AgentException(
'Running platform seems to be neither win32 nor *nix with any (sym)link support (or) provided link is wrong. Can\'t proceed with link read'
)
def validateDownload(obj, *args, **kwargs):
""" used by urlgrabber to check the files """
if (obj.filename.rpartition('.')[2] == 'prop'):
if (PackageUtil.validateProp(obj.filename) == False):
raise AgentException(
Errors.INVALID_PACKAGE,
'Package prop (%s) not valid' % obj.filename)
return
# figure the prop file name from the filename
if (obj.filename.rpartition('.')[2] == 'inprogress'):
propFilename = obj.filename.rpartition('.')[0] + '.prop'
else:
propFilename = obj.filename + '.prop'
if (PackageUtil.validatePackage(obj.filename, propFilename) == False):
raise AgentException(Errors.INVALID_PACKAGE,
'Package (%s) not valid' % obj.filename)
def symlink(target_dir, link_path): # TODO : rename to createlink
"""create sym linkPath"""
# lets not use symlink support by python even if present in win32, as of now
if (hasattr(os, 'symlink')):
return os.symlink(target_dir, link_path)
else:
raise AgentException(
'Running platform seems to be neither win32 nor *nix with any (sym)link support. Can\'t proceed with link creation'
)
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
failed = False
try:
activePath = os.path.join(
ServiceController.manifestPath(self._service), 'active')
oldManifest = None
# make sure that if the active path exists, it's a link
# if not log that and delete the link
if (os.path.exists(activePath) and not os.name == 'nt'
and not islink(activePath)):
self.__LOG.error('%s is not a link. Attempted to delete' %
activePath)
shutil.rmtree(activePath)
if (os.path.exists(activePath)):
oldManifest = os.path.basename(readlink(activePath))
else:
raise AgentException(
error=Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
errorMsg='No active manifest - cannot deactivate service')
self.__deactivateManifest(self._service, oldManifest)
self.__removeSymlink(self._service)
except SystemExit as exc:
failed = True
if (len(exc.args) == 2):
# ok we got {err code, err msg}
errorCode = exc.args[0]
errorMsg = exc.args[1]
raise exc
except AgentException as exc:
failed = True
errorMsg = 'Deactivate Manifest - Agent Exception - %s' % exc.getMsg(
)
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Deactivate Manifest - Unknown error - (%s) - %s - %s' \
% (self._service, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self.__LOG.warning(errorMsg)
self._updateStatus(httpStatus=500,
error=errorCode,
errorMsg=errorMsg)
self.__LOG.debug('Done: activate manifest for (%s)' %
(self._service))
self._updateProgress(100)
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
failed = False
activeManifest = None
try:
activePath = os.path.join(
ServiceController.manifestPath(self._service), 'active')
# make sure that the active path exists and it is a link
# Should we check this again since we already have a check in action controller
if not os.path.exists(activePath) or not islink(activePath):
raise AgentException(
error=Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
errorMsg='No active manifest - cannot reset service')
activeManifest = os.path.basename(readlink(activePath))
self.__shutdownManifest(self._service, activeManifest)
self.__deactivateManifest(self._service, activeManifest)
self.__activateManifest(self._service, activeManifest)
self.__startupManifest(self._service, activeManifest)
self.__LOG.info('Done: reset service for (%s/%s)' %
(self._service, activeManifest))
self._updateStatus(progress=100)
except AgentException as exc:
failed = True
errorMsg = 'Activate Manifest - Agent Exception - %s' % exc.getMsg(
)
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Activate Manifest - Unknown error - (%s/%s) - %s - %s' \
% (self._service, self._manifest, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self.__LOG.error(errorMsg)
if self._service and activeManifest:
try:
self.__LOG.info(
'Reset service %s failed, shutdown to cleanup' %
self._service)
self.__shutdownManifest(self._service, activeManifest)
except BaseException as excep:
self.__LOG.error('Cleanup failed - %s' % str(excep))
self._updateStatus(httpStatus=500,
error=errorCode,
errorMsg=errorMsg)
def runsyscmdwstdout(cmd):
''' run system command with capture of stdout '''
LOG.info('run system command %s ' % cmd)
process = Popen(cmd, stdout=PIPE, close_fds=True)
preprocessed = process.communicate()[0]
ret = process.wait()
if ret != 0:
error = 'system cmd (%s) returned error code (%d)' % (cmd, ret)
LOG.error(error)
raise AgentException(Errors.UTIL_SYSCMD_ERROR, error)
return preprocessed
def untar(self, packagePath, untarPath, nicelevel):
''' do real untar '''
cmd = ['tar', '-C', untarPath, '-x', '-f', packagePath]
# timeout 60 minute
execThread = ExecThread(None, cmd, None)
execThread.setTimeout(3600)
execThread.run()
status = execThread.getStatus()
if (status['error'] != None):
msg = 'untar cmd (%s) failed (%s - %s)' % (' '.join(cmd), status['error'], status['errorMsg'])
LOG.error(msg)
raise AgentException(Errors.PACKAGE_UNTAR_FAILURE, msg)
def readJson(service, filename):
""" read data in json format from under __service root folder """
if not filename.startswith('.'):
raise AgentException(Errors.INVALID_FILE_NAME, 'file name has to start with .')
filepath = os.path.join(servicePath(service), filename + '.json')
result = {}
try:
with open(filepath, 'rb') as fp:
result = json.load(fp)
except Exception:
# fine, file not exist
pass
return result
def copyFile(sourceDir, destinationDir, filename):
""" copy a file from source dir to dest dir """
# get owner of parent directory
sourceFn = os.path.join(sourceDir, filename)
if os.path.exists(destinationDir) and os.path.exists(sourceFn):
destUname = getuserofpath(os.path.join(destinationDir, filename))
cmd = sudoCmd('cp %s %s' % (sourceFn, destinationDir), destUname)
runsyscmd(cmd)
else:
raise AgentException(
Errors.INVALID_FILE_NAME,
'source %s %s or destination %s not exist' %
(sourceDir, filename, destinationDir))
def writeJson(service, filename, data):
""" write data in json format to .file under __service root folder """
if not filename.startswith('.'):
raise AgentException(Errors.INVALID_FILE_NAME, 'file name has to start with .')
filepath = os.path.join(servicePath(service), filename + '.json')
try:
with open(filepath, 'wb+') as fp:
jsonStr = json.dumps(data)
fp.write(jsonStr)
fp.write("\n")
except Exception:
pass # fine, directory does not exist
def rchown(path, uid, gid):
''' recursive chown. Given a path, change the uid and gid recrusively. Will not follow links.
@param path - string of path to chmod
@param uid - int uid
@param gid - int gid
@return - none
'''
if os.path.exists(path):
cmd = 'sudo chown -R %s:%s %s' % (str(uid), str(gid), path)
ret = os.system(cmd)
if (ret != 0):
error = 'chown cmd (%s) returned error code (%d)' % (cmd, ret)
LOG.error(error)
raise AgentException(Errors.UTIL_RCHOWN_ERROR, error)
def __stopProcesses(self):
""" stop all processes that run as app_user_account
refers to http://stackoverflow.com/questions/4669754/python-kill-all-processes-owned-by-user
"""
self._updateStatus(progress=10)
uname = pylons.config['app_user_account']
import pwd
uid = pwd.getpwnam(uname).pw_uid
pids = filter(lambda pid: pid.isdigit(), os.listdir('/proc'))
execThreads = []
# test if PID is owned by user
for pid in pids:
# check if PID still exist
if not os.path.exists(os.path.join('/proc', pid)):
LOG.debug("pid doesn't exist any more: %s" % pid)
continue
puid = os.stat(os.path.join('/proc', pid)).st_uid
if puid == uid:
cmd = utils.sudoCmd(['kill', '-9', pid], uname)
execThread = ExecThread(self._threadMgr, cmd)
execThread.setTimeout(self.__killTimeout)
execThread.start()
execThreads.append(execThread)
while (True):
self._checkStop()
running = False
for execThread in execThreads:
status = execThread.getStatus()
if (status['error'] != None):
raise AgentException(status['error'], status['errorMsg'])
if (execThread.isAlive()):
LOG.debug("process is still alive: %s" %
execThread.getCmd())
running = True
if (not running):
LOG.debug(
"stop processes finished: %s" %
[execThread.getCmd()[-1] for execThread in execThreads])
break
time.sleep(0.1)
self._updateStatus(progress=50)
def __prepManifest(self):
""" prepare data for manifest
"""
try:
if not self.__wisbVersion:
raise AgentException(Errors.MANIFEST_NOT_FOUND,
'wisb version not found')
agtPkg, agtcfgPkg = AgentUpdate.getAgentPkgs(
self.__wisbSource, self.__wisbVersion)
pyPkg = AgentUpdate.getLocalPyPkg()
LOG.info('Agent update using local python package %s' % pyPkg)
self.__packages = [agtPkg, agtcfgPkg, pyPkg]
except AgentException as exc:
errorMsg = 'Agent update - prepManifest - Agent exception - %s - %s' \
% (str(exc), traceback.format_exc(2))
raise exc
except Exception as exc:
errorMsg = 'Agent update - prepManifest - Unknown error - %s - %s' \
% (str(exc), traceback.format_exc(2))
error = Errors.UNKNOWN_ERROR
raise AgentException(error, errorMsg)
def _runExeThread(self, execThreadTuple):
''' run an execution thread, throw agent exception if it fails '''
execThread, isDummy = execThreadTuple
#dummy thread, ignore
if isDummy:
return {'progress': 0}
execThread.run()
status = execThread.getStatus()
if (status['error'] != None):
raise AgentException(status['error'], status['errorMsg'])
return status
def checkOwner(path):
""" check that file owner is either agent account (check config but most likely cronus) or the app account """
if (not os.name == 'nt'):
import pwd
agentUname = pylons.config['agent_user_account']
agentUid = pwd.getpwnam(agentUname).pw_uid
appUname = pylons.config['app_user_account']
appUid = pwd.getpwnam(appUname).pw_uid
file_stat = os.stat(path)
if (not file_stat.st_uid in [agentUid, appUid]):
raise AgentException(
Errors.SERVICE_DELETE_NOT_VALID,
'Path(%s) not owned by agent %s or application %s' %
(path, agentUname, appUname))
def getExecOutput(self, uuid):
""" get script output with an uuid """
LOG.info('get ouput for %s' % uuid)
try:
script = os.path.join(manifestutil.manifestPath('agent'), 'agent', 'cronus', 'scripts', 'execoutput')
LOG.info('execoutput script %s' % script)
if not uuid or uuid == '':
raise AgentException(Errors.INVALID_REQUEST, 'uuid cannot be empty')
tmp = [script, uuid]
cmds = []
for cmd in tmp:
cmds.append(cmd.encode('ascii', 'ignore'))
cmdout = utils.runsyscmdwstdout(cmds)
return cmdout
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Error when get execoutput %s - %s' %
(str(excep), traceback.format_exc(2)), controller = self)
def getLocalPyPkg():
"""
reuse of local python package instead of download it again from source of truth,
this should be the common use case for selfupdate without needing to update the python package
"""
activeManifest = manifestutil.getActiveManifest('agent')
activePyLink = os.path.join(
manifestutil.manifestPath('agent', activeManifest),
'python_package')
if (os.path.exists(activePyLink)):
activePyPath = readlink(activePyLink)
pyVersion = os.path.basename(activePyPath)
pyPkgName = ('python_package-%s' % pyVersion)
if AgentUpdate.nameRe.match(pyPkgName):
return ('http://localhost:12020/%s.cronus' % pyPkgName)
else:
raise AgentException(
Errors.PACKAGE_SCHEME_ERROR,
'package name %s is not valid' % pyPkgName)
def doRun(self):
""" Main body of the thread
Progress Info:
Look at the list of packages this manifest has and figure out the progress of this manifest
Progress 1 = manifest directory created
Progress 2-80 = all packages downloaded
Progress 81-99 = all packages untarred
Progress 100 = all links done
"""
inProgressPath = ManifestCreate.inProgress(ManifestController.manifestPath(self.__service, self.__manifest))
try:
if self.__service != 'agent':
utils.checkDiskFull()
installedPkgPath = ServiceController.installedPkgPath(self.__service)
# This shouldn'trn happen but make sure there isn'trn already a manifest directory in progress
if (os.path.isdir(inProgressPath)):
# probably another manifest create thread died out half way.
# Cleanup and reattempt manifest creation
LOG.debug('Manifest inprogress found for service/manifest (%s/%s). Will cleanup and retry' % (self.__service, self.__manifest))
shutil.rmtree(inProgressPath)
# make sure that the service path and installed package path exists
if (not os.path.isdir(ServiceController.manifestPath(self.__service)) or
not os.path.isdir(ServiceController.installedPkgPath(self.__service))):
errCode = Errors.SERVICE_NOT_FOUND
msg = 'Service (%s) for manifest (%s) not found' % (self.__service, self.__manifest)
self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
return
# ok create the manifest path
os.mkdir(inProgressPath)
self._updateProgress(1)
# figure out which of the packages are already there
remainingPackages = {}
for pkgUri in self.__packages:
pkgDict = PackageUtil.parseUri(pkgUri)
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
pkgName = pkgDict['packageName']
if (not os.path.exists(pkgPath)) or ((self.__forcePackages is not None) and pkgName in self.__forcePackages):
remainingPackages[pkgUri] = pkgDict
else:
symlink(pkgPath, os.path.join(inProgressPath, pkgDict['packageName']))
if self.__attemptDownload:
# now make sure all the packages are downloaded
try:
self._downloadPackages(remainingPackages.keys(), skipProp = self.__skipProp)
except AgentException as exc:
# check if it is download error, then modify exception appropriately
if exc.getCode() == Errors.DC_FAILED_DOWNLOAD:
exc = AgentException(Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED,
'Manifest (%s/%s) failed downloading package - %s'
% (self.__service, self.__manifest, exc.getMsg()))
raise exc
else:
if len(remainingPackages) > 0:
raise AgentException(Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST,
'Create Manifest (%s/%s) failed since package is not present and download has been disabled'
% (self.__service, self.__manifest))
LOG.info('Completed download all packages for (%s/%s)' % (self.__service, self.__manifest))
# now untar the packages
import re
pkgSuffix = '.%s' % re.sub(r"\W", "", self.__manifest)
self._untarPackages(remainingPackages, self.__service, ServiceController.installedPkgPath(self.__service), 0, self.__forcePackages, pkgSuffix)
LOG.info('Completed untar all packages for (%s/%s)' % (self.__service, self.__manifest))
# now create the links
for pkgDict in remainingPackages.itervalues():
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
linkPath = os.path.join(inProgressPath, pkgDict['packageName'])
# validate target folder does exist
if not os.path.exists(pkgPath):
raise AgentException(Errors.PACKAGE_PATH_ERROR, 'invalid untarred package at %s' % pkgPath)
symlink(pkgPath, linkPath)
# now move the inProgressPath to the final path
manifestContentPath = ManifestController.manifestContentPath(self.__service, self.__manifest)
os.rename(inProgressPath, ManifestController.manifestPath(self.__service, self.__manifest))
mfContentsFile = file(manifestContentPath, 'w')
for pkgUri in self.__packages:
mfContentsFile.write(('%s%s') % (pkgUri, os.linesep))
mfContentsFile.close()
LOG.info('Completed create manifest directories for (%s/%s)' % (self.__service, self.__manifest))
LOG.info('Completed create manifest for (%s/%s)' % (self.__service, self.__manifest))
self._updateStatus(progress = 100)
except AgentException as exc:
LOG.info(exc.getMsg())
self._updateStatus(httpStatus = 500, error = exc.getCode(), errorMsg = exc.getMsg())
except Exception as exc:
errCode = Errors.UNKNOWN_ERROR
msg = 'Unknown error for (%s/%s) - %s - %s' % (self.__service, self.__manifest,
str(exc), traceback.format_exc(2))
LOG.info(msg)
self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
finally:
# clean up intermediate progress
try:
shutil.rmtree(inProgressPath)
except OSError:
pass
def doRun(self):
""" Main body of the thread
Progress Info:
Look at the list of packages this manifest has and figure out the progress of this manifest
Progress 1 = manifest directory created
Progress 2-80 = all packages downloaded
Progress 81-99 = all packages untarred
Progress 100 = all links done
"""
try:
if self.__service != 'agent':
utils.checkDiskFull()
installedPkgPath = manifestutil.installedPkgRootPath(self.__service)
# figure out which of the packages are already there
remainingPackages = {}
pkgDict = PackageUtil.parseUri(self.__package)
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
remainingPackages[self.__package] = pkgDict
if (not os.path.exists(pkgPath)):
# now make sure all the packages are downloaded
try:
self._downloadPackages([self.__package])
except AgentException as exc:
# check if it is download error, then modify exception appropriately
if exc.getCode() == Errors.DC_FAILED_DOWNLOAD:
exc = AgentException(Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED,
'Manifest (%s/%s) failed downloading package - %s'
% (self.__service, self.__module, exc.getMsg()))
raise exc
LOG.info('Completed download packages for (%s/%s)' % (self.__service, self.__module))
# now untar the packages
self._untarPackages(remainingPackages, self.__service, ServiceController.installedPkgPath(self.__service))
LOG.info('Completed untar all packages for (%s/%s)' % (self.__service, self.__module))
# now create the links
for pkgDict in remainingPackages.itervalues():
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
modulePath = os.path.join(manifestutil.moduleRootPath(self.__service), self.__module)
# validate target folder does exist
if not os.path.exists(pkgPath):
raise AgentException(Errors.PACKAGE_PATH_ERROR, 'invalid untarred package at %s' % pkgPath)
# remove existing module
isExistingModule = os.path.exists(modulePath)
if isExistingModule:
execThread = self._getBuiltThread('deactivate')
if execThread:
super(ModuleCreate, self)._runExeThread(execThread)
rmlink(modulePath)
execThread = self._getBuiltThread('activate')
if execThread:
super(ModuleCreate, self)._runExeThread(execThread)
symlink(pkgPath, modulePath)
# load controllers from package
manifestutil.processModule(self.__service, self.__module, not isExistingModule)
LOG.info('Completed create module for (%s/%s)' % (self.__service, self.__module))
self._updateStatus(progress = 100)
except AgentException as exc:
LOG.info(exc.getMsg())
self._updateStatus(httpStatus = 500, error = exc.getCode(), errorMsg = exc.getMsg())
except Exception as exc:
errCode = Errors.UNKNOWN_ERROR
msg = 'Unknown error for (%s/%s) - %s - %s' % (self.__service, self.__module,
str(exc), traceback.format_exc(2))
LOG.info(msg)
self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
errorMsgRe = re.compile('.*errorMsg\".*:.*\"(.*)\"}')
match = errorMsgRe.match(ERROR)
if (match != None):
errorMsg = match.group(1)
errorlines = ERROR.split('\n')
errorjoin = None
ethread = ExecThread(NullThreadMgr, 'agent')
for line in errorlines:
ethread.processExecResponse(line)
line = line.rstrip('\n')
errorjoin = line if not errorjoin else (errorjoin + '\\n' + line)
threadjson = ethread.getResponse()
exc = AgentException('1', 'test');
errorMsg = exc.getMsg()
config = ConfigParser.RawConfigParser()
config.read('fact.prop')
env_vs = 'CRONUSAPP_HOME=%s LCM_CORRELATIONID=%s'.split(' ')
text = 'some'
text1 = text.split()
lines = ['line1', 'line2', 'line3']
for line in lines:
print line
arch = 'r1'.split(',')
print 'r1' in arch
