def _untarPackages(self, packages, service, untarRootPath, nicelevel=None, pgksNeedSuffix=None, pathSuffix=''):
"""
untar all the packages
@params packages - list of dictionary of parsed packages uri
@throws AgentException
"""
self._updateProgress(80)
count = 0
for pkgDict in packages.itervalues():
count += 1
# check if package path exists already
# if the package is already untarred, move on
# else create the package path
pkgName = pkgDict['packageName']
untarPath = os.path.join(untarRootPath,
pkgName,
'%s%s' % (pkgDict['packageVersion'], pathSuffix if ((pgksNeedSuffix is not None) and pkgName in pgksNeedSuffix) else ''))
if os.path.exists(untarPath):
# perhaps another thread has finished extraction, continue with next package
continue
os.makedirs(untarPath)
try:
self.untar(pkgDict['packagePath'], untarPath, nicelevel)
except AgentException:
rmrf(untarPath)
# Note: i. atleast self should have 'rx' so that we can proceed setting 'rx' for group and others
# ii. since cronus/cronusapp belong to diff group, lets give access to self, group and others.
# if both belong to same group in future, then just self, group should be enough
if (not os.name == 'nt'):
# ensure all parent dir of scripts dir have 'rx' so that we can really navigate to scripts dir and execute
uname = getuserofpath(untarPath)
chmod(untarPath, '+rx', sudoUser = uname)
cronusPath = os.path.join(untarPath, 'cronus')
if os.path.exists(cronusPath):
uname = getuserofpath(cronusPath)
chmod(cronusPath, '+rx', sudoUser = uname)
# now give all scripts 'rx' permission
scriptsPath = os.path.join(cronusPath, 'scripts')
if os.path.exists(scriptsPath):
uname = getuserofpath(scriptsPath)
rchmod(scriptsPath, '+rx', sudoUser = uname)
#Running as cronus user when higher privilege service (i.e. not chown all the package into the application user)
if (not isHigherPrivilegeService(service)):
import pwd
uname = pylons.config['app_user_account']
uid = pwd.getpwnam(uname).pw_uid
gid = pwd.getpwnam(uname).pw_gid
rchown(untarPath, uid, gid)
self._updateProgress(calcProgress(80, 99, float(count) / len(packages)))
def post(self, service):
""" Create a new service object """
try:
LOG.info('Got a post request for service ' + service)
path = ServiceController.servicePath(service)
# skip if the path already exists
if os.path.exists(path) and os.path.isdir(path):
return doneResult(request,
response,
httpStatus=201,
result=service,
controller=self)
os.makedirs(path)
os.makedirs(os.path.join(path, 'manifests'))
os.makedirs(os.path.join(path, 'installed-packages'))
os.makedirs(os.path.join(path, 'modules'))
os.makedirs(os.path.join(path, 'downloaded-packages'))
os.makedirs(os.path.join(path, '.appdata'))
os.makedirs(os.path.join(path, '.data'))
utils.rchmod(os.path.join(path, '.appdata'), '+rwx')
# verify that the path exists
if (not os.path.isdir(path)):
return errorResult(request,
response,
Errors.UNKNOWN_ERROR,
"Service(%s) was not created" % service,
controller=self)
return doneResult(request,
response,
result=service,
controller=self)
except OSError:
return errorResult(request,
response,
Errors.SERVICE_EXISTS,
"Service(%s) already exists(or %s)" %
(service, traceback.format_exc(2)),
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error when posting services %s - %s' %
(str(excep), traceback.format_exc(2)),
controller=self)
def post(self, service):
""" Create a new service object """
try:
LOG.info("Got a post request for service " + service)
path = ServiceController.servicePath(service)
# skip if the path already exists
if os.path.exists(path) and os.path.isdir(path):
return doneResult(request, response, httpStatus=201, result=service, controller=self)
os.makedirs(path)
os.makedirs(os.path.join(path, "manifests"))
os.makedirs(os.path.join(path, "installed-packages"))
os.makedirs(os.path.join(path, "modules"))
os.makedirs(os.path.join(path, "downloaded-packages"))
os.makedirs(os.path.join(path, ".appdata"))
os.makedirs(os.path.join(path, ".data"))
utils.rchmod(os.path.join(path, ".appdata"), "+rwx")
# verify that the path exists
if not os.path.isdir(path):
return errorResult(
request, response, Errors.UNKNOWN_ERROR, "Service(%s) was not created" % service, controller=self
)
return doneResult(request, response, result=service, controller=self)
except OSError:
return errorResult(
request,
response,
Errors.SERVICE_EXISTS,
"Service(%s) already exists(or %s)" % (service, traceback.format_exc(2)),
controller=self,
)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg="Unknown error when posting services %s - %s" % (str(excep), traceback.format_exc(2)),
controller=self,
)
def __startDownload(self):
""" actual download logic """
try:
LOG.info("Starting package download for package %s" %
self.__uriDict['package'])
# check to see if there's an in progress file,
# since PackageMgr guarantees that duplicate threads will not be spawned
# for same pkg, assume an existing thread was killed.
# attempt to clean up package n move
if (os.path.exists(self.__uriDict['inProgressPackagePath'])):
LOG.debug(
'In progress file (%s) already exists. Will validate and reattempt download if necessary'
% self.__uriDict['inProgressPackagePath'])
if os.path.exists(self.__uriDict['packagePath']):
if (os.path.exists(self.__uriDict['propPath']) and
PackageUtil.validateProp(self.__uriDict['propPath'])
and PackageUtil.validatePackage(
self.__uriDict['packagePath'],
self.__uriDict['propPath'])):
msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict[
'packagePath']
LOG.info(msg)
os.utime(self.__uriDict['packagePath'], None)
os.utime(self.__uriDict['propPath'], None)
self._updateStatus(progress=100)
# NOTE: this is a normal exit not an error!
return
LOG.warning(
'The package already exists. However package prop (%s) failed validation. Downloading package.'
% self.__uriDict['propPath'])
# Delete all traces of package before beginning download
LOG.info('Cleaning up all packages for %s ' %
self.__uriDict['packagePath'])
PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'],
self.__uriDict['packagePath'],
self.__uriDict['propPath'])
AgentThread._updateProgress(self, 0)
if not self.__skipProp:
# First, download .prop file
LOG.info(
'Starting download of prop file %s - %s' %
(self.__uriDict['propUri'], self.__uriDict['propPath']))
self.__download_prop_file()
try:
self.__prop = loadPropFile(self.__uriDict['propPath'])
except FileNotFoundError:
raise AgentException(
Errors.DC_MISSING_PROP_FILE,
'Prop file (%s) unable to read or did not parse' %
(self.__uriDict['propPath']))
AgentThread._updateProgress(self, 2)
self.__setProgressTimeouts()
if self.__uriDict['scheme'] == 'http':
# try download 3 times, with random sleep
for _ in range(3):
try:
sotimeout = float(
pylons.config['download_thread_sotimeout'])
proxies = json.loads(
pylons.config['urlgrabber_proxies'])
urlgrabber.urlgrab(
self.__uriDict['uri'],
self.__uriDict['inProgressPackagePath'],
checkfunc=None if self.__skipProp else
(PackageUtil.validateDownload, (), {}),
progress_obj=DownloadProgress(self),
throttle=float(pylons.config['package_throttle']),
bandwidth=int(pylons.config['package_bandwidth']),
keepalive=0,
timeout=sotimeout,
proxies=proxies)
break
except Exception as exc:
msg = 'Download error %s - %s' % (
str(exc), traceback.format_exc(3))
LOG.warning(msg)
randsleep = randint(30, 60)
time.sleep(randsleep)
else:
# oops! only http and bittorrent supported now
raise AgentException(
Errors.DC_UNSUPPORTED_PROTOCOL,
'Only http protocols is supported at the moment')
self._checkStop()
if not self.__skipProp:
if (not PackageUtil.validatePackage(
self.__uriDict['inProgressPackagePath'],
self.__uriDict['propPath'])):
raise AgentException(
Errors.DC_FAILED_VALIDATE, 'Package ' +
self.__uriDict['packagePath'] + ' failed validation')
os.utime(self.__uriDict['propPath'], None)
utils.rchmod(self.__uriDict['propPath'], "777", 'no')
LOG.info(
'Download complete, will now rename and do validation on this file %s'
% self.__uriDict['packagePath'])
os.rename(self.__uriDict['inProgressPackagePath'],
self.__uriDict['packagePath'])
os.utime(self.__uriDict['packagePath'], None)
utils.rchmod(self.__uriDict['packagePath'], "777", 'no')
LOG.info(
"Download complete, Validation completed, updating progress to 100"
)
self._updateStatus(progress=100)
except AgentException, exc:
self._updateStatus(httpStatus=500,
progress=0,
error=exc.getCode(),
errorMsg=exc.getMsg())
msg = 'Download error %s - %s' % (str(exc),
traceback.format_exc(3))
LOG.error(msg)
raise exc
def __startDownload(self):
""" actual download logic """
try:
LOG.info("Starting package download for package %s" % self.__uriDict['package'])
# check to see if there's an in progress file,
# since PackageMgr guarantees that duplicate threads will not be spawned
# for same pkg, assume an existing thread was killed.
# attempt to clean up package n move
if (os.path.exists(self.__uriDict['inProgressPackagePath'])):
LOG.debug('In progress file (%s) already exists. Cleanup and reattempt download'
% self.__uriDict['inProgressPackagePath'])
if os.path.exists(self.__uriDict['packagePath']):
if self.__skipProp or ((os.path.exists(self.__uriDict['propPath']) and
PackageUtil.validateProp(self.__uriDict['propPath']) and
PackageUtil.validatePackage(self.__uriDict['packagePath'],
self.__uriDict['propPath']))):
msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict['packagePath']
LOG.info(msg)
os.utime(self.__uriDict['packagePath'], None)
if os.path.exists(self.__uriDict['propPath']):
os.utime(self.__uriDict['propPath'], None)
self._updateStatus(progress = 100)
# NOTE: this is a normal exit not an error!
return
else:
LOG.warning('The package already exists. However package prop (%s) failed validation.'
% self.__uriDict['propPath'])
# Delete all traces of package before beginning download
LOG.debug('Cleaning up all packages for %s ' % self.__uriDict['packagePath'])
PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'],
self.__uriDict['packagePath'],
self.__uriDict['propPath'])
AgentThread._updateProgress(self, 0)
if self.__skipProp:
LOG.info('Skip download of prop file')
else:
# First, download .prop file
LOG.info('Starting download of prop file %s - %s' % (self.__uriDict['propUri'], self.__uriDict['propPath']))
self.__download_prop_file()
try:
self.__prop = loadPropFile(self.__uriDict['propPath'])
except FileNotFoundError:
raise AgentException(Errors.DC_MISSING_PROP_FILE,
'Prop file (%s) unable to read or did not parse' % (self.__uriDict['propPath']))
AgentThread._updateProgress(self, 2)
self.__setProgressTimeouts()
if self.__uriDict['scheme'] == 'http':
# try download 3 times, with random sleep
attempt = configutil.getConfigAsInt('download_thread_attempt')
for _ in range(attempt):
try:
sotimeout = float(pylons.config['download_thread_sotimeout'])
proxies = json.loads(pylons.config['urlgrabber_proxies'])
urlgrabber.urlgrab(self.__uriDict['uri'],
self.__uriDict['inProgressPackagePath'],
# checkfunc = None if self.__skipProp else (PackageUtil.validateDownload, (), {}),
progress_obj = DownloadProgress(self),
throttle = float(pylons.config['package_throttle']),
bandwidth = int(pylons.config['package_bandwidth']),
keepalive = 0,
timeout = sotimeout,
proxies = proxies)
break
except Exception as exc:
msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3))
LOG.warning(msg)
if _ == attempt-1:
raise exc
randsleep = randint(5, 10)
time.sleep(randsleep)
else:
# oops! only http supported now
raise AgentException(Errors.DC_UNSUPPORTED_PROTOCOL, 'Only http protocols is supported at the moment')
self._checkStop()
if self.__skipProp:
LOG.info('Skip validating against prop file')
else:
if (not PackageUtil.validatePackage(self.__uriDict['inProgressPackagePath'],
self.__uriDict['propPath'])):
raise AgentException(Errors.DC_FAILED_VALIDATE, 'Package ' +
self.__uriDict['packagePath'] + ' failed validation')
os.utime(self.__uriDict['propPath'], None)
utils.rchmod(self.__uriDict['propPath'], "777", 'no')
LOG.info('Download complete, rename this file %s' % self.__uriDict['packagePath'])
os.rename(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath'])
os.utime(self.__uriDict['packagePath'], None)
utils.rchmod(self.__uriDict['packagePath'], "777", 'no')
LOG.info("Download complete, updating progress to 100")
self._updateStatus(progress = 100)
except AgentException, exc:
self._updateStatus(httpStatus = 500, progress = 0, error = exc.getCode(), errorMsg = exc.getMsg())
msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3))
LOG.error(msg)
raise exc
def executeScript(self):
""" execute a script from remote location"""
scriptpath = None
try:
# parse the body
if (not request.body or request.body == ""):
LOG.error('invalid body found in post command')
return errorResult(request, response, 10001, 'No body found in post command', controller = self)
body = json.loads(request.body.encode('ascii', 'ignore'))
scriptloc = body['scriptLocation'] if 'scriptLocation' in body else None
scriptname = body['scriptName'] if 'scriptName' in body else None
hasSudo = ('sudoUser' in body and body['sudoUser'])
sudoUser = body['sudoUser'] if ('sudoUser' in body and body['sudoUser'] != 'root') else None
paramobj = body['params'] if 'params' in body else []
params = paramobj if type(paramobj) == list else paramobj.split()
LOG.info('%s %s %s %s %s' % (scriptloc, scriptname, hasSudo, sudoUser, params))
if not scriptloc:
return errorResult(request, response, Errors.INVALID_REQUEST, 'Script location not found', controller = self)
if not scriptname:
return errorResult(request, response, Errors.INVALID_REQUEST, 'Script name not found', controller = self)
scriptpath = os.path.join(self.dataPath(), scriptname)
LOG.info('scriptpath = %s' % scriptpath)
utils.runsyscmd('wget %s -O %s' % (scriptloc, scriptpath))
if not os.path.exists(scriptpath):
return errorResult(request, response, Errors.FILE_NOT_FOUND_ERROR, 'Failed to get script %s' % scriptpath, controller = self)
utils.rchmod(scriptpath, '+rx')
cmd = [scriptpath]
if hasSudo:
cmd.insert(0, 'sudo')
if sudoUser:
cmd.insert(1, sudoUser)
cmd.insert(1, '-u')
for param in params:
cmd.append(param)
LOG.info('cmd = %s' % cmd)
appGlobal = config['pylons.app_globals']
execThread = ExecThread(appGlobal.threadMgr, cmd)
execThread.setLogLevel('info')
contextutils.copyJobContexts(self, execThread)
execThread.start()
execThread.threadMgrEvent.wait()
return statusResult(request, response, execThread, controller = self)
except Exception as excp:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when executing cmd %s, %s - %s' %
(scriptpath, str(excp), traceback.format_exc(2)),
controller = self)
def _untarPackages(self, packages, service, untarRootPath, nicelevel=None, pgksNeedSuffix=None, pathSuffix=''):
"""
untar all the packages
@params packages - list of dictionary of parsed packages uri
@throws AgentException
"""
self._updateProgress(80)
count = 0
for pkgDict in packages.itervalues():
count += 1
# check if package path exists already
# if the package is already untarred, move on
# else create the package path
pkgName = pkgDict['packageName']
untarPath = os.path.join(untarRootPath,
pkgName,
'%s%s' % (pkgDict['packageVersion'], pathSuffix if ((pgksNeedSuffix is not None) and pkgName in pgksNeedSuffix) else ''))
if os.path.exists(untarPath):
# perhaps another thread has finished extraction, continue with next package
continue
os.makedirs(untarPath)
try:
self.untar(pkgDict['packagePath'], untarPath, nicelevel)
except AgentException:
rmrf(untarPath)
# Note: i. atleast self should have 'rx' so that we can proceed setting 'rx' for group and others
# if both belong to same group in future, then just self, group should be enough
# ensure all parent dir of scripts dir have 'rx' so that we can really navigate to scripts dir and execute
uname = getuserofpath(untarPath)
chmod(untarPath, '+rx', sudoUser = uname)
cronusPath = os.path.join(untarPath, 'cronus')
if os.path.exists(cronusPath):
uname = getuserofpath(cronusPath)
chmod(cronusPath, '+rx', sudoUser = uname)
# now give all scripts 'rx' permission
scriptsPath = os.path.join(cronusPath, 'scripts')
if os.path.exists(scriptsPath):
uname = getuserofpath(scriptsPath)
rchmod(scriptsPath, '+rx', sudoUser = uname)
# issue #16, now add symlink to .appdata for easy access
appdata_path = manifestutil.appDataPath(service)
link_path = os.path.join(untarPath, '.appdata')
LOG.info('Create .appdata symlink from %s to %s' % (link_path, appdata_path))
utils.symlink(appdata_path, link_path)
#Running as cronus user when higher privilege service (i.e. not chown all the package into the application user)
if (not isHigherPrivilegeService(service)):
uname = configutil.getAppUser()
uid, gid = utils.getUidGid(uname)
rchown(untarPath, uid, gid)
self._updateProgress(calcProgress(80, 99, float(count) / len(packages)))
def executeScript(self):
""" execute a script from remote location"""
scriptpath = None
try:
# parse the body
if (not request.body or request.body == ""):
LOG.error('invalid body found in post command')
return errorResult(request,
response,
10001,
'No body found in post command',
controller=self)
body = json.loads(request.body)
scriptloc = body['script-location'].encode(
'ascii', 'ignore') if 'script-location' in body else None
scriptname = body['script-name'].encode(
'ascii', 'ignore') if 'script-name' in body else None
needsudo = asbool(
body['need-sudo']) if 'need-sudo' in body else False
sudotgt = body['sudo-target'].encode(
'ascii', 'ignore') if 'sudo-target' in body else None
paramobj = body['params'] if 'params' in body else []
params = paramobj if type(paramobj) == list else paramobj.split()
LOG.info('%s %s %s %s %s' %
(scriptloc, scriptname, needsudo, sudotgt, params))
if scriptloc is None or scriptloc == '':
return errorResult(request,
response,
10003,
'Script location not found',
controller=self)
if scriptname is None or scriptname == '':
return errorResult(request,
response,
10003,
'Script name not found',
controller=self)
scriptpath = os.path.join(self.dataPath(), scriptname)
LOG.info('scriptpath = %s' % scriptpath)
os.system('wget %s -O %s' % (scriptloc, scriptpath))
if scriptpath is None or not os.path.exists(scriptpath):
return errorResult(request,
response,
10003,
'Failed to get script %s' % scriptpath,
controller=self)
rchmod(scriptpath, '+rx')
cmd = [scriptpath]
if needsudo:
cmd.insert(0, 'sudo')
if sudotgt is not None:
cmd.insert(1, sudotgt)
cmd.insert(1, '-u')
for param in params:
param = param.encode('ascii', 'ignore')
cmd.append(param)
LOG.info('cmd = %s' % cmd)
appGlobal = config['pylons.app_globals']
execThread = ExecThread(appGlobal.threadMgr, cmd)
execThread.setLogLevel('info')
execThread.start()
execThread.threadMgrEvent.wait()
return statusResult(request, response, execThread, controller=self)
except Exception as excp:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error when executing cmd %s, %s - %s' %
(scriptpath, str(excp), traceback.format_exc(2)),
controller=self)
