def createManifests(mf_path): os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0')) os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0')) latest = os.path.join(mf_path, 'm3.0') os.makedirs(os.path.join(latest, 'dummy_dir3.0')) utils.symlink(latest, os.path.join(mf_path, 'active')) return (['m1.0', 'm2.0', 'm3.0'], 'm3.0')
def testValidateLink(self): commonTearDown() commonSetup() basepath = pylons.config['agent_root'] badpath = os.path.join(basepath, 'not_exist') goodpath = os.path.join(basepath, 'packages') badlink = os.path.join(basepath, 'bad_link') goodlink = os.path.join(basepath, 'good_link') utils.symlink(badpath, badlink) assert (utils.validatelink(badlink) == False) utils.symlink(goodpath, goodlink) assert (utils.validatelink(goodlink) == True)
def __switchSymlink(self, service, manifest): """ remove old symlink and create new one """ self._LOG.info("Switch active link for %s-%s" % (service, manifest)) if (manifest == None): return #remove symlink activePath = self.__getSymlinkPath(service) self.__removeSymlink(activePath) # create the active link symlink(manifest, activePath) return True
def testValidateLink(self): commonTearDown() commonSetup() basepath = pylons.config['agent_root'] badpath = os.path.join(basepath, 'not_exist') goodpath = os.path.join(basepath, 'packages') badlink = os.path.join(basepath, 'bad_link') goodlink = os.path.join(basepath, 'good_link') utils.symlink(badpath, badlink) assert(utils.validatelink(badlink) == False) utils.symlink(goodpath, goodlink) assert(utils.validatelink(goodlink) == True)
def __switchSymlink(self, service, manifest): """ remove old symlink and create new one """ self.__LOG.debug("Switch active link for %s-%s" % (service, manifest)) if (manifest == None): return #remove symlink activePath = self.__getSymlinkPath(service) self.__removeSymlink(activePath) # create the active link symlink(manifest, activePath) return True
def test_delete_active_manifest(self): createManifest(self) self.assertTrue(os.path.isdir(ServiceController.manifestPath('foo'))) currentPath = os.getcwd() os.chdir(ServiceController.manifestPath('foo')) manifestPath = 'bar' #ManifestController.manifestPath('foo', 'bar'); use short manifest name instead of full path symlink(manifestPath, 'active') os.chdir(currentPath) response = self.app.delete(url(controller = 'manifest', action = 'delete', service = "foo", manifest = "bar"), expect_errors = True) self.assertEquals(500, response.status_int) body = json.loads(response.body) self.assertEquals(Errors.MANIFEST_DELETING_ACTIVE_MANIFEST, body['error'])
def testGet(self): path = manifestutil.manifestPath('foo', 'bar') os.makedirs(path) path = manifestutil.manifestPath('foo', 'baz') os.makedirs(path) activePath = os.path.join(ServiceController.manifestPath('foo'), 'active') symlink('bar', activePath) response = self.app.get(url(controller='service', service='foo', action='get'), expect_errors = True) body = json.loads(response.body) print "************** response = %s" % body assert body['progress'] == 100 assert body['result']['activemanifest'] == 'bar' assert body['result']['manifest'] == ['bar', 'baz']
def testGet(self): path = ManifestController.manifestPath('foo', 'bar') os.makedirs(path) path = ManifestController.manifestPath('foo', 'baz') os.makedirs(path) activePath = os.path.join(ServiceController.manifestPath('foo'), 'active') symlink('bar', activePath) response = self.app.get(url(controller='service', service='foo', action='get'), expect_errors=True) body = json.loads(response.body) print "************** response = %s" % body assert body['progress'] == 100 assert body['result']['activemanifest'] == 'bar' assert body['result']['manifest'] == ['bar', 'baz']
def doRun(self): """ Main body of the thread Progress Info: Look at the list of packages this manifest has and figure out the progress of this manifest Progress 1 = manifest directory created Progress 2-80 = all packages downloaded Progress 81-99 = all packages untarred Progress 100 = all links done """ inProgressPath = ManifestCreate.inProgress(ManifestController.manifestPath(self.__service, self.__manifest)) try: if self.__service != 'agent': utils.checkDiskFull() installedPkgPath = ServiceController.installedPkgPath(self.__service) # This shouldn'trn happen but make sure there isn'trn already a manifest directory in progress if (os.path.isdir(inProgressPath)): # probably another manifest create thread died out half way. # Cleanup and reattempt manifest creation LOG.debug('Manifest inprogress found for service/manifest (%s/%s). Will cleanup and retry' % (self.__service, self.__manifest)) shutil.rmtree(inProgressPath) # make sure that the service path and installed package path exists if (not os.path.isdir(ServiceController.manifestPath(self.__service)) or not os.path.isdir(ServiceController.installedPkgPath(self.__service))): errCode = Errors.SERVICE_NOT_FOUND msg = 'Service (%s) for manifest (%s) not found' % (self.__service, self.__manifest) self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg) return # ok create the manifest path os.mkdir(inProgressPath) self._updateProgress(1) # figure out which of the packages are already there remainingPackages = {} for pkgUri in self.__packages: pkgDict = PackageUtil.parseUri(pkgUri) pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion']) pkgName = pkgDict['packageName'] if (not os.path.exists(pkgPath)) or ((self.__forcePackages is not None) and pkgName in self.__forcePackages): remainingPackages[pkgUri] = pkgDict else: symlink(pkgPath, os.path.join(inProgressPath, pkgDict['packageName'])) if self.__attemptDownload: # now make sure all the packages are downloaded try: self._downloadPackages(remainingPackages.keys(), skipProp = self.__skipProp) except AgentException as exc: # check if it is download error, then modify exception appropriately if exc.getCode() == Errors.DC_FAILED_DOWNLOAD: exc = AgentException(Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED, 'Manifest (%s/%s) failed downloading package - %s' % (self.__service, self.__manifest, exc.getMsg())) raise exc else: if len(remainingPackages) > 0: raise AgentException(Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST, 'Create Manifest (%s/%s) failed since package is not present and download has been disabled' % (self.__service, self.__manifest)) LOG.info('Completed download all packages for (%s/%s)' % (self.__service, self.__manifest)) # now untar the packages import re pkgSuffix = '.%s' % re.sub(r"\W", "", self.__manifest) self._untarPackages(remainingPackages, self.__service, ServiceController.installedPkgPath(self.__service), 0, self.__forcePackages, pkgSuffix) LOG.info('Completed untar all packages for (%s/%s)' % (self.__service, self.__manifest)) # now create the links for pkgDict in remainingPackages.itervalues(): pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion']) linkPath = os.path.join(inProgressPath, pkgDict['packageName']) # validate target folder does exist if not os.path.exists(pkgPath): raise AgentException(Errors.PACKAGE_PATH_ERROR, 'invalid untarred package at %s' % pkgPath) symlink(pkgPath, linkPath) # now move the inProgressPath to the final path manifestContentPath = ManifestController.manifestContentPath(self.__service, self.__manifest) os.rename(inProgressPath, ManifestController.manifestPath(self.__service, self.__manifest)) mfContentsFile = file(manifestContentPath, 'w') for pkgUri in self.__packages: mfContentsFile.write(('%s%s') % (pkgUri, os.linesep)) mfContentsFile.close() LOG.info('Completed create manifest directories for (%s/%s)' % (self.__service, self.__manifest)) LOG.info('Completed create manifest for (%s/%s)' % (self.__service, self.__manifest)) self._updateStatus(progress = 100) except AgentException as exc: LOG.info(exc.getMsg()) self._updateStatus(httpStatus = 500, error = exc.getCode(), errorMsg = exc.getMsg()) except Exception as exc: errCode = Errors.UNKNOWN_ERROR msg = 'Unknown error for (%s/%s) - %s - %s' % (self.__service, self.__manifest, str(exc), traceback.format_exc(2)) LOG.info(msg) self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg) finally: # clean up intermediate progress try: shutil.rmtree(inProgressPath) except OSError: pass
def doRun(self): """ Main body of the thread Progress Info: Look at the list of packages this manifest has and figure out the progress of this manifest Progress 1 = manifest directory created Progress 2-80 = all packages downloaded Progress 81-99 = all packages untarred Progress 100 = all links done """ try: if self.__service != 'agent': utils.checkDiskFull() installedPkgPath = manifestutil.installedPkgRootPath(self.__service) # figure out which of the packages are already there remainingPackages = {} pkgDict = PackageUtil.parseUri(self.__package) pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion']) remainingPackages[self.__package] = pkgDict if (not os.path.exists(pkgPath)): # now make sure all the packages are downloaded try: self._downloadPackages([self.__package]) except AgentException as exc: # check if it is download error, then modify exception appropriately if exc.getCode() == Errors.DC_FAILED_DOWNLOAD: exc = AgentException(Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED, 'Manifest (%s/%s) failed downloading package - %s' % (self.__service, self.__module, exc.getMsg())) raise exc LOG.info('Completed download packages for (%s/%s)' % (self.__service, self.__module)) # now untar the packages self._untarPackages(remainingPackages, self.__service, ServiceController.installedPkgPath(self.__service)) LOG.info('Completed untar all packages for (%s/%s)' % (self.__service, self.__module)) # now create the links for pkgDict in remainingPackages.itervalues(): pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion']) modulePath = os.path.join(manifestutil.moduleRootPath(self.__service), self.__module) # validate target folder does exist if not os.path.exists(pkgPath): raise AgentException(Errors.PACKAGE_PATH_ERROR, 'invalid untarred package at %s' % pkgPath) # remove existing module isExistingModule = os.path.exists(modulePath) if isExistingModule: execThread = self._getBuiltThread('deactivate') if execThread: super(ModuleCreate, self)._runExeThread(execThread) rmlink(modulePath) execThread = self._getBuiltThread('activate') if execThread: super(ModuleCreate, self)._runExeThread(execThread) symlink(pkgPath, modulePath) # load controllers from package manifestutil.processModule(self.__service, self.__module, not isExistingModule) LOG.info('Completed create module for (%s/%s)' % (self.__service, self.__module)) self._updateStatus(progress = 100) except AgentException as exc: LOG.info(exc.getMsg()) self._updateStatus(httpStatus = 500, error = exc.getCode(), errorMsg = exc.getMsg()) except Exception as exc: errCode = Errors.UNKNOWN_ERROR msg = 'Unknown error for (%s/%s) - %s - %s' % (self.__service, self.__module, str(exc), traceback.format_exc(2)) LOG.info(msg) self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
def doRun(self): """ Main body of the thread Progress Info: Look at the list of packages this manifest has and figure out the progress of this manifest Progress 1 = manifest directory created Progress 2-80 = all packages downloaded Progress 81-99 = all packages untarred Progress 100 = all links done """ try: if self.__service != 'agent': utils.checkDiskFull() installedPkgPath = manifestutil.installedPkgRootPath( self.__service) # figure out which of the packages are already there remainingPackages = {} pkgDict = PackageUtil.parseUri(self.__package) pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion']) remainingPackages[self.__package] = pkgDict if (not os.path.exists(pkgPath)): # now make sure all the packages are downloaded try: self._downloadPackages([self.__package]) except AgentException as exc: # check if it is download error, then modify exception appropriately if exc.getCode() == Errors.DC_FAILED_DOWNLOAD: exc = AgentException( Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED, 'Manifest (%s/%s) failed downloading package - %s' % (self.__service, self.__module, exc.getMsg())) raise exc LOG.info('Completed download packages for (%s/%s)' % (self.__service, self.__module)) # now untar the packages self._untarPackages( remainingPackages, self.__service, ServiceController.installedPkgPath(self.__service)) LOG.info('Completed untar all packages for (%s/%s)' % (self.__service, self.__module)) # now create the links for pkgDict in remainingPackages.itervalues(): pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion']) modulePath = os.path.join( manifestutil.moduleRootPath(self.__service), self.__module) # validate target folder does exist if not os.path.exists(pkgPath): raise AgentException( Errors.PACKAGE_PATH_ERROR, 'invalid untarred package at %s' % pkgPath) # remove existing module isExistingModule = os.path.exists(modulePath) if isExistingModule: execThreadTuple = self._getBuiltThread('deactivate') super(ModuleCreate, self)._runExeThread(self, execThreadTuple) rmlink(modulePath) execThreadTuple = self._getBuiltThread('activate') super(ModuleCreate, self)._runExeThread(execThreadTuple) symlink(pkgPath, modulePath) # load controllers from package manifestutil.processModule(self.__service, self.__module, not isExistingModule) LOG.info('Completed create module for (%s/%s)' % (self.__service, self.__module)) self._updateStatus(progress=100) except AgentException as exc: LOG.info(exc.getMsg()) self._updateStatus(httpStatus=500, error=exc.getCode(), errorMsg=exc.getMsg()) except Exception as exc: errCode = Errors.UNKNOWN_ERROR msg = 'Unknown error for (%s/%s) - %s - %s' % ( self.__service, self.__manifest, str(exc), traceback.format_exc(2)) LOG.info(msg) self._updateStatus(httpStatus=500, error=errCode, errorMsg=msg)
def _untarPackages(self, packages, service, untarRootPath, nicelevel=None, pgksNeedSuffix=None, pathSuffix=''): """ untar all the packages @params packages - list of dictionary of parsed packages uri @throws AgentException """ self._updateProgress(80) count = 0 for pkgDict in packages.itervalues(): count += 1 # check if package path exists already # if the package is already untarred, move on # else create the package path pkgName = pkgDict['packageName'] untarPath = os.path.join(untarRootPath, pkgName, '%s%s' % (pkgDict['packageVersion'], pathSuffix if ((pgksNeedSuffix is not None) and pkgName in pgksNeedSuffix) else '')) if os.path.exists(untarPath): # perhaps another thread has finished extraction, continue with next package continue os.makedirs(untarPath) try: self.untar(pkgDict['packagePath'], untarPath, nicelevel) except AgentException: rmrf(untarPath) # Note: i. atleast self should have 'rx' so that we can proceed setting 'rx' for group and others # if both belong to same group in future, then just self, group should be enough # ensure all parent dir of scripts dir have 'rx' so that we can really navigate to scripts dir and execute uname = getuserofpath(untarPath) chmod(untarPath, '+rx', sudoUser = uname) cronusPath = os.path.join(untarPath, 'cronus') if os.path.exists(cronusPath): uname = getuserofpath(cronusPath) chmod(cronusPath, '+rx', sudoUser = uname) # now give all scripts 'rx' permission scriptsPath = os.path.join(cronusPath, 'scripts') if os.path.exists(scriptsPath): uname = getuserofpath(scriptsPath) rchmod(scriptsPath, '+rx', sudoUser = uname) # issue #16, now add symlink to .appdata for easy access appdata_path = manifestutil.appDataPath(service) link_path = os.path.join(untarPath, '.appdata') LOG.info('Create .appdata symlink from %s to %s' % (link_path, appdata_path)) utils.symlink(appdata_path, link_path) #Running as cronus user when higher privilege service (i.e. not chown all the package into the application user) if (not isHigherPrivilegeService(service)): uname = configutil.getAppUser() uid, gid = utils.getUidGid(uname) rchown(untarPath, uid, gid) self._updateProgress(calcProgress(80, 99, float(count) / len(packages)))