def runCommand(self, cfg, argSet, args, profile = False, callback = None, repos=None): args = args[1:] text = argSet.pop('text', False) binary = argSet.pop('binary', False) if len(args) < 2: return self.usage() checkin.addFiles(args[1:], text = text, binary = binary, repos = repos, defaultToText = False)
def runCommand(self, cfg, argSet, args, profile=False, callback=None, repos=None): args = args[1:] text = argSet.pop('text', False) binary = argSet.pop('binary', False) if len(args) < 2: return self.usage() checkin.addFiles(args[1:], text=text, binary=binary, repos=repos, defaultToText=False)
def _commitRecipe(conaryclient, cfg, recipePath, message, branch=None): repos = conaryclient.getRepos() conaryCompat = compat.ConaryVersion() recipeClass, pathList = _getPathList(repos, cfg, recipePath) sourceName = recipeClass.name + ':source' log.info("Creating a copy of %s in the rMake internal repository..." % recipeClass.name) cwd = os.getcwd() recipeDir = tempfile.mkdtemp() log.resetErrorOccurred() try: fileNames = [] # Create a source trove that matches the recipe we're trying to cook if not branch: branch = versions.Branch([cfg.buildLabel]) targetLabel = cfg.getTargetLabel(branch) if compat.ConaryVersion().supportsNewPkgBranch(): buildBranch = branch.createShadow(targetLabel) kw = dict(buildBranch=buildBranch) else: buildBranch = versions.Branch([targetLabel]) kw={} cfg.buildLabel = targetLabel if not repos.getTroveLeavesByBranch( { sourceName : { buildBranch : None } }).get(sourceName, None): # we pass it None for repos to avoid the label-based check for # existing packages. checkin.newTrove(None, cfg, recipeClass.name, dir=recipeDir, **kw) else: # see if this package exists on our build branch checkin.checkout(repos, cfg, recipeDir, ['%s=%s' % (sourceName, buildBranch)]) os.chdir(recipeDir) sourceState = state.ConaryStateFromFile(recipeDir + '/CONARY').getSourceState() fileNames = dict((os.path.basename(x), x) for x in pathList) for (pathId, baseName, fileId, version) in list(sourceState.iterFileList()): # update or remove any currently existing files if baseName not in fileNames: sourceState.removeFilePath(baseName) else: shutil.copyfile(fileNames[baseName], os.path.join(recipeDir, baseName)) del fileNames[baseName] for baseName, path in fileNames.iteritems(): shutil.copyfile(path, os.path.join(recipeDir, baseName)) if conaryCompat.stateFileVersion() > 0: # mark all the files as binary - this this version can # never be checked in, it doesn't really matter, but # conary likes us to give a value. for fileName in fileNames: isConfig = _getConfigInfo(fileName) checkin.addFiles([fileName], binary=not isConfig, text=isConfig) else: checkin.addFiles(fileNames) _doCommit(recipePath, repos, cfg, 'Temporary recipe build for rmake') newState = conaryCompat.ConaryStateFromFile(recipeDir + '/CONARY', repos=repos) return newState.getSourceState().getNameVersionFlavor() finally: os.chdir(cwd) shutil.rmtree(recipeDir)
def _shadowAndCommit(conaryclient, cfg, recipeDir, stateFile, message): repos = conaryclient.getRepos() conaryCompat = compat.ConaryVersion() oldSourceVersion = stateFile.getVersion() targetLabel = cfg.getTargetLabel(oldSourceVersion) if not targetLabel: raise errors.RmakeError( 'Cannot cook local recipes unless a target label is set') skipped, cs = conaryclient.createShadowChangeSet(str(targetLabel), [stateFile.getNameVersionFlavor()]) recipePath = recipeDir + '/' + stateFile.getName().split(':')[0] + '.recipe' recipeClass, pathList = _getPathList(repos, cfg, recipePath, relative=True) troveName = stateFile.getName() troveVersion = stateFile.getVersion() if not skipped: signAbsoluteChangeset(cs, None) repos.commitChangeSet(cs) log.info("Shadowing %s to internal repository..." % troveName) shadowBranch = troveVersion.createShadow(targetLabel).branch() shadowVersion = repos.findTrove(None, (troveName, str(shadowBranch), None), None)[0][1] cwd = os.getcwd() prefix = 'rmake-shadow-%s-' % troveName.split(':')[0] shadowSourceDir = tempfile.mkdtemp(prefix=prefix) try: log.info("Committing local changes to %s to the" " internal repository..." % troveName) log.resetErrorOccurred() checkin.checkout(repos, cfg, shadowSourceDir, ['%s=%s' % (troveName, shadowVersion)]) if compat.ConaryVersion().stateFileVersion() > 0: kw = dict(repos=repos) else: kw = {} # grab new and old state and make any modifications due to adding # or deleting of files (we assume files that don't exist are # autosource and can be ignored) oldState = conaryCompat.ConaryStateFromFile(recipeDir + '/CONARY', repos=repos).getSourceState() newConaryState = conaryCompat.ConaryStateFromFile( shadowSourceDir + '/CONARY', repos=repos) newState = newConaryState.getSourceState() neededFiles = set(x[1] for x in oldState.iterFileList() if os.path.exists(os.path.join(recipeDir, x[1]))) neededFiles.update(pathList) autoSourceFiles = set(x[1] for x in oldState.iterFileList() if oldState.fileIsAutoSource(x[0])) existingFiles = set(x[1] for x in newState.iterFileList() if os.path.exists(os.path.join(shadowSourceDir, x[1]))) toCopy = neededFiles & existingFiles toDel = existingFiles - neededFiles toAdd = neededFiles - existingFiles for sourceFile in (toCopy | toAdd): newPath = os.path.join(shadowSourceDir, sourceFile) if os.path.dirname(sourceFile): util.mkdirChain(os.path.dirname(newPath)) if os.path.isdir(sourceFile): util.mkdirChain(newPath) else: shutil.copyfile(os.path.join(recipeDir, sourceFile), newPath) os.chdir(shadowSourceDir) if hasattr(cfg.sourceSearchDir, '_getUnexpanded'): cfg.configKey('sourceSearchDir', cfg.sourceSearchDir._getUnexpanded()) for f in toDel: checkin.removeFile(f) if toDel: # toDel modifies the CONARY file on disk, so reload with the # changes made there. newState = conaryCompat.ConaryStateFromFile( shadowSourceDir + '/CONARY', repos=repos).getSourceState() if conaryCompat.stateFileVersion() == 0: checkin.addFiles(toAdd) else: oldPathIds = dict((x[1], x[0]) for x in oldState.iterFileList()) for path in toAdd: if path in oldPathIds: isConfig = oldState.fileIsConfig(oldPathIds[path]) else: isConfig = _getConfigInfo(path) checkin.addFiles([path], binary=not isConfig, text=isConfig) if toAdd: # get the new pathIDs for all the added troves, # since we can't set the refresh setting without the # needed pathIds newState = conaryCompat.ConaryStateFromFile( shadowSourceDir + '/CONARY', repos=repos).getSourceState() newPathIds = dict((x[1], x[0]) for x in newState.iterFileList()) for path in (toCopy | toAdd): if path in oldPathIds: isConfig = oldState.fileIsConfig(oldPathIds[path]) else: isConfig = _getConfigInfo(path) newState.fileIsConfig(newPathIds[path], isConfig) for path in autoSourceFiles: if path in newPathIds: needsRefresh = oldState.fileNeedsRefresh(oldPathIds[path]) newState.fileNeedsRefresh(newPathIds[path], needsRefresh) # if the factory changed, update it if newState.getFactory() != oldState.getFactory(): newState.setFactory(oldState.getFactory()) # we may have modified the state file. Write it back out to # disk so it will be picked up by the commit. newConaryState.setSourceState(newState) newConaryState.write(shadowSourceDir + '/CONARY') if message is None and compat.ConaryVersion().supportsCloneCallback(): message = 'Automated rMake commit' _doCommit('%s/%s' % (recipeDir, troveName), repos, cfg, message) newState = state.ConaryStateFromFile(shadowSourceDir + '/CONARY', **kw) return newState.getSourceState().getNameVersionFlavor() finally: os.chdir(cwd) if hasattr(cfg.sourceSearchDir, '_getUnexpanded'): cfg.configKey('sourceSearchDir', cfg.sourceSearchDir._getUnexpanded()) shutil.rmtree(shadowSourceDir)
def derive(repos, cfg, targetLabel, troveSpec, checkoutDir=None, extract=False, info=False, callback=None): """ Performs all the commands necessary to create a derived recipe. First it shadows the package, then it creates a checkout of the shadow and converts the checkout to a derived recipe package. Finally if extract = True, it installs an version of the binary package into a root. @param repos: trovesource to search for and derive packages from @param cfg: configuration to use when deriving the package @type cfg: ConaryConfiguration object @param targetLabel: label to derive from @type targetLabel: versions.Label @param checkoutDir: directory to create the checkout in. If None, defaults to currentDir + packageName. @param extract: If True, creates a subdirectory of the checkout named _ROOT_ with the contents of the binary of the derived package. @param info: If true, only display the information about the shadow that would be performed if the derive command were completed. @param callback: """ origDir = os.getcwd() try: if callback is None: callback = DeriveCallback() if isinstance(troveSpec, tuple): troveName, versionSpec, flavor = troveSpec versionSpec = str(versionSpec) troveSpec = cmdline.toTroveSpec(troveName, versionSpec, flavor) else: troveName, versionSpec, flavor = cmdline.parseTroveSpec(troveSpec) if isinstance(targetLabel, str): targetLabel = Label(targetLabel) troveName, versionSpec, flavor = cmdline.parseTroveSpec(troveSpec) result = repos.findTrove(cfg.buildLabel, (troveName, versionSpec, flavor), cfg.flavor) # findTrove shouldn't return multiple items for one package anymore # when a flavor is specified. troveToDerive, = result # displaying output along the screen allows there to be a record # of what operations were performed. Since this command is # an aggregate of several commands I think that is appropriate, # rather than simply using a progress callback. log.info('Shadowing %s=%s[%s] onto %s' % (troveToDerive[0], troveToDerive[1], troveToDerive[2], targetLabel)) if info: cfg.interactive = False error = branch.branch(repos, cfg, str(targetLabel), ['%s=%s[%s]' % troveToDerive], makeShadow=True, sourceOnly=True, binaryOnly=False, allowEmptyShadow=True, info=info) if info or error: return shadowedVersion = troveToDerive[1].createShadow(targetLabel) shadowedVersion = shadowedVersion.getSourceVersion(False) troveName = troveName.split(':')[0] checkoutDir = checkoutDir or troveName checkin.checkout(repos, cfg, checkoutDir, ["%s=%s" % (troveName, shadowedVersion)], callback=callback) os.chdir(checkoutDir) nvfs = repos.getTrovesBySource(troveToDerive[0] + ':source', troveToDerive[1].getSourceVersion()) trvs = repos.getTroves(nvfs) hasCapsule = [x for x in trvs if x.troveInfo.capsule.type()] if hasCapsule: derivedRecipeType = 'DerivedCapsuleRecipe' removeText = '' else: derivedRecipeType = 'DerivedPackageRecipe' removeText = \ """ # This appliance uses PHP as a command interpreter but does # not include a web server, so remove the file that creates # a dependency on the web server r.Remove('/etc/httpd/conf.d/php.conf') """ log.info('Rewriting recipe file') recipeName = troveName + '.recipe' className = util.convertPackageNameToClassName(troveName) derivedRecipe = """ class %(className)sRecipe(%(recipeBaseClass)s): name = '%(name)s' version = '%(version)s' def setup(r): ''' In this recipe, you can make modifications to the package. Examples: # This appliance has high-memory-use PHP scripts r.Replace('memory_limit = 8M', 'memory_limit = 32M', '/etc/php.ini') %(removeText)s # This appliance requires that a few binaries be replaced # with binaries built from a custom archive that includes # a Makefile that honors the DESTDIR variable for its # install target. r.addArchive('foo.tar.gz') r.Make() r.MakeInstall() # This appliance requires an extra configuration file r.Create('/etc/myconfigfile', contents='some data') ''' """ % dict(className=className, name=troveName, version=shadowedVersion.trailingRevision().getVersion(), recipeBaseClass=derivedRecipeType, removeText=removeText) open(recipeName, 'w').write(derivedRecipe) log.info('Removing extra files from checkout') conaryState = state.ConaryStateFromFile('CONARY', repos) sourceState = conaryState.getSourceState() # clear the factory since we don't care about how the parent trove was # created sourceState.setFactory('') addRecipe = True for (pathId, path, fileId, version) in list(sourceState.iterFileList()): if path == recipeName: addRecipe = False continue sourceState.removeFile(pathId) if util.exists(path): statInfo = os.lstat(path) try: if statInfo.st_mode & stat.S_IFDIR: os.rmdir(path) else: os.unlink(path) except OSError, e: log.warning("cannot remove %s: %s" % (path, e.strerror)) conaryState.write('CONARY') if addRecipe: checkin.addFiles([recipeName]) if extract: log.info('extracting files from %s=%s[%s]' % (troveToDerive)) # extract to _ROOT_ extractDir = os.path.join(os.getcwd(), '_ROOT_') ts = [(troveToDerive[0], (None, None), (troveToDerive[1], troveToDerive[2]), True)] cs = repos.createChangeSet(ts, recurse=True) ChangesetExploder(cs, extractDir) # extract to _OLD_ROOT_ secondDir = os.path.join(os.getcwd(), '_OLD_ROOT_') cs = repos.createChangeSet(ts, recurse=True) ChangesetExploder(cs, secondDir)
def derive(repos, cfg, targetLabel, troveSpec, checkoutDir=None, extract=False, info=False, callback=None): """ Performs all the commands necessary to create a derived recipe. First it shadows the package, then it creates a checkout of the shadow and converts the checkout to a derived recipe package. Finally if extract = True, it installs an version of the binary package into a root. @param repos: trovesource to search for and derive packages from @param cfg: configuration to use when deriving the package @type cfg: ConaryConfiguration object @param targetLabel: label to derive from @type targetLabel: versions.Label @param checkoutDir: directory to create the checkout in. If None, defaults to currentDir + packageName. @param extract: If True, creates a subdirectory of the checkout named _ROOT_ with the contents of the binary of the derived package. @param info: If true, only display the information about the shadow that would be performed if the derive command were completed. @param callback: """ origDir = os.getcwd() try: if callback is None: callback = DeriveCallback() if isinstance(troveSpec, tuple): troveName, versionSpec, flavor = troveSpec versionSpec = str(versionSpec) troveSpec = cmdline.toTroveSpec(troveName, versionSpec, flavor) else: troveName, versionSpec, flavor = cmdline.parseTroveSpec(troveSpec) if isinstance(targetLabel, str): targetLabel = Label(targetLabel) troveName, versionSpec, flavor = cmdline.parseTroveSpec(troveSpec) result = repos.findTrove(cfg.buildLabel, (troveName, versionSpec, flavor), cfg.flavor) # findTrove shouldn't return multiple items for one package anymore # when a flavor is specified. troveToDerive, = result # displaying output along the screen allows there to be a record # of what operations were performed. Since this command is # an aggregate of several commands I think that is appropriate, # rather than simply using a progress callback. log.info('Shadowing %s=%s[%s] onto %s' % (troveToDerive[0], troveToDerive[1], troveToDerive[2], targetLabel)) if info: cfg.interactive = False error = branch.branch(repos, cfg, str(targetLabel), ['%s=%s[%s]'%troveToDerive], makeShadow=True, sourceOnly=True, binaryOnly=False, allowEmptyShadow=True, info=info) if info or error: return shadowedVersion = troveToDerive[1].createShadow(targetLabel) shadowedVersion = shadowedVersion.getSourceVersion(False) troveName = troveName.split(':')[0] checkoutDir = checkoutDir or troveName checkin.checkout(repos, cfg, checkoutDir, ["%s=%s" % (troveName, shadowedVersion)], callback=callback) os.chdir(checkoutDir) nvfs = repos.getTrovesBySource(troveToDerive[0]+':source', troveToDerive[1].getSourceVersion()) trvs = repos.getTroves(nvfs) hasCapsule = [ x for x in trvs if x.troveInfo.capsule.type() ] if hasCapsule: derivedRecipeType = 'DerivedCapsuleRecipe' removeText = '' else: derivedRecipeType = 'DerivedPackageRecipe' removeText = \ """ # This appliance uses PHP as a command interpreter but does # not include a web server, so remove the file that creates # a dependency on the web server r.Remove('/etc/httpd/conf.d/php.conf') """ log.info('Rewriting recipe file') recipeName = troveName + '.recipe' className = util.convertPackageNameToClassName(troveName) derivedRecipe = """ class %(className)sRecipe(%(recipeBaseClass)s): name = '%(name)s' version = '%(version)s' def setup(r): ''' In this recipe, you can make modifications to the package. Examples: # This appliance has high-memory-use PHP scripts r.Replace('memory_limit = 8M', 'memory_limit = 32M', '/etc/php.ini') %(removeText)s # This appliance requires that a few binaries be replaced # with binaries built from a custom archive that includes # a Makefile that honors the DESTDIR variable for its # install target. r.addArchive('foo.tar.gz') r.Make() r.MakeInstall() # This appliance requires an extra configuration file r.Create('/etc/myconfigfile', contents='some data') ''' """ % dict(className=className, name=troveName, version=shadowedVersion.trailingRevision().getVersion(), recipeBaseClass=derivedRecipeType, removeText=removeText) open(recipeName, 'w').write(derivedRecipe) log.info('Removing extra files from checkout') conaryState = state.ConaryStateFromFile('CONARY', repos) sourceState = conaryState.getSourceState() # clear the factory since we don't care about how the parent trove was # created sourceState.setFactory('') addRecipe=True for (pathId, path, fileId, version) in list(sourceState.iterFileList()): if path == recipeName: addRecipe = False continue sourceState.removeFile(pathId) if util.exists(path): statInfo = os.lstat(path) try: if statInfo.st_mode & stat.S_IFDIR: os.rmdir(path) else: os.unlink(path) except OSError, e: log.warning("cannot remove %s: %s" % (path, e.strerror)) conaryState.write('CONARY') if addRecipe: checkin.addFiles([recipeName]) if extract: log.info('extracting files from %s=%s[%s]' % (troveToDerive)) # extract to _ROOT_ extractDir = os.path.join(os.getcwd(), '_ROOT_') ts = [ (troveToDerive[0], (None, None), (troveToDerive[1], troveToDerive[2]), True) ] cs = repos.createChangeSet(ts, recurse = True) ChangesetExploder(cs, extractDir) # extract to _OLD_ROOT_ secondDir = os.path.join(os.getcwd(), '_OLD_ROOT_') cs = repos.createChangeSet(ts, recurse = True) ChangesetExploder(cs, secondDir)