def testCheckoutMissingKey(self):
fingerprint = '95B457D16843B21EA3FC73BBC7C32FC1F94E405E'
# supply the pass phrase for our private key
keyCache = openpgpkey.getKeyCache()
keyCache.getPrivateKey(fingerprint, '111111')
self.cfg.signatureKey = fingerprint
self.addQuickTestComponent("test:source", "1.0-1-1")
signtrove.signTroves(self.cfg, ["test:source"])
repos = self.openRepository()
# utterly prevent the keycache from knowing about the key,
# but give it a place to store a keyserver retrieved key.
newKeyCache = openpgpkey.OpenPGPKeyFileCache()
tmpPath = mkdtemp()
pubRing = self.cfg.pubRing
self.cfg.pubRing = [tmpPath + '/pubring.gpg']
newKeyCache.publicPaths = self.cfg.pubRing
keyCacheCallback = openpgpkey.KeyCacheCallback(repos, self.cfg)
newKeyCache.setCallback(keyCacheCallback)
openpgpkey.setKeyCache(newKeyCache)
try:
checkout(repos, self.cfg, self.workDir, ["test"])
newKeyCache.getPublicKey(fingerprint)
finally:
self.cfg.pubRing = pubRing
openpgpkey.setKeyCache(keyCache)
util.rmtree(tmpPath)
def checkoutPackage(self, callData, buildCfg, troveName, troveVersion):
buildCfg = self._updateConfig(buildCfg)
repos = self._getRepos(buildCfg, caching=True)
workDir = buildCfg.root + "/tmp/rmake"
troveName = troveName.split(":")[0]
checkoutPath = "%s/%s-checkout" % (workDir, troveName)
util.mkdirChain(workDir)
# make sure we don't set the context during checkout, as
# the context doesn't exist at this point.
buildCfg.context = None
checkin.checkout(repos, buildCfg, checkoutPath, ["%s=%s" % (troveName, troveVersion)])
os.chmod(checkoutPath, 0775)
def runCommand(self, cfg, argSet, args, profile = False,
callback = None, repos = None):
args = args[1:]
if argSet.has_key("dir"):
dir = argSet['dir']
del argSet['dir']
else:
dir = None
if argSet or (len(args) < 2) or (dir and len(args) != 2):
# no args other than --dir, and --dir implies only one trove
return self.usage()
coArgs = [repos, cfg, dir, args[1:], callback]
checkin.checkout(*coArgs)
def checkoutPackage(self, callData, buildCfg, troveName, troveVersion):
buildCfg = self._updateConfig(buildCfg)
repos = self._getRepos(buildCfg, caching=True)
workDir = buildCfg.root + '/tmp/rmake'
troveName = troveName.split(':')[0]
checkoutPath = '%s/%s-checkout' % (workDir, troveName)
util.mkdirChain(workDir)
# make sure we don't set the context during checkout, as
# the context doesn't exist at this point.
buildCfg.context = None
checkin.checkout(repos, buildCfg, checkoutPath,
['%s=%s' % (troveName, troveVersion)])
os.chmod(checkoutPath, 0775)
def runCommand(self,
cfg,
argSet,
args,
profile=False,
callback=None,
repos=None):
args = args[1:]
if argSet.has_key("dir"):
dir = argSet['dir']
del argSet['dir']
else:
dir = None
if argSet or (len(args) < 2) or (dir and len(args) != 2):
# no args other than --dir, and --dir implies only one trove
return self.usage()
coArgs = [repos, cfg, dir, args[1:], callback]
checkin.checkout(*coArgs)
def checkout(self, package, label, targetDir=None):
"""
Create a subdirectory containing a checkout of a conary
source package. Similar to the C{cvc checkout} command.
@param package: name of package
@type package: string
@param label: label to find package on
@type label: string
@param targetDir: subdirectory into which to check out the package,
This is the final directory into which the checked-out files
will be placed, not the parent directory in which a subdirectory
will be created.
defaults to C{package}
@type targetDir: string
"""
cfg = self.getConaryConfig()
checkin.checkout(self._getRepositoryClient(), cfg, targetDir,
['%s=%s' % (package, label)])
def checkout(self, package, label, targetDir=None):
"""
Create a subdirectory containing a checkout of a conary
source package. Similar to the C{cvc checkout} command.
@param package: name of package
@type package: string
@param label: label to find package on
@type label: string
@param targetDir: subdirectory into which to check out the package,
This is the final directory into which the checked-out files
will be placed, not the parent directory in which a subdirectory
will be created.
defaults to C{package}
@type targetDir: string
"""
cfg = self.getConaryConfig()
checkin.checkout(self._getRepositoryClient(), cfg,
targetDir, ['%s=%s' % (package, label)])
def testSignedCheckout(self):
fingerprint = 'F7440D78FE813C882212C2BF8AC2828190B1E477'
# supply the pass phrase for our private key
keyCache = openpgpkey.getKeyCache()
keyCache.getPrivateKey(fingerprint, '111111')
self.addQuickTestComponent("test:source", "1.0-1-1")
self.cfg.signatureKey = fingerprint
self.cfg.quiet = True
# put the public key into the repo
keyRing = open(
resources.get_path('conary_test', 'archive', 'pubring.gpg'))
keyData = openpgpfile.exportKey(fingerprint, keyRing)
keyData.seek(0)
keyData = keyData.read()
repos = self.openRepository()
repos.addNewPGPKey(self.cfg.buildLabel, 'test', keyData)
signtrove.signTroves(self.cfg, ["test:source"])
# alter key's trust value and determine that the doUpdate
# code properly verifies trust thresholds
pubKey = keyCache.getPublicKey(fingerprint)
pubKey.trustLevel = openpgpfile.TRUST_UNTRUSTED
keyCache.publicDict[fingerprint] = pubKey
self.cfg.trustThreshold = openpgpfile.TRUST_ULTIMATE
self.logFilter.add()
try:
checkout(repos, self.cfg, self.workDir, ["test"])
self.fail("checkin.checkout did not properly verify trust levels")
except DigitalSignatureVerificationError:
self.logFilter.compare([
'warning: The trove test:source has '
'signatures generated with untrusted keys. You can either '
'resign the trove with a key that you trust, or add one of the '
'keys to the list of trusted keys (the trustedKeys '
'configuration option). The keys that were not trusted are: '
'90B1E477'
])
branch = stateFile.getBranch()
if (troveName, branch) not in committedSourcesByNB:
continue
stateVersion = stateFile.getVersion()
newVersion = committedSourcesByNB[troveName, branch]
if stateVersion != versions.NewVersion():
log.info('Updating %s after commit' % recipeDir)
try:
# Added in CNY-3035
checkin.nologUpdateSrc(repos, [recipeDir])
except checkin.builderrors.UpToDate:
pass # Don't mention if the source is already up to date
except checkin.builderrors.CheckinError, e:
e.logError()
except AttributeError:
checkin.updateSrc(repos, [recipeDir])
else:
log.info('Replacing CONARY file %s after initial commit' % recipeDir)
d = tempfile.mkdtemp(dir='/var/tmp',
prefix='rmake-update-%s' % troveName)
# check out the newly committed version of this recipe
# (too much work, since we only want the CONARY file not
# the file contents. Oh well.) and update the CONARY state
# file to it. This will not lose any local changes made after
# the build started but _will_ cause any added/removed files
# to need to be readded/removed via cvc add/remove
checkin.checkout(repos, cfg, d, ['%s=%s' % (troveName, newVersion)])
newConaryStateFile = state.ConaryStateFromFile(d + '/CONARY')
newConaryStateFile.setContext(context)
newConaryStateFile.write(stateFilePath)
def _commitRecipe(conaryclient, cfg, recipePath, message, branch=None):
repos = conaryclient.getRepos()
conaryCompat = compat.ConaryVersion()
recipeClass, pathList = _getPathList(repos, cfg, recipePath)
sourceName = recipeClass.name + ':source'
log.info("Creating a copy of %s in the rMake internal repository..." % recipeClass.name)
cwd = os.getcwd()
recipeDir = tempfile.mkdtemp()
log.resetErrorOccurred()
try:
fileNames = []
# Create a source trove that matches the recipe we're trying to cook
if not branch:
branch = versions.Branch([cfg.buildLabel])
targetLabel = cfg.getTargetLabel(branch)
if compat.ConaryVersion().supportsNewPkgBranch():
buildBranch = branch.createShadow(targetLabel)
kw = dict(buildBranch=buildBranch)
else:
buildBranch = versions.Branch([targetLabel])
kw={}
cfg.buildLabel = targetLabel
if not repos.getTroveLeavesByBranch(
{ sourceName : { buildBranch : None } }).get(sourceName, None):
# we pass it None for repos to avoid the label-based check for
# existing packages.
checkin.newTrove(None, cfg, recipeClass.name, dir=recipeDir, **kw)
else:
# see if this package exists on our build branch
checkin.checkout(repos, cfg, recipeDir,
['%s=%s' % (sourceName, buildBranch)])
os.chdir(recipeDir)
sourceState = state.ConaryStateFromFile(recipeDir + '/CONARY').getSourceState()
fileNames = dict((os.path.basename(x), x) for x in pathList)
for (pathId, baseName, fileId, version) in list(sourceState.iterFileList()):
# update or remove any currently existing files
if baseName not in fileNames:
sourceState.removeFilePath(baseName)
else:
shutil.copyfile(fileNames[baseName],
os.path.join(recipeDir, baseName))
del fileNames[baseName]
for baseName, path in fileNames.iteritems():
shutil.copyfile(path, os.path.join(recipeDir, baseName))
if conaryCompat.stateFileVersion() > 0:
# mark all the files as binary - this this version can
# never be checked in, it doesn't really matter, but
# conary likes us to give a value.
for fileName in fileNames:
isConfig = _getConfigInfo(fileName)
checkin.addFiles([fileName], binary=not isConfig, text=isConfig)
else:
checkin.addFiles(fileNames)
_doCommit(recipePath, repos, cfg, 'Temporary recipe build for rmake')
newState = conaryCompat.ConaryStateFromFile(recipeDir + '/CONARY',
repos=repos)
return newState.getSourceState().getNameVersionFlavor()
finally:
os.chdir(cwd)
shutil.rmtree(recipeDir)
def _shadowAndCommit(conaryclient, cfg, recipeDir, stateFile, message):
repos = conaryclient.getRepos()
conaryCompat = compat.ConaryVersion()
oldSourceVersion = stateFile.getVersion()
targetLabel = cfg.getTargetLabel(oldSourceVersion)
if not targetLabel:
raise errors.RmakeError(
'Cannot cook local recipes unless a target label is set')
skipped, cs = conaryclient.createShadowChangeSet(str(targetLabel),
[stateFile.getNameVersionFlavor()])
recipePath = recipeDir + '/' + stateFile.getName().split(':')[0] + '.recipe'
recipeClass, pathList = _getPathList(repos, cfg, recipePath, relative=True)
troveName = stateFile.getName()
troveVersion = stateFile.getVersion()
if not skipped:
signAbsoluteChangeset(cs, None)
repos.commitChangeSet(cs)
log.info("Shadowing %s to internal repository..." % troveName)
shadowBranch = troveVersion.createShadow(targetLabel).branch()
shadowVersion = repos.findTrove(None,
(troveName, str(shadowBranch),
None), None)[0][1]
cwd = os.getcwd()
prefix = 'rmake-shadow-%s-' % troveName.split(':')[0]
shadowSourceDir = tempfile.mkdtemp(prefix=prefix)
try:
log.info("Committing local changes to %s to the"
" internal repository..." % troveName)
log.resetErrorOccurred()
checkin.checkout(repos, cfg, shadowSourceDir,
['%s=%s' % (troveName, shadowVersion)])
if compat.ConaryVersion().stateFileVersion() > 0:
kw = dict(repos=repos)
else:
kw = {}
# grab new and old state and make any modifications due to adding
# or deleting of files (we assume files that don't exist are
# autosource and can be ignored)
oldState = conaryCompat.ConaryStateFromFile(recipeDir + '/CONARY',
repos=repos).getSourceState()
newConaryState = conaryCompat.ConaryStateFromFile(
shadowSourceDir + '/CONARY',
repos=repos)
newState = newConaryState.getSourceState()
neededFiles = set(x[1] for x in oldState.iterFileList()
if os.path.exists(os.path.join(recipeDir, x[1])))
neededFiles.update(pathList)
autoSourceFiles = set(x[1] for x in oldState.iterFileList()
if oldState.fileIsAutoSource(x[0]))
existingFiles = set(x[1] for x in newState.iterFileList()
if os.path.exists(os.path.join(shadowSourceDir, x[1])))
toCopy = neededFiles & existingFiles
toDel = existingFiles - neededFiles
toAdd = neededFiles - existingFiles
for sourceFile in (toCopy | toAdd):
newPath = os.path.join(shadowSourceDir, sourceFile)
if os.path.dirname(sourceFile):
util.mkdirChain(os.path.dirname(newPath))
if os.path.isdir(sourceFile):
util.mkdirChain(newPath)
else:
shutil.copyfile(os.path.join(recipeDir, sourceFile), newPath)
os.chdir(shadowSourceDir)
if hasattr(cfg.sourceSearchDir, '_getUnexpanded'):
cfg.configKey('sourceSearchDir',
cfg.sourceSearchDir._getUnexpanded())
for f in toDel:
checkin.removeFile(f)
if toDel:
# toDel modifies the CONARY file on disk, so reload with the
# changes made there.
newState = conaryCompat.ConaryStateFromFile(
shadowSourceDir + '/CONARY',
repos=repos).getSourceState()
if conaryCompat.stateFileVersion() == 0:
checkin.addFiles(toAdd)
else:
oldPathIds = dict((x[1], x[0]) for x in oldState.iterFileList())
for path in toAdd:
if path in oldPathIds:
isConfig = oldState.fileIsConfig(oldPathIds[path])
else:
isConfig = _getConfigInfo(path)
checkin.addFiles([path], binary=not isConfig, text=isConfig)
if toAdd:
# get the new pathIDs for all the added troves,
# since we can't set the refresh setting without the
# needed pathIds
newState = conaryCompat.ConaryStateFromFile(
shadowSourceDir + '/CONARY',
repos=repos).getSourceState()
newPathIds = dict((x[1], x[0]) for x in newState.iterFileList())
for path in (toCopy | toAdd):
if path in oldPathIds:
isConfig = oldState.fileIsConfig(oldPathIds[path])
else:
isConfig = _getConfigInfo(path)
newState.fileIsConfig(newPathIds[path], isConfig)
for path in autoSourceFiles:
if path in newPathIds:
needsRefresh = oldState.fileNeedsRefresh(oldPathIds[path])
newState.fileNeedsRefresh(newPathIds[path], needsRefresh)
# if the factory changed, update it
if newState.getFactory() != oldState.getFactory():
newState.setFactory(oldState.getFactory())
# we may have modified the state file. Write it back out to
# disk so it will be picked up by the commit.
newConaryState.setSourceState(newState)
newConaryState.write(shadowSourceDir + '/CONARY')
if message is None and compat.ConaryVersion().supportsCloneCallback():
message = 'Automated rMake commit'
_doCommit('%s/%s' % (recipeDir, troveName), repos, cfg, message)
newState = state.ConaryStateFromFile(shadowSourceDir + '/CONARY', **kw)
return newState.getSourceState().getNameVersionFlavor()
finally:
os.chdir(cwd)
if hasattr(cfg.sourceSearchDir, '_getUnexpanded'):
cfg.configKey('sourceSearchDir',
cfg.sourceSearchDir._getUnexpanded())
shutil.rmtree(shadowSourceDir)
checkin.nologUpdateSrc(repos, [recipeDir])
except checkin.builderrors.UpToDate:
pass # Don't mention if the source is already up to date
except checkin.builderrors.CheckinError, e:
e.logError()
except AttributeError:
checkin.updateSrc(repos, [recipeDir])
else:
curDir = os.getcwd()
try:
os.chdir(recipeDir)
checkin.updateSrc(repos)
finally:
os.chdir(curDir)
else:
log.info('Replacing CONARY file %s after initial commit' %
recipeDir)
d = tempfile.mkdtemp(dir='/var/tmp',
prefix='rmake-update-%s' % troveName)
# check out the newly committed version of this recipe
# (too much work, since we only want the CONARY file not
# the file contents. Oh well.) and update the CONARY state
# file to it. This will not lose any local changes made after
# the build started but _will_ cause any added/removed files
# to need to be readded/removed via cvc add/remove
checkin.checkout(repos, cfg, d,
['%s=%s' % (troveName, newVersion)])
newConaryStateFile = state.ConaryStateFromFile(d + '/CONARY')
newConaryStateFile.setContext(context)
newConaryStateFile.write(stateFilePath)
def derive(repos,
cfg,
targetLabel,
troveSpec,
checkoutDir=None,
extract=False,
info=False,
callback=None):
"""
Performs all the commands necessary to create a derived recipe.
First it shadows the package, then it creates a checkout of the shadow
and converts the checkout to a derived recipe package.
Finally if extract = True, it installs an version of the binary
package into a root.
@param repos: trovesource to search for and derive packages from
@param cfg: configuration to use when deriving the package
@type cfg: ConaryConfiguration object
@param targetLabel: label to derive from
@type targetLabel: versions.Label
@param checkoutDir: directory to create the checkout in. If None,
defaults to currentDir + packageName.
@param extract: If True, creates a subdirectory of the checkout named
_ROOT_ with the contents of the binary of the derived
package.
@param info: If true, only display the information about the shadow
that would be performed if the derive command were
completed.
@param callback:
"""
origDir = os.getcwd()
try:
if callback is None:
callback = DeriveCallback()
if isinstance(troveSpec, tuple):
troveName, versionSpec, flavor = troveSpec
versionSpec = str(versionSpec)
troveSpec = cmdline.toTroveSpec(troveName, versionSpec, flavor)
else:
troveName, versionSpec, flavor = cmdline.parseTroveSpec(troveSpec)
if isinstance(targetLabel, str):
targetLabel = Label(targetLabel)
troveName, versionSpec, flavor = cmdline.parseTroveSpec(troveSpec)
result = repos.findTrove(cfg.buildLabel,
(troveName, versionSpec, flavor), cfg.flavor)
# findTrove shouldn't return multiple items for one package anymore
# when a flavor is specified.
troveToDerive, = result
# displaying output along the screen allows there to be a record
# of what operations were performed. Since this command is
# an aggregate of several commands I think that is appropriate,
# rather than simply using a progress callback.
log.info('Shadowing %s=%s[%s] onto %s' %
(troveToDerive[0], troveToDerive[1], troveToDerive[2],
targetLabel))
if info:
cfg.interactive = False
error = branch.branch(repos,
cfg,
str(targetLabel), ['%s=%s[%s]' % troveToDerive],
makeShadow=True,
sourceOnly=True,
binaryOnly=False,
allowEmptyShadow=True,
info=info)
if info or error:
return
shadowedVersion = troveToDerive[1].createShadow(targetLabel)
shadowedVersion = shadowedVersion.getSourceVersion(False)
troveName = troveName.split(':')[0]
checkoutDir = checkoutDir or troveName
checkin.checkout(repos,
cfg,
checkoutDir, ["%s=%s" % (troveName, shadowedVersion)],
callback=callback)
os.chdir(checkoutDir)
nvfs = repos.getTrovesBySource(troveToDerive[0] + ':source',
troveToDerive[1].getSourceVersion())
trvs = repos.getTroves(nvfs)
hasCapsule = [x for x in trvs if x.troveInfo.capsule.type()]
if hasCapsule:
derivedRecipeType = 'DerivedCapsuleRecipe'
removeText = ''
else:
derivedRecipeType = 'DerivedPackageRecipe'
removeText = \
"""
# This appliance uses PHP as a command interpreter but does
# not include a web server, so remove the file that creates
# a dependency on the web server
r.Remove('/etc/httpd/conf.d/php.conf')
"""
log.info('Rewriting recipe file')
recipeName = troveName + '.recipe'
className = util.convertPackageNameToClassName(troveName)
derivedRecipe = """
class %(className)sRecipe(%(recipeBaseClass)s):
name = '%(name)s'
version = '%(version)s'
def setup(r):
'''
In this recipe, you can make modifications to the package.
Examples:
# This appliance has high-memory-use PHP scripts
r.Replace('memory_limit = 8M', 'memory_limit = 32M', '/etc/php.ini')
%(removeText)s
# This appliance requires that a few binaries be replaced
# with binaries built from a custom archive that includes
# a Makefile that honors the DESTDIR variable for its
# install target.
r.addArchive('foo.tar.gz')
r.Make()
r.MakeInstall()
# This appliance requires an extra configuration file
r.Create('/etc/myconfigfile', contents='some data')
'''
""" % dict(className=className,
name=troveName,
version=shadowedVersion.trailingRevision().getVersion(),
recipeBaseClass=derivedRecipeType,
removeText=removeText)
open(recipeName, 'w').write(derivedRecipe)
log.info('Removing extra files from checkout')
conaryState = state.ConaryStateFromFile('CONARY', repos)
sourceState = conaryState.getSourceState()
# clear the factory since we don't care about how the parent trove was
# created
sourceState.setFactory('')
addRecipe = True
for (pathId, path, fileId,
version) in list(sourceState.iterFileList()):
if path == recipeName:
addRecipe = False
continue
sourceState.removeFile(pathId)
if util.exists(path):
statInfo = os.lstat(path)
try:
if statInfo.st_mode & stat.S_IFDIR:
os.rmdir(path)
else:
os.unlink(path)
except OSError, e:
log.warning("cannot remove %s: %s" % (path, e.strerror))
conaryState.write('CONARY')
if addRecipe:
checkin.addFiles([recipeName])
if extract:
log.info('extracting files from %s=%s[%s]' % (troveToDerive))
# extract to _ROOT_
extractDir = os.path.join(os.getcwd(), '_ROOT_')
ts = [(troveToDerive[0], (None, None), (troveToDerive[1],
troveToDerive[2]), True)]
cs = repos.createChangeSet(ts, recurse=True)
ChangesetExploder(cs, extractDir)
# extract to _OLD_ROOT_
secondDir = os.path.join(os.getcwd(), '_OLD_ROOT_')
cs = repos.createChangeSet(ts, recurse=True)
ChangesetExploder(cs, secondDir)
def _commitRecipe(conaryclient, cfg, recipePath, message, branch=None):
repos = conaryclient.getRepos()
conaryCompat = compat.ConaryVersion()
recipeClass, pathList = _getPathList(repos, cfg, recipePath)
sourceName = recipeClass.name + ':source'
log.info("Creating a copy of %s in the rMake internal repository..." % recipeClass.name)
cwd = os.getcwd()
recipeDir = tempfile.mkdtemp()
log.resetErrorOccurred()
try:
fileNames = []
# Create a source trove that matches the recipe we're trying to cook
if not branch:
branch = versions.Branch([cfg.buildLabel])
targetLabel = cfg.getTargetLabel(branch)
if compat.ConaryVersion().supportsNewPkgBranch():
buildBranch = branch.createShadow(targetLabel)
kw = dict(buildBranch=buildBranch)
else:
buildBranch = versions.Branch([targetLabel])
kw={}
cfg.buildLabel = targetLabel
if not repos.getTroveLeavesByBranch(
{ sourceName : { buildBranch : None } }).get(sourceName, None):
# we pass it None for repos to avoid the label-based check for
# existing packages.
checkin.newTrove(None, cfg, recipeClass.name, dir=recipeDir, **kw)
else:
# see if this package exists on our build branch
checkin.checkout(repos, cfg, recipeDir,
['%s=%s' % (sourceName, buildBranch)])
os.chdir(recipeDir)
sourceState = state.ConaryStateFromFile(recipeDir + '/CONARY').getSourceState()
fileNames = dict((os.path.basename(x), x) for x in pathList)
for (pathId, baseName, fileId, version) in list(sourceState.iterFileList()):
# update or remove any currently existing files
if baseName not in fileNames:
sourceState.removeFilePath(baseName)
else:
shutil.copyfile(fileNames[baseName],
os.path.join(recipeDir, baseName))
del fileNames[baseName]
for baseName, path in fileNames.iteritems():
shutil.copyfile(path, os.path.join(recipeDir, baseName))
if conaryCompat.stateFileVersion() > 0:
# mark all the files as binary - this this version can
# never be checked in, it doesn't really matter, but
# conary likes us to give a value.
for fileName in fileNames:
isConfig = _getConfigInfo(fileName)
checkin.addFiles([fileName], binary=not isConfig, text=isConfig)
else:
checkin.addFiles(fileNames)
_doCommit(recipePath, repos, cfg, 'Temporary recipe build for rmake')
newState = conaryCompat.ConaryStateFromFile(recipeDir + '/CONARY',
repos=repos)
return newState.getSourceState().getNameVersionFlavor()
finally:
os.chdir(cwd)
shutil.rmtree(recipeDir)
def _shadowAndCommit(conaryclient, cfg, recipeDir, stateFile, message):
repos = conaryclient.getRepos()
conaryCompat = compat.ConaryVersion()
oldSourceVersion = stateFile.getVersion()
targetLabel = cfg.getTargetLabel(oldSourceVersion)
if not targetLabel:
raise errors.RmakeError(
'Cannot cook local recipes unless a target label is set')
skipped, cs = conaryclient.createShadowChangeSet(str(targetLabel),
[stateFile.getNameVersionFlavor()])
recipePath = recipeDir + '/' + stateFile.getName().split(':')[0] + '.recipe'
recipeClass, pathList = _getPathList(repos, cfg, recipePath, relative=True)
troveName = stateFile.getName()
troveVersion = stateFile.getVersion()
if not skipped:
signAbsoluteChangeset(cs, None)
repos.commitChangeSet(cs)
log.info("Shadowing %s to internal repository..." % troveName)
shadowBranch = troveVersion.createShadow(targetLabel).branch()
shadowVersion = repos.findTrove(None,
(troveName, str(shadowBranch),
None), None)[0][1]
cwd = os.getcwd()
prefix = 'rmake-shadow-%s-' % troveName.split(':')[0]
shadowSourceDir = tempfile.mkdtemp(prefix=prefix)
try:
log.info("Committing local changes to %s to the"
" internal repository..." % troveName)
log.resetErrorOccurred()
checkin.checkout(repos, cfg, shadowSourceDir,
['%s=%s' % (troveName, shadowVersion)])
if compat.ConaryVersion().stateFileVersion() > 0:
kw = dict(repos=repos)
else:
kw = {}
# grab new and old state and make any modifications due to adding
# or deleting of files (we assume files that don't exist are
# autosource and can be ignored)
oldState = conaryCompat.ConaryStateFromFile(recipeDir + '/CONARY',
repos=repos).getSourceState()
newConaryState = conaryCompat.ConaryStateFromFile(
shadowSourceDir + '/CONARY',
repos=repos)
newState = newConaryState.getSourceState()
neededFiles = set(x[1] for x in oldState.iterFileList()
if os.path.exists(os.path.join(recipeDir, x[1])))
neededFiles.update(pathList)
autoSourceFiles = set(x[1] for x in oldState.iterFileList()
if oldState.fileIsAutoSource(x[0]))
existingFiles = set(x[1] for x in newState.iterFileList()
if os.path.exists(os.path.join(shadowSourceDir, x[1])))
toCopy = neededFiles & existingFiles
toDel = existingFiles - neededFiles
toAdd = neededFiles - existingFiles
for sourceFile in (toCopy | toAdd):
newPath = os.path.join(shadowSourceDir, sourceFile)
if os.path.dirname(sourceFile):
util.mkdirChain(os.path.dirname(newPath))
if os.path.isdir(sourceFile):
util.mkdirChain(newPath)
else:
shutil.copyfile(os.path.join(recipeDir, sourceFile), newPath)
os.chdir(shadowSourceDir)
if hasattr(cfg.sourceSearchDir, '_getUnexpanded'):
cfg.configKey('sourceSearchDir',
cfg.sourceSearchDir._getUnexpanded())
for f in toDel:
checkin.removeFile(f)
if toDel:
# toDel modifies the CONARY file on disk, so reload with the
# changes made there.
newState = conaryCompat.ConaryStateFromFile(
shadowSourceDir + '/CONARY',
repos=repos).getSourceState()
if conaryCompat.stateFileVersion() == 0:
checkin.addFiles(toAdd)
else:
oldPathIds = dict((x[1], x[0]) for x in oldState.iterFileList())
for path in toAdd:
if path in oldPathIds:
isConfig = oldState.fileIsConfig(oldPathIds[path])
else:
isConfig = _getConfigInfo(path)
checkin.addFiles([path], binary=not isConfig, text=isConfig)
if toAdd:
# get the new pathIDs for all the added troves,
# since we can't set the refresh setting without the
# needed pathIds
newState = conaryCompat.ConaryStateFromFile(
shadowSourceDir + '/CONARY',
repos=repos).getSourceState()
newPathIds = dict((x[1], x[0]) for x in newState.iterFileList())
for path in (toCopy | toAdd):
if path in oldPathIds:
isConfig = oldState.fileIsConfig(oldPathIds[path])
else:
isConfig = _getConfigInfo(path)
newState.fileIsConfig(newPathIds[path], isConfig)
for path in autoSourceFiles:
if path in newPathIds:
needsRefresh = oldState.fileNeedsRefresh(oldPathIds[path])
newState.fileNeedsRefresh(newPathIds[path], needsRefresh)
# if the factory changed, update it
if newState.getFactory() != oldState.getFactory():
newState.setFactory(oldState.getFactory())
# we may have modified the state file. Write it back out to
# disk so it will be picked up by the commit.
newConaryState.setSourceState(newState)
newConaryState.write(shadowSourceDir + '/CONARY')
if message is None and compat.ConaryVersion().supportsCloneCallback():
message = 'Automated rMake commit'
_doCommit('%s/%s' % (recipeDir, troveName), repos, cfg, message)
newState = state.ConaryStateFromFile(shadowSourceDir + '/CONARY', **kw)
return newState.getSourceState().getNameVersionFlavor()
finally:
os.chdir(cwd)
if hasattr(cfg.sourceSearchDir, '_getUnexpanded'):
cfg.configKey('sourceSearchDir',
cfg.sourceSearchDir._getUnexpanded())
shutil.rmtree(shadowSourceDir)
def derive(repos, cfg, targetLabel, troveSpec, checkoutDir=None,
extract=False, info=False, callback=None):
"""
Performs all the commands necessary to create a derived recipe.
First it shadows the package, then it creates a checkout of the shadow
and converts the checkout to a derived recipe package.
Finally if extract = True, it installs an version of the binary
package into a root.
@param repos: trovesource to search for and derive packages from
@param cfg: configuration to use when deriving the package
@type cfg: ConaryConfiguration object
@param targetLabel: label to derive from
@type targetLabel: versions.Label
@param checkoutDir: directory to create the checkout in. If None,
defaults to currentDir + packageName.
@param extract: If True, creates a subdirectory of the checkout named
_ROOT_ with the contents of the binary of the derived
package.
@param info: If true, only display the information about the shadow
that would be performed if the derive command were
completed.
@param callback:
"""
origDir = os.getcwd()
try:
if callback is None:
callback = DeriveCallback()
if isinstance(troveSpec, tuple):
troveName, versionSpec, flavor = troveSpec
versionSpec = str(versionSpec)
troveSpec = cmdline.toTroveSpec(troveName, versionSpec, flavor)
else:
troveName, versionSpec, flavor = cmdline.parseTroveSpec(troveSpec)
if isinstance(targetLabel, str):
targetLabel = Label(targetLabel)
troveName, versionSpec, flavor = cmdline.parseTroveSpec(troveSpec)
result = repos.findTrove(cfg.buildLabel,
(troveName, versionSpec, flavor),
cfg.flavor)
# findTrove shouldn't return multiple items for one package anymore
# when a flavor is specified.
troveToDerive, = result
# displaying output along the screen allows there to be a record
# of what operations were performed. Since this command is
# an aggregate of several commands I think that is appropriate,
# rather than simply using a progress callback.
log.info('Shadowing %s=%s[%s] onto %s' % (troveToDerive[0],
troveToDerive[1],
troveToDerive[2],
targetLabel))
if info:
cfg.interactive = False
error = branch.branch(repos, cfg, str(targetLabel),
['%s=%s[%s]'%troveToDerive],
makeShadow=True, sourceOnly=True,
binaryOnly=False, allowEmptyShadow=True,
info=info)
if info or error:
return
shadowedVersion = troveToDerive[1].createShadow(targetLabel)
shadowedVersion = shadowedVersion.getSourceVersion(False)
troveName = troveName.split(':')[0]
checkoutDir = checkoutDir or troveName
checkin.checkout(repos, cfg, checkoutDir,
["%s=%s" % (troveName, shadowedVersion)],
callback=callback)
os.chdir(checkoutDir)
nvfs = repos.getTrovesBySource(troveToDerive[0]+':source',
troveToDerive[1].getSourceVersion())
trvs = repos.getTroves(nvfs)
hasCapsule = [ x for x in trvs if x.troveInfo.capsule.type() ]
if hasCapsule:
derivedRecipeType = 'DerivedCapsuleRecipe'
removeText = ''
else:
derivedRecipeType = 'DerivedPackageRecipe'
removeText = \
"""
# This appliance uses PHP as a command interpreter but does
# not include a web server, so remove the file that creates
# a dependency on the web server
r.Remove('/etc/httpd/conf.d/php.conf')
"""
log.info('Rewriting recipe file')
recipeName = troveName + '.recipe'
className = util.convertPackageNameToClassName(troveName)
derivedRecipe = """
class %(className)sRecipe(%(recipeBaseClass)s):
name = '%(name)s'
version = '%(version)s'
def setup(r):
'''
In this recipe, you can make modifications to the package.
Examples:
# This appliance has high-memory-use PHP scripts
r.Replace('memory_limit = 8M', 'memory_limit = 32M', '/etc/php.ini')
%(removeText)s
# This appliance requires that a few binaries be replaced
# with binaries built from a custom archive that includes
# a Makefile that honors the DESTDIR variable for its
# install target.
r.addArchive('foo.tar.gz')
r.Make()
r.MakeInstall()
# This appliance requires an extra configuration file
r.Create('/etc/myconfigfile', contents='some data')
'''
""" % dict(className=className,
name=troveName,
version=shadowedVersion.trailingRevision().getVersion(),
recipeBaseClass=derivedRecipeType,
removeText=removeText)
open(recipeName, 'w').write(derivedRecipe)
log.info('Removing extra files from checkout')
conaryState = state.ConaryStateFromFile('CONARY', repos)
sourceState = conaryState.getSourceState()
# clear the factory since we don't care about how the parent trove was
# created
sourceState.setFactory('')
addRecipe=True
for (pathId, path, fileId, version) in list(sourceState.iterFileList()):
if path == recipeName:
addRecipe = False
continue
sourceState.removeFile(pathId)
if util.exists(path):
statInfo = os.lstat(path)
try:
if statInfo.st_mode & stat.S_IFDIR:
os.rmdir(path)
else:
os.unlink(path)
except OSError, e:
log.warning("cannot remove %s: %s" % (path, e.strerror))
conaryState.write('CONARY')
if addRecipe:
checkin.addFiles([recipeName])
if extract:
log.info('extracting files from %s=%s[%s]' % (troveToDerive))
# extract to _ROOT_
extractDir = os.path.join(os.getcwd(), '_ROOT_')
ts = [ (troveToDerive[0], (None, None),
(troveToDerive[1], troveToDerive[2]), True) ]
cs = repos.createChangeSet(ts, recurse = True)
ChangesetExploder(cs, extractDir)
# extract to _OLD_ROOT_
secondDir = os.path.join(os.getcwd(), '_OLD_ROOT_')
cs = repos.createChangeSet(ts, recurse = True)
ChangesetExploder(cs, secondDir)
