def testURLArguments(self):
# standard url
url1 = lookaside.laUrl('http://foo.example.com/bar.tar')
self.assertEqual(url1.filePath(), '/foo.example.com/bar.tar')
# url with arguments
url2 = lookaside.laUrl('http://foo.example.com/bar.tar?arg=value')
self.assertEqual(url2.filePath(), '/foo.example.com/bar.tar?arg=value')
# mirror url with arguments
mirrorUrl = lookaside.laUrl('mirror://testmirror.com/baz.tar?arg=bif')
url3 = lookaside.laUrl('http://foo.example.com/bar.tar?arg=value',
parent=mirrorUrl)
self.assertEqual(url3.filePath(), '/testmirror.com/baz.tar?arg=bif')
# url with arguments and no filename
url4 = lookaside.laUrl('http://foo.example.com/?arg=value')
self.assertEqual(url4.filePath(), '/foo.example.com/?arg=value')
# CNY-3674
url5 = lookaside.laUrl(
'lookaside://lp:lightdm/lp:lightdm--466.tar.bz2')
self.assertEqual(url5.host, 'lp:lightdm')
def testURLArguments(self):
# standard url
url1 = lookaside.laUrl('http://foo.example.com/bar.tar')
self.assertEqual(url1.filePath(), '/foo.example.com/bar.tar')
# url with arguments
url2 = lookaside.laUrl('http://foo.example.com/bar.tar?arg=value')
self.assertEqual(url2.filePath(),
'/foo.example.com/bar.tar?arg=value')
# mirror url with arguments
mirrorUrl = lookaside.laUrl('mirror://testmirror.com/baz.tar?arg=bif')
url3 = lookaside.laUrl('http://foo.example.com/bar.tar?arg=value',
parent=mirrorUrl)
self.assertEqual(url3.filePath(),
'/testmirror.com/baz.tar?arg=bif')
# url with arguments and no filename
url4 = lookaside.laUrl('http://foo.example.com/?arg=value')
self.assertEqual(url4.filePath(), '/foo.example.com/?arg=value')
# CNY-3674
url5 = lookaside.laUrl('lookaside://lp:lightdm/lp:lightdm--466.tar.bz2')
self.assertEqual(url5.host, 'lp:lightdm')
def _merge(self):
changeSet = ChangeSet()
deleteDirs = set()
doCommit = False
# If this is not None then all ephemeral sources will still be fetched
# but will be placed in this directory instead.
if self.helper.plan.ephemeralSourceDir:
ephDir = self.helper.makeEphemeralDir()
else:
ephDir = None
def _addFile(path, contents, isText):
if path in oldFiles:
# Always recycle pathId if available.
pathId, _, oldFileId, oldFileVersion = oldFiles[path]
else:
pathId = hashlib.md5(path).digest()
oldFileId = oldFileVersion = None
fileHelper = filetypes.RegularFile(contents=contents,
config=isText)
fileStream = fileHelper.get(pathId)
fileStream.flags.isSource(set=True)
fileId = fileStream.fileId()
# If the fileId matches, recycle the fileVersion too.
if fileId == oldFileId:
fileVersion = oldFileVersion
else:
fileVersion = newTrove.getVersion()
filesToAdd[fileId] = (fileStream, fileHelper.contents, isText)
newTrove.addFile(pathId, path, fileVersion, fileId)
for package, (recipeText, recipeObj), oldTrove in zip(
self.packages, self.recipes, self.oldTroves):
filesToAdd = {}
oldFiles = {}
if oldTrove is not None:
for pathId, path, fileId, fileVer in oldTrove.iterFileList():
oldFiles[path] = (pathId, path, fileId, fileVer)
newTrove = Trove(package.name, package.nextVersion, deps.Flavor())
newTrove.setFactory(package.targetConfig.factory)
# Add upstream files to new trove. Recycle pathids from the old
# version.
# LAZY: assume that everything other than the recipe is binary.
# Conary has a magic module, but it only accepts filenames!
for path, contents in package.recipeFiles.iteritems():
isText = path == package.getRecipeName()
_addFile(path, contents, isText)
# Collect requested auto sources from recipe. Unknown recipe types
# will not be loaded so recipeObj will be the class, so assume
# these have no sources.
if not inspect.isclass(recipeObj):
recipeFiles = dict((os.path.basename(x.getPath()), x)
for x in recipeObj.getSourcePathList())
newFiles = set(x[1] for x in newTrove.iterFileList())
needFiles = set(recipeFiles) - newFiles
for autoPath in needFiles:
source = recipeFiles[autoPath]
if getattr(source, 'contents', None
) and not source.sourcename:
# Ignore trove scripts that have inline contents
continue
if not autoPath:
log.error("bob does not support 'gussed' filenames; "
"please use a full path for source '%s' in "
"package %s", source.getPath(), package.name)
raise RuntimeError("Unsupported source action")
if (autoPath in oldFiles
and not self.helper.plan.refreshSources
and not source.ephemeral):
# File exists in old version.
pathId, path, fileId, fileVer = oldFiles[autoPath]
newTrove.addFile(pathId, path, fileVer, fileId)
continue
if source.ephemeral and not ephDir:
continue
# File doesn't exist; need to create it.
if source.ephemeral:
laUrl = lookaside.laUrl(source.getPath())
tempDir = joinPaths(ephDir,
os.path.dirname(laUrl.filePath()))
mkdirChain(tempDir)
else:
tempDir = tempfile.mkdtemp()
deleteDirs.add(tempDir)
snapshot = _getSnapshot(self.helper, package, source,
tempDir)
if not source.ephemeral and snapshot:
autoPathId = hashlib.md5(autoPath).digest()
autoObj = FileFromFilesystem(snapshot, autoPathId)
autoObj.flags.isAutoSource(set=True)
autoObj.flags.isSource(set=True)
autoFileId = autoObj.fileId()
autoContents = filecontents.FromFilesystem(snapshot)
filesToAdd[autoFileId] = (autoObj, autoContents, False)
newTrove.addFile(autoPathId, autoPath,
newTrove.getVersion(), autoFileId)
# If the old and new troves are identical, just use the old one.
if oldTrove and _sourcesIdentical(
oldTrove, newTrove, [self.oldChangeSet, filesToAdd]):
package.setDownstreamVersion(oldTrove.getVersion())
log.debug('Skipped %s=%s', oldTrove.getName(),
oldTrove.getVersion())
continue
# Add files and contents to changeset.
for fileId, (fileObj, fileContents, cfgFile) in filesToAdd.items():
changeSet.addFileContents(fileObj.pathId(), fileObj.fileId(),
ChangedFileTypes.file, fileContents, cfgFile)
changeSet.addFile(None, fileObj.fileId(), fileObj.freeze())
# Create a changelog entry.
changeLog = ChangeLog(
name=self.helper.cfg.name, contact=self.helper.cfg.contact,
message=self.helper.plan.commitMessage + '\n')
newTrove.changeChangeLog(changeLog)
# Calculate trove digests and add the trove to the changeset
newTrove.invalidateDigests()
newTrove.computeDigests()
newTroveCs = newTrove.diff(None, absolute=True)[0]
changeSet.newTrove(newTroveCs)
doCommit = True
package.setDownstreamVersion(newTrove.getVersion())
log.debug('Created %s=%s', newTrove.getName(), newTrove.getVersion())
if doCommit:
cook.signAbsoluteChangesetByConfig(changeSet, self.helper.cfg)
f = tempfile.NamedTemporaryFile(dir=os.getcwd(), suffix='.ccs',
delete=False)
f.close()
changeSet.writeToFile(f.name)
try:
self.helper.getRepos().commitChangeSet(changeSet)
except:
log.error("Error committing changeset to repository, "
"failed changeset is saved at %s", f.name)
raise
else:
os.unlink(f.name)
for path in deleteDirs:
shutil.rmtree(path)
def populateLcache(self):
"""
Populate a repository lookaside cache
"""
class lcachePopulationState:
"""Used to track the state of the lcache to enable it to be
efficiently populated on-demand"""
classes = None
sourcePaths = {}
completedActions = set()
pathMap = {}
if not self.laReposCache.repos:
return
if not self._lcstate:
repos = self.laReposCache.repos
self._lcstate = lcachePopulationState()
# build a list containing this recipe class and any ancestor class
# from which it descends
recipeClass = self.__class__
classes = [recipeClass]
bases = list(recipeClass.__bases__)
while bases:
parent = bases.pop()
bases.extend(list(parent.__bases__))
if issubclass(parent, Recipe):
classes.append(parent)
# reverse the class list, this way the files will be found in the
# youngest descendant first
classes.reverse()
self._lcstate.classes = classes
for rclass in self._lcstate.classes:
if not rclass._trove:
continue
srcName = rclass._trove.getName()
srcVersion = rclass._trove.getVersion()
# CNY-31: walk over the files in the trove we found upstream
# (which we may have modified to remove the non-autosourced
# files.
# Also, if an autosource file is marked as needing to be
# refreshed in the Conary state file, the lookaside cache has
# to win, so don't populate it with the repository file)
fileList = []
for pathId, path, fileId, version in rclass._trove.iterFileList():
assert path[0] != "/"
# we might need to retrieve this source file
# to enable a build, so we need to find the
# sha1 hash of it since that's how it's indexed
# in the file store
if isinstance(version, versions.NewVersion):
# don't try and look up things on the NewVersion label!
continue
fileList.append((pathId, path, fileId, version))
fileObjs = repos.getFileVersions([(x[0], x[2], x[3]) for x in fileList])
for i in range(len(fileList)):
fileObj = fileObjs[i]
if isinstance(fileObj, files_mod.RegularFile):
(pathId, path, fileId, version) = fileList[i]
self._lcstate.pathMap[path] = (srcName, srcVersion, pathId, path, fileId, version, fileObj)
# populate the repository source lookaside cache from the :source
# components
sourcePaths = self._lcstate.sourcePaths
actions = set(self.getSourcePathList()) - self._lcstate.completedActions
for a in actions:
self._lcstate.completedActions.add(a)
ps = a.getPathAndSuffix()
# check if we have an autosourced file
if any(x in ps[0] for x in ["://", ":pserver:", ":ext:"]):
# use guess name if it is provided
k = os.path.basename(ps[0]) or ps[1]
assert k
sourcePaths[k] = ps
else:
sourcePaths[ps[0]] = ps
if a.ephemeral:
raise RecipeFileError("File '%s' is marked as ephemeral " "but is not autosourced" % (ps[0],))
pathMap = self._lcstate.pathMap
delList = []
for path in pathMap:
fullPath = None
if path in sourcePaths:
fullPath = lookaside.laUrl(sourcePaths[path][0]).filePath()
elif path.find("/") == -1: # we might have a guessed name
for k in sourcePaths:
if k and path.startswith(k) and sourcePaths[k][2]:
for sk in sourcePaths[k][2]:
if path.endswith(sk) and len(k) + len(sk) == len(path) - 1:
fullUrl = sourcePaths[k][0] + k + "." + sk
fullPath = lookaside.laUrl(fullUrl).filePath()
(srcName, srcVersion, pathId, path, fileId, version, fileObj) = pathMap[path]
if not fullPath:
if fileObj.flags.isAutoSource():
continue
else:
fullPath = path
self.laReposCache.addFileHash(
fullPath,
srcName,
srcVersion,
pathId,
path,
fileId,
version,
fileObj.contents.sha1(),
fileObj.inode.perms(),
)
delList.append(path)
for path in delList:
if path in sourcePaths:
del sourcePaths[path]
del pathMap[path]
def populateLcache(self):
"""
Populate a repository lookaside cache
"""
class lcachePopulationState:
"""Used to track the state of the lcache to enable it to be
efficiently populated on-demand"""
classes = None
sourcePaths = {}
completedActions = set()
pathMap = {}
if not self.laReposCache.repos:
return
if not self._lcstate:
repos = self.laReposCache.repos
self._lcstate = lcachePopulationState()
# build a list containing this recipe class and any ancestor class
# from which it descends
recipeClass = self.__class__
classes = [recipeClass]
bases = list(recipeClass.__bases__)
while bases:
parent = bases.pop()
bases.extend(list(parent.__bases__))
if issubclass(parent, Recipe):
classes.append(parent)
# reverse the class list, this way the files will be found in the
# youngest descendant first
classes.reverse()
self._lcstate.classes = classes
for rclass in self._lcstate.classes:
if not rclass._trove:
continue
srcName = rclass._trove.getName()
srcVersion = rclass._trove.getVersion()
# CNY-31: walk over the files in the trove we found upstream
# (which we may have modified to remove the non-autosourced
# files.
# Also, if an autosource file is marked as needing to be
# refreshed in the Conary state file, the lookaside cache has
# to win, so don't populate it with the repository file)
fileList = []
for pathId, path, fileId, version in \
rclass._trove.iterFileList():
assert (path[0] != "/")
# we might need to retrieve this source file
# to enable a build, so we need to find the
# sha1 hash of it since that's how it's indexed
# in the file store
if isinstance(version, versions.NewVersion):
# don't try and look up things on the NewVersion label!
continue
fileList.append((pathId, path, fileId, version))
fileObjs = repos.getFileVersions([(x[0], x[2], x[3])
for x in fileList])
for i in range(len(fileList)):
fileObj = fileObjs[i]
if isinstance(fileObj, files_mod.RegularFile):
(pathId, path, fileId, version) = fileList[i]
self._lcstate.pathMap[path] = (srcName, srcVersion,
pathId, path, fileId,
version, fileObj)
# populate the repository source lookaside cache from the :source
# components
sourcePaths = self._lcstate.sourcePaths
actions = set(
self.getSourcePathList()) - self._lcstate.completedActions
for a in actions:
self._lcstate.completedActions.add(a)
ps = a.getPathAndSuffix()
# check if we have an autosourced file
if any(x in ps[0] for x in ['://', ':pserver:', ':ext:']):
# use guess name if it is provided
k = os.path.basename(ps[0]) or ps[1]
assert (k)
sourcePaths[k] = ps
else:
sourcePaths[ps[0]] = ps
if a.ephemeral:
raise RecipeFileError("File '%s' is marked as ephemeral "
"but is not autosourced" % (ps[0], ))
pathMap = self._lcstate.pathMap
delList = []
for path in pathMap:
fullPath = None
if path in sourcePaths:
fullPath = lookaside.laUrl(sourcePaths[path][0]).filePath()
elif path.find("/") == -1: # we might have a guessed name
for k in sourcePaths:
if k and path.startswith(k) and sourcePaths[k][2]:
for sk in sourcePaths[k][2]:
if path.endswith(sk) and \
len(k) + len(sk) == len(path)-1:
fullUrl = sourcePaths[k][0] + k + '.' + sk
fullPath = \
lookaside.laUrl(fullUrl).filePath()
(srcName, srcVersion, pathId, path, fileId, version, fileObj) = \
pathMap[path]
if not fullPath:
if fileObj.flags.isAutoSource():
continue
else:
fullPath = path
self.laReposCache.addFileHash(fullPath, srcName, srcVersion,
pathId, path, fileId, version,
fileObj.contents.sha1(),
fileObj.inode.perms())
delList.append(path)
for path in delList:
if path in sourcePaths:
del sourcePaths[path]
del pathMap[path]
def _merge(self):
changeSet = ChangeSet()
deleteDirs = set()
doCommit = False
# If this is not None then all ephemeral sources will still be fetched
# but will be placed in this directory instead.
if self.helper.plan.ephemeralSourceDir:
ephDir = self.helper.makeEphemeralDir()
else:
ephDir = None
def _addFile(path, contents, isText):
if path in oldFiles:
# Always recycle pathId if available.
pathId, _, oldFileId, oldFileVersion = oldFiles[path]
else:
pathId = hashlib.md5(path).digest()
oldFileId = oldFileVersion = None
fileHelper = filetypes.RegularFile(contents=contents,
config=isText)
fileStream = fileHelper.get(pathId)
fileStream.flags.isSource(set=True)
fileId = fileStream.fileId()
# If the fileId matches, recycle the fileVersion too.
if fileId == oldFileId:
fileVersion = oldFileVersion
else:
fileVersion = newTrove.getVersion()
filesToAdd[fileId] = (fileStream, fileHelper.contents, isText)
newTrove.addFile(pathId, path, fileVersion, fileId)
for package, (recipeText, recipeObj), oldTrove in zip(
self.packages, self.recipes, self.oldTroves):
filesToAdd = {}
oldFiles = {}
if oldTrove is not None:
for pathId, path, fileId, fileVer in oldTrove.iterFileList():
oldFiles[path] = (pathId, path, fileId, fileVer)
newTrove = Trove(package.name, package.nextVersion, deps.Flavor())
newTrove.setFactory(package.targetConfig.factory)
# Add upstream files to new trove. Recycle pathids from the old
# version.
# LAZY: assume that everything other than the recipe is binary.
# Conary has a magic module, but it only accepts filenames!
for path, contents in package.recipeFiles.iteritems():
isText = path == package.getRecipeName()
_addFile(path, contents, isText)
# Collect requested auto sources from recipe. Unknown recipe types
# will not be loaded so recipeObj will be the class, so assume
# these have no sources.
if not inspect.isclass(recipeObj):
recipeFiles = dict((os.path.basename(x.getPath()), x)
for x in recipeObj.getSourcePathList())
newFiles = set(x[1] for x in newTrove.iterFileList())
needFiles = set(recipeFiles) - newFiles
for autoPath in needFiles:
source = recipeFiles[autoPath]
if (autoPath in oldFiles
and not self.helper.plan.refreshSources
and not source.ephemeral):
# File exists in old version.
pathId, path, fileId, fileVer = oldFiles[autoPath]
newTrove.addFile(pathId, path, fileVer, fileId)
continue
if source.ephemeral and not ephDir:
continue
# File doesn't exist; need to create it.
if source.ephemeral:
laUrl = lookaside.laUrl(source.getPath())
tempDir = joinPaths(ephDir,
os.path.dirname(laUrl.filePath()))
mkdirChain(tempDir)
else:
tempDir = tempfile.mkdtemp()
deleteDirs.add(tempDir)
snapshot = _getSnapshot(self.helper, package, source,
tempDir)
if not source.ephemeral and snapshot:
autoPathId = hashlib.md5(autoPath).digest()
autoObj = FileFromFilesystem(snapshot, autoPathId)
autoObj.flags.isAutoSource(set=True)
autoObj.flags.isSource(set=True)
autoFileId = autoObj.fileId()
autoContents = filecontents.FromFilesystem(snapshot)
filesToAdd[autoFileId] = (autoObj, autoContents, False)
newTrove.addFile(autoPathId, autoPath,
newTrove.getVersion(), autoFileId)
# If the old and new troves are identical, just use the old one.
if oldTrove and _sourcesIdentical(
oldTrove, newTrove, [self.oldChangeSet, filesToAdd]):
package.setDownstreamVersion(oldTrove.getVersion())
log.debug('Skipped %s=%s', oldTrove.getName(),
oldTrove.getVersion())
continue
# Add files and contents to changeset.
for fileId, (fileObj, fileContents, cfgFile) in filesToAdd.items():
changeSet.addFileContents(fileObj.pathId(), fileObj.fileId(),
ChangedFileTypes.file, fileContents, cfgFile)
changeSet.addFile(None, fileObj.fileId(), fileObj.freeze())
# Create a changelog entry.
changeLog = ChangeLog(
name=self.helper.cfg.name, contact=self.helper.cfg.contact,
message=self.helper.plan.commitMessage + '\n')
newTrove.changeChangeLog(changeLog)
# Calculate trove digests and add the trove to the changeset
newTrove.invalidateDigests()
newTrove.computeDigests()
newTroveCs = newTrove.diff(None, absolute=True)[0]
changeSet.newTrove(newTroveCs)
doCommit = True
package.setDownstreamVersion(newTrove.getVersion())
log.debug('Created %s=%s', newTrove.getName(), newTrove.getVersion())
if doCommit:
cook.signAbsoluteChangesetByConfig(changeSet, self.helper.cfg)
f = tempfile.NamedTemporaryFile(dir=os.getcwd(), suffix='.ccs',
delete=False)
f.close()
changeSet.writeToFile(f.name)
try:
self.helper.getRepos().commitChangeSet(changeSet)
except:
log.error("Error committing changeset to repository, "
"failed changeset is saved at %s", f.name)
raise
else:
os.unlink(f.name)
for path in deleteDirs:
shutil.rmtree(path)
