def _addArcToGraph(self, graphInstance, dep):
basename = gitwrapper.originURLBasename(dep.gitURL)
mirror = repomirrorcache.get(dep.gitURL)
distance = mirror.distanceFromMaster(dep.hash)
requiringBasename = gitwrapper.originURLBasename(dep.requiringURL)
graphInstance.addArc(
requiringBasename, basename, style=self._lineStyleFromDependencyType(dep.type),
** self._attributesFromDistanceFromMaster(distance))
def __init__(self):
self._requirements = []
upsetoManifest = upseto.manifest.Manifest.fromLocalDirOrNew()
for requirement in upsetoManifest.requirements():
self._requirements.append((
gitwrapper.originURLBasename(requirement['originURL']),
requirement['hash']))
solventManifest = solvent.manifest.Manifest.fromLocalDirOrNew()
for requirement in solventManifest.requirements():
self._requirements.append((
gitwrapper.originURLBasename(requirement['originURL']),
requirement['hash']))
def includeRecursiveByBasenames(self, projectBasename):
self._reachableBasenames.add(projectBasename)
while True:
before = len(self._reachableBasenames)
for dependency in self._dependencies:
if dependency.requiringURL is None:
continue
basename = gitwrapper.originURLBasename(dependency.requiringURL)
if basename in self._reachableBasenames:
self._filtered.add(dependency)
self._reachableBasenames.add(gitwrapper.originURLBasename(dependency.gitURL))
if len(self._reachableBasenames) == before:
break
def _addReferencesFromManifests(self, dependency):
mirror = repomirrorcache.get(dependency.gitURL)
for requirement in mirror.upsetoManifest(dependency.hash).requirements():
basename = gitwrapper.originURLBasename(requirement['originURL'])
self._references.add((basename, requirement['hash']))
for requirement in mirror.solventManifest(dependency.hash).requirements():
basename = gitwrapper.originURLBasename(requirement['originURL'])
self._references.add((basename, requirement['hash']))
try:
label = mirror.dirbalakManifest(dependency.hash).buildRootFSLabel()
except KeyError:
return
self._references.add(self._basenameAndHashFromLabel(label))
def __init__(self):
mani = manifest.Manifest.fromLocalDirOrNew()
self._requirements = []
for requirement in mani.requirements():
self._requirements.append((
gitwrapper.originURLBasename(requirement['originURL']),
requirement['hash']))
def _testBasenameConsistency(self, originURL):
basename = gitwrapper.originURLBasename(originURL)
if basename in self._allBasenames:
if originURL != self._allBasenames[originURL]:
raise Exception(
"Both requirements '%s' and '%s' have the same basename" %
(originURL, self._allBasenames[originURL]))
def traverse(self, mani):
"""
Recursivly visit all dependencies, once. Yields a 'Dependency'
named tuple for each dependency visited (root not included).
the 'manifest' field might be none if this project does not
have an upseto manifest.
"""
if mani.originURL() in self._visitedOriginURLs:
return
self._visitedOriginURLs.add(mani.originURL())
for requirement in mani.requirements():
basename = gitwrapper.originURLBasename(requirement['originURL'])
projectDir = os.path.join(self._baseDir, basename)
submanifest = manifest.Manifest.fromDir(projectDir) if \
manifest.Manifest.exists(projectDir) else None
dependency = Dependency(
requirement=requirement,
projectDir=projectDir,
manifest=submanifest,
parentOriginURL=mani.originURL(),
basename=basename)
yield dependency
# refresh manifest in case it was changed by caller (as in
# FulfillRequirements which checks out the code)
refreshedSubmanifest = manifest.Manifest.fromDir(projectDir) if \
manifest.Manifest.exists(projectDir) else None
if refreshedSubmanifest is not None:
for x in self.traverse(refreshedSubmanifest):
yield x
def __init__(self):
mani = manifest.Manifest.fromLocalDirOrNew()
self._requirements = []
for requirement in mani.requirements():
self._requirements.append(
(gitwrapper.originURLBasename(requirement['originURL']),
requirement['hash']))
def _testBasenameConsistency(self, originURL):
basename = gitwrapper.originURLBasename(originURL)
if basename in self._allBasenames:
if originURL != self._allBasenames[originURL]:
raise Exception(
"Both requirements '%s' and '%s' have the same basename" % (
originURL, self._allBasenames[originURL]))
def traverse(self, mani, level=0):
"""
Recursivly visit all dependencies, once. Yields a 'Dependency'
named tuple for each dependency visited (root not included).
the 'manifest' field might be none if this project does not
have an upseto manifest.
"""
if mani.originURL() in self._visitedOriginURLs:
return
self._visitedOriginURLs.add(mani.originURL())
for requirement in mani.dirtyFirstRequirements():
basename = gitwrapper.originURLBasename(requirement['originURL'])
projectDir = os.path.join(self._baseDir, basename)
submanifest = manifest.Manifest.fromDir(projectDir) if \
manifest.Manifest.exists(projectDir) else None
dependency = Dependency(requirement=requirement,
projectDir=projectDir,
manifest=submanifest,
parentOriginURL=mani.originURL(),
basename=basename,
level=level)
yield dependency
# refresh manifest in case it was changed by caller (as in
# FulfillRequirements which checks out the code)
refreshedSubmanifest = manifest.Manifest.fromDir(projectDir) if \
manifest.Manifest.exists(projectDir) else None
if refreshedSubmanifest is not None:
for x in self.traverse(refreshedSubmanifest, level=level + 1):
yield x
def includeRecursiveByExactHashes(self, url, hash):
basename = gitwrapper.originURLBasename(url)
self._reachableHashes.add((basename, hash))
while True:
before = len(self._reachableHashes)
for dependency in self._dependencies:
if dependency.requiringURL is None:
continue
basename = gitwrapper.originURLBasename(dependency.requiringURL)
exact = (basename, dependency.requiringURLHash)
if exact in self._reachableHashes:
self._filtered.add(dependency)
basename = gitwrapper.originURLBasename(dependency.gitURL)
self._reachableHashes.add((basename, dependency.hash))
if len(self._reachableHashes) == before:
break
def recalculate(self):
labels = solventofficiallabels.SolventOfficialLabels(self._officialObjectStore)
self._reversedMap = dict()
self._cantBeBuilt = dict()
filtered = traversefilterbuildbanned.TraverseFilterBuildBanned(
self._multiverse,
self._multiverse.getTraverse().dependencies())
for dep in filtered.dependencies():
basename = gitwrapper.originURLBasename(dep.gitURL)
if basename not in self._multiverse.projects:
logging.info("Will not build project '%(project)s' not in the multiverse file", dict(
project=basename))
continue
project = self._multiverse.projects[basename]
hexHash = dep.hash if dep.hash != 'origin/master' else dep.masterHash
projectDict = dict(
basename=basename, hash=dep.hash, gitURL=dep.gitURL, submit=False,
hexHash=hexHash, buildRootFS=project.buildRootFS())
buildState = self._buildState.get(dep.gitURL, hexHash)
projectDict.update(buildState)
unbuiltRequirements = self._unbuiltRequirements(dep.gitURL, dep.hash, labels)
if unbuiltRequirements:
self._cantBeBuilt[(basename, dep.hash)] = unbuiltRequirements
continue
if project.buildBanned():
logging.info(
"Will not put project '%(project)s' in queue, is build banned '%(message)s'",
dict(project=basename, message=project.buildBanned()))
continue
if dep.hash == "origin/master":
mirror = repomirrorcache.get(dep.gitURL)
if labels.built(basename, mirror.hash('origin/master')):
self._put(projectDict, self.MASTERS_REBUILD)
else:
if buildState['failures'] > 0 and buildState['successes'] == 0:
projectDict['submit'] = True
self._put(projectDict, self.MASTERS_WHICH_BUILD_ONLY_FAILED)
else:
projectDict['submit'] = True
self._put(projectDict, self.MASTERS_NOT_BUILT)
else:
if not labels.built(basename, dep.hash):
projectDict['submit'] = True
projectDict['requiringBasename'] = gitwrapper.originURLBasename(dep.requiringURL)
self._put(projectDict, self.NON_MASTER_DEPENDENCIES)
self._reverseMap()
self._toJS()
def _gitURLBuildBanned(self, gitURL):
if gitURL is None:
return False
basename = gitwrapper.originURLBasename(gitURL)
project = self._multiverse.projects.get(basename, None)
if project is None:
return True
return bool(project.buildBanned())
def findRequirementByBasename(self, basename):
for requirement in self._data['requirements']:
if gitwrapper.originURLBasename(
requirement['originURL']) == basename:
return requirement
raise Exception(
"Origin URL with the basename '%s' was not found in requirement list"
% basename)
def _addNode(self, gitURL, hash, broken):
basename = gitwrapper.originURLBasename(gitURL).replace('-', '_')
name = self._nodeName(gitURL, hash)
hash = 'master' if hash == 'origin/master' else hash[:8]
attributes = dict(label=basename + "\\n" + hash, cluster=basename)
if broken:
attributes['color'] = 'red'
self._graph.setNodeAttributes(name, **attributes)
def _configureLogbeam(self, gitURL, logbeamBuildID):
basename = gitwrapper.originURLBasename(gitURL)
under = os.path.join(config.LOGBEAM_ROOT_DIR, basename, logbeamBuildID)
environ = dict(os.environ)
if os.path.exists("/etc/dirbalak.logbeam.config"):
with open("/etc/dirbalak.logbeam.config") as f:
environ['LOGBEAM_CONFIG'] = f.read()
conf = subprocess.check_output(["logbeam", "createConfig", "--under", under], env=environ)
self._ssh.ftp.putContents("/etc/logbeam.config", conf)
def fetchRepoFromWebHook(repo):
global multiverseInstance
if multiverseInstance is None:
return
basename = gitwrapper.originURLBasename(repo)
if basename not in multiverseInstance.projects:
logging.error("Webhook called for unfamiliar repo '%(repo)s'", dict(repo=repo))
return
project = multiverseInstance.projects[basename]
project.needsFetch('Webhook triggered fetch')
def _unbuiltRequirements(self, gitURL, hash, labels):
result = []
for dep in self._multiverse.getTraverse().dependencies():
if dep.requiringURL != gitURL or dep.requiringURLHash != hash:
continue
basename = gitwrapper.originURLBasename(dep.gitURL)
mirror = repomirrorcache.get(dep.gitURL)
hexHash = dep.hash if dep.hash != 'origin/master' else mirror.hash('origin/master')
if not labels.built(basename, hexHash):
result.append(dict(basename=basename, hash=dep.hash))
return result
def rereadMultiverseFile(self, filename):
with open(filename) as f:
data = yaml.load(f.read())
for projectData in data['PROJECTS']:
basename = gitwrapper.originURLBasename(projectData['gitURL'])
if basename in self.projects:
projectInstance = self.projects[basename]
projectInstance.update(** projectData)
else:
projectInstance = project.Project(fetchThread=self._fetchThread, ** projectData)
self.projects[projectInstance.basename()] = projectInstance
self.needsFetch("Reread multiverse file")
def __init__(self, gitURL, owner, group, fetchThread, defaultRootFS=False, buildBanned=None):
self._gitURL = gitURL
self._owner = owner
self._group = group
self._fetchThread = fetchThread
self._defaultRootFS = defaultRootFS
self._buildBanned = buildBanned
self._basename = gitwrapper.originURLBasename(gitURL)
self._mirror = repomirrorcache.get(gitURL)
self._traverse = None
self._masterBuildHistory = projectmasterbuildhistory.ProjectMasterBuildHistory(
basename=self._basename)
def _dependedBy(self):
result = []
for dep in self._traverse.dependencies():
if dep.gitURL != self._gitURL:
continue
if dep.requiringURLHash != 'origin/master':
continue
result.append(dict(
basename=gitwrapper.originURLBasename(dep.requiringURL),
hash=dep.hash,
distanceFromMaster=self._mirror.distanceFromMaster(dep.hash),
type=dep.type))
return result
def updateAllDependencies(self, gitURL):
dependsOn = list(self._dependsOn(gitURL))
script = [
"#!/bin/bash",
"set -e",
"cd /tmp",
"rm -fr scriptolog",
"mkdir scriptolog",
"cd scriptolog",
"git clone %s" % gitURL,
"cd %s" % gitwrapper.originURLBasename(gitURL)] + \
self._updatedUpsetoManifest(dependsOn) + \
self._updatedSolventManifest(dependsOn) + [
"git add .",
"git commit -m 'Updated all deps to latest master'",
"git push origin master",
"cd /tmp",
"rm -fr scriptolog"]
return "\n".join(script)
logging.info("Saved '%(graphicOutput)s'", dict(graphicOutput=args.graphicOutput))
if args.dotOutput:
graph = describeInstance.makeGraph()
graph.saveDot(args.dotOutput)
elif args.cmd == "discover":
projects = list(args.gitURL)
if args.currentProject:
projects.append(gitwrapper.GitWrapper('.').originURL())
clusterMap = dict()
if args.multiverseFile:
with open(args.multiverseFile) as f:
multiverse = yaml.load(f.read())
if not args.noFetch:
repomirrorcache.prepopulate(p['gitURL'] for p in multiverse['PROJECTS'])
clusterMap = {
gitwrapper.originURLBasename(p['gitURL']): p['group'] for p in multiverse['PROJECTS']}
if args.projectsFromMultiverse:
projects += [p['gitURL'] for p in multiverse['PROJECTS']]
if len(projects) == 0:
raise Exception("No projects specified in command line")
discoverInstance = discover.Discover(
projects=projects, objectStore=args.officialObjectStore,
clusterMap=clusterMap,
dirbalakBuildRootFSArcs=not args.noDirbalakBuildRootFSArcs,
solventRootFSArcs=not args.noSolventRootFSArcs)
print discoverInstance.renderText()
if args.graphicOutput:
graph = discoverInstance.makeGraph()
graph.saveSvg(args.graphicOutput)
logging.info("Saved '%(graphicOutput)s'", dict(graphicOutput=args.graphicOutput))
if args.dotOutput:
def _configureLogbeam(self, gitURL, logbeamBuildID):
basename = gitwrapper.originURLBasename(gitURL)
under = os.path.join(config.LOGBEAM_ROOT_DIR, basename, logbeamBuildID)
conf = subprocess.check_output(["logbeam", "createConfig", "--under", under])
self._ssh.ftp.putContents("/etc/logbeam.config", conf)
def _addNodeToGraph(self, graphInstance, gitURL):
basename = gitwrapper.originURLBasename(gitURL)
attributes = self._getNodeAttributesCallback(gitURL)
attributes['label'] = basename
graphInstance.setNodeAttributes(basename, **attributes)
def _nodeName(self, gitURL, hash):
basename = gitwrapper.originURLBasename(gitURL)
return basename + '/' + hash
def _requirementByBasename(self, basename):
for requirement in self._data['requirements']:
if gitwrapper.originURLBasename(requirement['originURL']) == basename:
return requirement
raise Exception("Origin URL with the basename '%s' was not found in requirement list", basename)
def originURLBasename(self):
return gitwrapper.originURLBasename(self._originURL)
