def _addArcToGraph(self, graphInstance, dep): basename = gitwrapper.originURLBasename(dep.gitURL) mirror = repomirrorcache.get(dep.gitURL) distance = mirror.distanceFromMaster(dep.hash) requiringBasename = gitwrapper.originURLBasename(dep.requiringURL) graphInstance.addArc( requiringBasename, basename, style=self._lineStyleFromDependencyType(dep.type), ** self._attributesFromDistanceFromMaster(distance))
def __init__(self): self._requirements = [] upsetoManifest = upseto.manifest.Manifest.fromLocalDirOrNew() for requirement in upsetoManifest.requirements(): self._requirements.append(( gitwrapper.originURLBasename(requirement['originURL']), requirement['hash'])) solventManifest = solvent.manifest.Manifest.fromLocalDirOrNew() for requirement in solventManifest.requirements(): self._requirements.append(( gitwrapper.originURLBasename(requirement['originURL']), requirement['hash']))
def includeRecursiveByBasenames(self, projectBasename): self._reachableBasenames.add(projectBasename) while True: before = len(self._reachableBasenames) for dependency in self._dependencies: if dependency.requiringURL is None: continue basename = gitwrapper.originURLBasename(dependency.requiringURL) if basename in self._reachableBasenames: self._filtered.add(dependency) self._reachableBasenames.add(gitwrapper.originURLBasename(dependency.gitURL)) if len(self._reachableBasenames) == before: break
def _addReferencesFromManifests(self, dependency): mirror = repomirrorcache.get(dependency.gitURL) for requirement in mirror.upsetoManifest(dependency.hash).requirements(): basename = gitwrapper.originURLBasename(requirement['originURL']) self._references.add((basename, requirement['hash'])) for requirement in mirror.solventManifest(dependency.hash).requirements(): basename = gitwrapper.originURLBasename(requirement['originURL']) self._references.add((basename, requirement['hash'])) try: label = mirror.dirbalakManifest(dependency.hash).buildRootFSLabel() except KeyError: return self._references.add(self._basenameAndHashFromLabel(label))
def __init__(self): mani = manifest.Manifest.fromLocalDirOrNew() self._requirements = [] for requirement in mani.requirements(): self._requirements.append(( gitwrapper.originURLBasename(requirement['originURL']), requirement['hash']))
def _testBasenameConsistency(self, originURL): basename = gitwrapper.originURLBasename(originURL) if basename in self._allBasenames: if originURL != self._allBasenames[originURL]: raise Exception( "Both requirements '%s' and '%s' have the same basename" % (originURL, self._allBasenames[originURL]))
def traverse(self, mani): """ Recursivly visit all dependencies, once. Yields a 'Dependency' named tuple for each dependency visited (root not included). the 'manifest' field might be none if this project does not have an upseto manifest. """ if mani.originURL() in self._visitedOriginURLs: return self._visitedOriginURLs.add(mani.originURL()) for requirement in mani.requirements(): basename = gitwrapper.originURLBasename(requirement['originURL']) projectDir = os.path.join(self._baseDir, basename) submanifest = manifest.Manifest.fromDir(projectDir) if \ manifest.Manifest.exists(projectDir) else None dependency = Dependency( requirement=requirement, projectDir=projectDir, manifest=submanifest, parentOriginURL=mani.originURL(), basename=basename) yield dependency # refresh manifest in case it was changed by caller (as in # FulfillRequirements which checks out the code) refreshedSubmanifest = manifest.Manifest.fromDir(projectDir) if \ manifest.Manifest.exists(projectDir) else None if refreshedSubmanifest is not None: for x in self.traverse(refreshedSubmanifest): yield x
def __init__(self): mani = manifest.Manifest.fromLocalDirOrNew() self._requirements = [] for requirement in mani.requirements(): self._requirements.append( (gitwrapper.originURLBasename(requirement['originURL']), requirement['hash']))
def _testBasenameConsistency(self, originURL): basename = gitwrapper.originURLBasename(originURL) if basename in self._allBasenames: if originURL != self._allBasenames[originURL]: raise Exception( "Both requirements '%s' and '%s' have the same basename" % ( originURL, self._allBasenames[originURL]))
def traverse(self, mani, level=0): """ Recursivly visit all dependencies, once. Yields a 'Dependency' named tuple for each dependency visited (root not included). the 'manifest' field might be none if this project does not have an upseto manifest. """ if mani.originURL() in self._visitedOriginURLs: return self._visitedOriginURLs.add(mani.originURL()) for requirement in mani.dirtyFirstRequirements(): basename = gitwrapper.originURLBasename(requirement['originURL']) projectDir = os.path.join(self._baseDir, basename) submanifest = manifest.Manifest.fromDir(projectDir) if \ manifest.Manifest.exists(projectDir) else None dependency = Dependency(requirement=requirement, projectDir=projectDir, manifest=submanifest, parentOriginURL=mani.originURL(), basename=basename, level=level) yield dependency # refresh manifest in case it was changed by caller (as in # FulfillRequirements which checks out the code) refreshedSubmanifest = manifest.Manifest.fromDir(projectDir) if \ manifest.Manifest.exists(projectDir) else None if refreshedSubmanifest is not None: for x in self.traverse(refreshedSubmanifest, level=level + 1): yield x
def includeRecursiveByExactHashes(self, url, hash): basename = gitwrapper.originURLBasename(url) self._reachableHashes.add((basename, hash)) while True: before = len(self._reachableHashes) for dependency in self._dependencies: if dependency.requiringURL is None: continue basename = gitwrapper.originURLBasename(dependency.requiringURL) exact = (basename, dependency.requiringURLHash) if exact in self._reachableHashes: self._filtered.add(dependency) basename = gitwrapper.originURLBasename(dependency.gitURL) self._reachableHashes.add((basename, dependency.hash)) if len(self._reachableHashes) == before: break
def recalculate(self): labels = solventofficiallabels.SolventOfficialLabels(self._officialObjectStore) self._reversedMap = dict() self._cantBeBuilt = dict() filtered = traversefilterbuildbanned.TraverseFilterBuildBanned( self._multiverse, self._multiverse.getTraverse().dependencies()) for dep in filtered.dependencies(): basename = gitwrapper.originURLBasename(dep.gitURL) if basename not in self._multiverse.projects: logging.info("Will not build project '%(project)s' not in the multiverse file", dict( project=basename)) continue project = self._multiverse.projects[basename] hexHash = dep.hash if dep.hash != 'origin/master' else dep.masterHash projectDict = dict( basename=basename, hash=dep.hash, gitURL=dep.gitURL, submit=False, hexHash=hexHash, buildRootFS=project.buildRootFS()) buildState = self._buildState.get(dep.gitURL, hexHash) projectDict.update(buildState) unbuiltRequirements = self._unbuiltRequirements(dep.gitURL, dep.hash, labels) if unbuiltRequirements: self._cantBeBuilt[(basename, dep.hash)] = unbuiltRequirements continue if project.buildBanned(): logging.info( "Will not put project '%(project)s' in queue, is build banned '%(message)s'", dict(project=basename, message=project.buildBanned())) continue if dep.hash == "origin/master": mirror = repomirrorcache.get(dep.gitURL) if labels.built(basename, mirror.hash('origin/master')): self._put(projectDict, self.MASTERS_REBUILD) else: if buildState['failures'] > 0 and buildState['successes'] == 0: projectDict['submit'] = True self._put(projectDict, self.MASTERS_WHICH_BUILD_ONLY_FAILED) else: projectDict['submit'] = True self._put(projectDict, self.MASTERS_NOT_BUILT) else: if not labels.built(basename, dep.hash): projectDict['submit'] = True projectDict['requiringBasename'] = gitwrapper.originURLBasename(dep.requiringURL) self._put(projectDict, self.NON_MASTER_DEPENDENCIES) self._reverseMap() self._toJS()
def _gitURLBuildBanned(self, gitURL): if gitURL is None: return False basename = gitwrapper.originURLBasename(gitURL) project = self._multiverse.projects.get(basename, None) if project is None: return True return bool(project.buildBanned())
def findRequirementByBasename(self, basename): for requirement in self._data['requirements']: if gitwrapper.originURLBasename( requirement['originURL']) == basename: return requirement raise Exception( "Origin URL with the basename '%s' was not found in requirement list" % basename)
def _addNode(self, gitURL, hash, broken): basename = gitwrapper.originURLBasename(gitURL).replace('-', '_') name = self._nodeName(gitURL, hash) hash = 'master' if hash == 'origin/master' else hash[:8] attributes = dict(label=basename + "\\n" + hash, cluster=basename) if broken: attributes['color'] = 'red' self._graph.setNodeAttributes(name, **attributes)
def _configureLogbeam(self, gitURL, logbeamBuildID): basename = gitwrapper.originURLBasename(gitURL) under = os.path.join(config.LOGBEAM_ROOT_DIR, basename, logbeamBuildID) environ = dict(os.environ) if os.path.exists("/etc/dirbalak.logbeam.config"): with open("/etc/dirbalak.logbeam.config") as f: environ['LOGBEAM_CONFIG'] = f.read() conf = subprocess.check_output(["logbeam", "createConfig", "--under", under], env=environ) self._ssh.ftp.putContents("/etc/logbeam.config", conf)
def fetchRepoFromWebHook(repo): global multiverseInstance if multiverseInstance is None: return basename = gitwrapper.originURLBasename(repo) if basename not in multiverseInstance.projects: logging.error("Webhook called for unfamiliar repo '%(repo)s'", dict(repo=repo)) return project = multiverseInstance.projects[basename] project.needsFetch('Webhook triggered fetch')
def _unbuiltRequirements(self, gitURL, hash, labels): result = [] for dep in self._multiverse.getTraverse().dependencies(): if dep.requiringURL != gitURL or dep.requiringURLHash != hash: continue basename = gitwrapper.originURLBasename(dep.gitURL) mirror = repomirrorcache.get(dep.gitURL) hexHash = dep.hash if dep.hash != 'origin/master' else mirror.hash('origin/master') if not labels.built(basename, hexHash): result.append(dict(basename=basename, hash=dep.hash)) return result
def rereadMultiverseFile(self, filename): with open(filename) as f: data = yaml.load(f.read()) for projectData in data['PROJECTS']: basename = gitwrapper.originURLBasename(projectData['gitURL']) if basename in self.projects: projectInstance = self.projects[basename] projectInstance.update(** projectData) else: projectInstance = project.Project(fetchThread=self._fetchThread, ** projectData) self.projects[projectInstance.basename()] = projectInstance self.needsFetch("Reread multiverse file")
def __init__(self, gitURL, owner, group, fetchThread, defaultRootFS=False, buildBanned=None): self._gitURL = gitURL self._owner = owner self._group = group self._fetchThread = fetchThread self._defaultRootFS = defaultRootFS self._buildBanned = buildBanned self._basename = gitwrapper.originURLBasename(gitURL) self._mirror = repomirrorcache.get(gitURL) self._traverse = None self._masterBuildHistory = projectmasterbuildhistory.ProjectMasterBuildHistory( basename=self._basename)
def _dependedBy(self): result = [] for dep in self._traverse.dependencies(): if dep.gitURL != self._gitURL: continue if dep.requiringURLHash != 'origin/master': continue result.append(dict( basename=gitwrapper.originURLBasename(dep.requiringURL), hash=dep.hash, distanceFromMaster=self._mirror.distanceFromMaster(dep.hash), type=dep.type)) return result
def updateAllDependencies(self, gitURL): dependsOn = list(self._dependsOn(gitURL)) script = [ "#!/bin/bash", "set -e", "cd /tmp", "rm -fr scriptolog", "mkdir scriptolog", "cd scriptolog", "git clone %s" % gitURL, "cd %s" % gitwrapper.originURLBasename(gitURL)] + \ self._updatedUpsetoManifest(dependsOn) + \ self._updatedSolventManifest(dependsOn) + [ "git add .", "git commit -m 'Updated all deps to latest master'", "git push origin master", "cd /tmp", "rm -fr scriptolog"] return "\n".join(script)
logging.info("Saved '%(graphicOutput)s'", dict(graphicOutput=args.graphicOutput)) if args.dotOutput: graph = describeInstance.makeGraph() graph.saveDot(args.dotOutput) elif args.cmd == "discover": projects = list(args.gitURL) if args.currentProject: projects.append(gitwrapper.GitWrapper('.').originURL()) clusterMap = dict() if args.multiverseFile: with open(args.multiverseFile) as f: multiverse = yaml.load(f.read()) if not args.noFetch: repomirrorcache.prepopulate(p['gitURL'] for p in multiverse['PROJECTS']) clusterMap = { gitwrapper.originURLBasename(p['gitURL']): p['group'] for p in multiverse['PROJECTS']} if args.projectsFromMultiverse: projects += [p['gitURL'] for p in multiverse['PROJECTS']] if len(projects) == 0: raise Exception("No projects specified in command line") discoverInstance = discover.Discover( projects=projects, objectStore=args.officialObjectStore, clusterMap=clusterMap, dirbalakBuildRootFSArcs=not args.noDirbalakBuildRootFSArcs, solventRootFSArcs=not args.noSolventRootFSArcs) print discoverInstance.renderText() if args.graphicOutput: graph = discoverInstance.makeGraph() graph.saveSvg(args.graphicOutput) logging.info("Saved '%(graphicOutput)s'", dict(graphicOutput=args.graphicOutput)) if args.dotOutput:
def _configureLogbeam(self, gitURL, logbeamBuildID): basename = gitwrapper.originURLBasename(gitURL) under = os.path.join(config.LOGBEAM_ROOT_DIR, basename, logbeamBuildID) conf = subprocess.check_output(["logbeam", "createConfig", "--under", under]) self._ssh.ftp.putContents("/etc/logbeam.config", conf)
def _addNodeToGraph(self, graphInstance, gitURL): basename = gitwrapper.originURLBasename(gitURL) attributes = self._getNodeAttributesCallback(gitURL) attributes['label'] = basename graphInstance.setNodeAttributes(basename, **attributes)
def _nodeName(self, gitURL, hash): basename = gitwrapper.originURLBasename(gitURL) return basename + '/' + hash
def _requirementByBasename(self, basename): for requirement in self._data['requirements']: if gitwrapper.originURLBasename(requirement['originURL']) == basename: return requirement raise Exception("Origin URL with the basename '%s' was not found in requirement list", basename)
def originURLBasename(self): return gitwrapper.originURLBasename(self._originURL)