def setLayers(self, bitbakes, layers): """ a word of attention: by convention, the first layer for any build will be poky! """ assert self.be.sourcedir is not None assert len(bitbakes) == 1 # set layers in the layersource raise NotImplementedException("Not implemented: SSH setLayers") # 3. configure the build environment, so we have a conf/bblayers.conf assert self.pokydirname is not None self._setupBE() # 4. update the bblayers.conf bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") if not self._pathexists(bblayerconf): raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) import uuid local_bblayerconf = "/tmp/" + uuid.uuid4() + "-bblayer.conf" self.pullFile(bblayerconf, local_bblayerconf) BuildEnvironmentController._updateBBLayers(local_bblayerconf, layerlist) self.pushFile(local_bblayerconf, bblayerconf) os.unlink(local_bblayerconf) self.islayerset = True return True
def setLayers(self, bitbake, layers, targets): """ a word of attention: by convention, the first layer for any build will be poky! """ assert self.be.sourcedir is not None # set layers in the layersource # 1. get a list of repos with branches, and map dirpaths for each layer gitrepos = {} gitrepos[(bitbake.giturl, bitbake.commit)] = [] gitrepos[(bitbake.giturl, bitbake.commit)].append( ("bitbake", bitbake.dirpath) ) for layer in layers: # we don't process local URLs if layer.giturl.startswith("file://"): continue if not (layer.giturl, layer.commit) in gitrepos: gitrepos[(layer.giturl, layer.commit)] = [] gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) ) logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) # 2. Note for future use if the current source directory is a # checked-out git repos that could match a layer's vcs_url and therefore # be used to speed up cloning (rather than fetching it again). cached_layers = {} try: for remotes in self._shellcmd("git remote -v", self.be.sourcedir).split("\n"): try: remote = remotes.split("\t")[1].split(" ")[0] if remote not in cached_layers: cached_layers[remote] = self.be.sourcedir except IndexError: pass except ShellCmdException: # ignore any errors in collecting git remotes this is an optional # step pass logger.info("Using pre-checked out source for layer %s", cached_layers) layerlist = [] # 3. checkout the repositories for giturl, commit in gitrepos.keys(): localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) # make sure our directory is a git repository if os.path.exists(localdirname): localremotes = self._shellcmd("git remote -v", localdirname) if not giturl in localremotes: raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl)) else: if giturl in cached_layers: logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname)) self._shellcmd("git remote remove origin", localdirname) self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname) else: logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname)) self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname)) # branch magic name "HEAD" will inhibit checkout if commit != "HEAD": logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname) # take the localdirname as poky dir if we can find the oe-init-build-env if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname) self.pokydirname = localdirname # make sure we have a working bitbake if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname) self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake'))) # verify our repositories for name, dirpath in gitrepos[(giturl, commit)]: localdirpath = os.path.join(localdirname, dirpath) logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath) if not os.path.exists(localdirpath): raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) if name != "bitbake": layerlist.append(localdirpath.rstrip("/")) logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) # 4. update the bblayers.conf bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") if not os.path.exists(bblayerconf): raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) # 5. create custom layer and add custom recipes to it layerpath = os.path.join(self.be.sourcedir, "_meta-toaster-custom") if os.path.isdir(layerpath): shutil.rmtree(layerpath) # remove leftovers from previous builds for target in targets: try: customrecipe = CustomImageRecipe.objects.get(name=target.target, project=bitbake.req.project) except CustomImageRecipe.DoesNotExist: continue # not a custom recipe, skip # create directory structure for name in ("conf", "recipes"): path = os.path.join(layerpath, name) if not os.path.isdir(path): os.makedirs(path) # create layer.oonf config = os.path.join(layerpath, "conf", "layer.conf") if not os.path.isfile(config): with open(config, "w") as conf: conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n') # create recipe recipe = os.path.join(layerpath, "recipes", "%s.bb" % target.target) with open(recipe, "w") as recipef: recipef.write("require %s\n" % customrecipe.base_recipe.recipe.file_path) packages = [pkg.name for pkg in customrecipe.packages.all()] if packages: recipef.write('IMAGE_INSTALL = "%s"\n' % ' '.join(packages)) # create *Layer* objects needed for build machinery to work layer = Layer.objects.get_or_create(name="Toaster Custom layer", summary="Layer for custom recipes", vcs_url="file://%s" % layerpath)[0] breq = target.req lver = Layer_Version.objects.get_or_create(project=breq.project, layer=layer, dirpath=layerpath, build=breq.build)[0] ProjectLayer.objects.get_or_create(project=breq.project, layercommit=lver, optional=False) BRLayer.objects.get_or_create(req=breq, name=layer.name, dirpath=layerpath, giturl="file://%s" % layerpath) if os.path.isdir(layerpath): layerlist.append(layerpath) BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist) self.islayerset = True return True
def setLayers(self, bitbakes, layers): """ a word of attention: by convention, the first layer for any build will be poky! """ assert self.be.sourcedir is not None assert len(bitbakes) == 1 # set layers in the layersource # 1. get a list of repos with branches, and map dirpaths for each layer gitrepos = {} gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = [] gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append( ("bitbake", bitbakes[0].dirpath) ) for layer in layers: # we don't process local URLs if layer.giturl.startswith("file://"): continue if not (layer.giturl, layer.commit) in gitrepos: gitrepos[(layer.giturl, layer.commit)] = [] gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) ) logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) # 2. find checked-out git repos in the sourcedir directory that may help faster cloning cached_layers = {} for ldir in os.listdir(self.be.sourcedir): fldir = os.path.join(self.be.sourcedir, ldir) if os.path.isdir(fldir): try: for line in self._shellcmd("git remote -v", fldir).split("\n"): try: remote = line.split("\t")[1].split(" ")[0] if remote not in cached_layers: cached_layers[remote] = fldir except IndexError: pass except ShellCmdException: # ignore any errors in collecting git remotes pass layerlist = [] # 3. checkout the repositories for giturl, commit in gitrepos.keys(): localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) # make sure our directory is a git repository if os.path.exists(localdirname): localremotes = self._shellcmd("git remote -v", localdirname) if not giturl in localremotes: raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl)) else: if giturl in cached_layers: logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname)) self._shellcmd("git remote remove origin", localdirname) self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname) else: logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname)) self._shellcmd("git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname)) # branch magic name "HEAD" will inhibit checkout if commit != "HEAD": logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) self._shellcmd("git fetch --all && git checkout \"%s\" && git rebase \"origin/%s\"" % (commit, commit) , localdirname) # take the localdirname as poky dir if we can find the oe-init-build-env if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname) self.pokydirname = localdirname # make sure we have a working bitbake if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname) self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) # verify our repositories for name, dirpath in gitrepos[(giturl, commit)]: localdirpath = os.path.join(localdirname, dirpath) logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath) if not os.path.exists(localdirpath): raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) if name != "bitbake": layerlist.append(localdirpath.rstrip("/")) logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) # 4. configure the build environment, so we have a conf/bblayers.conf assert self.pokydirname is not None self._setupBE() # 5. update the bblayers.conf bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") if not os.path.exists(bblayerconf): raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist) self.islayerset = True return True
def setLayers(self, bitbake, layers, targets): """ a word of attention: by convention, the first layer for any build will be poky! """ assert self.be.sourcedir is not None # set layers in the layersource # 1. get a list of repos with branches, and map dirpaths for each layer gitrepos = {} gitrepos[(bitbake.giturl, bitbake.commit)] = [] gitrepos[(bitbake.giturl, bitbake.commit)].append( ("bitbake", bitbake.dirpath)) for layer in layers: # we don't process local URLs if layer.giturl.startswith("file://"): continue if not (layer.giturl, layer.commit) in gitrepos: gitrepos[(layer.giturl, layer.commit)] = [] gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath)) logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) # 2. Note for future use if the current source directory is a # checked-out git repos that could match a layer's vcs_url and therefore # be used to speed up cloning (rather than fetching it again). cached_layers = {} try: for remotes in self._shellcmd("git remote -v", self.be.sourcedir).split("\n"): try: remote = remotes.split("\t")[1].split(" ")[0] if remote not in cached_layers: cached_layers[remote] = self.be.sourcedir except IndexError: pass except ShellCmdException: # ignore any errors in collecting git remotes this is an optional # step pass logger.info("Using pre-checked out source for layer %s", cached_layers) layerlist = [] # Verify HEAD layers layerlist = self.getHeadLayers(gitrepos, layers) # If the layerlist doesn't have already cloned layers then checkout repositories if len(layerlist) == 0: # 3. checkout the repositories for giturl, commit in gitrepos.keys(): localdirname = os.path.join( self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) logger.debug( "localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) # make sure our directory is a git repository if os.path.exists(localdirname): localremotes = self._shellcmd("git remote -v", localdirname) if not giturl in localremotes: raise BuildSetupException( "Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join( localremotes.split("\n")), giturl)) else: if giturl in cached_layers: logger.debug( "localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname)) self._shellcmd("git remote remove origin", localdirname) self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname) else: logger.debug( "localhostbecontroller: cloning %s in %s" % (giturl, localdirname)) self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname)) # branch magic name "HEAD" will inhibit checkout if commit != "HEAD": logger.debug( "localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit self._shellcmd( 'git fetch --all && git reset --hard "%s"' % ref, localdirname) # take the localdirname as poky dir if we can find the oe-init-build-env if self.pokydirname is None and os.path.exists( os.path.join(localdirname, "oe-init-build-env")): logger.debug( "localhostbecontroller: selected poky dir name %s" % localdirname) self.pokydirname = localdirname # make sure we have a working bitbake if not os.path.exists( os.path.join(self.pokydirname, 'bitbake')): logger.debug( "localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname) self._shellcmd( "git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake'))) # verify our repositories for name, dirpath in gitrepos[(giturl, commit)]: localdirpath = os.path.join(localdirname, dirpath) logger.debug( "localhostbecontroller: localdirpath expected '%s'" % localdirpath) if not os.path.exists(localdirpath): raise BuildSetupException( "Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) if name != "bitbake": layerlist.append(localdirpath.rstrip("/")) logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) # 4. update the bblayers.conf bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") if not os.path.exists(bblayerconf): raise BuildSetupException( "BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) # 5. create custom layer and add custom recipes to it layerpath = os.path.join(self.be.sourcedir, "_meta-toaster-custom") if os.path.isdir(layerpath): shutil.rmtree(layerpath) # remove leftovers from previous builds for target in targets: try: customrecipe = CustomImageRecipe.objects.get( name=target.target, project=bitbake.req.project) except CustomImageRecipe.DoesNotExist: continue # not a custom recipe, skip # create directory structure for name in ("conf", "recipes"): path = os.path.join(layerpath, name) if not os.path.isdir(path): os.makedirs(path) # create layer.oonf config = os.path.join(layerpath, "conf", "layer.conf") if not os.path.isfile(config): with open(config, "w") as conf: conf.write( 'BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n' ) # Update the Layer_Version dirpath that has our base_recipe in # to be able to read the base recipe to then generate the # custom recipe. br_layer_base_recipe = layers.get( layer_version=customrecipe.base_recipe.layer_version) br_layer_base_dirpath = \ os.path.join(self.be.sourcedir, self.getGitCloneDirectory( br_layer_base_recipe.giturl, br_layer_base_recipe.commit), customrecipe.base_recipe.layer_version.dirpath ) customrecipe.base_recipe.layer_version.dirpath = \ br_layer_base_dirpath customrecipe.base_recipe.layer_version.save() # create recipe recipe_path = \ os.path.join(layerpath, "recipes", "%s.bb" % target.target) with open(recipe_path, "w") as recipef: recipef.write(customrecipe.generate_recipe_file_contents()) # Update the layer and recipe objects customrecipe.layer_version.dirpath = layerpath customrecipe.layer_version.save() customrecipe.file_path = recipe_path customrecipe.save() # create *Layer* objects needed for build machinery to work BRLayer.objects.get_or_create(req=target.req, name=layer.name, dirpath=layerpath, giturl="file://%s" % layerpath) if os.path.isdir(layerpath): layerlist.append(layerpath) BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist) self.islayerset = True return True
def setLayers(self, bitbakes, layers): """ a word of attention: by convention, the first layer for any build will be poky! """ assert self.be.sourcedir is not None assert len(bitbakes) == 1 # set layers in the layersource # 1. get a list of repos, and map dirpaths for each layer gitrepos = {} gitrepos[bitbakes[0].giturl] = [] gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) for layer in layers: # we don't process local URLs if layer.giturl.startswith("file://"): continue if not layer.giturl in gitrepos: gitrepos[layer.giturl] = [] gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) for giturl in gitrepos.keys(): commitid = gitrepos[giturl][0][2] for e in gitrepos[giturl]: if commitid != e[2]: import pprint raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos)) logger.debug("localhostbecontroller, our git repos are %s" % gitrepos) layerlist = [] # 2. checkout the repositories for giturl in gitrepos.keys(): localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl)) logger.debug("localhostbecontroller: giturl %s checking out in current directory %s" % (giturl, localdirname)) # make sure our directory is a git repository if os.path.exists(localdirname): if not giturl in self._shellcmd("git remote -v", localdirname): raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) else: self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) # checkout the needed commit commit = gitrepos[giturl][0][2] # branch magic name "HEAD" will inhibit checkout if commit != "HEAD": logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname) # take the localdirname as poky dir if we can find the oe-init-build-env if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname) self.pokydirname = localdirname # make sure we have a working bitbake if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname) self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) # verify our repositories for name, dirpath, commit in gitrepos[giturl]: localdirpath = os.path.join(localdirname, dirpath) if not os.path.exists(localdirpath): raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) if name != "bitbake": layerlist.append(localdirpath.rstrip("/")) logger.debug("localhostbecontroller: current layer list %s " % layerlist) # 3. configure the build environment, so we have a conf/bblayers.conf assert self.pokydirname is not None self._setupBE() # 4. update the bblayers.conf bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") if not os.path.exists(bblayerconf): raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist) self.islayerset = True return True
def setLayers(self, bitbakes, layers): """ a word of attention: by convention, the first layer for any build will be poky! """ assert self.be.sourcedir is not None assert len(bitbakes) == 1 # set layers in the layersource # 1. get a list of repos, and map dirpaths for each layer gitrepos = {} gitrepos[bitbakes[0].giturl] = [] gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) for layer in layers: # we don't process local URLs if layer.giturl.startswith("file://"): continue if not layer.giturl in gitrepos: gitrepos[layer.giturl] = [] gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) for giturl in gitrepos.keys(): commitid = gitrepos[giturl][0][2] for e in gitrepos[giturl]: if commitid != e[2]: import pprint raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos)) layerlist = [] # 2. checkout the repositories for giturl in gitrepos.keys(): localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl)) print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname # make sure our directory is a git repository if os.path.exists(localdirname): if not giturl in self._shellcmd("git remote -v", localdirname): raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) else: self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) # checkout the needed commit commit = gitrepos[giturl][0][2] # branch magic name "HEAD" will inhibit checkout if commit != "HEAD": print "DEBUG: checking out commit ", commit, "to", localdirname self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname) # take the localdirname as poky dir if we can find the oe-init-build-env if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): print "DEBUG: selected poky dir name", localdirname self.pokydirname = localdirname # make sure we have a working bitbake if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): print "DEBUG: checking bitbake into the poky dirname %s " % self.pokydirname self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) # verify our repositories for name, dirpath, commit in gitrepos[giturl]: localdirpath = os.path.join(localdirname, dirpath) if not os.path.exists(localdirpath): raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) if name != "bitbake": layerlist.append(localdirpath.rstrip("/")) print "DEBUG: current layer list ", layerlist # 3. configure the build environment, so we have a conf/bblayers.conf assert self.pokydirname is not None self._setupBE() # 4. update the bblayers.conf bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") if not os.path.exists(bblayerconf): raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist) self.islayerset = True return True
def setLayers(self, bitbakes, layers): """ a word of attention: by convention, the first layer for any build will be poky! """ assert self.be.sourcedir is not None assert len(bitbakes) == 1 # set layers in the layersource # 1. get a list of repos with branches, and map dirpaths for each layer gitrepos = {} gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = [] gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append( ("bitbake", bitbakes[0].dirpath)) for layer in layers: # we don't process local URLs if layer.giturl.startswith("file://"): continue if not (layer.giturl, layer.commit) in gitrepos: gitrepos[(layer.giturl, layer.commit)] = [] gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath)) logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) # 2. find checked-out git repos in the sourcedir directory that may help faster cloning cached_layers = {} for ldir in os.listdir(self.be.sourcedir): fldir = os.path.join(self.be.sourcedir, ldir) if os.path.isdir(fldir): try: for line in self._shellcmd("git remote -v", fldir).split("\n"): try: remote = line.split("\t")[1].split(" ")[0] if remote not in cached_layers: cached_layers[remote] = fldir except IndexError: pass except ShellCmdException: # ignore any errors in collecting git remotes pass layerlist = [] # 3. checkout the repositories for giturl, commit in gitrepos.keys(): localdirname = os.path.join( self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) logger.debug( "localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) # make sure our directory is a git repository if os.path.exists(localdirname): localremotes = self._shellcmd("git remote -v", localdirname) if not giturl in localremotes: raise BuildSetupException( "Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join( localremotes.split("\n")), giturl)) else: if giturl in cached_layers: logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname)) self._shellcmd("git remote remove origin", localdirname) self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname) else: logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname)) self._shellcmd( "git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname)) # branch magic name "HEAD" will inhibit checkout if commit != "HEAD": logger.debug( "localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) self._shellcmd( "git fetch --all && git checkout \"%s\" && git rebase \"origin/%s\"" % (commit, commit), localdirname) # take the localdirname as poky dir if we can find the oe-init-build-env if self.pokydirname is None and os.path.exists( os.path.join(localdirname, "oe-init-build-env")): logger.debug( "localhostbecontroller: selected poky dir name %s" % localdirname) self.pokydirname = localdirname # make sure we have a working bitbake if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): logger.debug( "localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname) self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) # verify our repositories for name, dirpath in gitrepos[(giturl, commit)]: localdirpath = os.path.join(localdirname, dirpath) logger.debug( "localhostbecontroller: localdirpath expected '%s'" % localdirpath) if not os.path.exists(localdirpath): raise BuildSetupException( "Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) if name != "bitbake": layerlist.append(localdirpath.rstrip("/")) logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) # 4. configure the build environment, so we have a conf/bblayers.conf assert self.pokydirname is not None self._setupBE() # 5. update the bblayers.conf bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") if not os.path.exists(bblayerconf): raise BuildSetupException( "BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist) self.islayerset = True return True
def setLayers(self, bitbakes, layers, targets): """ a word of attention: by convention, the first layer for any build will be poky! """ assert self.be.sourcedir is not None assert len(bitbakes) == 1 # set layers in the layersource # 1. get a list of repos with branches, and map dirpaths for each layer gitrepos = {} gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = [] gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append( ("bitbake", bitbakes[0].dirpath)) for layer in layers: # we don't process local URLs if layer.giturl.startswith("file://"): continue if not (layer.giturl, layer.commit) in gitrepos: gitrepos[(layer.giturl, layer.commit)] = [] gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath)) logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) # 2. find checked-out git repos in the sourcedir directory that may help faster cloning cached_layers = {} for ldir in os.listdir(self.be.sourcedir): fldir = os.path.join(self.be.sourcedir, ldir) if os.path.isdir(fldir): try: for line in self._shellcmd("git remote -v", fldir).split("\n"): try: remote = line.split("\t")[1].split(" ")[0] if remote not in cached_layers: cached_layers[remote] = fldir except IndexError: pass except ShellCmdException: # ignore any errors in collecting git remotes pass layerlist = [] # 3. checkout the repositories for giturl, commit in gitrepos.keys(): localdirname = os.path.join( self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) logger.debug( "localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) # make sure our directory is a git repository if os.path.exists(localdirname): localremotes = self._shellcmd("git remote -v", localdirname) if not giturl in localremotes: raise BuildSetupException( "Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join( localremotes.split("\n")), giturl)) else: if giturl in cached_layers: logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname)) self._shellcmd("git remote remove origin", localdirname) self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname) else: logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname)) self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname)) # branch magic name "HEAD" will inhibit checkout if commit != "HEAD": logger.debug( "localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit self._shellcmd( 'git fetch --all && git reset --hard "%s"' % ref, localdirname) # take the localdirname as poky dir if we can find the oe-init-build-env if self.pokydirname is None and os.path.exists( os.path.join(localdirname, "oe-init-build-env")): logger.debug( "localhostbecontroller: selected poky dir name %s" % localdirname) self.pokydirname = localdirname # make sure we have a working bitbake if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): logger.debug( "localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname) self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) # verify our repositories for name, dirpath in gitrepos[(giturl, commit)]: localdirpath = os.path.join(localdirname, dirpath) logger.debug( "localhostbecontroller: localdirpath expected '%s'" % localdirpath) if not os.path.exists(localdirpath): raise BuildSetupException( "Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) if name != "bitbake": layerlist.append(localdirpath.rstrip("/")) logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) # 4. configure the build environment, so we have a conf/bblayers.conf assert self.pokydirname is not None self._setupBE() # 5. update the bblayers.conf bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") if not os.path.exists(bblayerconf): raise BuildSetupException( "BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) # 6. create custom layer and add custom recipes to it layerpath = os.path.join(self.be.sourcedir, "_meta-toaster-custom") if os.path.isdir(layerpath): shutil.rmtree(layerpath) # remove leftovers from previous builds for target in targets: try: customrecipe = CustomImageRecipe.objects.get( name=target.target, project=bitbakes[0].req.project) except CustomImageRecipe.DoesNotExist: continue # not a custom recipe, skip # create directory structure for name in ("conf", "recipes"): path = os.path.join(layerpath, name) if not os.path.isdir(path): os.makedirs(path) # create layer.oonf config = os.path.join(layerpath, "conf", "layer.conf") if not os.path.isfile(config): with open(config, "w") as conf: conf.write( 'BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n' ) # create recipe recipe = os.path.join(layerpath, "recipes", "%s.bb" % target.target) with open(recipe, "w") as recipef: recipef.write("require %s\n" % customrecipe.base_recipe.recipe.file_path) packages = [pkg.name for pkg in customrecipe.packages.all()] if packages: recipef.write('IMAGE_INSTALL = "%s"\n' % ' '.join(packages)) # create *Layer* objects needed for build machinery to work layer = Layer.objects.get_or_create( name="Toaster Custom layer", summary="Layer for custom recipes", vcs_url="file://%s" % layerpath)[0] breq = target.req lver = Layer_Version.objects.get_or_create(project=breq.project, layer=layer, dirpath=layerpath, build=breq.build)[0] ProjectLayer.objects.get_or_create(project=breq.project, layercommit=lver, optional=False) BRLayer.objects.get_or_create(req=breq, name=layer.name, dirpath=layerpath, giturl="file://%s" % layerpath) if os.path.isdir(layerpath): layerlist.append(layerpath) BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist) self.islayerset = True return True