def setLayers(self, bitbakes, layers):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
assert len(bitbakes) == 1
# set layers in the layersource
raise NotImplementedException("Not implemented: SSH setLayers")
# 3. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()
# 4. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not self._pathexists(bblayerconf):
raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
import uuid
local_bblayerconf = "/tmp/" + uuid.uuid4() + "-bblayer.conf"
self.pullFile(bblayerconf, local_bblayerconf)
BuildEnvironmentController._updateBBLayers(local_bblayerconf, layerlist)
self.pushFile(local_bblayerconf, bblayerconf)
os.unlink(local_bblayerconf)
self.islayerset = True
return True
def setLayers(self, bitbakes, layers):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
assert len(bitbakes) == 1
# set layers in the layersource
raise NotImplementedException("Not implemented: SSH setLayers")
# 3. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()
# 4. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not self._pathexists(bblayerconf):
raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
import uuid
local_bblayerconf = "/tmp/" + uuid.uuid4() + "-bblayer.conf"
self.pullFile(bblayerconf, local_bblayerconf)
BuildEnvironmentController._updateBBLayers(local_bblayerconf, layerlist)
self.pushFile(local_bblayerconf, bblayerconf)
os.unlink(local_bblayerconf)
self.islayerset = True
return True
def setLayers(self, bitbake, layers, targets):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
# set layers in the layersource
# 1. get a list of repos with branches, and map dirpaths for each layer
gitrepos = {}
gitrepos[(bitbake.giturl, bitbake.commit)] = []
gitrepos[(bitbake.giturl, bitbake.commit)].append( ("bitbake", bitbake.dirpath) )
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not (layer.giturl, layer.commit) in gitrepos:
gitrepos[(layer.giturl, layer.commit)] = []
gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) )
logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos))
# 2. Note for future use if the current source directory is a
# checked-out git repos that could match a layer's vcs_url and therefore
# be used to speed up cloning (rather than fetching it again).
cached_layers = {}
try:
for remotes in self._shellcmd("git remote -v", self.be.sourcedir).split("\n"):
try:
remote = remotes.split("\t")[1].split(" ")[0]
if remote not in cached_layers:
cached_layers[remote] = self.be.sourcedir
except IndexError:
pass
except ShellCmdException:
# ignore any errors in collecting git remotes this is an optional
# step
pass
logger.info("Using pre-checked out source for layer %s", cached_layers)
layerlist = []
# 3. checkout the repositories
for giturl, commit in gitrepos.keys():
localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
# make sure our directory is a git repository
if os.path.exists(localdirname):
localremotes = self._shellcmd("git remote -v", localdirname)
if not giturl in localremotes:
raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
else:
if giturl in cached_layers:
logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
self._shellcmd("git remote remove origin", localdirname)
self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
else:
logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname))
self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname))
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit
self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
self.pokydirname = localdirname
# make sure we have a working bitbake
if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')))
# verify our repositories
for name, dirpath in gitrepos[(giturl, commit)]:
localdirpath = os.path.join(localdirname, dirpath)
logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
if not os.path.exists(localdirpath):
raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
# 4. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not os.path.exists(bblayerconf):
raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
# 5. create custom layer and add custom recipes to it
layerpath = os.path.join(self.be.sourcedir, "_meta-toaster-custom")
if os.path.isdir(layerpath):
shutil.rmtree(layerpath) # remove leftovers from previous builds
for target in targets:
try:
customrecipe = CustomImageRecipe.objects.get(name=target.target,
project=bitbake.req.project)
except CustomImageRecipe.DoesNotExist:
continue # not a custom recipe, skip
# create directory structure
for name in ("conf", "recipes"):
path = os.path.join(layerpath, name)
if not os.path.isdir(path):
os.makedirs(path)
# create layer.oonf
config = os.path.join(layerpath, "conf", "layer.conf")
if not os.path.isfile(config):
with open(config, "w") as conf:
conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
# create recipe
recipe = os.path.join(layerpath, "recipes", "%s.bb" % target.target)
with open(recipe, "w") as recipef:
recipef.write("require %s\n" % customrecipe.base_recipe.recipe.file_path)
packages = [pkg.name for pkg in customrecipe.packages.all()]
if packages:
recipef.write('IMAGE_INSTALL = "%s"\n' % ' '.join(packages))
# create *Layer* objects needed for build machinery to work
layer = Layer.objects.get_or_create(name="Toaster Custom layer",
summary="Layer for custom recipes",
vcs_url="file://%s" % layerpath)[0]
breq = target.req
lver = Layer_Version.objects.get_or_create(project=breq.project, layer=layer,
dirpath=layerpath, build=breq.build)[0]
ProjectLayer.objects.get_or_create(project=breq.project, layercommit=lver,
optional=False)
BRLayer.objects.get_or_create(req=breq, name=layer.name, dirpath=layerpath,
giturl="file://%s" % layerpath)
if os.path.isdir(layerpath):
layerlist.append(layerpath)
BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
self.islayerset = True
return True
def setLayers(self, bitbakes, layers):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
assert len(bitbakes) == 1
# set layers in the layersource
# 1. get a list of repos with branches, and map dirpaths for each layer
gitrepos = {}
gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = []
gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append( ("bitbake", bitbakes[0].dirpath) )
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not (layer.giturl, layer.commit) in gitrepos:
gitrepos[(layer.giturl, layer.commit)] = []
gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) )
logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos))
# 2. find checked-out git repos in the sourcedir directory that may help faster cloning
cached_layers = {}
for ldir in os.listdir(self.be.sourcedir):
fldir = os.path.join(self.be.sourcedir, ldir)
if os.path.isdir(fldir):
try:
for line in self._shellcmd("git remote -v", fldir).split("\n"):
try:
remote = line.split("\t")[1].split(" ")[0]
if remote not in cached_layers:
cached_layers[remote] = fldir
except IndexError:
pass
except ShellCmdException:
# ignore any errors in collecting git remotes
pass
layerlist = []
# 3. checkout the repositories
for giturl, commit in gitrepos.keys():
localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
# make sure our directory is a git repository
if os.path.exists(localdirname):
localremotes = self._shellcmd("git remote -v", localdirname)
if not giturl in localremotes:
raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
else:
if giturl in cached_layers:
logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
self._shellcmd("git remote remove origin", localdirname)
self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
else:
logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname))
self._shellcmd("git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname))
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
self._shellcmd("git fetch --all && git checkout \"%s\" && git rebase \"origin/%s\"" % (commit, commit) , localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
self.pokydirname = localdirname
# make sure we have a working bitbake
if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
# verify our repositories
for name, dirpath in gitrepos[(giturl, commit)]:
localdirpath = os.path.join(localdirname, dirpath)
logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
if not os.path.exists(localdirpath):
raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
# 4. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()
# 5. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not os.path.exists(bblayerconf):
raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
self.islayerset = True
return True
def setLayers(self, bitbake, layers, targets):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
# set layers in the layersource
# 1. get a list of repos with branches, and map dirpaths for each layer
gitrepos = {}
gitrepos[(bitbake.giturl, bitbake.commit)] = []
gitrepos[(bitbake.giturl, bitbake.commit)].append(
("bitbake", bitbake.dirpath))
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not (layer.giturl, layer.commit) in gitrepos:
gitrepos[(layer.giturl, layer.commit)] = []
gitrepos[(layer.giturl, layer.commit)].append(
(layer.name, layer.dirpath))
logger.debug("localhostbecontroller, our git repos are %s" %
pformat(gitrepos))
# 2. Note for future use if the current source directory is a
# checked-out git repos that could match a layer's vcs_url and therefore
# be used to speed up cloning (rather than fetching it again).
cached_layers = {}
try:
for remotes in self._shellcmd("git remote -v",
self.be.sourcedir).split("\n"):
try:
remote = remotes.split("\t")[1].split(" ")[0]
if remote not in cached_layers:
cached_layers[remote] = self.be.sourcedir
except IndexError:
pass
except ShellCmdException:
# ignore any errors in collecting git remotes this is an optional
# step
pass
logger.info("Using pre-checked out source for layer %s", cached_layers)
layerlist = []
# Verify HEAD layers
layerlist = self.getHeadLayers(gitrepos, layers)
# If the layerlist doesn't have already cloned layers then checkout repositories
if len(layerlist) == 0:
# 3. checkout the repositories
for giturl, commit in gitrepos.keys():
localdirname = os.path.join(
self.be.sourcedir,
self.getGitCloneDirectory(giturl, commit))
logger.debug(
"localhostbecontroller: giturl %s:%s checking out in current directory %s"
% (giturl, commit, localdirname))
# make sure our directory is a git repository
if os.path.exists(localdirname):
localremotes = self._shellcmd("git remote -v",
localdirname)
if not giturl in localremotes:
raise BuildSetupException(
"Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something."
% (localdirname, ", ".join(
localremotes.split("\n")), giturl))
else:
if giturl in cached_layers:
logger.debug(
"localhostbecontroller git-copying %s to %s" %
(cached_layers[giturl], localdirname))
self._shellcmd("git clone \"%s\" \"%s\"" %
(cached_layers[giturl], localdirname))
self._shellcmd("git remote remove origin",
localdirname)
self._shellcmd("git remote add origin \"%s\"" % giturl,
localdirname)
else:
logger.debug(
"localhostbecontroller: cloning %s in %s" %
(giturl, localdirname))
self._shellcmd('git clone "%s" "%s"' %
(giturl, localdirname))
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
logger.debug(
"localhostbecontroller: checking out commit %s to %s "
% (commit, localdirname))
ref = commit if re.match('^[a-fA-F0-9]+$',
commit) else 'origin/%s' % commit
self._shellcmd(
'git fetch --all && git reset --hard "%s"' % ref,
localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(
os.path.join(localdirname, "oe-init-build-env")):
logger.debug(
"localhostbecontroller: selected poky dir name %s" %
localdirname)
self.pokydirname = localdirname
# make sure we have a working bitbake
if not os.path.exists(
os.path.join(self.pokydirname, 'bitbake')):
logger.debug(
"localhostbecontroller: checking bitbake into the poky dirname %s "
% self.pokydirname)
self._shellcmd(
"git clone -b \"%s\" \"%s\" \"%s\" " %
(bitbake.commit, bitbake.giturl,
os.path.join(self.pokydirname, 'bitbake')))
# verify our repositories
for name, dirpath in gitrepos[(giturl, commit)]:
localdirpath = os.path.join(localdirname, dirpath)
logger.debug(
"localhostbecontroller: localdirpath expected '%s'" %
localdirpath)
if not os.path.exists(localdirpath):
raise BuildSetupException(
"Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting."
% (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
logger.debug("localhostbecontroller: current layer list %s " %
pformat(layerlist))
# 4. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not os.path.exists(bblayerconf):
raise BuildSetupException(
"BE is not consistent: bblayers.conf file missing at %s" %
bblayerconf)
# 5. create custom layer and add custom recipes to it
layerpath = os.path.join(self.be.sourcedir, "_meta-toaster-custom")
if os.path.isdir(layerpath):
shutil.rmtree(layerpath) # remove leftovers from previous builds
for target in targets:
try:
customrecipe = CustomImageRecipe.objects.get(
name=target.target, project=bitbake.req.project)
except CustomImageRecipe.DoesNotExist:
continue # not a custom recipe, skip
# create directory structure
for name in ("conf", "recipes"):
path = os.path.join(layerpath, name)
if not os.path.isdir(path):
os.makedirs(path)
# create layer.oonf
config = os.path.join(layerpath, "conf", "layer.conf")
if not os.path.isfile(config):
with open(config, "w") as conf:
conf.write(
'BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n'
)
# Update the Layer_Version dirpath that has our base_recipe in
# to be able to read the base recipe to then generate the
# custom recipe.
br_layer_base_recipe = layers.get(
layer_version=customrecipe.base_recipe.layer_version)
br_layer_base_dirpath = \
os.path.join(self.be.sourcedir,
self.getGitCloneDirectory(
br_layer_base_recipe.giturl,
br_layer_base_recipe.commit),
customrecipe.base_recipe.layer_version.dirpath
)
customrecipe.base_recipe.layer_version.dirpath = \
br_layer_base_dirpath
customrecipe.base_recipe.layer_version.save()
# create recipe
recipe_path = \
os.path.join(layerpath, "recipes", "%s.bb" % target.target)
with open(recipe_path, "w") as recipef:
recipef.write(customrecipe.generate_recipe_file_contents())
# Update the layer and recipe objects
customrecipe.layer_version.dirpath = layerpath
customrecipe.layer_version.save()
customrecipe.file_path = recipe_path
customrecipe.save()
# create *Layer* objects needed for build machinery to work
BRLayer.objects.get_or_create(req=target.req,
name=layer.name,
dirpath=layerpath,
giturl="file://%s" % layerpath)
if os.path.isdir(layerpath):
layerlist.append(layerpath)
BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
self.islayerset = True
return True
def setLayers(self, bitbakes, layers):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
assert len(bitbakes) == 1
# set layers in the layersource
# 1. get a list of repos, and map dirpaths for each layer
gitrepos = {}
gitrepos[bitbakes[0].giturl] = []
gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not layer.giturl in gitrepos:
gitrepos[layer.giturl] = []
gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit))
for giturl in gitrepos.keys():
commitid = gitrepos[giturl][0][2]
for e in gitrepos[giturl]:
if commitid != e[2]:
import pprint
raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos))
logger.debug("localhostbecontroller, our git repos are %s" % gitrepos)
layerlist = []
# 2. checkout the repositories
for giturl in gitrepos.keys():
localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl))
logger.debug("localhostbecontroller: giturl %s checking out in current directory %s" % (giturl, localdirname))
# make sure our directory is a git repository
if os.path.exists(localdirname):
if not giturl in self._shellcmd("git remote -v", localdirname):
raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
else:
self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname))
# checkout the needed commit
commit = gitrepos[giturl][0][2]
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
self.pokydirname = localdirname
# make sure we have a working bitbake
if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
# verify our repositories
for name, dirpath, commit in gitrepos[giturl]:
localdirpath = os.path.join(localdirname, dirpath)
if not os.path.exists(localdirpath):
raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
logger.debug("localhostbecontroller: current layer list %s " % layerlist)
# 3. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()
# 4. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not os.path.exists(bblayerconf):
raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
self.islayerset = True
return True
def setLayers(self, bitbakes, layers):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
assert len(bitbakes) == 1
# set layers in the layersource
# 1. get a list of repos, and map dirpaths for each layer
gitrepos = {}
gitrepos[bitbakes[0].giturl] = []
gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not layer.giturl in gitrepos:
gitrepos[layer.giturl] = []
gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit))
for giturl in gitrepos.keys():
commitid = gitrepos[giturl][0][2]
for e in gitrepos[giturl]:
if commitid != e[2]:
import pprint
raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos))
layerlist = []
# 2. checkout the repositories
for giturl in gitrepos.keys():
localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl))
print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname
# make sure our directory is a git repository
if os.path.exists(localdirname):
if not giturl in self._shellcmd("git remote -v", localdirname):
raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
else:
self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname))
# checkout the needed commit
commit = gitrepos[giturl][0][2]
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
print "DEBUG: checking out commit ", commit, "to", localdirname
self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
print "DEBUG: selected poky dir name", localdirname
self.pokydirname = localdirname
# make sure we have a working bitbake
if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
print "DEBUG: checking bitbake into the poky dirname %s " % self.pokydirname
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
# verify our repositories
for name, dirpath, commit in gitrepos[giturl]:
localdirpath = os.path.join(localdirname, dirpath)
if not os.path.exists(localdirpath):
raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
print "DEBUG: current layer list ", layerlist
# 3. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()
# 4. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not os.path.exists(bblayerconf):
raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
self.islayerset = True
return True
def setLayers(self, bitbakes, layers):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
assert len(bitbakes) == 1
# set layers in the layersource
# 1. get a list of repos with branches, and map dirpaths for each layer
gitrepos = {}
gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = []
gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append(
("bitbake", bitbakes[0].dirpath))
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not (layer.giturl, layer.commit) in gitrepos:
gitrepos[(layer.giturl, layer.commit)] = []
gitrepos[(layer.giturl, layer.commit)].append(
(layer.name, layer.dirpath))
logger.debug("localhostbecontroller, our git repos are %s" %
pformat(gitrepos))
# 2. find checked-out git repos in the sourcedir directory that may help faster cloning
cached_layers = {}
for ldir in os.listdir(self.be.sourcedir):
fldir = os.path.join(self.be.sourcedir, ldir)
if os.path.isdir(fldir):
try:
for line in self._shellcmd("git remote -v",
fldir).split("\n"):
try:
remote = line.split("\t")[1].split(" ")[0]
if remote not in cached_layers:
cached_layers[remote] = fldir
except IndexError:
pass
except ShellCmdException:
# ignore any errors in collecting git remotes
pass
layerlist = []
# 3. checkout the repositories
for giturl, commit in gitrepos.keys():
localdirname = os.path.join(
self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
logger.debug(
"localhostbecontroller: giturl %s:%s checking out in current directory %s"
% (giturl, commit, localdirname))
# make sure our directory is a git repository
if os.path.exists(localdirname):
localremotes = self._shellcmd("git remote -v", localdirname)
if not giturl in localremotes:
raise BuildSetupException(
"Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something."
% (localdirname, ", ".join(
localremotes.split("\n")), giturl))
else:
if giturl in cached_layers:
logger.debug("localhostbecontroller git-copying %s to %s" %
(cached_layers[giturl], localdirname))
self._shellcmd("git clone \"%s\" \"%s\"" %
(cached_layers[giturl], localdirname))
self._shellcmd("git remote remove origin", localdirname)
self._shellcmd("git remote add origin \"%s\"" % giturl,
localdirname)
else:
logger.debug("localhostbecontroller: cloning %s:%s in %s" %
(giturl, commit, localdirname))
self._shellcmd(
"git clone \"%s\" --single-branch --branch \"%s\" \"%s\""
% (giturl, commit, localdirname))
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
logger.debug(
"localhostbecontroller: checking out commit %s to %s " %
(commit, localdirname))
self._shellcmd(
"git fetch --all && git checkout \"%s\" && git rebase \"origin/%s\""
% (commit, commit), localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(
os.path.join(localdirname, "oe-init-build-env")):
logger.debug(
"localhostbecontroller: selected poky dir name %s" %
localdirname)
self.pokydirname = localdirname
# make sure we have a working bitbake
if not os.path.exists(os.path.join(self.pokydirname,
'bitbake')):
logger.debug(
"localhostbecontroller: checking bitbake into the poky dirname %s "
% self.pokydirname)
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " %
(bitbakes[0].commit, bitbakes[0].giturl,
os.path.join(self.pokydirname, 'bitbake')))
# verify our repositories
for name, dirpath in gitrepos[(giturl, commit)]:
localdirpath = os.path.join(localdirname, dirpath)
logger.debug(
"localhostbecontroller: localdirpath expected '%s'" %
localdirpath)
if not os.path.exists(localdirpath):
raise BuildSetupException(
"Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting."
% (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
logger.debug("localhostbecontroller: current layer list %s " %
pformat(layerlist))
# 4. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()
# 5. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not os.path.exists(bblayerconf):
raise BuildSetupException(
"BE is not consistent: bblayers.conf file missing at %s" %
bblayerconf)
BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
self.islayerset = True
return True
def setLayers(self, bitbakes, layers, targets):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
assert len(bitbakes) == 1
# set layers in the layersource
# 1. get a list of repos with branches, and map dirpaths for each layer
gitrepos = {}
gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = []
gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append(
("bitbake", bitbakes[0].dirpath))
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not (layer.giturl, layer.commit) in gitrepos:
gitrepos[(layer.giturl, layer.commit)] = []
gitrepos[(layer.giturl, layer.commit)].append(
(layer.name, layer.dirpath))
logger.debug("localhostbecontroller, our git repos are %s" %
pformat(gitrepos))
# 2. find checked-out git repos in the sourcedir directory that may help faster cloning
cached_layers = {}
for ldir in os.listdir(self.be.sourcedir):
fldir = os.path.join(self.be.sourcedir, ldir)
if os.path.isdir(fldir):
try:
for line in self._shellcmd("git remote -v",
fldir).split("\n"):
try:
remote = line.split("\t")[1].split(" ")[0]
if remote not in cached_layers:
cached_layers[remote] = fldir
except IndexError:
pass
except ShellCmdException:
# ignore any errors in collecting git remotes
pass
layerlist = []
# 3. checkout the repositories
for giturl, commit in gitrepos.keys():
localdirname = os.path.join(
self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
logger.debug(
"localhostbecontroller: giturl %s:%s checking out in current directory %s"
% (giturl, commit, localdirname))
# make sure our directory is a git repository
if os.path.exists(localdirname):
localremotes = self._shellcmd("git remote -v", localdirname)
if not giturl in localremotes:
raise BuildSetupException(
"Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something."
% (localdirname, ", ".join(
localremotes.split("\n")), giturl))
else:
if giturl in cached_layers:
logger.debug("localhostbecontroller git-copying %s to %s" %
(cached_layers[giturl], localdirname))
self._shellcmd("git clone \"%s\" \"%s\"" %
(cached_layers[giturl], localdirname))
self._shellcmd("git remote remove origin", localdirname)
self._shellcmd("git remote add origin \"%s\"" % giturl,
localdirname)
else:
logger.debug("localhostbecontroller: cloning %s in %s" %
(giturl, localdirname))
self._shellcmd('git clone "%s" "%s"' %
(giturl, localdirname))
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
logger.debug(
"localhostbecontroller: checking out commit %s to %s " %
(commit, localdirname))
ref = commit if re.match('^[a-fA-F0-9]+$',
commit) else 'origin/%s' % commit
self._shellcmd(
'git fetch --all && git reset --hard "%s"' % ref,
localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(
os.path.join(localdirname, "oe-init-build-env")):
logger.debug(
"localhostbecontroller: selected poky dir name %s" %
localdirname)
self.pokydirname = localdirname
# make sure we have a working bitbake
if not os.path.exists(os.path.join(self.pokydirname,
'bitbake')):
logger.debug(
"localhostbecontroller: checking bitbake into the poky dirname %s "
% self.pokydirname)
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " %
(bitbakes[0].commit, bitbakes[0].giturl,
os.path.join(self.pokydirname, 'bitbake')))
# verify our repositories
for name, dirpath in gitrepos[(giturl, commit)]:
localdirpath = os.path.join(localdirname, dirpath)
logger.debug(
"localhostbecontroller: localdirpath expected '%s'" %
localdirpath)
if not os.path.exists(localdirpath):
raise BuildSetupException(
"Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting."
% (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
logger.debug("localhostbecontroller: current layer list %s " %
pformat(layerlist))
# 4. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()
# 5. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not os.path.exists(bblayerconf):
raise BuildSetupException(
"BE is not consistent: bblayers.conf file missing at %s" %
bblayerconf)
# 6. create custom layer and add custom recipes to it
layerpath = os.path.join(self.be.sourcedir, "_meta-toaster-custom")
if os.path.isdir(layerpath):
shutil.rmtree(layerpath) # remove leftovers from previous builds
for target in targets:
try:
customrecipe = CustomImageRecipe.objects.get(
name=target.target, project=bitbakes[0].req.project)
except CustomImageRecipe.DoesNotExist:
continue # not a custom recipe, skip
# create directory structure
for name in ("conf", "recipes"):
path = os.path.join(layerpath, name)
if not os.path.isdir(path):
os.makedirs(path)
# create layer.oonf
config = os.path.join(layerpath, "conf", "layer.conf")
if not os.path.isfile(config):
with open(config, "w") as conf:
conf.write(
'BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n'
)
# create recipe
recipe = os.path.join(layerpath, "recipes",
"%s.bb" % target.target)
with open(recipe, "w") as recipef:
recipef.write("require %s\n" %
customrecipe.base_recipe.recipe.file_path)
packages = [pkg.name for pkg in customrecipe.packages.all()]
if packages:
recipef.write('IMAGE_INSTALL = "%s"\n' %
' '.join(packages))
# create *Layer* objects needed for build machinery to work
layer = Layer.objects.get_or_create(
name="Toaster Custom layer",
summary="Layer for custom recipes",
vcs_url="file://%s" % layerpath)[0]
breq = target.req
lver = Layer_Version.objects.get_or_create(project=breq.project,
layer=layer,
dirpath=layerpath,
build=breq.build)[0]
ProjectLayer.objects.get_or_create(project=breq.project,
layercommit=lver,
optional=False)
BRLayer.objects.get_or_create(req=breq,
name=layer.name,
dirpath=layerpath,
giturl="file://%s" % layerpath)
if os.path.isdir(layerpath):
layerlist.append(layerpath)
BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
self.islayerset = True
return True