def setLayers(self, bitbakes, layers):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
assert len(bitbakes) == 1
# set layers in the layersource
raise NotImplementedException("Not implemented: SSH setLayers")
# 3. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()
# 4. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not self._pathexists(bblayerconf):
raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
import uuid
local_bblayerconf = "/tmp/" + uuid.uuid4() + "-bblayer.conf"
self.pullFile(bblayerconf, local_bblayerconf)
BuildEnvironmentController._updateBBLayers(local_bblayerconf, layerlist)
self.pushFile(local_bblayerconf, bblayerconf)
os.unlink(local_bblayerconf)
self.islayerset = True
return True
def startBBServer(self):
assert self.pokydirname and os.path.exists(self.pokydirname)
assert self.islayerset
# find our own toasterui listener/bitbake
from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path
toaster = _reduce_canon_path(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
"../../../bin/toaster"))
assert os.path.exists(toaster) and os.path.isfile(toaster)
# restart bitbake server and toastergui observer
self._shellcmd("bash -c 'source %s restart-bitbake'" % toaster,
self.be.builddir)
logger.debug("localhostbecontroller: restarted bitbake server")
# read port number from bitbake.lock
self.be.bbport = ""
bblock = os.path.join(self.be.builddir, 'bitbake.lock')
if os.path.exists(bblock):
with open(bblock) as fplock:
for line in fplock:
if ":" in line:
self.be.bbport = line.split(":")[-1].strip()
logger.debug("localhostbecontroller: bitbake port %s",
self.be.bbport)
break
if not self.be.bbport:
raise BuildSetupException(
"localhostbecontroller: can't read bitbake port from %s" %
bblock)
self.be.bbaddress = "localhost"
self.be.bbstate = BuildEnvironment.SERVER_STARTED
self.be.save()
def setLayers(self, bitbake, layers, targets):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
# set layers in the layersource
# 1. get a list of repos with branches, and map dirpaths for each layer
gitrepos = {}
gitrepos[(bitbake.giturl, bitbake.commit)] = []
gitrepos[(bitbake.giturl, bitbake.commit)].append(
("bitbake", bitbake.dirpath))
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not (layer.giturl, layer.commit) in gitrepos:
gitrepos[(layer.giturl, layer.commit)] = []
gitrepos[(layer.giturl, layer.commit)].append(
(layer.name, layer.dirpath))
logger.debug("localhostbecontroller, our git repos are %s" %
pformat(gitrepos))
# 2. Note for future use if the current source directory is a
# checked-out git repos that could match a layer's vcs_url and therefore
# be used to speed up cloning (rather than fetching it again).
cached_layers = {}
try:
for remotes in self._shellcmd("git remote -v",
self.be.sourcedir).split("\n"):
try:
remote = remotes.split("\t")[1].split(" ")[0]
if remote not in cached_layers:
cached_layers[remote] = self.be.sourcedir
except IndexError:
pass
except ShellCmdException:
# ignore any errors in collecting git remotes this is an optional
# step
pass
logger.info("Using pre-checked out source for layer %s", cached_layers)
layerlist = []
# Verify HEAD layers
layerlist = self.getHeadLayers(gitrepos, layers)
# If the layerlist doesn't have already cloned layers then checkout repositories
if len(layerlist) == 0:
# 3. checkout the repositories
for giturl, commit in gitrepos.keys():
localdirname = os.path.join(
self.be.sourcedir,
self.getGitCloneDirectory(giturl, commit))
logger.debug(
"localhostbecontroller: giturl %s:%s checking out in current directory %s"
% (giturl, commit, localdirname))
# make sure our directory is a git repository
if os.path.exists(localdirname):
localremotes = self._shellcmd("git remote -v",
localdirname)
if not giturl in localremotes:
raise BuildSetupException(
"Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something."
% (localdirname, ", ".join(
localremotes.split("\n")), giturl))
else:
if giturl in cached_layers:
logger.debug(
"localhostbecontroller git-copying %s to %s" %
(cached_layers[giturl], localdirname))
self._shellcmd("git clone \"%s\" \"%s\"" %
(cached_layers[giturl], localdirname))
self._shellcmd("git remote remove origin",
localdirname)
self._shellcmd("git remote add origin \"%s\"" % giturl,
localdirname)
else:
logger.debug(
"localhostbecontroller: cloning %s in %s" %
(giturl, localdirname))
self._shellcmd('git clone "%s" "%s"' %
(giturl, localdirname))
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
logger.debug(
"localhostbecontroller: checking out commit %s to %s "
% (commit, localdirname))
ref = commit if re.match('^[a-fA-F0-9]+$',
commit) else 'origin/%s' % commit
self._shellcmd(
'git fetch --all && git reset --hard "%s"' % ref,
localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(
os.path.join(localdirname, "oe-init-build-env")):
logger.debug(
"localhostbecontroller: selected poky dir name %s" %
localdirname)
self.pokydirname = localdirname
# make sure we have a working bitbake
if not os.path.exists(
os.path.join(self.pokydirname, 'bitbake')):
logger.debug(
"localhostbecontroller: checking bitbake into the poky dirname %s "
% self.pokydirname)
self._shellcmd(
"git clone -b \"%s\" \"%s\" \"%s\" " %
(bitbake.commit, bitbake.giturl,
os.path.join(self.pokydirname, 'bitbake')))
# verify our repositories
for name, dirpath in gitrepos[(giturl, commit)]:
localdirpath = os.path.join(localdirname, dirpath)
logger.debug(
"localhostbecontroller: localdirpath expected '%s'" %
localdirpath)
if not os.path.exists(localdirpath):
raise BuildSetupException(
"Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting."
% (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
logger.debug("localhostbecontroller: current layer list %s " %
pformat(layerlist))
# 4. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not os.path.exists(bblayerconf):
raise BuildSetupException(
"BE is not consistent: bblayers.conf file missing at %s" %
bblayerconf)
# 5. create custom layer and add custom recipes to it
layerpath = os.path.join(self.be.sourcedir, "_meta-toaster-custom")
if os.path.isdir(layerpath):
shutil.rmtree(layerpath) # remove leftovers from previous builds
for target in targets:
try:
customrecipe = CustomImageRecipe.objects.get(
name=target.target, project=bitbake.req.project)
except CustomImageRecipe.DoesNotExist:
continue # not a custom recipe, skip
# create directory structure
for name in ("conf", "recipes"):
path = os.path.join(layerpath, name)
if not os.path.isdir(path):
os.makedirs(path)
# create layer.oonf
config = os.path.join(layerpath, "conf", "layer.conf")
if not os.path.isfile(config):
with open(config, "w") as conf:
conf.write(
'BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n'
)
# Update the Layer_Version dirpath that has our base_recipe in
# to be able to read the base recipe to then generate the
# custom recipe.
br_layer_base_recipe = layers.get(
layer_version=customrecipe.base_recipe.layer_version)
br_layer_base_dirpath = \
os.path.join(self.be.sourcedir,
self.getGitCloneDirectory(
br_layer_base_recipe.giturl,
br_layer_base_recipe.commit),
customrecipe.base_recipe.layer_version.dirpath
)
customrecipe.base_recipe.layer_version.dirpath = \
br_layer_base_dirpath
customrecipe.base_recipe.layer_version.save()
# create recipe
recipe_path = \
os.path.join(layerpath, "recipes", "%s.bb" % target.target)
with open(recipe_path, "w") as recipef:
recipef.write(customrecipe.generate_recipe_file_contents())
# Update the layer and recipe objects
customrecipe.layer_version.dirpath = layerpath
customrecipe.layer_version.save()
customrecipe.file_path = recipe_path
customrecipe.save()
# create *Layer* objects needed for build machinery to work
BRLayer.objects.get_or_create(req=target.req,
name=layer.name,
dirpath=layerpath,
giturl="file://%s" % layerpath)
if os.path.isdir(layerpath):
layerlist.append(layerpath)
BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
self.islayerset = True
return True
def setLayers(self, bitbakes, layers):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
assert len(bitbakes) == 1
# set layers in the layersource
# 1. get a list of repos, and map dirpaths for each layer
gitrepos = {}
gitrepos[bitbakes[0].giturl] = []
gitrepos[bitbakes[0].giturl].append(
("bitbake", bitbakes[0].dirpath, bitbakes[0].commit))
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not layer.giturl in gitrepos:
gitrepos[layer.giturl] = []
gitrepos[layer.giturl].append(
(layer.name, layer.dirpath, layer.commit))
for giturl in gitrepos.keys():
commitid = gitrepos[giturl][0][2]
for e in gitrepos[giturl]:
if commitid != e[2]:
raise BuildSetupException(
"More than one commit per git url, unsupported configuration"
)
layerlist = []
# 2. checkout the repositories
for giturl in gitrepos.keys():
import os
localdirname = os.path.join(self.be.sourcedir,
_getgitcheckoutdirectoryname(giturl))
print "DEBUG: giturl ", giturl, "checking out in current directory", localdirname
# make sure our directory is a git repository
if self._pathexists(localdirname):
if not giturl in self._shellcmd("git remote -v", localdirname):
raise BuildSetupException(
"Existing git repository at %s, but with different remotes (not '%s'). Aborting."
% (localdirname, giturl))
else:
self._shellcmd("git clone \"%s\" \"%s\"" %
(giturl, localdirname))
# checkout the needed commit
commit = gitrepos[giturl][0][2]
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
print "DEBUG: checking out commit ", commit, "to", localdirname
self._shellcmd(
"git fetch --all && git checkout \"%s\"" % commit,
localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and self._pathexists(
os.path.join(localdirname, "oe-init-build-env")):
print "DEBUG: selected poky dir name", localdirname
self.pokydirname = localdirname
# verify our repositories
for name, dirpath, commit in gitrepos[giturl]:
localdirpath = os.path.join(localdirname, dirpath)
if not self._pathexists(localdirpath):
raise BuildSetupException(
"Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting."
% (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath)
print "DEBUG: current layer list ", layerlist
# 3. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()
# 4. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not self._pathexists(bblayerconf):
raise BuildSetupException(
"BE is not consistent: bblayers.conf file missing at %s" %
bblayerconf)
conflines = open(bblayerconf, "r").readlines()
bblayerconffile = open(bblayerconf, "w")
for i in xrange(len(conflines)):
if conflines[i].startswith("# line added by toaster"):
i += 2
else:
bblayerconffile.write(conflines[i])
bblayerconffile.write(
"\n# line added by toaster build control\nBBLAYERS = \"" +
" ".join(layerlist) + "\"")
bblayerconffile.close()
self.islayerset = True
return True
def setLayers(self, bitbakes, layers):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
assert len(bitbakes) == 1
# set layers in the layersource
# 1. get a list of repos, and map dirpaths for each layer
gitrepos = {}
gitrepos[bitbakes[0].giturl] = []
gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not layer.giturl in gitrepos:
gitrepos[layer.giturl] = []
gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit))
for giturl in gitrepos.keys():
commitid = gitrepos[giturl][0][2]
for e in gitrepos[giturl]:
if commitid != e[2]:
import pprint
raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos))
logger.debug("localhostbecontroller, our git repos are %s" % gitrepos)
layerlist = []
# 2. checkout the repositories
for giturl in gitrepos.keys():
localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl))
logger.debug("localhostbecontroller: giturl %s checking out in current directory %s" % (giturl, localdirname))
# make sure our directory is a git repository
if os.path.exists(localdirname):
if not giturl in self._shellcmd("git remote -v", localdirname):
raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
else:
self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname))
# checkout the needed commit
commit = gitrepos[giturl][0][2]
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
self.pokydirname = localdirname
# make sure we have a working bitbake
if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
# verify our repositories
for name, dirpath, commit in gitrepos[giturl]:
localdirpath = os.path.join(localdirname, dirpath)
if not os.path.exists(localdirpath):
raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
logger.debug("localhostbecontroller: current layer list %s " % layerlist)
# 3. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()
# 4. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not os.path.exists(bblayerconf):
raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
self.islayerset = True
return True
def triggerBuild(self, bitbake, layers, variables, targets, brbe):
layers = self.setLayers(bitbake, layers, targets)
# init build environment from the clone
builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
oe_init = os.path.join(self.pokydirname, 'oe-init-build-env')
# init build environment
self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
self.be.sourcedir)
# update bblayers.conf
bblconfpath = os.path.join(builddir, "conf/bblayers.conf")
conflines = open(bblconfpath, "r").readlines()
skip = False
with open(bblconfpath, 'w') as bblayers:
for line in conflines:
if line.startswith("# line added by toaster"):
skip = True
continue
if skip:
skip = False
else:
bblayers.write(line)
bblayers.write('# line added by toaster build control\n'
'BBLAYERS = "%s"' % ' '.join(layers))
# write configuration file
confpath = os.path.join(builddir, 'conf/toaster.conf')
with open(confpath, 'w') as conf:
for var in variables:
conf.write('%s="%s"\n' % (var.name, var.value))
conf.write('INHERIT+="toaster buildhistory"')
# run bitbake server from the clone
bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
self._shellcmd(
'bash -c \"source %s %s; BITBAKE_UI="" %s --read %s '
'--server-only -t xmlrpc -B 0.0.0.0:0\"' %
(oe_init, builddir, bitbake, confpath), self.be.sourcedir)
# read port number from bitbake.lock
self.be.bbport = ""
bblock = os.path.join(builddir, 'bitbake.lock')
with open(bblock) as fplock:
for line in fplock:
if ":" in line:
self.be.bbport = line.split(":")[-1].strip()
logger.debug("localhostbecontroller: bitbake port %s",
self.be.bbport)
break
if not self.be.bbport:
raise BuildSetupException(
"localhostbecontroller: can't read bitbake port from %s" %
bblock)
self.be.bbaddress = "localhost"
self.be.bbstate = BuildEnvironment.SERVER_STARTED
self.be.lock = BuildEnvironment.LOCK_RUNNING
self.be.save()
bbtargets = ''
for target in targets:
task = target.task
if task:
if not task.startswith('do_'):
task = 'do_' + task
task = ':%s' % task
bbtargets += '%s%s ' % (target.target, task)
# run build with local bitbake. stop the server after the build.
log = os.path.join(builddir, 'toaster_ui.log')
local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
'bitbake')
self._shellcmd(['bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:-1" '
'%s %s -u toasterui --token="" >>%s 2>&1;'
'BITBAKE_UI="" BBSERVER=0.0.0.0:-1 %s -m)&\"' \
% (brbe, local_bitbake, bbtargets, log, bitbake)],
builddir, nowait=True)
logger.debug('localhostbecontroller: Build launched, exiting. '
'Follow build logs at %s' % log)
def startBBServer(self):
assert self.pokydirname and os.path.exists(self.pokydirname)
assert self.islayerset
# find our own toasterui listener/bitbake
from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path
own_bitbake = _reduce_canon_path(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
"../../../bin/bitbake"))
assert os.path.exists(own_bitbake) and os.path.isfile(own_bitbake)
logger.debug("localhostbecontroller: running the listener at %s" %
own_bitbake)
toaster_ui_log_filepath = os.path.join(self.be.builddir,
"toaster_ui.log")
# get the file length; we need to detect the _last_ start of the toaster UI, not the first
toaster_ui_log_filelength = 0
if os.path.exists(toaster_ui_log_filepath):
with open(toaster_ui_log_filepath, "w") as f:
f.seek(0, 2) # jump to the end
toaster_ui_log_filelength = f.tell()
cmd = "bash -c \"source %s/oe-init-build-env %s 2>&1 >toaster_server.log && bitbake --read %s/conf/toaster-pre.conf --postread %s/conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 2>&1 >>toaster_server.log \"" % (
self.pokydirname, self.be.builddir, self.be.builddir,
self.be.builddir)
port = "-1"
logger.debug("localhostbecontroller: starting builder \n%s\n" % cmd)
cmdoutput = self._shellcmd(cmd)
with open(self.be.builddir + "/toaster_server.log", "r") as f:
for i in f.readlines():
if i.startswith("Bitbake server address"):
port = i.split(" ")[-1]
logger.debug(
"localhostbecontroller: Found bitbake server port %s" %
port)
cmd = "bash -c \"source %s/oe-init-build-env-memres -1 %s && %s --observe-only -u toasterui --remote-server=0.0.0.0:-1 -t xmlrpc\"" % \
(self.pokydirname, self.be.builddir, own_bitbake)
# Use a copy of the current environment and add the DATABASE_URL
# for the bitbake observer process.
env = os.environ.copy()
env['DATABASE_URL'] = settings.getDATABASE_URL()
with open(toaster_ui_log_filepath, "a+") as f:
p = subprocess.Popen(cmd,
cwd=self.be.builddir,
shell=True,
stdout=f,
stderr=f,
env=env)
def _toaster_ui_started(filepath, filepos=0):
if not os.path.exists(filepath):
return False
with open(filepath, "r") as f:
f.seek(filepos)
for line in f:
if line.startswith("NOTE: ToasterUI waiting for events"):
return True
return False
retries = 0
started = False
while not started and retries < 50:
started = _toaster_ui_started(toaster_ui_log_filepath,
toaster_ui_log_filelength)
import time
logger.debug(
"localhostbecontroller: Waiting bitbake server to start")
time.sleep(0.5)
retries += 1
if not started:
toaster_ui_log = open(
os.path.join(self.be.builddir, "toaster_ui.log"), "r").read()
toaster_server_log = open(
os.path.join(self.be.builddir, "toaster_server.log"),
"r").read()
raise BuildSetupException(
"localhostbecontroller: Bitbake server did not start in 25 seconds, aborting (Error: '%s' '%s')"
% (toaster_ui_log, toaster_server_log))
logger.debug("localhostbecontroller: Started bitbake server")
while port == "-1":
# the port specification is "autodetect"; read the bitbake.lock file
with open("%s/bitbake.lock" % self.be.builddir, "r") as f:
for line in f.readlines():
if ":" in line:
port = line.split(":")[1].strip()
logger.debug(
"localhostbecontroller: Autodetected bitbake port %s",
port)
break
assert self.be.sourcedir and os.path.exists(self.be.builddir)
self.be.bbaddress = "localhost"
self.be.bbport = port
self.be.bbstate = BuildEnvironment.SERVER_STARTED
self.be.save()
def setLayers(self, bitbakes, layers):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
assert len(bitbakes) == 1
# set layers in the layersource
# 1. get a list of repos with branches, and map dirpaths for each layer
gitrepos = {}
gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = []
gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append(
("bitbake", bitbakes[0].dirpath))
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not (layer.giturl, layer.commit) in gitrepos:
gitrepos[(layer.giturl, layer.commit)] = []
gitrepos[(layer.giturl, layer.commit)].append(
(layer.name, layer.dirpath))
logger.debug("localhostbecontroller, our git repos are %s" %
pformat(gitrepos))
# 2. find checked-out git repos in the sourcedir directory that may help faster cloning
cached_layers = {}
for ldir in os.listdir(self.be.sourcedir):
fldir = os.path.join(self.be.sourcedir, ldir)
if os.path.isdir(fldir):
try:
for line in self._shellcmd("git remote -v",
fldir).split("\n"):
try:
remote = line.split("\t")[1].split(" ")[0]
if remote not in cached_layers:
cached_layers[remote] = fldir
except IndexError:
pass
except ShellCmdException:
# ignore any errors in collecting git remotes
pass
layerlist = []
# 3. checkout the repositories
for giturl, commit in gitrepos.keys():
localdirname = os.path.join(
self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
logger.debug(
"localhostbecontroller: giturl %s:%s checking out in current directory %s"
% (giturl, commit, localdirname))
# make sure our directory is a git repository
if os.path.exists(localdirname):
localremotes = self._shellcmd("git remote -v", localdirname)
if not giturl in localremotes:
raise BuildSetupException(
"Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something."
% (localdirname, ", ".join(
localremotes.split("\n")), giturl))
else:
if giturl in cached_layers:
logger.debug("localhostbecontroller git-copying %s to %s" %
(cached_layers[giturl], localdirname))
self._shellcmd("git clone \"%s\" \"%s\"" %
(cached_layers[giturl], localdirname))
self._shellcmd("git remote remove origin", localdirname)
self._shellcmd("git remote add origin \"%s\"" % giturl,
localdirname)
else:
logger.debug("localhostbecontroller: cloning %s:%s in %s" %
(giturl, commit, localdirname))
self._shellcmd(
"git clone \"%s\" --single-branch --branch \"%s\" \"%s\""
% (giturl, commit, localdirname))
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
logger.debug(
"localhostbecontroller: checking out commit %s to %s " %
(commit, localdirname))
self._shellcmd(
"git fetch --all && git checkout \"%s\" && git rebase \"origin/%s\""
% (commit, commit), localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(
os.path.join(localdirname, "oe-init-build-env")):
logger.debug(
"localhostbecontroller: selected poky dir name %s" %
localdirname)
self.pokydirname = localdirname
# make sure we have a working bitbake
if not os.path.exists(os.path.join(self.pokydirname,
'bitbake')):
logger.debug(
"localhostbecontroller: checking bitbake into the poky dirname %s "
% self.pokydirname)
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " %
(bitbakes[0].commit, bitbakes[0].giturl,
os.path.join(self.pokydirname, 'bitbake')))
# verify our repositories
for name, dirpath in gitrepos[(giturl, commit)]:
localdirpath = os.path.join(localdirname, dirpath)
logger.debug(
"localhostbecontroller: localdirpath expected '%s'" %
localdirpath)
if not os.path.exists(localdirpath):
raise BuildSetupException(
"Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting."
% (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
logger.debug("localhostbecontroller: current layer list %s " %
pformat(layerlist))
# 4. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()
# 5. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not os.path.exists(bblayerconf):
raise BuildSetupException(
"BE is not consistent: bblayers.conf file missing at %s" %
bblayerconf)
BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
self.islayerset = True
return True
def setLayers(self, bitbakes, layers, targets):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
assert len(bitbakes) == 1
# set layers in the layersource
# 1. get a list of repos with branches, and map dirpaths for each layer
gitrepos = {}
gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = []
gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append(
("bitbake", bitbakes[0].dirpath))
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not (layer.giturl, layer.commit) in gitrepos:
gitrepos[(layer.giturl, layer.commit)] = []
gitrepos[(layer.giturl, layer.commit)].append(
(layer.name, layer.dirpath))
logger.debug("localhostbecontroller, our git repos are %s" %
pformat(gitrepos))
# 2. find checked-out git repos in the sourcedir directory that may help faster cloning
cached_layers = {}
for ldir in os.listdir(self.be.sourcedir):
fldir = os.path.join(self.be.sourcedir, ldir)
if os.path.isdir(fldir):
try:
for line in self._shellcmd("git remote -v",
fldir).split("\n"):
try:
remote = line.split("\t")[1].split(" ")[0]
if remote not in cached_layers:
cached_layers[remote] = fldir
except IndexError:
pass
except ShellCmdException:
# ignore any errors in collecting git remotes
pass
layerlist = []
# 3. checkout the repositories
for giturl, commit in gitrepos.keys():
localdirname = os.path.join(
self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
logger.debug(
"localhostbecontroller: giturl %s:%s checking out in current directory %s"
% (giturl, commit, localdirname))
# make sure our directory is a git repository
if os.path.exists(localdirname):
localremotes = self._shellcmd("git remote -v", localdirname)
if not giturl in localremotes:
raise BuildSetupException(
"Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something."
% (localdirname, ", ".join(
localremotes.split("\n")), giturl))
else:
if giturl in cached_layers:
logger.debug("localhostbecontroller git-copying %s to %s" %
(cached_layers[giturl], localdirname))
self._shellcmd("git clone \"%s\" \"%s\"" %
(cached_layers[giturl], localdirname))
self._shellcmd("git remote remove origin", localdirname)
self._shellcmd("git remote add origin \"%s\"" % giturl,
localdirname)
else:
logger.debug("localhostbecontroller: cloning %s in %s" %
(giturl, localdirname))
self._shellcmd('git clone "%s" "%s"' %
(giturl, localdirname))
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
logger.debug(
"localhostbecontroller: checking out commit %s to %s " %
(commit, localdirname))
ref = commit if re.match('^[a-fA-F0-9]+$',
commit) else 'origin/%s' % commit
self._shellcmd(
'git fetch --all && git reset --hard "%s"' % ref,
localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(
os.path.join(localdirname, "oe-init-build-env")):
logger.debug(
"localhostbecontroller: selected poky dir name %s" %
localdirname)
self.pokydirname = localdirname
# make sure we have a working bitbake
if not os.path.exists(os.path.join(self.pokydirname,
'bitbake')):
logger.debug(
"localhostbecontroller: checking bitbake into the poky dirname %s "
% self.pokydirname)
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " %
(bitbakes[0].commit, bitbakes[0].giturl,
os.path.join(self.pokydirname, 'bitbake')))
# verify our repositories
for name, dirpath in gitrepos[(giturl, commit)]:
localdirpath = os.path.join(localdirname, dirpath)
logger.debug(
"localhostbecontroller: localdirpath expected '%s'" %
localdirpath)
if not os.path.exists(localdirpath):
raise BuildSetupException(
"Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting."
% (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
logger.debug("localhostbecontroller: current layer list %s " %
pformat(layerlist))
# 4. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()
# 5. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not os.path.exists(bblayerconf):
raise BuildSetupException(
"BE is not consistent: bblayers.conf file missing at %s" %
bblayerconf)
# 6. create custom layer and add custom recipes to it
layerpath = os.path.join(self.be.sourcedir, "_meta-toaster-custom")
if os.path.isdir(layerpath):
shutil.rmtree(layerpath) # remove leftovers from previous builds
for target in targets:
try:
customrecipe = CustomImageRecipe.objects.get(
name=target.target, project=bitbakes[0].req.project)
except CustomImageRecipe.DoesNotExist:
continue # not a custom recipe, skip
# create directory structure
for name in ("conf", "recipes"):
path = os.path.join(layerpath, name)
if not os.path.isdir(path):
os.makedirs(path)
# create layer.oonf
config = os.path.join(layerpath, "conf", "layer.conf")
if not os.path.isfile(config):
with open(config, "w") as conf:
conf.write(
'BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n'
)
# create recipe
recipe = os.path.join(layerpath, "recipes",
"%s.bb" % target.target)
with open(recipe, "w") as recipef:
recipef.write("require %s\n" %
customrecipe.base_recipe.recipe.file_path)
packages = [pkg.name for pkg in customrecipe.packages.all()]
if packages:
recipef.write('IMAGE_INSTALL = "%s"\n' %
' '.join(packages))
# create *Layer* objects needed for build machinery to work
layer = Layer.objects.get_or_create(
name="Toaster Custom layer",
summary="Layer for custom recipes",
vcs_url="file://%s" % layerpath)[0]
breq = target.req
lver = Layer_Version.objects.get_or_create(project=breq.project,
layer=layer,
dirpath=layerpath,
build=breq.build)[0]
ProjectLayer.objects.get_or_create(project=breq.project,
layercommit=lver,
optional=False)
BRLayer.objects.get_or_create(req=breq,
name=layer.name,
dirpath=layerpath,
giturl="file://%s" % layerpath)
if os.path.isdir(layerpath):
layerlist.append(layerpath)
BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
self.islayerset = True
return True
def startBBServer(self):
assert self.pokydirname and os.path.exists(self.pokydirname)
assert self.islayerset
# find our own toasterui listener/bitbake
from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path
own_bitbake = _reduce_canon_path(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
"../../../bin/bitbake"))
assert os.path.exists(own_bitbake) and os.path.isfile(own_bitbake)
logger.debug("localhostbecontroller: running the listener at %s" %
own_bitbake)
cmd = "bash -c \"source %s/oe-init-build-env %s && bitbake --read conf/toaster-pre.conf --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 && DATABASE_URL=%s BBSERVER=0.0.0.0:-1 daemon -d -i -D %s -o toaster_ui.log -- %s --observe-only -u toasterui &\"" % (
self.pokydirname, self.be.builddir, self.dburl, self.be.builddir,
own_bitbake)
port = "-1"
cmdoutput = self._shellcmd(cmd)
for i in cmdoutput.split("\n"):
if i.startswith("Bitbake server address"):
port = i.split(" ")[-1]
logger.debug(
"localhostbecontroller: Found bitbake server port %s" %
port)
def _toaster_ui_started(filepath):
if not os.path.exists(filepath):
return False
with open(filepath, "r") as f:
for line in f:
if line.startswith("NOTE: ToasterUI waiting for events"):
return True
return False
retries = 0
started = False
while not started and retries < 10:
started = _toaster_ui_started(
os.path.join(self.be.builddir, "toaster_ui.log"))
import time
logger.debug(
"localhostbecontroller: Waiting bitbake server to start")
time.sleep(0.5)
retries += 1
if not started:
raise BuildSetupException(
"localhostbecontroller: Bitbake server did not start in 5 seconds, aborting (Error: '%s')"
% (cmdoutput))
logger.debug("localhostbecontroller: Started bitbake server")
while port == "-1":
# the port specification is "autodetect"; read the bitbake.lock file
with open("%s/bitbake.lock" % self.be.builddir, "r") as f:
for line in f.readlines():
if ":" in line:
port = line.split(":")[1].strip()
logger.debug(
"localhostbecontroller: Autodetected bitbake port %s",
port)
break
assert self.be.sourcedir and os.path.exists(self.be.builddir)
self.be.bbaddress = "localhost"
self.be.bbport = port
self.be.bbstate = BuildEnvironment.SERVER_STARTED
self.be.save()