def parseBaseConfiguration(self):
data_hash = hashlib.sha256()
try:
self.data = self.parseConfigurationFiles(self.prefiles,
self.postfiles)
if self.data.getVar("BB_WORKERCONTEXT", False) is None:
bb.fetch.fetcher_init(self.data)
bb.parse.init_parser(self.data)
bb.codeparser.parser_cache_init(self.data)
bb.event.fire(bb.event.ConfigParsed(), self.data)
reparse_cnt = 0
while self.data.getVar("BB_INVALIDCONF", False) is True:
if reparse_cnt > 20:
logger.error(
"Configuration has been re-parsed over 20 times, "
"breaking out of the loop...")
raise Exception(
"Too deep config re-parse loop. Check locations where "
"BB_INVALIDCONF is being set (ConfigParsed event handlers)"
)
self.data.setVar("BB_INVALIDCONF", False)
self.data = self.parseConfigurationFiles(
self.prefiles, self.postfiles)
reparse_cnt += 1
bb.event.fire(bb.event.ConfigParsed(), self.data)
bb.parse.init_parser(self.data)
data_hash.update(self.data.get_hash().encode('utf-8'))
self.mcdata[''] = self.data
multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
for config in multiconfig:
mcdata = self.parseConfigurationFiles(self.prefiles,
self.postfiles, config)
bb.event.fire(bb.event.ConfigParsed(), mcdata)
self.mcdata[config] = mcdata
data_hash.update(mcdata.get_hash().encode('utf-8'))
if multiconfig:
bb.event.fire(bb.event.MultiConfigParsed(self.mcdata),
self.data)
self.data_hash = data_hash.hexdigest()
except (SyntaxError, bb.BBHandledException):
raise bb.BBHandledException()
except bb.data_smart.ExpansionError as e:
logger.error(str(e))
raise bb.BBHandledException()
except Exception:
logger.exception("Error parsing configuration files")
raise bb.BBHandledException()
# Create a copy so we can reset at a later date when UIs disconnect
self.origdata = self.data
self.data = bb.data.createCopy(self.origdata)
self.mcdata[''] = self.data
def better_exec(code, context, text=None, realfile="<code>"):
"""
Similiar to better_compile, better_exec will
print the lines that are responsible for the
error.
"""
import bb.parse
if not text:
text = code
if not hasattr(code, "co_filename"):
code = better_compile(code, realfile, realfile)
try:
exec(code, get_context(), context)
except (bb.BBHandledException, bb.parse.SkipRecipe, bb.build.FuncFailed,
bb.data_smart.ExpansionError):
# Error already shown so passthrough, no need for traceback
raise
except Exception as e:
(t, value, tb) = sys.exc_info()
try:
_print_exception(t, value, tb, realfile, text, context)
except Exception as e:
logger.error("Exception handler error: %s" % str(e))
e = bb.BBHandledException(e)
raise e
def better_compile(text, file, realfile, mode="exec", lineno=0):
"""
A better compile method. This method
will print the offending lines.
"""
try:
cache = bb.methodpool.compile_cache(text)
if cache:
return cache
# We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though
text2 = "\n" * int(lineno) + text
code = compile(text2, realfile, mode)
bb.methodpool.compile_cache_add(text, code)
return code
except Exception as e:
error = []
# split the text into lines again
body = text.split('\n')
error.append("Error in compiling python function in %s, line %s:\n" %
(realfile, lineno))
if hasattr(e, "lineno"):
error.append("The code lines resulting in this error were:")
error.extend(_print_trace(body, e.lineno))
else:
error.append("The function causing this error was:")
for line in body:
error.append(line)
error.append("%s: %s" % (e.__class__.__name__, str(e)))
logger.error("\n".join(error))
e = bb.BBHandledException(e)
raise e
def better_compile(text, file, realfile, mode="exec"):
"""
A better compile method. This method
will print the offending lines.
"""
try:
return compile(text, file, mode)
except Exception as e:
error = []
# split the text into lines again
body = text.split('\n')
error.append("Error in compiling python function in %s:\n" % realfile)
if e.lineno:
error.append("The code lines resulting in this error were:")
error.extend(_print_trace(body, e.lineno))
else:
error.append("The function causing this error was:")
for line in body:
error.append(line)
error.append("%s: %s" % (e.__class__.__name__, str(e)))
logger.error("\n".join(error))
e = bb.BBHandledException(e)
raise e
def better_exec(code, context, text=None, realfile="<code>"):
"""
Similiar to better_compile, better_exec will
print the lines that are responsible for the
error.
"""
import bb.parse
if not text:
text = code
if not hasattr(code, "co_filename"):
code = better_compile(code, realfile, realfile)
try:
exec(code, get_context(), context)
except Exception as e:
(t, value, tb) = sys.exc_info()
if t in [bb.parse.SkipPackage, bb.build.FuncFailed]:
raise
try:
_print_exception(t, value, tb, realfile, text, context)
except Exception as e:
logger.error("Exception handler error: %s" % str(e))
e = bb.BBHandledException(e)
raise e
def finalize(fn, d, variant = None):
saved_handlers = bb.event.get_handlers().copy()
try:
# Found renamed variables. Exit immediately
if d.getVar("_FAILPARSINGERRORHANDLED", False) == True:
raise bb.BBHandledException()
for var in d.getVar('__BBHANDLERS', False) or []:
# try to add the handler
handlerfn = d.getVarFlag(var, "filename", False)
if not handlerfn:
bb.fatal("Undefined event handler function '%s'" % var)
handlerln = int(d.getVarFlag(var, "lineno", False))
bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln, data=d)
bb.event.fire(bb.event.RecipePreFinalise(fn), d)
bb.data.expandKeys(d)
bb.event.fire(bb.event.RecipePostKeyExpansion(fn), d)
runAnonFuncs(d)
tasklist = d.getVar('__BBTASKS', False) or []
bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d)
bb.build.add_tasks(tasklist, d)
bb.parse.siggen.finalise(fn, d, variant)
d.setVar('BBINCLUDED', bb.parse.get_file_depends(d))
bb.event.fire(bb.event.RecipeParsed(fn), d)
finally:
bb.event.set_handlers(saved_handlers)
def runCommand(self, command):
self.connection.send(command)
if not self.recv.poll(30):
logger.info("No reply from server in 30s")
if not self.recv.poll(30):
raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s)")
ret, exc = self.recv.get()
# Should probably turn all exceptions in exc back into exceptions?
# For now, at least handle BBHandledException
if exc and ("BBHandledException" in exc or "SystemExit" in exc):
raise bb.BBHandledException()
return ret, exc
def wrapped(fn, *args):
try:
return func(fn, *args)
except IOError as exc:
import traceback
parselog.critical(traceback.format_exc())
parselog.critical("Unable to parse %s: %s" % (fn, exc))
raise bb.BBHandledException()
except bb.data_smart.ExpansionError as exc:
import traceback
bbdir = os.path.dirname(__file__) + os.sep
exc_class, exc, tb = sys.exc_info()
for tb in iter(lambda: tb.tb_next, None):
# Skip frames in bitbake itself, we only want the metadata
fn, _, _, _ = traceback.extract_tb(tb, 1)[0]
if not fn.startswith(bbdir):
break
parselog.critical("Unable to parse %s" % fn,
exc_info=(exc_class, exc, tb))
raise bb.BBHandledException()
except bb.parse.ParseError as exc:
parselog.critical(str(exc))
raise bb.BBHandledException()
def better_compile(text, file, realfile, mode="exec"):
"""
A better compile method. This method
will print the offending lines.
"""
try:
return compile(text, file, mode)
except Exception as e:
# split the text into lines again
body = text.split('\n')
logger.error("Error in compiling python function in %s", realfile)
logger.error(str(e))
if e.lineno:
logger.error("The lines leading to this error were:")
logger.error("\t%d:%s:'%s'", e.lineno, e.__class__.__name__,
body[e.lineno - 1])
_print_trace(body, e.lineno)
else:
logger.error("The function causing this error was:")
for line in body:
logger.error(line)
e = bb.BBHandledException(e)
raise e
def parseConfigurationFiles(self, prefiles, postfiles, mc="default"):
data = bb.data.createCopy(self.basedata)
data.setVar("BB_CURRENT_MC", mc)
# Parse files for loading *before* bitbake.conf and any includes
for f in prefiles:
data = parse_config_file(f, data)
layerconf = self._findLayerConf(data)
if layerconf:
parselog.debug(2, "Found bblayers.conf (%s)", layerconf)
# By definition bblayers.conf is in conf/ of TOPDIR.
# We may have been called with cwd somewhere else so reset TOPDIR
data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf)))
data = parse_config_file(layerconf, data)
layers = (data.getVar('BBLAYERS') or "").split()
broken_layers = []
data = bb.data.createCopy(data)
approved = bb.utils.approved_variables()
# Check whether present layer directories exist
for layer in layers:
if not os.path.isdir(layer):
broken_layers.append(layer)
if broken_layers:
parselog.critical(
"The following layer directories do not exist:")
for layer in broken_layers:
parselog.critical(" %s", layer)
parselog.critical("Please check BBLAYERS in %s" % (layerconf))
raise bb.BBHandledException()
for layer in layers:
parselog.debug(2, "Adding layer %s", layer)
if 'HOME' in approved and '~' in layer:
layer = os.path.expanduser(layer)
if layer.endswith('/'):
layer = layer.rstrip('/')
data.setVar('LAYERDIR', layer)
data.setVar('LAYERDIR_RE', re.escape(layer))
data = parse_config_file(
os.path.join(layer, "conf", "layer.conf"), data)
data.expandVarref('LAYERDIR')
data.expandVarref('LAYERDIR_RE')
data.delVar('LAYERDIR_RE')
data.delVar('LAYERDIR')
bbfiles_dynamic = (data.getVar('BBFILES_DYNAMIC') or "").split()
collections = (data.getVar('BBFILE_COLLECTIONS') or "").split()
invalid = []
for entry in bbfiles_dynamic:
parts = entry.split(":", 1)
if len(parts) != 2:
invalid.append(entry)
continue
l, f = parts
invert = l[0] == "!"
if invert:
l = l[1:]
if (l in collections and not invert) or (l not in collections
and invert):
data.appendVar("BBFILES", " " + f)
if invalid:
bb.fatal(
"BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:\n %s"
% "\n ".join(invalid))
layerseries = set((data.getVar("LAYERSERIES_CORENAMES")
or "").split())
collections_tmp = collections[:]
for c in collections:
collections_tmp.remove(c)
if c in collections_tmp:
bb.fatal(
"Found duplicated BBFILE_COLLECTIONS '%s', check bblayers.conf or layer.conf to fix it."
% c)
compat = set((data.getVar("LAYERSERIES_COMPAT_%s" % c)
or "").split())
if compat and not (compat & layerseries):
bb.fatal(
"Layer %s is not compatible with the core layer which only supports these series: %s (layer is compatible with %s)"
% (c, " ".join(layerseries), " ".join(compat)))
elif not compat and not data.getVar("BB_WORKERCONTEXT"):
bb.warn(
"Layer %s should set LAYERSERIES_COMPAT_%s in its conf/layer.conf file to list the core layer names it is compatible with."
% (c, c))
if not data.getVar("BBPATH"):
msg = "The BBPATH variable is not set"
if not layerconf:
msg += (
" and bitbake did not find a conf/bblayers.conf file in"
" the expected location.\nMaybe you accidentally"
" invoked bitbake from the wrong directory?")
raise SystemExit(msg)
data = parse_config_file(os.path.join("conf", "bitbake.conf"), data)
# Parse files for loading *after* bitbake.conf and any includes
for p in postfiles:
data = parse_config_file(p, data)
# Handle any INHERITs and inherit the base class
bbclasses = ["base"] + (data.getVar('INHERIT') or "").split()
for bbclass in bbclasses:
data = _inherit(bbclass, data)
# Nomally we only register event handlers at the end of parsing .bb files
# We register any handlers we've found so far here...
for var in data.getVar('__BBHANDLERS', False) or []:
handlerfn = data.getVarFlag(var, "filename", False)
if not handlerfn:
parselog.critical("Undefined event handler function '%s'" %
var)
raise bb.BBHandledException()
handlerln = int(data.getVarFlag(var, "lineno", False))
bb.event.register(var, data.getVar(var, False),
(data.getVarFlag(var, "eventmask")
or "").split(), handlerfn, handlerln)
data.setVar('BBINCLUDED', bb.parse.get_file_depends(data))
return data
def better_exec(code, context, text=None, realfile="<code>"):
"""
Similiar to better_compile, better_exec will
print the lines that are responsible for the
error.
"""
import bb.parse
if not text:
text = code
if not hasattr(code, "co_filename"):
code = better_compile(code, realfile, realfile)
try:
exec(code, _context, context)
except Exception as e:
(t, value, tb) = sys.exc_info()
if t in [bb.parse.SkipPackage, bb.build.FuncFailed]:
raise
import traceback
exception = traceback.format_exception_only(t, value)
logger.error('Error executing a python function in %s:\n%s', realfile,
''.join(exception))
# Strip 'us' from the stack (better_exec call)
tb = tb.tb_next
textarray = text.split('\n')
linefailed = traceback.tb_lineno(tb)
tbextract = traceback.extract_tb(tb)
tbformat = "\n".join(traceback.format_list(tbextract))
logger.error(
"The stack trace of python calls that resulted in this exception/failure was:"
)
for line in tbformat.split('\n'):
logger.error(line)
logger.error("The code that was being executed was:")
_print_trace(textarray, linefailed)
logger.error("[From file: '%s', lineno: %s, function: %s]",
tbextract[0][0], tbextract[0][1], tbextract[0][2])
# See if this is a function we constructed and has calls back into other functions in
# "text". If so, try and improve the context of the error by diving down the trace
level = 0
nexttb = tb.tb_next
while nexttb is not None and (level + 1) < len(tbextract):
if tbextract[level][0] == tbextract[level + 1][0] and tbextract[
level + 1][2] == tbextract[level][0]:
_print_trace(textarray, tbextract[level + 1][1])
logger.error("[From file: '%s', lineno: %s, function: %s]",
tbextract[level + 1][0], tbextract[level + 1][1],
tbextract[level + 1][2])
elif "d" in context and tbextract[level + 1][2]:
d = context["d"]
functionname = tbextract[level + 1][2]
text = d.getVar(functionname, True)
if text:
_print_trace(text.split('\n'), tbextract[level + 1][1])
logger.error("[From file: '%s', lineno: %s, function: %s]",
tbextract[level + 1][0],
tbextract[level + 1][1],
tbextract[level + 1][2])
else:
break
else:
break
nexttb = tb.tb_next
level = level + 1
e = bb.BBHandledException(e)
raise e
def dummyfatal(msg):
print("ERROR: %s" % msg)
raise bb.BBHandledException()
def parseBaseConfiguration(self, worker=False):
data_hash = hashlib.sha256()
try:
self.data = self.parseConfigurationFiles(self.prefiles,
self.postfiles)
if self.data.getVar("BB_WORKERCONTEXT",
False) is None and not worker:
bb.fetch.fetcher_init(self.data)
bb.parse.init_parser(self.data)
bb.codeparser.parser_cache_init(self.data)
bb.event.fire(bb.event.ConfigParsed(), self.data)
reparse_cnt = 0
while self.data.getVar("BB_INVALIDCONF", False) is True:
if reparse_cnt > 20:
logger.error(
"Configuration has been re-parsed over 20 times, "
"breaking out of the loop...")
raise Exception(
"Too deep config re-parse loop. Check locations where "
"BB_INVALIDCONF is being set (ConfigParsed event handlers)"
)
self.data.setVar("BB_INVALIDCONF", False)
self.data = self.parseConfigurationFiles(
self.prefiles, self.postfiles)
reparse_cnt += 1
bb.event.fire(bb.event.ConfigParsed(), self.data)
bb.parse.init_parser(self.data)
data_hash.update(self.data.get_hash().encode('utf-8'))
self.mcdata[''] = self.data
multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
for config in multiconfig:
if config[0].isdigit():
bb.fatal(
"Multiconfig name '%s' is invalid as multiconfigs cannot start with a digit"
% config)
mcdata = self.parseConfigurationFiles(self.prefiles,
self.postfiles, config)
bb.event.fire(bb.event.ConfigParsed(), mcdata)
self.mcdata[config] = mcdata
data_hash.update(mcdata.get_hash().encode('utf-8'))
if multiconfig:
bb.event.fire(bb.event.MultiConfigParsed(self.mcdata),
self.data)
self.data_hash = data_hash.hexdigest()
except (SyntaxError, bb.BBHandledException):
raise bb.BBHandledException()
except bb.data_smart.ExpansionError as e:
logger.error(str(e))
raise bb.BBHandledException()
except Exception:
logger.exception("Error parsing configuration files")
raise bb.BBHandledException()
# Handle obsolete variable names
d = self.data
renamedvars = d.getVarFlags('BB_RENAMED_VARIABLES') or {}
renamedvars.update(bb.data_smart.bitbake_renamed_vars)
issues = False
for v in renamedvars:
if d.getVar(v) != None or d.hasOverrides(v):
issues = True
loginfo = {}
history = d.varhistory.get_variable_refs(v)
for h in history:
for line in history[h]:
loginfo = {'file': h, 'line': line}
bb.data.data_smart._print_rename_error(
v, loginfo, renamedvars)
if not history:
bb.data.data_smart._print_rename_error(
v, loginfo, renamedvars)
if issues:
raise bb.BBHandledException()
# Create a copy so we can reset at a later date when UIs disconnect
self.origdata = self.data
self.data = bb.data.createCopy(self.origdata)
self.mcdata[''] = self.data
