def propagate_runq_task_build(self):
"""always build all tasks depending on other tasks to build"""
rowcount = 0
while True:
self.dbc.execute(
"UPDATE"
" runq.task "
"SET"
" build=1 "
"WHERE"
" build IS NULL"
" AND EXISTS"
" (SELECT *"
" FROM runq.depend, runq.task AS parent_task"
" WHERE runq.depend.task=runq.task.task"
" AND runq.depend.parent_task=parent_task.task"
" AND parent_task.build=1"
" LIMIT 1)")
if rowcount == -1:
die("propagate_runq_task_build did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
debug("set build flag on %d tasks due to propagation" % (rowcount))
return
def prepare(self, runq):
meta = self.meta()
buildhash = self.cookbook.baker.runq.get_task_buildhash(self)
debug("buildhash=%s" % (repr(buildhash)))
meta.set("TASK_BUILDHASH", buildhash)
def prepare_stage(deptype):
stage = {}
recdepends = []
get_package_filename = self.cookbook.baker.runq.get_package_filename
packages = self.cookbook.baker.runq.get_depend_packages(
self, deptype) or []
for package in packages:
package = self.cookbook.get_package(id=package)
filename = get_package_filename(package)
if not filename in stage:
stage[filename] = package
return stage
meta["__stage"] = prepare_stage("DEPENDS")
meta["__rstage"] = prepare_stage("RDEPENDS")
meta["__fstage"] = prepare_stage("FDEPENDS")
meta.set_flag("__stage", "nohash", True)
meta.set_flag("__rstage", "nohash", True)
meta.set_flag("__fstage", "nohash", True)
return
def prepare(self, runq):
meta = self.meta()
buildhash = self.cookbook.baker.runq.get_task_buildhash(self)
debug("buildhash=%s"%(repr(buildhash)))
meta.set("TASK_BUILDHASH", buildhash)
def prepare_stage(deptype):
stage = {}
recdepends = []
get_package_filename = self.cookbook.baker.runq.get_package_filename
packages = self.cookbook.baker.runq.get_depend_packages(
self, deptype) or []
for package in packages:
package = self.cookbook.get_package(id=package)
filename = get_package_filename(package)
if not filename in stage:
stage[filename] = package
return stage
meta["__stage"] = prepare_stage("DEPENDS")
meta["__rstage"] = prepare_stage("RDEPENDS")
meta["__fstage"] = prepare_stage("FDEPENDS")
meta.set_flag("__stage", "nohash", True)
meta.set_flag("__rstage", "nohash", True)
meta.set_flag("__fstage", "nohash", True)
return
def propagate_runq_task_build(self):
"""always build all tasks depending on other tasks to build"""
rowcount = 0
while True:
self.dbc.execute(
"UPDATE"
" runq.task "
"SET"
" build=1 "
"WHERE"
" build IS NULL"
" AND EXISTS"
" (SELECT *"
" FROM runq.depend, runq.task AS parent_task"
" WHERE runq.depend.task=runq.task.task"
" AND runq.depend.parent_task=parent_task.task"
" AND parent_task.build=1"
" LIMIT 1)")
if rowcount == -1:
die("propagate_runq_task_build did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
debug("set build flag on %d tasks due to propagation"%(rowcount))
return
def prepare(self):
meta = self.meta()
self.weight = self.get_weight(meta)
buildhash = self.cookbook.baker.runq.get_task_buildhash(self)
debug("buildhash=%s"%(repr(buildhash)))
meta.set("TASK_BUILDHASH", buildhash)
@oelite.profiling.profile_calls
def prepare_stage(deptype):
stage = {}
recdepends = []
get_package_filename = self.cookbook.baker.runq.get_package_filename
packages = self.cookbook.baker.runq.get_depend_packages(
self, deptype) or []
for package in packages:
package = self.cookbook.get_package(id=package)
filename = get_package_filename(package)
if not filename in stage:
stage[filename] = package
return stage
meta["__stage"] = prepare_stage("DEPENDS")
meta["__rstage"] = prepare_stage("RDEPENDS")
meta["__fstage"] = prepare_stage("FDEPENDS")
def prepare(self):
meta = self.meta()
self.weight = self.get_weight(meta)
buildhash = self.cookbook.baker.runq.get_task_buildhash(self)
debug("buildhash=%s" % (repr(buildhash)))
meta.set("TASK_BUILDHASH", buildhash)
@oelite.profiling.profile_calls
def prepare_stage(deptype):
stage = {}
recdepends = []
get_package_filename = self.cookbook.baker.runq.get_package_filename
packages = self.cookbook.baker.runq.get_depend_packages(
self, deptype) or []
for package in packages:
package = self.cookbook.get_package(id=package)
filename = get_package_filename(package)
if not filename in stage:
stage[filename] = package
return stage
meta["__stage"] = prepare_stage("DEPENDS")
meta["__rstage"] = prepare_stage("RDEPENDS")
meta["__fstage"] = prepare_stage("FDEPENDS")
def add_task_depends(task_names, recipes):
for task_name in task_names:
for recipe in recipes:
task = self.cookbook.get_task(recipe, task_name)
if task:
task_depends.add(task)
else:
debug("not adding unsupported task %s:%s" % (recipe, task_name))
def add_task_depends(task_names, recipes):
for task_name in task_names:
for recipe in recipes:
task = self.cookbook.get_task(recipe, task_name)
if task:
task_depends.add(task)
else:
debug("not adding unsupported task %s:%s" %
(recipe, task_name))
def set_task_build_on_nostamp_tasks(self):
rowcount = self.dbc.execute(
"UPDATE runq.task SET build=1 "
"WHERE build IS NULL AND EXISTS "
"(SELECT * FROM task"
" WHERE id=runq.task.task AND nostamp=1)").rowcount
if rowcount == -1:
die("set_task_build_on_nostamp_tasks did not work out")
debug("set build flag on %d nostamp tasks"%(rowcount))
return
def start(self, task):
self.count += 1
debug("")
debug("Preparing %s"%(task))
task.prepare()
info("%s started - %d / %d "%(task, self.count, self.total))
task.build_started()
self.add(task)
task.start()
def start(self, task):
self.count += 1
debug("")
debug("Preparing %s" % (task))
task.prepare()
info("%s started - %d / %d " % (task, self.count, self.total))
task.build_started()
self.add(task)
task.start()
def set_task_build_on_nostamp_tasks(self):
rowcount = self.dbc.execute(
"UPDATE runq.task SET build=1 "
"WHERE build IS NULL AND EXISTS "
"(SELECT * FROM task"
" WHERE id=runq.task.task AND nostamp=1)").rowcount
if rowcount == -1:
die("set_task_build_on_nostamp_tasks did not work out")
debug("set build flag on %d nostamp tasks" % (rowcount))
return
def arch_is_compatible(meta, arch_type):
compatible_archs = meta.get("COMPATIBLE_%s_ARCHS"%arch_type)
if compatible_archs is None:
return True
arch = meta.get(arch_type + "_ARCH")
for compatible_arch in compatible_archs.split():
if re.match(compatible_arch, arch):
return True
debug("skipping %s_ARCH incompatible recipe %s:%s"%(
arch_type, recipe_type, meta.get("PN")))
return False
def arch_is_compatible(meta, arch_type):
compatible_archs = meta.get("COMPATIBLE_%s_ARCHS"%arch_type)
if compatible_archs is None:
return True
arch = meta.get(arch_type + "_ARCH")
for compatible_arch in compatible_archs.split():
if re.match(compatible_arch, arch):
return True
debug("skipping %s_ARCH incompatible recipe %s:%s"%(
arch_type, recipe_type, meta.get("PN")))
return False
def compatible_use_flags(meta):
flags = meta.get("COMPATIBLE_IF_FLAGS")
if not flags:
return True
for name in flags.split():
val = meta.get("USE_"+name)
if not val:
debug("skipping %s:%s_%s (required %s USE flag not set)"%(
recipe_type, meta.get("PN"), meta.get("PV"),
name))
return False
return True
def compatible_use_flags(meta):
flags = meta.get("COMPATIBLE_IF_FLAGS")
if not flags:
return True
for name in flags.split():
val = meta.get("USE_"+name)
if not val or val == "0":
debug("skipping %s:%s_%s (required %s USE flag not set)"%(
recipe_type, meta.get("PN"), meta.get("PV"),
name))
return False
return True
def bake(self):
self.setup_tmpdir()
oelite.profiling.init(self.config)
# task(s) to do
if self.options.task:
tasks_todo = self.options.task
elif "OE_DEFAULT_TASK" in self.config:
tasks_todo = self.config.get("OE_DEFAULT_TASK", 1)
else:
#tasks_todo = "all"
tasks_todo = "build"
self.tasks_todo = tasks_todo.split(",")
if self.options.rebuild:
self.options.rebuild = max(self.options.rebuild)
self.options.prebake = False
else:
self.options.rebuild = None
if self.options.relax:
self.options.relax = max(self.options.relax)
else:
default_relax = self.config.get("DEFAULT_RELAX", 1)
if default_relax and default_relax != "0":
self.options.relax = int(default_relax)
else:
self.options.relax = None
# init build quue
self.runq = OEliteRunQueue(self.config, self.cookbook,
self.options.rebuild, self.options.relax)
# first, add complete dependency tree, with complete
# task-to-task and task-to-package/task dependency information
debug("Building dependency tree")
rusage = oelite.profiling.Rusage("Building dependency tree")
for task in self.tasks_todo:
task = oelite.task.task_name(task)
try:
for thing in self.recipes_todo:
if not self.runq.add_recipe(thing, task):
die("No such recipe: %s" % (thing))
for thing in self.things_todo:
thing = oelite.item.OEliteItem(thing)
if not self.runq.add_something(thing, task):
die("No such thing: %s" % (thing))
except RecursiveDepends, e:
die("dependency loop: %s\n\t--> %s" %
(e.args[1], "\n\t--> ".join(e.args[0])))
except NoSuchTask, e:
die("No such task: %s: %s" % (thing, e.__str__()))
def set_task_build_on_hashdiff(self):
rowcount = 0
while True:
self.dbc.execute(
"UPDATE runq.task SET build=1 "
"WHERE build IS NULL AND relax IS NULL AND tmphash != metahash")
if rowcount == -1:
die("set_task_build_on_hashdiff did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
debug("set build flag on %d tasks with tmphash != metahash"%(rowcount))
return
def bake(self):
self.setup_tmpdir()
# task(s) to do
if self.options.task:
tasks_todo = self.options.task
elif "OE_DEFAULT_TASK" in self.config:
tasks_todo = self.config.get("OE_DEFAULT_TASK", 1)
else:
#tasks_todo = "all"
tasks_todo = "build"
self.tasks_todo = tasks_todo.split(",")
if self.options.rebuild:
self.options.rebuild = max(self.options.rebuild)
self.options.prebake = False
else:
self.options.rebuild = None
if self.options.relax:
self.options.relax = max(self.options.relax)
else:
default_relax = self.config.get("DEFAULT_RELAX", 1)
if default_relax and default_relax != "0":
self.options.relax = int(default_relax)
else:
self.options.relax = None
# init build quue
self.runq = OEliteRunQueue(self.config, self.cookbook,
self.options.rebuild, self.options.relax)
# first, add complete dependency tree, with complete
# task-to-task and task-to-package/task dependency information
debug("Building dependency tree")
start = datetime.datetime.now()
for task in self.tasks_todo:
task = oelite.task.task_name(task)
try:
for thing in self.recipes_todo:
if not self.runq.add_recipe(thing, task):
die("No such recipe: %s"%(thing))
for thing in self.things_todo:
thing = oelite.item.OEliteItem(thing)
if not self.runq.add_something(thing, task):
die("No such thing: %s"%(thing))
except RecursiveDepends, e:
die("dependency loop: %s\n\t--> %s"%(
e.args[1], "\n\t--> ".join(e.args[0])))
except NoSuchTask, e:
die("No such task: %s: %s"%(thing, e.__str__()))
def __init__(self, baker):
self.baker = baker
self.config = baker.config
self.oeparser = baker.oeparser
self.init_layer_meta()
self.db = sqlite.connect(":memory:", isolation_level=None)
if not self.db:
raise Exception("could not create in-memory sqlite db")
self.db.text_factory = str
self.dbc = CursorWrapper(self.db.cursor(), profile=False)
self.init_db()
self.recipes = {}
self.packages = {}
self.tasks = {}
self.cachedir = self.config.get("CACHEDIR") or ""
self.debug = self.baker.debug
fail = False
recipefiles = self.list_recipefiles()
total = len(recipefiles)
count = 0
rusage = oelite.profiling.Rusage("recipe parsing")
for recipefile in recipefiles:
count += 1
if self.debug:
debug("Adding %s to cookbook [%s/%s]"%(
self.shortfilename(recipefile), count, total))
else:
oelite.util.progress_info("Adding recipes to cookbook",
total, count)
try:
if not self.add_recipefile(recipefile):
fail = True
except KeyboardInterrupt:
if os.isatty(sys.stdout.fileno()) and not self.debug:
print
die("Aborted while building cookbook")
except oelite.parse.ParseError, e:
if os.isatty(sys.stdout.fileno()) and not self.debug:
print
e.print_details()
err("Parse error in %s"%(self.shortfilename(recipefile)))
fail = True
except Exception, e:
import traceback
if os.isatty(sys.stdout.fileno()) and not self.debug:
print
traceback.print_exc()
err("Uncaught Python exception in %s"%(
self.shortfilename(recipefile)))
fail = True
def cpu_families_is_compatible(meta, arch_type):
compatible_cpu_fams = meta.get("COMPATIBLE_%s_CPU_FAMILIES"%arch_type)
if compatible_cpu_fams is None:
return True
cpu_fams = meta.get(arch_type + "_CPU_FAMILIES")
if not cpu_fams:
return False
for compatible_cpu_fam in compatible_cpu_fams.split():
for cpu_fam in cpu_fams.split():
if re.match(compatible_cpu_fam, cpu_fam):
return True
debug("skipping %s_CPU_FAMILIES incompatible recipe %s:%s"%(
arch_type, recipe_type, meta.get("PN")))
return False
def cpu_families_is_compatible(meta, arch_type):
compatible_cpu_fams = meta.get("COMPATIBLE_%s_CPU_FAMILIES"%arch_type)
if compatible_cpu_fams is None:
return True
cpu_fams = meta.get(arch_type + "_CPU_FAMILIES")
if not cpu_fams:
return False
for compatible_cpu_fam in compatible_cpu_fams.split():
for cpu_fam in cpu_fams.split():
if re.match(compatible_cpu_fam, cpu_fam):
return True
debug("skipping %s_CPU_FAMILIES incompatible recipe %s:%s"%(
arch_type, recipe_type, meta.get("PN")))
return False
def resolve_dependency(item, recursion_path, deptype):
if not rec_deptype:
if self.assume_provided(item):
debug("ASSUME_PROVIDED %s" % (item))
return set([])
try:
(recipe, package) = self.get_recipe_provider(item)
except NoProvider, e:
if ignore_missing:
return set([])
if len(e.args) < 2 and len(recursion_path[0]):
raise NoProvider(e.args[0], recursion_path[0][-1])
raise
return set([package])
def resolve_dependency(item, recursion_path, deptype):
if not rec_deptype:
if self.assume_provided(item):
debug("ASSUME_PROVIDED %s"%(item))
return set([])
try:
(recipe, package) = self.get_recipe_provider(item)
except NoProvider, e:
if ignore_missing:
return set([])
if len(e.args) < 2 and len(recursion_path[0]):
raise NoProvider(e.args[0], recursion_path[0][-1])
raise
return set([package])
def machine_is_compatible(meta):
compatible_machines = meta.get("COMPATIBLE_MACHINES")
if compatible_machines is None:
return True
machine = meta.get("MACHINE")
if machine is None:
debug("skipping MACHINE incompatible recipe %s:%s"%(
recipe_type, meta.get("PN")))
return False
for compatible_machine in compatible_machines.split():
if re.match(compatible_machine, machine):
return True
debug("skipping MACHINE incompatible recipe %s:%s"%(
recipe_type, meta.get("PN")))
return False
def set_task_build_on_hashdiff(self):
rowcount = 0
while True:
self.dbc.execute(
"UPDATE runq.task SET build=1 "
"WHERE build IS NULL AND relax IS NULL AND tmphash != metahash"
)
if rowcount == -1:
die("set_task_build_on_hashdiff did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
debug("set build flag on %d tasks with tmphash != metahash" %
(rowcount))
return
def machine_is_compatible(meta):
compatible_machines = meta.get("COMPATIBLE_MACHINES")
if compatible_machines is None:
return True
machine = meta.get("MACHINE")
if machine is None:
debug("skipping MACHINE incompatible recipe %s:%s"%(
recipe_type, meta.get("PN")))
return False
for compatible_machine in compatible_machines.split():
if re.match(compatible_machine, machine):
return True
debug("skipping MACHINE incompatible recipe %s:%s"%(
recipe_type, meta.get("PN")))
return False
def set_task_build_on_retired_tasks(self):
rowcount = 0
while True:
self.dbc.execute(
"UPDATE runq.task SET build=1 "
"WHERE build IS NULL AND EXISTS "
"(SELECT * FROM runq.depend, runq.task AS parent_task"
" WHERE runq.depend.task=runq.task.task"
" AND runq.depend.parent_task=parent_task.task"
" AND parent_task.mtime > runq.task.mtime)")
if rowcount == -1:
die("set_task_build_on_retired_tasks did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
debug("set build flag on %d retired tasks"%(rowcount))
return
def set_task_build_on_retired_tasks(self):
rowcount = 0
while True:
self.dbc.execute(
"UPDATE runq.task SET build=1 "
"WHERE build IS NULL AND EXISTS "
"(SELECT * FROM runq.depend, runq.task AS parent_task"
" WHERE runq.depend.task=runq.task.task"
" AND runq.depend.parent_task=parent_task.task"
" AND parent_task.mtime > runq.task.mtime)")
if rowcount == -1:
die("set_task_build_on_retired_tasks did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
debug("set build flag on %d retired tasks" % (rowcount))
return
def prune_runq_tasks(self):
rowcount = self.dbc.execute(
"UPDATE"
" runq.task "
"SET"
" build=NULL "
"WHERE"
" prime IS NULL AND NOT EXISTS"
" (SELECT *"
" FROM runq.depend"
" WHERE runq.depend.parent_task=runq.task.task"
" LIMIT 1"
")").rowcount
if rowcount == -1:
die("prune_runq_tasks did not work out")
if rowcount:
debug("pruned %d tasks that does not need to be build" % rowcount)
return rowcount
def prune_runq_tasks(self):
rowcount = self.dbc.execute(
"UPDATE"
" runq.task "
"SET"
" build=NULL "
"WHERE"
" prime IS NULL AND NOT EXISTS"
" (SELECT *"
" FROM runq.depend"
" WHERE runq.depend.parent_task=runq.task.task"
" LIMIT 1"
")").rowcount
if rowcount == -1:
die("prune_runq_tasks did not work out")
if rowcount:
debug("pruned %d tasks that does not need to be build"%rowcount)
return rowcount
def prune_runq_depends_with_nobody_depending_on_it(self):
#c = self.dbc.cursor()
rowcount = 0
while True:
self.dbc.execute(
"DELETE FROM runq.depend "
"WHERE prime IS NULL AND NOT EXISTS "
"(SELECT * FROM runq.depend AS next_depend"
" WHERE next_depend.parent_task=runq.depend.task"
" LIMIT 1"
")")
if rowcount == -1:
die("prune_runq_depends_with_no_depending_tasks did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
if rowcount:
debug("pruned %d dependencies which where not needed anyway"%rowcount)
return rowcount
def prune_runq_depends_nobuild(self):
rowcount = 0
while True:
self.dbc.execute("UPDATE runq.depend SET parent_task=NULL "
"WHERE parent_task IS NOT NULL AND NOT EXISTS "
"(SELECT * FROM runq.task"
" WHERE runq.task.build=1"
" AND runq.task.task=runq.depend.parent_task"
" LIMIT 1"
")")
if rowcount == -1:
die("prune_runq_depends_nobuild did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
if rowcount:
debug("pruned %d dependencies that did not have to be rebuilt" %
rowcount)
return rowcount
def prune_runq_depends_nobuild(self):
rowcount = 0
while True:
self.dbc.execute(
"UPDATE runq.depend SET parent_task=NULL "
"WHERE parent_task IS NOT NULL AND NOT EXISTS "
"(SELECT * FROM runq.task"
" WHERE runq.task.build=1"
" AND runq.task.task=runq.depend.parent_task"
" LIMIT 1"
")")
if rowcount == -1:
die("prune_runq_depends_nobuild did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
if rowcount:
debug("pruned %d dependencies that did not have to be rebuilt"%rowcount)
return rowcount
def recipe_is_compatible(meta):
incompatible_recipes = meta.get("INCOMPATIBLE_RECIPES")
if incompatible_recipes is None:
return True
pn = meta.get("PN")
pv = meta.get("PV")
for incompatible_recipe in incompatible_recipes.split():
if "_" in incompatible_recipe:
incompatible_recipe = incompatible_recipe.rsplit("_", 1)
else:
incompatible_recipe = (incompatible_recipe, None)
if not re.match("%s$"%(incompatible_recipe[0]), pn):
continue
if incompatible_recipe[1] is None:
return False
if re.match("%s$"%(incompatible_recipe[1]), pv):
debug("skipping incompatible recipe %s:%s_%s"%(
recipe_type, pn, pv))
return False
return True
def recipe_is_compatible(meta):
incompatible_recipes = meta.get("INCOMPATIBLE_RECIPES")
if incompatible_recipes is None:
return True
pn = meta.get("PN")
pv = meta.get("PV")
for incompatible_recipe in incompatible_recipes.split():
if "_" in incompatible_recipe:
incompatible_recipe = incompatible_recipe.rsplit("_", 1)
else:
incompatible_recipe = (incompatible_recipe, None)
if not re.match("%s$"%(incompatible_recipe[0]), pn):
continue
if incompatible_recipe[1] is None:
return False
if re.match("%s$"%(incompatible_recipe[1]), pv):
debug("skipping incompatible recipe %s:%s_%s"%(
recipe_type, pn, pv))
return False
return True
def prune_runq_depends_with_nobody_depending_on_it(self):
#c = self.dbc.cursor()
rowcount = 0
while True:
self.dbc.execute("DELETE FROM runq.depend "
"WHERE prime IS NULL AND NOT EXISTS "
"(SELECT * FROM runq.depend AS next_depend"
" WHERE next_depend.parent_task=runq.depend.task"
" LIMIT 1"
")")
if rowcount == -1:
die("prune_runq_depends_with_no_depending_tasks did not work out"
)
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
if rowcount:
debug("pruned %d dependencies which where not needed anyway" %
rowcount)
return rowcount
def import_env(self):
whitelist = OE_ENV_WHITELIST
if "OE_ENV_WHITELIST" in os.environ:
whitelist += os.environ["OE_ENV_WHITELIST"].split()
if "OE_ENV_WHITELIST" in self:
whitelist += self.get("OE_ENV_WHITELIST", True).split()
debug("whitelist=%s" % (whitelist))
debug("Whitelist filtered shell environment:")
hasher = hashlib.md5()
for var in whitelist:
if var in os.environ:
debug("> %s=%s" % (var, os.environ[var]))
if not var in self and var in os.environ:
env_val = os.environ[var]
self[var] = env_val
hasher.update("%s=%r\n" % (var, env_val))
self['__env_signature'] = hasher.hexdigest()
def import_env(self):
whitelist = OE_ENV_WHITELIST
if "OE_ENV_WHITELIST" in os.environ:
whitelist += os.environ["OE_ENV_WHITELIST"].split()
if "OE_ENV_WHITELIST" in self:
whitelist += self.get("OE_ENV_WHITELIST", True).split()
debug("whitelist=%s"%(whitelist))
debug("Whitelist filtered shell environment:")
hasher = hashlib.md5()
for var in whitelist:
if var in os.environ:
debug("> %s=%s"%(var, os.environ[var]))
if not var in self and var in os.environ:
env_val = os.environ[var]
self[var] = env_val
hasher.update("%s=%r\n"%(var, env_val))
self['__env_signature'] = hasher.hexdigest()
def get_recipe(self, id=None, task=None, package=None,
filename=None, type=None, name=None, version=None,
strict=True, default_type="machine"):
"""
Get recipe from cookbook. Returns recipe object if arguments
match a single recipe. Returns None if recipe is not found.
Throws MultipleRecipes exception if more than one recipe is
found.
"""
recipes = self.get_recipes(
id=id, task=task, package=package,
filename=filename, type=type, name=name, version=version)
if len(recipes) == 0:
recipe = None
elif len(recipes) == 1:
recipe = recipes[0]
elif strict:
warn("multiple recipes found in %s.%s: returning None!"%(
self.__class__.__name__, inspect.stack()[0][3]))
for recipe in recipes:
info("%s:%s_%s"%(recipe.type, recipe.name, recipe.version))
recipe = None
else:
chosen = [ recipes[0] ]
for other in recipes[1:]:
if chosen[0].priority > other.priority:
continue
if chosen[0].priority < other.priority:
chosen = [ other ]
continue
vercmp = bb.utils.vercmp_part(chosen[0].version, other.version)
if vercmp < 0:
chosen = [ other ]
if vercmp == 0:
#debug("chosen=%s\nother=%s"%(chosen, other))
#die("you have to be more precise")
chosen.append(other)
if len(chosen) == 1:
recipe = chosen[0]
elif not default_type:
warn("multiple recipes found in %s.%s: returning None!"%(
self.__class__.__name__, inspect.stack()[0][3]))
for recipe in chosen:
info("%s:%s_%s"%(recipe.type, recipe.name, recipe.version))
recipe = None
else:
# there is multiple recipes with the same priority and version,
# so let's try to pick the default type
defaults_chosen = []
for choice in chosen:
if choice.type == default_type:
defaults_chosen.append(choice)
if len(defaults_chosen) == 1:
recipe = defaults_chosen[0]
elif not defaults_chosen:
debug("multiple recipes, but none with default_type (%s)"%(
default_type))
recipe = None
else:
warn("multiple recipes found in %s.%s: returning None!"%(
self.__class__.__name__, inspect.stack()[0][3]))
for recipe in defaults_chosen:
info("%s:%s_%s"%(recipe.type, recipe.name,
recipe.version))
recipe = None
return recipe
def find_prebaked_package(self, package):
"""return full-path filename string or None"""
package_deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR")
prebake_url_cache_dir = self.config.get("PREBAKE_CACHE_DIR")
if not package_deploy_dir:
die("PACKAGE_DEPLOY_DIR not defined")
if self.options.prebake:
prebake_path = self.config.get("PREBAKE_PATH") or []
if prebake_path:
prebake_path = prebake_path.split(":")
prebake_path.insert(0, package_deploy_dir)
prebake_path.insert(0, prebake_url_cache_dir)
else:
prebake_path = [package_deploy_dir]
debug("package=%s"%(repr(package)))
recipe = self.cookbook.get_recipe(package=package)
if not recipe:
raise NoSuchRecipe()
metahash = self.runq.get_package_metahash(package)
debug("got metahash=%s"%(metahash))
package = self.cookbook.get_package(id=package)
if not package:
raise NoSuchPackage()
filename = "%s_%s_%s.tar"%(package.name, recipe.version, metahash)
debug("prebake_path=%s"%(prebake_path))
#test local paths first
for base_dir in prebake_path:
path = os.path.join(
base_dir,
package.type,
package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
filename)
debug("checking for prebake: %s"%(path))
if os.path.exists(path):
debug("found prebake: %s"%(path))
return path
#then test URLs from PREBAKE_URL
url_prefix = self.config.get("PREBAKE_URL")
if url_prefix is not None:
package_path =os.path.join(
package.type,
package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
filename)
downloaded_file =os.path.join(
prebake_url_cache_dir,
package_path)
url = os.path.join(url_prefix, package_path)
if oelite.fetch.url.grab(url, downloaded_file, timeout=1, retry=1):
if os.path.exists(downloaded_file) and os.path.getsize(downloaded_file) > 0:
debug("using prebake from web: %s"%(url))
return downloaded_file
else:
os.unlink(downloaded_file)
return None
def __init__(self, options, args, config):
self.options = options
self.debug = self.options.debug
self.debug_loglines = getattr(self.options, 'debug_loglines', None)
# Bakery 3 compatibility, configure the logging module
if (not hasattr(oebakery, "__version__") or
oebakery.__version__.split(".")[0] < 4):
logging.basicConfig(format="%(message)s")
if self.debug:
logging.getLogger().setLevel(logging.DEBUG)
else:
logging.getLogger().setLevel(logging.INFO)
self.config = oelite.meta.DictMeta(meta=config)
self.config["OE_IMPORTS"] = INITIAL_OE_IMPORTS
self.config.import_env()
os.environ.clear()
self.config.pythonfunc_init()
self.topdir = self.config.get("TOPDIR", True)
self.set_manifest_origin()
# FIXME: self.config.freeze("TOPDIR")
self.confparser = confparse.ConfParser(self.config)
self.confparser.parse("conf/oe-lite.conf")
oelite.pyexec.exechooks(self.config, "post_conf_parse")
# FIXME: refactor oelite.arch.init to a post_conf_parse hook
oelite.arch.init(self.config)
# Handle any INHERITs and inherit the base class
inherits = ["core"] + (self.config.get("INHERIT", 1) or "").split()
# and inherit rmwork when needed
try:
rmwork = self.options.rmwork
if rmwork is None:
rmwork = self.config.get("RMWORK", True)
if rmwork == "0":
rmwork = False
if rmwork:
debug("rmwork")
inherits.append("rmwork")
self.options.rmwork = True
except AttributeError:
pass
self.oeparser = oeparse.OEParser(self.config)
for inherit in inherits:
self.oeparser.reset_lexstate()
self.oeparser.parse("classes/%s.oeclass"%(inherit), require=True)
oelite.pyexec.exechooks(self.config, "post_common_inherits")
self.cookbook = CookBook(self)
# things (ritem, item, recipe, or package) to do
if args:
self.things_todo = args
elif "OE_DEFAULT_THING" in self.config:
self.things_todo = self.config.get("OE_DEFAULT_THING", 1).split()
else:
self.things_todo = [ "world" ]
recipe_types = ("machine", "native", "sdk",
"cross", "sdk-cross", "canadian-cross")
def thing_todo(thing):
if not thing in self.things_todo:
self.things_todo.append(thing)
def dont_do_thing(thing):
while thing in self.things_todo:
self.things_todo.remove(thing)
self.recipes_todo = set()
if "universe" in self.things_todo:
dont_do_thing("universe")
for recipe_type in recipe_types:
thing_todo(recipe_type + ":world")
if "world" in self.things_todo:
dont_do_thing("world")
thing_todo("machine:world")
for recipe_type in recipe_types:
world = recipe_type + ":world"
if world in self.things_todo:
dont_do_thing(world)
for recipe in self.cookbook.get_recipes(type=recipe_type):
self.recipes_todo.add(recipe)
return
build_count = self.runq.set_buildhash_for_build_tasks()
nobuild_count = self.runq.set_buildhash_for_nobuild_tasks()
if (build_count + nobuild_count) != total:
die("build_count + nobuild_count != total")
deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR", True)
packages = self.runq.get_packages_to_build()
for package in packages:
package = self.cookbook.get_package(id=package)
recipe = self.cookbook.get_recipe(package=package.id)
buildhash = self.runq.get_package_buildhash(package.id)
filename = os.path.join(
deploy_dir, package.type,
package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
"%s_%s_%s.tar"%(package.name, recipe.version, buildhash))
debug("will use from build: %s"%(filename))
self.runq.set_package_filename(package.id, filename)
self.runq.mark_primary_runq_depends()
self.runq.prune_runq_depends_nobuild()
self.runq.prune_runq_depends_with_nobody_depending_on_it()
self.runq.prune_runq_tasks()
remaining = self.runq.number_of_tasks_to_build()
debug("%d tasks remains"%remaining)
recipes = self.runq.get_recipes_with_tasks_to_build()
if not recipes:
info("Nothing to do")
return 0
build_count = self.runq.set_buildhash_for_build_tasks()
nobuild_count = self.runq.set_buildhash_for_nobuild_tasks()
if (build_count + nobuild_count) != total:
die("build_count + nobuild_count != total")
deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR", True)
packages = self.runq.get_packages_to_build()
for package in packages:
package = self.cookbook.get_package(id=package)
recipe = self.cookbook.get_recipe(package=package.id)
buildhash = self.runq.get_package_buildhash(package.id)
filename = os.path.join(
deploy_dir, package.type,
package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
"%s_%s_%s.tar" % (package.name, recipe.version, buildhash))
debug("will use from build: %s" % (filename))
self.runq.set_package_filename(package.id, filename)
self.runq.mark_primary_runq_depends()
self.runq.prune_runq_depends_nobuild()
self.runq.prune_runq_depends_with_nobody_depending_on_it()
self.runq.prune_runq_tasks()
remaining = self.runq.number_of_tasks_to_build()
debug("%d tasks remains" % remaining)
recipes = self.runq.get_recipes_with_tasks_to_build()
if not recipes:
info("Nothing to do")
return 0
def __init__(self, options, args, config):
self.options = options
self.debug = self.options.debug
self.debug_loglines = getattr(self.options, 'debug_loglines', None)
# Bakery 3 compatibility, configure the logging module
if (not hasattr(oebakery, "__version__")
or oebakery.__version__.split(".")[0] < 4):
logging.basicConfig(format="%(message)s")
if self.debug:
logging.getLogger().setLevel(logging.DEBUG)
else:
logging.getLogger().setLevel(logging.INFO)
self.config = oelite.meta.DictMeta(meta=config)
self.config["OE_IMPORTS"] = INITIAL_OE_IMPORTS
self.config.import_env()
os.environ.clear()
self.config.pythonfunc_init()
self.topdir = self.config.get("TOPDIR", True)
self.set_manifest_origin()
# FIXME: self.config.freeze("TOPDIR")
self.confparser = confparse.ConfParser(self.config)
self.confparser.parse("conf/oe-lite.conf")
oelite.pyexec.exechooks(self.config, "post_conf_parse")
# FIXME: refactor oelite.arch.init to a post_conf_parse hook
oelite.arch.init(self.config)
# Handle any INHERITs and inherit the base class
inherits = ["core"] + (self.config.get("INHERIT", 1) or "").split()
# and inherit rmwork when needed
try:
rmwork = self.options.rmwork
if rmwork is None:
rmwork = self.config.get("RMWORK", True)
if rmwork == "0":
rmwork = False
if rmwork:
debug("rmwork")
inherits.append("rmwork")
self.options.rmwork = True
except AttributeError:
pass
self.oeparser = oeparse.OEParser(self.config)
for inherit in inherits:
self.oeparser.reset_lexstate()
self.oeparser.parse("classes/%s.oeclass" % (inherit), require=True)
oelite.pyexec.exechooks(self.config, "post_common_inherits")
self.cookbook = CookBook(self)
# things (ritem, item, recipe, or package) to do
if args:
self.things_todo = args
elif "OE_DEFAULT_THING" in self.config:
self.things_todo = self.config.get("OE_DEFAULT_THING", 1).split()
else:
self.things_todo = ["world"]
recipe_types = ("machine", "native", "sdk", "cross", "sdk-cross",
"canadian-cross")
def thing_todo(thing):
if not thing in self.things_todo:
self.things_todo.append(thing)
def dont_do_thing(thing):
while thing in self.things_todo:
self.things_todo.remove(thing)
self.recipes_todo = set()
if "universe" in self.things_todo:
dont_do_thing("universe")
for recipe_type in recipe_types:
thing_todo(recipe_type + ":world")
if "world" in self.things_todo:
dont_do_thing("world")
thing_todo("machine:world")
for recipe_type in recipe_types:
world = recipe_type + ":world"
if world in self.things_todo:
dont_do_thing(world)
for recipe in self.cookbook.get_recipes(type=recipe_type):
self.recipes_todo.add(recipe)
return
def get_recipe(self, id=None, task=None, package=None,
filename=None, type=None, name=None, version=None,
strict=True, default_type="machine"):
"""
Get recipe from cookbook. Returns recipe object if arguments
match a single recipe. Returns None if recipe is not found.
Throws MultipleRecipes exception if more than one recipe is
found.
"""
recipes = self.get_recipes(
id=id, task=task, package=package,
filename=filename, type=type, name=name, version=version)
if len(recipes) == 0:
recipe = None
elif len(recipes) == 1:
recipe = recipes[0]
elif strict:
warn("multiple recipes found in %s.%s: returning None!"%(
self.__class__.__name__, inspect.stack()[0][3]))
for recipe in recipes:
info("%s:%s_%s"%(recipe.type, recipe.name, recipe.version))
recipe = None
else:
chosen = [ recipes[0] ]
for other in recipes[1:]:
if chosen[0].priority > other.priority:
continue
if chosen[0].priority < other.priority:
chosen = [ other ]
continue
vercmp = bb.utils.vercmp_part(chosen[0].version, other.version)
if vercmp < 0:
chosen = [ other ]
if vercmp == 0:
#debug("chosen=%s\nother=%s"%(chosen, other))
#die("you have to be more precise")
chosen.append(other)
if len(chosen) == 1:
recipe = chosen[0]
elif not default_type:
warn("multiple recipes found in %s.%s: returning None!"%(
self.__class__.__name__, inspect.stack()[0][3]))
for recipe in chosen:
info("%s:%s_%s"%(recipe.type, recipe.name, recipe.version))
recipe = None
else:
# there is multiple recipes with the same priority and version,
# so let's try to pick the default type
defaults_chosen = []
for choice in chosen:
if choice.type == default_type:
defaults_chosen.append(choice)
if len(defaults_chosen) == 1:
recipe = defaults_chosen[0]
elif not defaults_chosen:
debug("multiple recipes, but none with default_type (%s)"%(
default_type))
recipe = None
else:
warn("multiple recipes found in %s.%s: returning None!"%(
self.__class__.__name__, inspect.stack()[0][3]))
for recipe in defaults_chosen:
info("%s:%s_%s"%(recipe.type, recipe.name,
recipe.version))
recipe = None
return recipe
def find_prebaked_package(self, package):
"""return full-path filename string or None"""
package_deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR")
prebake_url_cache_dir = self.config.get("PREBAKE_CACHE_DIR")
if not package_deploy_dir:
die("PACKAGE_DEPLOY_DIR not defined")
if self.options.prebake:
prebake_path = self.config.get("PREBAKE_PATH") or []
if prebake_path:
prebake_path = prebake_path.split(":")
prebake_path.insert(0, package_deploy_dir)
prebake_path.insert(0, prebake_url_cache_dir)
else:
prebake_path = [package_deploy_dir]
debug("package=%s" % (repr(package)))
recipe = self.cookbook.get_recipe(package=package)
if not recipe:
raise NoSuchRecipe()
metahash = self.runq.get_package_metahash(package)
debug("got metahash=%s" % (metahash))
package = self.cookbook.get_package(id=package)
if not package:
raise NoSuchPackage()
filename = "%s_%s_%s.tar" % (package.name, recipe.version, metahash)
debug("prebake_path=%s" % (prebake_path))
#test local paths first
for base_dir in prebake_path:
path = os.path.join(
base_dir, package.type,
package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
filename)
debug("checking for prebake: %s" % (path))
if os.path.exists(path):
debug("found prebake: %s" % (path))
return path
#then test URLs from PREBAKE_URL
url_prefix = self.config.get("PREBAKE_URL")
if url_prefix is not None:
package_path = os.path.join(
package.type,
package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
filename)
downloaded_file = os.path.join(prebake_url_cache_dir, package_path)
url = os.path.join(url_prefix, package_path)
if oelite.fetch.url.grab(url, downloaded_file, timeout=1, retry=1):
if os.path.exists(downloaded_file
) and os.path.getsize(downloaded_file) > 0:
debug("using prebake from web: %s" % (url))
return downloaded_file
else:
os.unlink(downloaded_file)
return None
build_count = self.runq.set_buildhash_for_build_tasks()
nobuild_count = self.runq.set_buildhash_for_nobuild_tasks()
if (build_count + nobuild_count) != total:
die("build_count + nobuild_count != total")
deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR", True)
packages = self.runq.get_packages_to_build()
for package in packages:
package = self.cookbook.get_package(id=package)
recipe = self.cookbook.get_recipe(package=package.id)
buildhash = self.runq.get_package_buildhash(package.id)
filename = os.path.join(
deploy_dir, package.type,
package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
"%s_%s_%s.tar" % (package.name, recipe.version, buildhash))
debug("will use from build: %s" % (filename))
self.runq.set_package_filename(package.id, filename)
self.runq.mark_primary_runq_depends()
self.runq.prune_runq_depends_nobuild()
self.runq.prune_runq_depends_with_nobody_depending_on_it()
self.runq.prune_runq_tasks()
remaining = self.runq.number_of_tasks_to_build()
debug("%d tasks remains" % remaining)
recipes = self.runq.get_recipes_with_tasks_to_build()
if not recipes:
info("Nothing to do")
return 0
def debug(*args):
oebakery.debug(" ".join(args))
