def list_recipefiles(self, sort=True):
OERECIPES = (self.config["OERECIPES"] or "").split(":")
if not OERECIPES:
die("OERECIPES not defined")
files = []
for f in OERECIPES:
if os.path.isdir(f):
dirfiles = find_recipoefiles(f)
files.append(dirfiles)
elif os.path.isfile(f):
files.append(f)
else:
for file in glob.iglob(f):
files.append(file)
oerecipes = []
for f in files:
if f.endswith(".oe"):
oerecipes.append(f)
else:
warn("skipping %s: unknown file extension"%(f))
if sort:
oerecipes.sort()
return oerecipes
def set_buildhash_for_build_tasks(self):
rowcount = self.dbc.execute(
"UPDATE runq.task SET buildhash=metahash WHERE build=1").rowcount
if rowcount == -1:
die("unable to determine rowcount in "
"set_buildhash_for_build_tasks")
return rowcount
def prune_runq_depends_with_nobody_depending_on_it(self):
#c = self.dbc.cursor()
rowcount = 0
start = oelite.util.now()
while True:
# The code below, until the executemany() call, implements
# what was previously done with this horribly-performing
# single SQL statement:
#
# self.dbc.execute(
# "DELETE FROM runq.depend "
# "WHERE prime IS NULL AND NOT EXISTS "
# "(SELECT * FROM runq.depend AS next_depend"
# " WHERE next_depend.parent_task=runq.depend.task"
# " LIMIT 1"
# ")")
dump = self.dbc.execute("SELECT rowid, prime, task, parent_task "
"FROM runq.depend").fetchall()
has_dependant = set([])
for x in dump:
has_dependant.add(x[3])
to_delete = [(x[0], ) for x in dump if not x[1] and x[2] not in has_dependant]
if not to_delete:
break
self.dbc.executemany("DELETE FROM runq.depend WHERE rowid=?", to_delete)
rc = self.dbc.rowcount
if rc == -1:
die("prune_runq_depends_with_no_depending_tasks did not work out")
assert(rc == len(to_delete))
rowcount += rc
oelite.util.timing_info("pruned %d dependencies which where not needed anyway"%rowcount, start)
def propagate_runq_task_build(self):
"""always build all tasks depending on other tasks to build"""
rowcount = 0
while True:
self.dbc.execute(
"UPDATE"
" runq.task "
"SET"
" build=1 "
"WHERE"
" build IS NULL"
" AND EXISTS"
" (SELECT *"
" FROM runq.depend, runq.task AS parent_task"
" WHERE runq.depend.task=runq.task.task"
" AND runq.depend.parent_task=parent_task.task"
" AND parent_task.build=1"
" LIMIT 1)")
if rowcount == -1:
die("propagate_runq_task_build did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
debug("set build flag on %d tasks due to propagation" % (rowcount))
return
def list_recipefiles(self, sort=True):
OERECIPES = (self.config["OERECIPES"] or "").split(":")
if not OERECIPES:
die("OERECIPES not defined")
files = []
for f in OERECIPES:
if os.path.isdir(f):
dirfiles = find_recipoefiles(f)
files.append(dirfiles)
elif os.path.isfile(f):
files.append(f)
else:
for file in glob.iglob(f):
files.append(file)
oerecipes = []
for f in files:
if f.endswith(".oe"):
oerecipes.append(f)
else:
warn("skipping %s: unknown file extension"%(f))
if sort:
oerecipes.sort()
return oerecipes
def propagate_runq_task_build(self):
"""always build all tasks depending on other tasks to build"""
rowcount = 0
while True:
self.dbc.execute(
"UPDATE"
" runq.task "
"SET"
" build=1 "
"WHERE"
" build IS NULL"
" AND EXISTS"
" (SELECT *"
" FROM runq.depend, runq.task AS parent_task"
" WHERE runq.depend.task=runq.task.task"
" AND runq.depend.parent_task=parent_task.task"
" AND parent_task.build=1"
" LIMIT 1)")
if rowcount == -1:
die("propagate_runq_task_build did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
debug("set build flag on %d tasks due to propagation"%(rowcount))
return
def set_buildhash_for_nobuild_tasks(self):
rowcount = self.dbc.execute(
"UPDATE runq.task SET buildhash=tmphash WHERE build IS NULL"
).rowcount
if rowcount == -1:
die("unable to determine rowcount in "
"set_buildhash_for_nobuild_tasks")
return rowcount
def mark_primary_runq_depends(self):
rowcount = self.dbc.execute(
"UPDATE runq.depend SET prime=1 WHERE EXISTS "
"(SELECT * FROM runq.task"
" WHERE runq.task.prime=1 AND runq.task.task=runq.depend.task"
")").rowcount
if rowcount == -1:
die("mark_primary_runq_depends did not work out")
return rowcount
def mark_primary_runq_depends(self):
rowcount = self.dbc.execute(
"UPDATE runq.depend SET prime=1 WHERE EXISTS "
"(SELECT * FROM runq.task"
" WHERE runq.task.prime=1 AND runq.task.task=runq.depend.task"
")").rowcount
if rowcount == -1:
die("mark_primary_runq_depends did not work out")
return rowcount
def set_task_build_on_nostamp_tasks(self):
rowcount = self.dbc.execute(
"UPDATE runq.task SET build=1 "
"WHERE build IS NULL AND EXISTS "
"(SELECT * FROM task"
" WHERE id=runq.task.task AND nostamp=1)").rowcount
if rowcount == -1:
die("set_task_build_on_nostamp_tasks did not work out")
debug("set build flag on %d nostamp tasks"%(rowcount))
return
def set_task_build_on_nostamp_tasks(self):
rowcount = self.dbc.execute(
"UPDATE runq.task SET build=1 "
"WHERE build IS NULL AND EXISTS "
"(SELECT * FROM task"
" WHERE id=runq.task.task AND nostamp=1)").rowcount
if rowcount == -1:
die("set_task_build_on_nostamp_tasks did not work out")
debug("set build flag on %d nostamp tasks" % (rowcount))
return
def find_prebaked_package(self, package):
"""return full-path filename string or None"""
package_deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR")
prebake_url_cache_dir = self.config.get("PREBAKE_CACHE_DIR")
if not package_deploy_dir:
die("PACKAGE_DEPLOY_DIR not defined")
if self.options.prebake:
prebake_path = self.config.get("PREBAKE_PATH") or []
if prebake_path:
prebake_path = prebake_path.split(":")
prebake_path.insert(0, package_deploy_dir)
prebake_path.insert(0, prebake_url_cache_dir)
else:
prebake_path = [package_deploy_dir]
debug("package=%s"%(repr(package)))
recipe = self.cookbook.get_recipe(package=package)
if not recipe:
raise NoSuchRecipe()
metahash = self.runq.get_package_metahash(package)
debug("got metahash=%s"%(metahash))
package = self.cookbook.get_package(id=package)
if not package:
raise NoSuchPackage()
filename = "%s_%s_%s.tar"%(package.name, recipe.version, metahash)
debug("prebake_path=%s"%(prebake_path))
#test local paths first
for base_dir in prebake_path:
path = os.path.join(
base_dir,
package.type,
package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
filename)
debug("checking for prebake: %s"%(path))
if os.path.exists(path):
debug("found prebake: %s"%(path))
return path
#then test URLs from PREBAKE_URL
url_prefix = self.config.get("PREBAKE_URL")
if url_prefix is not None:
package_path =os.path.join(
package.type,
package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
filename)
downloaded_file =os.path.join(
prebake_url_cache_dir,
package_path)
url = os.path.join(url_prefix, package_path)
if oelite.fetch.url.grab(url, downloaded_file, timeout=1, retry=1):
if os.path.exists(downloaded_file) and os.path.getsize(downloaded_file) > 0:
debug("using prebake from web: %s"%(url))
return downloaded_file
else:
os.unlink(downloaded_file)
return None
def read_stamp(self):
stampfile = self.stampfile_path()[1]
if not os.path.exists(stampfile):
return (None, None)
if not os.path.isfile(stampfile):
die("bad hash file: %s"%(stampfile))
if os.path.getsize(stampfile) == 0:
return (None, None)
mtime = os.stat(stampfile).st_mtime
with open(stampfile, "r") as stampfile:
tmphash = stampfile.read()
return (mtime, tmphash)
def read_stamp(self):
stampfile = self.stampfile_path()[1]
if not os.path.exists(stampfile):
return (None, None)
if not os.path.isfile(stampfile):
die("bad hash file: %s" % (stampfile))
if os.path.getsize(stampfile) == 0:
return (None, None)
mtime = os.stat(stampfile).st_mtime
with open(stampfile, "r") as stampfile:
tmphash = stampfile.read()
return (mtime, tmphash)
def find_prebaked_package(self, package):
"""return full-path filename string or None"""
package_deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR")
prebake_url_cache_dir = self.config.get("PREBAKE_CACHE_DIR")
if not package_deploy_dir:
die("PACKAGE_DEPLOY_DIR not defined")
if self.options.prebake:
prebake_path = self.config.get("PREBAKE_PATH") or []
if prebake_path:
prebake_path = prebake_path.split(":")
prebake_path.insert(0, package_deploy_dir)
prebake_path.insert(0, prebake_url_cache_dir)
else:
prebake_path = [package_deploy_dir]
debug("package=%s" % (repr(package)))
recipe = self.cookbook.get_recipe(package=package)
if not recipe:
raise NoSuchRecipe()
metahash = self.runq.get_package_metahash(package)
debug("got metahash=%s" % (metahash))
package = self.cookbook.get_package(id=package)
if not package:
raise NoSuchPackage()
filename = "%s_%s_%s.tar" % (package.name, recipe.version, metahash)
debug("prebake_path=%s" % (prebake_path))
#test local paths first
for base_dir in prebake_path:
path = os.path.join(
base_dir, package.type,
package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
filename)
debug("checking for prebake: %s" % (path))
if os.path.exists(path):
debug("found prebake: %s" % (path))
return path
#then test URLs from PREBAKE_URL
url_prefix = self.config.get("PREBAKE_URL")
if url_prefix is not None:
package_path = os.path.join(
package.type,
package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
filename)
downloaded_file = os.path.join(prebake_url_cache_dir, package_path)
url = os.path.join(url_prefix, package_path)
if oelite.fetch.url.grab(url, downloaded_file, timeout=1, retry=1):
if os.path.exists(downloaded_file
) and os.path.getsize(downloaded_file) > 0:
debug("using prebake from web: %s" % (url))
return downloaded_file
else:
os.unlink(downloaded_file)
return None
def add_package_depends(task_names, deptype, depends):
if not depends:
return
if not deptype in package_depends.keys():
package_depends[deptype] = set([])
for task_name in task_names:
for package in depends:
assert isinstance(package, oelite.package.OElitePackage)
recipe = package.recipe.id
task = self.cookbook.get_task(recipe=recipe, name=task_name)
if not task:
die("cannot add unsupported task %s:%s" % (package.recipe.name, task_name))
package_depends[deptype].add((task, package))
def set_task_build_on_hashdiff(self):
rowcount = 0
while True:
self.dbc.execute(
"UPDATE runq.task SET build=1 "
"WHERE build IS NULL AND relax IS NULL AND tmphash != metahash")
if rowcount == -1:
die("set_task_build_on_hashdiff did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
debug("set build flag on %d tasks with tmphash != metahash"%(rowcount))
return
def __init__(self, baker):
self.baker = baker
self.config = baker.config
self.oeparser = baker.oeparser
self.init_layer_meta()
self.db = sqlite.connect(":memory:", isolation_level=None)
if not self.db:
raise Exception("could not create in-memory sqlite db")
self.db.text_factory = str
self.dbc = CursorWrapper(self.db.cursor(), profile=False)
self.init_db()
self.recipes = {}
self.packages = {}
self.tasks = {}
self.cachedir = self.config.get("CACHEDIR") or ""
self.debug = self.baker.debug
fail = False
recipefiles = self.list_recipefiles()
total = len(recipefiles)
count = 0
rusage = oelite.profiling.Rusage("recipe parsing")
for recipefile in recipefiles:
count += 1
if self.debug:
debug("Adding %s to cookbook [%s/%s]"%(
self.shortfilename(recipefile), count, total))
else:
oelite.util.progress_info("Adding recipes to cookbook",
total, count)
try:
if not self.add_recipefile(recipefile):
fail = True
except KeyboardInterrupt:
if os.isatty(sys.stdout.fileno()) and not self.debug:
print
die("Aborted while building cookbook")
except oelite.parse.ParseError, e:
if os.isatty(sys.stdout.fileno()) and not self.debug:
print
e.print_details()
err("Parse error in %s"%(self.shortfilename(recipefile)))
fail = True
except Exception, e:
import traceback
if os.isatty(sys.stdout.fileno()) and not self.debug:
print
traceback.print_exc()
err("Uncaught Python exception in %s"%(
self.shortfilename(recipefile)))
fail = True
def set_task_build_on_hashdiff(self):
rowcount = 0
while True:
self.dbc.execute(
"UPDATE runq.task SET build=1 "
"WHERE build IS NULL AND relax IS NULL AND tmphash != metahash"
)
if rowcount == -1:
die("set_task_build_on_hashdiff did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
debug("set build flag on %d tasks with tmphash != metahash" %
(rowcount))
return
def add_package_depends(task_names, deptype, depends):
if not depends:
return
if not deptype in package_depends.keys():
package_depends[deptype] = set([])
for task_name in task_names:
for package in depends:
assert isinstance(package, oelite.package.OElitePackage)
recipe = package.recipe.id
task = self.cookbook.get_task(recipe=recipe,
name=task_name)
if not task:
die("cannot add unsupported task %s:%s" %
(package.recipe.name, task_name))
package_depends[deptype].add((task, package))
def prune_runq_tasks(self):
start = oelite.util.now()
rowcount = self.dbc.execute(
"UPDATE"
" runq.task "
"SET"
" build=NULL "
"WHERE"
" prime IS NULL AND NOT EXISTS"
" (SELECT *"
" FROM runq.depend"
" WHERE runq.depend.parent_task=runq.task.task"
" LIMIT 1"
")").rowcount
if rowcount == -1:
die("prune_runq_tasks did not work out")
oelite.util.timing_info("pruned %d tasks that does not need to be build"%rowcount, start)
def set_task_build_on_retired_tasks(self):
rowcount = 0
while True:
self.dbc.execute(
"UPDATE runq.task SET build=1 "
"WHERE build IS NULL AND EXISTS "
"(SELECT * FROM runq.depend, runq.task AS parent_task"
" WHERE runq.depend.task=runq.task.task"
" AND runq.depend.parent_task=parent_task.task"
" AND parent_task.mtime > runq.task.mtime)")
if rowcount == -1:
die("set_task_build_on_retired_tasks did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
debug("set build flag on %d retired tasks"%(rowcount))
return
def set_task_build_on_retired_tasks(self):
rowcount = 0
while True:
self.dbc.execute(
"UPDATE runq.task SET build=1 "
"WHERE build IS NULL AND EXISTS "
"(SELECT * FROM runq.depend, runq.task AS parent_task"
" WHERE runq.depend.task=runq.task.task"
" AND runq.depend.parent_task=parent_task.task"
" AND parent_task.mtime > runq.task.mtime)")
if rowcount == -1:
die("set_task_build_on_retired_tasks did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
debug("set build flag on %d retired tasks" % (rowcount))
return
def setup_tmpdir(self):
tmpdir = os.path.realpath(self.config.get("TMPDIR", 1) or "tmp")
#debug("TMPDIR = %s"%tmpdir)
try:
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
if (os.path.islink(tmpdir) and
not os.path.exists(os.path.realpath(tmpdir))):
os.makedirs(os.path.realpath(tmpdir))
except Exception, e:
die("failed to setup TMPDIR: %s"%e)
import traceback
e.print_exception(type(e), e, True)
def prune_runq_tasks(self):
rowcount = self.dbc.execute(
"UPDATE"
" runq.task "
"SET"
" build=NULL "
"WHERE"
" prime IS NULL AND NOT EXISTS"
" (SELECT *"
" FROM runq.depend"
" WHERE runq.depend.parent_task=runq.task.task"
" LIMIT 1"
")").rowcount
if rowcount == -1:
die("prune_runq_tasks did not work out")
if rowcount:
debug("pruned %d tasks that does not need to be build" % rowcount)
return rowcount
def setup_tmpdir(self):
tmpdir = os.path.realpath(self.config.get("TMPDIR", 1) or "tmp")
#debug("TMPDIR = %s"%tmpdir)
try:
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
if (os.path.islink(tmpdir)
and not os.path.exists(os.path.realpath(tmpdir))):
os.makedirs(os.path.realpath(tmpdir))
except Exception, e:
die("failed to setup TMPDIR: %s" % e)
import traceback
e.print_exception(type(e), e, True)
def prune_runq_tasks(self):
rowcount = self.dbc.execute(
"UPDATE"
" runq.task "
"SET"
" build=NULL "
"WHERE"
" prime IS NULL AND NOT EXISTS"
" (SELECT *"
" FROM runq.depend"
" WHERE runq.depend.parent_task=runq.task.task"
" LIMIT 1"
")").rowcount
if rowcount == -1:
die("prune_runq_tasks did not work out")
if rowcount:
debug("pruned %d tasks that does not need to be build"%rowcount)
return rowcount
def prune_runq_tasks(self):
start = oelite.util.now()
rowcount = self.dbc.execute(
"UPDATE"
" runq.task "
"SET"
" build=NULL "
"WHERE"
" prime IS NULL AND NOT EXISTS"
" (SELECT *"
" FROM runq.depend"
" WHERE runq.depend.parent_task=runq.task.task"
" LIMIT 1"
")").rowcount
if rowcount == -1:
die("prune_runq_tasks did not work out")
oelite.util.timing_info(
"pruned %d tasks that does not need to be build" % rowcount, start)
def prune_runq_depends_nobuild(self):
rowcount = 0
while True:
self.dbc.execute("UPDATE runq.depend SET parent_task=NULL "
"WHERE parent_task IS NOT NULL AND NOT EXISTS "
"(SELECT * FROM runq.task"
" WHERE runq.task.build=1"
" AND runq.task.task=runq.depend.parent_task"
" LIMIT 1"
")")
if rowcount == -1:
die("prune_runq_depends_nobuild did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
if rowcount:
debug("pruned %d dependencies that did not have to be rebuilt" %
rowcount)
return rowcount
def prune_runq_depends_nobuild(self):
rowcount = 0
while True:
self.dbc.execute(
"UPDATE runq.depend SET parent_task=NULL "
"WHERE parent_task IS NOT NULL AND NOT EXISTS "
"(SELECT * FROM runq.task"
" WHERE runq.task.build=1"
" AND runq.task.task=runq.depend.parent_task"
" LIMIT 1"
")")
if rowcount == -1:
die("prune_runq_depends_nobuild did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
if rowcount:
debug("pruned %d dependencies that did not have to be rebuilt"%rowcount)
return rowcount
def prune_runq_depends_with_nobody_depending_on_it(self):
#c = self.dbc.cursor()
rowcount = 0
while True:
self.dbc.execute(
"DELETE FROM runq.depend "
"WHERE prime IS NULL AND NOT EXISTS "
"(SELECT * FROM runq.depend AS next_depend"
" WHERE next_depend.parent_task=runq.depend.task"
" LIMIT 1"
")")
if rowcount == -1:
die("prune_runq_depends_with_no_depending_tasks did not work out")
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
if rowcount:
debug("pruned %d dependencies which where not needed anyway"%rowcount)
return rowcount
def prune_runq_depends_nobuild(self):
rowcount = 0
start = oelite.util.now()
while True:
self.dbc.execute(
"UPDATE runq.depend SET parent_task=NULL "
"WHERE parent_task IS NOT NULL AND NOT EXISTS "
"(SELECT * FROM runq.task"
" WHERE runq.task.build=1"
" AND runq.task.task=runq.depend.parent_task"
" LIMIT 1"
")")
rc = self.dbc.rowcount
if rc == -1:
die("prune_runq_depends_nobuild did not work out")
if not rc:
break
rowcount += rc
oelite.util.timing_info("pruned %d dependencies that did not have to be rebuilt"%rowcount, start)
def bake(self):
self.setup_tmpdir()
oelite.profiling.init(self.config)
# task(s) to do
if self.options.task:
tasks_todo = self.options.task
elif "OE_DEFAULT_TASK" in self.config:
tasks_todo = self.config.get("OE_DEFAULT_TASK", 1)
else:
#tasks_todo = "all"
tasks_todo = "build"
self.tasks_todo = tasks_todo.split(",")
if self.options.rebuild:
self.options.rebuild = max(self.options.rebuild)
self.options.prebake = False
else:
self.options.rebuild = None
if self.options.relax:
self.options.relax = max(self.options.relax)
else:
default_relax = self.config.get("DEFAULT_RELAX", 1)
if default_relax and default_relax != "0":
self.options.relax = int(default_relax)
else:
self.options.relax = None
# init build quue
self.runq = OEliteRunQueue(self.config, self.cookbook,
self.options.rebuild, self.options.relax)
# first, add complete dependency tree, with complete
# task-to-task and task-to-package/task dependency information
debug("Building dependency tree")
rusage = oelite.profiling.Rusage("Building dependency tree")
for task in self.tasks_todo:
task = oelite.task.task_name(task)
try:
for thing in self.recipes_todo:
if not self.runq.add_recipe(thing, task):
die("No such recipe: %s" % (thing))
for thing in self.things_todo:
thing = oelite.item.OEliteItem(thing)
if not self.runq.add_something(thing, task):
die("No such thing: %s" % (thing))
except RecursiveDepends, e:
die("dependency loop: %s\n\t--> %s" %
(e.args[1], "\n\t--> ".join(e.args[0])))
except NoSuchTask, e:
die("No such task: %s: %s" % (thing, e.__str__()))
def prune_runq_depends_with_nobody_depending_on_it(self):
#c = self.dbc.cursor()
rowcount = 0
while True:
self.dbc.execute("DELETE FROM runq.depend "
"WHERE prime IS NULL AND NOT EXISTS "
"(SELECT * FROM runq.depend AS next_depend"
" WHERE next_depend.parent_task=runq.depend.task"
" LIMIT 1"
")")
if rowcount == -1:
die("prune_runq_depends_with_no_depending_tasks did not work out"
)
if not self.dbc.rowcount:
break
rowcount += self.dbc.rowcount
if rowcount:
debug("pruned %d dependencies which where not needed anyway" %
rowcount)
return rowcount
def prune_runq_depends_nobuild(self):
rowcount = 0
start = oelite.util.now()
while True:
self.dbc.execute("UPDATE runq.depend SET parent_task=NULL "
"WHERE parent_task IS NOT NULL AND NOT EXISTS "
"(SELECT * FROM runq.task"
" WHERE runq.task.build=1"
" AND runq.task.task=runq.depend.parent_task"
" LIMIT 1"
")")
rc = self.dbc.rowcount
if rc == -1:
die("prune_runq_depends_nobuild did not work out")
if not rc:
break
rowcount += rc
oelite.util.timing_info(
"pruned %d dependencies that did not have to be rebuilt" %
rowcount, start)
def bake(self):
self.setup_tmpdir()
# task(s) to do
if self.options.task:
tasks_todo = self.options.task
elif "OE_DEFAULT_TASK" in self.config:
tasks_todo = self.config.get("OE_DEFAULT_TASK", 1)
else:
#tasks_todo = "all"
tasks_todo = "build"
self.tasks_todo = tasks_todo.split(",")
if self.options.rebuild:
self.options.rebuild = max(self.options.rebuild)
self.options.prebake = False
else:
self.options.rebuild = None
if self.options.relax:
self.options.relax = max(self.options.relax)
else:
default_relax = self.config.get("DEFAULT_RELAX", 1)
if default_relax and default_relax != "0":
self.options.relax = int(default_relax)
else:
self.options.relax = None
# init build quue
self.runq = OEliteRunQueue(self.config, self.cookbook,
self.options.rebuild, self.options.relax)
# first, add complete dependency tree, with complete
# task-to-task and task-to-package/task dependency information
debug("Building dependency tree")
start = datetime.datetime.now()
for task in self.tasks_todo:
task = oelite.task.task_name(task)
try:
for thing in self.recipes_todo:
if not self.runq.add_recipe(thing, task):
die("No such recipe: %s"%(thing))
for thing in self.things_todo:
thing = oelite.item.OEliteItem(thing)
if not self.runq.add_something(thing, task):
die("No such thing: %s"%(thing))
except RecursiveDepends, e:
die("dependency loop: %s\n\t--> %s"%(
e.args[1], "\n\t--> ".join(e.args[0])))
except NoSuchTask, e:
die("No such task: %s: %s"%(thing, e.__str__()))
def show(self):
if len(self.things_todo) == 0:
die("you must specify something to show")
if len(self.recipes_todo) > 0:
die("you cannot show world")
thing = oelite.item.OEliteItem(self.things_todo[0])
recipe = self.cookbook.get_recipe(
type=thing.type, name=thing.name, version=thing.version,
strict=False)
if not recipe:
die("Cannot find %s"%(thing))
if self.options.task:
if self.options.task.startswith("do_"):
task = self.options.task
else:
task = "do_" + self.options.task
self.runq = OEliteRunQueue(self.config, self.cookbook)
self.runq._add_recipe(recipe, task)
task = self.cookbook.get_task(recipe=recipe, name=task)
task.prepare(self.runq)
meta = task.meta()
else:
meta = recipe.meta
#meta.dump(pretty=False, nohash=False, flags=True,
# ignore_flags=("filename", "lineno"),
meta.dump(pretty=True, nohash=(not self.options.nohash),
only=(self.things_todo[1:] or None))
return 0
def show(self):
if len(self.things_todo) == 0:
die("you must specify something to show")
if len(self.recipes_todo) > 0:
die("you cannot show world")
thing = oelite.item.OEliteItem(self.things_todo[0])
recipe = self.cookbook.get_recipe(type=thing.type,
name=thing.name,
version=thing.version,
strict=False)
if not recipe:
die("Cannot find %s" % (thing))
if self.options.task:
if self.options.task.startswith("do_"):
task = self.options.task
else:
task = "do_" + self.options.task
self.runq = OEliteRunQueue(self.config, self.cookbook)
self.runq._add_recipe(recipe, task)
task = self.cookbook.get_task(recipe=recipe, name=task)
task.prepare(self.runq)
meta = task.meta()
else:
meta = recipe.meta
#meta.dump(pretty=False, nohash=False, flags=True,
# ignore_flags=("filename", "lineno"),
meta.dump(pretty=True,
nohash=(not self.options.nohash),
only=(self.things_todo[1:] or None))
return 0
def prune_runq_depends_with_nobody_depending_on_it(self):
#c = self.dbc.cursor()
rowcount = 0
start = oelite.util.now()
while True:
# The code below, until the executemany() call, implements
# what was previously done with this horribly-performing
# single SQL statement:
#
# self.dbc.execute(
# "DELETE FROM runq.depend "
# "WHERE prime IS NULL AND NOT EXISTS "
# "(SELECT * FROM runq.depend AS next_depend"
# " WHERE next_depend.parent_task=runq.depend.task"
# " LIMIT 1"
# ")")
dump = self.dbc.execute("SELECT rowid, prime, task, parent_task "
"FROM runq.depend").fetchall()
has_dependant = set([])
for x in dump:
has_dependant.add(x[3])
to_delete = [(x[0], ) for x in dump
if not x[1] and x[2] not in has_dependant]
if not to_delete:
break
self.dbc.executemany("DELETE FROM runq.depend WHERE rowid=?",
to_delete)
rc = self.dbc.rowcount
if rc == -1:
die("prune_runq_depends_with_no_depending_tasks did not work out"
)
assert (rc == len(to_delete))
rowcount += rc
oelite.util.timing_info(
"pruned %d dependencies which where not needed anyway" % rowcount,
start)
depends = set([])
depends.update(items)
packages = set([])
for depend in depends:
try:
_packages = resolve_dependency(
oelite.item.OEliteItem(depend, (deptype, context)),
([], []), rec_deptype)
except NoProvider, e:
if len(e.args) < 2:
_needed_by = needed_by
else:
_needed_by = e.args[1]
raise die("No provider for %s (needed by %s)"%(
e.args[0], _needed_by))
packages.update(_packages)
return packages
def get_recipe_provider(self, item):
package = self.get_provider(item)
if not package:
raise NoProvider(item)
recipe = self.cookbook.get_recipe(package=package)
return (recipe, package)
def _set_provider(self, item, package):
t = (item.type, item.name, item.version)
assert t not in self._provider
for j in xrange(len(unresolved_deps)-1, -1, -1):
recipe_dep = unresolved_deps[j]
if not recipe_dep in resolved_recipes:
continue
recipe_dep_extra_arch = recipe_dep.meta.get("EXTRA_ARCH")
if recipe_dep_extra_arch:
# FIXME: sanity check for inconsistent EXTRA_ARCH here
recipe.meta.set("EXTRA_ARCH", recipe_dep_extra_arch)
del unresolved_deps[j]
if len(unresolved_deps) == 0:
resolved_recipes.add(recipe)
del unresolved_recipes[i]
progress = True
oelite.pyexec.exechooks(recipe.meta, "post_extra_arch")
if not progress:
die("recipe EXTRA_ARCH resolving deadlocked!")
# update runq task list, checking recipe and src hashes and
# determining which tasks needs to be run
# examing each task, computing it's hash, and checking if the
# task has already been built, and with the same hash.
task = self.runq.get_metahashable_task()
total = self.runq.number_of_runq_tasks()
count = 0
start = datetime.datetime.now()
while task:
oelite.util.progress_info("Calculating task metadata hashes",
total, count)
recipe = task.recipe
if task.nostamp:
depends = set([])
depends.update(items)
packages = set([])
for depend in depends:
try:
_packages = resolve_dependency(
oelite.item.OEliteItem(depend, (deptype, context)),
([], []), rec_deptype)
except NoProvider, e:
if len(e.args) < 2:
_needed_by = needed_by
else:
_needed_by = e.args[1]
raise die("No provider for %s (needed by %s)" %
(e.args[0], _needed_by))
packages.update(_packages)
return packages
def get_recipe_provider(self, item):
package = self.get_provider(item)
if not package:
raise NoProvider(item)
recipe = self.cookbook.get_recipe(package=package)
return (recipe, package)
def _set_provider(self, item, package):
if item.version is None:
self.dbc.execute(
"INSERT INTO runq.provider (type, item, package) VALUES (?, ?, ?)",
if os.isatty(sys.stdout.fileno()) and not self.debug:
print
e.print_details()
err("Parse error in %s"%(self.shortfilename(recipefile)))
fail = True
except Exception, e:
import traceback
if os.isatty(sys.stdout.fileno()) and not self.debug:
print
traceback.print_exc()
err("Uncaught Python exception in %s"%(
self.shortfilename(recipefile)))
fail = True
rusage.end()
if fail:
die("Errors while adding recipes to cookbook")
self.create_world_recipes()
# All recipes parsed, no reason to hold on to the layer
# metadata (this frees a few MB of memory).
del self.layer_meta
#print "when instantiating from a parsed oefile, do some 'finalizing', ie. collapsing of overrides and append, and remember to save expand_cache also"
return
def init_layer_meta(self):
self.layer_meta = {}
oepath = self.config.get('OEPATH').split(':')
layer_priority = 0
continue
recipe_dep_extra_arch = recipe_dep.meta.get("EXTRA_ARCH")
if recipe_dep_extra_arch:
# FIXME: sanity check for inconsistent EXTRA_ARCH here
recipe.meta.set("EXTRA_ARCH", recipe_dep_extra_arch)
del unresolved_deps[j]
if len(unresolved_deps) == 0:
resolved_recipes.add(recipe)
del unresolved_recipes[i]
progress = True
oelite.pyexec.exechooks(recipe.meta, "post_extra_arch")
if not progress:
foo = ""
for r, deps in unresolved_recipes:
foo += "\n %s(%s)" % (r, ",".join(map(str, deps)))
die("recipe EXTRA_ARCH resolving deadlocked!" + foo)
# update runq task list, checking recipe and src hashes and
# determining which tasks needs to be run
# examing each task, computing it's hash, and checking if the
# task has already been built, and with the same hash.
task = self.runq.get_metahashable_task()
total = self.runq.number_of_runq_tasks()
count = 0
rusage = oelite.profiling.Rusage("Calculating task metadata hashes")
while task:
oelite.util.progress_info("Calculating task metadata hashes",
total, count)
recipe = task.recipe
if task.nostamp:
def fatal(*args):
oebakery.die(" ".join(args))
class OEliteBaker:
@oelite.profiling.profile_rusage_delta
def __init__(self, options, args, config):
self.options = options
self.debug = self.options.debug
self.debug_loglines = getattr(self.options, 'debug_loglines', None)
# Bakery 3 compatibility, configure the logging module
if (not hasattr(oebakery, "__version__")
or oebakery.__version__.split(".")[0] < 4):
logging.basicConfig(format="%(message)s")
if self.debug:
logging.getLogger().setLevel(logging.DEBUG)
else:
logging.getLogger().setLevel(logging.INFO)
self.config = oelite.meta.DictMeta(meta=config)
self.config["OE_IMPORTS"] = INITIAL_OE_IMPORTS
self.config.import_env()
os.environ.clear()
self.config.pythonfunc_init()
self.topdir = self.config.get("TOPDIR", True)
self.set_manifest_origin()
# FIXME: self.config.freeze("TOPDIR")
self.confparser = confparse.ConfParser(self.config)
self.confparser.parse("conf/oe-lite.conf")
oelite.pyexec.exechooks(self.config, "post_conf_parse")
# FIXME: refactor oelite.arch.init to a post_conf_parse hook
oelite.arch.init(self.config)
# Handle any INHERITs and inherit the base class
inherits = ["core"] + (self.config.get("INHERIT", 1) or "").split()
self.oeparser = oeparse.OEParser(self.config)
for inherit in inherits:
self.oeparser.reset_lexstate()
self.oeparser.parse("classes/%s.oeclass" % (inherit), require=True)
oelite.pyexec.exechooks(self.config, "post_common_inherits")
self.cookbook = CookBook(self)
# things (ritem, item, recipe, or package) to do
if args:
self.things_todo = args
elif "OE_DEFAULT_THING" in self.config:
self.things_todo = self.config.get("OE_DEFAULT_THING", 1).split()
else:
self.things_todo = ["world"]
recipe_types = ("machine", "native", "sdk", "cross", "sdk-cross",
"canadian-cross")
def thing_todo(thing):
if not thing in self.things_todo:
self.things_todo.append(thing)
def dont_do_thing(thing):
while thing in self.things_todo:
self.things_todo.remove(thing)
self.recipes_todo = set()
if "universe" in self.things_todo:
dont_do_thing("universe")
for recipe_type in recipe_types:
thing_todo(recipe_type + ":world")
return
def __del__(self):
return
def set_manifest_origin(self):
if not os.path.exists(os.path.join(self.topdir, '.git')):
return
url = oelite.util.shcmd("git config --get remote.origin.url",
quiet=True,
silent_errorcodes=[1])
if not url:
return
url = os.path.dirname(url.strip())
if url.startswith('file:///'):
url = url[7:]
srcuri = None
protocols = ('git', 'ssh', 'http', 'https', 'ftp', 'ftps', 'rsync')
if url.startswith('/'):
srcuri = 'git://%s' % (url)
protocol = 'file'
elif url.startswith('git://'):
srcuri = url
protocol = None
elif not url.split('://')[0] in protocols:
url = 'ssh://' + url.replace(':', '/', 1)
if srcuri is None:
for protocol in protocols:
if url.startswith('%s://' % (protocol)):
srcuri = 'git://%s' % (url[len(protocol) + 3:])
break
self.config.set('MANIFEST_ORIGIN_URL', url)
self.config.set_flag('MANIFEST_ORIGIN_URL', 'nohash', True)
if srcuri is None:
logging.warning("unsupported manifest origin: %s" % (url))
return
self.config.set('MANIFEST_ORIGIN_SRCURI', srcuri)
self.config.set_flag('MANIFEST_ORIGIN_SRCURI', 'nohash', True)
if protocol:
self.config.set('MANIFEST_ORIGIN_PARAMS',
';protocol=%s' % (protocol))
else:
self.config.set('MANIFEST_ORIGIN_PARAMS', '')
self.config.set_flag('MANIFEST_ORIGIN_PARAMS', 'nohash', True)
def show(self):
if len(self.things_todo) == 0:
die("you must specify something to show")
if len(self.recipes_todo) > 0:
die("you cannot show world")
thing = oelite.item.OEliteItem(self.things_todo[0])
recipe = self.cookbook.get_recipe(type=thing.type,
name=thing.name,
version=thing.version,
strict=False)
if not recipe:
die("Cannot find %s" % (thing))
if self.options.task:
if self.options.task.startswith("do_"):
task = self.options.task
else:
task = "do_" + self.options.task
self.runq = OEliteRunQueue(self.config, self.cookbook)
self.runq._add_recipe(recipe, task)
task = self.cookbook.get_task(recipe=recipe, name=task)
task.prepare()
meta = task.meta()
else:
meta = recipe.meta
#meta.dump(pretty=False, nohash=False, flags=True,
# ignore_flags=re.compile("filename|lineno"),
meta.dump(pretty=True,
show_nohash=(not self.options.nohash),
dynvar_replacement=(not self.options.nodynvar),
only=(self.things_todo[1:] or None))
return 0
def bake(self):
self.setup_tmpdir()
oelite.profiling.init(self.config)
# task(s) to do
if self.options.task:
tasks_todo = self.options.task
elif "OE_DEFAULT_TASK" in self.config:
tasks_todo = self.config.get("OE_DEFAULT_TASK", 1)
else:
#tasks_todo = "all"
tasks_todo = "build"
self.tasks_todo = tasks_todo.split(",")
if self.options.rebuild:
self.options.rebuild = max(self.options.rebuild)
self.options.prebake = False
else:
self.options.rebuild = None
if self.options.relax:
self.options.relax = max(self.options.relax)
else:
default_relax = self.config.get("DEFAULT_RELAX", 1)
if default_relax and default_relax != "0":
self.options.relax = int(default_relax)
else:
self.options.relax = None
# init build quue
self.runq = OEliteRunQueue(self.config, self.cookbook,
self.options.rebuild, self.options.relax)
# first, add complete dependency tree, with complete
# task-to-task and task-to-package/task dependency information
debug("Building dependency tree")
rusage = oelite.profiling.Rusage("Building dependency tree")
for task in self.tasks_todo:
task = oelite.task.task_name(task)
try:
for thing in self.recipes_todo:
if not self.runq.add_recipe(thing, task):
die("No such recipe: %s" % (thing))
for thing in self.things_todo:
thing = oelite.item.OEliteItem(thing)
if not self.runq.add_something(thing, task):
die("No such thing: %s" % (thing))
except RecursiveDepends, e:
die("dependency loop: %s\n\t--> %s" %
(e.args[1], "\n\t--> ".join(e.args[0])))
except NoSuchTask, e:
die("No such task: %s: %s" % (thing, e.__str__()))
except oebakery.FatalError, e:
die("Failed to add %s:%s to runqueue" % (thing, task))
except oelite.parse.ParseError, e:
if os.isatty(sys.stdout.fileno()) and not self.debug:
print
e.print_details()
err("Parse error in %s"%(self.shortfilename(recipefile)))
fail = True
except Exception, e:
import traceback
if os.isatty(sys.stdout.fileno()) and not self.debug:
print
traceback.print_exc()
err("Uncaught Python exception in %s"%(
self.shortfilename(recipefile)))
fail = True
if fail:
die("Errors while adding recipes to cookbook")
#print "when instantiating from a parsed oefile, do some 'finalizing', ie. collapsing of overrides and append, and remember to save expand_cache also"
return
def init_layer_meta(self):
self.layer_meta = {}
oepath = self.config.get('OEPATH').split(':')
layer_priority = 0
max_layer_height = 0
def layer_height_roundup(priority):
return (priority+99)/100*100
layer_conf_files = []
for layer in reversed(oepath):
layer_conf = os.path.join(layer, 'conf', 'layer.conf')
