def execute_runqueue_initVars(self):
self.stats = RunQueueStats(len(self.runq_fnid))
self.runq_buildable = []
self.runq_running = []
self.runq_complete = []
self.build_pids = {}
self.build_pipes = {}
self.failed_fnids = []
# Mark initial buildable tasks
for task in range(self.stats.total):
self.runq_running.append(0)
self.runq_complete.append(0)
if len(self.runq_depends[task]) == 0:
self.runq_buildable.append(1)
else:
self.runq_buildable.append(0)
self.state = runQueueRunning
event.fire(
bb.event.StampUpdate(self.target_pairs, self.dataCache.stamp),
self.cfgData)
def exec_task(fn, task, d, profile = False):
try:
quieterr = False
if d.getVarFlag(task, "quieterrors") is not None:
quieterr = True
if profile:
profname = "profile-%s.log" % (d.getVar("PN", True) + "-" + task)
try:
import cProfile as profile
except:
import profile
prof = profile.Profile()
ret = profile.Profile.runcall(prof, _exec_task, fn, task, d, quieterr)
prof.dump_stats(profname)
bb.utils.process_profilelog(profname)
return ret
else:
return _exec_task(fn, task, d, quieterr)
except Exception:
from traceback import format_exc
if not quieterr:
logger.error("Build of %s failed" % (task))
logger.error(format_exc())
failedevent = TaskFailed(task, None, d, True)
event.fire(failedevent, d)
return 1
def exec_task(fn, task, d, profile = False):
try:
quieterr = False
if d.getVarFlag(task, "quieterrors", False) is not None:
quieterr = True
if profile:
profname = "profile-%s.log" % (d.getVar("PN") + "-" + task)
try:
import cProfile as profile
except:
import profile
prof = profile.Profile()
ret = profile.Profile.runcall(prof, _exec_task, fn, task, d, quieterr)
prof.dump_stats(profname)
bb.utils.process_profilelog(profname)
return ret
else:
return _exec_task(fn, task, d, quieterr)
except Exception:
from traceback import format_exc
if not quieterr:
logger.error("Build of %s failed" % (task))
logger.error(format_exc())
failedevent = TaskFailed(task, None, d, True)
event.fire(failedevent, d)
return 1
def exec_task(fn, task, d):
try:
return _exec_task(fn, task, d)
except Exception:
from traceback import format_exc
logger.error("Build of %s failed" % (task))
logger.error(format_exc())
failedevent = TaskFailed(task, None, d)
event.fire(failedevent, d)
return 1
def exec_task(fn, task, d):
try:
quieterr = False
if d.getVarFlag(task, "quieterrors") is not None:
quieterr = True
return _exec_task(fn, task, d, quieterr)
except Exception:
from traceback import format_exc
if not quieterr:
logger.error("Build of %s failed" % (task))
logger.error(format_exc())
failedevent = TaskFailed(task, None, d)
event.fire(failedevent, d)
return 1
def exec_task(fn, task, d):
try:
quieterr = False
if d.getVarFlag(task, "quieterrors") is not None:
quieterr = True
return _exec_task(fn, task, d, quieterr)
except Exception:
from traceback import format_exc
if not quieterr:
logger.error("Build of %s failed" % (task))
logger.error(format_exc())
failedevent = TaskFailed(task, None, d, True)
event.fire(failedevent, d)
return 1
def exec_task(task, d):
"""Execute an BB 'task'
The primary difference between executing a task versus executing
a function is that a task exists in the task digraph, and therefore
has dependencies amongst other tasks."""
# Check whther this is a valid task
if not data.getVarFlag(task, 'task', d):
raise EventException("No such task", InvalidTask(task, d))
try:
bb.msg.debug(1, bb.msg.domain.Build, "Executing task %s" % task)
old_overrides = data.getVar('OVERRIDES', d, 0)
localdata = data.createCopy(d)
data.setVar('OVERRIDES', 'task-%s:%s' % (task[3:], old_overrides), localdata)
data.update_data(localdata)
data.expandKeys(localdata)
event.fire(TaskStarted(task, localdata))
exec_func(task, localdata)
event.fire(TaskSucceeded(task, localdata))
except FuncFailed, message:
# Try to extract the optional logfile
try:
(msg, logfile) = message
except:
logfile = None
msg = message
bb.msg.note(1, bb.msg.domain.Build, "Task failed: %s" % message )
failedevent = TaskFailed(msg, logfile, task, d)
event.fire(failedevent)
raise EventException("Function failed in task: %s" % message, failedevent)
def execute(graph, item):
if data.getVarFlag(item, 'task', d):
if item in task_cache:
return 1
if task != item:
# deeper than toplevel, exec w/ deps
exec_task(item, d)
return 1
try:
bb.msg.debug(1, bb.msg.domain.Build, "Executing task %s" % item)
old_overrides = data.getVar('OVERRIDES', d, 0)
localdata = data.createCopy(d)
data.setVar('OVERRIDES', 'task_%s:%s' % (item, old_overrides), localdata)
data.update_data(localdata)
event.fire(TaskStarted(item, localdata))
exec_func(item, localdata)
event.fire(TaskSucceeded(item, localdata))
task_cache.append(item)
data.setVar('_task_cache', task_cache, d)
except FuncFailed, reason:
bb.msg.note(1, bb.msg.domain.Build, "Task failed: %s" % reason )
failedevent = TaskFailed(item, d)
event.fire(failedevent)
raise EventException("Function failed in task: %s" % reason, failedevent)
def exec_task(fn, task, d):
"""Execute a BB 'task'
Execution of a task involves a bit more setup than executing a function,
running it with its own local metadata, and with some useful variables set.
"""
# Check whther this is a valid task
if not data.getVarFlag(task, 'task', d):
raise InvalidTask(task, d)
try:
logger.debug(1, "Executing task %s", task)
old_overrides = data.getVar('OVERRIDES', d, 0)
localdata = data.createCopy(d)
data.setVar('OVERRIDES', 'task-%s:%s' % (task[3:], old_overrides), localdata)
data.update_data(localdata)
data.expandKeys(localdata)
data.setVar('BB_FILENAME', fn, d)
data.setVar('BB_CURRENTTASK', task[3:], d)
event.fire(TaskStarted(task, localdata), localdata)
exec_func(task, localdata)
event.fire(TaskSucceeded(task, localdata), localdata)
except FuncFailed as exc:
event.fire(TaskFailed(exc.name, exc.logfile, localdata), localdata)
raise
# make stamp, or cause event and raise exception
if not data.getVarFlag(task, 'nostamp', d) and not data.getVarFlag(task, 'selfstamp', d):
make_stamp(task, d)
def exec_task(task, d):
"""Execute an BB 'task'
The primary difference between executing a task versus executing
a function is that a task exists in the task digraph, and therefore
has dependencies amongst other tasks."""
# Check whther this is a valid task
if not data.getVarFlag(task, 'task', d):
raise EventException("No such task", InvalidTask(task, d))
try:
bb.msg.debug(1, bb.msg.domain.Build, "Executing task %s" % task)
old_overrides = data.getVar('OVERRIDES', d, 0)
localdata = data.createCopy(d)
data.setVar('OVERRIDES', 'task-%s:%s' % (task[3:], old_overrides),
localdata)
data.update_data(localdata)
data.expandKeys(localdata)
event.fire(TaskStarted(task, localdata), localdata)
exec_func(task, localdata)
event.fire(TaskSucceeded(task, localdata), localdata)
except FuncFailed, message:
# Try to extract the optional logfile
try:
(msg, logfile) = message
except:
logfile = None
msg = message
bb.msg.note(1, bb.msg.domain.Build, "Task failed: %s" % message)
failedevent = TaskFailed(msg, logfile, task, d)
event.fire(failedevent, d)
raise EventException("Function failed in task: %s" % message,
failedevent)
def execute(graph, item):
if data.getVarFlag(item, 'task', d):
if item in task_cache:
return 1
if task != item:
# deeper than toplevel, exec w/ deps
exec_task(item, d)
return 1
try:
debug(1, "Executing task %s" % item)
old_overrides = data.getVar('OVERRIDES', d, 0)
from copy import deepcopy
localdata = deepcopy(d)
data.setVar('OVERRIDES', 'task_%s:%s' % (item, old_overrides), localdata)
data.update_data(localdata)
event.fire(TaskStarted(item, localdata))
exec_func(item, localdata)
event.fire(TaskSucceeded(item, localdata))
task_cache.append(item)
except FuncFailed, reason:
note( "Task failed: %s" % reason )
failedevent = TaskFailed(item, d)
event.fire(failedevent)
raise EventException(None, failedevent)
def execute_runqueue_initVars(self):
self.stats = RunQueueStats(len(self.runq_fnid))
self.runq_buildable = []
self.runq_running = []
self.runq_complete = []
self.build_pids = {}
self.failed_fnids = []
# Mark initial buildable tasks
for task in range(self.stats.total):
self.runq_running.append(0)
self.runq_complete.append(0)
if len(self.runq_depends[task]) == 0:
self.runq_buildable.append(1)
else:
self.runq_buildable.append(0)
self.state = runQueueRunning
event.fire(bb.event.StampUpdate(self.target_pairs, self.dataCache.stamp, self.cfgData))
def exec_task(fn, task, d):
"""Execute a BB 'task'
Execution of a task involves a bit more setup than executing a function,
running it with its own local metadata, and with some useful variables set.
"""
if not data.getVarFlag(task, 'task', d):
raise InvalidTask(task, d)
logger.debug(1, "Executing task %s", task)
localdata = _task_data(fn, task, d)
tempdir = localdata.getVar('T', True)
if not tempdir:
bb.fatal("T variable not set, unable to build")
bb.utils.mkdirhier(tempdir)
loglink = os.path.join(tempdir, 'log.{0}'.format(task))
logfn = os.path.join(tempdir, 'log.{0}.{1}'.format(task, os.getpid()))
if loglink:
bb.utils.remove(loglink)
try:
os.symlink(logfn, loglink)
except OSError:
pass
prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
logfile = open(logfn, 'w')
event.fire(TaskStarted(task, localdata), localdata)
try:
for func in (prefuncs or '').split():
exec_func(func, localdata, logfile=logfile)
exec_func(task, localdata, logfile=logfile)
for func in (postfuncs or '').split():
exec_func(func, localdata, logfile=logfile)
except FuncFailed as exc:
event.fire(TaskFailed(exc.name, exc.logfile, localdata), localdata)
raise
finally:
logfile.close()
if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
logger.debug(2, "Zero size logfn %s, removing", logfn)
bb.utils.remove(logfn)
bb.utils.remove(loglink)
event.fire(TaskSucceeded(task, localdata), localdata)
if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(
task, 'selfstamp'):
make_stamp(task, localdata)
def exec_task(fn, task, d):
"""Execute a BB 'task'
Execution of a task involves a bit more setup than executing a function,
running it with its own local metadata, and with some useful variables set.
"""
if not data.getVarFlag(task, 'task', d):
raise InvalidTask(task, d)
logger.debug(1, "Executing task %s", task)
localdata = _task_data(fn, task, d)
tempdir = localdata.getVar('T', True)
if not tempdir:
bb.fatal("T variable not set, unable to build")
bb.utils.mkdirhier(tempdir)
loglink = os.path.join(tempdir, 'log.{0}'.format(task))
logfn = os.path.join(tempdir, 'log.{0}.{1}'.format(task, os.getpid()))
if loglink:
bb.utils.remove(loglink)
try:
os.symlink(logfn, loglink)
except OSError:
pass
prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
logfile = open(logfn, 'w')
event.fire(TaskStarted(task, localdata), localdata)
try:
for func in (prefuncs or '').split():
exec_func(func, localdata, logfile=logfile)
exec_func(task, localdata, logfile=logfile)
for func in (postfuncs or '').split():
exec_func(func, localdata, logfile=logfile)
except FuncFailed as exc:
event.fire(TaskFailed(exc.name, exc.logfile, localdata), localdata)
raise
finally:
logfile.close()
if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
logger.debug(2, "Zero size logfn %s, removing", logfn)
bb.utils.remove(logfn)
bb.utils.remove(loglink)
event.fire(TaskSucceeded(task, localdata), localdata)
if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(task, 'selfstamp'):
make_stamp(task, localdata)
def execute(graph, item):
if data.getVarFlag(item, 'task', d):
if item in task_cache:
return 1
if task != item:
# deeper than toplevel, exec w/ deps
exec_task(item, d)
return 1
try:
debug(1, "Executing task %s" % item)
event.fire(TaskStarted(item, d))
exec_func(item, d)
event.fire(TaskSucceeded(item, d))
task_cache.append(item)
except FuncFailed, reason:
note( "Task failed: %s" % reason )
failedevent = TaskFailed(item, d)
event.fire(failedevent)
raise EventException(None, failedevent)
def execute_runqueue_internal(self):
"""
Run the tasks in a queue prepared by prepare_runqueue
"""
bb.msg.note(1, bb.msg.domain.RunQueue, "Executing runqueue")
self.execute_runqueue_initVars()
if len(self.runq_fnid) == 0:
# nothing to do
return []
def sigint_handler(signum, frame):
raise KeyboardInterrupt
event.fire(
bb.event.StampUpdate(self.target_pairs, self.dataCache.stamp,
self.cfgdata))
while True:
task = self.sched.next()
if task is not None:
fn = self.taskData.fn_index[self.runq_fnid[task]]
taskname = self.runq_task[task]
if self.check_stamp_task(task):
bb.msg.debug(
2, bb.msg.domain.RunQueue,
"Stamp current task %s (%s)" %
(task, self.get_user_idstring(task)))
self.runq_running[task] = 1
self.task_complete(task)
self.stats.taskCompleted()
self.stats.taskSkipped()
continue
bb.msg.note(
1, bb.msg.domain.RunQueue,
"Running task %d of %d (ID: %s, %s)" %
(self.stats.completed + self.active_builds + 1,
len(self.runq_fnid), task, self.get_user_idstring(task)))
sys.stdout.flush()
sys.stderr.flush()
try:
pid = os.fork()
except OSError, e:
bb.msg.fatal(
bb.msg.domain.RunQueue,
"fork failed: %d (%s)" % (e.errno, e.strerror))
if pid == 0:
# Bypass master process' handling
self.master_process = False
# Stop Ctrl+C being sent to children
# signal.signal(signal.SIGINT, signal.SIG_IGN)
# Make the child the process group leader
os.setpgid(0, 0)
newsi = os.open('/dev/null', os.O_RDWR)
os.dup2(newsi, sys.stdin.fileno())
self.cooker.configuration.cmd = taskname[3:]
bb.data.setVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY", self,
self.cooker.configuration.data)
try:
self.cooker.tryBuild(fn)
except bb.build.EventException:
bb.msg.error(
bb.msg.domain.Build,
"Build of " + fn + " " + taskname + " failed")
sys.exit(1)
except:
bb.msg.error(
bb.msg.domain.Build,
"Build of " + fn + " " + taskname + " failed")
raise
sys.exit(0)
self.build_pids[pid] = task
self.runq_running[task] = 1
self.active_builds = self.active_builds + 1
if self.active_builds < self.number_tasks:
continue
if self.active_builds > 0:
result = os.waitpid(-1, 0)
self.active_builds = self.active_builds - 1
task = self.build_pids[result[0]]
if result[1] != 0:
del self.build_pids[result[0]]
bb.msg.error(
bb.msg.domain.RunQueue, "Task %s (%s) failed" %
(task, self.get_user_idstring(task)))
self.failed_fnids.append(self.runq_fnid[task])
self.stats.taskFailed()
if not self.taskData.abort:
continue
break
self.task_complete(task)
self.stats.taskCompleted()
del self.build_pids[result[0]]
continue
return
def _exec_task(fn, task, d, quieterr):
"""Execute a BB 'task'
Execution of a task involves a bit more setup than executing a function,
running it with its own local metadata, and with some useful variables set.
"""
if not d.getVarFlag(task, 'task', False):
event.fire(TaskInvalid(task, d), d)
logger.error("No such task: %s" % task)
return 1
logger.debug(1, "Executing task %s", task)
localdata = _task_data(fn, task, d)
tempdir = localdata.getVar('T')
if not tempdir:
bb.fatal("T variable not set, unable to build")
# Change nice level if we're asked to
nice = localdata.getVar("BB_TASK_NICE_LEVEL")
if nice:
curnice = os.nice(0)
nice = int(nice) - curnice
newnice = os.nice(nice)
logger.debug(1, "Renice to %s " % newnice)
ionice = localdata.getVar("BB_TASK_IONICE_LEVEL")
if ionice:
try:
cls, prio = ionice.split(".", 1)
bb.utils.ioprio_set(os.getpid(), int(cls), int(prio))
except:
bb.warn("Invalid ionice level %s" % ionice)
bb.utils.mkdirhier(tempdir)
# Determine the logfile to generate
logfmt = localdata.getVar('BB_LOGFMT') or 'log.{task}.{pid}'
logbase = logfmt.format(task=task, pid=os.getpid())
# Document the order of the tasks...
logorder = os.path.join(tempdir, 'log.task_order')
try:
with open(logorder, 'a') as logorderfile:
logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase))
except OSError:
logger.exception("Opening log file '%s'", logorder)
pass
# Setup the courtesy link to the logfn
loglink = os.path.join(tempdir, 'log.{0}'.format(task))
logfn = os.path.join(tempdir, logbase)
if loglink:
bb.utils.remove(loglink)
try:
os.symlink(logbase, loglink)
except OSError:
pass
prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
class ErrorCheckHandler(logging.Handler):
def __init__(self):
self.triggered = False
logging.Handler.__init__(self, logging.ERROR)
def emit(self, record):
if getattr(record, 'forcelog', False):
self.triggered = False
else:
self.triggered = True
# Handle logfiles
try:
bb.utils.mkdirhier(os.path.dirname(logfn))
logfile = open(logfn, 'w')
except OSError:
logger.exception("Opening log file '%s'", logfn)
pass
# Dup the existing fds so we dont lose them
osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
# Replace those fds with our own
with open('/dev/null', 'r') as si:
os.dup2(si.fileno(), osi[1])
os.dup2(logfile.fileno(), oso[1])
os.dup2(logfile.fileno(), ose[1])
# Ensure Python logging goes to the logfile
handler = logging.StreamHandler(logfile)
handler.setFormatter(logformatter)
# Always enable full debug output into task logfiles
handler.setLevel(logging.DEBUG - 2)
bblogger.addHandler(handler)
errchk = ErrorCheckHandler()
bblogger.addHandler(errchk)
localdata.setVar('BB_LOGFILE', logfn)
localdata.setVar('BB_RUNTASK', task)
localdata.setVar('BB_TASK_LOGGER', bblogger)
flags = localdata.getVarFlags(task)
try:
try:
event.fire(TaskStarted(task, logfn, flags, localdata), localdata)
except (bb.BBHandledException, SystemExit):
return 1
except FuncFailed as exc:
logger.error(str(exc))
return 1
try:
for func in (prefuncs or '').split():
exec_func(func, localdata)
exec_func(task, localdata)
for func in (postfuncs or '').split():
exec_func(func, localdata)
except FuncFailed as exc:
if quieterr:
event.fire(TaskFailedSilent(task, logfn, localdata), localdata)
else:
errprinted = errchk.triggered
logger.error(str(exc))
event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata)
return 1
except bb.BBHandledException:
event.fire(TaskFailed(task, logfn, localdata, True), localdata)
return 1
finally:
sys.stdout.flush()
sys.stderr.flush()
bblogger.removeHandler(handler)
# Restore the backup fds
os.dup2(osi[0], osi[1])
os.dup2(oso[0], oso[1])
os.dup2(ose[0], ose[1])
# Close the backup fds
os.close(osi[0])
os.close(oso[0])
os.close(ose[0])
logfile.close()
if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
logger.debug(2, "Zero size logfn %s, removing", logfn)
bb.utils.remove(logfn)
bb.utils.remove(loglink)
event.fire(TaskSucceeded(task, logfn, localdata), localdata)
if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False):
make_stamp(task, localdata)
return 0
def _exec_task(fn, task, d, quieterr):
"""Execute a BB 'task'
Execution of a task involves a bit more setup than executing a function,
running it with its own local metadata, and with some useful variables set.
"""
if not data.getVarFlag(task, 'task', d):
event.fire(TaskInvalid(task, d), d)
logger.error("No such task: %s" % task)
return 1
logger.debug(1, "Executing task %s", task)
localdata = _task_data(fn, task, d)
tempdir = localdata.getVar('T', True)
if not tempdir:
bb.fatal("T variable not set, unable to build")
bb.utils.mkdirhier(tempdir)
loglink = os.path.join(tempdir, 'log.{0}'.format(task))
logfn = os.path.join(tempdir, 'log.{0}.{1}'.format(task, os.getpid()))
if loglink:
bb.utils.remove(loglink)
try:
os.symlink(logfn, loglink)
except OSError:
pass
prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
class ErrorCheckHandler(logging.Handler):
def __init__(self):
self.triggered = False
logging.Handler.__init__(self, logging.ERROR)
def emit(self, record):
self.triggered = True
# Handle logfiles
si = file('/dev/null', 'r')
try:
logfile = file(logfn, 'w')
except OSError:
logger.exception("Opening log file '%s'", logfn)
pass
# Dup the existing fds so we dont lose them
osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
# Replace those fds with our own
os.dup2(si.fileno(), osi[1])
os.dup2(logfile.fileno(), oso[1])
os.dup2(logfile.fileno(), ose[1])
# Ensure python logging goes to the logfile
handler = logging.StreamHandler(logfile)
handler.setFormatter(logformatter)
# Always enable full debug output into task logfiles
handler.setLevel(logging.DEBUG - 2)
bblogger.addHandler(handler)
errchk = ErrorCheckHandler()
bblogger.addHandler(errchk)
localdata.setVar('BB_LOGFILE', logfn)
event.fire(TaskStarted(task, localdata), localdata)
try:
for func in (prefuncs or '').split():
exec_func(func, localdata)
exec_func(task, localdata)
for func in (postfuncs or '').split():
exec_func(func, localdata)
except FuncFailed as exc:
if quieterr:
event.fire(TaskFailedSilent(task, logfn, localdata), localdata)
else:
errprinted = errchk.triggered
logger.error(str(exc))
event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata)
return 1
finally:
sys.stdout.flush()
sys.stderr.flush()
bblogger.removeHandler(handler)
# Restore the backup fds
os.dup2(osi[0], osi[1])
os.dup2(oso[0], oso[1])
os.dup2(ose[0], ose[1])
# Close the backup fds
os.close(osi[0])
os.close(oso[0])
os.close(ose[0])
si.close()
logfile.close()
if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
logger.debug(2, "Zero size logfn %s, removing", logfn)
bb.utils.remove(logfn)
bb.utils.remove(loglink)
event.fire(TaskSucceeded(task, localdata), localdata)
if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(task, 'selfstamp'):
make_stamp(task, localdata)
return 0
def _exec_task(fn, task, d, quieterr):
"""Execute a BB 'task'
Execution of a task involves a bit more setup than executing a function,
running it with its own local metadata, and with some useful variables set.
"""
if not d.getVarFlag(task, 'task'):
event.fire(TaskInvalid(task, d), d)
logger.error("No such task: %s" % task)
return 1
logger.debug(1, "Executing task %s", task)
localdata = _task_data(fn, task, d)
tempdir = localdata.getVar('T', True)
if not tempdir:
bb.fatal("T variable not set, unable to build")
# Change nice level if we're asked to
nice = localdata.getVar("BB_TASK_NICE_LEVEL", True)
if nice:
curnice = os.nice(0)
nice = int(nice) - curnice
newnice = os.nice(nice)
logger.debug(1, "Renice to %s " % newnice)
bb.utils.mkdirhier(tempdir)
# Determine the logfile to generate
logfmt = localdata.getVar('BB_LOGFMT', True) or 'log.{task}.{pid}'
logbase = logfmt.format(task=task, pid=os.getpid())
# Document the order of the tasks...
logorder = os.path.join(tempdir, 'log.task_order')
try:
with open(logorder, 'a') as logorderfile:
logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase))
except OSError:
logger.exception("Opening log file '%s'", logorder)
pass
# Setup the courtesy link to the logfn
loglink = os.path.join(tempdir, 'log.{0}'.format(task))
logfn = os.path.join(tempdir, logbase)
if loglink:
bb.utils.remove(loglink)
try:
os.symlink(logbase, loglink)
except OSError:
pass
prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
class ErrorCheckHandler(logging.Handler):
def __init__(self):
self.triggered = False
logging.Handler.__init__(self, logging.ERROR)
def emit(self, record):
self.triggered = True
# Handle logfiles
si = open('/dev/null', 'r')
try:
bb.utils.mkdirhier(os.path.dirname(logfn))
logfile = open(logfn, 'w')
except OSError:
logger.exception("Opening log file '%s'", logfn)
pass
# Dup the existing fds so we dont lose them
osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
# Replace those fds with our own
os.dup2(si.fileno(), osi[1])
os.dup2(logfile.fileno(), oso[1])
os.dup2(logfile.fileno(), ose[1])
# Ensure python logging goes to the logfile
handler = logging.StreamHandler(logfile)
handler.setFormatter(logformatter)
# Always enable full debug output into task logfiles
handler.setLevel(logging.DEBUG - 2)
bblogger.addHandler(handler)
errchk = ErrorCheckHandler()
bblogger.addHandler(errchk)
localdata.setVar('BB_LOGFILE', logfn)
localdata.setVar('BB_RUNTASK', task)
flags = localdata.getVarFlags(task)
event.fire(TaskStarted(task, logfn, flags, localdata), localdata)
try:
for func in (prefuncs or '').split():
exec_func(func, localdata)
exec_func(task, localdata)
for func in (postfuncs or '').split():
exec_func(func, localdata)
except FuncFailed as exc:
if quieterr:
event.fire(TaskFailedSilent(task, logfn, localdata), localdata)
else:
errprinted = errchk.triggered
logger.error(str(exc))
event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata)
return 1
finally:
sys.stdout.flush()
sys.stderr.flush()
bblogger.removeHandler(handler)
# Restore the backup fds
os.dup2(osi[0], osi[1])
os.dup2(oso[0], oso[1])
os.dup2(ose[0], ose[1])
# Close the backup fds
os.close(osi[0])
os.close(oso[0])
os.close(ose[0])
si.close()
logfile.close()
if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
logger.debug(2, "Zero size logfn %s, removing", logfn)
bb.utils.remove(logfn)
bb.utils.remove(loglink)
event.fire(TaskSucceeded(task, logfn, localdata), localdata)
if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(task, 'selfstamp'):
make_stamp(task, localdata)
return 0
def execute_runqueue_internal(self):
"""
Run the tasks in a queue prepared by prepare_runqueue
"""
bb.msg.note(1, bb.msg.domain.RunQueue, "Executing runqueue")
self.execute_runqueue_initVars()
if len(self.runq_fnid) == 0:
# nothing to do
return []
def sigint_handler(signum, frame):
raise KeyboardInterrupt
event.fire(bb.event.StampUpdate(self.target_pairs, self.dataCache.stamp, self.cfgdata))
while True:
task = self.sched.next()
if task is not None:
fn = self.taskData.fn_index[self.runq_fnid[task]]
taskname = self.runq_task[task]
if self.check_stamp_task(task):
bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task)))
self.runq_running[task] = 1
self.task_complete(task)
self.stats.taskCompleted()
self.stats.taskSkipped()
continue
bb.msg.note(1, bb.msg.domain.RunQueue, "Running task %d of %d (ID: %s, %s)" % (self.stats.completed + self.active_builds + 1, len(self.runq_fnid), task, self.get_user_idstring(task)))
sys.stdout.flush()
sys.stderr.flush()
try:
pid = os.fork()
except OSError, e:
bb.msg.fatal(bb.msg.domain.RunQueue, "fork failed: %d (%s)" % (e.errno, e.strerror))
if pid == 0:
# Bypass master process' handling
self.master_process = False
# Stop Ctrl+C being sent to children
# signal.signal(signal.SIGINT, signal.SIG_IGN)
# Make the child the process group leader
os.setpgid(0, 0)
newsi = os.open('/dev/null', os.O_RDWR)
os.dup2(newsi, sys.stdin.fileno())
self.cooker.configuration.cmd = taskname[3:]
bb.data.setVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY", self, self.cooker.configuration.data)
try:
self.cooker.tryBuild(fn)
except bb.build.EventException:
bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
sys.exit(1)
except:
bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
raise
sys.exit(0)
self.build_pids[pid] = task
self.runq_running[task] = 1
self.active_builds = self.active_builds + 1
if self.active_builds < self.number_tasks:
continue
if self.active_builds > 0:
result = os.waitpid(-1, 0)
self.active_builds = self.active_builds - 1
task = self.build_pids[result[0]]
if result[1] != 0:
del self.build_pids[result[0]]
bb.msg.error(bb.msg.domain.RunQueue, "Task %s (%s) failed" % (task, self.get_user_idstring(task)))
self.failed_fnids.append(self.runq_fnid[task])
self.stats.taskFailed()
if not self.taskData.abort:
continue
break
self.task_complete(task)
self.stats.taskCompleted()
del self.build_pids[result[0]]
continue
return
def _exec_task(fn, task, d):
"""Execute a BB 'task'
Execution of a task involves a bit more setup than executing a function,
running it with its own local metadata, and with some useful variables set.
"""
if not data.getVarFlag(task, "task", d):
event.fire(TaskInvalid(task, d), d)
logger.error("No such task: %s" % task)
return 1
logger.debug(1, "Executing task %s", task)
localdata = _task_data(fn, task, d)
tempdir = localdata.getVar("T", True)
if not tempdir:
bb.fatal("T variable not set, unable to build")
bb.utils.mkdirhier(tempdir)
loglink = os.path.join(tempdir, "log.{0}".format(task))
logfn = os.path.join(tempdir, "log.{0}.{1}".format(task, os.getpid()))
if loglink:
bb.utils.remove(loglink)
try:
os.symlink(logfn, loglink)
except OSError:
pass
prefuncs = localdata.getVarFlag(task, "prefuncs", expand=True)
postfuncs = localdata.getVarFlag(task, "postfuncs", expand=True)
# Handle logfiles
si = file("/dev/null", "r")
try:
logfile = file(logfn, "w")
except OSError:
logger.exception("Opening log file '%s'", logfn)
pass
# Dup the existing fds so we dont lose them
osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
# Replace those fds with our own
os.dup2(si.fileno(), osi[1])
os.dup2(logfile.fileno(), oso[1])
os.dup2(logfile.fileno(), ose[1])
# Ensure python logging goes to the logfile
handler = logging.StreamHandler(logfile)
handler.setFormatter(logformatter)
bblogger.addHandler(handler)
localdata.setVar("BB_LOGFILE", logfn)
event.fire(TaskStarted(task, localdata), localdata)
try:
for func in (prefuncs or "").split():
exec_func(func, localdata)
exec_func(task, localdata)
for func in (postfuncs or "").split():
exec_func(func, localdata)
except FuncFailed as exc:
logger.error(str(exc))
event.fire(TaskFailed(task, logfn, localdata), localdata)
return 1
finally:
sys.stdout.flush()
sys.stderr.flush()
bblogger.removeHandler(handler)
# Restore the backup fds
os.dup2(osi[0], osi[1])
os.dup2(oso[0], oso[1])
os.dup2(ose[0], ose[1])
# Close the backup fds
os.close(osi[0])
os.close(oso[0])
os.close(ose[0])
si.close()
logfile.close()
if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
logger.debug(2, "Zero size logfn %s, removing", logfn)
bb.utils.remove(logfn)
bb.utils.remove(loglink)
event.fire(TaskSucceeded(task, localdata), localdata)
if not localdata.getVarFlag(task, "nostamp") and not localdata.getVarFlag(task, "selfstamp"):
make_stamp(task, localdata)
return 0
def _exec_task(fn, task, d, quieterr):
"""Execute a BB 'task'
Execution of a task involves a bit more setup than executing a function,
running it with its own local metadata, and with some useful variables set.
"""
if not data.getVarFlag(task, 'task', d):
event.fire(TaskInvalid(task, d), d)
logger.error("No such task: %s" % task)
return 1
logger.debug(1, "Executing task %s", task)
localdata = _task_data(fn, task, d)
tempdir = localdata.getVar('T', True)
if not tempdir:
bb.fatal("T variable not set, unable to build")
bb.utils.mkdirhier(tempdir)
loglink = os.path.join(tempdir, 'log.{0}'.format(task))
logfn = os.path.join(tempdir, 'log.{0}.{1}'.format(task, os.getpid()))
if loglink:
bb.utils.remove(loglink)
try:
os.symlink(logfn, loglink)
except OSError:
pass
prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
# Handle logfiles
si = file('/dev/null', 'r')
try:
logfile = file(logfn, 'w')
except OSError:
logger.exception("Opening log file '%s'", logfn)
pass
# Dup the existing fds so we dont lose them
osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
# Replace those fds with our own
os.dup2(si.fileno(), osi[1])
os.dup2(logfile.fileno(), oso[1])
os.dup2(logfile.fileno(), ose[1])
# Ensure python logging goes to the logfile
handler = logging.StreamHandler(logfile)
handler.setFormatter(logformatter)
bblogger.addHandler(handler)
localdata.setVar('BB_LOGFILE', logfn)
event.fire(TaskStarted(task, localdata), localdata)
try:
for func in (prefuncs or '').split():
exec_func(func, localdata)
exec_func(task, localdata)
for func in (postfuncs or '').split():
exec_func(func, localdata)
except FuncFailed as exc:
if not quieterr:
logger.error(str(exc))
event.fire(TaskFailed(task, logfn, localdata), localdata)
return 1
finally:
sys.stdout.flush()
sys.stderr.flush()
bblogger.removeHandler(handler)
# Restore the backup fds
os.dup2(osi[0], osi[1])
os.dup2(oso[0], oso[1])
os.dup2(ose[0], ose[1])
# Close the backup fds
os.close(osi[0])
os.close(oso[0])
os.close(ose[0])
si.close()
logfile.close()
if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
logger.debug(2, "Zero size logfn %s, removing", logfn)
bb.utils.remove(logfn)
bb.utils.remove(loglink)
event.fire(TaskSucceeded(task, localdata), localdata)
if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(
task, 'selfstamp'):
make_stamp(task, localdata)
return 0
