def execute_runqueue_initVars(self): self.stats = RunQueueStats(len(self.runq_fnid)) self.runq_buildable = [] self.runq_running = [] self.runq_complete = [] self.build_pids = {} self.build_pipes = {} self.failed_fnids = [] # Mark initial buildable tasks for task in range(self.stats.total): self.runq_running.append(0) self.runq_complete.append(0) if len(self.runq_depends[task]) == 0: self.runq_buildable.append(1) else: self.runq_buildable.append(0) self.state = runQueueRunning event.fire( bb.event.StampUpdate(self.target_pairs, self.dataCache.stamp), self.cfgData)
def exec_task(fn, task, d, profile = False): try: quieterr = False if d.getVarFlag(task, "quieterrors") is not None: quieterr = True if profile: profname = "profile-%s.log" % (d.getVar("PN", True) + "-" + task) try: import cProfile as profile except: import profile prof = profile.Profile() ret = profile.Profile.runcall(prof, _exec_task, fn, task, d, quieterr) prof.dump_stats(profname) bb.utils.process_profilelog(profname) return ret else: return _exec_task(fn, task, d, quieterr) except Exception: from traceback import format_exc if not quieterr: logger.error("Build of %s failed" % (task)) logger.error(format_exc()) failedevent = TaskFailed(task, None, d, True) event.fire(failedevent, d) return 1
def exec_task(fn, task, d, profile = False): try: quieterr = False if d.getVarFlag(task, "quieterrors", False) is not None: quieterr = True if profile: profname = "profile-%s.log" % (d.getVar("PN") + "-" + task) try: import cProfile as profile except: import profile prof = profile.Profile() ret = profile.Profile.runcall(prof, _exec_task, fn, task, d, quieterr) prof.dump_stats(profname) bb.utils.process_profilelog(profname) return ret else: return _exec_task(fn, task, d, quieterr) except Exception: from traceback import format_exc if not quieterr: logger.error("Build of %s failed" % (task)) logger.error(format_exc()) failedevent = TaskFailed(task, None, d, True) event.fire(failedevent, d) return 1
def exec_task(fn, task, d): try: return _exec_task(fn, task, d) except Exception: from traceback import format_exc logger.error("Build of %s failed" % (task)) logger.error(format_exc()) failedevent = TaskFailed(task, None, d) event.fire(failedevent, d) return 1
def exec_task(fn, task, d): try: quieterr = False if d.getVarFlag(task, "quieterrors") is not None: quieterr = True return _exec_task(fn, task, d, quieterr) except Exception: from traceback import format_exc if not quieterr: logger.error("Build of %s failed" % (task)) logger.error(format_exc()) failedevent = TaskFailed(task, None, d) event.fire(failedevent, d) return 1
def exec_task(fn, task, d): try: quieterr = False if d.getVarFlag(task, "quieterrors") is not None: quieterr = True return _exec_task(fn, task, d, quieterr) except Exception: from traceback import format_exc if not quieterr: logger.error("Build of %s failed" % (task)) logger.error(format_exc()) failedevent = TaskFailed(task, None, d, True) event.fire(failedevent, d) return 1
def exec_task(task, d): """Execute an BB 'task' The primary difference between executing a task versus executing a function is that a task exists in the task digraph, and therefore has dependencies amongst other tasks.""" # Check whther this is a valid task if not data.getVarFlag(task, 'task', d): raise EventException("No such task", InvalidTask(task, d)) try: bb.msg.debug(1, bb.msg.domain.Build, "Executing task %s" % task) old_overrides = data.getVar('OVERRIDES', d, 0) localdata = data.createCopy(d) data.setVar('OVERRIDES', 'task-%s:%s' % (task[3:], old_overrides), localdata) data.update_data(localdata) data.expandKeys(localdata) event.fire(TaskStarted(task, localdata)) exec_func(task, localdata) event.fire(TaskSucceeded(task, localdata)) except FuncFailed, message: # Try to extract the optional logfile try: (msg, logfile) = message except: logfile = None msg = message bb.msg.note(1, bb.msg.domain.Build, "Task failed: %s" % message ) failedevent = TaskFailed(msg, logfile, task, d) event.fire(failedevent) raise EventException("Function failed in task: %s" % message, failedevent)
def execute(graph, item): if data.getVarFlag(item, 'task', d): if item in task_cache: return 1 if task != item: # deeper than toplevel, exec w/ deps exec_task(item, d) return 1 try: bb.msg.debug(1, bb.msg.domain.Build, "Executing task %s" % item) old_overrides = data.getVar('OVERRIDES', d, 0) localdata = data.createCopy(d) data.setVar('OVERRIDES', 'task_%s:%s' % (item, old_overrides), localdata) data.update_data(localdata) event.fire(TaskStarted(item, localdata)) exec_func(item, localdata) event.fire(TaskSucceeded(item, localdata)) task_cache.append(item) data.setVar('_task_cache', task_cache, d) except FuncFailed, reason: bb.msg.note(1, bb.msg.domain.Build, "Task failed: %s" % reason ) failedevent = TaskFailed(item, d) event.fire(failedevent) raise EventException("Function failed in task: %s" % reason, failedevent)
def exec_task(fn, task, d): """Execute a BB 'task' Execution of a task involves a bit more setup than executing a function, running it with its own local metadata, and with some useful variables set. """ # Check whther this is a valid task if not data.getVarFlag(task, 'task', d): raise InvalidTask(task, d) try: logger.debug(1, "Executing task %s", task) old_overrides = data.getVar('OVERRIDES', d, 0) localdata = data.createCopy(d) data.setVar('OVERRIDES', 'task-%s:%s' % (task[3:], old_overrides), localdata) data.update_data(localdata) data.expandKeys(localdata) data.setVar('BB_FILENAME', fn, d) data.setVar('BB_CURRENTTASK', task[3:], d) event.fire(TaskStarted(task, localdata), localdata) exec_func(task, localdata) event.fire(TaskSucceeded(task, localdata), localdata) except FuncFailed as exc: event.fire(TaskFailed(exc.name, exc.logfile, localdata), localdata) raise # make stamp, or cause event and raise exception if not data.getVarFlag(task, 'nostamp', d) and not data.getVarFlag(task, 'selfstamp', d): make_stamp(task, d)
def exec_task(task, d): """Execute an BB 'task' The primary difference between executing a task versus executing a function is that a task exists in the task digraph, and therefore has dependencies amongst other tasks.""" # Check whther this is a valid task if not data.getVarFlag(task, 'task', d): raise EventException("No such task", InvalidTask(task, d)) try: bb.msg.debug(1, bb.msg.domain.Build, "Executing task %s" % task) old_overrides = data.getVar('OVERRIDES', d, 0) localdata = data.createCopy(d) data.setVar('OVERRIDES', 'task-%s:%s' % (task[3:], old_overrides), localdata) data.update_data(localdata) data.expandKeys(localdata) event.fire(TaskStarted(task, localdata), localdata) exec_func(task, localdata) event.fire(TaskSucceeded(task, localdata), localdata) except FuncFailed, message: # Try to extract the optional logfile try: (msg, logfile) = message except: logfile = None msg = message bb.msg.note(1, bb.msg.domain.Build, "Task failed: %s" % message) failedevent = TaskFailed(msg, logfile, task, d) event.fire(failedevent, d) raise EventException("Function failed in task: %s" % message, failedevent)
def execute(graph, item): if data.getVarFlag(item, 'task', d): if item in task_cache: return 1 if task != item: # deeper than toplevel, exec w/ deps exec_task(item, d) return 1 try: debug(1, "Executing task %s" % item) old_overrides = data.getVar('OVERRIDES', d, 0) from copy import deepcopy localdata = deepcopy(d) data.setVar('OVERRIDES', 'task_%s:%s' % (item, old_overrides), localdata) data.update_data(localdata) event.fire(TaskStarted(item, localdata)) exec_func(item, localdata) event.fire(TaskSucceeded(item, localdata)) task_cache.append(item) except FuncFailed, reason: note( "Task failed: %s" % reason ) failedevent = TaskFailed(item, d) event.fire(failedevent) raise EventException(None, failedevent)
def execute_runqueue_initVars(self): self.stats = RunQueueStats(len(self.runq_fnid)) self.runq_buildable = [] self.runq_running = [] self.runq_complete = [] self.build_pids = {} self.failed_fnids = [] # Mark initial buildable tasks for task in range(self.stats.total): self.runq_running.append(0) self.runq_complete.append(0) if len(self.runq_depends[task]) == 0: self.runq_buildable.append(1) else: self.runq_buildable.append(0) self.state = runQueueRunning event.fire(bb.event.StampUpdate(self.target_pairs, self.dataCache.stamp, self.cfgData))
def exec_task(fn, task, d): """Execute a BB 'task' Execution of a task involves a bit more setup than executing a function, running it with its own local metadata, and with some useful variables set. """ if not data.getVarFlag(task, 'task', d): raise InvalidTask(task, d) logger.debug(1, "Executing task %s", task) localdata = _task_data(fn, task, d) tempdir = localdata.getVar('T', True) if not tempdir: bb.fatal("T variable not set, unable to build") bb.utils.mkdirhier(tempdir) loglink = os.path.join(tempdir, 'log.{0}'.format(task)) logfn = os.path.join(tempdir, 'log.{0}.{1}'.format(task, os.getpid())) if loglink: bb.utils.remove(loglink) try: os.symlink(logfn, loglink) except OSError: pass prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) logfile = open(logfn, 'w') event.fire(TaskStarted(task, localdata), localdata) try: for func in (prefuncs or '').split(): exec_func(func, localdata, logfile=logfile) exec_func(task, localdata, logfile=logfile) for func in (postfuncs or '').split(): exec_func(func, localdata, logfile=logfile) except FuncFailed as exc: event.fire(TaskFailed(exc.name, exc.logfile, localdata), localdata) raise finally: logfile.close() if os.path.exists(logfn) and os.path.getsize(logfn) == 0: logger.debug(2, "Zero size logfn %s, removing", logfn) bb.utils.remove(logfn) bb.utils.remove(loglink) event.fire(TaskSucceeded(task, localdata), localdata) if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag( task, 'selfstamp'): make_stamp(task, localdata)
def exec_task(fn, task, d): """Execute a BB 'task' Execution of a task involves a bit more setup than executing a function, running it with its own local metadata, and with some useful variables set. """ if not data.getVarFlag(task, 'task', d): raise InvalidTask(task, d) logger.debug(1, "Executing task %s", task) localdata = _task_data(fn, task, d) tempdir = localdata.getVar('T', True) if not tempdir: bb.fatal("T variable not set, unable to build") bb.utils.mkdirhier(tempdir) loglink = os.path.join(tempdir, 'log.{0}'.format(task)) logfn = os.path.join(tempdir, 'log.{0}.{1}'.format(task, os.getpid())) if loglink: bb.utils.remove(loglink) try: os.symlink(logfn, loglink) except OSError: pass prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) logfile = open(logfn, 'w') event.fire(TaskStarted(task, localdata), localdata) try: for func in (prefuncs or '').split(): exec_func(func, localdata, logfile=logfile) exec_func(task, localdata, logfile=logfile) for func in (postfuncs or '').split(): exec_func(func, localdata, logfile=logfile) except FuncFailed as exc: event.fire(TaskFailed(exc.name, exc.logfile, localdata), localdata) raise finally: logfile.close() if os.path.exists(logfn) and os.path.getsize(logfn) == 0: logger.debug(2, "Zero size logfn %s, removing", logfn) bb.utils.remove(logfn) bb.utils.remove(loglink) event.fire(TaskSucceeded(task, localdata), localdata) if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(task, 'selfstamp'): make_stamp(task, localdata)
def execute(graph, item): if data.getVarFlag(item, 'task', d): if item in task_cache: return 1 if task != item: # deeper than toplevel, exec w/ deps exec_task(item, d) return 1 try: debug(1, "Executing task %s" % item) event.fire(TaskStarted(item, d)) exec_func(item, d) event.fire(TaskSucceeded(item, d)) task_cache.append(item) except FuncFailed, reason: note( "Task failed: %s" % reason ) failedevent = TaskFailed(item, d) event.fire(failedevent) raise EventException(None, failedevent)
def execute_runqueue_internal(self): """ Run the tasks in a queue prepared by prepare_runqueue """ bb.msg.note(1, bb.msg.domain.RunQueue, "Executing runqueue") self.execute_runqueue_initVars() if len(self.runq_fnid) == 0: # nothing to do return [] def sigint_handler(signum, frame): raise KeyboardInterrupt event.fire( bb.event.StampUpdate(self.target_pairs, self.dataCache.stamp, self.cfgdata)) while True: task = self.sched.next() if task is not None: fn = self.taskData.fn_index[self.runq_fnid[task]] taskname = self.runq_task[task] if self.check_stamp_task(task): bb.msg.debug( 2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task))) self.runq_running[task] = 1 self.task_complete(task) self.stats.taskCompleted() self.stats.taskSkipped() continue bb.msg.note( 1, bb.msg.domain.RunQueue, "Running task %d of %d (ID: %s, %s)" % (self.stats.completed + self.active_builds + 1, len(self.runq_fnid), task, self.get_user_idstring(task))) sys.stdout.flush() sys.stderr.flush() try: pid = os.fork() except OSError, e: bb.msg.fatal( bb.msg.domain.RunQueue, "fork failed: %d (%s)" % (e.errno, e.strerror)) if pid == 0: # Bypass master process' handling self.master_process = False # Stop Ctrl+C being sent to children # signal.signal(signal.SIGINT, signal.SIG_IGN) # Make the child the process group leader os.setpgid(0, 0) newsi = os.open('/dev/null', os.O_RDWR) os.dup2(newsi, sys.stdin.fileno()) self.cooker.configuration.cmd = taskname[3:] bb.data.setVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY", self, self.cooker.configuration.data) try: self.cooker.tryBuild(fn) except bb.build.EventException: bb.msg.error( bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed") sys.exit(1) except: bb.msg.error( bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed") raise sys.exit(0) self.build_pids[pid] = task self.runq_running[task] = 1 self.active_builds = self.active_builds + 1 if self.active_builds < self.number_tasks: continue if self.active_builds > 0: result = os.waitpid(-1, 0) self.active_builds = self.active_builds - 1 task = self.build_pids[result[0]] if result[1] != 0: del self.build_pids[result[0]] bb.msg.error( bb.msg.domain.RunQueue, "Task %s (%s) failed" % (task, self.get_user_idstring(task))) self.failed_fnids.append(self.runq_fnid[task]) self.stats.taskFailed() if not self.taskData.abort: continue break self.task_complete(task) self.stats.taskCompleted() del self.build_pids[result[0]] continue return
def _exec_task(fn, task, d, quieterr): """Execute a BB 'task' Execution of a task involves a bit more setup than executing a function, running it with its own local metadata, and with some useful variables set. """ if not d.getVarFlag(task, 'task', False): event.fire(TaskInvalid(task, d), d) logger.error("No such task: %s" % task) return 1 logger.debug(1, "Executing task %s", task) localdata = _task_data(fn, task, d) tempdir = localdata.getVar('T') if not tempdir: bb.fatal("T variable not set, unable to build") # Change nice level if we're asked to nice = localdata.getVar("BB_TASK_NICE_LEVEL") if nice: curnice = os.nice(0) nice = int(nice) - curnice newnice = os.nice(nice) logger.debug(1, "Renice to %s " % newnice) ionice = localdata.getVar("BB_TASK_IONICE_LEVEL") if ionice: try: cls, prio = ionice.split(".", 1) bb.utils.ioprio_set(os.getpid(), int(cls), int(prio)) except: bb.warn("Invalid ionice level %s" % ionice) bb.utils.mkdirhier(tempdir) # Determine the logfile to generate logfmt = localdata.getVar('BB_LOGFMT') or 'log.{task}.{pid}' logbase = logfmt.format(task=task, pid=os.getpid()) # Document the order of the tasks... logorder = os.path.join(tempdir, 'log.task_order') try: with open(logorder, 'a') as logorderfile: logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase)) except OSError: logger.exception("Opening log file '%s'", logorder) pass # Setup the courtesy link to the logfn loglink = os.path.join(tempdir, 'log.{0}'.format(task)) logfn = os.path.join(tempdir, logbase) if loglink: bb.utils.remove(loglink) try: os.symlink(logbase, loglink) except OSError: pass prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) class ErrorCheckHandler(logging.Handler): def __init__(self): self.triggered = False logging.Handler.__init__(self, logging.ERROR) def emit(self, record): if getattr(record, 'forcelog', False): self.triggered = False else: self.triggered = True # Handle logfiles try: bb.utils.mkdirhier(os.path.dirname(logfn)) logfile = open(logfn, 'w') except OSError: logger.exception("Opening log file '%s'", logfn) pass # Dup the existing fds so we dont lose them osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] # Replace those fds with our own with open('/dev/null', 'r') as si: os.dup2(si.fileno(), osi[1]) os.dup2(logfile.fileno(), oso[1]) os.dup2(logfile.fileno(), ose[1]) # Ensure Python logging goes to the logfile handler = logging.StreamHandler(logfile) handler.setFormatter(logformatter) # Always enable full debug output into task logfiles handler.setLevel(logging.DEBUG - 2) bblogger.addHandler(handler) errchk = ErrorCheckHandler() bblogger.addHandler(errchk) localdata.setVar('BB_LOGFILE', logfn) localdata.setVar('BB_RUNTASK', task) localdata.setVar('BB_TASK_LOGGER', bblogger) flags = localdata.getVarFlags(task) try: try: event.fire(TaskStarted(task, logfn, flags, localdata), localdata) except (bb.BBHandledException, SystemExit): return 1 except FuncFailed as exc: logger.error(str(exc)) return 1 try: for func in (prefuncs or '').split(): exec_func(func, localdata) exec_func(task, localdata) for func in (postfuncs or '').split(): exec_func(func, localdata) except FuncFailed as exc: if quieterr: event.fire(TaskFailedSilent(task, logfn, localdata), localdata) else: errprinted = errchk.triggered logger.error(str(exc)) event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata) return 1 except bb.BBHandledException: event.fire(TaskFailed(task, logfn, localdata, True), localdata) return 1 finally: sys.stdout.flush() sys.stderr.flush() bblogger.removeHandler(handler) # Restore the backup fds os.dup2(osi[0], osi[1]) os.dup2(oso[0], oso[1]) os.dup2(ose[0], ose[1]) # Close the backup fds os.close(osi[0]) os.close(oso[0]) os.close(ose[0]) logfile.close() if os.path.exists(logfn) and os.path.getsize(logfn) == 0: logger.debug(2, "Zero size logfn %s, removing", logfn) bb.utils.remove(logfn) bb.utils.remove(loglink) event.fire(TaskSucceeded(task, logfn, localdata), localdata) if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False): make_stamp(task, localdata) return 0
def _exec_task(fn, task, d, quieterr): """Execute a BB 'task' Execution of a task involves a bit more setup than executing a function, running it with its own local metadata, and with some useful variables set. """ if not data.getVarFlag(task, 'task', d): event.fire(TaskInvalid(task, d), d) logger.error("No such task: %s" % task) return 1 logger.debug(1, "Executing task %s", task) localdata = _task_data(fn, task, d) tempdir = localdata.getVar('T', True) if not tempdir: bb.fatal("T variable not set, unable to build") bb.utils.mkdirhier(tempdir) loglink = os.path.join(tempdir, 'log.{0}'.format(task)) logfn = os.path.join(tempdir, 'log.{0}.{1}'.format(task, os.getpid())) if loglink: bb.utils.remove(loglink) try: os.symlink(logfn, loglink) except OSError: pass prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) class ErrorCheckHandler(logging.Handler): def __init__(self): self.triggered = False logging.Handler.__init__(self, logging.ERROR) def emit(self, record): self.triggered = True # Handle logfiles si = file('/dev/null', 'r') try: logfile = file(logfn, 'w') except OSError: logger.exception("Opening log file '%s'", logfn) pass # Dup the existing fds so we dont lose them osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] # Replace those fds with our own os.dup2(si.fileno(), osi[1]) os.dup2(logfile.fileno(), oso[1]) os.dup2(logfile.fileno(), ose[1]) # Ensure python logging goes to the logfile handler = logging.StreamHandler(logfile) handler.setFormatter(logformatter) # Always enable full debug output into task logfiles handler.setLevel(logging.DEBUG - 2) bblogger.addHandler(handler) errchk = ErrorCheckHandler() bblogger.addHandler(errchk) localdata.setVar('BB_LOGFILE', logfn) event.fire(TaskStarted(task, localdata), localdata) try: for func in (prefuncs or '').split(): exec_func(func, localdata) exec_func(task, localdata) for func in (postfuncs or '').split(): exec_func(func, localdata) except FuncFailed as exc: if quieterr: event.fire(TaskFailedSilent(task, logfn, localdata), localdata) else: errprinted = errchk.triggered logger.error(str(exc)) event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata) return 1 finally: sys.stdout.flush() sys.stderr.flush() bblogger.removeHandler(handler) # Restore the backup fds os.dup2(osi[0], osi[1]) os.dup2(oso[0], oso[1]) os.dup2(ose[0], ose[1]) # Close the backup fds os.close(osi[0]) os.close(oso[0]) os.close(ose[0]) si.close() logfile.close() if os.path.exists(logfn) and os.path.getsize(logfn) == 0: logger.debug(2, "Zero size logfn %s, removing", logfn) bb.utils.remove(logfn) bb.utils.remove(loglink) event.fire(TaskSucceeded(task, localdata), localdata) if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(task, 'selfstamp'): make_stamp(task, localdata) return 0
def _exec_task(fn, task, d, quieterr): """Execute a BB 'task' Execution of a task involves a bit more setup than executing a function, running it with its own local metadata, and with some useful variables set. """ if not d.getVarFlag(task, 'task'): event.fire(TaskInvalid(task, d), d) logger.error("No such task: %s" % task) return 1 logger.debug(1, "Executing task %s", task) localdata = _task_data(fn, task, d) tempdir = localdata.getVar('T', True) if not tempdir: bb.fatal("T variable not set, unable to build") # Change nice level if we're asked to nice = localdata.getVar("BB_TASK_NICE_LEVEL", True) if nice: curnice = os.nice(0) nice = int(nice) - curnice newnice = os.nice(nice) logger.debug(1, "Renice to %s " % newnice) bb.utils.mkdirhier(tempdir) # Determine the logfile to generate logfmt = localdata.getVar('BB_LOGFMT', True) or 'log.{task}.{pid}' logbase = logfmt.format(task=task, pid=os.getpid()) # Document the order of the tasks... logorder = os.path.join(tempdir, 'log.task_order') try: with open(logorder, 'a') as logorderfile: logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase)) except OSError: logger.exception("Opening log file '%s'", logorder) pass # Setup the courtesy link to the logfn loglink = os.path.join(tempdir, 'log.{0}'.format(task)) logfn = os.path.join(tempdir, logbase) if loglink: bb.utils.remove(loglink) try: os.symlink(logbase, loglink) except OSError: pass prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) class ErrorCheckHandler(logging.Handler): def __init__(self): self.triggered = False logging.Handler.__init__(self, logging.ERROR) def emit(self, record): self.triggered = True # Handle logfiles si = open('/dev/null', 'r') try: bb.utils.mkdirhier(os.path.dirname(logfn)) logfile = open(logfn, 'w') except OSError: logger.exception("Opening log file '%s'", logfn) pass # Dup the existing fds so we dont lose them osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] # Replace those fds with our own os.dup2(si.fileno(), osi[1]) os.dup2(logfile.fileno(), oso[1]) os.dup2(logfile.fileno(), ose[1]) # Ensure python logging goes to the logfile handler = logging.StreamHandler(logfile) handler.setFormatter(logformatter) # Always enable full debug output into task logfiles handler.setLevel(logging.DEBUG - 2) bblogger.addHandler(handler) errchk = ErrorCheckHandler() bblogger.addHandler(errchk) localdata.setVar('BB_LOGFILE', logfn) localdata.setVar('BB_RUNTASK', task) flags = localdata.getVarFlags(task) event.fire(TaskStarted(task, logfn, flags, localdata), localdata) try: for func in (prefuncs or '').split(): exec_func(func, localdata) exec_func(task, localdata) for func in (postfuncs or '').split(): exec_func(func, localdata) except FuncFailed as exc: if quieterr: event.fire(TaskFailedSilent(task, logfn, localdata), localdata) else: errprinted = errchk.triggered logger.error(str(exc)) event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata) return 1 finally: sys.stdout.flush() sys.stderr.flush() bblogger.removeHandler(handler) # Restore the backup fds os.dup2(osi[0], osi[1]) os.dup2(oso[0], oso[1]) os.dup2(ose[0], ose[1]) # Close the backup fds os.close(osi[0]) os.close(oso[0]) os.close(ose[0]) si.close() logfile.close() if os.path.exists(logfn) and os.path.getsize(logfn) == 0: logger.debug(2, "Zero size logfn %s, removing", logfn) bb.utils.remove(logfn) bb.utils.remove(loglink) event.fire(TaskSucceeded(task, logfn, localdata), localdata) if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(task, 'selfstamp'): make_stamp(task, localdata) return 0
def execute_runqueue_internal(self): """ Run the tasks in a queue prepared by prepare_runqueue """ bb.msg.note(1, bb.msg.domain.RunQueue, "Executing runqueue") self.execute_runqueue_initVars() if len(self.runq_fnid) == 0: # nothing to do return [] def sigint_handler(signum, frame): raise KeyboardInterrupt event.fire(bb.event.StampUpdate(self.target_pairs, self.dataCache.stamp, self.cfgdata)) while True: task = self.sched.next() if task is not None: fn = self.taskData.fn_index[self.runq_fnid[task]] taskname = self.runq_task[task] if self.check_stamp_task(task): bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task))) self.runq_running[task] = 1 self.task_complete(task) self.stats.taskCompleted() self.stats.taskSkipped() continue bb.msg.note(1, bb.msg.domain.RunQueue, "Running task %d of %d (ID: %s, %s)" % (self.stats.completed + self.active_builds + 1, len(self.runq_fnid), task, self.get_user_idstring(task))) sys.stdout.flush() sys.stderr.flush() try: pid = os.fork() except OSError, e: bb.msg.fatal(bb.msg.domain.RunQueue, "fork failed: %d (%s)" % (e.errno, e.strerror)) if pid == 0: # Bypass master process' handling self.master_process = False # Stop Ctrl+C being sent to children # signal.signal(signal.SIGINT, signal.SIG_IGN) # Make the child the process group leader os.setpgid(0, 0) newsi = os.open('/dev/null', os.O_RDWR) os.dup2(newsi, sys.stdin.fileno()) self.cooker.configuration.cmd = taskname[3:] bb.data.setVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY", self, self.cooker.configuration.data) try: self.cooker.tryBuild(fn) except bb.build.EventException: bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed") sys.exit(1) except: bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed") raise sys.exit(0) self.build_pids[pid] = task self.runq_running[task] = 1 self.active_builds = self.active_builds + 1 if self.active_builds < self.number_tasks: continue if self.active_builds > 0: result = os.waitpid(-1, 0) self.active_builds = self.active_builds - 1 task = self.build_pids[result[0]] if result[1] != 0: del self.build_pids[result[0]] bb.msg.error(bb.msg.domain.RunQueue, "Task %s (%s) failed" % (task, self.get_user_idstring(task))) self.failed_fnids.append(self.runq_fnid[task]) self.stats.taskFailed() if not self.taskData.abort: continue break self.task_complete(task) self.stats.taskCompleted() del self.build_pids[result[0]] continue return
def _exec_task(fn, task, d): """Execute a BB 'task' Execution of a task involves a bit more setup than executing a function, running it with its own local metadata, and with some useful variables set. """ if not data.getVarFlag(task, "task", d): event.fire(TaskInvalid(task, d), d) logger.error("No such task: %s" % task) return 1 logger.debug(1, "Executing task %s", task) localdata = _task_data(fn, task, d) tempdir = localdata.getVar("T", True) if not tempdir: bb.fatal("T variable not set, unable to build") bb.utils.mkdirhier(tempdir) loglink = os.path.join(tempdir, "log.{0}".format(task)) logfn = os.path.join(tempdir, "log.{0}.{1}".format(task, os.getpid())) if loglink: bb.utils.remove(loglink) try: os.symlink(logfn, loglink) except OSError: pass prefuncs = localdata.getVarFlag(task, "prefuncs", expand=True) postfuncs = localdata.getVarFlag(task, "postfuncs", expand=True) # Handle logfiles si = file("/dev/null", "r") try: logfile = file(logfn, "w") except OSError: logger.exception("Opening log file '%s'", logfn) pass # Dup the existing fds so we dont lose them osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] # Replace those fds with our own os.dup2(si.fileno(), osi[1]) os.dup2(logfile.fileno(), oso[1]) os.dup2(logfile.fileno(), ose[1]) # Ensure python logging goes to the logfile handler = logging.StreamHandler(logfile) handler.setFormatter(logformatter) bblogger.addHandler(handler) localdata.setVar("BB_LOGFILE", logfn) event.fire(TaskStarted(task, localdata), localdata) try: for func in (prefuncs or "").split(): exec_func(func, localdata) exec_func(task, localdata) for func in (postfuncs or "").split(): exec_func(func, localdata) except FuncFailed as exc: logger.error(str(exc)) event.fire(TaskFailed(task, logfn, localdata), localdata) return 1 finally: sys.stdout.flush() sys.stderr.flush() bblogger.removeHandler(handler) # Restore the backup fds os.dup2(osi[0], osi[1]) os.dup2(oso[0], oso[1]) os.dup2(ose[0], ose[1]) # Close the backup fds os.close(osi[0]) os.close(oso[0]) os.close(ose[0]) si.close() logfile.close() if os.path.exists(logfn) and os.path.getsize(logfn) == 0: logger.debug(2, "Zero size logfn %s, removing", logfn) bb.utils.remove(logfn) bb.utils.remove(loglink) event.fire(TaskSucceeded(task, localdata), localdata) if not localdata.getVarFlag(task, "nostamp") and not localdata.getVarFlag(task, "selfstamp"): make_stamp(task, localdata) return 0
def _exec_task(fn, task, d, quieterr): """Execute a BB 'task' Execution of a task involves a bit more setup than executing a function, running it with its own local metadata, and with some useful variables set. """ if not data.getVarFlag(task, 'task', d): event.fire(TaskInvalid(task, d), d) logger.error("No such task: %s" % task) return 1 logger.debug(1, "Executing task %s", task) localdata = _task_data(fn, task, d) tempdir = localdata.getVar('T', True) if not tempdir: bb.fatal("T variable not set, unable to build") bb.utils.mkdirhier(tempdir) loglink = os.path.join(tempdir, 'log.{0}'.format(task)) logfn = os.path.join(tempdir, 'log.{0}.{1}'.format(task, os.getpid())) if loglink: bb.utils.remove(loglink) try: os.symlink(logfn, loglink) except OSError: pass prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) # Handle logfiles si = file('/dev/null', 'r') try: logfile = file(logfn, 'w') except OSError: logger.exception("Opening log file '%s'", logfn) pass # Dup the existing fds so we dont lose them osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] # Replace those fds with our own os.dup2(si.fileno(), osi[1]) os.dup2(logfile.fileno(), oso[1]) os.dup2(logfile.fileno(), ose[1]) # Ensure python logging goes to the logfile handler = logging.StreamHandler(logfile) handler.setFormatter(logformatter) bblogger.addHandler(handler) localdata.setVar('BB_LOGFILE', logfn) event.fire(TaskStarted(task, localdata), localdata) try: for func in (prefuncs or '').split(): exec_func(func, localdata) exec_func(task, localdata) for func in (postfuncs or '').split(): exec_func(func, localdata) except FuncFailed as exc: if not quieterr: logger.error(str(exc)) event.fire(TaskFailed(task, logfn, localdata), localdata) return 1 finally: sys.stdout.flush() sys.stderr.flush() bblogger.removeHandler(handler) # Restore the backup fds os.dup2(osi[0], osi[1]) os.dup2(oso[0], oso[1]) os.dup2(ose[0], ose[1]) # Close the backup fds os.close(osi[0]) os.close(oso[0]) os.close(ose[0]) si.close() logfile.close() if os.path.exists(logfn) and os.path.getsize(logfn) == 0: logger.debug(2, "Zero size logfn %s, removing", logfn) bb.utils.remove(logfn) bb.utils.remove(loglink) event.fire(TaskSucceeded(task, localdata), localdata) if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag( task, 'selfstamp'): make_stamp(task, localdata) return 0