def load(self, name): names = self.expand(name).split(':') for n in names: log.trace('opening: %s' % (n)) if path.exists(n): try: mc = open(path.host(n), 'r') macros = self.parse(mc) mc.close() self.files += [n] return except IOError as err: pass raise error.general('opening macro file: %s' % \ (path.host(self.expand(name))))
def run_jobs(self, jobs): if path.exists(self.build_dir) and not self.options['no-clean']: log.notice('Cleaning: %s' % (self.build_dir)) path.removeall(self.build_dir) self.start = _now() self.end = _now() self.duration = self.end - self.start self.average = self.duration env_path = os.environ['PATH'] os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \ os.pathsep + os.environ['PATH'] job_count, build_job_count = jobs_option_parse(self.options['jobs']) builds = self._create_build_jobs(jobs, build_job_count) active_jobs = [] self.jobs_completed = 0 try: while len(builds) > 0 or len(active_jobs) > 0: new_jobs = job_count - len(active_jobs) if new_jobs > 0: active_jobs += builds[:new_jobs] builds = builds[new_jobs:] finished_jobs = [] for job in active_jobs: state = job.current_state() if state == 'ready': job.run() elif state != 'running': finished_jobs += [job] for job in finished_jobs: self._update_file_counts(job.get_file_counts()) job.log_output() job.clean() active_jobs.remove(job) self.jobs_completed += 1 if self.options['dry-run']: time.sleep(0) else: time.sleep(0.100) except: for job in active_jobs: try: job.kill() except: pass raise self.end = _now() os.environ['PATH'] = env_path self.duration = self.end - self.start if self.jobs_completed == 0: self.jobs_completed = 1 self._finished() self.average = self.duration / self.jobs_completed log.notice('Average BSP Build Time: %s' % (str(self.average))) log.notice('Total Time %s' % (str(self.duration)))
def _run(self, args, check=False): e = execute.capture_execution() if path.exists(self.path): cwd = self.path else: cwd = None cmd = [self.git] + args log.trace('cmd: (%s) %s' % (str(cwd), ' '.join(cmd))) exit_code, proc, output = e.spawn(cmd, cwd=path.host(cwd)) log.trace(output) if check: self._git_exit_code(exit_code, cmd, output) return exit_code, output
def _command(cmd, cwd): e = execute.capture_execution() cwd = path.abspath(cwd) log.output('>> cwd: %s' % (cwd)) log.output('> %s' % (cmd)) exit_code, proc, output = e.shell(cmd, cwd=path.host(cwd)) output_split = output.split(os.linesep) if len(output_split) >= 1 and len(output_split[0]) > 0: log.output(['> ' + l for l in output_split]) log.output('> exit: %d' % (exit_code)) if exit_code != 0: err = 'executing failure: (exit:%d) %s' % (exit_code, cmd) raise error.general(err) return output
def _load_released_version_config(): '''Local worker to load a configuration file.''' top = _top() for ver in [ os.path.join(top, 'VERSION'), os.path.join('..', 'VERSION'), rtems.configuration_file('rtems-version.ini') ]: if path.exists(path.join(ver)): v = configparser.SafeConfigParser() try: v.read(path.host(ver)) except Exception as e: raise error.general('Invalid version config format: %s: %s' % (ver, e)) return ver, v return None, None
def _execute(self, phase): exit_code = 0 cmd = self.commands[phase] try: # This should locked; not sure how to do that self.proc = execute.capture_execution(log=self.output) log.output(wrap(('run:', self.build.key(), cmd), lineend='\\')) if not self.commands['dry-run']: exit_code, proc, output = self.proc.shell( cmd, cwd=path.host(self._build_dir())) except: traceback.print_exc() self.lock.acquire() if self.proc is not None: self.proc.kill() self.lock.release() exit_code = 1 self.lock.acquire() self.proc = None self.lock.release() return exit_code == 0
def __init__(self, rtems): self.rtems = path.host(rtems) self.reset()
def build_arch_bsp(self, arch, bsp): if not self.config.bsp_present(arch, bsp): raise error.general('BSP not found: %s/%s' % (arch, bsp)) log.output('-' * 70) log.notice('] BSP: %s/%s' % (arch, bsp)) log.notice('. Creating: %s' % (self._path(arch, bsp))) self._arch_bsp_dir_clean(arch, bsp) self._arch_bsp_dir_make(arch, bsp) variations = self._variations(arch, bsp) build_set = self._build_set(variations) bsp_start = datetime.datetime.now() bsp_warnings = warnings_counter(self.rtems) env_path = os.environ['PATH'] os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \ os.pathsep + os.environ['PATH'] for bs in sorted(build_set.keys()): warnings = warnings_counter(self.rtems) start = datetime.datetime.now() log.output('- ' * 35) log.notice('. Configuring: %s' % (bs)) try: result = '+ Pass' bpath = self._build_dir(arch, bsp, bs) path.mkdir(bpath) config_cmd = self._config_command(build_set[bs], arch, bsp) cmd = config_cmd e = execute.capture_execution(log = warnings) log.output('run: ' + cmd) if self.options['dry-run']: exit_code = 0 else: exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath)) if exit_code != 0: result = '- FAIL' self.errors['configure'] += 1 log.notice('- Configure failed: %s' % (bs)) log.output('cmd failed: %s' % (cmd)) if self.options['stop-on-error']: raise error.general('Configuring %s failed' % (bs)) else: log.notice('. Building: %s' % (bs)) cmd = 'make' if 'jobs' in self.options: cmd += ' -j %s' % (self.options['jobs']) log.output('run: ' + cmd) if self.options['dry-run']: exit_code = 0 else: exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath)) if exit_code != 0: result = '- FAIL' self.errors['build'] += 1 log.notice('- FAIL: %s: %s' % (bs, self._error_str())) log.output('cmd failed: %s' % (cmd)) if self.options['stop-on-error']: raise error.general('Building %s failed' % (bs)) files = self._count_files(arch, bsp, bs) log.notice('%s: %s: warnings:%d exes:%d objs:%s libs:%d' % \ (result, bs, warnings.get(), files['exes'], files['objs'], files['libs'])) log.notice(' %s' % (self._error_str())) self.results.add(result[0] == '+', arch, bsp, config_cmd, warnings.get()) finally: end = datetime.datetime.now() if not self.options['no-clean']: log.notice('. Cleaning: %s' % (self._build_dir(arch, bsp, bs))) path.removeall(self._build_dir(arch, bsp, bs)) log.notice('^ Time %s' % (str(end - start))) log.output('Warnings Report:') log.output(warnings.report()) warnings.accumulate(bsp_warnings) warnings.accumulate(self.warnings) bsp_end = datetime.datetime.now() log.notice('^ BSP Time %s' % (str(bsp_end - bsp_start))) log.output('BSP Warnings Report:') log.output(bsp_warnings.report()) os.environ['PATH'] = env_path
def __init__(self, source_base, groups): self.lock = threading.Lock() self.source_base = path.host(source_base) self.groups = groups self.reset()
def build_arch_bsp(self, arch, bsp): if not self.config.bsp_present(arch, bsp): raise error.general('BSP not found: %s/%s' % (arch, bsp)) log.output('-' * 70) log.notice('] BSP: %s/%s' % (arch, bsp)) log.notice('. Creating: %s' % (self._path(arch, bsp))) self._arch_bsp_dir_clean(arch, bsp) self._arch_bsp_dir_make(arch, bsp) variations = self._variations(arch, bsp) build_set = self._build_set(variations) bsp_start = datetime.datetime.now() bsp_warnings = warnings_counter(self.rtems) env_path = os.environ['PATH'] os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \ os.pathsep + os.environ['PATH'] for bs in sorted(build_set.keys()): warnings = warnings_counter(self.rtems) start = datetime.datetime.now() log.output('- ' * 35) log.notice('. Configuring: %s' % (bs)) try: result = '+ Pass' bpath = self._build_dir(arch, bsp, bs) path.mkdir(bpath) config_cmd = self._config_command(build_set[bs], arch, bsp) cmd = config_cmd e = execute.capture_execution(log=warnings) log.output('run: ' + cmd) if self.options['dry-run']: exit_code = 0 else: exit_code, proc, output = e.shell(cmd, cwd=path.host(bpath)) if exit_code != 0: result = '- FAIL' self.errors['configure'] += 1 log.notice('- Configure failed: %s' % (bs)) log.output('cmd failed: %s' % (cmd)) if self.options['stop-on-error']: raise error.general('Configuring %s failed' % (bs)) else: log.notice('. Building: %s' % (bs)) cmd = 'make' if 'jobs' in self.options: cmd += ' -j %s' % (self.options['jobs']) log.output('run: ' + cmd) if self.options['dry-run']: exit_code = 0 else: exit_code, proc, output = e.shell(cmd, cwd=path.host(bpath)) if exit_code != 0: result = '- FAIL' self.errors['build'] += 1 log.notice('- FAIL: %s: %s' % (bs, self._error_str())) log.output('cmd failed: %s' % (cmd)) if self.options['stop-on-error']: raise error.general('Building %s failed' % (bs)) files = self._count_files(arch, bsp, bs) log.notice('%s: %s: warnings:%d exes:%d objs:%s libs:%d' % \ (result, bs, warnings.get(), files['exes'], files['objs'], files['libs'])) log.notice(' %s' % (self._error_str())) self.results.add(result[0] == '+', arch, bsp, config_cmd, warnings.get()) finally: end = datetime.datetime.now() if not self.options['no-clean']: log.notice('. Cleaning: %s' % (self._build_dir(arch, bsp, bs))) path.removeall(self._build_dir(arch, bsp, bs)) log.notice('^ Time %s' % (str(end - start))) log.output('Warnings Report:') log.output(warnings.report()) warnings.accumulate(bsp_warnings) warnings.accumulate(self.warnings) bsp_end = datetime.datetime.now() log.notice('^ BSP Time %s' % (str(bsp_end - bsp_start))) log.output('BSP Warnings Report:') log.output(bsp_warnings.report()) os.environ['PATH'] = env_path
def clone(self, url, _path): ec, output = self._run(['clone', url, path.host(_path)], check=True)
def load(self, name): def common_end(left, right): end = '' while len(left) and len(right): if left[-1] != right[-1]: return end end = left[-1] + end left = left[:-1] right = right[:-1] return end if self.load_depth == 0: self.in_error = False self.lc = 0 self.name = name self.conditionals = {} self.load_depth += 1 save_name = self.name save_lc = self.lc self.name = name self.lc = 0 # # Locate the config file. Expand any macros then add the # extension. Check if the file exists, therefore directly # referenced. If not see if the file contains ':' or the path # separator. If it does split the path else use the standard config dir # path in the defaults. # exname = self.expand(name) # # Macro could add an extension. # if exname.endswith('.cfg'): configname = exname else: configname = '%s.cfg' % (exname) name = '%s.cfg' % (name) if ':' in configname: cfgname = path.basename(configname) else: cfgname = common_end(configname, name) if not path.exists(configname): if ':' in configname: configdirs = path.dirname(configname).split(':') else: configdirs = self.define('_configdir').split(':') for cp in configdirs: configname = path.join(path.abspath(cp), cfgname) if path.exists(configname): break configname = None if configname is None: raise error.general('no config file found: %s' % (cfgname)) try: log.trace('config: %s: _open: %s' % (self.init_name, path.host(configname))) config = open(path.host(configname), 'r') except IOError as err: raise error.general('error opening config file: %s' % (path.host(configname))) self.configpath += [configname] self._includes += [configname] try: dir = None info = None data = [] while True: r = self._parse(config, dir, info) if r[0] == 'control': if r[1] == '%end': break log.warning("unexpected '%s'" % (r[1])) elif r[0] == 'directive': if r[1] == '%include': self.load(r[2][0]) continue dir, info, data = self._process_directive(r, dir, info, data) elif r[0] == 'data': dir, info, data = self._process_data(r, dir, info, data) else: self._error("%d: invalid parse state: '%s" % (self.lc, r[0])) if dir is not None: self._directive_extend(dir, data) except: config.close() raise config.close() self.name = save_name self.lc = save_lc self.load_depth -= 1