def get_package_from_registry(package): for d in ['tmp', 'vendor']: try: os.makedirs(d) except FileExistsError: pass db_path = os.path.join('tmp', 'registry.db') # download the registry database if not os.path.exists(db_path): urllib.request.urlretrieve(REGISTRY_DB_URL, db_path) # look for the package db = sqlite3.connect(db_path) c = db.execute('SELECT type, uri FROM packages WHERE name = ?', (package, )) r = c.fetchone() if r is None: error("unknown package '{}' in registry".format(package)) type_, uri = r # download it if type_ == 'github': name = 'github-{}'.format(uri.replace('/', '-')) repo_path = os.path.join('vendor', name) if not os.path.exists(repo_path): repo_url = 'git://github.com/{}'.format(uri) progress('cloning ' + repo_url) subprocess.run(['git', 'clone', repo_url, repo_path]) # TODO: use pure python
def run_make(make, makefile, prettify=False): """Executes make(1)""" makeflags = ['-j' + str(multiprocessing.cpu_count())] try: p = subprocess.Popen(make.split(' ') + ['-r', '-f', makefile] + makeflags, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) except Exception as e: error('failed to execute make: ' + str(e)) while True: l = p.stdout.readline().decode('utf-8').rstrip() if l == '': break if prettify and l.startswith('--> '): try: cmd, rest = l.lstrip('--> ').split(' ', 1) print(colored('{:<8}'.format(cmd), 'magenta', attrs=['bold']), colored(rest, 'yellow')) except ValueError: print(l) else: print(l) return p.wait()
def _load_include(package, include, config, enable_if): include_yml = load_yaml(os.path.join(get_package_dir(package), include)) try: # FIXME include_if = not enable_if or 'include_if' not in include_yml or \ eval(include_yml['include_if'], copy(config.getdict())) except Exception as e: error("eval(include_if) in {}: {}".format(package, str(e))) if include_if: include_yml.pop('include_if', None) yml = include_yml else: yml = {} return yml
def load_global_config(config, enable_if): for cs in config: if enable_if and cs.get('if') and not eval( cs['if'], copy(global_config.getdict())): continue for k, v in cs.items(): if k == 'if': continue for mode in ['append', 'append_words', 'default']: if v.get(mode): global_config._set(mode, k, v[mode]) break else: error("unsupported global config: '{}'".format(repr(v)))
def main(args): try: command = args[0] except IndexError: command = 'help' if command not in resea.command_list: error('command not found: {}'.format(command)) m = import_module('resea.commands.{}'.format(command.replace('-', '_'))) exit_code = m.main(args[1:]) if exit_code is None: exit_code = 0 return exit_code
def main(args_): parser = argparse.ArgumentParser(prog='resea new', description='create a new package') parser.add_argument('--git', action='store_true', help='git init in the created directory') parser.add_argument('dir', help='directory') args = parser.parse_args(args_) package_name = os.path.basename(args.dir) if re.match(PACKAGE_NAME_REGEX, package_name) is None: error('The package name must be lowercase_with_underscore (regex: {})' .format(PACKAGE_NAME_REGEX)) # # generate the package directory # generating('MKDIR', package_name) try: os.mkdir(package_name) except FileExistsError: error("Directory already exists: '{}'".format(package_name)) # # Variables used in templates # author = exec_cmd('git config --get user.name', ignore_failure=True).strip() email = exec_cmd('git config --get user.email', ignore_failure=True).strip() # # generate directories and files # for path, tmpl in FILES: f = '{}/{}'.format(package_name, path) dir = os.path.dirname(f) if not os.path.exists(dir): generating('MKDIR', dir) os.mkdir(dir) generating('GEN', f) open(f, 'w').write(jinja2.Template(tmpl).render(**locals())) # intialize as a new Git repository if args.git: exec_cmd('git init .', cwd=package_name)
def get_package_dir(package, search_registry=True): """Returns a path to the package.""" paths = [os.path.abspath('..')] for base in ['.'] + glob.glob('vendor/*'): paths += load_reseapath(base) for path in paths: d = os.path.join(path, package) if os.path.exists(os.path.join(d, 'package.yaml')): return d if search_registry: get_package_from_registry(package) return get_package_dir(package, search_registry=False) error("package not found: '{}'".format(package))
def doctor(): resea.commands.clean.main([]) tmp_dir = os.path.join('tmp', 'doctor') shutil.rmtree(tmp_dir, ignore_errors=True) os.makedirs(tmp_dir) plan('Validate the package') progress('check for the existence of README.md') if not os.path.exists('README.md'): notice('README.md not found') progress('validate package.yaml') try: yml = load_yaml('package.yaml', validator=validate_package_yml) except FileNotFoundError: error("'package.yaml' not found") if yml['category'] in ['application', 'library', 'lang', 'hal']: load_packages([yml['name']] + yml['depends'], {}) lang = yml.get("lang") if lang is None: error("lang is not speicified in package.yaml") # run lang's doctor lang_html_path = os.path.join(tmp_dir, 'lang.html') doctor = expand_var(get_var('LANGS')[lang]['doctor']) subprocess.Popen('{} {} {}'.format(doctor, lang_html_path, tmp_dir), shell=True).wait() # generate index.html progress('Generating index.html') with open(lang_html_path) as f: lang_html = f.read() else: lang_html = '' index_html_path = os.path.join(tmp_dir, 'index.html') with open(index_html_path, 'w') as f: f.write( render(INDEX_HTML, { 'lang': lang_html, 'created_at': str(datetime.datetime.now()) })) return index_html_path
def load_cmdline_config(args): for arg in args: for op, func in [('+=', global_config.append_words), ('=', global_config.set)]: if op in arg: break else: error( 'invalid default variable (should be "VAR_NAME=yaml" form): {}' .format(arg)) k, v = map(lambda x: x.strip(), arg.split(op, 1)) val = loads_yaml(v) if val is None: val = '' func(k, val)
def generate_package_doc(indir, outdir, revision): os.makedirs(outdir, exist_ok=True) try: yml = load_yaml(os.path.join(indir, 'package.yaml'), validator=validate_package_yml) except FileNotFoundError: error("'package.yaml' not found") yml.update({ 'title': 'Resea Documentation - {}'.format(yml['name']), 'css': PACKAGE_DOC_CSS, 'readme': md2html(os.path.join(indir, 'README.md')), 'revision': revision }) generating('GENDOC', os.path.join(outdir, 'index.html')) with open(os.path.join(outdir, 'index.html'), 'w') as f: f.write(render(PACKAGE_DOC, yml)) generate_documentation_dir(os.path.join(indir, 'Documentation'), outdir, revision)
def scaffold(args): try: yml = load_yaml('package.yaml', validator=validate_package_yml) except FileNotFoundError: error("'package.yaml' not found (are you in a package directory?)") load_packages([yml['name']] + yml['depends'], {}) lang = yml.get('lang') if lang is None: error("lang is not speicified in package.yaml") bin = expand_var(get_var('LANGS')[lang]['scaffold']) if bin is None: error("'{}' lang does not support scaffolding".format(lang)) subprocess.Popen(' '.join([bin, ".", "package.yaml"]), shell=True).wait()
def run_emulator(cmd, test=False, save_log=None, wait=False): if save_log is None: save_log = '/dev/null' # prepend a header to the log file f = open(save_log, 'a') f.write('================ {} ================\n'.format( str(datetime.datetime.now()))) # run test program speicified in HAL_RUN config p = subprocess.Popen(' '.join(cmd), shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, preexec_fn=os.setsid) atexit.register(atexit_handler, p) try: # parse log messages bugs = 0 passed = 0 failed = 0 while True: b = p.stdout.readline() try: l = b.decode('utf-8').strip() except UnicodeDecodeError: notice('cannot decode utf-8 string [{}]'.format(b)) if l == "" and p.poll() is not None: if test: error("the test program finished without 'TEST: end'") sys.exit(1) result = try_parse(l) if result == 'end': exit_code = 1 f.write(l + '\n') lprint(l) if bugs > 1: fail('{} bugs found'.format(bugs)) exit_code = 1 if test: if failed == 0: success('All {} tests passed'.format(passed)) else: fail('{} tests failed'.format(failed)) exit_code = 2 if wait: progress('Waiting for termination') p.wait() else: p.terminate() p.kill() return exit_code elif result == 'pass': passed += 1 f.write(l + '\n') lprint(l) elif result == 'fail': failed += 1 f.write(l + '\n') lprint(l) else: f.write(l + '\n') try: if lprint(l) == 'BUG': bugs += 1 except SystemExit: # kernel panic p.terminate() p.kill() return False except KeyboardInterrupt: pass
def update(): try: subprocess.run(['sudo', 'apt-get', 'update', '-y'], check=True) except subprocess.CalledProcessError: error('failed to update the list of packages')
def build(args): """Builds an executable.""" # load package.yaml in the current directory try: yml = load_yaml('package.yaml', validator=validate_package_yml) except FileNotFoundError: error("'package.yaml' not found (are you in a package directory?)") global_config.setdefault('ENV', 'release') global_config.setdefault('MAKE', 'make') global_config.setdefault('TEST', False) global_config.set('BUILD_DIR', 'build/' + get_var('ENV')) global_config.set('EXECUTABLE_PATH', expand_var('{{ BUILD_DIR }}/application')) load_configsets(args.configset) load_cmdline_config(args.config) if get_var('HAL', default=None) is None: error('HAL is not speicified') # resolve dependencies progress('Loading packages') if get_var('PACKAGES', default=[]) != []: packages = get_var('PACKAGES', default=[]) else: packages = [yml['name']] packages = list(set([get_var('HAL')] + packages)) if get_var('TEST'): if 'kernel' not in packages: packages.append('kernel') if yml['name'] not in packages: error("test target '" + yml['name'] + "' is not included in builtin packages " + '(did you include it in PACKAGES?)') ymls = load_packages(packages, enable_if=True, update_env=True) build_dir = get_var('BUILD_DIR') plan('Building {CATEGORY} with {HAL} HAL in {BUILD_DIR} ({ENV})'.format( **global_config.getdict())) # install os requirements os_requirements_pickle = os.path.join(build_dir, 'os_requirements.pickle') os_requirements = list(map(lambda y: y['os_requirements'], ymls.values())) if not is_object_equals_to_pickle(os_requirements, os_requirements_pickle): plan('Checking OS requirements') for x in os_requirements: install_os_requirements(x) sources = [] autogen_files = [] libs = [] stubs = [] deps_files = [] stub_files = [] target_deps = [] for package in ymls.keys(): package_dir = get_package_dir(package) libs += get_var('LIBS', package, default=[]) ext_lang = {} for lang_name, lang in get_var('LANGS').items(): ext_lang[lang['ext']] = lang if 'stub' not in lang: continue stub_file = os.path.join( build_dir, 'stubs', lang_name, lang['stub']['prefix'] + package + lang['stub']['suffix']) stubs.append((stub_file, os.path.join(package_dir, 'package.yaml'), expand_var( get_var('LANGS', package)['cpp']['genstub'], package))) stub_files.append(stub_file) source_files = [] for src in get_var('SOURCES', package, default=[]): base, ext = os.path.splitext(src) deps_file = os.path.join(build_dir, 'deps', package, base + '.deps') deps_files.append(deps_file) sources.append( ([(os.path.join(get_package_dir(package), src), os.path.join(build_dir, 'objs', package, base + '.o'), deps_file)], expand_var(ext_lang[ext.lstrip('.')]['abbrev'], package), expand_var(ext_lang[ext.lstrip('.')]['compile'], package), expand_var(ext_lang[ext.lstrip('.')]['mkdeps'], package))) for f in get_var('FILES', package, default=[]): path = os.path.join(package_dir, f['path']) cmd = expand_var(f['cmd'], package) autogen_files.append((path, cmd)) if f.get('rebuild_on') == 'always': try: os.remove(path) except FileNotFoundError: pass if f.get('required_on') == 'link': target_deps += [path] target_deps += libs # start if get_var('TEST'): genstart_args = ['--test', '--test-target', yml['name']] else: genstart_args = [] genstart = ' '.join([get_var('HAL_GENSTART'), '--with-threading'] + genstart_args + list(set(get_var('BUILTIN_APPS')))) start_file = os.path.join(build_dir, 'start.' + get_var('LANGS')['cpp']['ext']) start_deps_file = os.path.join(build_dir, 'start.deps') start_obj_file = os.path.join(build_dir, 'start.o') lang = get_var('HAL_START_LANG') sources.append( ([(start_file, start_obj_file, start_deps_file)], expand_var(get_var('LANGS', package)[lang]['abbrev'], package), expand_var(get_var('LANGS', package)[lang]['compile'], package), expand_var(get_var('LANGS', package)[lang]['mkdeps'], package))) # target if get_var('TEST'): global_config.set('CATEGORY', 'application') category = get_var('CATEGORY') if category == 'application': hal_link = get_var('HAL_LINK') elif category == 'library': hal_link = get_var('HAL_LIBLINK') else: error("unknown category '{}'".format(category)) target = os.path.join(build_dir, category) for files, _, _, _ in sources: for _, obj, _ in files: target_deps.append(obj) buildconfig = (global_config.getdict(), [c.getdict() for c in local_config.values()]) buildconfig_pickle = os.path.join(build_dir, 'buildconfig.pickle') generate_makefile = True if os.path.exists(build_dir): # clean up if build config have been changed if is_object_equals_to_pickle(buildconfig, buildconfig_pickle): generate_makefile = False else: plan('detected build config changes; cleaning the build directory') progress('deleting {}'.format(build_dir)) shutil.rmtree(build_dir) os.makedirs(build_dir, exist_ok=True) # save the build config and the os requirements to detect changes pickle.dump(buildconfig, open(buildconfig_pickle, 'wb')) pickle.dump(os_requirements, open(os_requirements_pickle, 'wb')) makefile = build_dir + '/Makefile' # generate makefile if needed if generate_makefile: with open(makefile, 'w') as f: f.write(render(MAKEFILE_TEMPLATE, locals())) # Everything is ready now. Let's start building! progress('executing make') if run_make(get_var('MAKE'), makefile, args.prettify) != 0: error('something went wrong in make(1)')
def update(): try: subprocess.run(['brew', 'update'], check=True) except subprocess.CalledProcessError: error('failed to update the list of packages')
def brew_tap(tap): try: subprocess.run(['brew', 'tap', tap], check=True) except subprocess.CalledProcessError: error('failed to tap {}')
def brew_install(args): try: subprocess.run(['brew', 'install'] + args, check=True) except subprocess.CalledProcessError: error('failed to install {}'.format(args))
def apt_install(args): try: subprocess.run(['sudo', 'apt-get', 'install', '-y'] + args, check=True) except subprocess.CalledProcessError: error('failed to install {}'.format(args))