def run(args):
build(args)
plan('Launching an emulator')
cmd = [get_var('HAL_RUN'), get_var('EXECUTABLE_PATH')]
progress(' '.join(cmd))
run_emulator(cmd)
def get_package_from_registry(package):
for d in ['tmp', 'vendor']:
try:
os.makedirs(d)
except FileExistsError:
pass
db_path = os.path.join('tmp', 'registry.db')
# download the registry database
if not os.path.exists(db_path):
urllib.request.urlretrieve(REGISTRY_DB_URL, db_path)
# look for the package
db = sqlite3.connect(db_path)
c = db.execute('SELECT type, uri FROM packages WHERE name = ?',
(package, ))
r = c.fetchone()
if r is None:
error("unknown package '{}' in registry".format(package))
type_, uri = r
# download it
if type_ == 'github':
name = 'github-{}'.format(uri.replace('/', '-'))
repo_path = os.path.join('vendor', name)
if not os.path.exists(repo_path):
repo_url = 'git://github.com/{}'.format(uri)
progress('cloning ' + repo_url)
subprocess.run(['git', 'clone', repo_url,
repo_path]) # TODO: use pure python
def main(argv_):
parser = argparse.ArgumentParser(prog='resea doctor',
description='diagnose the package')
parser.add_argument('--open-browser',
action='store_true',
help='open results in a web browser')
args = parser.parse_args(argv_)
index_html_path = doctor()
if args.open_browser:
progress('Opening the results')
webbrowser.open_new_tab('file://' + os.path.abspath(index_html_path))
success('Done all diagnosis')
def alldocs(args):
packages = []
# FIXME
files = glob('package.yaml') + glob('*/package.yaml') + \
glob('*/*/package.yaml') + glob('*/*/*/package.yaml') + \
glob('*/*/*/*/package.yaml') + glob('*/*/*/*/*/package.yaml')
kwargs = {'revision': args.revision}
os.makedirs(args.outdir)
generate_documentation_dir('Documentation', os.path.join(args.outdir),
**kwargs)
for f in files:
if '/packages/' in f:
continue
path = os.path.dirname(f)
package = os.path.basename(path)
progress('Generating documentaion: {}'.format(package))
generate_package_doc(path,
os.path.join(args.outdir, 'packages', package),
**kwargs)
packages.append({'name': package, 'summary': load_yaml(f)['summary']})
progress('Generating packages index')
generate_package_index_doc(
packages, os.path.join(args.outdir, 'packages', 'index.html'),
**kwargs)
progress('Generating documentation index')
generate_index_doc(os.path.join(args.outdir, 'index.html'), **kwargs)
def doctor():
resea.commands.clean.main([])
tmp_dir = os.path.join('tmp', 'doctor')
shutil.rmtree(tmp_dir, ignore_errors=True)
os.makedirs(tmp_dir)
plan('Validate the package')
progress('check for the existence of README.md')
if not os.path.exists('README.md'):
notice('README.md not found')
progress('validate package.yaml')
try:
yml = load_yaml('package.yaml', validator=validate_package_yml)
except FileNotFoundError:
error("'package.yaml' not found")
if yml['category'] in ['application', 'library', 'lang', 'hal']:
load_packages([yml['name']] + yml['depends'], {})
lang = yml.get("lang")
if lang is None:
error("lang is not speicified in package.yaml")
# run lang's doctor
lang_html_path = os.path.join(tmp_dir, 'lang.html')
doctor = expand_var(get_var('LANGS')[lang]['doctor'])
subprocess.Popen('{} {} {}'.format(doctor, lang_html_path, tmp_dir),
shell=True).wait()
# generate index.html
progress('Generating index.html')
with open(lang_html_path) as f:
lang_html = f.read()
else:
lang_html = ''
index_html_path = os.path.join(tmp_dir, 'index.html')
with open(index_html_path, 'w') as f:
f.write(
render(INDEX_HTML, {
'lang': lang_html,
'created_at': str(datetime.datetime.now())
}))
return index_html_path
def build(args):
"""Builds an executable."""
# load package.yaml in the current directory
try:
yml = load_yaml('package.yaml', validator=validate_package_yml)
except FileNotFoundError:
error("'package.yaml' not found (are you in a package directory?)")
global_config.setdefault('ENV', 'release')
global_config.setdefault('MAKE', 'make')
global_config.setdefault('TEST', False)
global_config.set('BUILD_DIR', 'build/' + get_var('ENV'))
global_config.set('EXECUTABLE_PATH',
expand_var('{{ BUILD_DIR }}/application'))
load_configsets(args.configset)
load_cmdline_config(args.config)
if get_var('HAL', default=None) is None:
error('HAL is not speicified')
# resolve dependencies
progress('Loading packages')
if get_var('PACKAGES', default=[]) != []:
packages = get_var('PACKAGES', default=[])
else:
packages = [yml['name']]
packages = list(set([get_var('HAL')] + packages))
if get_var('TEST'):
if 'kernel' not in packages:
packages.append('kernel')
if yml['name'] not in packages:
error("test target '" + yml['name'] +
"' is not included in builtin packages " +
'(did you include it in PACKAGES?)')
ymls = load_packages(packages, enable_if=True, update_env=True)
build_dir = get_var('BUILD_DIR')
plan('Building {CATEGORY} with {HAL} HAL in {BUILD_DIR} ({ENV})'.format(
**global_config.getdict()))
# install os requirements
os_requirements_pickle = os.path.join(build_dir, 'os_requirements.pickle')
os_requirements = list(map(lambda y: y['os_requirements'], ymls.values()))
if not is_object_equals_to_pickle(os_requirements, os_requirements_pickle):
plan('Checking OS requirements')
for x in os_requirements:
install_os_requirements(x)
sources = []
autogen_files = []
libs = []
stubs = []
deps_files = []
stub_files = []
target_deps = []
for package in ymls.keys():
package_dir = get_package_dir(package)
libs += get_var('LIBS', package, default=[])
ext_lang = {}
for lang_name, lang in get_var('LANGS').items():
ext_lang[lang['ext']] = lang
if 'stub' not in lang:
continue
stub_file = os.path.join(
build_dir, 'stubs', lang_name,
lang['stub']['prefix'] + package + lang['stub']['suffix'])
stubs.append((stub_file, os.path.join(package_dir, 'package.yaml'),
expand_var(
get_var('LANGS', package)['cpp']['genstub'],
package)))
stub_files.append(stub_file)
source_files = []
for src in get_var('SOURCES', package, default=[]):
base, ext = os.path.splitext(src)
deps_file = os.path.join(build_dir, 'deps', package,
base + '.deps')
deps_files.append(deps_file)
sources.append(
([(os.path.join(get_package_dir(package), src),
os.path.join(build_dir, 'objs', package,
base + '.o'), deps_file)],
expand_var(ext_lang[ext.lstrip('.')]['abbrev'], package),
expand_var(ext_lang[ext.lstrip('.')]['compile'], package),
expand_var(ext_lang[ext.lstrip('.')]['mkdeps'], package)))
for f in get_var('FILES', package, default=[]):
path = os.path.join(package_dir, f['path'])
cmd = expand_var(f['cmd'], package)
autogen_files.append((path, cmd))
if f.get('rebuild_on') == 'always':
try:
os.remove(path)
except FileNotFoundError:
pass
if f.get('required_on') == 'link':
target_deps += [path]
target_deps += libs
# start
if get_var('TEST'):
genstart_args = ['--test', '--test-target', yml['name']]
else:
genstart_args = []
genstart = ' '.join([get_var('HAL_GENSTART'), '--with-threading'] +
genstart_args + list(set(get_var('BUILTIN_APPS'))))
start_file = os.path.join(build_dir,
'start.' + get_var('LANGS')['cpp']['ext'])
start_deps_file = os.path.join(build_dir, 'start.deps')
start_obj_file = os.path.join(build_dir, 'start.o')
lang = get_var('HAL_START_LANG')
sources.append(
([(start_file, start_obj_file, start_deps_file)],
expand_var(get_var('LANGS', package)[lang]['abbrev'], package),
expand_var(get_var('LANGS', package)[lang]['compile'], package),
expand_var(get_var('LANGS', package)[lang]['mkdeps'], package)))
# target
if get_var('TEST'):
global_config.set('CATEGORY', 'application')
category = get_var('CATEGORY')
if category == 'application':
hal_link = get_var('HAL_LINK')
elif category == 'library':
hal_link = get_var('HAL_LIBLINK')
else:
error("unknown category '{}'".format(category))
target = os.path.join(build_dir, category)
for files, _, _, _ in sources:
for _, obj, _ in files:
target_deps.append(obj)
buildconfig = (global_config.getdict(),
[c.getdict() for c in local_config.values()])
buildconfig_pickle = os.path.join(build_dir, 'buildconfig.pickle')
generate_makefile = True
if os.path.exists(build_dir):
# clean up if build config have been changed
if is_object_equals_to_pickle(buildconfig, buildconfig_pickle):
generate_makefile = False
else:
plan('detected build config changes; cleaning the build directory')
progress('deleting {}'.format(build_dir))
shutil.rmtree(build_dir)
os.makedirs(build_dir, exist_ok=True)
# save the build config and the os requirements to detect changes
pickle.dump(buildconfig, open(buildconfig_pickle, 'wb'))
pickle.dump(os_requirements, open(os_requirements_pickle, 'wb'))
makefile = build_dir + '/Makefile'
# generate makefile if needed
if generate_makefile:
with open(makefile, 'w') as f:
f.write(render(MAKEFILE_TEMPLATE, locals()))
# Everything is ready now. Let's start building!
progress('executing make')
if run_make(get_var('MAKE'), makefile, args.prettify) != 0:
error('something went wrong in make(1)')
def run_emulator(cmd, test=False, save_log=None, wait=False):
if save_log is None:
save_log = '/dev/null'
# prepend a header to the log file
f = open(save_log, 'a')
f.write('================ {} ================\n'.format(
str(datetime.datetime.now())))
# run test program speicified in HAL_RUN config
p = subprocess.Popen(' '.join(cmd),
shell=True,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
preexec_fn=os.setsid)
atexit.register(atexit_handler, p)
try:
# parse log messages
bugs = 0
passed = 0
failed = 0
while True:
b = p.stdout.readline()
try:
l = b.decode('utf-8').strip()
except UnicodeDecodeError:
notice('cannot decode utf-8 string [{}]'.format(b))
if l == "" and p.poll() is not None:
if test:
error("the test program finished without 'TEST: end'")
sys.exit(1)
result = try_parse(l)
if result == 'end':
exit_code = 1
f.write(l + '\n')
lprint(l)
if bugs > 1:
fail('{} bugs found'.format(bugs))
exit_code = 1
if test:
if failed == 0:
success('All {} tests passed'.format(passed))
else:
fail('{} tests failed'.format(failed))
exit_code = 2
if wait:
progress('Waiting for termination')
p.wait()
else:
p.terminate()
p.kill()
return exit_code
elif result == 'pass':
passed += 1
f.write(l + '\n')
lprint(l)
elif result == 'fail':
failed += 1
f.write(l + '\n')
lprint(l)
else:
f.write(l + '\n')
try:
if lprint(l) == 'BUG':
bugs += 1
except SystemExit:
# kernel panic
p.terminate()
p.kill()
return False
except KeyboardInterrupt:
pass
def atexit_handler(p):
if p.poll() is None:
progress('Terminating the emulator')
os.killpg(p.pid, signal.SIGTERM)