def get_package_from_registry(package):
for d in ['tmp', 'vendor']:
try:
os.makedirs(d)
except FileExistsError:
pass
db_path = os.path.join('tmp', 'registry.db')
# download the registry database
if not os.path.exists(db_path):
urllib.request.urlretrieve(REGISTRY_DB_URL, db_path)
# look for the package
db = sqlite3.connect(db_path)
c = db.execute('SELECT type, uri FROM packages WHERE name = ?',
(package, ))
r = c.fetchone()
if r is None:
error("unknown package '{}' in registry".format(package))
type_, uri = r
# download it
if type_ == 'github':
name = 'github-{}'.format(uri.replace('/', '-'))
repo_path = os.path.join('vendor', name)
if not os.path.exists(repo_path):
repo_url = 'git://github.com/{}'.format(uri)
progress('cloning ' + repo_url)
subprocess.run(['git', 'clone', repo_url,
repo_path]) # TODO: use pure python
def run_make(make, makefile, prettify=False):
"""Executes make(1)"""
makeflags = ['-j' + str(multiprocessing.cpu_count())]
try:
p = subprocess.Popen(make.split(' ') + ['-r', '-f', makefile] +
makeflags,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
except Exception as e:
error('failed to execute make: ' + str(e))
while True:
l = p.stdout.readline().decode('utf-8').rstrip()
if l == '':
break
if prettify and l.startswith('--> '):
try:
cmd, rest = l.lstrip('--> ').split(' ', 1)
print(colored('{:<8}'.format(cmd), 'magenta', attrs=['bold']),
colored(rest, 'yellow'))
except ValueError:
print(l)
else:
print(l)
return p.wait()
def _load_include(package, include, config, enable_if):
include_yml = load_yaml(os.path.join(get_package_dir(package), include))
try:
# FIXME
include_if = not enable_if or 'include_if' not in include_yml or \
eval(include_yml['include_if'], copy(config.getdict()))
except Exception as e:
error("eval(include_if) in {}: {}".format(package, str(e)))
if include_if:
include_yml.pop('include_if', None)
yml = include_yml
else:
yml = {}
return yml
def load_global_config(config, enable_if):
for cs in config:
if enable_if and cs.get('if') and not eval(
cs['if'], copy(global_config.getdict())):
continue
for k, v in cs.items():
if k == 'if':
continue
for mode in ['append', 'append_words', 'default']:
if v.get(mode):
global_config._set(mode, k, v[mode])
break
else:
error("unsupported global config: '{}'".format(repr(v)))
def main(args):
try:
command = args[0]
except IndexError:
command = 'help'
if command not in resea.command_list:
error('command not found: {}'.format(command))
m = import_module('resea.commands.{}'.format(command.replace('-', '_')))
exit_code = m.main(args[1:])
if exit_code is None:
exit_code = 0
return exit_code
def main(args_):
parser = argparse.ArgumentParser(prog='resea new',
description='create a new package')
parser.add_argument('--git', action='store_true',
help='git init in the created directory')
parser.add_argument('dir', help='directory')
args = parser.parse_args(args_)
package_name = os.path.basename(args.dir)
if re.match(PACKAGE_NAME_REGEX, package_name) is None:
error('The package name must be lowercase_with_underscore (regex: {})'
.format(PACKAGE_NAME_REGEX))
#
# generate the package directory
#
generating('MKDIR', package_name)
try:
os.mkdir(package_name)
except FileExistsError:
error("Directory already exists: '{}'".format(package_name))
#
# Variables used in templates
#
author = exec_cmd('git config --get user.name',
ignore_failure=True).strip()
email = exec_cmd('git config --get user.email',
ignore_failure=True).strip()
#
# generate directories and files
#
for path, tmpl in FILES:
f = '{}/{}'.format(package_name, path)
dir = os.path.dirname(f)
if not os.path.exists(dir):
generating('MKDIR', dir)
os.mkdir(dir)
generating('GEN', f)
open(f, 'w').write(jinja2.Template(tmpl).render(**locals()))
# intialize as a new Git repository
if args.git:
exec_cmd('git init .', cwd=package_name)
def get_package_dir(package, search_registry=True):
"""Returns a path to the package."""
paths = [os.path.abspath('..')]
for base in ['.'] + glob.glob('vendor/*'):
paths += load_reseapath(base)
for path in paths:
d = os.path.join(path, package)
if os.path.exists(os.path.join(d, 'package.yaml')):
return d
if search_registry:
get_package_from_registry(package)
return get_package_dir(package, search_registry=False)
error("package not found: '{}'".format(package))
def doctor():
resea.commands.clean.main([])
tmp_dir = os.path.join('tmp', 'doctor')
shutil.rmtree(tmp_dir, ignore_errors=True)
os.makedirs(tmp_dir)
plan('Validate the package')
progress('check for the existence of README.md')
if not os.path.exists('README.md'):
notice('README.md not found')
progress('validate package.yaml')
try:
yml = load_yaml('package.yaml', validator=validate_package_yml)
except FileNotFoundError:
error("'package.yaml' not found")
if yml['category'] in ['application', 'library', 'lang', 'hal']:
load_packages([yml['name']] + yml['depends'], {})
lang = yml.get("lang")
if lang is None:
error("lang is not speicified in package.yaml")
# run lang's doctor
lang_html_path = os.path.join(tmp_dir, 'lang.html')
doctor = expand_var(get_var('LANGS')[lang]['doctor'])
subprocess.Popen('{} {} {}'.format(doctor, lang_html_path, tmp_dir),
shell=True).wait()
# generate index.html
progress('Generating index.html')
with open(lang_html_path) as f:
lang_html = f.read()
else:
lang_html = ''
index_html_path = os.path.join(tmp_dir, 'index.html')
with open(index_html_path, 'w') as f:
f.write(
render(INDEX_HTML, {
'lang': lang_html,
'created_at': str(datetime.datetime.now())
}))
return index_html_path
def load_cmdline_config(args):
for arg in args:
for op, func in [('+=', global_config.append_words),
('=', global_config.set)]:
if op in arg:
break
else:
error(
'invalid default variable (should be "VAR_NAME=yaml" form): {}'
.format(arg))
k, v = map(lambda x: x.strip(), arg.split(op, 1))
val = loads_yaml(v)
if val is None:
val = ''
func(k, val)
def generate_package_doc(indir, outdir, revision):
os.makedirs(outdir, exist_ok=True)
try:
yml = load_yaml(os.path.join(indir, 'package.yaml'),
validator=validate_package_yml)
except FileNotFoundError:
error("'package.yaml' not found")
yml.update({
'title': 'Resea Documentation - {}'.format(yml['name']),
'css': PACKAGE_DOC_CSS,
'readme': md2html(os.path.join(indir, 'README.md')),
'revision': revision
})
generating('GENDOC', os.path.join(outdir, 'index.html'))
with open(os.path.join(outdir, 'index.html'), 'w') as f:
f.write(render(PACKAGE_DOC, yml))
generate_documentation_dir(os.path.join(indir, 'Documentation'), outdir,
revision)
def scaffold(args):
try:
yml = load_yaml('package.yaml', validator=validate_package_yml)
except FileNotFoundError:
error("'package.yaml' not found (are you in a package directory?)")
load_packages([yml['name']] + yml['depends'], {})
lang = yml.get('lang')
if lang is None:
error("lang is not speicified in package.yaml")
bin = expand_var(get_var('LANGS')[lang]['scaffold'])
if bin is None:
error("'{}' lang does not support scaffolding".format(lang))
subprocess.Popen(' '.join([bin, ".", "package.yaml"]), shell=True).wait()
def run_emulator(cmd, test=False, save_log=None, wait=False):
if save_log is None:
save_log = '/dev/null'
# prepend a header to the log file
f = open(save_log, 'a')
f.write('================ {} ================\n'.format(
str(datetime.datetime.now())))
# run test program speicified in HAL_RUN config
p = subprocess.Popen(' '.join(cmd),
shell=True,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
preexec_fn=os.setsid)
atexit.register(atexit_handler, p)
try:
# parse log messages
bugs = 0
passed = 0
failed = 0
while True:
b = p.stdout.readline()
try:
l = b.decode('utf-8').strip()
except UnicodeDecodeError:
notice('cannot decode utf-8 string [{}]'.format(b))
if l == "" and p.poll() is not None:
if test:
error("the test program finished without 'TEST: end'")
sys.exit(1)
result = try_parse(l)
if result == 'end':
exit_code = 1
f.write(l + '\n')
lprint(l)
if bugs > 1:
fail('{} bugs found'.format(bugs))
exit_code = 1
if test:
if failed == 0:
success('All {} tests passed'.format(passed))
else:
fail('{} tests failed'.format(failed))
exit_code = 2
if wait:
progress('Waiting for termination')
p.wait()
else:
p.terminate()
p.kill()
return exit_code
elif result == 'pass':
passed += 1
f.write(l + '\n')
lprint(l)
elif result == 'fail':
failed += 1
f.write(l + '\n')
lprint(l)
else:
f.write(l + '\n')
try:
if lprint(l) == 'BUG':
bugs += 1
except SystemExit:
# kernel panic
p.terminate()
p.kill()
return False
except KeyboardInterrupt:
pass
def update():
try:
subprocess.run(['sudo', 'apt-get', 'update', '-y'], check=True)
except subprocess.CalledProcessError:
error('failed to update the list of packages')
def build(args):
"""Builds an executable."""
# load package.yaml in the current directory
try:
yml = load_yaml('package.yaml', validator=validate_package_yml)
except FileNotFoundError:
error("'package.yaml' not found (are you in a package directory?)")
global_config.setdefault('ENV', 'release')
global_config.setdefault('MAKE', 'make')
global_config.setdefault('TEST', False)
global_config.set('BUILD_DIR', 'build/' + get_var('ENV'))
global_config.set('EXECUTABLE_PATH',
expand_var('{{ BUILD_DIR }}/application'))
load_configsets(args.configset)
load_cmdline_config(args.config)
if get_var('HAL', default=None) is None:
error('HAL is not speicified')
# resolve dependencies
progress('Loading packages')
if get_var('PACKAGES', default=[]) != []:
packages = get_var('PACKAGES', default=[])
else:
packages = [yml['name']]
packages = list(set([get_var('HAL')] + packages))
if get_var('TEST'):
if 'kernel' not in packages:
packages.append('kernel')
if yml['name'] not in packages:
error("test target '" + yml['name'] +
"' is not included in builtin packages " +
'(did you include it in PACKAGES?)')
ymls = load_packages(packages, enable_if=True, update_env=True)
build_dir = get_var('BUILD_DIR')
plan('Building {CATEGORY} with {HAL} HAL in {BUILD_DIR} ({ENV})'.format(
**global_config.getdict()))
# install os requirements
os_requirements_pickle = os.path.join(build_dir, 'os_requirements.pickle')
os_requirements = list(map(lambda y: y['os_requirements'], ymls.values()))
if not is_object_equals_to_pickle(os_requirements, os_requirements_pickle):
plan('Checking OS requirements')
for x in os_requirements:
install_os_requirements(x)
sources = []
autogen_files = []
libs = []
stubs = []
deps_files = []
stub_files = []
target_deps = []
for package in ymls.keys():
package_dir = get_package_dir(package)
libs += get_var('LIBS', package, default=[])
ext_lang = {}
for lang_name, lang in get_var('LANGS').items():
ext_lang[lang['ext']] = lang
if 'stub' not in lang:
continue
stub_file = os.path.join(
build_dir, 'stubs', lang_name,
lang['stub']['prefix'] + package + lang['stub']['suffix'])
stubs.append((stub_file, os.path.join(package_dir, 'package.yaml'),
expand_var(
get_var('LANGS', package)['cpp']['genstub'],
package)))
stub_files.append(stub_file)
source_files = []
for src in get_var('SOURCES', package, default=[]):
base, ext = os.path.splitext(src)
deps_file = os.path.join(build_dir, 'deps', package,
base + '.deps')
deps_files.append(deps_file)
sources.append(
([(os.path.join(get_package_dir(package), src),
os.path.join(build_dir, 'objs', package,
base + '.o'), deps_file)],
expand_var(ext_lang[ext.lstrip('.')]['abbrev'], package),
expand_var(ext_lang[ext.lstrip('.')]['compile'], package),
expand_var(ext_lang[ext.lstrip('.')]['mkdeps'], package)))
for f in get_var('FILES', package, default=[]):
path = os.path.join(package_dir, f['path'])
cmd = expand_var(f['cmd'], package)
autogen_files.append((path, cmd))
if f.get('rebuild_on') == 'always':
try:
os.remove(path)
except FileNotFoundError:
pass
if f.get('required_on') == 'link':
target_deps += [path]
target_deps += libs
# start
if get_var('TEST'):
genstart_args = ['--test', '--test-target', yml['name']]
else:
genstart_args = []
genstart = ' '.join([get_var('HAL_GENSTART'), '--with-threading'] +
genstart_args + list(set(get_var('BUILTIN_APPS'))))
start_file = os.path.join(build_dir,
'start.' + get_var('LANGS')['cpp']['ext'])
start_deps_file = os.path.join(build_dir, 'start.deps')
start_obj_file = os.path.join(build_dir, 'start.o')
lang = get_var('HAL_START_LANG')
sources.append(
([(start_file, start_obj_file, start_deps_file)],
expand_var(get_var('LANGS', package)[lang]['abbrev'], package),
expand_var(get_var('LANGS', package)[lang]['compile'], package),
expand_var(get_var('LANGS', package)[lang]['mkdeps'], package)))
# target
if get_var('TEST'):
global_config.set('CATEGORY', 'application')
category = get_var('CATEGORY')
if category == 'application':
hal_link = get_var('HAL_LINK')
elif category == 'library':
hal_link = get_var('HAL_LIBLINK')
else:
error("unknown category '{}'".format(category))
target = os.path.join(build_dir, category)
for files, _, _, _ in sources:
for _, obj, _ in files:
target_deps.append(obj)
buildconfig = (global_config.getdict(),
[c.getdict() for c in local_config.values()])
buildconfig_pickle = os.path.join(build_dir, 'buildconfig.pickle')
generate_makefile = True
if os.path.exists(build_dir):
# clean up if build config have been changed
if is_object_equals_to_pickle(buildconfig, buildconfig_pickle):
generate_makefile = False
else:
plan('detected build config changes; cleaning the build directory')
progress('deleting {}'.format(build_dir))
shutil.rmtree(build_dir)
os.makedirs(build_dir, exist_ok=True)
# save the build config and the os requirements to detect changes
pickle.dump(buildconfig, open(buildconfig_pickle, 'wb'))
pickle.dump(os_requirements, open(os_requirements_pickle, 'wb'))
makefile = build_dir + '/Makefile'
# generate makefile if needed
if generate_makefile:
with open(makefile, 'w') as f:
f.write(render(MAKEFILE_TEMPLATE, locals()))
# Everything is ready now. Let's start building!
progress('executing make')
if run_make(get_var('MAKE'), makefile, args.prettify) != 0:
error('something went wrong in make(1)')
def update():
try:
subprocess.run(['brew', 'update'], check=True)
except subprocess.CalledProcessError:
error('failed to update the list of packages')
def brew_tap(tap):
try:
subprocess.run(['brew', 'tap', tap], check=True)
except subprocess.CalledProcessError:
error('failed to tap {}')
def brew_install(args):
try:
subprocess.run(['brew', 'install'] + args, check=True)
except subprocess.CalledProcessError:
error('failed to install {}'.format(args))
def apt_install(args):
try:
subprocess.run(['sudo', 'apt-get', 'install', '-y'] + args, check=True)
except subprocess.CalledProcessError:
error('failed to install {}'.format(args))
