def _write_file(self, data: str, file_id: str) -> None: #
''' Write the data to disk ans store the id for the generated data '''
self.generated_files[file_id] = self._gen_filename(file_id)
out_file = self.out_dir / self.generated_files[file_id]
out_file.write_text(data, encoding='ascii')
mlog.log('Generated', mlog.bold(out_file.name))
def generate_hotdoc_config(self):
cwd = os.path.abspath(os.curdir)
ncwd = os.path.join(self.sourcedir, self.subdir)
mlog.log('Generating Hotdoc configuration for: ', mlog.bold(self.name))
os.chdir(ncwd)
self.hotdoc.run_hotdoc(self.flatten_config_command())
os.chdir(cwd)
def load_impl(self) -> ReferenceManual:
mlog.log('Loading YAML reference manual')
with mlog.nested():
manual = ReferenceManual(
functions=[
self._load_function(x) for x in self.func_dir.iterdir()
],
objects=mesonlib.listify([
[
self._load_object(ObjectType.ELEMENTARY, x)
for x in self.elem_dir.iterdir()
],
[
self._load_object(ObjectType.RETURNED, x)
for x in self.objs_dir.iterdir()
],
[
self._load_object(ObjectType.BUILTIN, x)
for x in self.builtin_dir.iterdir()
],
[self._load_module(x) for x in self.modules_dir.iterdir()]
],
flatten=True))
if not self.strict:
mlog.warning(
'YAML reference manual loaded using the best-effort fastyaml loader. Results are not guaranteed to be stable or correct.'
)
return manual
def generate_hotdoc_config(self):
cwd = os.path.abspath(os.curdir)
ncwd = os.path.join(self.sourcedir, self.subdir)
mlog.log('Generating Hotdoc configuration for: ', mlog.bold(self.name))
os.chdir(ncwd)
self.hotdoc.run_hotdoc(self.flatten_config_command())
os.chdir(cwd)
def _load_function(self,
path: Path,
obj: T.Optional[Object] = None) -> Function:
path_label = path.relative_to(self.yaml_dir).as_posix()
mlog.log('Loading', mlog.bold(path_label))
raw = self._load(self.read_file(path),
self.template.s_function,
label=path_label)
return self._process_function_base(raw)
def load_impl(self) -> ReferenceManual:
mlog.log('Loading YAML refererence manual')
with mlog.nested():
return ReferenceManual(
functions=[self._load_function(x) for x in self.func_dir.iterdir()],
objects=mesonlib.listify([
[self._load_object(ObjectType.ELEMENTARY, x) for x in self.elem_dir.iterdir()],
[self._load_object(ObjectType.RETURNED, x) for x in self.objs_dir.iterdir()],
[self._load_object(ObjectType.BUILTIN, x) for x in self.builtin_dir.iterdir()],
[self._load_module(x) for x in self.modules_dir.iterdir()]
], flatten=True)
)
def generate(self) -> None:
mlog.log('Generating markdown files...')
with mlog.nested():
self._write_functions()
for obj in self.objects:
if not self.enable_modules and (
obj.obj_type == ObjectType.MODULE
or obj.defined_by_module is not None):
continue
self._write_object(obj)
self._root_refman_docs()
self._configure_sitemap()
self._generate_link_def()
def validate_and_resolve(self, manual: ReferenceManual) -> ReferenceManual:
mlog.log('Validating loaded manual...')
# build type map and func map for methods
for obj in manual.objects:
assert obj.name not in self.type_map, f'Duplicate object name {obj.name}'
self.type_map[obj.name] = obj
for m in obj.methods:
mid = f'{obj.name}.{m.name}'
assert mid not in self.type_map, f'Duplicate metod {mid}'
self.func_map[mid] = m
# Build func map for functions
for func in manual.functions:
assert func.name not in [*self.func_map.keys()
], f'Duplicate function {func.name}'
self.func_map[func.name] = func
mlog.log('Validating functions...')
for func in manual.functions:
mlog.log(' -- validating', mlog.bold(func.name))
self._validate_func(func)
mlog.log('Validating objects...')
for obj in manual.objects:
mlog.log(' -- validating', mlog.bold(obj.name))
self._validate_named_object(obj)
self._validate_feature_check(obj)
# Resolve and validate inheritence
if obj.extends:
assert obj.extends in self.type_map, f'Unknown extends object {obj.extends} in {obj.name}'
obj.extends_obj = self.type_map[obj.extends]
obj.extends_obj.extended_by += [obj]
# Only returned objects can be associated with module
if obj.obj_type is not ObjectType.RETURNED:
assert obj.defined_by_module is None
for m in obj.methods:
assert m.obj is obj
self._validate_func(m)
# Resolve inherited methods
for obj in manual.objects:
inherited_methods = obj.inherited_methods
curr = obj.extends_obj
while curr is not None:
inherited_methods += curr.methods
curr = curr.extends_obj
return manual
def set_meson_command(mainfile):
global python_command
global meson_command
# On UNIX-like systems `meson` is a Python script
# On Windows `meson` and `meson.exe` are wrapper exes
if not mainfile.endswith('.py'):
meson_command = [mainfile]
elif os.path.isabs(mainfile) and mainfile.endswith('mesonmain.py'):
# Can't actually run meson with an absolute path to mesonmain.py, it must be run as -m mesonbuild.mesonmain
meson_command = python_command + ['-m', 'mesonbuild.mesonmain']
else:
# Either run uninstalled, or full path to meson-script.py
meson_command = python_command + [mainfile]
# We print this value for unit tests.
if 'MESON_COMMAND_TESTS' in os.environ:
mlog.log('meson_command is {!r}'.format(meson_command))
def _load_object(self, obj_type: ObjectType, path: Path) -> Object:
path_label = path.relative_to(self.yaml_dir).as_posix()
mlog.log(f'Loading', mlog.bold(path_label))
raw = load(self.read_file(path), s_object, label=path_label).data
def as_methods(mlist: T.List[Function]) -> T.List[Method]:
res: T.List[Method] = []
for i in mlist:
assert isinstance(i, Method)
res += [i]
return res
methods = raw.pop('methods', [])
obj = Object(methods=[], obj_type=obj_type, **raw)
obj.methods = as_methods([self._process_function_base(x, obj) for x in methods])
return obj
def set_meson_command(mainfile):
global python_command
global meson_command
# On UNIX-like systems `meson` is a Python script
# On Windows `meson` and `meson.exe` are wrapper exes
if not mainfile.endswith('.py'):
meson_command = [mainfile]
elif os.path.isabs(mainfile) and mainfile.endswith('mesonmain.py'):
# Can't actually run meson with an absolute path to mesonmain.py, it must be run as -m mesonbuild.mesonmain
meson_command = python_command + ['-m', 'mesonbuild.mesonmain']
else:
# Either run uninstalled, or full path to meson-script.py
meson_command = python_command + [mainfile]
# We print this value for unit tests.
if 'MESON_COMMAND_TESTS' in os.environ:
mlog.log('meson_command is {!r}'.format(meson_command))
def generate(self) -> None:
mlog.log('\n\n', mlog.bold('=== Functions ==='), '\n')
for f in self.functions:
self._generate_function(f)
mlog.log('\n\n', mlog.bold('=== Elementary ==='), '\n')
for obj in self.elementary:
self._generate_object(obj)
mlog.log('\n\n', mlog.bold('=== Builtins ==='), '\n')
for obj in self.builtins:
self._generate_object(obj)
mlog.log('\n\n', mlog.bold('=== Returned objects ==='), '\n')
for obj in self.returned:
self._generate_object(obj)
mlog.log('\n\n', mlog.bold('=== Modules ==='), '\n')
for obj in self.modules:
self._generate_object(obj)
for mod_obj in self.extract_returned_by_module(obj):
self._generate_object(mod_obj)
def _configure_sitemap(self) -> None:
'''
Replaces the `@REFMAN_PLACEHOLDER@` placeholder with the reference
manual sitemap. The structure of the sitemap is derived from the
file IDs.
'''
raw = self.sitemap_in.read_text(encoding='utf-8')
out = ''
for l in raw.split('\n'):
if '@REFMAN_PLACEHOLDER@' not in l:
out += f'{l}\n'
continue
mlog.log('Generating', mlog.bold(self.sitemap_out.as_posix()))
base_indent = l.replace('@REFMAN_PLACEHOLDER@', '')
for k in sorted(self.generated_files.keys()):
indent = base_indent + '\t' * k.count('.')
out += f'{indent}{self.generated_files[k]}\n'
self.sitemap_out.write_text(out, encoding='utf-8')
def declaration(self, state, args, kwargs):
if len(args) != 2:
raise InterpreterException('declaration takes exactly two arguments.')
check_stringlist(args)
disabled, required, _ = extract_required_kwarg(kwargs, state.subproject)
if disabled:
return ModuleReturnValue(False, [False])
lang, compiler = self._compiler(state)
parser = declaration_parsers.get(lang)
if not parser:
raise InterpreterException('no declaration parser for language %s' % lang)
compiler_args = self._compile_args(compiler, state, kwargs)
compiler_args += compiler.get_werror_args()
pkg = args[0]
decl = args[1]
tree = parser.parse(decl)
name = extract_declaration_name(tree)
proto = rewrite_declaration(tree, name)
check_prototype = not is_lone_identifier(tree)
if check_prototype:
checker = prototype_checkers.get(lang)
if not checker:
raise InterpreterException('no checker program for language %s' % lang)
prog = checker(tree, pkg, name)
ok = compiler.compiles(prog, state.environment, compiler_args, None)
status = mlog.green('YES') if ok else mlog.red('NO')
mlog.log('Checking that', mlog.bold(name, True), 'has prototype', mlog.bold(proto, True), ':', status)
else:
ok = compiler.has_header_symbol(pkg, name, '', state.environment, compiler_args, None)
self._checklog('declaration for', name, ok)
self._set_config_var(state, ok, name, kwargs)
if not ok and required:
raise InterpreterException('{} declaration {} required but not found.'.format(lang, name))
return ModuleReturnValue(ok, [ok])
def install_help(srcdir, blddir, sources, media, langs, install_dir, destdir,
project_id, symlinks):
c_install_dir = os.path.join(install_dir, 'C', project_id)
for lang in langs + ['C']:
indir = destdir_join(destdir,
os.path.join(install_dir, lang, project_id))
os.makedirs(indir, exist_ok=True)
for source in sources:
infile = os.path.join(srcdir if lang == 'C' else blddir, lang,
source)
outfile = os.path.join(indir, source)
mlog.log('Installing %s to %s.' % (infile, outfile))
shutil.copyfile(infile, outfile)
shutil.copystat(infile, outfile)
for m in media:
infile = os.path.join(srcdir, lang, m)
outfile = os.path.join(indir, m)
if not os.path.exists(infile):
if lang == 'C':
mlog.warning(
'Media file "%s" did not exist in C directory' % m)
elif symlinks:
srcfile = os.path.join(c_install_dir, m)
mlog.log('Symlinking %s to %s.' % (outfile, srcfile))
if '/' in m or '\\' in m:
os.makedirs(os.path.dirname(outfile), exist_ok=True)
os.symlink(srcfile, outfile)
continue
symfile = os.path.join(install_dir, m)
mlog.log('Installing %s to %s.' % (infile, outfile))
if '/' in m or '\\' in m:
os.makedirs(os.path.dirname(outfile), exist_ok=True)
shutil.copyfile(infile, outfile)
shutil.copystat(infile, outfile)
def run(args):
options = parser.parse_args(args)
if options.target is None or options.filename is None:
sys.exit("Must specify both target and filename.")
print('This tool is highly experimental, use with care.')
rewriter = mesonbuild.astinterpreter.AstInterpreter(options.sourcedir, '')
try:
if options.commands[0] == 'add':
rewriter.add_source(options.target, options.filename)
elif options.commands[0] == 'remove':
rewriter.remove_source(options.target, options.filename)
else:
sys.exit('Unknown command: ' + options.commands[0])
except Exception as e:
if isinstance(e, MesonException):
if hasattr(e, 'file') and hasattr(e, 'lineno') and hasattr(
e, 'colno'):
mlog.log(
mlog.red(
'\nMeson encountered an error in file %s, line %d, column %d:'
% (e.file, e.lineno, e.colno)))
else:
mlog.log(mlog.red('\nMeson encountered an error:'))
mlog.log(e)
else:
traceback.print_exc()
return 1
return 0
def _generate_object(self, obj: Object) -> None:
tags = []
tags += [{
ObjectType.ELEMENTARY: mlog.yellow('[elementary]'),
ObjectType.BUILTIN: mlog.green('[builtin]'),
ObjectType.MODULE: mlog.blue('[module]'),
ObjectType.RETURNED: mlog.cyan('[returned]'),
}[obj.obj_type]]
if obj.is_container:
tags += [mlog.red('[container]')]
mlog.log()
mlog.log('Object', mlog.bold(obj.name), *tags)
with my_nested():
desc = obj.description
if '\n' in desc:
desc = desc[:desc.index('\n')]
mlog.log('Description:', mlog.bold(desc))
mlog.log('Returned by:', mlog.bold(str([x.name for x in obj.returned_by])))
mlog.log('Methods:')
with my_nested():
for m in obj.methods:
self._generate_function(m)
def run(args):
options = parser.parse_args(args)
if options.target is None or options.filename is None:
sys.exit("Must specify both target and filename.")
print('This tool is highly experimental, use with care.')
rewriter = mesonbuild.astinterpreter.AstInterpreter(options.sourcedir, '')
try:
if options.commands[0] == 'add':
rewriter.add_source(options.target, options.filename)
elif options.commands[0] == 'remove':
rewriter.remove_source(options.target, options.filename)
else:
sys.exit('Unknown command: ' + options.commands[0])
except Exception as e:
if isinstance(e, MesonException):
if hasattr(e, 'file') and hasattr(e, 'lineno') and hasattr(e, 'colno'):
mlog.log(mlog.red('\nMeson encountered an error in file %s, line %d, column %d:' % (e.file, e.lineno, e.colno)))
else:
mlog.log(mlog.red('\nMeson encountered an error:'))
mlog.log(e)
else:
traceback.print_exc()
return 1
return 0
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('case', type=pathlib.Path, help='The test case to run')
parser.add_argument('--subtest',
type=int,
action='append',
dest='subtests',
help='which subtests to run')
parser.add_argument('--backend',
action='store',
help="Which backend to use")
parser.add_argument('--cross-file',
action='store',
help='File describing cross compilation environment.')
parser.add_argument('--native-file',
action='store',
help='File describing native compilation environment.')
parser.add_argument('--use-tmpdir',
action='store_true',
help='Use tmp directory for temporary files.')
args = T.cast('ArgumentType', parser.parse_args())
setup_commands(args.backend)
detect_system_compiler(args)
print_tool_versions()
test = TestDef(args.case, args.case.stem, [])
tests = load_test_json(test, False)
if args.subtests:
tests = [t for i, t in enumerate(tests) if i in args.subtests]
results = [run_test(t, t.args, '', True) for t in tests]
failed = False
for test, result in zip(tests, results):
if (result is None) or ('MESON_SKIP_TEST' in result.stdo):
msg = mlog.yellow('SKIP:')
elif result.msg:
msg = mlog.red('FAIL:')
failed = True
else:
msg = mlog.green('PASS:'******'MESON_SKIP_TEST' not in result.stdo:
mlog.log('reason:', result.msg)
if result.step is BuildStep.configure:
# For configure failures, instead of printing stdout,
# print the meson log if available since it's a superset
# of stdout and often has very useful information.
mlog.log(result.mlog)
else:
mlog.log(result.stdo)
for cmd_res in result.cicmds:
mlog.log(cmd_res)
mlog.log(result.stde)
exit(1 if failed else 0)
def _generate_function(self, func: Function) -> None:
mlog.log()
mlog.log('Function', mlog.bold(func.name))
with my_nested():
desc = func.description
if '\n' in desc:
desc = desc[:desc.index('\n')]
mlog.log('Description:', mlog.bold(desc))
mlog.log('Return type:', mlog.bold(self._types_to_string(func.returns)))
mlog.log('Pos args: ', mlog.bold(str([x.name for x in func.posargs])))
mlog.log('Opt args: ', mlog.bold(str([x.name for x in func.optargs])))
mlog.log('Varargs: ', mlog.bold(func.varargs.name if func.varargs is not None else 'null'))
mlog.log('Kwargs: ', mlog.bold(str(list(func.kwargs.keys()))))
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('case', type=pathlib.Path, help='The test case to run')
parser.add_argument('--subtest',
type=int,
action='append',
dest='subtests',
help='which subtests to run')
parser.add_argument('--backend',
action='store',
help="Which backend to use")
args = T.cast('ArgumentType', parser.parse_args())
test = TestDef(args.case, args.case.stem, [])
tests = load_test_json(test, False)
if args.subtests:
tests = [t for i, t in enumerate(tests) if i in args.subtests]
with mesonlib.TemporaryDirectoryWinProof() as build_dir:
fake_opts = get_fake_options('/')
env = environment.Environment(None, build_dir, fake_opts)
try:
comp = env.compiler_from_language(
'c', mesonlib.MachineChoice.HOST).get_id()
except mesonlib.MesonException:
raise RuntimeError('Could not detect C compiler')
backend, backend_args = guess_backend(args.backend,
shutil.which('msbuild'))
_cmds = get_backend_commands(backend, False)
commands = (_cmds[0], _cmds[1], _cmds[3], _cmds[4])
results = [
run_test(t, t.args, comp, backend, backend_args, commands, False, True)
for t in tests
]
failed = False
for test, result in zip(tests, results):
if result is None:
msg = mlog.yellow('SKIP:')
elif result.msg:
msg = mlog.red('FAIL:')
failed = True
else:
msg = mlog.green('PASS:'******'reason:', result.msg)
if result.step is BuildStep.configure:
# For configure failures, instead of printing stdout,
# print the meson log if available since it's a superset
# of stdout and often has very useful information.
mlog.log(result.mlog)
else:
mlog.log(result.stdo)
for cmd_res in result.cicmds:
mlog.log(cmd_res)
mlog.log(result.stde)
exit(1 if failed else 0)
results = [
run_test(t, t.args, comp, backend, backend_args, commands, False, True)
for t in tests
]
failed = False
for test, result in zip(tests, results):
if (result is None) or (
('MESON_SKIP_TEST' in result.stdo) and
(skippable(str(args.case.parent), test.path.as_posix()))):
msg = mlog.yellow('SKIP:')
elif result.msg:
msg = mlog.red('FAIL:')
failed = True
else:
msg = mlog.green('PASS:'******'MESON_SKIP_TEST' not in result.stdo:
mlog.log('reason:', result.msg)
if result.step is BuildStep.configure:
# For configure failures, instead of printing stdout,
# print the meson log if available since it's a superset
# of stdout and often has very useful information.
mlog.log(result.mlog)
else:
mlog.log(result.stdo)
for cmd_res in result.cicmds:
mlog.log(cmd_res)
mlog.log(result.stde)
exit(1 if failed else 0)
help='Path to source directory.')
parser.add_argument('--target', default=None,
help='Name of target to edit.')
parser.add_argument('--filename', default=None,
help='Name of source file to add or remove to target.')
parser.add_argument('commands', nargs='+')
if __name__ == '__main__':
options = parser.parse_args()
if options.target is None or options.filename is None:
sys.exit("Must specify both target and filename.")
print('This tool is highly experimental, use with care.')
rewriter = mesonbuild.astinterpreter.AstInterpreter(options.sourcedir, '')
try:
if options.commands[0] == 'add':
rewriter.add_source(options.target, options.filename)
elif options.commands[0] == 'remove':
rewriter.remove_source(options.target, options.filename)
else:
sys.exit('Unknown command: ' + options.commands[0])
except Exception as e:
if isinstance(e, MesonException):
if hasattr(e, 'file') and hasattr(e, 'lineno') and hasattr(e, 'colno'):
mlog.log(mlog.red('\nMeson encountered an error in file %s, line %d, column %d:' % (e.file, e.lineno, e.colno)))
else:
mlog.log(mlog.red('\nMeson encountered an error:'))
mlog.log(e)
else:
traceback.print_exc()
sys.exit(1)
help='Name of source file to add or remove to target.')
parser.add_argument('commands', nargs='+')
if __name__ == '__main__':
options = parser.parse_args()
if options.target is None or options.filename is None:
sys.exit("Must specify both target and filename.")
print('This tool is highly experimental, use with care.')
rewriter = mesonbuild.astinterpreter.AstInterpreter(options.sourcedir, '')
try:
if options.commands[0] == 'add':
rewriter.add_source(options.target, options.filename)
elif options.commands[0] == 'remove':
rewriter.remove_source(options.target, options.filename)
else:
sys.exit('Unknown command: ' + options.commands[0])
except Exception as e:
if isinstance(e, MesonException):
if hasattr(e, 'file') and hasattr(e, 'lineno') and hasattr(
e, 'colno'):
mlog.log(
mlog.red(
'\nMeson encountered an error in file %s, line %d, column %d:'
% (e.file, e.lineno, e.colno)))
else:
mlog.log(mlog.red('\nMeson encountered an error:'))
mlog.log(e)
else:
traceback.print_exc()
sys.exit(1)
def _checklog(self, which, what, ok):
status = mlog.green('YES') if ok else mlog.red('NO')
mlog.log('Checking that', which, mlog.bold(what, True), 'exists:', status)
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('case', type=pathlib.Path, help='The test case to run')
parser.add_argument('--subtest',
type=int,
action='append',
dest='subtests',
help='which subtests to run')
parser.add_argument('--backend',
action='store',
help="Which backend to use")
parser.add_argument('--cross-file',
action='store',
help='File describing cross compilation environment.')
parser.add_argument('--native-file',
action='store',
help='File describing native compilation environment.')
parser.add_argument('--use-tmpdir',
action='store_true',
help='Use tmp directory for temporary files.')
args = T.cast('ArgumentType', parser.parse_args())
setup_commands(args.backend)
detect_system_compiler(args)
print_tool_versions()
test = TestDef(args.case, args.case.stem, [])
tests = load_test_json(test, False)
if args.subtests:
tests = [t for i, t in enumerate(tests) if i in args.subtests]
def should_fail(path: pathlib.Path) -> str:
dir_ = path.parent.stem
# FIXME: warning tets might not be handled correctly still…
if dir_.startswith(('failing', 'warning')):
if ' ' in dir_:
return dir_.split(' ')[1]
return 'meson'
return ''
results = [
run_test(t, t.args, should_fail(t.path), args.use_tmpdir)
for t in tests
]
failed = False
for test, result in zip(tests, results):
if result is None:
is_skipped = True
skip_reason = 'not run because preconditions were not met'
else:
for l in result.stdo.splitlines():
if test_emits_skip_msg(l):
is_skipped = True
offset = l.index('MESON_SKIP_TEST') + 16
skip_reason = l[offset:].strip()
break
else:
is_skipped = False
skip_reason = ''
if is_skipped:
msg = mlog.yellow('SKIP:')
elif result.msg:
msg = mlog.red('FAIL:')
failed = True
else:
msg = mlog.green('PASS:'******'Reason:'), skip_reason)
if result is not None and result.msg and 'MESON_SKIP_TEST' not in result.stdo:
mlog.log('reason:', result.msg)
if result.step is BuildStep.configure:
# For configure failures, instead of printing stdout,
# print the meson log if available since it's a superset
# of stdout and often has very useful information.
mlog.log(result.mlog)
else:
mlog.log(result.stdo)
for cmd_res in result.cicmds:
mlog.log(cmd_res)
mlog.log(result.stde)
exit(1 if failed else 0)