def test_programs(args):
# TODO: This is a copy-paste of test_backend()
if args.build:
from tools.build import build_grammar, build_caramel
build_grammar(args)
build_caramel(args)
if args.interactive and len(args.test_files) > 0:
logger.warn('Running in interactive mode, ignoring test files.')
args.test_files = []
if args.all:
args.test_files = None
# Run the tests
backend_tests = BackendTests()
if args.interactive:
backend_tests.add_test('interactive test', '', False)
else:
backend_tests.discover(PATHS['programs-test-dir'],
only=args.test_files)
backend_tests.run_all(
open_gui=args.gui,
open_gui_on_failure=args.gui_on_failure,
show_stdout=args.stdout,
show_stderr=args.stderr,
)
print_failed_tests()
def build(args):
if not args.grammar and not args.caramel and not args.all:
logger.warn('Nothing to build.')
return
logger.info('Build started...')
start_time = time()
if args.debug and not args.caramel:
logger.warn('--release ignored because -c / --caramel is absent.')
args.debug = False
if args.all:
args.grammar = True
args.caramel = True
if args.grammar:
build_grammar(args)
if args.caramel:
build_caramel(args)
total_time = time() - start_time
logger.info('Build finished. Total time: {}.'.format(
colored(seconds_to_string(total_time), color='yellow')))
def test_semantic(args):
logger.info('Running semantic tests...')
if args.build:
from tools.build import build_grammar, build_caramel
build_grammar(args)
build_caramel(args)
if args.interactive and len(args.test_files) > 0:
logger.warn('Running in interactive mode, ignoring test files.')
args.test_files = []
if args.all:
args.test_files = None
# Run the tests
semantic_tests = SemanticTests()
if args.interactive:
semantic_tests.add_test('interactive test', '', False)
else:
semantic_tests.discover(PATHS['semantic-test-dir'],
only=args.test_files)
semantic_tests.run_all(
open_gui=args.gui,
open_gui_on_failure=args.gui_on_failure,
show_stdout=args.stdout,
show_stderr=args.stderr,
)
print_failed_tests()
def test_backend(args):
# TODO: Refactor => factorize semantic and back-end?
logger.info('Running back-end tests...')
if args.build:
from tools.build import build_grammar, build_caramel
build_grammar(args)
build_caramel(args)
if args.interactive and len(args.test_files) > 0:
logger.warn('Running in interactive mode, ignoring test files.')
args.test_files = []
if args.all:
args.test_files = None
# Run the tests
backend_tests = BackendTests()
if args.interactive:
backend_tests.add_test('interactive test', '', False)
else:
backend_tests.discover(PATHS['backend-test-dir'], only=args.test_files)
backend_tests.run_all(
open_gui=args.gui,
open_gui_on_failure=args.gui_on_failure,
show_stdout=args.stdout,
show_stderr=args.stderr,
)
print_failed_tests()
def test_grammar(args):
logger.info('Running grammar tests...')
if args.build:
from tools.build import build_grammar
build_grammar(args)
if args.interactive and len(args.test_files) > 0:
logger.warn('Running in interactive mode, ignoring test files.')
args.test_files = []
if args.all:
args.test_files = None
# Run the tests
grammar_tests = GrammarTests()
grammar_tests.check_build()
if args.interactive:
grammar_tests.add_test('interactive test', '', False)
else:
grammar_tests.discover(PATHS['grammar-test-dir'], only=args.test_files)
grammar_tests.run_all(
open_gui=args.gui,
open_gui_on_failure=args.gui_on_failure,
show_stdout=args.stdout,
show_stderr=args.stderr,
)
print_failed_tests()
def main():
try:
_chef()
except KeyboardInterrupt:
logger.warn('Chef was ordered to stop early.')
exit(1)
except Exception as e:
logger.critical(
colored('An exception occurred:', 'red', None, ['bold']), e)
raise e
def print_failed_tests():
if len(failed_tests) > 0:
logger.warn('{} failed tests:'.format(len(failed_tests)), failed_tests)
def execute(self,
open_gui=False,
open_gui_on_failure=False,
show_stdout=False,
show_stderr=False):
start_time = time()
logger.info('Testing {}...'.format(self.display_name))
# Get the GCC outputs
gcc_flags = '-O0 -mno-red-zone -Wno-implicit-function-declaration' # -Wall -Wextra -Wpedantic'
gcc_build_command = 'gcc {} {} -o ./build/cpp-bin/gcc.out'.format(
gcc_flags, self.full_path)
gcc_run_command = './build/cpp-bin/gcc.out'
logger.trace('GCC build command:', gcc_build_command)
exec_(gcc_build_command)
logger.trace('GCC exec command:', gcc_run_command)
with subprocess.Popen(shlex.split(gcc_run_command),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as test_process:
test_process.wait()
# Get stdout and stderr
gcc_stdout = list(
map(lambda s: s.decode("utf-8"),
test_process.stdout.readlines()))
gcc_stderr = list(
map(lambda s: s.decode("utf-8"),
test_process.stderr.readlines()))
initial_cwd = os.getcwd()
os.chdir('./build/cpp-bin')
# Get the Caramel outputs
compile_command = './Caramel --good-defaults {}'.format(
os.path.join('../..', self.full_path))
assemble_command = 'gcc ./assembly.s -no-pie -o ./caramel.out'
run_command = './caramel.out'
# Compile with Caramel
logger.trace('Compile command:', compile_command)
if len(self.full_path) == 0: # Interactive test
print('Enter back-end test input: (ended by ^D)')
with subprocess.Popen(shlex.split(compile_command),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={'LD_LIBRARY_PATH':
'../../lib'}) as test_process:
test_process.wait()
# Get stdout and stderr
out_str = list(
map(lambda s: s.decode("utf-8"),
test_process.stdout.readlines()))
error_str = list(
map(lambda s: s.decode("utf-8"),
test_process.stderr.readlines()))
# Save the test state
caramel_state = {
'stderr': error_str,
'stdout_lines': sum(len(line.strip()) for line in out_str),
'stderr_lines': sum(len(line.strip()) for line in error_str),
'return_code': test_process.returncode,
'time': time() - start_time
}
if caramel_state['stdout_lines'] != 0:
logger.warn('Caramel wrote on stdout!')
# Assemble with GCC
logger.trace('Assemble command:', assemble_command)
exec_(assemble_command)
# Execute
try:
logger.trace('Run command:', run_command)
with subprocess.Popen(shlex.split(run_command),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as test_process:
test_process.wait()
# Get stdout and stderr
out_str = list(
map(lambda s: s.decode("utf-8"),
test_process.stdout.readlines()))
error_str = list(
map(lambda s: s.decode("utf-8"),
test_process.stderr.readlines()))
# Save the test state
self.state = {
'stdout_lines':
sum(len(line.strip()) for line in out_str),
'stderr_lines':
sum(len(line.strip()) for line in error_str),
'caramel_stderr':
caramel_state['stderr'],
'caramel_stderr_lines':
caramel_state['stderr_lines'],
'gcc_stdout_lines':
sum(len(line.strip()) for line in gcc_stdout),
'gcc_stderr_lines':
sum(len(line.strip()) for line in gcc_stderr),
'correct_stdout':
out_str == gcc_stdout,
'return_code':
test_process.returncode,
'time':
time() - start_time
}
if self.state['stderr_lines'] != 0:
logger.warn(
'Unhandled: the test program wrote on stderr. Ignoring.'
)
# Determine if unexpected errors, or successes, occurred
errors = not self.state['correct_stdout']
self.succeeded = errors if self.should_fail else not errors
# Feed our user
if self.succeeded:
logger.info(
'Test {}'.format(self.display_name),
colored('succeeded.', color='green', attrs=['bold']),
colored('[%s]' % seconds_to_string(self.state['time']),
color='yellow'))
else:
logger.info(
'Test {}'.format(self.display_name),
colored('failed #{}.'.format(
_return_code_to_str(test_process.returncode)),
color='red',
attrs=['bold']),
colored('[%s]' % seconds_to_string(self.state['time']),
color='yellow'))
failed_tests.append(self.display_name)
if open_gui_on_failure and not open_gui:
self.execute(open_gui=True, open_gui_on_failure=False)
# Show stdout or stderr if asked
if show_stdout or open_gui:
if self.state['stdout_lines'] == 0 and self.state[
'gcc_stdout_lines'] == 0:
print(colored('No stdout output.', attrs=['bold']))
else:
print('\n'.join([
'#' * 20,
colored('GCC stdout:', attrs=['bold']),
''.join(gcc_stdout),
]))
print('\n'.join([
colored('Caramel-compiled stdout:',
attrs=['bold']),
''.join(out_str),
'-' * 20,
]))
if show_stderr or open_gui:
if self.state['caramel_stderr_lines'] == 0 and self.state[
'gcc_stderr_lines'] == 0:
print(colored('No stderr output.', attrs=['bold']))
else:
print('\n'.join([
'#' * 20,
colored('GCC stderr:', attrs=['bold']),
colored(''.join(gcc_stderr), color='grey'),
]))
print('\n'.join([
colored('Caramel stderr:', attrs=['bold']),
''.join(self.state['caramel_stderr']),
'-' * 20,
]))
except FileNotFoundError:
print("Caramel's stderr:")
print(''.join(caramel_state['stderr']))
exit(1)
os.chdir(initial_cwd)
def _chef():
start_time = time()
# create the top-level parser
parser = argparse.ArgumentParser(
description='The Caramel Jack of all trades.')
group_verbosity = parser.add_mutually_exclusive_group()
group_verbosity.add_argument(
'--verbose',
'-v',
help='increase the verbosity (repeat for even more verbosity)',
dest='verbosity',
action='count',
default=4)
group_verbosity.add_argument(
'--quiet',
'-q',
help='decrease the verbosity (repeat for less verbosity)',
action='count',
default=0)
subparsers = parser.add_subparsers(title='Available commands',
dest='subcommand_name')
# create the parser for the "clean" command
parser_clean = subparsers.add_parser(
'clean', help='Ask the Chef to clean up his workplace.')
parser_clean.set_defaults(func=tools.clean.clean)
# create the parser for the "build" command
parser_build = subparsers.add_parser(
'build', help='Make the Chef cook some Caramel.')
parser_build.add_argument('-g',
'--grammar',
help='build the grammar',
action='store_true')
parser_build.add_argument('-c',
'--caramel',
help='build the compiler',
action='store_true')
parser_build.add_argument('-d',
'--debug',
help='build as debug',
action='store_true')
parser_build.add_argument('-a',
'--all',
help='build everything',
action='store_true')
parser_build.set_defaults(func=tools.build.build)
# create the parser for the "test" command
parser_test = subparsers.add_parser('test',
help='Test the Caramel quality.')
# create the "test" command common arguments
def test_common(sub_test_parser: argparse.ArgumentParser):
sub_test_parser.add_argument('-b',
'--build',
help='build before running tests',
action='store_true')
sub_test_parser.add_argument('-O',
'--stdout',
help='show the tests stdout output',
action='store_true')
sub_test_parser.add_argument('-E',
'--stderr',
help='show the tests stderr output',
action='store_true')
sub_test_parser.add_argument('-i',
'--interactive',
help='run a test in interactive mode',
action='store_true')
group_test_grammar_gui = sub_test_parser.add_mutually_exclusive_group()
group_test_grammar_gui.add_argument(
'-g',
'--gui',
help='open a GUI when executing test',
action='store_true')
group_test_grammar_gui.add_argument('-G',
'--gui-on-failure',
help='open a GUI on failed tests',
action='store_true')
sub_test_parser.add_argument('-a',
'--all',
help='run all tests',
action='store_true')
sub_test_parser.add_argument('test_files',
nargs='*',
help='test files to test')
# Create the "test" sub-commands
test_subparsers = parser_test.add_subparsers(
title='Available sub-commands')
# Create the parser for the "test grammar" command
parser_test_grammar = test_subparsers.add_parser(
'grammar', help='Test the Caramel grammar.')
parser_test_grammar.set_defaults(func=tools.test.test_grammar)
test_common(parser_test_grammar)
# Create the parser for the "test semantic" command
parser_test_semantic = test_subparsers.add_parser(
'semantic', help='Test the Caramel semantic analysis.')
parser_test_semantic.set_defaults(func=tools.test.test_semantic)
test_common(parser_test_semantic)
parser_test_semantic.add_argument('-d',
'--debug',
help='run Caramel as debug',
action='store_true')
# Create the parser for the "test backend" command
parser_test_backend = test_subparsers.add_parser(
'backend', help='Test the Caramel back-end.')
parser_test_backend.set_defaults(func=tools.test.test_backend)
test_common(parser_test_backend)
parser_test_backend.add_argument('-d',
'--debug',
help='run Caramel as debug',
action='store_true')
# Create the parser for the "test programs" command
parser_test_programs = test_subparsers.add_parser(
'programs', help='Test the execution of some example programs.')
parser_test_programs.set_defaults(func=tools.test.test_programs)
test_common(parser_test_programs)
parser_test_programs.add_argument('-d',
'--debug',
help='run Caramel as debug',
action='store_true')
# Create the parser for the "test all" command
parser_test_all = test_subparsers.add_parser('all', help='Run all tests.')
parser_test_all.set_defaults(func=tools.test.test_all)
test_common(parser_test_all)
parser_test_all.add_argument('-d',
'--debug',
help='run Caramel as debug',
action='store_true')
# parse the command line and call the appropriate submodule
args = parser.parse_args()
logger.level = LoggerLevel(args.verbosity - args.quiet)
if args.subcommand_name is None:
logger.warn('You forgot to specify the subcommand. Use -h for help.')
parser.print_usage()
exit(1)
else:
args.func(args)
logger.info('Completed in {}.'.format(
colored(seconds_to_string(time() - start_time), color='yellow')))