def test_comment(self):
txt = '<!-- In the interest of restricting article length, please limit this section to two or three short ' \
'paragraphs and add any substantial information to the main Issues in anarchism article. Thank you. ' \
'--> '
parser = Parser()
ast = parser.parse(txt, Grammar.comment)
print(ast, Compiler().render(ast))
def test_link(self):
txt = '[[File:Nearest_stars_rotating_red-green.gif|alt=Rotating 3D image of the nearest stars|thumb|Animated 3D map of the nearest stars, centered on the Sun. {{3d glasses|color=red green}}]]'
txt2 = '[[File:William Shea.jpg|thumb|upright|[[William Shea]] was instrumental in returning [[National League|National League baseball| [[asd|{{asd}}]]]] to [[New York City]] after five years of absence.]]'
txt3 = '[[asd]]'
parser = Parser()
ast = parser.parse(txt2, Grammar.link)
print(ast)
return ast
def test_headings(self):
txt = '==asd=='
txt3 = '===asd==='
txt4 = '====asd===='
txt5 = '=====asd====='
txt6 = '======asd======'
parser = Parser()
ast = parser.parse(txt, expression=Grammar.headings)
print(ast)
return ast
def shell(fn, text):
# Generate tokens
lexer = Lexer(fn, text)
tokens, exception = lexer.make_tokens()
if exception:
return None, exception
# Generate AST
parser = Parser(tokens)
ast = parser.parse()
return ast.node, ast.error
def test_parse(self, name='wikitext'):
with (DATA_FOLDER / name).open(encoding="utf8") as f:
text = f.read()
t0 = time.time()
# lexer = Lexer()
# tokens = lexer.tokenize(text)
parser = Parser()
ast = parser.parse(text)
t1 = time.time()
# print(ast)
print('Ast built in: ', t1 - t0)
return ast
def test_template(self):
parser = Parser()
ast = parser.parse('{{asd}}', Grammar.template)
print(ast)
return ast
class Interpreter:
"""
Math expressions interpreter
Supported types:
Numbers(int or floats)
Complex Numbers(using i)
Matrices
Variables
Functions
Variables and functions could be defined usign assignment operator
Also, simple equations of degree 0-2 is supported. Every term should be in correct form:
[coefficient][*][variable][^degree]
Predefined math functions: sin, cos, tan, log, abs, sqrt, exp
Predefined matrix functions: inv, transp
Predefined special commands: vars, funcs, plot, linreg
"""
def __init__(self):
self._variables = self._init_predefined_variables()
self._functions = self._init_predefined_functions()
self._parser = Parser()
self._tokenizer = Tokenizer()
def _init_predefined_variables(self):
variables = {}
for var_name, var_val in DEFINED_VARS.items():
variables[var_name] = Variable(var_name, var_val)
return variables
def _init_predefined_functions(self):
functions = {}
for func_name, func in DEFINED_FUNCS.items():
func_obj = SpecialNumericFunction(func_name, func)
functions[func_name] = func_obj
functions["inv"] = MatrixInversionFunc("inv")
functions["transp"] = MatrixTransposeFunc("transp")
return functions
def read_eval_print_loop(self) -> None:
"""
Infinite REP loop which stops after key interrupt
"""
while True:
try:
input_string = input(">")
except (EOFError, KeyboardInterrupt):
break
self.eval_print_string(input_string)
def read_eval_print_file(self, filename: str) -> None:
file = open(filename, "r")
for line in file:
self.eval_print_string(line)
def eval_print_string(self, input_string: str) -> None:
try:
if input_string:
output_string = self.eval_string(input_string)
if output_string is not None:
print(output_string)
except (ParsingError, EvalException, MathException, TokenizationError) as e:
print("ERROR: ", str(e))
def eval_string(self, string: str) -> str:
"""
:param string: input to interpreter
:return: output as string
"""
tokens = self._tokenizer.tokenize(string)
objs = self._parser.parse(tokens)
op_type, left, right = self._recognize_operation_type(objs)
if op_type == "assignment":
eval_res = self._make_assignment(left, right)
elif op_type == "evaluation":
eval_res = Expression(left).evaluate(self._variables, self._functions)
elif op_type == "equation":
equation = Equation(left, right[:-1], self._variables, self._functions)
eval_res = equation.solve()
elif op_type == "special":
spec_comm = SPECIAL_COMMANDS[left[0].name]
eval_res = spec_comm.evaluate(left[0].input, self._variables, self._functions)
elif op_type == "print_func": # kostyl
if left[0].name not in self._functions:
raise FunctionNotExists(left[0].name)
f = self._functions[left[0].name]
return str(f.body)
else:
raise Exception("Shouldn't be here man")
return str(eval_res) if eval_res else None
def _make_assignment(self, left: List, right: List) -> str:
"""
Tries to assign right part to left
:param left: list of objects on left part of assignment
:param right: list of objects on right part of assignment
:return: string which describes assignment
"""
if len(left) != 1:
raise WrongAssingmentLeftPart(left[0] if len(left) else None)
left = left[0]
if isinstance(left, Variable):
right_part_evaluated = Expression(right).evaluate(self._variables, self._functions)
left.val = right_part_evaluated
self._variables[left.name] = left
output = right_part_evaluated
elif isinstance(left, AFunction):
if len(left.input) != 1 or not isinstance(left.input.body[0], Variable):
raise WrongAssingmentLeftPart(left.input)
left_input_variable = left.input.body[0]
func_body = Expression(right)
if self._func_body_is_recursive(left.name, right):
raise FunctionIsRecursive(left.name)
func_body.evaluate_variables(self._variables, exceptions=[left_input_variable])
left.body = func_body
left.input = left_input_variable
self._functions[left.name] = left
output = str(left)
else:
raise WrongAssingmentLeftPart(left)
return output
def _func_body_is_recursive(self, func_name, func_body):
"""
Checks if func_body contains any calls to func_name, including one that nested
:param func_name: name of checked function
:param func_body: list of objects
:return: True or False
"""
for obj in func_body:
if isinstance(obj, AFunction):
if obj.name == func_name:
return True
if self._func_body_is_recursive(func_name, obj.input.body):
return True
if obj.name in self._functions and isinstance(self._functions[obj.name], UserDefinedFunction):
if self._func_body_is_recursive(func_name, self._functions[obj.name].body.body):
return True
return False
@staticmethod
def _recognize_operation_type(expr: List) \
-> Tuple[str, List, List]:
"""
Goes through expression, tries to find strange errors and recognize,
what type of expression this is: "evaluation", "assignment" or "equation"
:param expr: list with operators and operands
:return: (one of "evaluation", "assignment" or "equation",
list of expression parts, splitted by "=" operator)
"""
assignment_indices = []
question_mark = False
# find assignment operators and question marks. Raise exception if question mark not at the end
for i, obj in enumerate(expr):
if isinstance(obj, Operator) and obj.op == "=":
assignment_indices.append(i)
elif isinstance(obj, Operator) and obj.op == "?":
if i != len(expr)-1:
raise UnexpectedToken(obj.op)
question_mark = True
if len(assignment_indices) > 1:
raise TooManyAssignments()
# split input in two parts by assignment operator
if not assignment_indices:
left, right = expr, None
else:
left = expr[:assignment_indices[0]]
right = expr[assignment_indices[0]+1:]
if Interpreter._is_special_command(left, right):
op_type = "special"
elif (question_mark and len(right) == 1 and len(left) == 1 and isinstance(left[0], AFunction) and
len(left[0].input.body) == 1 and isinstance(left[0].input.body[0], Variable)):
op_type = "print_func" # stupid case for function definition printing
elif right is None or (question_mark and len(right) == 1):
op_type = "evaluation" # 'expression = ?' or no assignment operator line
elif question_mark and len(right) > 1:
op_type = "equation"
elif assignment_indices and not question_mark:
op_type = "assignment"
else:
raise Exception("Shouldn't be here man")
return op_type, left, right
@staticmethod
def _is_special_command(left, right):
special_funcs = []
for obj in left:
if isinstance(obj, AFunction) and obj.name in SPECIAL_COMMANDS:
special_funcs.append(obj)
if special_funcs:
if len(left) > 1 or right is not None:
raise WrongSpecialCommandUse()
return True
else:
return False
class InterpreterTest(unittest.TestCase):
def setUp(self) -> None:
self.source = TestSource()
self.lexer = TestLexer(self.source)
self.parser = Parser(self.lexer)
self.interpreter = Interpreter()
def interpret(self, text):
self.source.put_text(text)
self.lexer.lex()
ast = self.parser.parse()
return self.interpreter.interpret(ast)
def test_interpreting_int_value(self):
result = self.interpret('1;')
self.assertEqual('1', str(result))
def test_interpreting_double_value(self):
result = self.interpret('1.0;')
self.assertEqual('1.0', str(result))
def test_interpreting_string_value(self):
result = self.interpret('"hakuna matata";')
self.assertEqual('hakuna matata', str(result))
def test_interpreting_bool_value(self):
result = self.interpret('true;')
self.assertEqual('true', str(result))
def test_interpreting_phys_value(self):
result = self.interpret('3&|m/s|;')
self.assertEqual('3*(m^1/s^1)', str(result))
def test_interpreting_unit_value(self):
result = self.interpret('|m/s*s*s|;')
self.assertEqual('(m^1/s^3)', str(result))
def test_interpreting_if_statement(self):
statement = """
int x = 5;
if (x==5){
bool y = true;
}
else{
bool y=false;
}
y;
"""
result = self.interpret(statement)
self.assertEqual('true', str(result))
def test_interpreting_while_statement(self):
statement = """
int counter = 1; int x = 2;
while(counter < 5){
x = x * 2;
counter = counter + 1;
}
x;
"""
result = self.interpret(statement)
self.assertEqual('32', str(result))
def test_interpreting_function_statement(self):
statement = """
function multiply_phys_values(a:phys,b:phys)->phys{
return a*b;
}
phys x = 1&|m/s|; phys y = 3&|m/s*s|;
phys result = multiply_phys_values(x,y);
result;
"""
result = self.interpret(statement)
self.assertEqual('3*(m^2/s^3)', str(result))
def test_interpreting_multiple_ifs_statement(self):
statement = """
int x = 5;
int result = 0;
if (x < 4){
result = 1;
}
elseif(x > 6){
result = 2;
}
elseif (x==5){
result = 3;
}
result;
"""
result = self.interpret(statement)
self.assertEqual('3', str(result))
def main():
parser = argparse.ArgumentParser(description="Esoteric C compiler")
parser.add_argument('infiles',
metavar='infile',
type=str,
nargs='+',
help="Input files, can be either C or ASM")
parser.add_argument('-o',
dest='outfile',
metavar='outfile',
type=str,
default='a.out',
required=False,
help="Place the output into <outfile>")
parser.add_argument(
'-E',
dest='preprocess_only',
action='store_const',
const=True,
default=False,
help="Preprocess only; do not compile, assemble or link.")
parser.add_argument('-S',
dest='compile_only',
action='store_const',
const=True,
default=False,
help="Compile only; do not assemble or link.")
parser.add_argument('-c',
dest='assemble_only',
action='store_const',
const=True,
default=False,
help="Compile and assemble, but do not link.")
parser.add_argument('-D',
dest='defines',
metavar='macro[=val]',
nargs=1,
action='append',
help='Predefine name as a macro [with value]')
parser.add_argument('-I',
dest='includes',
metavar='path',
nargs=1,
action='append',
help="Path to search for unfound #include's")
parser.add_argument('--dump-ir',
dest='dump_ir',
action='store_const',
const=True,
default=False,
help="Dump the IR into a file")
parser.add_argument('--dump-ast',
dest='dump_ast',
action='store_const',
const=True,
default=False,
help="Dump the AST into a file")
args = parser.parse_args()
################################################
# preprocess all files
################################################
preprocessor = pcpp.Preprocessor()
preprocessor.add_path('.')
if args.includes is not None:
for path in args.includes:
preprocessor.add_path(path)
# TODO: pass defines
#
# Figure all the files
#
files = []
asms = []
objects = []
for file in args.infiles:
if file.endswith('.c'):
preprocessor.parse(open(file), file)
s = StringIO()
preprocessor.write(s)
code = s.getvalue()
files.append((code, file))
elif file.endswith('.S'):
asms.append((file, open(file).read()))
elif file.endswith('.o'):
obj = pickle.Unpickler(open(file, 'rb')).load()
objects.append((obj, file))
else:
assert False, f"Unknown file extension {file}"
#
# If preprocess just print the preprocessed files
#
if args.preprocess_only:
for code, file in files:
print(code)
return
#
# Compile all c files
#
for code, file in files:
# Parse the code into an ast
parser = Parser(code, filename=file)
parser.parse()
if parser.got_errors:
exit(1)
assert not parser.got_errors
# Optimize the AST
opt = Optimizer(parser)
opt.optimize()
if args.dump_ast:
with open(file[:-2] + '.ast', 'w') as f:
for func in opt.parser.func_list:
f.write(str(func) + '\n')
# Now we need to translate it into
# the ir code
trans = IrTranslator(parser)
trans.translate()
if args.dump_ir:
with open(file[:-2] + '.ir', 'w') as f:
p = Printer()
for proc in trans.proc_list:
f.write(proc.get_name() + ":\n")
for inst in proc.get_body():
f.write('\t' + p.print_instruction(inst) + '\n')
# Now run it through the ir translator for
# the dcpu16
code_trans = Dcpu16Translator()
for proc in trans.proc_list:
code_trans.translate_procedure(proc)
asm = code_trans.get_asm()
# Run the code through the peephole optimizer
optimizer = Dcpu16PeepholeOptimizer()
asm = optimizer.optimize(asm)
# Add externs for any unknown label
for func in parser.func_list:
if func.prototype:
asm += f'\n.extern {func.name}\n'
# Add global vars definitions
for var in parser.global_vars:
if var.storage == StorageClass.EXTERN:
asm += f'\n.extern {var.ident.name}\n'
else:
if var.storage != StorageClass.STATIC:
asm += f'\n.global {var.ident.name}\n'
asm += f'{var.ident.name}:\n'
if var.value is None:
asm += f'\t.fill {var.typ.sizeof()}, 0\n'
else:
asm += f'\t.dw {var.value}\n'
asms.append((asm, file))
#
# If we only do compilation then save the assembly files
#
if args.compile_only:
for asm, file in asms:
with open(file[:-2] + '.S', 'w') as f:
f.write(asm)
return
#
# Assemble all assembly files
#
for asm, file in asms:
asm = Dcpu16Assembler(asm, file)
asm.parse()
asm.fix_labels()
if asm.got_errors:
exit(1)
assert not asm.got_errors
objects.append((asm.get_object(), file))
#
# If only assemble save the object files
#
if args.assemble_only:
for obj, file in objects:
pickle.Pickler(open(file[:-2] + '.o', 'wb')).dump(obj)
return
#
# Link everything
#
linker = Dcpu16Linker()
for obj, file in objects:
linker.append_object(obj)
linker.link(BinaryType.RAW)
#
# Output the final binary
#
with open(args.outfile, 'wb') as f:
for word in linker.get_words():
f.write(struct.pack('>H', word))
def evaluate(self, source_type, file_path=None):
lexer = StdInLexer() if source_type == 'stdin' else FileLexer(
file_path)
parser = Parser(lexer)
ast = parser.parse()
return self.interpreter.interpret(ast)
class ParserTest(unittest.TestCase):
def setUp(self) -> None:
self.source = TestSource()
self.lexer = TestLexer(self.source)
self.parser = Parser(self.lexer)
def parse(self, text):
self.source.put_text(text)
self.lexer.lex()
return self.parser.parse()
def test_parsing_int_value(self):
result = self.parse('1;')
self.assertEqual('(int value:1)', str(result))
def test_parsing_double_value(self):
result = self.parse('1.0;')
self.assertEqual('(double value:1.0)', str(result))
def test_parsing_string_value(self):
result = self.parse('"text";')
self.assertEqual('(string value:text)', str(result))
def test_parsing_true_value(self):
result = self.parse('true;')
self.assertEqual('(true)', str(result))
def test_parsing_false_value(self):
result = self.parse('false;')
self.assertEqual('(false)', str(result))
def test_parsing_phys_value(self):
result = self.parse('3&|m/s|;')
self.assertEqual('((Phys: int value:3*(Unit:m^1s^-1)))', str(result))
def test_parsing_unit_value(self):
result = self.parse('|m*m/s*n*x|;')
self.assertEqual('((Unit:m^2s^-1n^-1x^-1))', str(result))
def test_parsing_while_statement_value(self):
result = self.parse('while (true) {1;}')
self.assertEqual('((While: true Do:(int value:1)))', str(result))
def test_parsing_if_statement_value(self):
result = self.parse('if (false) {"x";} else{3;}')
self.assertEqual('((If:(false, (string value:x))(int value:3)))',
str(result))
def test_parsing_variable_assignment_value(self):
result = self.parse('int x = 5;')
self.assertEqual('((Assignment: int identifier:x=int value:5))',
str(result))
def test_parsing_variable_access_value(self):
result = self.parse('x;')
self.assertEqual('(identifier:x)', str(result))
def test_parsing_function_definition(self):
result = self.parse('function add (a:int, b:int)->int{ return a+b; }')
self.assertEqual(
'((Function:identifier:add->int Args:[(identifier:a:int), (identifier:b:int)]'
' Body:(<Return> (identifier:a+identifier:b))))', str(result))
def test_parsing_type_int(self):
result = self.parse('int x = 4;')
self.assertEqual('((Assignment: int identifier:x=int value:4))',
str(result))
def test_parsing_type_string(self):
result = self.parse('string s = "string";')
self.assertEqual(
'((Assignment: string identifier:s=string value:string))',
str(result))
def test_parsing_type_bool(self):
result = self.parse('bool v = true;')
self.assertEqual('((Assignment: bool identifier:v=true))', str(result))
def test_parsing_type_double(self):
result = self.parse('double v = 2.0;')
self.assertEqual(
'((Assignment: double identifier:v=double value:2.0))',
str(result))
def test_parsing_type_phys(self):
result = self.parse('phys v = 3&|m/s|;')
self.assertEqual(
'((Assignment: phys identifier:v=(Phys: int value:3*(Unit:m^1s^-1))))',
str(result))
def test_parsing_type_unit(self):
result = self.parse('unit u = |a*s*x/c|;')
self.assertEqual(
'((Assignment: unit identifier:u=(Unit:a^1s^1x^1c^-1)))',
str(result))
