def get_imported_declarations(base_path, imports, imported=None):
"""Given a list of imports and a list of paths to search for them in,
return a list of declarations containing the imported declarations. The
returned list of declarations can then be prepended to the the declarations
in the input file. This function will recursively import if required.
:param base_file str: the absolute path to the file we are currently
compiling
:param imports [str]: a list of imports -- these are relative paths to
.fky files
:return: a list of declarations from the imported files
"""
log.info("Finding declarations for imports '{}' relative to {}.".format(
", ".join(imports), base_path))
if imported is None:
imported = set([base_path])
else:
log.debug("'{}' already imported.".format(", ".join(imported)))
imported.add(base_path)
base_path = os.path.abspath(base_path) # just in case...
if not os.path.isdir(base_path):
base_dir = os.path.dirname(base_path)
else:
log.debug(
"Base path is a directory; you are probably running the REPL.")
base_dir = base_path
decls = []
log.debug("Building parser to read imported source files...")
parser = FunkyParser()
parser.build()
log.debug("Done building parser.")
def do_import(base_dir, imp):
filename = search_for_import(imp, [base_dir] + SEARCH_PATHS)
if filename in imported: return
imported.add(filename)
with open(filename, "r") as f:
source = f.read()
parsed = parser.do_parse(source)
for sub_imp in parsed.body.imports:
log.debug("Found more imports, handling them.")
new_base_dir = os.path.dirname(filename)
do_import(new_base_dir, sub_imp)
decls.extend(parsed.body.toplevel_declarations)
for imp in imports:
do_import(base_dir, imp)
log.info("Found all imported declarations.")
return decls
def test_sanity(self):
sanity_fails = [
"""
module test with
tau = 2 * pi
""",
"""
module test with
f 0 0 = 0
f x y z = 1
""",
"""
module test with
f x x = 1
""",
"""
module test with
to_int = 10
""",
"""
module test with
to_str = 10
""",
"""
module test with
to_float = 10
""",
"""
module test with
newtype Test = P Integer | P Float
""",
"""
module test with
newtype Test = P Integer
newtype Test2 = P Float
""",
"""
module test with
x = 2
x = 3
""",
]
parser = FunkyParser()
parser.build()
for test in sanity_fails:
ast = parser.do_parse(test)
with self.assertRaises(FunkyRenamingError):
do_rename(ast)
def __init__(self, lazy=False):
start = time.time()
super().__init__()
self.intro = cgreen("\nfunkyi ({}) repl".format(__version__)) + \
"\nReady!\nFor help, use the ':help' command.\n"
self.prompt = cyellow("funkyi> ")
log.debug("Lazy mode {}.".format("enabled" if lazy else "disabled"))
if lazy:
print(cblue("Lazy evalation enabled."))
else:
print(cblue("Lazy evalation disabled."))
# create the various parsers:
log.debug("Creating required parsers...")
self.decl_parser = FunkyParser()
self.decl_parser.build(start="TOPLEVEL_DECLARATIONS")
self.expr_parser = FunkyParser()
self.expr_parser.build(start="EXP")
self.newtype_parser = FunkyParser()
self.newtype_parser.build(start="ADT_DECLARATION")
self.setfix_parser = FunkyParser()
self.setfix_parser.build(start="FIXITY_DECLARATION")
self.import_parser = FunkyParser()
self.import_parser.build(start="IMPORT_STATEMENT")
if lazy:
log.debug("Using lazy code generator for REPL.")
self.py_generator = LazyPythonCodeGenerator()
else:
log.debug("Using strict code generator for REPL.")
self.py_generator = StrictPythonCodeGenerator()
log.debug("Done creating parsers.")
self.reset()
end = time.time()
print(cblue("Startup completed ({0:.3f}s).".format(end - start)))
def test_invalid_parse(self):
invalid_programs = [
"""
module invalid with
main = f x + + 2
""",
"""
module invalid some rubbish with
nothing
""",
"""
module invalid with
n = * 5 2 # '*' needs to be '(*)' for this to be valid!
""",
"""
module invalid with
10 + 10"""
]
parser = FunkyParser()
parser.build()
for invalid_program in invalid_programs:
with self.assertRaises(FunkySyntaxError):
parser.do_parse(invalid_program)
def compiler_lex_and_parse(source, dump_pretty, dump_lexed, dump_parsed):
"""Lexes and parses the source code to get the syntax tree.
:param source str: the Funky source code
:param dump_pretty bool: if true, dump the syntax-highlighted, prettified
code to stdout
:param dump_lexed bool: if true, dump the lexed code to stdout
:param dump_parsed bool: if true, dump the syntax tree to stdout
"""
# lex and parse code
parser = FunkyParser()
parser.build(dump_pretty=dump_pretty, dump_lexed=dump_lexed)
# lexing is done in the same step as parsing -- so we have to tell the
# parser whether we want the lexer's output to be displayed
parsed = parser.do_parse(source)
if dump_parsed:
print(cblue("## DUMPED PARSE TREE"))
print(parsed)
print("")
log.info("Parsing source code completed.")
return parsed
class FunkyShell(CustomCmd):
def __init__(self, lazy=False):
start = time.time()
super().__init__()
self.intro = cgreen("\nfunkyi ({}) repl".format(__version__)) + \
"\nReady!\nFor help, use the ':help' command.\n"
self.prompt = cyellow("funkyi> ")
log.debug("Lazy mode {}.".format("enabled" if lazy else "disabled"))
if lazy:
print(cblue("Lazy evalation enabled."))
else:
print(cblue("Lazy evalation disabled."))
# create the various parsers:
log.debug("Creating required parsers...")
self.decl_parser = FunkyParser()
self.decl_parser.build(start="TOPLEVEL_DECLARATIONS")
self.expr_parser = FunkyParser()
self.expr_parser.build(start="EXP")
self.newtype_parser = FunkyParser()
self.newtype_parser.build(start="ADT_DECLARATION")
self.setfix_parser = FunkyParser()
self.setfix_parser.build(start="FIXITY_DECLARATION")
self.import_parser = FunkyParser()
self.import_parser.build(start="IMPORT_STATEMENT")
if lazy:
log.debug("Using lazy code generator for REPL.")
self.py_generator = LazyPythonCodeGenerator()
else:
log.debug("Using strict code generator for REPL.")
self.py_generator = StrictPythonCodeGenerator()
log.debug("Done creating parsers.")
self.reset()
end = time.time()
print(cblue("Startup completed ({0:.3f}s).".format(end - start)))
@report_errors
@atomic
def do_begin_block(self, arg):
"""Start a block of definitions."""
block_prompt = cyellow("block> ")
end_block = ":end_block" # <- type this to end the block
lines = []
try:
while True:
inp = input(block_prompt)
if inp:
if inp == end_block:
break
lines.append(" " + inp)
except KeyboardInterrupt:
print("^C\n{}".format(cred("Cancelled block.")))
return
self.parse_and_add_declarations(lines)
@report_errors
@atomic
@needs_argument
def do_type(self, arg):
"""Show the type of an expression. E.g.: :type 5"""
expr = self.get_core(arg)
self.global_let.expr = expr
do_type_inference(self.global_let, self.global_types)
print("{} :: {}".format(arg, self.global_let.inferred_type))
@needs_argument
def do_lazy(self, arg):
"""Use ':lazy on' to use lazy evaluation and ':lazy off' to use strict evaluation."""
SWAPPED = "Swapped to {} code generator.".format
ALREADY = "Already using {} code generator; ignoring.".format
if arg.lower() == "on":
log.debug("Now using lazy code generator for REPL.")
if isinstance(self.py_generator, StrictPythonCodeGenerator):
self.py_generator = LazyPythonCodeGenerator()
print(cgreen(SWAPPED("lazy")))
else:
print(ALREADY("lazy"))
elif arg.lower() == "off":
log.debug("Now using strict code generator for REPL.")
if isinstance(self.py_generator, LazyPythonCodeGenerator):
self.py_generator = StrictPythonCodeGenerator()
print(cgreen(SWAPPED("strict")))
else:
print(ALREADY("strict"))
else:
print(cred("Invalid option '{}'. Please specify 'on' or "
"'off'.".format(arg)))
@needs_argument
def do_color(self, arg):
"""Use ':color on' to enable colors and ':color off' to disable them."""
SWAPPED = "Turned colors {}.".format
ALREADY = "Colors are already {}.".format
if arg.lower() == "on":
log.debug("Enabling colors.")
if not funky.globals.USE_COLORS:
funky.globals.USE_COLORS = True
print(cgreen(SWAPPED("on")))
else:
print(ALREADY("on"))
elif arg.lower() == "off":
log.debug("Disabling colors.")
if funky.globals.USE_COLORS:
funky.globals.USE_COLORS = False
print(cgreen(SWAPPED("off")))
else:
print(ALREADY("off"))
else:
print(cred("Invalid option '{}'. Please specify 'on' or "
"'off'.".format(arg)))
@needs_argument
def do_unicode(self, arg):
"""Use ':unicode on' to enable unicode characters and ':unicode off' to disable them."""
SWAPPED = "Unicode printing is now {}.".format
ALREADY = "Unicode printing is already {}.".format
if arg.lower() == "on":
log.debug("Enabling unicode printing.")
if not funky.globals.USE_UNICODE:
funky.globals.USE_UNICODE = True
print(cgreen(SWAPPED("on")))
else:
print(ALREADY("on"))
elif arg.lower() == "off":
log.debug("Disabling unicode printing.")
if funky.globals.USE_UNICODE:
funky.globals.USE_UNICODE = False
print(cgreen(SWAPPED("off")))
else:
print(ALREADY("off"))
else:
print(cred("Invalid option '{}'. Please specify 'on' or "
"'off'.".format(arg)))
def do_list(self, arg):
"""List the current bindings in desuguared intermediate code."""
if not (self.global_types or self.global_let.binds):
print(cgreen("Nothing currently registered."))
return
print(cgreen("Currently registered bindings:"))
if self.global_types:
print("\n".join(str(b) for b in self.global_types))
if self.global_let.binds:
print("\n".join(str(b) for b in self.global_let.binds))
def do_binds(self, arg):
"""List the available bindings."""
if not self.scope.local:
print("No bindings.")
return
print(cgreen("Available bindings:"))
self.scope.pprint_local_binds()
@report_errors
@atomic
@needs_argument
def do_newtype(self, arg):
"""Create an ADT. E.g.: :newtype List = Cons Integer List | Nil"""
parsed = self.newtype_parser.do_parse("newtype {}".format(arg))
self.add_typedefs([parsed])
@report_errors
@atomic
@needs_argument
def do_show(self, arg):
"""Show the compiled code for an expression. E.g.: :show 1 + 1"""
code = self.get_compiled(arg)
print(code)
@report_errors
@needs_argument
def do_setfix(self, arg):
"""Change the fixity of an operator. E.g.: :setfix leftassoc 8 **"""
self.setfix_parser.do_parse("setfix {}".format(arg))
@report_errors
@atomic
@needs_argument
def do_import(self, arg):
"""Import a .fky file into the REPL. E.g.: :import "stdlib.fky"."""
start = time.time()
try:
import_stmt = self.import_parser.do_parse("import {}".format(arg))
cwd = os.path.abspath(os.getcwd())
imports_source = get_imported_declarations(cwd, [import_stmt],
imported=self.imported)
typedefs, code = split_typedefs_and_code(imports_source)
self.add_typedefs(typedefs)
self.add_declarations(code)
end = time.time()
print(cgreen("Successfully imported {0} ({1:.3f}s).".format(arg,
end - start)))
except FunkyError as e:
# if an error occurs, extend the error message to tell the user
# that the error occurred while importing *this* file.
new_msg = "{} (while importing {})".format(e.args[0], arg)
e.args = (new_msg,) + e.args[1:]
raise
@needs_argument
def do_typeclass(self, arg):
"""Prints a quick summary of a typeclass."""
try:
typeclass = TYPECLASSES[arg]
print(arg, typeclass.constraints_str())
except KeyError:
print(cred("Typeclass '{}' does not exist.".format(arg)))
def do_reset(self, arg):
"""Reset the environment (clear the current list of bindings)."""
self.reset()
print(cgreen("All bindings reset."))
def reset(self):
"""Resets the scope and bindings."""
self.imported = set([])
self.scope = Scope()
# global_types is the collection of user-defined type declarations.
self.global_types = []
# global_let is a core let whose bindings are just the bindings the
# user has introduced, and whose expression is 'dynamic' -- it is
# changed each time the user asks for an expression to be evaluated and
# recompiled as a new program to give the new result.
self.global_let = CoreLet([], CoreLiteral(0))
def get_state(self):
return (
copy.copy(self.imported),
copy.deepcopy(self.scope),
copy.deepcopy(self.global_types),
copy.deepcopy(self.global_let)
)
def revert_state(self, state):
self.imported, self.scope, self.global_types, self.global_let = state
def get_core(self, source):
"""Converts a string of Funky code into the intermediate language.
:param source: the source code to convert to the intermediate language
:return: the core code
"""
parsed = self.expr_parser.do_parse(source)
rename(parsed, self.scope)
check_scope_for_errors(self.scope)
core_tree, _ = do_desugar(parsed)
return core_tree
def get_compiled(self, source):
"""Converts a string of funky code into the target source language.
:param source: the source code to convert to the target source language
:return: the compiled code in the target source language
"""
core_expr = self.get_core(source)
self.global_let.expr = core_expr
do_type_inference(self.global_let, self.global_types)
target_source = self.py_generator.do_generate_code(self.global_let,
self.global_types)
return target_source
def parse_and_add_declarations(self, lines):
"""Add a new block of declarations to global_let.
:param lines: the Funky source code lines in the new block of
declarations
"""
# shoehorn the lines into a 'fake' with clause so that they can be
# parsed correctly.
parsed = self.decl_parser.do_parse("func = 0 with\n{}".format(
"\n".join(lines)))
# extract the parsed declarations back out from our 'fake' with clause.
declarations = parsed[0].expression.declarations
self.add_declarations(declarations)
def add_typedefs(self, typedefs):
for typedef in typedefs:
rename(typedef, self.scope)
typedef = desugar(typedef)
self.global_types.append(typedef)
def add_declarations(self, declarations):
self.global_let.expr = CoreLiteral(0)
# rename and desugar each declaration one-by-one, and append each to
# the (new_) global_let binds
for decl in declarations:
rename(decl, self.scope)
core_tree, _ = do_desugar(decl)
self.global_let.binds.append(core_tree)
check_scope_for_errors(self.scope)
self.global_let.binds = condense_function_binds(self.global_let.binds)
self.global_let.create_dependency_graph()
# type infer the new global let to check for inconsistencies
do_type_inference(self.global_let, self.global_types)
def do_EOF(self, line):
"""Exit safely."""
print("^D\nEOF, exiting.")
exit(0)
@report_errors
@atomic
def default(self, arg):
"""This is called when the user does not type in any command in
particular. Since this is a REPL, we should interpret the given
text as an expression or a declaration
"""
# if there are comments, drop them
try:
arg = arg[:arg.index("#")]
if not arg:
return
except ValueError:
pass
try:
# try treating as an expression...
code = self.get_compiled(arg)
print(cblue("= "), end="")
try:
exec(code, {"__name__" : "__main__"})
except Exception as e:
print(cred(str(e)))
except FunkyParsingError:
# if that didn't work, try treating as a declaration
self.parse_and_add_declarations([arg])
def emptyline(self):
"""Empty lines in the REPL do nothing."""
pass
def test_valid_parse(self):
valid_programs = ["""
module test with
main = 10 + 10
""",
"""
module test with
factorial 0 = 1
factorial n = n * factorial (n - 1)
main = factorial 5""",
"""
module test with
area r = pi * r ** 2.0
with pi = 3.141
""",
"""
# Treesort algorithm for integer in Funky.
module treesort with
import "stdlib.fky"
import "intlist.fky"
newtype Tree = Branch Tree Integer Tree | Empty
insert e Empty = Branch Empty e Empty
insert e (Branch l v r) given e <= v = Branch (insert e l) v r
given otherwise = Branch l v (insert e r)
inorder Empty = Nil
inorder (Branch l v r) = inorder l ~concatenate~
unit v ~concatenate~
inorder r
treesort list = inorder (foldr insert Empty list)
main = treesort my_list
with my_list = Cons 5 (Cons 1 (Cons 7 (Cons 7 (Cons 3 Nil))))
""",
"""
# Representing and evaluating expression trees in Funky.
module expressiontree with
newtype Expression = Const Float
| BinOp Expression
(Expression -> Expression -> Float)
Expression
| UnOp (Expression -> Float)
Expression
evaluate (Const x) = x
evaluate (BinOp a f b) = f a b
evaluate (UnOp f x) = f x
add exp1 exp2 = (evaluate exp1) + (evaluate exp2)
sub exp1 exp2 = (evaluate exp1) - (evaluate exp2)
mul exp1 exp2 = (evaluate exp1) * (evaluate exp2)
div exp1 exp2 = (evaluate exp1) / (evaluate exp2)
test_exp = (BinOp (Const 5.0) add (BinOp (Const 3.0) div (Const 2.0)))
main = evaluate test_exp
""",
"""
# Demonstrating the use of Funky's random library to generate a random float.
module randomfloat with
import "random.fky"
seed = 51780
main = randfloat seed
"""]
parser = FunkyParser()
parser.build()
for valid_program in valid_programs:
ast = parser.do_parse(valid_program)
self.assertTrue(isinstance(ast, ASTNode))
from unittest import TestCase
import funky.globals
from funky.parse.funky_parser import FunkyParser
from funky.rename.rename import do_rename
from funky.desugar.desugar import do_desugar
from funky.infer.infer import do_type_inference
from funky.infer import FunkyTypeError
parser = FunkyParser()
parser.build()
def get_type_str(source):
ast = parser.do_parse(source)
# NOTE: we do not include imports for simplicity :)
do_rename(ast)
core_tree, typedefs = do_desugar(ast)
do_type_inference(core_tree, typedefs)
return str(core_tree.inferred_type)
class TestBasicInference(TestCase):
def setUp(self):
funky.globals.USE_UNICODE = False
funky.globals.USE_COLOR = False
def test_basic_type_inference(self):
tests = {