def setup(self, *args, **kwargs):
"""Sets parameters for the compiler."""
if self.comp is None:
self.comp = Compiler(*args, **kwargs)
else:
self.comp.setup(*args, **kwargs)
class Command(object):
"""The Coconut command-line interface."""
comp = None # current coconut.compiler.Compiler
show = False # corresponds to --display flag
runner = None # the current Runner
jobs = 0 # corresponds to --jobs flag
executor = None # runs --jobs
exit_code = 0 # exit status to return
errmsg = None # error message to display
mypy_args = None # corresponds to --mypy flag
def __init__(self):
"""Creates the CLI."""
self.prompt = Prompt()
def start(self, run=False):
"""Processes command-line arguments."""
if run:
args, argv = ["--run", "--quiet", "--target", "sys"], []
for i in range(1, len(sys.argv)):
arg = sys.argv[i]
if arg.startswith("-"):
args.append(arg) # coconut option
else:
args.append(arg) # source file
argv = sys.argv[i:]
break
sys.argv = argv
else:
args = None
self.cmd(args)
def cmd(self, args=None, interact=True):
"""Processes command-line arguments."""
if args is None:
args = arguments.parse_args()
else:
args = arguments.parse_args(args)
self.exit_code = 0
with self.handling_exceptions():
self.use_args(args, interact)
self.exit_on_error()
def setup(self, *args, **kwargs):
"""Sets parameters for the compiler."""
if self.comp is None:
self.comp = Compiler(*args, **kwargs)
else:
self.comp.setup(*args, **kwargs)
def exit_on_error(self):
"""Exits if exit_code is abnormal."""
if self.exit_code:
if self.errmsg is not None:
logger.show_error("Exiting due to " + self.errmsg + ".")
self.errmsg = None
if self.jobs != 0:
kill_children()
sys.exit(self.exit_code)
def set_recursion_limit(self, limit):
"""Sets the Python recursion limit."""
if limit < minimum_recursion_limit:
raise CoconutException("--recursion-limit must be at least " + str(minimum_recursion_limit))
else:
sys.setrecursionlimit(limit)
def use_args(self, args, interact=True):
"""Handles command-line arguments."""
logger.quiet, logger.verbose = args.quiet, args.verbose
if DEVELOP:
logger.tracing = args.trace
if args.recursion_limit is not None:
self.set_recursion_limit(args.recursion_limit)
if args.jobs is not None:
self.set_jobs(args.jobs)
if args.display:
self.show = True
if args.style is not None:
self.prompt.set_style(args.style)
if args.documentation:
launch_documentation()
if args.tutorial:
launch_tutorial()
self.setup(
target=args.target,
strict=args.strict,
minify=args.minify,
line_numbers=args.line_numbers,
keep_lines=args.keep_lines,
)
if args.mypy is not None:
self.set_mypy_args(args.mypy)
if args.source is not None:
if args.interact and args.run:
logger.warn("extraneous --run argument passed; --interact implies --run")
if args.package and self.mypy:
logger.warn("extraneous --package argument passed; --mypy implies --package")
if args.standalone and args.package:
raise CoconutException("cannot compile as both --package and --standalone")
if args.standalone and self.mypy:
raise CoconutException("cannot compile as both --package (implied by --mypy) and --standalone")
if (args.run or args.interact) and os.path.isdir(args.source):
if args.run:
raise CoconutException("source path must point to file not directory when --run is enabled")
if args.interact:
raise CoconutException("source path must point to file not directory when --run (implied by --interact) is enabled")
if args.watch and os.path.isfile(args.source):
raise CoconutException("source path must point to directory not file when --watch is enabled")
if args.dest is None:
if args.nowrite:
dest = None # no dest
else:
dest = True # auto-generate dest
elif args.nowrite:
raise CoconutException("destination path cannot be given when --nowrite is enabled")
elif os.path.isfile(args.dest):
raise CoconutException("destination path must point to directory not file")
else:
dest = args.dest
if args.package or self.mypy:
package = True
elif args.standalone:
package = False
else:
package = None # auto-decide package
with self.running_jobs():
filepaths = self.compile_path(args.source, dest, package, args.run or args.interact, args.force)
self.run_mypy(filepaths)
elif (args.run
or args.nowrite
or args.force
or args.package
or args.standalone
or args.watch):
raise CoconutException("a source file/folder must be specified when options that depend on the source are enabled")
if args.code is not None:
self.execute(self.comp.parse_block(args.code))
stdin = not sys.stdin.isatty() # check if input was piped in
if stdin:
self.execute(self.comp.parse_block(sys.stdin.read()))
if args.jupyter is not None:
self.start_jupyter(args.jupyter)
if args.interact or (interact and not (
stdin
or args.source
or args.code
or args.tutorial
or args.documentation
or args.watch
or args.jupyter is not None
)):
self.start_prompt()
if args.watch:
self.watch(args.source, dest, package, args.run, args.force)
def register_error(self, code=1, errmsg=None):
"""Updates the exit code."""
if errmsg is not None:
if self.errmsg is None:
self.errmsg = errmsg
elif errmsg not in self.errmsg:
self.errmsg += ", " + errmsg
if code is not None:
self.exit_code = max(self.exit_code, code)
@contextmanager
def handling_exceptions(self):
"""Performs proper exception handling."""
try:
with handling_broken_process_pool():
yield
except SystemExit as err:
self.register_error(err.code)
except BaseException as err:
if isinstance(err, CoconutException):
logger.print_exc()
elif not isinstance(err, KeyboardInterrupt):
traceback.print_exc()
self.register_error(errmsg=err.__class__.__name__)
def compile_path(self, path, write=True, package=None, run=False, force=False):
"""Compiles a path and returns paths to compiled files."""
path = fixpath(path)
if write is not None and write is not True:
write = fixpath(write)
if os.path.isfile(path):
if package is None:
package = False
destpath = self.compile_file(path, write, package, run, force)
if destpath is None:
return []
else:
return [destpath]
elif os.path.isdir(path):
if package is None:
package = True
return self.compile_folder(path, write, package, run, force)
else:
raise CoconutException("could not find source path", path)
def compile_folder(self, directory, write=True, package=True, run=False, force=False):
"""Compiles a directory and returns paths to compiled files."""
filepaths = []
for dirpath, dirnames, filenames in os.walk(directory):
if write is None or write is True:
writedir = write
else:
writedir = os.path.join(write, os.path.relpath(dirpath, directory))
for filename in filenames:
if os.path.splitext(filename)[1] in code_exts:
destpath = self.compile_file(os.path.join(dirpath, filename), writedir, package, run, force)
if destpath is not None:
filepaths.append(destpath)
for name in dirnames[:]:
if not is_special_dir(name) and name.startswith("."):
if logger.verbose:
logger.show_tabulated("Skipped directory", name, "(explicitly pass as source to override).")
dirnames.remove(name) # directories removed from dirnames won't appear in further os.walk iteration
return filepaths
def compile_file(self, filepath, write=True, package=False, run=False, force=False):
"""Compiles a file and returns the compiled file's path."""
if write is None:
destpath = None
elif write is True:
destpath = filepath
else:
destpath = os.path.join(write, os.path.basename(filepath))
if destpath is not None:
base, ext = os.path.splitext(os.path.splitext(destpath)[0])
if not ext:
ext = comp_ext
destpath = fixpath(base + ext)
if filepath == destpath:
raise CoconutException("cannot compile " + showpath(filepath) + " to itself", extra="incorrect file extension")
self.compile(filepath, destpath, package, run, force)
return destpath
def compile(self, codepath, destpath=None, package=False, run=False, force=False):
"""Compiles a source Coconut file to a destination Python file."""
logger.show_tabulated("Compiling", showpath(codepath), "...")
with openfile(codepath, "r") as opened:
code = readfile(opened)
if destpath is not None:
destdir = os.path.dirname(destpath)
if not os.path.exists(destdir):
os.makedirs(destdir)
if package is True:
self.create_package(destdir)
foundhash = None if force else self.hashashof(destpath, code, package)
if foundhash:
logger.show_tabulated("Left unchanged", showpath(destpath), "(pass --force to override).")
if self.show:
print(foundhash)
if run:
self.execute_file(destpath)
else:
if package is True:
compile_method = "parse_package"
elif package is False:
compile_method = "parse_file"
else:
raise CoconutInternalException("invalid value for package", package)
def callback(compiled):
if destpath is None:
logger.show_tabulated("Finished", showpath(codepath), "without writing to file.")
else:
with openfile(destpath, "w") as opened:
writefile(opened, compiled)
logger.show_tabulated("Compiled to", showpath(destpath), ".")
if self.show:
print(compiled)
if run:
if destpath is None:
self.execute(compiled, path=codepath, allow_show=False)
else:
self.execute_file(destpath)
self.submit_comp_job(codepath, callback, compile_method, code)
def submit_comp_job(self, path, callback, method, *args, **kwargs):
"""Submits a job on self.comp to be run in parallel."""
if self.executor is None:
with self.handling_exceptions():
callback(getattr(self.comp, method)(*args, **kwargs))
else:
path = showpath(path)
with logger.in_path(path): # pickle the compiler in the path context
future = self.executor.submit(multiprocess_wrapper(self.comp, method), *args, **kwargs)
def callback_wrapper(completed_future):
"""Ensures that all errors are always caught, since errors raised in a callback won't be propagated."""
with logger.in_path(path): # handle errors in the path context
with self.handling_exceptions():
result = completed_future.result()
callback(result)
future.add_done_callback(callback_wrapper)
def set_jobs(self, jobs):
"""Sets --jobs."""
if jobs == "sys":
self.jobs = None
else:
try:
jobs = int(jobs)
except ValueError:
jobs = -1
if jobs < 0:
raise CoconutException("--jobs must be an integer >= 0 or 'sys'")
else:
self.jobs = jobs
@contextmanager
def running_jobs(self):
"""Initialize multiprocessing."""
with self.handling_exceptions():
if self.jobs == 0:
yield
else:
from concurrent.futures import ProcessPoolExecutor
try:
with ProcessPoolExecutor(self.jobs) as self.executor:
yield
finally:
self.executor = None
self.exit_on_error()
def create_package(self, dirpath):
"""Sets up a package directory."""
dirpath = fixpath(dirpath)
filepath = os.path.join(dirpath, "__coconut__.py")
with openfile(filepath, "w") as opened:
writefile(opened, self.comp.getheader("__coconut__"))
def hashashof(self, destpath, code, package):
"""Determines if a file has the hash of the code."""
if destpath is not None and os.path.isfile(destpath):
with openfile(destpath, "r") as opened:
compiled = readfile(opened)
hashash = gethash(compiled)
if hashash is not None and hashash == self.comp.genhash(package, code):
return compiled
return None
def get_input(self, more=False):
"""Prompts for code input."""
try:
return self.prompt.input(more)
except KeyboardInterrupt:
printerr("\nKeyboardInterrupt")
except EOFError:
print()
self.exit_runner()
return None
def start_running(self):
"""Starts running the Runner."""
self.comp.bind()
self.check_runner()
self.running = True
def start_prompt(self):
"""Starts the interpreter."""
print("Coconut Interpreter:")
print('(type "exit()" or press Ctrl-D to end)')
self.start_running()
while self.running:
code = self.get_input()
if code:
compiled = self.handle_input(code)
if compiled:
self.execute(compiled, use_eval=None)
def exit_runner(self, exit_code=0):
"""Exits the interpreter."""
self.register_error(exit_code)
self.running = False
def handle_input(self, code):
"""Compiles Coconut interpreter input."""
if not self.prompt.multiline:
if not should_indent(code):
try:
return self.comp.parse_block(code)
except CoconutException:
pass
while True:
line = self.get_input(more=True)
if line is None:
return None
elif line:
code += "\n" + line
else:
break
try:
return self.comp.parse_block(code)
except CoconutException:
logger.print_exc()
return None
def execute(self, compiled=None, path=None, use_eval=False, allow_show=True):
"""Executes compiled code."""
self.check_runner()
if compiled is not None:
if allow_show and self.show:
print(compiled)
if path is not None: # path means header is included, and thus encoding must be removed
compiled = rem_encoding(compiled)
self.runner.run(compiled, use_eval=use_eval, path=path, all_errors_exit=(path is not None))
self.run_mypy(code=self.runner.was_run_code())
def execute_file(self, destpath):
"""Executes compiled file."""
self.check_runner()
self.runner.run_file(destpath)
def check_runner(self):
"""Makes sure there is a runner."""
if os.getcwd() not in sys.path:
sys.path.append(os.getcwd())
if self.runner is None:
self.runner = Runner(self.comp, exit=self.exit_runner, store=self.mypy)
@property
def mypy(self):
"""Whether using MyPy or not."""
return self.mypy_args is not None
def set_mypy_args(self, mypy_args=None):
"""Sets MyPy arguments."""
if mypy_args is None:
self.mypy_args = None
else:
self.mypy_errs = []
self.mypy_args = list(mypy_args)
for arg in self.mypy_args:
if arg == "--fast-parser":
logger.warn("unnecessary --mypy argument", arg, extra="passed automatically if available")
elif arg == "--py2" or arg == "-2":
logger.warn("unnecessary --mypy argument", arg, extra="passed automatically when needed")
elif arg == "--python-version":
logger.warn("unnecessary --mypy argument", arg, extra="current --target passed as version automatically")
if "--fast-parser" not in self.mypy_args:
try:
import typed_ast # NOQA
except ImportError:
if self.comp.target != "3":
logger.warn("missing typed_ast; MyPy may not properly analyze type annotations",
extra="run 'pip install typed_ast' or pass '--target 3' to fix")
else:
self.mypy_args.append("--fast-parser")
if not ("--py2" in self.mypy_args or "-2" in self.mypy_args) and not self.comp.target.startswith("3"):
self.mypy_args.append("--py2")
if "--python-version" not in self.mypy_args:
self.mypy_args += ["--python-version", ".".join(str(v) for v in self.comp.target_info_len2)]
def run_mypy(self, paths=[], code=None):
"""Run MyPy with arguments."""
set_mypy_path(stub_dir)
if self.mypy:
from coconut.command.mypy import mypy_run
args = paths + self.mypy_args
if code is not None:
args += ["-c", code]
for line, is_err in mypy_run(args):
if code is None or line not in self.mypy_errs:
if is_err:
printerr(line)
else:
print(line)
if line not in self.mypy_errs:
self.mypy_errs.append(line)
def start_jupyter(self, args):
"""Starts Jupyter with the Coconut kernel."""
install_func = functools.partial(run_cmd, show_output=logger.verbose or not args)
try:
install_func(["jupyter", "--version"])
except CalledProcessError:
jupyter = "ipython"
else:
jupyter = "jupyter"
for icoconut_kernel_dir in icoconut_kernel_dirs:
install_args = [jupyter, "kernelspec", "install", icoconut_kernel_dir, "--replace"]
try:
install_func(install_args)
except CalledProcessError:
user_install_args = install_args + ["--user"]
try:
install_func(user_install_args)
except CalledProcessError:
logger.warn("kernel install failed on command'", " ".join(install_args))
self.register_error(errmsg="Jupyter error")
if args:
if args[0] == "console":
ver = "2" if PY2 else "3"
try:
install_func(["python" + ver, "-m", "coconut.main", "--version"])
except CalledProcessError:
kernel_name = "coconut"
else:
kernel_name = "coconut" + ver
run_args = [jupyter, "console", "--kernel", kernel_name] + args[1:]
elif args[0] == "notebook":
run_args = [jupyter, "notebook"] + args[1:]
else:
raise CoconutException('first argument after --jupyter must be either "console" or "notebook"')
self.register_error(run_cmd(run_args, raise_errs=False), errmsg="Jupyter error")
def watch(self, source, write=True, package=None, run=False, force=False):
"""Watches a source and recompiles on change."""
from coconut.command.watch import Observer, RecompilationWatcher
source = fixpath(source)
print()
logger.show_tabulated("Watching", showpath(source), "(press Ctrl-C to end)...")
def recompile(path):
if os.path.isfile(path) and os.path.splitext(path)[1] in code_exts:
with self.handling_exceptions():
self.run_mypy(self.compile_path(path, write, package, run, force))
observer = Observer()
observer.schedule(RecompilationWatcher(recompile), source, recursive=True)
with self.running_jobs():
observer.start()
try:
while True:
time.sleep(watch_interval)
except KeyboardInterrupt:
logger.show("Got KeyboardInterrupt; stopping watcher.")
finally:
observer.stop()
observer.join()
class Command(object):
"""Coconut command-line interface."""
comp = None # current coconut.compiler.Compiler
show = False # corresponds to --display flag
runner = None # the current Runner
jobs = 0 # corresponds to --jobs flag
executor = None # runs --jobs
exit_code = 0 # exit status to return
errmsg = None # error message to display
mypy_args = None # corresponds to --mypy flag
def __init__(self):
"""Create the CLI."""
self.prompt = Prompt()
def start(self, run=False):
"""Process command-line arguments."""
if run:
args, argv = list(coconut_run_args), []
# for coconut-run, all args beyond the source file should be wrapped in an --argv
for i in range(1, len(sys.argv)):
arg = sys.argv[i]
args.append(arg)
if not arg.startswith("-"): # was source file
argv = sys.argv[i + 1:]
break
args += ["--argv"] + argv
else:
args = None
self.cmd(args)
def cmd(self, args=None, interact=True):
"""Process command-line arguments."""
if args is None:
parsed_args = arguments.parse_args()
else:
parsed_args = arguments.parse_args(args)
self.exit_code = 0
with self.handling_exceptions():
self.use_args(parsed_args, interact, original_args=args)
self.exit_on_error()
def setup(self, *args, **kwargs):
"""Set parameters for the compiler."""
if self.comp is None:
self.comp = Compiler(*args, **kwargs)
else:
self.comp.setup(*args, **kwargs)
def exit_on_error(self):
"""Exit if exit_code is abnormal."""
if self.exit_code:
if self.errmsg is not None:
logger.show_error("Exiting due to " + self.errmsg + ".")
self.errmsg = None
if self.jobs != 0:
kill_children()
sys.exit(self.exit_code)
def use_args(self, args, interact=True, original_args=None):
"""Handle command-line arguments."""
logger.quiet, logger.verbose = args.quiet, args.verbose
if DEVELOP:
logger.tracing = args.trace
logger.log("Using " + PYPARSING + ".")
if original_args is not None:
logger.log("Directly passed args:", original_args)
logger.log("Parsed args:", args)
if args.recursion_limit is not None:
set_recursion_limit(args.recursion_limit)
if args.jobs is not None:
self.set_jobs(args.jobs)
if args.display:
self.show = True
if args.style is not None:
self.prompt.set_style(args.style)
if args.documentation:
launch_documentation()
if args.tutorial:
launch_tutorial()
self.setup(
target=args.target,
strict=args.strict,
minify=args.minify,
line_numbers=args.line_numbers,
keep_lines=args.keep_lines,
no_tco=args.no_tco or args.mypy is not None,
)
if args.mypy is not None:
if args.no_tco:
logger.warn(
"extraneous --no-tco argument passed; --mypy implies --no-tco"
)
self.set_mypy_args(args.mypy)
if args.argv is not None:
sys.argv = [args.source if args.source is not None else ""]
sys.argv.extend(args.argv)
if args.source is not None:
if args.interact and args.run:
logger.warn(
"extraneous --run argument passed; --interact implies --run"
)
if args.package and self.mypy:
logger.warn(
"extraneous --package argument passed; --mypy implies --package"
)
if args.standalone and args.package:
raise CoconutException(
"cannot compile as both --package and --standalone")
if args.standalone and self.mypy:
raise CoconutException(
"cannot compile as both --package (implied by --mypy) and --standalone"
)
if args.no_write and self.mypy:
raise CoconutException(
"cannot compile with --no-write when using --mypy")
if (args.run or args.interact) and os.path.isdir(args.source):
if args.run:
raise CoconutException(
"source path must point to file not directory when --run is enabled"
)
if args.interact:
raise CoconutException(
"source path must point to file not directory when --run (implied by --interact) is enabled"
)
if args.watch and os.path.isfile(args.source):
raise CoconutException(
"source path must point to directory not file when --watch is enabled"
)
if args.dest is None:
if args.no_write:
dest = None # no dest
else:
dest = True # auto-generate dest
elif args.no_write:
raise CoconutException(
"destination path cannot be given when --no-write is enabled"
)
elif os.path.isfile(args.dest):
raise CoconutException(
"destination path must point to directory not file")
else:
dest = args.dest
if args.package or self.mypy:
package = True
elif args.standalone:
package = False
else:
package = None # auto-decide package
with self.running_jobs(exit_on_error=not args.watch):
filepaths = self.compile_path(args.source, dest, package,
args.run or args.interact,
args.force)
self.run_mypy(filepaths)
elif (args.run or args.no_write or args.force or args.package
or args.standalone or args.watch):
raise CoconutException(
"a source file/folder must be specified when options that depend on the source are enabled"
)
if args.code is not None:
self.execute(self.comp.parse_block(args.code))
got_stdin = False
if stdin_readable():
logger.log("Reading piped input from stdin...")
self.execute(self.comp.parse_block(sys.stdin.read()))
got_stdin = True
if args.jupyter is not None:
self.start_jupyter(args.jupyter)
if args.interact or (interact
and not (got_stdin or args.source or args.code
or args.tutorial or args.documentation or
args.watch or args.jupyter is not None)):
self.start_prompt()
if args.watch:
self.watch(args.source, dest, package, args.run, args.force)
def register_error(self, code=1, errmsg=None):
"""Update the exit code."""
if errmsg is not None:
if self.errmsg is None:
self.errmsg = errmsg
elif errmsg not in self.errmsg:
self.errmsg += ", " + errmsg
if code is not None:
self.exit_code = max(self.exit_code, code)
@contextmanager
def handling_exceptions(self):
"""Perform proper exception handling."""
try:
with handling_broken_process_pool():
yield
except SystemExit as err:
self.register_error(err.code)
except BaseException as err:
if isinstance(err, CoconutException):
logger.display_exc()
elif not isinstance(err, KeyboardInterrupt):
traceback.print_exc()
printerr(report_this_text)
self.register_error(errmsg=err.__class__.__name__)
def compile_path(self, path, write=True, package=None, *args, **kwargs):
"""Compile a path and returns paths to compiled files."""
path = fixpath(path)
if write is not None and write is not True:
write = fixpath(write)
if os.path.isfile(path):
if package is None:
package = False
destpath = self.compile_file(path, write, package, *args, **kwargs)
return [destpath] if destpath is not None else []
elif os.path.isdir(path):
if package is None:
package = True
return self.compile_folder(path, write, package, *args, **kwargs)
else:
raise CoconutException("could not find source path", path)
def compile_folder(self,
directory,
write=True,
package=True,
*args,
**kwargs):
"""Compile a directory and returns paths to compiled files."""
filepaths = []
for dirpath, dirnames, filenames in os.walk(directory):
if write is None or write is True:
writedir = write
else:
writedir = os.path.join(write,
os.path.relpath(dirpath, directory))
for filename in filenames:
if os.path.splitext(filename)[1] in code_exts:
destpath = self.compile_file(
os.path.join(dirpath, filename), writedir, package,
*args, **kwargs)
if destpath is not None:
filepaths.append(destpath)
for name in dirnames[:]:
if not is_special_dir(name) and name.startswith("."):
if logger.verbose:
logger.show_tabulated(
"Skipped directory", name,
"(explicitly pass as source to override).")
dirnames.remove(
name
) # directories removed from dirnames won't appear in further os.walk iterations
return filepaths
def compile_file(self,
filepath,
write=True,
package=False,
*args,
**kwargs):
"""Compile a file and returns the compiled file's path."""
if write is None:
destpath = None
elif write is True:
destpath = filepath
else:
# add the name of file to the destination path
destpath = os.path.join(write, os.path.basename(filepath))
if destpath is not None:
base, ext = os.path.splitext(os.path.splitext(destpath)[0])
if not ext:
ext = comp_ext
destpath = fixpath(base + ext)
if filepath == destpath:
raise CoconutException("cannot compile " + showpath(filepath) +
" to itself",
extra="incorrect file extension")
self.compile(filepath, destpath, package, *args, **kwargs)
return destpath
def compile(self,
codepath,
destpath=None,
package=False,
run=False,
force=False,
show_unchanged=True):
"""Compile a source Coconut file to a destination Python file."""
with openfile(codepath, "r") as opened:
code = readfile(opened)
if destpath is not None:
destdir = os.path.dirname(destpath)
if not os.path.exists(destdir):
os.makedirs(destdir)
if package is True:
self.create_package(destdir)
foundhash = None if force else self.has_hash_of(
destpath, code, package)
if foundhash:
if show_unchanged:
logger.show_tabulated("Left unchanged", showpath(destpath),
"(pass --force to override).")
if self.show:
print(foundhash)
if run:
self.execute_file(destpath)
else:
logger.show_tabulated("Compiling", showpath(codepath), "...")
if package is True:
compile_method = "parse_package"
elif package is False:
compile_method = "parse_file"
else:
raise CoconutInternalException("invalid value for package",
package)
def callback(compiled):
if destpath is None:
logger.show_tabulated("Compiled", showpath(codepath),
"without writing to file.")
else:
with openfile(destpath, "w") as opened:
writefile(opened, compiled)
logger.show_tabulated("Compiled to", showpath(destpath),
".")
if self.show:
print(compiled)
if run:
if destpath is None:
self.execute(compiled, path=codepath, allow_show=False)
else:
self.execute_file(destpath)
self.submit_comp_job(codepath, callback, compile_method, code)
def submit_comp_job(self, path, callback, method, *args, **kwargs):
"""Submits a job on self.comp to be run in parallel."""
if self.executor is None:
with self.handling_exceptions():
callback(getattr(self.comp, method)(*args, **kwargs))
else:
path = showpath(path)
with logger.in_path(
path): # pickle the compiler in the path context
future = self.executor.submit(
multiprocess_wrapper(self.comp, method), *args, **kwargs)
def callback_wrapper(completed_future):
"""Ensures that all errors are always caught, since errors raised in a callback won't be propagated."""
with logger.in_path(path): # handle errors in the path context
with self.handling_exceptions():
result = completed_future.result()
callback(result)
future.add_done_callback(callback_wrapper)
def set_jobs(self, jobs):
"""Set --jobs."""
if jobs == "sys":
self.jobs = None
else:
try:
jobs = int(jobs)
except ValueError:
jobs = -1 # will raise error below
if jobs < 0:
raise CoconutException(
"--jobs must be an integer >= 0 or 'sys'")
self.jobs = jobs
@contextmanager
def running_jobs(self, exit_on_error=True):
"""Initialize multiprocessing."""
with self.handling_exceptions():
if self.jobs == 0:
yield
else:
from concurrent.futures import ProcessPoolExecutor
try:
with ProcessPoolExecutor(self.jobs) as self.executor:
yield
finally:
self.executor = None
if exit_on_error:
self.exit_on_error()
def create_package(self, dirpath):
"""Set up a package directory."""
dirpath = fixpath(dirpath)
filepath = os.path.join(dirpath, "__coconut__.py")
with openfile(filepath, "w") as opened:
writefile(opened, self.comp.getheader("__coconut__"))
def has_hash_of(self, destpath, code, package):
"""Determine if a file has the hash of the code."""
if destpath is not None and os.path.isfile(destpath):
with openfile(destpath, "r") as opened:
compiled = readfile(opened)
hashash = gethash(compiled)
if hashash is not None and hashash == self.comp.genhash(
package, code):
return compiled
return None
def get_input(self, more=False):
"""Prompt for code input."""
received = None
try:
received = self.prompt.input(more)
except KeyboardInterrupt:
print()
printerr("KeyboardInterrupt")
except EOFError:
print()
self.exit_runner()
else:
if received.startswith(exit_chars):
self.exit_runner()
received = None
return received
def start_running(self):
"""Start running the Runner."""
self.comp.warm_up()
self.check_runner()
self.running = True
def start_prompt(self):
"""Start the interpreter."""
logger.show("Coconut Interpreter:")
logger.show("(type 'exit()' or press Ctrl-D to end)")
self.start_running()
while self.running:
try:
code = self.get_input()
if code:
compiled = self.handle_input(code)
if compiled:
self.execute(compiled, use_eval=None)
except KeyboardInterrupt:
printerr("\nKeyboardInterrupt")
def exit_runner(self, exit_code=0):
"""Exit the interpreter."""
self.register_error(exit_code)
self.running = False
def handle_input(self, code):
"""Compile Coconut interpreter input."""
if not self.prompt.multiline:
if not should_indent(code):
try:
return self.comp.parse_block(code)
except CoconutException:
pass
while True:
line = self.get_input(more=True)
if line is None:
return None
elif line:
code += "\n" + line
else:
break
try:
return self.comp.parse_block(code)
except CoconutException:
logger.display_exc()
return None
def execute(self,
compiled=None,
path=None,
use_eval=False,
allow_show=True):
"""Execute compiled code."""
self.check_runner()
if compiled is not None:
if allow_show and self.show:
print(compiled)
if path is not None: # path means header is included, and thus encoding must be removed
compiled = rem_encoding(compiled)
self.runner.run(compiled,
use_eval=use_eval,
path=path,
all_errors_exit=(path is not None))
self.run_mypy(code=self.runner.was_run_code())
def execute_file(self, destpath):
"""Execute compiled file."""
self.check_runner()
self.runner.run_file(destpath)
def check_runner(self):
"""Make sure there is a runner."""
if os.getcwd() not in sys.path:
sys.path.append(os.getcwd())
if self.runner is None:
self.runner = Runner(self.comp,
exit=self.exit_runner,
store=self.mypy)
@property
def mypy(self):
"""Whether using MyPy or not."""
return self.mypy_args is not None
def set_mypy_args(self, mypy_args=None):
"""Set MyPy arguments."""
if mypy_args is None:
self.mypy_args = None
else:
self.mypy_errs = []
self.mypy_args = list(mypy_args)
for arg in self.mypy_args:
if arg == "--py2" or arg == "-2":
logger.warn("unnecessary --mypy argument",
arg,
extra="passed automatically when needed")
elif arg == "--python-version":
logger.warn(
"unnecessary --mypy argument",
arg,
extra="current --target passed as version automatically"
)
if not ("--py2" in self.mypy_args or "-2" in self.mypy_args
) and not self.comp.target.startswith("3"):
self.mypy_args.append("--py2")
if "--python-version" not in self.mypy_args:
self.mypy_args += [
"--python-version",
".".join(str(v) for v in self.comp.target_info_len2)
]
if logger.verbose:
for arg in verbose_mypy_args:
if arg not in self.mypy_args:
self.mypy_args.append(arg)
logger.log("MyPy args:", self.mypy_args)
def run_mypy(self, paths=(), code=None):
"""Run MyPy with arguments."""
if self.mypy:
set_mypy_path(stub_dir)
from coconut.command.mypy import mypy_run
args = list(paths) + self.mypy_args
if code is not None:
args += ["-c", code]
for line, is_err in mypy_run(args):
if code is None or line not in self.mypy_errs:
if is_err:
printerr(line)
else:
print(line)
if line not in self.mypy_errs:
self.mypy_errs.append(line)
def start_jupyter(self, args):
"""Start Jupyter with the Coconut kernel."""
install_func = functools.partial(run_cmd, show_output=logger.verbose)
try:
install_func(["jupyter", "--version"])
except CalledProcessError:
jupyter = "ipython"
else:
jupyter = "jupyter"
# always install kernels if given no args, otherwise only if there's a kernel missing
do_install = not args
if not do_install:
kernel_list = run_cmd([jupyter, "kernelspec", "list"],
show_output=False,
raise_errs=False)
do_install = any(ker not in kernel_list
for ker in icoconut_kernel_names)
if do_install:
success = True
for icoconut_kernel_dir in icoconut_kernel_dirs:
install_args = [
jupyter, "kernelspec", "install", icoconut_kernel_dir,
"--replace"
]
try:
install_func(install_args)
except CalledProcessError:
user_install_args = install_args + ["--user"]
try:
install_func(user_install_args)
except CalledProcessError:
logger.warn("kernel install failed on command'",
" ".join(install_args))
self.register_error(errmsg="Jupyter error")
success = False
if success:
logger.show_sig(
"Successfully installed Coconut Jupyter kernel.")
if args:
if args[0] == "console":
ver = "2" if PY2 else "3"
try:
install_func(
["python" + ver, "-m", "coconut.main", "--version"])
except CalledProcessError:
kernel_name = "coconut"
else:
kernel_name = "coconut" + ver
run_args = [jupyter, "console", "--kernel", kernel_name
] + args[1:]
elif args[0] == "notebook":
run_args = [jupyter, "notebook"] + args[1:]
else:
raise CoconutException(
"first argument after --jupyter must be either 'console' or 'notebook'"
)
self.register_error(run_cmd(run_args, raise_errs=False),
errmsg="Jupyter error")
def watch(self, source, write=True, package=None, run=False, force=False):
"""Watch a source and recompiles on change."""
from coconut.command.watch import Observer, RecompilationWatcher
source = fixpath(source)
logger.show()
logger.show_tabulated("Watching", showpath(source),
"(press Ctrl-C to end)...")
def recompile(path):
path = fixpath(path)
if os.path.isfile(path) and os.path.splitext(path)[1] in code_exts:
with self.handling_exceptions():
if write is True or write is None:
writedir = write
else:
# correct the compilation path based on the relative position of path to source
dirpath = os.path.dirname(path)
writedir = os.path.join(
write, os.path.relpath(dirpath, source))
filepaths = self.compile_path(path,
writedir,
package,
run,
force,
show_unchanged=False)
self.run_mypy(filepaths)
watcher = RecompilationWatcher(recompile)
observer = Observer()
observer.schedule(watcher, source, recursive=True)
with self.running_jobs():
observer.start()
try:
while True:
time.sleep(watch_interval)
watcher.keep_watching()
except KeyboardInterrupt:
logger.show_sig("Got KeyboardInterrupt; stopping watcher.")
finally:
observer.stop()
observer.join()
"; skipping coconut.icoconut loading")
else:
raise CoconutException(
"--jupyter flag requires Jupyter library",
extra="run 'pip install coconut[jupyter]' to fix",
)
else:
LOAD_MODULE = True
# -----------------------------------------------------------------------------------------------------------------------
# GLOBALS:
# -----------------------------------------------------------------------------------------------------------------------
COMPILER = Compiler(
target="sys",
line_numbers=True,
keep_lines=True,
)
RUNNER = Runner(COMPILER)
parse_block_memo = {}
def memoized_parse_block(code):
"""Memoized version of parse_block."""
internal_assert(lambda: code not in parse_block_memo.values(),
"attempted recompilation of", code)
success, result = parse_block_memo.get(code, (None, None))
if success is None:
try:
name += c
else:
right = i
break
if i == len(code) - 1:
right = len(code)
if get_bounds:
return name, left, right
else:
return name
#-----------------------------------------------------------------------------------------------------------------------
# KERNEL:
#-----------------------------------------------------------------------------------------------------------------------
comp = Compiler(target="sys") # from coconut.compiler
class CoconutKernel(Kernel):
"""Jupyter kernel for Coconut."""
_runner = None # current ..command.Runner
implementation = "icoconut"
implementation_version = VERSION
language = "coconut"
language_version = VERSION
banner = version_banner
language_info = {
"name": "coconut",
"mimetype": mimetype,
"file_extension": code_exts[0],
"codemirror_mode": {
def setup(self, *args, **kwargs):
"""Set parameters for the compiler."""
if self.comp is None:
self.comp = Compiler(*args, **kwargs)
else:
self.comp.setup(*args, **kwargs)
py_syntax_version,
mimetype,
version_banner,
tutorial_url,
documentation_url,
code_exts,
)
from coconut.compiler import Compiler
from coconut.compiler.util import should_indent
from coconut.command.util import Runner
#-----------------------------------------------------------------------------------------------------------------------
# GLOBALS:
#-----------------------------------------------------------------------------------------------------------------------
COMPILER = Compiler(target="sys")
RUNNER = Runner(COMPILER)
parse_block_memo = {}
def memoized_parse_block(code, none_if_not_found=False):
"""Memoized version of parse_block."""
try:
result = parse_block_memo[code]
except KeyError:
if none_if_not_found:
return None
try:
parsed = COMPILER.parse_block(code)
except Exception as err:
class Command(object):
"""The Coconut command-line interface."""
comp = None # current coconut.compiler.Compiler
show = False # corresponds to --display flag
running = False # whether the interpreter is currently active
runner = None # the current Runner
target = None # corresponds to --target flag
jobs = 0 # corresponds to --jobs flag
executor = None # runs --jobs
exit_code = 0 # exit status to return
errmsg = None # error message to display
def __init__(self):
"""Creates the CLI."""
self.prompt = Prompt()
def start(self):
"""Processes command-line arguments."""
self.cmd(arguments.parse_args())
def setup(self, *args, **kwargs):
"""Sets parameters for the compiler."""
if self.comp is None:
self.comp = Compiler(*args, **kwargs)
else:
self.comp.setup(*args, **kwargs)
def cmd(self, args, interact=True):
"""Processes command-line arguments."""
self.exit_code = 0
with self.handling_exceptions(True):
self.use_args(args, interact)
self.exit_on_error()
def exit_on_error(self):
"""Exits if exit_code is abnormal."""
if self.exit_code:
if self.errmsg is not None:
logger.show_error("Exiting due to " + self.errmsg + ".")
self.errmsg = None
if self.jobs != 0:
kill_children()
sys.exit(self.exit_code)
def set_recursion_limit(self, limit):
"""Sets the Python recursion limit."""
if limit < minimum_recursion_limit:
raise CoconutException("--recursion-limit must be at least " +
str(minimum_recursion_limit))
else:
sys.setrecursionlimit(limit)
def use_args(self, args, interact=True):
"""Handles command-line arguments."""
logger.quiet, logger.verbose = args.quiet, args.verbose
if args.recursion_limit is not None:
self.set_recursion_limit(args.recursion_limit)
if args.jobs is not None:
self.set_jobs(args.jobs)
if args.tutorial:
self.launch_tutorial()
if args.documentation:
self.launch_documentation()
if args.style is not None:
self.prompt.set_style(args.style)
if args.display:
self.show = True
self.setup(
target=args.target,
strict=args.strict,
minify=args.minify,
line_numbers=args.line_numbers,
keep_lines=args.keep_lines,
)
if args.source is not None:
if args.run and os.path.isdir(args.source):
raise CoconutException(
"source path must point to file not directory when --run is enabled"
)
elif args.watch and os.path.isfile(args.source):
raise CoconutException(
"source path must point to directory not file when --watch is enabled"
)
if args.dest is None:
if args.nowrite:
dest = None # no dest
else:
dest = True # auto-generate dest
elif args.nowrite:
raise CoconutException(
"destination path cannot be given when --nowrite is enabled"
)
elif os.path.isfile(args.dest):
raise CoconutException(
"destination path must point to directory not file")
else:
dest = args.dest
if args.package and args.standalone:
raise CoconutException(
"cannot compile as both --package and --standalone")
elif args.package:
package = True
elif args.standalone:
package = False
else:
package = None # auto-decide package
with self.running_jobs():
self.compile_path(args.source, dest, package, args.run,
args.force)
elif (args.run or args.nowrite or args.force or args.package
or args.standalone or args.watch):
raise CoconutException(
"a source file/folder must be specified when options that depend on the source are enabled"
)
if args.code is not None:
self.execute(self.comp.parse_block(args.code))
stdin = not sys.stdin.isatty() # check if input was piped in
if stdin:
self.execute(self.comp.parse_block(sys.stdin.read()))
if args.jupyter is not None:
self.start_jupyter(args.jupyter)
if args.interact or (interact
and not (stdin or args.source or args.code
or args.tutorial or args.documentation or
args.watch or args.jupyter is not None)):
self.start_prompt()
if args.watch:
self.watch(args.source, dest, package, args.run, args.force)
def register_error(self, code=1, errmsg=None):
"""Updates the exit code."""
if errmsg is not None:
if self.errmsg is None:
self.errmsg = errmsg
elif errmsg not in self.errmsg:
self.errmsg += ", " + errmsg
self.exit_code = max(self.exit_code, code)
@contextmanager
def handling_exceptions(self, blanket=False, errmsg=None):
"""Performs proper exception handling."""
if blanket:
try:
with handle_broken_process_pool():
yield
except KeyboardInterrupt:
self.register_error(errmsg="KeyboardInterrupt")
except SystemExit as err:
self.register_error(err.code)
except:
logger.print_exc()
self.register_error(errmsg=errmsg)
else:
try:
yield
except CoconutException:
logger.print_exc()
self.register_error(errmsg=errmsg)
def compile_path(self,
path,
write=True,
package=None,
run=False,
force=False):
"""Compiles a path."""
path = fixpath(path)
if write is not None and write is not True:
write = fixpath(write)
if os.path.isfile(path):
if package is None:
package = False
self.compile_file(path, write, package, run, force)
elif os.path.isdir(path):
if package is None:
package = True
self.compile_folder(path, write, package, run, force)
else:
raise CoconutException("could not find source path", path)
def compile_folder(self,
directory,
write=True,
package=True,
run=False,
force=False):
"""Compiles a directory."""
for dirpath, dirnames, filenames in os.walk(directory):
if write is None or write is True:
writedir = write
else:
writedir = os.path.join(write,
os.path.relpath(dirpath, directory))
for filename in filenames:
if os.path.splitext(filename)[1] in code_exts:
self.compile_file(os.path.join(dirpath, filename),
writedir, package, run, force)
for name in dirnames[:]:
if name != "." * len(name) and name.startswith("."):
if logger.verbose:
logger.show_tabulated(
"Skipped directory", name,
"(explicitly pass as source to override).")
dirnames.remove(
name
) # directories removed from dirnames won't appear in further os.walk iteration
def compile_file(self,
filepath,
write=True,
package=False,
run=False,
force=False):
"""Compiles a file."""
if write is None:
destpath = None
elif write is True:
destpath = filepath
else:
destpath = os.path.join(write, os.path.basename(filepath))
if destpath is not None:
base, ext = os.path.splitext(os.path.splitext(destpath)[0])
if not ext:
ext = comp_ext
destpath = base + ext
if filepath == destpath:
raise CoconutException("cannot compile " + showpath(filepath) +
" to itself (incorrect file extension)")
else:
self.compile(filepath, destpath, package, run, force)
def compile(self,
codepath,
destpath=None,
package=False,
run=False,
force=False):
"""Compiles a source Coconut file to a destination Python file."""
logger.show_tabulated("Compiling", showpath(codepath), "...")
with openfile(codepath, "r") as opened:
code = readfile(opened)
if destpath is not None:
destdir = os.path.dirname(destpath)
if not os.path.exists(destdir):
os.makedirs(destdir)
if package is True:
self.create_package(destdir)
foundhash = None if force else self.hashashof(destpath, code, package)
if foundhash:
logger.show_tabulated("Left unchanged", showpath(destpath),
"(pass --force to override).")
if run:
self.execute(foundhash, path=destpath, isolate=True)
elif self.show:
print(foundhash)
else:
if package is True:
compile_method = "parse_module"
elif package is False:
compile_method = "parse_file"
else:
raise CoconutInternalException("invalid value for package",
package)
def callback(compiled):
if destpath is None:
logger.show_tabulated("Finished", showpath(codepath),
"without writing to file.")
else:
with openfile(destpath, "w") as opened:
writefile(opened, compiled)
logger.show_tabulated("Compiled to", showpath(destpath),
".")
if run:
runpath = destpath if destpath is not None else codepath
self.execute(compiled, path=runpath, isolate=True)
elif self.show:
print(compiled)
self.submit_comp_job(codepath, callback, compile_method, code)
def submit_comp_job(self, path, callback, method, *args, **kwargs):
"""Submits a job on self.comp to be run in parallel."""
if self.executor is None:
with self.handling_exceptions():
callback(getattr(self.comp, method)(*args, **kwargs))
else:
path = showpath(path)
with logger.in_path(
path): # pickle the compiler in the path context
future = self.executor.submit(
multiprocess_wrapper(self.comp, method), *args, **kwargs)
def callback_wrapper(completed_future):
"""Ensures that all errors are always caught, since errors raised in a callback won't be propagated."""
with logger.in_path(path): # handle errors in the path context
with self.handling_exceptions(True, "compilation error"):
result = completed_future.result()
callback(result)
future.add_done_callback(callback_wrapper)
def set_jobs(self, jobs):
"""Sets --jobs."""
if jobs == "sys":
self.jobs = None
else:
try:
jobs = int(jobs)
except ValueError:
jobs = -1
if jobs < 0:
raise CoconutException(
"--jobs must be an integer >= 0 or 'sys'")
else:
self.jobs = jobs
@contextmanager
def running_jobs(self):
"""Initialize multiprocessing."""
if self.jobs == 0:
yield
else:
from concurrent.futures import ProcessPoolExecutor
with self.handling_exceptions(True):
with ProcessPoolExecutor(self.jobs) as self.executor:
with ensure_time_elapsed():
yield
self.executor = None
self.exit_on_error()
def create_package(self, dirpath):
"""Sets up a package directory."""
filepath = os.path.join(fixpath(dirpath), "__coconut__.py")
with openfile(filepath, "w") as opened:
writefile(opened, self.comp.headers("package"))
def hashashof(self, destpath, code, package):
"""Determines if a file has the hash of the code."""
if destpath is not None and os.path.isfile(destpath):
with openfile(destpath, "r") as opened:
compiled = readfile(opened)
hashash = gethash(compiled)
if hashash is not None and hashash == self.comp.genhash(
package, code):
return compiled
return None
def get_input(self, more=False):
"""Prompts for code input."""
try:
return self.prompt.input(more)
except KeyboardInterrupt:
logger.printerr("\nKeyboardInterrupt")
except EOFError:
print()
self.exit_runner()
except ValueError:
logger.print_exc()
self.exit_runner()
return None
def start_running(self):
"""Starts running the Runner."""
self.check_runner()
self.running = True
def start_prompt(self):
"""Starts the interpreter."""
logger.print("Coconut Interpreter:")
logger.print('(type "exit()" or press Ctrl-D to end)')
self.start_running()
while self.running:
code = self.get_input()
if code:
compiled = self.handle_input(code)
if compiled:
self.execute(compiled, error=False, print_expr=True)
def exit_runner(self, exit_code=0):
"""Exits the interpreter."""
self.register_error(exit_code)
self.running = False
def handle_input(self, code):
"""Compiles Coconut interpreter input."""
if not self.prompt.multiline:
if not self.comp.should_indent(code):
try:
return self.comp.parse_block(code)
except CoconutException:
pass
while True:
line = self.get_input(more=True)
if line is None:
return None
elif line.strip():
code += "\n" + line
else:
break
try:
return self.comp.parse_block(code)
except CoconutException:
logger.print_exc()
return None
def execute(self,
compiled=None,
error=True,
path=None,
isolate=False,
print_expr=False):
"""Executes compiled code."""
self.check_runner(path, isolate)
if compiled is not None:
if self.show:
print(compiled)
if isolate: # isolate means header is included, and thus encoding must be removed
compiled = rem_encoding(compiled)
if print_expr:
self.runner.run(compiled, error)
else:
self.runner.run(compiled, error, run_func=None)
def check_runner(self, path=None, isolate=False):
"""Makes sure there is a runner."""
if isolate or path is not None or self.runner is None:
self.start_runner(path, isolate)
def start_runner(self, path=None, isolate=False):
"""Starts the runner."""
sys.path.insert(0, os.getcwd())
if isolate:
comp = None
else:
comp = self.comp
self.runner = Runner(comp, self.exit_runner, path)
def launch_tutorial(self):
"""Opens the Coconut tutorial."""
import webbrowser
webbrowser.open(tutorial_url, 2)
def launch_documentation(self):
"""Opens the Coconut documentation."""
import webbrowser
webbrowser.open(documentation_url, 2)
def start_jupyter(self, args):
"""Starts Jupyter with the Coconut kernel."""
def install_func(cmd):
"""Runs an installation command."""
logger.log_cmd(cmd)
if args and not logger.verbose:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
else:
subprocess.check_call(cmd)
try:
install_func(["jupyter", "--version"])
except subprocess.CalledProcessError:
jupyter = "ipython"
else:
jupyter = "jupyter"
for icoconut_kernel_dir in icoconut_kernel_dirs:
install_args = [
jupyter, "kernelspec", "install", icoconut_kernel_dir,
"--replace"
]
try:
install_func(install_args)
except subprocess.CalledProcessError:
user_install_args = install_args + ["--user"]
try:
install_func(user_install_args)
except subprocess.CalledProcessError:
args = "kernel install failed on command'", " ".join(
install_args)
self.comp.warn(*args)
self.register_error(errmsg="Jupyter error")
if args:
if args[0] == "console":
ver = "2" if PY2 else "3"
try:
install_func(
["python" + ver, "-m", "coconut.main", "--version"])
except subprocess.CalledProcessError:
kernel_name = "coconut"
else:
kernel_name = "coconut" + ver
run_args = [jupyter, "console", "--kernel", kernel_name
] + args[1:]
elif args[0] == "notebook":
run_args = [jupyter, "notebook"] + args[1:]
else:
raise CoconutException(
'first argument after --jupyter must be either "console" or "notebook"'
)
logger.log_cmd(run_args)
self.register_error(subprocess.call(run_args),
errmsg="Jupyter error")
def watch(self, source, write=True, package=None, run=False, force=False):
"""Watches a source and recompiles on change."""
from coconut.command.watch import Observer, RecompilationWatcher
source = fixpath(source)
logger.print()
logger.show_tabulated("Watching", showpath(source),
"(press Ctrl-C to end)...")
def recompile(path):
if os.path.isfile(path) and os.path.splitext(path)[1] in code_exts:
with self.handling_exceptions():
self.compile_path(path, write, package, run, force)
observer = Observer()
observer.schedule(RecompilationWatcher(recompile),
source,
recursive=True)
with self.running_jobs():
observer.start()
try:
while True:
time.sleep(watch_interval)
except KeyboardInterrupt:
logger.show("Got KeyboardInterrupt; stopping watcher.")
finally:
observer.stop()
observer.join()
class Command(object):
"""Coconut command-line interface."""
comp = None # current coconut.compiler.Compiler
show = False # corresponds to --display flag
runner = None # the current Runner
jobs = 0 # corresponds to --jobs flag
executor = None # runs --jobs
exit_code = 0 # exit status to return
errmsg = None # error message to display
mypy_args = None # corresponds to --mypy flag
def __init__(self):
"""Create the CLI."""
self.prompt = Prompt()
def start(self, run=False):
"""Process command-line arguments."""
if run:
args, argv = [], []
# for coconut-run, all args beyond the source file should be wrapped in an --argv
for i in range(1, len(sys.argv)):
arg = sys.argv[i]
args.append(arg)
# if arg is source file, put everything else in argv
if not arg.startswith("-") and canparse(arguments, args[:-1]):
argv = sys.argv[i + 1:]
break
if "--verbose" in args:
args = list(coconut_run_verbose_args) + args
else:
args = list(coconut_run_args) + args
args += ["--argv"] + argv
else:
args = None
self.cmd(args)
def cmd(self, args=None, interact=True):
"""Process command-line arguments."""
if args is None:
parsed_args = arguments.parse_args()
else:
parsed_args = arguments.parse_args(args)
self.exit_code = 0
with self.handling_exceptions():
self.use_args(parsed_args, interact, original_args=args)
self.exit_on_error()
def setup(self, *args, **kwargs):
"""Set parameters for the compiler."""
if self.comp is None:
self.comp = Compiler(*args, **kwargs)
else:
self.comp.setup(*args, **kwargs)
def exit_on_error(self):
"""Exit if exit_code is abnormal."""
if self.exit_code:
if self.errmsg is not None:
logger.show("Exiting due to " + self.errmsg + ".")
self.errmsg = None
if self.using_jobs:
kill_children()
sys.exit(self.exit_code)
def use_args(self, args, interact=True, original_args=None):
"""Handle command-line arguments."""
logger.quiet, logger.verbose = args.quiet, args.verbose
if DEVELOP:
logger.tracing = args.trace
logger.log("Using " + PYPARSING + ".")
if original_args is not None:
logger.log("Directly passed args:", original_args)
logger.log("Parsed args:", args)
if args.recursion_limit is not None:
set_recursion_limit(args.recursion_limit)
if args.jobs is not None:
self.set_jobs(args.jobs)
if args.display:
self.show = True
if args.style is not None:
self.prompt.set_style(args.style)
if args.history_file is not None:
self.prompt.set_history_file(args.history_file)
if args.documentation:
launch_documentation()
if args.tutorial:
launch_tutorial()
self.setup(
target=args.target,
strict=args.strict,
minify=args.minify,
line_numbers=args.line_numbers,
keep_lines=args.keep_lines,
no_tco=args.no_tco,
)
if args.mypy is not None:
self.set_mypy_args(args.mypy)
if args.argv is not None:
sys.argv = [args.source if args.source is not None else ""]
sys.argv.extend(args.argv)
if args.source is not None:
if args.interact and args.run:
logger.warn("extraneous --run argument passed; --interact implies --run")
if args.package and self.mypy:
logger.warn("extraneous --package argument passed; --mypy implies --package")
if args.standalone and args.package:
raise CoconutException("cannot compile as both --package and --standalone")
if args.standalone and self.mypy:
raise CoconutException("cannot compile as both --package (implied by --mypy) and --standalone")
if args.no_write and self.mypy:
raise CoconutException("cannot compile with --no-write when using --mypy")
if (args.run or args.interact) and os.path.isdir(args.source):
if args.run:
raise CoconutException("source path must point to file not directory when --run is enabled")
if args.interact:
raise CoconutException("source path must point to file not directory when --run (implied by --interact) is enabled")
if args.watch and os.path.isfile(args.source):
raise CoconutException("source path must point to directory not file when --watch is enabled")
if args.dest is None:
if args.no_write:
dest = False # no dest
else:
dest = True # auto-generate dest
elif args.no_write:
raise CoconutException("destination path cannot be given when --no-write is enabled")
else:
dest = args.dest
if args.package or self.mypy:
package = True
elif args.standalone:
package = False
else:
package = None # auto-decide package
with self.running_jobs(exit_on_error=not args.watch):
filepaths = self.compile_path(args.source, dest, package, args.run or args.interact, args.force)
self.run_mypy(filepaths)
elif (
args.run
or args.no_write
or args.force
or args.package
or args.standalone
or args.watch
):
raise CoconutException("a source file/folder must be specified when options that depend on the source are enabled")
if args.code is not None:
self.execute(self.comp.parse_block(args.code))
got_stdin = False
if args.jupyter is not None:
self.start_jupyter(args.jupyter)
elif stdin_readable():
logger.log("Reading piped input from stdin...")
self.execute(self.comp.parse_block(sys.stdin.read()))
got_stdin = True
if args.interact or (interact and not (
got_stdin
or args.source
or args.code
or args.tutorial
or args.documentation
or args.watch
or args.jupyter is not None
)):
self.start_prompt()
if args.watch:
self.watch(args.source, dest, package, args.run, args.force)
def register_error(self, code=1, errmsg=None):
"""Update the exit code."""
if errmsg is not None:
if self.errmsg is None:
self.errmsg = errmsg
elif errmsg not in self.errmsg:
self.errmsg += ", " + errmsg
if code is not None:
self.exit_code = max(self.exit_code, code)
@contextmanager
def handling_exceptions(self):
"""Perform proper exception handling."""
try:
if self.using_jobs:
with handling_broken_process_pool():
yield
else:
yield
except SystemExit as err:
self.register_error(err.code)
except BaseException as err:
if isinstance(err, CoconutException):
logger.display_exc()
elif not isinstance(err, KeyboardInterrupt):
traceback.print_exc()
printerr(report_this_text)
self.register_error(errmsg=err.__class__.__name__)
def compile_path(self, path, write=True, package=None, *args, **kwargs):
"""Compile a path and returns paths to compiled files."""
path = fixpath(path)
if not isinstance(write, bool):
write = fixpath(write)
if os.path.isfile(path):
if package is None:
package = False
destpath = self.compile_file(path, write, package, *args, **kwargs)
return [destpath] if destpath is not None else []
elif os.path.isdir(path):
if package is None:
package = True
return self.compile_folder(path, write, package, *args, **kwargs)
else:
raise CoconutException("could not find source path", path)
def compile_folder(self, directory, write=True, package=True, *args, **kwargs):
"""Compile a directory and returns paths to compiled files."""
if not isinstance(write, bool) and os.path.isfile(write):
raise CoconutException("destination path cannot point to a file when compiling a directory")
filepaths = []
for dirpath, dirnames, filenames in os.walk(directory):
if isinstance(write, bool):
writedir = write
else:
writedir = os.path.join(write, os.path.relpath(dirpath, directory))
for filename in filenames:
if os.path.splitext(filename)[1] in code_exts:
with self.handling_exceptions():
destpath = self.compile_file(os.path.join(dirpath, filename), writedir, package, *args, **kwargs)
if destpath is not None:
filepaths.append(destpath)
for name in dirnames[:]:
if not is_special_dir(name) and name.startswith("."):
if logger.verbose:
logger.show_tabulated("Skipped directory", name, "(explicitly pass as source to override).")
dirnames.remove(name) # directories removed from dirnames won't appear in further os.walk iterations
return filepaths
def compile_file(self, filepath, write=True, package=False, *args, **kwargs):
"""Compile a file and returns the compiled file's path."""
set_ext = False
if write is False:
destpath = None
elif write is True:
destpath = filepath
set_ext = True
elif os.path.splitext(write)[1]:
# write is a file; it is the destination filepath
destpath = write
else:
# write is a dir; make the destination filepath by adding the filename
destpath = os.path.join(write, os.path.basename(filepath))
set_ext = True
if set_ext:
base, ext = os.path.splitext(os.path.splitext(destpath)[0])
if not ext:
ext = comp_ext
destpath = fixpath(base + ext)
if filepath == destpath:
raise CoconutException("cannot compile " + showpath(filepath) + " to itself", extra="incorrect file extension")
self.compile(filepath, destpath, package, *args, **kwargs)
return destpath
def compile(self, codepath, destpath=None, package=False, run=False, force=False, show_unchanged=True):
"""Compile a source Coconut file to a destination Python file."""
with openfile(codepath, "r") as opened:
code = readfile(opened)
if destpath is not None:
destdir = os.path.dirname(destpath)
if not os.path.exists(destdir):
os.makedirs(destdir)
if package is True:
self.create_package(destdir)
foundhash = None if force else self.has_hash_of(destpath, code, package)
if foundhash:
if show_unchanged:
logger.show_tabulated("Left unchanged", showpath(destpath), "(pass --force to override).")
if self.show:
print(foundhash)
if run:
self.execute_file(destpath)
else:
logger.show_tabulated("Compiling", showpath(codepath), "...")
if package is True:
compile_method = "parse_package"
elif package is False:
compile_method = "parse_file"
else:
raise CoconutInternalException("invalid value for package", package)
def callback(compiled):
if destpath is None:
logger.show_tabulated("Compiled", showpath(codepath), "without writing to file.")
else:
with openfile(destpath, "w") as opened:
writefile(opened, compiled)
logger.show_tabulated("Compiled to", showpath(destpath), ".")
if self.show:
print(compiled)
if run:
if destpath is None:
self.execute(compiled, path=codepath, allow_show=False)
else:
self.execute_file(destpath)
self.submit_comp_job(codepath, callback, compile_method, code)
def submit_comp_job(self, path, callback, method, *args, **kwargs):
"""Submits a job on self.comp to be run in parallel."""
if self.executor is None:
with self.handling_exceptions():
callback(getattr(self.comp, method)(*args, **kwargs))
else:
path = showpath(path)
with logger.in_path(path): # pickle the compiler in the path context
future = self.executor.submit(multiprocess_wrapper(self.comp, method), *args, **kwargs)
def callback_wrapper(completed_future):
"""Ensures that all errors are always caught, since errors raised in a callback won't be propagated."""
with logger.in_path(path): # handle errors in the path context
with self.handling_exceptions():
result = completed_future.result()
callback(result)
future.add_done_callback(callback_wrapper)
def set_jobs(self, jobs):
"""Set --jobs."""
if jobs == "sys":
self.jobs = None
else:
try:
jobs = int(jobs)
except ValueError:
jobs = -1 # will raise error below
if jobs < 0:
raise CoconutException("--jobs must be an integer >= 0 or 'sys'")
self.jobs = jobs
@property
def using_jobs(self):
"""Determine whether or not multiprocessing is being used."""
return self.jobs != 0
@contextmanager
def running_jobs(self, exit_on_error=True):
"""Initialize multiprocessing."""
with self.handling_exceptions():
if self.using_jobs:
from concurrent.futures import ProcessPoolExecutor
try:
with ProcessPoolExecutor(self.jobs) as self.executor:
yield
finally:
self.executor = None
else:
yield
if exit_on_error:
self.exit_on_error()
def create_package(self, dirpath):
"""Set up a package directory."""
dirpath = fixpath(dirpath)
filepath = os.path.join(dirpath, "__coconut__.py")
with openfile(filepath, "w") as opened:
writefile(opened, self.comp.getheader("__coconut__"))
def has_hash_of(self, destpath, code, package):
"""Determine if a file has the hash of the code."""
if destpath is not None and os.path.isfile(destpath):
with openfile(destpath, "r") as opened:
compiled = readfile(opened)
hashash = gethash(compiled)
if hashash is not None and hashash == self.comp.genhash(package, code):
return compiled
return None
def get_input(self, more=False):
"""Prompt for code input."""
received = None
try:
received = self.prompt.input(more)
except KeyboardInterrupt:
print()
printerr("KeyboardInterrupt")
except EOFError:
print()
self.exit_runner()
else:
if received.startswith(exit_chars):
self.exit_runner()
received = None
return received
def start_running(self):
"""Start running the Runner."""
self.comp.warm_up()
self.check_runner()
self.running = True
def start_prompt(self):
"""Start the interpreter."""
logger.show("Coconut Interpreter:")
logger.show("(type 'exit()' or press Ctrl-D to end)")
self.start_running()
while self.running:
try:
code = self.get_input()
if code:
compiled = self.handle_input(code)
if compiled:
self.execute(compiled, use_eval=None)
except KeyboardInterrupt:
printerr("\nKeyboardInterrupt")
def exit_runner(self, exit_code=0):
"""Exit the interpreter."""
self.register_error(exit_code)
self.running = False
def handle_input(self, code):
"""Compile Coconut interpreter input."""
if not self.prompt.multiline:
if not should_indent(code):
try:
return self.comp.parse_block(code)
except CoconutException:
pass
while True:
line = self.get_input(more=True)
if line is None:
return None
elif line:
code += "\n" + line
else:
break
try:
return self.comp.parse_block(code)
except CoconutException:
logger.display_exc()
return None
def execute(self, compiled=None, path=None, use_eval=False, allow_show=True):
"""Execute compiled code."""
self.check_runner()
if compiled is not None:
if allow_show and self.show:
print(compiled)
if path is not None: # path means header is included, and thus encoding must be removed
compiled = rem_encoding(compiled)
self.runner.run(compiled, use_eval=use_eval, path=path, all_errors_exit=(path is not None))
self.run_mypy(code=self.runner.was_run_code())
def execute_file(self, destpath):
"""Execute compiled file."""
self.check_runner()
self.runner.run_file(destpath)
def check_runner(self):
"""Make sure there is a runner."""
if os.getcwd() not in sys.path:
sys.path.append(os.getcwd())
if self.runner is None:
self.runner = Runner(self.comp, exit=self.exit_runner, store=self.mypy)
@property
def mypy(self):
"""Whether using MyPy or not."""
return self.mypy_args is not None
def set_mypy_args(self, mypy_args=None):
"""Set MyPy arguments."""
if mypy_args is None:
self.mypy_args = None
else:
self.mypy_errs = []
self.mypy_args = list(mypy_args)
if not any(arg.startswith("--python-version") for arg in mypy_args):
self.mypy_args += [
"--python-version",
".".join(str(v) for v in get_target_info_len2(self.comp.target, mode="nearest")),
]
if logger.verbose:
for arg in verbose_mypy_args:
if arg not in self.mypy_args:
self.mypy_args.append(arg)
logger.log("MyPy args:", self.mypy_args)
def run_mypy(self, paths=(), code=None):
"""Run MyPy with arguments."""
if self.mypy:
set_mypy_path(stub_dir)
from coconut.command.mypy import mypy_run
args = list(paths) + self.mypy_args
if code is not None:
args += ["-c", code]
for line, is_err in mypy_run(args):
if code is None or line not in self.mypy_errs:
printerr(line)
if line not in self.mypy_errs:
self.mypy_errs.append(line)
self.register_error(errmsg="MyPy error")
def start_jupyter(self, args):
"""Start Jupyter with the Coconut kernel."""
install_func = partial(run_cmd, show_output=logger.verbose)
try:
install_func(["jupyter", "--version"])
except CalledProcessError:
jupyter = "ipython"
else:
jupyter = "jupyter"
# always install kernels if given no args, otherwise only if there's a kernel missing
do_install = not args
if not do_install:
kernel_list = run_cmd([jupyter, "kernelspec", "list"], show_output=False, raise_errs=False)
do_install = any(ker not in kernel_list for ker in icoconut_kernel_names)
if do_install:
success = True
for icoconut_kernel_dir in icoconut_kernel_dirs:
install_args = [jupyter, "kernelspec", "install", icoconut_kernel_dir, "--replace"]
try:
install_func(install_args)
except CalledProcessError:
user_install_args = install_args + ["--user"]
try:
install_func(user_install_args)
except CalledProcessError:
logger.warn("kernel install failed on command'", " ".join(install_args))
self.register_error(errmsg="Jupyter error")
success = False
if success:
logger.show_sig("Successfully installed Coconut Jupyter kernel.")
if args:
if args[0] == "console":
ver = "2" if PY2 else "3"
try:
install_func(["python" + ver, "-m", "coconut.main", "--version"])
except CalledProcessError:
kernel_name = "coconut"
else:
kernel_name = "coconut" + ver
run_args = [jupyter, "console", "--kernel", kernel_name] + args[1:]
else:
run_args = [jupyter] + args
self.register_error(run_cmd(run_args, raise_errs=False), errmsg="Jupyter error")
def watch(self, source, write=True, package=None, run=False, force=False):
"""Watch a source and recompiles on change."""
from coconut.command.watch import Observer, RecompilationWatcher
source = fixpath(source)
logger.show()
logger.show_tabulated("Watching", showpath(source), "(press Ctrl-C to end)...")
def recompile(path):
path = fixpath(path)
if os.path.isfile(path) and os.path.splitext(path)[1] in code_exts:
with self.handling_exceptions():
if write is True or write is None:
writedir = write
else:
# correct the compilation path based on the relative position of path to source
dirpath = os.path.dirname(path)
writedir = os.path.join(write, os.path.relpath(dirpath, source))
filepaths = self.compile_path(path, writedir, package, run, force, show_unchanged=False)
self.run_mypy(filepaths)
watcher = RecompilationWatcher(recompile)
observer = Observer()
observer.schedule(watcher, source, recursive=True)
with self.running_jobs():
observer.start()
try:
while True:
time.sleep(watch_interval)
watcher.keep_watching()
except KeyboardInterrupt:
logger.show_sig("Got KeyboardInterrupt; stopping watcher.")
finally:
observer.stop()
observer.join()
def __init__(self, pyfun):
self.pyfun = pyfun
self.c = Compiler()
self.c.compile(pyfun.__code__)
self.ircfg = self.c.ircfg
def compile_coconut(cls, source):
"""Compile the given Coconut source text."""
if cls.coconut_compiler is None:
cls.coconut_compiler = Compiler(**coconut_encoding_kwargs)
return cls.coconut_compiler.parse_sys(source)
def __init__(self, pyfun):
self.pyfun = pyfun
self.c = Compiler()
self.c.compile(pyfun.__code__)
self.ircfg = self.c.ircfg
class Command(object):
"""Coconut command-line interface."""
comp = None # current coconut.compiler.Compiler
show = False # corresponds to --display flag
runner = None # the current Runner
jobs = 0 # corresponds to --jobs flag
executor = None # runs --jobs
exit_code = 0 # exit status to return
errmsg = None # error message to display
mypy_args = None # corresponds to --mypy flag
argv_args = None # corresponds to --argv flag
def __init__(self):
"""Create the CLI."""
self.prompt = Prompt()
def start(self, run=False):
"""Endpoint for coconut and coconut-run."""
if run:
args, argv = [], []
# for coconut-run, all args beyond the source file should be wrapped in an --argv
for i in range(1, len(sys.argv)):
arg = sys.argv[i]
args.append(arg)
# if arg is source file, put everything else in argv
if not arg.startswith("-") and can_parse(arguments, args[:-1]):
argv = sys.argv[i + 1:]
break
for run_arg in (coconut_run_verbose_args if "--verbose" in args else coconut_run_args):
if run_arg not in args:
args.append(run_arg)
self.cmd(args, argv=argv)
else:
self.cmd()
def cmd(self, args=None, argv=None, interact=True):
"""Process command-line arguments."""
if args is None:
parsed_args = arguments.parse_args()
else:
parsed_args = arguments.parse_args(args)
if argv is not None:
if parsed_args.argv is not None:
raise CoconutException("cannot pass --argv/--args when using coconut-run (coconut-run interprets any arguments after the source file as --argv/--args)")
parsed_args.argv = argv
self.exit_code = 0
with self.handling_exceptions():
self.use_args(parsed_args, interact, original_args=args)
self.exit_on_error()
def setup(self, *args, **kwargs):
"""Set parameters for the compiler."""
if self.comp is None:
self.comp = Compiler(*args, **kwargs)
else:
self.comp.setup(*args, **kwargs)
def exit_on_error(self):
"""Exit if exit_code is abnormal."""
if self.exit_code:
if self.errmsg is not None:
logger.show("Exiting with error: " + self.errmsg)
self.errmsg = None
if self.using_jobs:
kill_children()
sys.exit(self.exit_code)
def use_args(self, args, interact=True, original_args=None):
"""Handle command-line arguments."""
# fix args
if not DEVELOP:
args.trace = args.profile = False
# set up logger
logger.quiet, logger.verbose, logger.tracing = args.quiet, args.verbose, args.trace
if args.trace or args.profile:
unset_fast_pyparsing_reprs()
if args.profile:
collect_timing_info()
logger.log(cli_version)
if original_args is not None:
logger.log("Directly passed args:", original_args)
logger.log("Parsed args:", args)
# validate general command args
if args.mypy is not None and args.line_numbers:
logger.warn("extraneous --line-numbers argument passed; --mypy implies --line-numbers")
if args.site_install and args.site_uninstall:
raise CoconutException("cannot --site-install and --site-uninstall simultaneously")
# process general command args
if args.recursion_limit is not None:
set_recursion_limit(args.recursion_limit)
if args.jobs is not None:
self.set_jobs(args.jobs)
if args.display:
self.show = True
if args.style is not None:
self.prompt.set_style(args.style)
if args.history_file is not None:
self.prompt.set_history_file(args.history_file)
if args.vi_mode:
self.prompt.vi_mode = True
if args.docs:
launch_documentation()
if args.tutorial:
launch_tutorial()
if args.site_uninstall:
self.site_uninstall()
if args.site_install:
self.site_install()
if args.argv is not None:
self.argv_args = list(args.argv)
# additional validation after processing
if args.profile and self.jobs != 0:
raise CoconutException("--profile incompatible with --jobs {jobs}".format(jobs=args.jobs))
# process general compiler args
self.setup(
target=args.target,
strict=args.strict,
minify=args.minify,
line_numbers=args.line_numbers or args.mypy is not None,
keep_lines=args.keep_lines,
no_tco=args.no_tco,
no_wrap=args.no_wrap,
)
# process mypy args (must come after compiler setup)
if args.mypy is not None:
self.set_mypy_args(args.mypy)
if args.source is not None:
# warnings if source is given
if args.interact and args.run:
logger.warn("extraneous --run argument passed; --interact implies --run")
if args.package and self.mypy:
logger.warn("extraneous --package argument passed; --mypy implies --package")
# errors if source is given
if args.standalone and args.package:
raise CoconutException("cannot compile as both --package and --standalone")
if args.standalone and self.mypy:
raise CoconutException("cannot compile as both --package (implied by --mypy) and --standalone")
if args.no_write and self.mypy:
raise CoconutException("cannot compile with --no-write when using --mypy")
# process all source, dest pairs
src_dest_package_triples = [
self.process_source_dest(src, dst, args)
for src, dst in (
[(args.source, args.dest)]
+ (getattr(args, "and") or [])
)
]
# do compilation
with self.running_jobs(exit_on_error=not args.watch):
filepaths = []
for source, dest, package in src_dest_package_triples:
filepaths += self.compile_path(source, dest, package, run=args.run or args.interact, force=args.force)
self.run_mypy(filepaths)
# validate args if no source is given
elif (
args.run
or args.no_write
or args.force
or args.package
or args.standalone
or args.watch
):
raise CoconutException("a source file/folder must be specified when options that depend on the source are enabled")
elif getattr(args, "and"):
raise CoconutException("--and should only be used for extra source/dest pairs, not the first source/dest pair")
# handle extra cli tasks
if args.code is not None:
self.execute(self.comp.parse_block(args.code))
got_stdin = False
if args.jupyter is not None:
self.start_jupyter(args.jupyter)
elif stdin_readable():
logger.log("Reading piped input from stdin...")
self.execute(self.comp.parse_block(sys.stdin.read()))
got_stdin = True
if args.interact or (
interact and not (
got_stdin
or args.source
or args.code
or args.tutorial
or args.docs
or args.watch
or args.site_uninstall
or args.site_install
or args.jupyter is not None
or args.mypy == [mypy_install_arg]
)
):
self.start_prompt()
if args.watch:
# src_dest_package_triples is always available here
self.watch(src_dest_package_triples, args.run, args.force)
if args.profile:
print_timing_info()
def process_source_dest(self, source, dest, args):
"""Determine the correct source, dest, package mode to use for the given source, dest, and args."""
# determine source
processed_source = fixpath(source)
# validate args
if (args.run or args.interact) and os.path.isdir(processed_source):
if args.run:
raise CoconutException("source path %r must point to file not directory when --run is enabled" % (source,))
if args.interact:
raise CoconutException("source path %r must point to file not directory when --run (implied by --interact) is enabled" % (source,))
if args.watch and os.path.isfile(processed_source):
raise CoconutException("source path %r must point to directory not file when --watch is enabled" % (source,))
# determine dest
if dest is None:
if args.no_write:
processed_dest = False # no dest
else:
processed_dest = True # auto-generate dest
elif args.no_write:
raise CoconutException("destination path cannot be given when --no-write is enabled")
else:
processed_dest = dest
# determine package mode
if args.package or self.mypy:
package = True
elif args.standalone:
package = False
else:
# auto-decide package
if os.path.isfile(source):
package = False
elif os.path.isdir(source):
package = True
else:
raise CoconutException("could not find source path", source)
return processed_source, processed_dest, package
def register_error(self, code=1, errmsg=None):
"""Update the exit code."""
if errmsg is not None:
if self.errmsg is None:
self.errmsg = errmsg
elif errmsg not in self.errmsg:
self.errmsg += ", " + errmsg
if code is not None:
self.exit_code = code or self.exit_code
@contextmanager
def handling_exceptions(self):
"""Perform proper exception handling."""
try:
if self.using_jobs:
with handling_broken_process_pool():
yield
else:
yield
except SystemExit as err:
self.register_error(err.code)
except BaseException as err:
if isinstance(err, CoconutException):
logger.print_exc()
elif not isinstance(err, KeyboardInterrupt):
logger.print_exc()
printerr(report_this_text)
self.register_error(errmsg=err.__class__.__name__)
def compile_path(self, path, write=True, package=True, **kwargs):
"""Compile a path and returns paths to compiled files."""
if not isinstance(write, bool):
write = fixpath(write)
if os.path.isfile(path):
destpath = self.compile_file(path, write, package, **kwargs)
return [destpath] if destpath is not None else []
elif os.path.isdir(path):
return self.compile_folder(path, write, package, **kwargs)
else:
raise CoconutException("could not find source path", path)
def compile_folder(self, directory, write=True, package=True, **kwargs):
"""Compile a directory and returns paths to compiled files."""
if not isinstance(write, bool) and os.path.isfile(write):
raise CoconutException("destination path cannot point to a file when compiling a directory")
filepaths = []
for dirpath, dirnames, filenames in os.walk(directory):
if isinstance(write, bool):
writedir = write
else:
writedir = os.path.join(write, os.path.relpath(dirpath, directory))
for filename in filenames:
if os.path.splitext(filename)[1] in code_exts:
with self.handling_exceptions():
destpath = self.compile_file(os.path.join(dirpath, filename), writedir, package, **kwargs)
if destpath is not None:
filepaths.append(destpath)
for name in dirnames[:]:
if not is_special_dir(name) and name.startswith("."):
if logger.verbose:
logger.show_tabulated("Skipped directory", name, "(explicitly pass as source to override).")
dirnames.remove(name) # directories removed from dirnames won't appear in further os.walk iterations
return filepaths
def compile_file(self, filepath, write=True, package=False, force=False, **kwargs):
"""Compile a file and returns the compiled file's path."""
set_ext = False
if write is False:
destpath = None
elif write is True:
destpath = filepath
set_ext = True
elif os.path.splitext(write)[1]:
# write is a file; it is the destination filepath
destpath = write
else:
# write is a dir; make the destination filepath by adding the filename
destpath = os.path.join(write, os.path.basename(filepath))
set_ext = True
if set_ext:
base, ext = os.path.splitext(os.path.splitext(destpath)[0])
if not ext:
ext = comp_ext
destpath = fixpath(base + ext)
if filepath == destpath:
raise CoconutException("cannot compile " + showpath(filepath) + " to itself", extra="incorrect file extension")
if destpath is not None:
dest_ext = os.path.splitext(destpath)[1]
if dest_ext in code_exts:
if force:
logger.warn("found destination path with " + dest_ext + " extension; compiling anyway due to --force")
else:
raise CoconutException(
"found destination path with " + dest_ext + " extension; aborting compilation",
extra="pass --force to override",
)
self.compile(filepath, destpath, package, force=force, **kwargs)
return destpath
def compile(self, codepath, destpath=None, package=False, run=False, force=False, show_unchanged=True):
"""Compile a source Coconut file to a destination Python file."""
with univ_open(codepath, "r") as opened:
code = readfile(opened)
package_level = -1
if destpath is not None:
destpath = fixpath(destpath)
destdir = os.path.dirname(destpath)
if not os.path.exists(destdir):
os.makedirs(destdir)
if package is True:
package_level = self.get_package_level(codepath)
if package_level == 0:
self.create_package(destdir)
foundhash = None if force else self.has_hash_of(destpath, code, package_level)
if foundhash:
if show_unchanged:
logger.show_tabulated("Left unchanged", showpath(destpath), "(pass --force to override).")
if self.show:
print(foundhash)
if run:
self.execute_file(destpath, argv_source_path=codepath)
else:
logger.show_tabulated("Compiling", showpath(codepath), "...")
def callback(compiled):
if destpath is None:
logger.show_tabulated("Compiled", showpath(codepath), "without writing to file.")
else:
with univ_open(destpath, "w") as opened:
writefile(opened, compiled)
logger.show_tabulated("Compiled to", showpath(destpath), ".")
if self.show:
print(compiled)
if run:
if destpath is None:
self.execute(compiled, path=codepath, allow_show=False)
else:
self.execute_file(destpath, argv_source_path=codepath)
if package is True:
self.submit_comp_job(codepath, callback, "parse_package", code, package_level=package_level)
elif package is False:
self.submit_comp_job(codepath, callback, "parse_file", code)
else:
raise CoconutInternalException("invalid value for package", package)
def get_package_level(self, codepath):
"""Get the relative level to the base directory of the package."""
package_level = -1
check_dir = os.path.dirname(os.path.abspath(codepath))
while check_dir:
has_init = False
for ext in code_exts:
init_file = os.path.join(check_dir, "__init__" + ext)
if os.path.exists(init_file):
has_init = True
break
if has_init:
package_level += 1
check_dir = os.path.dirname(check_dir)
else:
break
if package_level < 0:
if self.comp.strict:
logger.warn("missing __init__" + code_exts[0] + " in package", check_dir, extra="remove --strict to dismiss")
package_level = 0
return package_level
return 0
def create_package(self, dirpath):
"""Set up a package directory."""
filepath = os.path.join(dirpath, "__coconut__.py")
with univ_open(filepath, "w") as opened:
writefile(opened, self.comp.getheader("__coconut__"))
def submit_comp_job(self, path, callback, method, *args, **kwargs):
"""Submits a job on self.comp to be run in parallel."""
if self.executor is None:
with self.handling_exceptions():
callback(getattr(self.comp, method)(*args, **kwargs))
else:
path = showpath(path)
with logger.in_path(path): # pickle the compiler in the path context
future = self.executor.submit(multiprocess_wrapper(self.comp, method), *args, **kwargs)
def callback_wrapper(completed_future):
"""Ensures that all errors are always caught, since errors raised in a callback won't be propagated."""
with logger.in_path(path): # handle errors in the path context
with self.handling_exceptions():
result = completed_future.result()
callback(result)
future.add_done_callback(callback_wrapper)
def set_jobs(self, jobs):
"""Set --jobs."""
if jobs == "sys":
self.jobs = None
else:
try:
jobs = int(jobs)
except ValueError:
jobs = -1 # will raise error below
if jobs < 0:
raise CoconutException("--jobs must be an integer >= 0 or 'sys'")
self.jobs = jobs
@property
def using_jobs(self):
"""Determine whether or not multiprocessing is being used."""
return self.jobs is None or self.jobs > 1
@contextmanager
def running_jobs(self, exit_on_error=True):
"""Initialize multiprocessing."""
with self.handling_exceptions():
if self.using_jobs:
from concurrent.futures import ProcessPoolExecutor
try:
with ProcessPoolExecutor(self.jobs) as self.executor:
yield
finally:
self.executor = None
else:
yield
if exit_on_error:
self.exit_on_error()
def has_hash_of(self, destpath, code, package_level):
"""Determine if a file has the hash of the code."""
if destpath is not None and os.path.isfile(destpath):
with univ_open(destpath, "r") as opened:
compiled = readfile(opened)
hashash = gethash(compiled)
if hashash is not None and hashash == self.comp.genhash(code, package_level):
return True
return False
def get_input(self, more=False):
"""Prompt for code input."""
received = None
try:
received = self.prompt.input(more)
except KeyboardInterrupt:
print()
printerr("KeyboardInterrupt")
except EOFError:
print()
self.exit_runner()
else:
if received.startswith(exit_chars):
self.exit_runner()
received = None
return received
def start_running(self):
"""Start running the Runner."""
self.comp.warm_up()
self.check_runner()
self.running = True
def start_prompt(self):
"""Start the interpreter."""
logger.show(
"Coconut Interpreter v{co_ver}:".format(
co_ver=VERSION,
),
)
logger.show("(enter 'exit()' or press Ctrl-D to end)")
self.start_running()
while self.running:
try:
code = self.get_input()
if code:
compiled = self.handle_input(code)
if compiled:
self.execute(compiled, use_eval=None)
except KeyboardInterrupt:
printerr("\nKeyboardInterrupt")
def exit_runner(self, exit_code=0):
"""Exit the interpreter."""
self.register_error(exit_code)
self.running = False
def handle_input(self, code):
"""Compile Coconut interpreter input."""
if not self.prompt.multiline:
if not should_indent(code):
try:
return self.comp.parse_block(code)
except CoconutException:
pass
while True:
line = self.get_input(more=True)
if line is None:
return None
elif line:
code += "\n" + line
else:
break
try:
return self.comp.parse_block(code)
except CoconutException:
logger.print_exc()
return None
def execute(self, compiled=None, path=None, use_eval=False, allow_show=True):
"""Execute compiled code."""
self.check_runner()
if compiled is not None:
if allow_show and self.show:
print(compiled)
if path is None: # header is not included
if not self.mypy:
no_str_code = self.comp.remove_strs(compiled)
result = mypy_builtin_regex.search(no_str_code)
if result:
logger.warn("found mypy-only built-in " + repr(result.group(0)) + "; pass --mypy to use mypy-only built-ins at the interpreter")
else: # header is included
compiled = rem_encoding(compiled)
self.runner.run(compiled, use_eval=use_eval, path=path, all_errors_exit=path is not None)
self.run_mypy(code=self.runner.was_run_code())
def execute_file(self, destpath, **kwargs):
"""Execute compiled file."""
self.check_runner(**kwargs)
self.runner.run_file(destpath)
def check_runner(self, set_sys_vars=True, argv_source_path=""):
"""Make sure there is a runner."""
if set_sys_vars:
# set sys.path
if os.getcwd() not in sys.path:
sys.path.append(os.getcwd())
# set sys.argv
if self.argv_args is not None:
sys.argv = [argv_source_path] + self.argv_args
# set up runner
if self.runner is None:
self.runner = Runner(self.comp, exit=self.exit_runner, store=self.mypy)
@property
def mypy(self):
"""Whether using MyPy or not."""
return self.mypy_args is not None
def set_mypy_args(self, mypy_args=None):
"""Set MyPy arguments."""
if mypy_args is None:
self.mypy_args = None
elif mypy_install_arg in mypy_args:
if mypy_args != [mypy_install_arg]:
raise CoconutException("'--mypy install' cannot be used alongside other --mypy arguments")
stub_dir = set_mypy_path()
logger.show_sig("Successfully installed MyPy stubs into " + repr(stub_dir))
self.mypy_args = None
else:
self.mypy_args = list(mypy_args)
if not any(arg.startswith("--python-version") for arg in self.mypy_args):
self.mypy_args += [
"--python-version",
ver_tuple_to_str(get_target_info_smart(self.comp.target, mode="mypy")),
]
if not any(arg.startswith("--python-executable") for arg in self.mypy_args):
self.mypy_args += [
"--python-executable",
sys.executable,
]
add_mypy_args = default_mypy_args + (verbose_mypy_args if logger.verbose else ())
for arg in add_mypy_args:
no_arg = invert_mypy_arg(arg)
arg_prefixes = (arg,) + ((no_arg,) if no_arg is not None else ())
if not any(arg.startswith(arg_prefixes) for arg in self.mypy_args):
self.mypy_args.append(arg)
logger.log("MyPy args:", self.mypy_args)
self.mypy_errs = []
def run_mypy(self, paths=(), code=None):
"""Run MyPy with arguments."""
if self.mypy:
set_mypy_path()
from coconut.command.mypy import mypy_run
args = list(paths) + self.mypy_args
if code is not None: # interpreter
args += ["-c", code]
for line, is_err in mypy_run(args):
logger.log("[MyPy]", line)
if line.startswith(mypy_silent_err_prefixes):
if code is None: # file
printerr(line)
self.register_error(errmsg="MyPy error")
elif not line.startswith(mypy_silent_non_err_prefixes):
if code is None: # file
printerr(line)
if any(infix in line for infix in mypy_err_infixes):
self.register_error(errmsg="MyPy error")
if line not in self.mypy_errs:
if code is not None: # interpreter
printerr(line)
self.mypy_errs.append(line)
def run_silent_cmd(self, *args):
"""Same as run_cmd$(show_output=logger.verbose)."""
return run_cmd(*args, show_output=logger.verbose)
def install_jupyter_kernel(self, jupyter, kernel_dir):
"""Install the given kernel via the command line and return whether successful."""
install_args = jupyter + ["kernelspec", "install", kernel_dir, "--replace"]
try:
self.run_silent_cmd(install_args)
except CalledProcessError:
user_install_args = install_args + ["--user"]
try:
self.run_silent_cmd(user_install_args)
except CalledProcessError:
logger.warn("kernel install failed on command", " ".join(install_args))
self.register_error(errmsg="Jupyter kernel error")
return False
return True
def remove_jupyter_kernel(self, jupyter, kernel_name):
"""Remove the given kernel via the command line and return whether successful."""
remove_args = jupyter + ["kernelspec", "remove", kernel_name, "-f"]
try:
self.run_silent_cmd(remove_args)
except CalledProcessError:
logger.warn("kernel removal failed on command", " ".join(remove_args))
self.register_error(errmsg="Jupyter kernel error")
return False
return True
def install_default_jupyter_kernels(self, jupyter, kernel_list):
"""Install icoconut default kernels."""
logger.show_sig("Installing Jupyter kernels '" + "', '".join(icoconut_default_kernel_names) + "'...")
overall_success = True
for old_kernel_name in icoconut_old_kernel_names:
if old_kernel_name in kernel_list:
success = self.remove_jupyter_kernel(jupyter, old_kernel_name)
overall_success = overall_success and success
for kernel_dir in icoconut_default_kernel_dirs:
success = self.install_jupyter_kernel(jupyter, kernel_dir)
overall_success = overall_success and success
if overall_success:
return icoconut_default_kernel_names
else:
return []
def get_jupyter_kernels(self, jupyter):
"""Get the currently installed Jupyter kernels."""
raw_kernel_list = run_cmd(jupyter + ["kernelspec", "list"], show_output=False, raise_errs=False)
kernel_list = []
for line in raw_kernel_list.splitlines():
kernel_list.append(line.split()[0])
return kernel_list
def start_jupyter(self, args):
"""Start Jupyter with the Coconut kernel."""
# get the correct jupyter command
for jupyter in (
[sys.executable, "-m", "jupyter"],
[sys.executable, "-m", "ipython"],
["jupyter"],
):
try:
self.run_silent_cmd(jupyter + ["--help"]) # --help is much faster than --version
except CalledProcessError:
logger.warn("failed to find Jupyter command at " + repr(" ".join(jupyter)))
else:
break
# get a list of installed kernels
kernel_list = self.get_jupyter_kernels(jupyter)
newly_installed_kernels = []
# always update the custom kernel, but only reinstall it if it isn't already there or given no args
custom_kernel_dir = install_custom_kernel(logger=logger)
if custom_kernel_dir is not None and (icoconut_custom_kernel_name not in kernel_list or not args):
logger.show_sig("Installing Jupyter kernel {name!r}...".format(name=icoconut_custom_kernel_name))
if self.install_jupyter_kernel(jupyter, custom_kernel_dir):
newly_installed_kernels.append(icoconut_custom_kernel_name)
if not args:
# install default kernels if given no args
newly_installed_kernels += self.install_default_jupyter_kernels(jupyter, kernel_list)
run_args = None
else:
# use the custom kernel if it exists
if icoconut_custom_kernel_name in kernel_list or icoconut_custom_kernel_name in newly_installed_kernels:
kernel = icoconut_custom_kernel_name
# otherwise determine which default kernel to use and install them if necessary
else:
ver = "2" if PY2 else "3"
try:
self.run_silent_cmd(["python" + ver, "-m", "coconut.main", "--version"])
except CalledProcessError:
kernel = "coconut_py"
else:
kernel = "coconut_py" + ver
if kernel not in kernel_list:
newly_installed_kernels += self.install_default_jupyter_kernels(jupyter, kernel_list)
logger.warn("could not find {name!r} kernel; using {kernel!r} kernel instead".format(name=icoconut_custom_kernel_name, kernel=kernel))
# pass the kernel to the console or otherwise just launch Jupyter now that we know our kernel is available
if args[0] == "console":
run_args = jupyter + ["console", "--kernel", kernel] + args[1:]
else:
run_args = jupyter + args
if newly_installed_kernels:
logger.show_sig("Successfully installed Jupyter kernels: '" + "', '".join(newly_installed_kernels) + "'")
# run the Jupyter command
if run_args is not None:
self.register_error(run_cmd(run_args, raise_errs=False), errmsg="Jupyter error")
def watch(self, src_dest_package_triples, run=False, force=False):
"""Watch a source and recompile on change."""
from coconut.command.watch import Observer, RecompilationWatcher
for src, _, _ in src_dest_package_triples:
logger.show()
logger.show_tabulated("Watching", showpath(src), "(press Ctrl-C to end)...")
def recompile(path, src, dest, package):
path = fixpath(path)
if os.path.isfile(path) and os.path.splitext(path)[1] in code_exts:
with self.handling_exceptions():
if dest is True or dest is None:
writedir = dest
else:
# correct the compilation path based on the relative position of path to src
dirpath = os.path.dirname(path)
writedir = os.path.join(dest, os.path.relpath(dirpath, src))
filepaths = self.compile_path(path, writedir, package, run=run, force=force, show_unchanged=False)
self.run_mypy(filepaths)
observer = Observer()
watchers = []
for src, dest, package in src_dest_package_triples:
watcher = RecompilationWatcher(recompile, src, dest, package)
observer.schedule(watcher, src, recursive=True)
watchers.append(watcher)
with self.running_jobs():
observer.start()
try:
while True:
time.sleep(watch_interval)
for wcher in watchers:
wcher.keep_watching()
except KeyboardInterrupt:
logger.show_sig("Got KeyboardInterrupt; stopping watcher.")
finally:
observer.stop()
observer.join()
def get_python_lib(self):
"""Get current Python lib location."""
from distutils import sysconfig # expensive, so should only be imported here
return fixpath(sysconfig.get_python_lib())
def site_install(self):
"""Add Coconut's pth file to site-packages."""
python_lib = self.get_python_lib()
shutil.copy(coconut_pth_file, python_lib)
logger.show_sig("Added %s to %s." % (os.path.basename(coconut_pth_file), python_lib))
def site_uninstall(self):
"""Remove Coconut's pth file from site-packages."""
python_lib = self.get_python_lib()
pth_file = os.path.join(python_lib, os.path.basename(coconut_pth_file))
if os.path.isfile(pth_file):
os.remove(pth_file)
logger.show_sig("Removed %s from %s." % (os.path.basename(coconut_pth_file), python_lib))
else:
raise CoconutException("failed to find %s file to remove" % (os.path.basename(coconut_pth_file),))