def setup(self, *args, **kwargs): """Sets parameters for the compiler.""" if self.comp is None: self.comp = Compiler(*args, **kwargs) else: self.comp.setup(*args, **kwargs)
class Command(object): """The Coconut command-line interface.""" comp = None # current coconut.compiler.Compiler show = False # corresponds to --display flag runner = None # the current Runner jobs = 0 # corresponds to --jobs flag executor = None # runs --jobs exit_code = 0 # exit status to return errmsg = None # error message to display mypy_args = None # corresponds to --mypy flag def __init__(self): """Creates the CLI.""" self.prompt = Prompt() def start(self, run=False): """Processes command-line arguments.""" if run: args, argv = ["--run", "--quiet", "--target", "sys"], [] for i in range(1, len(sys.argv)): arg = sys.argv[i] if arg.startswith("-"): args.append(arg) # coconut option else: args.append(arg) # source file argv = sys.argv[i:] break sys.argv = argv else: args = None self.cmd(args) def cmd(self, args=None, interact=True): """Processes command-line arguments.""" if args is None: args = arguments.parse_args() else: args = arguments.parse_args(args) self.exit_code = 0 with self.handling_exceptions(): self.use_args(args, interact) self.exit_on_error() def setup(self, *args, **kwargs): """Sets parameters for the compiler.""" if self.comp is None: self.comp = Compiler(*args, **kwargs) else: self.comp.setup(*args, **kwargs) def exit_on_error(self): """Exits if exit_code is abnormal.""" if self.exit_code: if self.errmsg is not None: logger.show_error("Exiting due to " + self.errmsg + ".") self.errmsg = None if self.jobs != 0: kill_children() sys.exit(self.exit_code) def set_recursion_limit(self, limit): """Sets the Python recursion limit.""" if limit < minimum_recursion_limit: raise CoconutException("--recursion-limit must be at least " + str(minimum_recursion_limit)) else: sys.setrecursionlimit(limit) def use_args(self, args, interact=True): """Handles command-line arguments.""" logger.quiet, logger.verbose = args.quiet, args.verbose if DEVELOP: logger.tracing = args.trace if args.recursion_limit is not None: self.set_recursion_limit(args.recursion_limit) if args.jobs is not None: self.set_jobs(args.jobs) if args.display: self.show = True if args.style is not None: self.prompt.set_style(args.style) if args.documentation: launch_documentation() if args.tutorial: launch_tutorial() self.setup( target=args.target, strict=args.strict, minify=args.minify, line_numbers=args.line_numbers, keep_lines=args.keep_lines, ) if args.mypy is not None: self.set_mypy_args(args.mypy) if args.source is not None: if args.interact and args.run: logger.warn("extraneous --run argument passed; --interact implies --run") if args.package and self.mypy: logger.warn("extraneous --package argument passed; --mypy implies --package") if args.standalone and args.package: raise CoconutException("cannot compile as both --package and --standalone") if args.standalone and self.mypy: raise CoconutException("cannot compile as both --package (implied by --mypy) and --standalone") if (args.run or args.interact) and os.path.isdir(args.source): if args.run: raise CoconutException("source path must point to file not directory when --run is enabled") if args.interact: raise CoconutException("source path must point to file not directory when --run (implied by --interact) is enabled") if args.watch and os.path.isfile(args.source): raise CoconutException("source path must point to directory not file when --watch is enabled") if args.dest is None: if args.nowrite: dest = None # no dest else: dest = True # auto-generate dest elif args.nowrite: raise CoconutException("destination path cannot be given when --nowrite is enabled") elif os.path.isfile(args.dest): raise CoconutException("destination path must point to directory not file") else: dest = args.dest if args.package or self.mypy: package = True elif args.standalone: package = False else: package = None # auto-decide package with self.running_jobs(): filepaths = self.compile_path(args.source, dest, package, args.run or args.interact, args.force) self.run_mypy(filepaths) elif (args.run or args.nowrite or args.force or args.package or args.standalone or args.watch): raise CoconutException("a source file/folder must be specified when options that depend on the source are enabled") if args.code is not None: self.execute(self.comp.parse_block(args.code)) stdin = not sys.stdin.isatty() # check if input was piped in if stdin: self.execute(self.comp.parse_block(sys.stdin.read())) if args.jupyter is not None: self.start_jupyter(args.jupyter) if args.interact or (interact and not ( stdin or args.source or args.code or args.tutorial or args.documentation or args.watch or args.jupyter is not None )): self.start_prompt() if args.watch: self.watch(args.source, dest, package, args.run, args.force) def register_error(self, code=1, errmsg=None): """Updates the exit code.""" if errmsg is not None: if self.errmsg is None: self.errmsg = errmsg elif errmsg not in self.errmsg: self.errmsg += ", " + errmsg if code is not None: self.exit_code = max(self.exit_code, code) @contextmanager def handling_exceptions(self): """Performs proper exception handling.""" try: with handling_broken_process_pool(): yield except SystemExit as err: self.register_error(err.code) except BaseException as err: if isinstance(err, CoconutException): logger.print_exc() elif not isinstance(err, KeyboardInterrupt): traceback.print_exc() self.register_error(errmsg=err.__class__.__name__) def compile_path(self, path, write=True, package=None, run=False, force=False): """Compiles a path and returns paths to compiled files.""" path = fixpath(path) if write is not None and write is not True: write = fixpath(write) if os.path.isfile(path): if package is None: package = False destpath = self.compile_file(path, write, package, run, force) if destpath is None: return [] else: return [destpath] elif os.path.isdir(path): if package is None: package = True return self.compile_folder(path, write, package, run, force) else: raise CoconutException("could not find source path", path) def compile_folder(self, directory, write=True, package=True, run=False, force=False): """Compiles a directory and returns paths to compiled files.""" filepaths = [] for dirpath, dirnames, filenames in os.walk(directory): if write is None or write is True: writedir = write else: writedir = os.path.join(write, os.path.relpath(dirpath, directory)) for filename in filenames: if os.path.splitext(filename)[1] in code_exts: destpath = self.compile_file(os.path.join(dirpath, filename), writedir, package, run, force) if destpath is not None: filepaths.append(destpath) for name in dirnames[:]: if not is_special_dir(name) and name.startswith("."): if logger.verbose: logger.show_tabulated("Skipped directory", name, "(explicitly pass as source to override).") dirnames.remove(name) # directories removed from dirnames won't appear in further os.walk iteration return filepaths def compile_file(self, filepath, write=True, package=False, run=False, force=False): """Compiles a file and returns the compiled file's path.""" if write is None: destpath = None elif write is True: destpath = filepath else: destpath = os.path.join(write, os.path.basename(filepath)) if destpath is not None: base, ext = os.path.splitext(os.path.splitext(destpath)[0]) if not ext: ext = comp_ext destpath = fixpath(base + ext) if filepath == destpath: raise CoconutException("cannot compile " + showpath(filepath) + " to itself", extra="incorrect file extension") self.compile(filepath, destpath, package, run, force) return destpath def compile(self, codepath, destpath=None, package=False, run=False, force=False): """Compiles a source Coconut file to a destination Python file.""" logger.show_tabulated("Compiling", showpath(codepath), "...") with openfile(codepath, "r") as opened: code = readfile(opened) if destpath is not None: destdir = os.path.dirname(destpath) if not os.path.exists(destdir): os.makedirs(destdir) if package is True: self.create_package(destdir) foundhash = None if force else self.hashashof(destpath, code, package) if foundhash: logger.show_tabulated("Left unchanged", showpath(destpath), "(pass --force to override).") if self.show: print(foundhash) if run: self.execute_file(destpath) else: if package is True: compile_method = "parse_package" elif package is False: compile_method = "parse_file" else: raise CoconutInternalException("invalid value for package", package) def callback(compiled): if destpath is None: logger.show_tabulated("Finished", showpath(codepath), "without writing to file.") else: with openfile(destpath, "w") as opened: writefile(opened, compiled) logger.show_tabulated("Compiled to", showpath(destpath), ".") if self.show: print(compiled) if run: if destpath is None: self.execute(compiled, path=codepath, allow_show=False) else: self.execute_file(destpath) self.submit_comp_job(codepath, callback, compile_method, code) def submit_comp_job(self, path, callback, method, *args, **kwargs): """Submits a job on self.comp to be run in parallel.""" if self.executor is None: with self.handling_exceptions(): callback(getattr(self.comp, method)(*args, **kwargs)) else: path = showpath(path) with logger.in_path(path): # pickle the compiler in the path context future = self.executor.submit(multiprocess_wrapper(self.comp, method), *args, **kwargs) def callback_wrapper(completed_future): """Ensures that all errors are always caught, since errors raised in a callback won't be propagated.""" with logger.in_path(path): # handle errors in the path context with self.handling_exceptions(): result = completed_future.result() callback(result) future.add_done_callback(callback_wrapper) def set_jobs(self, jobs): """Sets --jobs.""" if jobs == "sys": self.jobs = None else: try: jobs = int(jobs) except ValueError: jobs = -1 if jobs < 0: raise CoconutException("--jobs must be an integer >= 0 or 'sys'") else: self.jobs = jobs @contextmanager def running_jobs(self): """Initialize multiprocessing.""" with self.handling_exceptions(): if self.jobs == 0: yield else: from concurrent.futures import ProcessPoolExecutor try: with ProcessPoolExecutor(self.jobs) as self.executor: yield finally: self.executor = None self.exit_on_error() def create_package(self, dirpath): """Sets up a package directory.""" dirpath = fixpath(dirpath) filepath = os.path.join(dirpath, "__coconut__.py") with openfile(filepath, "w") as opened: writefile(opened, self.comp.getheader("__coconut__")) def hashashof(self, destpath, code, package): """Determines if a file has the hash of the code.""" if destpath is not None and os.path.isfile(destpath): with openfile(destpath, "r") as opened: compiled = readfile(opened) hashash = gethash(compiled) if hashash is not None and hashash == self.comp.genhash(package, code): return compiled return None def get_input(self, more=False): """Prompts for code input.""" try: return self.prompt.input(more) except KeyboardInterrupt: printerr("\nKeyboardInterrupt") except EOFError: print() self.exit_runner() return None def start_running(self): """Starts running the Runner.""" self.comp.bind() self.check_runner() self.running = True def start_prompt(self): """Starts the interpreter.""" print("Coconut Interpreter:") print('(type "exit()" or press Ctrl-D to end)') self.start_running() while self.running: code = self.get_input() if code: compiled = self.handle_input(code) if compiled: self.execute(compiled, use_eval=None) def exit_runner(self, exit_code=0): """Exits the interpreter.""" self.register_error(exit_code) self.running = False def handle_input(self, code): """Compiles Coconut interpreter input.""" if not self.prompt.multiline: if not should_indent(code): try: return self.comp.parse_block(code) except CoconutException: pass while True: line = self.get_input(more=True) if line is None: return None elif line: code += "\n" + line else: break try: return self.comp.parse_block(code) except CoconutException: logger.print_exc() return None def execute(self, compiled=None, path=None, use_eval=False, allow_show=True): """Executes compiled code.""" self.check_runner() if compiled is not None: if allow_show and self.show: print(compiled) if path is not None: # path means header is included, and thus encoding must be removed compiled = rem_encoding(compiled) self.runner.run(compiled, use_eval=use_eval, path=path, all_errors_exit=(path is not None)) self.run_mypy(code=self.runner.was_run_code()) def execute_file(self, destpath): """Executes compiled file.""" self.check_runner() self.runner.run_file(destpath) def check_runner(self): """Makes sure there is a runner.""" if os.getcwd() not in sys.path: sys.path.append(os.getcwd()) if self.runner is None: self.runner = Runner(self.comp, exit=self.exit_runner, store=self.mypy) @property def mypy(self): """Whether using MyPy or not.""" return self.mypy_args is not None def set_mypy_args(self, mypy_args=None): """Sets MyPy arguments.""" if mypy_args is None: self.mypy_args = None else: self.mypy_errs = [] self.mypy_args = list(mypy_args) for arg in self.mypy_args: if arg == "--fast-parser": logger.warn("unnecessary --mypy argument", arg, extra="passed automatically if available") elif arg == "--py2" or arg == "-2": logger.warn("unnecessary --mypy argument", arg, extra="passed automatically when needed") elif arg == "--python-version": logger.warn("unnecessary --mypy argument", arg, extra="current --target passed as version automatically") if "--fast-parser" not in self.mypy_args: try: import typed_ast # NOQA except ImportError: if self.comp.target != "3": logger.warn("missing typed_ast; MyPy may not properly analyze type annotations", extra="run 'pip install typed_ast' or pass '--target 3' to fix") else: self.mypy_args.append("--fast-parser") if not ("--py2" in self.mypy_args or "-2" in self.mypy_args) and not self.comp.target.startswith("3"): self.mypy_args.append("--py2") if "--python-version" not in self.mypy_args: self.mypy_args += ["--python-version", ".".join(str(v) for v in self.comp.target_info_len2)] def run_mypy(self, paths=[], code=None): """Run MyPy with arguments.""" set_mypy_path(stub_dir) if self.mypy: from coconut.command.mypy import mypy_run args = paths + self.mypy_args if code is not None: args += ["-c", code] for line, is_err in mypy_run(args): if code is None or line not in self.mypy_errs: if is_err: printerr(line) else: print(line) if line not in self.mypy_errs: self.mypy_errs.append(line) def start_jupyter(self, args): """Starts Jupyter with the Coconut kernel.""" install_func = functools.partial(run_cmd, show_output=logger.verbose or not args) try: install_func(["jupyter", "--version"]) except CalledProcessError: jupyter = "ipython" else: jupyter = "jupyter" for icoconut_kernel_dir in icoconut_kernel_dirs: install_args = [jupyter, "kernelspec", "install", icoconut_kernel_dir, "--replace"] try: install_func(install_args) except CalledProcessError: user_install_args = install_args + ["--user"] try: install_func(user_install_args) except CalledProcessError: logger.warn("kernel install failed on command'", " ".join(install_args)) self.register_error(errmsg="Jupyter error") if args: if args[0] == "console": ver = "2" if PY2 else "3" try: install_func(["python" + ver, "-m", "coconut.main", "--version"]) except CalledProcessError: kernel_name = "coconut" else: kernel_name = "coconut" + ver run_args = [jupyter, "console", "--kernel", kernel_name] + args[1:] elif args[0] == "notebook": run_args = [jupyter, "notebook"] + args[1:] else: raise CoconutException('first argument after --jupyter must be either "console" or "notebook"') self.register_error(run_cmd(run_args, raise_errs=False), errmsg="Jupyter error") def watch(self, source, write=True, package=None, run=False, force=False): """Watches a source and recompiles on change.""" from coconut.command.watch import Observer, RecompilationWatcher source = fixpath(source) print() logger.show_tabulated("Watching", showpath(source), "(press Ctrl-C to end)...") def recompile(path): if os.path.isfile(path) and os.path.splitext(path)[1] in code_exts: with self.handling_exceptions(): self.run_mypy(self.compile_path(path, write, package, run, force)) observer = Observer() observer.schedule(RecompilationWatcher(recompile), source, recursive=True) with self.running_jobs(): observer.start() try: while True: time.sleep(watch_interval) except KeyboardInterrupt: logger.show("Got KeyboardInterrupt; stopping watcher.") finally: observer.stop() observer.join()
class Command(object): """Coconut command-line interface.""" comp = None # current coconut.compiler.Compiler show = False # corresponds to --display flag runner = None # the current Runner jobs = 0 # corresponds to --jobs flag executor = None # runs --jobs exit_code = 0 # exit status to return errmsg = None # error message to display mypy_args = None # corresponds to --mypy flag def __init__(self): """Create the CLI.""" self.prompt = Prompt() def start(self, run=False): """Process command-line arguments.""" if run: args, argv = list(coconut_run_args), [] # for coconut-run, all args beyond the source file should be wrapped in an --argv for i in range(1, len(sys.argv)): arg = sys.argv[i] args.append(arg) if not arg.startswith("-"): # was source file argv = sys.argv[i + 1:] break args += ["--argv"] + argv else: args = None self.cmd(args) def cmd(self, args=None, interact=True): """Process command-line arguments.""" if args is None: parsed_args = arguments.parse_args() else: parsed_args = arguments.parse_args(args) self.exit_code = 0 with self.handling_exceptions(): self.use_args(parsed_args, interact, original_args=args) self.exit_on_error() def setup(self, *args, **kwargs): """Set parameters for the compiler.""" if self.comp is None: self.comp = Compiler(*args, **kwargs) else: self.comp.setup(*args, **kwargs) def exit_on_error(self): """Exit if exit_code is abnormal.""" if self.exit_code: if self.errmsg is not None: logger.show_error("Exiting due to " + self.errmsg + ".") self.errmsg = None if self.jobs != 0: kill_children() sys.exit(self.exit_code) def use_args(self, args, interact=True, original_args=None): """Handle command-line arguments.""" logger.quiet, logger.verbose = args.quiet, args.verbose if DEVELOP: logger.tracing = args.trace logger.log("Using " + PYPARSING + ".") if original_args is not None: logger.log("Directly passed args:", original_args) logger.log("Parsed args:", args) if args.recursion_limit is not None: set_recursion_limit(args.recursion_limit) if args.jobs is not None: self.set_jobs(args.jobs) if args.display: self.show = True if args.style is not None: self.prompt.set_style(args.style) if args.documentation: launch_documentation() if args.tutorial: launch_tutorial() self.setup( target=args.target, strict=args.strict, minify=args.minify, line_numbers=args.line_numbers, keep_lines=args.keep_lines, no_tco=args.no_tco or args.mypy is not None, ) if args.mypy is not None: if args.no_tco: logger.warn( "extraneous --no-tco argument passed; --mypy implies --no-tco" ) self.set_mypy_args(args.mypy) if args.argv is not None: sys.argv = [args.source if args.source is not None else ""] sys.argv.extend(args.argv) if args.source is not None: if args.interact and args.run: logger.warn( "extraneous --run argument passed; --interact implies --run" ) if args.package and self.mypy: logger.warn( "extraneous --package argument passed; --mypy implies --package" ) if args.standalone and args.package: raise CoconutException( "cannot compile as both --package and --standalone") if args.standalone and self.mypy: raise CoconutException( "cannot compile as both --package (implied by --mypy) and --standalone" ) if args.no_write and self.mypy: raise CoconutException( "cannot compile with --no-write when using --mypy") if (args.run or args.interact) and os.path.isdir(args.source): if args.run: raise CoconutException( "source path must point to file not directory when --run is enabled" ) if args.interact: raise CoconutException( "source path must point to file not directory when --run (implied by --interact) is enabled" ) if args.watch and os.path.isfile(args.source): raise CoconutException( "source path must point to directory not file when --watch is enabled" ) if args.dest is None: if args.no_write: dest = None # no dest else: dest = True # auto-generate dest elif args.no_write: raise CoconutException( "destination path cannot be given when --no-write is enabled" ) elif os.path.isfile(args.dest): raise CoconutException( "destination path must point to directory not file") else: dest = args.dest if args.package or self.mypy: package = True elif args.standalone: package = False else: package = None # auto-decide package with self.running_jobs(exit_on_error=not args.watch): filepaths = self.compile_path(args.source, dest, package, args.run or args.interact, args.force) self.run_mypy(filepaths) elif (args.run or args.no_write or args.force or args.package or args.standalone or args.watch): raise CoconutException( "a source file/folder must be specified when options that depend on the source are enabled" ) if args.code is not None: self.execute(self.comp.parse_block(args.code)) got_stdin = False if stdin_readable(): logger.log("Reading piped input from stdin...") self.execute(self.comp.parse_block(sys.stdin.read())) got_stdin = True if args.jupyter is not None: self.start_jupyter(args.jupyter) if args.interact or (interact and not (got_stdin or args.source or args.code or args.tutorial or args.documentation or args.watch or args.jupyter is not None)): self.start_prompt() if args.watch: self.watch(args.source, dest, package, args.run, args.force) def register_error(self, code=1, errmsg=None): """Update the exit code.""" if errmsg is not None: if self.errmsg is None: self.errmsg = errmsg elif errmsg not in self.errmsg: self.errmsg += ", " + errmsg if code is not None: self.exit_code = max(self.exit_code, code) @contextmanager def handling_exceptions(self): """Perform proper exception handling.""" try: with handling_broken_process_pool(): yield except SystemExit as err: self.register_error(err.code) except BaseException as err: if isinstance(err, CoconutException): logger.display_exc() elif not isinstance(err, KeyboardInterrupt): traceback.print_exc() printerr(report_this_text) self.register_error(errmsg=err.__class__.__name__) def compile_path(self, path, write=True, package=None, *args, **kwargs): """Compile a path and returns paths to compiled files.""" path = fixpath(path) if write is not None and write is not True: write = fixpath(write) if os.path.isfile(path): if package is None: package = False destpath = self.compile_file(path, write, package, *args, **kwargs) return [destpath] if destpath is not None else [] elif os.path.isdir(path): if package is None: package = True return self.compile_folder(path, write, package, *args, **kwargs) else: raise CoconutException("could not find source path", path) def compile_folder(self, directory, write=True, package=True, *args, **kwargs): """Compile a directory and returns paths to compiled files.""" filepaths = [] for dirpath, dirnames, filenames in os.walk(directory): if write is None or write is True: writedir = write else: writedir = os.path.join(write, os.path.relpath(dirpath, directory)) for filename in filenames: if os.path.splitext(filename)[1] in code_exts: destpath = self.compile_file( os.path.join(dirpath, filename), writedir, package, *args, **kwargs) if destpath is not None: filepaths.append(destpath) for name in dirnames[:]: if not is_special_dir(name) and name.startswith("."): if logger.verbose: logger.show_tabulated( "Skipped directory", name, "(explicitly pass as source to override).") dirnames.remove( name ) # directories removed from dirnames won't appear in further os.walk iterations return filepaths def compile_file(self, filepath, write=True, package=False, *args, **kwargs): """Compile a file and returns the compiled file's path.""" if write is None: destpath = None elif write is True: destpath = filepath else: # add the name of file to the destination path destpath = os.path.join(write, os.path.basename(filepath)) if destpath is not None: base, ext = os.path.splitext(os.path.splitext(destpath)[0]) if not ext: ext = comp_ext destpath = fixpath(base + ext) if filepath == destpath: raise CoconutException("cannot compile " + showpath(filepath) + " to itself", extra="incorrect file extension") self.compile(filepath, destpath, package, *args, **kwargs) return destpath def compile(self, codepath, destpath=None, package=False, run=False, force=False, show_unchanged=True): """Compile a source Coconut file to a destination Python file.""" with openfile(codepath, "r") as opened: code = readfile(opened) if destpath is not None: destdir = os.path.dirname(destpath) if not os.path.exists(destdir): os.makedirs(destdir) if package is True: self.create_package(destdir) foundhash = None if force else self.has_hash_of( destpath, code, package) if foundhash: if show_unchanged: logger.show_tabulated("Left unchanged", showpath(destpath), "(pass --force to override).") if self.show: print(foundhash) if run: self.execute_file(destpath) else: logger.show_tabulated("Compiling", showpath(codepath), "...") if package is True: compile_method = "parse_package" elif package is False: compile_method = "parse_file" else: raise CoconutInternalException("invalid value for package", package) def callback(compiled): if destpath is None: logger.show_tabulated("Compiled", showpath(codepath), "without writing to file.") else: with openfile(destpath, "w") as opened: writefile(opened, compiled) logger.show_tabulated("Compiled to", showpath(destpath), ".") if self.show: print(compiled) if run: if destpath is None: self.execute(compiled, path=codepath, allow_show=False) else: self.execute_file(destpath) self.submit_comp_job(codepath, callback, compile_method, code) def submit_comp_job(self, path, callback, method, *args, **kwargs): """Submits a job on self.comp to be run in parallel.""" if self.executor is None: with self.handling_exceptions(): callback(getattr(self.comp, method)(*args, **kwargs)) else: path = showpath(path) with logger.in_path( path): # pickle the compiler in the path context future = self.executor.submit( multiprocess_wrapper(self.comp, method), *args, **kwargs) def callback_wrapper(completed_future): """Ensures that all errors are always caught, since errors raised in a callback won't be propagated.""" with logger.in_path(path): # handle errors in the path context with self.handling_exceptions(): result = completed_future.result() callback(result) future.add_done_callback(callback_wrapper) def set_jobs(self, jobs): """Set --jobs.""" if jobs == "sys": self.jobs = None else: try: jobs = int(jobs) except ValueError: jobs = -1 # will raise error below if jobs < 0: raise CoconutException( "--jobs must be an integer >= 0 or 'sys'") self.jobs = jobs @contextmanager def running_jobs(self, exit_on_error=True): """Initialize multiprocessing.""" with self.handling_exceptions(): if self.jobs == 0: yield else: from concurrent.futures import ProcessPoolExecutor try: with ProcessPoolExecutor(self.jobs) as self.executor: yield finally: self.executor = None if exit_on_error: self.exit_on_error() def create_package(self, dirpath): """Set up a package directory.""" dirpath = fixpath(dirpath) filepath = os.path.join(dirpath, "__coconut__.py") with openfile(filepath, "w") as opened: writefile(opened, self.comp.getheader("__coconut__")) def has_hash_of(self, destpath, code, package): """Determine if a file has the hash of the code.""" if destpath is not None and os.path.isfile(destpath): with openfile(destpath, "r") as opened: compiled = readfile(opened) hashash = gethash(compiled) if hashash is not None and hashash == self.comp.genhash( package, code): return compiled return None def get_input(self, more=False): """Prompt for code input.""" received = None try: received = self.prompt.input(more) except KeyboardInterrupt: print() printerr("KeyboardInterrupt") except EOFError: print() self.exit_runner() else: if received.startswith(exit_chars): self.exit_runner() received = None return received def start_running(self): """Start running the Runner.""" self.comp.warm_up() self.check_runner() self.running = True def start_prompt(self): """Start the interpreter.""" logger.show("Coconut Interpreter:") logger.show("(type 'exit()' or press Ctrl-D to end)") self.start_running() while self.running: try: code = self.get_input() if code: compiled = self.handle_input(code) if compiled: self.execute(compiled, use_eval=None) except KeyboardInterrupt: printerr("\nKeyboardInterrupt") def exit_runner(self, exit_code=0): """Exit the interpreter.""" self.register_error(exit_code) self.running = False def handle_input(self, code): """Compile Coconut interpreter input.""" if not self.prompt.multiline: if not should_indent(code): try: return self.comp.parse_block(code) except CoconutException: pass while True: line = self.get_input(more=True) if line is None: return None elif line: code += "\n" + line else: break try: return self.comp.parse_block(code) except CoconutException: logger.display_exc() return None def execute(self, compiled=None, path=None, use_eval=False, allow_show=True): """Execute compiled code.""" self.check_runner() if compiled is not None: if allow_show and self.show: print(compiled) if path is not None: # path means header is included, and thus encoding must be removed compiled = rem_encoding(compiled) self.runner.run(compiled, use_eval=use_eval, path=path, all_errors_exit=(path is not None)) self.run_mypy(code=self.runner.was_run_code()) def execute_file(self, destpath): """Execute compiled file.""" self.check_runner() self.runner.run_file(destpath) def check_runner(self): """Make sure there is a runner.""" if os.getcwd() not in sys.path: sys.path.append(os.getcwd()) if self.runner is None: self.runner = Runner(self.comp, exit=self.exit_runner, store=self.mypy) @property def mypy(self): """Whether using MyPy or not.""" return self.mypy_args is not None def set_mypy_args(self, mypy_args=None): """Set MyPy arguments.""" if mypy_args is None: self.mypy_args = None else: self.mypy_errs = [] self.mypy_args = list(mypy_args) for arg in self.mypy_args: if arg == "--py2" or arg == "-2": logger.warn("unnecessary --mypy argument", arg, extra="passed automatically when needed") elif arg == "--python-version": logger.warn( "unnecessary --mypy argument", arg, extra="current --target passed as version automatically" ) if not ("--py2" in self.mypy_args or "-2" in self.mypy_args ) and not self.comp.target.startswith("3"): self.mypy_args.append("--py2") if "--python-version" not in self.mypy_args: self.mypy_args += [ "--python-version", ".".join(str(v) for v in self.comp.target_info_len2) ] if logger.verbose: for arg in verbose_mypy_args: if arg not in self.mypy_args: self.mypy_args.append(arg) logger.log("MyPy args:", self.mypy_args) def run_mypy(self, paths=(), code=None): """Run MyPy with arguments.""" if self.mypy: set_mypy_path(stub_dir) from coconut.command.mypy import mypy_run args = list(paths) + self.mypy_args if code is not None: args += ["-c", code] for line, is_err in mypy_run(args): if code is None or line not in self.mypy_errs: if is_err: printerr(line) else: print(line) if line not in self.mypy_errs: self.mypy_errs.append(line) def start_jupyter(self, args): """Start Jupyter with the Coconut kernel.""" install_func = functools.partial(run_cmd, show_output=logger.verbose) try: install_func(["jupyter", "--version"]) except CalledProcessError: jupyter = "ipython" else: jupyter = "jupyter" # always install kernels if given no args, otherwise only if there's a kernel missing do_install = not args if not do_install: kernel_list = run_cmd([jupyter, "kernelspec", "list"], show_output=False, raise_errs=False) do_install = any(ker not in kernel_list for ker in icoconut_kernel_names) if do_install: success = True for icoconut_kernel_dir in icoconut_kernel_dirs: install_args = [ jupyter, "kernelspec", "install", icoconut_kernel_dir, "--replace" ] try: install_func(install_args) except CalledProcessError: user_install_args = install_args + ["--user"] try: install_func(user_install_args) except CalledProcessError: logger.warn("kernel install failed on command'", " ".join(install_args)) self.register_error(errmsg="Jupyter error") success = False if success: logger.show_sig( "Successfully installed Coconut Jupyter kernel.") if args: if args[0] == "console": ver = "2" if PY2 else "3" try: install_func( ["python" + ver, "-m", "coconut.main", "--version"]) except CalledProcessError: kernel_name = "coconut" else: kernel_name = "coconut" + ver run_args = [jupyter, "console", "--kernel", kernel_name ] + args[1:] elif args[0] == "notebook": run_args = [jupyter, "notebook"] + args[1:] else: raise CoconutException( "first argument after --jupyter must be either 'console' or 'notebook'" ) self.register_error(run_cmd(run_args, raise_errs=False), errmsg="Jupyter error") def watch(self, source, write=True, package=None, run=False, force=False): """Watch a source and recompiles on change.""" from coconut.command.watch import Observer, RecompilationWatcher source = fixpath(source) logger.show() logger.show_tabulated("Watching", showpath(source), "(press Ctrl-C to end)...") def recompile(path): path = fixpath(path) if os.path.isfile(path) and os.path.splitext(path)[1] in code_exts: with self.handling_exceptions(): if write is True or write is None: writedir = write else: # correct the compilation path based on the relative position of path to source dirpath = os.path.dirname(path) writedir = os.path.join( write, os.path.relpath(dirpath, source)) filepaths = self.compile_path(path, writedir, package, run, force, show_unchanged=False) self.run_mypy(filepaths) watcher = RecompilationWatcher(recompile) observer = Observer() observer.schedule(watcher, source, recursive=True) with self.running_jobs(): observer.start() try: while True: time.sleep(watch_interval) watcher.keep_watching() except KeyboardInterrupt: logger.show_sig("Got KeyboardInterrupt; stopping watcher.") finally: observer.stop() observer.join()
"; skipping coconut.icoconut loading") else: raise CoconutException( "--jupyter flag requires Jupyter library", extra="run 'pip install coconut[jupyter]' to fix", ) else: LOAD_MODULE = True # ----------------------------------------------------------------------------------------------------------------------- # GLOBALS: # ----------------------------------------------------------------------------------------------------------------------- COMPILER = Compiler( target="sys", line_numbers=True, keep_lines=True, ) RUNNER = Runner(COMPILER) parse_block_memo = {} def memoized_parse_block(code): """Memoized version of parse_block.""" internal_assert(lambda: code not in parse_block_memo.values(), "attempted recompilation of", code) success, result = parse_block_memo.get(code, (None, None)) if success is None: try:
name += c else: right = i break if i == len(code) - 1: right = len(code) if get_bounds: return name, left, right else: return name #----------------------------------------------------------------------------------------------------------------------- # KERNEL: #----------------------------------------------------------------------------------------------------------------------- comp = Compiler(target="sys") # from coconut.compiler class CoconutKernel(Kernel): """Jupyter kernel for Coconut.""" _runner = None # current ..command.Runner implementation = "icoconut" implementation_version = VERSION language = "coconut" language_version = VERSION banner = version_banner language_info = { "name": "coconut", "mimetype": mimetype, "file_extension": code_exts[0], "codemirror_mode": {
def setup(self, *args, **kwargs): """Set parameters for the compiler.""" if self.comp is None: self.comp = Compiler(*args, **kwargs) else: self.comp.setup(*args, **kwargs)
py_syntax_version, mimetype, version_banner, tutorial_url, documentation_url, code_exts, ) from coconut.compiler import Compiler from coconut.compiler.util import should_indent from coconut.command.util import Runner #----------------------------------------------------------------------------------------------------------------------- # GLOBALS: #----------------------------------------------------------------------------------------------------------------------- COMPILER = Compiler(target="sys") RUNNER = Runner(COMPILER) parse_block_memo = {} def memoized_parse_block(code, none_if_not_found=False): """Memoized version of parse_block.""" try: result = parse_block_memo[code] except KeyError: if none_if_not_found: return None try: parsed = COMPILER.parse_block(code) except Exception as err:
class Command(object): """The Coconut command-line interface.""" comp = None # current coconut.compiler.Compiler show = False # corresponds to --display flag running = False # whether the interpreter is currently active runner = None # the current Runner target = None # corresponds to --target flag jobs = 0 # corresponds to --jobs flag executor = None # runs --jobs exit_code = 0 # exit status to return errmsg = None # error message to display def __init__(self): """Creates the CLI.""" self.prompt = Prompt() def start(self): """Processes command-line arguments.""" self.cmd(arguments.parse_args()) def setup(self, *args, **kwargs): """Sets parameters for the compiler.""" if self.comp is None: self.comp = Compiler(*args, **kwargs) else: self.comp.setup(*args, **kwargs) def cmd(self, args, interact=True): """Processes command-line arguments.""" self.exit_code = 0 with self.handling_exceptions(True): self.use_args(args, interact) self.exit_on_error() def exit_on_error(self): """Exits if exit_code is abnormal.""" if self.exit_code: if self.errmsg is not None: logger.show_error("Exiting due to " + self.errmsg + ".") self.errmsg = None if self.jobs != 0: kill_children() sys.exit(self.exit_code) def set_recursion_limit(self, limit): """Sets the Python recursion limit.""" if limit < minimum_recursion_limit: raise CoconutException("--recursion-limit must be at least " + str(minimum_recursion_limit)) else: sys.setrecursionlimit(limit) def use_args(self, args, interact=True): """Handles command-line arguments.""" logger.quiet, logger.verbose = args.quiet, args.verbose if args.recursion_limit is not None: self.set_recursion_limit(args.recursion_limit) if args.jobs is not None: self.set_jobs(args.jobs) if args.tutorial: self.launch_tutorial() if args.documentation: self.launch_documentation() if args.style is not None: self.prompt.set_style(args.style) if args.display: self.show = True self.setup( target=args.target, strict=args.strict, minify=args.minify, line_numbers=args.line_numbers, keep_lines=args.keep_lines, ) if args.source is not None: if args.run and os.path.isdir(args.source): raise CoconutException( "source path must point to file not directory when --run is enabled" ) elif args.watch and os.path.isfile(args.source): raise CoconutException( "source path must point to directory not file when --watch is enabled" ) if args.dest is None: if args.nowrite: dest = None # no dest else: dest = True # auto-generate dest elif args.nowrite: raise CoconutException( "destination path cannot be given when --nowrite is enabled" ) elif os.path.isfile(args.dest): raise CoconutException( "destination path must point to directory not file") else: dest = args.dest if args.package and args.standalone: raise CoconutException( "cannot compile as both --package and --standalone") elif args.package: package = True elif args.standalone: package = False else: package = None # auto-decide package with self.running_jobs(): self.compile_path(args.source, dest, package, args.run, args.force) elif (args.run or args.nowrite or args.force or args.package or args.standalone or args.watch): raise CoconutException( "a source file/folder must be specified when options that depend on the source are enabled" ) if args.code is not None: self.execute(self.comp.parse_block(args.code)) stdin = not sys.stdin.isatty() # check if input was piped in if stdin: self.execute(self.comp.parse_block(sys.stdin.read())) if args.jupyter is not None: self.start_jupyter(args.jupyter) if args.interact or (interact and not (stdin or args.source or args.code or args.tutorial or args.documentation or args.watch or args.jupyter is not None)): self.start_prompt() if args.watch: self.watch(args.source, dest, package, args.run, args.force) def register_error(self, code=1, errmsg=None): """Updates the exit code.""" if errmsg is not None: if self.errmsg is None: self.errmsg = errmsg elif errmsg not in self.errmsg: self.errmsg += ", " + errmsg self.exit_code = max(self.exit_code, code) @contextmanager def handling_exceptions(self, blanket=False, errmsg=None): """Performs proper exception handling.""" if blanket: try: with handle_broken_process_pool(): yield except KeyboardInterrupt: self.register_error(errmsg="KeyboardInterrupt") except SystemExit as err: self.register_error(err.code) except: logger.print_exc() self.register_error(errmsg=errmsg) else: try: yield except CoconutException: logger.print_exc() self.register_error(errmsg=errmsg) def compile_path(self, path, write=True, package=None, run=False, force=False): """Compiles a path.""" path = fixpath(path) if write is not None and write is not True: write = fixpath(write) if os.path.isfile(path): if package is None: package = False self.compile_file(path, write, package, run, force) elif os.path.isdir(path): if package is None: package = True self.compile_folder(path, write, package, run, force) else: raise CoconutException("could not find source path", path) def compile_folder(self, directory, write=True, package=True, run=False, force=False): """Compiles a directory.""" for dirpath, dirnames, filenames in os.walk(directory): if write is None or write is True: writedir = write else: writedir = os.path.join(write, os.path.relpath(dirpath, directory)) for filename in filenames: if os.path.splitext(filename)[1] in code_exts: self.compile_file(os.path.join(dirpath, filename), writedir, package, run, force) for name in dirnames[:]: if name != "." * len(name) and name.startswith("."): if logger.verbose: logger.show_tabulated( "Skipped directory", name, "(explicitly pass as source to override).") dirnames.remove( name ) # directories removed from dirnames won't appear in further os.walk iteration def compile_file(self, filepath, write=True, package=False, run=False, force=False): """Compiles a file.""" if write is None: destpath = None elif write is True: destpath = filepath else: destpath = os.path.join(write, os.path.basename(filepath)) if destpath is not None: base, ext = os.path.splitext(os.path.splitext(destpath)[0]) if not ext: ext = comp_ext destpath = base + ext if filepath == destpath: raise CoconutException("cannot compile " + showpath(filepath) + " to itself (incorrect file extension)") else: self.compile(filepath, destpath, package, run, force) def compile(self, codepath, destpath=None, package=False, run=False, force=False): """Compiles a source Coconut file to a destination Python file.""" logger.show_tabulated("Compiling", showpath(codepath), "...") with openfile(codepath, "r") as opened: code = readfile(opened) if destpath is not None: destdir = os.path.dirname(destpath) if not os.path.exists(destdir): os.makedirs(destdir) if package is True: self.create_package(destdir) foundhash = None if force else self.hashashof(destpath, code, package) if foundhash: logger.show_tabulated("Left unchanged", showpath(destpath), "(pass --force to override).") if run: self.execute(foundhash, path=destpath, isolate=True) elif self.show: print(foundhash) else: if package is True: compile_method = "parse_module" elif package is False: compile_method = "parse_file" else: raise CoconutInternalException("invalid value for package", package) def callback(compiled): if destpath is None: logger.show_tabulated("Finished", showpath(codepath), "without writing to file.") else: with openfile(destpath, "w") as opened: writefile(opened, compiled) logger.show_tabulated("Compiled to", showpath(destpath), ".") if run: runpath = destpath if destpath is not None else codepath self.execute(compiled, path=runpath, isolate=True) elif self.show: print(compiled) self.submit_comp_job(codepath, callback, compile_method, code) def submit_comp_job(self, path, callback, method, *args, **kwargs): """Submits a job on self.comp to be run in parallel.""" if self.executor is None: with self.handling_exceptions(): callback(getattr(self.comp, method)(*args, **kwargs)) else: path = showpath(path) with logger.in_path( path): # pickle the compiler in the path context future = self.executor.submit( multiprocess_wrapper(self.comp, method), *args, **kwargs) def callback_wrapper(completed_future): """Ensures that all errors are always caught, since errors raised in a callback won't be propagated.""" with logger.in_path(path): # handle errors in the path context with self.handling_exceptions(True, "compilation error"): result = completed_future.result() callback(result) future.add_done_callback(callback_wrapper) def set_jobs(self, jobs): """Sets --jobs.""" if jobs == "sys": self.jobs = None else: try: jobs = int(jobs) except ValueError: jobs = -1 if jobs < 0: raise CoconutException( "--jobs must be an integer >= 0 or 'sys'") else: self.jobs = jobs @contextmanager def running_jobs(self): """Initialize multiprocessing.""" if self.jobs == 0: yield else: from concurrent.futures import ProcessPoolExecutor with self.handling_exceptions(True): with ProcessPoolExecutor(self.jobs) as self.executor: with ensure_time_elapsed(): yield self.executor = None self.exit_on_error() def create_package(self, dirpath): """Sets up a package directory.""" filepath = os.path.join(fixpath(dirpath), "__coconut__.py") with openfile(filepath, "w") as opened: writefile(opened, self.comp.headers("package")) def hashashof(self, destpath, code, package): """Determines if a file has the hash of the code.""" if destpath is not None and os.path.isfile(destpath): with openfile(destpath, "r") as opened: compiled = readfile(opened) hashash = gethash(compiled) if hashash is not None and hashash == self.comp.genhash( package, code): return compiled return None def get_input(self, more=False): """Prompts for code input.""" try: return self.prompt.input(more) except KeyboardInterrupt: logger.printerr("\nKeyboardInterrupt") except EOFError: print() self.exit_runner() except ValueError: logger.print_exc() self.exit_runner() return None def start_running(self): """Starts running the Runner.""" self.check_runner() self.running = True def start_prompt(self): """Starts the interpreter.""" logger.print("Coconut Interpreter:") logger.print('(type "exit()" or press Ctrl-D to end)') self.start_running() while self.running: code = self.get_input() if code: compiled = self.handle_input(code) if compiled: self.execute(compiled, error=False, print_expr=True) def exit_runner(self, exit_code=0): """Exits the interpreter.""" self.register_error(exit_code) self.running = False def handle_input(self, code): """Compiles Coconut interpreter input.""" if not self.prompt.multiline: if not self.comp.should_indent(code): try: return self.comp.parse_block(code) except CoconutException: pass while True: line = self.get_input(more=True) if line is None: return None elif line.strip(): code += "\n" + line else: break try: return self.comp.parse_block(code) except CoconutException: logger.print_exc() return None def execute(self, compiled=None, error=True, path=None, isolate=False, print_expr=False): """Executes compiled code.""" self.check_runner(path, isolate) if compiled is not None: if self.show: print(compiled) if isolate: # isolate means header is included, and thus encoding must be removed compiled = rem_encoding(compiled) if print_expr: self.runner.run(compiled, error) else: self.runner.run(compiled, error, run_func=None) def check_runner(self, path=None, isolate=False): """Makes sure there is a runner.""" if isolate or path is not None or self.runner is None: self.start_runner(path, isolate) def start_runner(self, path=None, isolate=False): """Starts the runner.""" sys.path.insert(0, os.getcwd()) if isolate: comp = None else: comp = self.comp self.runner = Runner(comp, self.exit_runner, path) def launch_tutorial(self): """Opens the Coconut tutorial.""" import webbrowser webbrowser.open(tutorial_url, 2) def launch_documentation(self): """Opens the Coconut documentation.""" import webbrowser webbrowser.open(documentation_url, 2) def start_jupyter(self, args): """Starts Jupyter with the Coconut kernel.""" def install_func(cmd): """Runs an installation command.""" logger.log_cmd(cmd) if args and not logger.verbose: subprocess.check_output(cmd, stderr=subprocess.STDOUT) else: subprocess.check_call(cmd) try: install_func(["jupyter", "--version"]) except subprocess.CalledProcessError: jupyter = "ipython" else: jupyter = "jupyter" for icoconut_kernel_dir in icoconut_kernel_dirs: install_args = [ jupyter, "kernelspec", "install", icoconut_kernel_dir, "--replace" ] try: install_func(install_args) except subprocess.CalledProcessError: user_install_args = install_args + ["--user"] try: install_func(user_install_args) except subprocess.CalledProcessError: args = "kernel install failed on command'", " ".join( install_args) self.comp.warn(*args) self.register_error(errmsg="Jupyter error") if args: if args[0] == "console": ver = "2" if PY2 else "3" try: install_func( ["python" + ver, "-m", "coconut.main", "--version"]) except subprocess.CalledProcessError: kernel_name = "coconut" else: kernel_name = "coconut" + ver run_args = [jupyter, "console", "--kernel", kernel_name ] + args[1:] elif args[0] == "notebook": run_args = [jupyter, "notebook"] + args[1:] else: raise CoconutException( 'first argument after --jupyter must be either "console" or "notebook"' ) logger.log_cmd(run_args) self.register_error(subprocess.call(run_args), errmsg="Jupyter error") def watch(self, source, write=True, package=None, run=False, force=False): """Watches a source and recompiles on change.""" from coconut.command.watch import Observer, RecompilationWatcher source = fixpath(source) logger.print() logger.show_tabulated("Watching", showpath(source), "(press Ctrl-C to end)...") def recompile(path): if os.path.isfile(path) and os.path.splitext(path)[1] in code_exts: with self.handling_exceptions(): self.compile_path(path, write, package, run, force) observer = Observer() observer.schedule(RecompilationWatcher(recompile), source, recursive=True) with self.running_jobs(): observer.start() try: while True: time.sleep(watch_interval) except KeyboardInterrupt: logger.show("Got KeyboardInterrupt; stopping watcher.") finally: observer.stop() observer.join()
class Command(object): """Coconut command-line interface.""" comp = None # current coconut.compiler.Compiler show = False # corresponds to --display flag runner = None # the current Runner jobs = 0 # corresponds to --jobs flag executor = None # runs --jobs exit_code = 0 # exit status to return errmsg = None # error message to display mypy_args = None # corresponds to --mypy flag def __init__(self): """Create the CLI.""" self.prompt = Prompt() def start(self, run=False): """Process command-line arguments.""" if run: args, argv = [], [] # for coconut-run, all args beyond the source file should be wrapped in an --argv for i in range(1, len(sys.argv)): arg = sys.argv[i] args.append(arg) # if arg is source file, put everything else in argv if not arg.startswith("-") and canparse(arguments, args[:-1]): argv = sys.argv[i + 1:] break if "--verbose" in args: args = list(coconut_run_verbose_args) + args else: args = list(coconut_run_args) + args args += ["--argv"] + argv else: args = None self.cmd(args) def cmd(self, args=None, interact=True): """Process command-line arguments.""" if args is None: parsed_args = arguments.parse_args() else: parsed_args = arguments.parse_args(args) self.exit_code = 0 with self.handling_exceptions(): self.use_args(parsed_args, interact, original_args=args) self.exit_on_error() def setup(self, *args, **kwargs): """Set parameters for the compiler.""" if self.comp is None: self.comp = Compiler(*args, **kwargs) else: self.comp.setup(*args, **kwargs) def exit_on_error(self): """Exit if exit_code is abnormal.""" if self.exit_code: if self.errmsg is not None: logger.show("Exiting due to " + self.errmsg + ".") self.errmsg = None if self.using_jobs: kill_children() sys.exit(self.exit_code) def use_args(self, args, interact=True, original_args=None): """Handle command-line arguments.""" logger.quiet, logger.verbose = args.quiet, args.verbose if DEVELOP: logger.tracing = args.trace logger.log("Using " + PYPARSING + ".") if original_args is not None: logger.log("Directly passed args:", original_args) logger.log("Parsed args:", args) if args.recursion_limit is not None: set_recursion_limit(args.recursion_limit) if args.jobs is not None: self.set_jobs(args.jobs) if args.display: self.show = True if args.style is not None: self.prompt.set_style(args.style) if args.history_file is not None: self.prompt.set_history_file(args.history_file) if args.documentation: launch_documentation() if args.tutorial: launch_tutorial() self.setup( target=args.target, strict=args.strict, minify=args.minify, line_numbers=args.line_numbers, keep_lines=args.keep_lines, no_tco=args.no_tco, ) if args.mypy is not None: self.set_mypy_args(args.mypy) if args.argv is not None: sys.argv = [args.source if args.source is not None else ""] sys.argv.extend(args.argv) if args.source is not None: if args.interact and args.run: logger.warn("extraneous --run argument passed; --interact implies --run") if args.package and self.mypy: logger.warn("extraneous --package argument passed; --mypy implies --package") if args.standalone and args.package: raise CoconutException("cannot compile as both --package and --standalone") if args.standalone and self.mypy: raise CoconutException("cannot compile as both --package (implied by --mypy) and --standalone") if args.no_write and self.mypy: raise CoconutException("cannot compile with --no-write when using --mypy") if (args.run or args.interact) and os.path.isdir(args.source): if args.run: raise CoconutException("source path must point to file not directory when --run is enabled") if args.interact: raise CoconutException("source path must point to file not directory when --run (implied by --interact) is enabled") if args.watch and os.path.isfile(args.source): raise CoconutException("source path must point to directory not file when --watch is enabled") if args.dest is None: if args.no_write: dest = False # no dest else: dest = True # auto-generate dest elif args.no_write: raise CoconutException("destination path cannot be given when --no-write is enabled") else: dest = args.dest if args.package or self.mypy: package = True elif args.standalone: package = False else: package = None # auto-decide package with self.running_jobs(exit_on_error=not args.watch): filepaths = self.compile_path(args.source, dest, package, args.run or args.interact, args.force) self.run_mypy(filepaths) elif ( args.run or args.no_write or args.force or args.package or args.standalone or args.watch ): raise CoconutException("a source file/folder must be specified when options that depend on the source are enabled") if args.code is not None: self.execute(self.comp.parse_block(args.code)) got_stdin = False if args.jupyter is not None: self.start_jupyter(args.jupyter) elif stdin_readable(): logger.log("Reading piped input from stdin...") self.execute(self.comp.parse_block(sys.stdin.read())) got_stdin = True if args.interact or (interact and not ( got_stdin or args.source or args.code or args.tutorial or args.documentation or args.watch or args.jupyter is not None )): self.start_prompt() if args.watch: self.watch(args.source, dest, package, args.run, args.force) def register_error(self, code=1, errmsg=None): """Update the exit code.""" if errmsg is not None: if self.errmsg is None: self.errmsg = errmsg elif errmsg not in self.errmsg: self.errmsg += ", " + errmsg if code is not None: self.exit_code = max(self.exit_code, code) @contextmanager def handling_exceptions(self): """Perform proper exception handling.""" try: if self.using_jobs: with handling_broken_process_pool(): yield else: yield except SystemExit as err: self.register_error(err.code) except BaseException as err: if isinstance(err, CoconutException): logger.display_exc() elif not isinstance(err, KeyboardInterrupt): traceback.print_exc() printerr(report_this_text) self.register_error(errmsg=err.__class__.__name__) def compile_path(self, path, write=True, package=None, *args, **kwargs): """Compile a path and returns paths to compiled files.""" path = fixpath(path) if not isinstance(write, bool): write = fixpath(write) if os.path.isfile(path): if package is None: package = False destpath = self.compile_file(path, write, package, *args, **kwargs) return [destpath] if destpath is not None else [] elif os.path.isdir(path): if package is None: package = True return self.compile_folder(path, write, package, *args, **kwargs) else: raise CoconutException("could not find source path", path) def compile_folder(self, directory, write=True, package=True, *args, **kwargs): """Compile a directory and returns paths to compiled files.""" if not isinstance(write, bool) and os.path.isfile(write): raise CoconutException("destination path cannot point to a file when compiling a directory") filepaths = [] for dirpath, dirnames, filenames in os.walk(directory): if isinstance(write, bool): writedir = write else: writedir = os.path.join(write, os.path.relpath(dirpath, directory)) for filename in filenames: if os.path.splitext(filename)[1] in code_exts: with self.handling_exceptions(): destpath = self.compile_file(os.path.join(dirpath, filename), writedir, package, *args, **kwargs) if destpath is not None: filepaths.append(destpath) for name in dirnames[:]: if not is_special_dir(name) and name.startswith("."): if logger.verbose: logger.show_tabulated("Skipped directory", name, "(explicitly pass as source to override).") dirnames.remove(name) # directories removed from dirnames won't appear in further os.walk iterations return filepaths def compile_file(self, filepath, write=True, package=False, *args, **kwargs): """Compile a file and returns the compiled file's path.""" set_ext = False if write is False: destpath = None elif write is True: destpath = filepath set_ext = True elif os.path.splitext(write)[1]: # write is a file; it is the destination filepath destpath = write else: # write is a dir; make the destination filepath by adding the filename destpath = os.path.join(write, os.path.basename(filepath)) set_ext = True if set_ext: base, ext = os.path.splitext(os.path.splitext(destpath)[0]) if not ext: ext = comp_ext destpath = fixpath(base + ext) if filepath == destpath: raise CoconutException("cannot compile " + showpath(filepath) + " to itself", extra="incorrect file extension") self.compile(filepath, destpath, package, *args, **kwargs) return destpath def compile(self, codepath, destpath=None, package=False, run=False, force=False, show_unchanged=True): """Compile a source Coconut file to a destination Python file.""" with openfile(codepath, "r") as opened: code = readfile(opened) if destpath is not None: destdir = os.path.dirname(destpath) if not os.path.exists(destdir): os.makedirs(destdir) if package is True: self.create_package(destdir) foundhash = None if force else self.has_hash_of(destpath, code, package) if foundhash: if show_unchanged: logger.show_tabulated("Left unchanged", showpath(destpath), "(pass --force to override).") if self.show: print(foundhash) if run: self.execute_file(destpath) else: logger.show_tabulated("Compiling", showpath(codepath), "...") if package is True: compile_method = "parse_package" elif package is False: compile_method = "parse_file" else: raise CoconutInternalException("invalid value for package", package) def callback(compiled): if destpath is None: logger.show_tabulated("Compiled", showpath(codepath), "without writing to file.") else: with openfile(destpath, "w") as opened: writefile(opened, compiled) logger.show_tabulated("Compiled to", showpath(destpath), ".") if self.show: print(compiled) if run: if destpath is None: self.execute(compiled, path=codepath, allow_show=False) else: self.execute_file(destpath) self.submit_comp_job(codepath, callback, compile_method, code) def submit_comp_job(self, path, callback, method, *args, **kwargs): """Submits a job on self.comp to be run in parallel.""" if self.executor is None: with self.handling_exceptions(): callback(getattr(self.comp, method)(*args, **kwargs)) else: path = showpath(path) with logger.in_path(path): # pickle the compiler in the path context future = self.executor.submit(multiprocess_wrapper(self.comp, method), *args, **kwargs) def callback_wrapper(completed_future): """Ensures that all errors are always caught, since errors raised in a callback won't be propagated.""" with logger.in_path(path): # handle errors in the path context with self.handling_exceptions(): result = completed_future.result() callback(result) future.add_done_callback(callback_wrapper) def set_jobs(self, jobs): """Set --jobs.""" if jobs == "sys": self.jobs = None else: try: jobs = int(jobs) except ValueError: jobs = -1 # will raise error below if jobs < 0: raise CoconutException("--jobs must be an integer >= 0 or 'sys'") self.jobs = jobs @property def using_jobs(self): """Determine whether or not multiprocessing is being used.""" return self.jobs != 0 @contextmanager def running_jobs(self, exit_on_error=True): """Initialize multiprocessing.""" with self.handling_exceptions(): if self.using_jobs: from concurrent.futures import ProcessPoolExecutor try: with ProcessPoolExecutor(self.jobs) as self.executor: yield finally: self.executor = None else: yield if exit_on_error: self.exit_on_error() def create_package(self, dirpath): """Set up a package directory.""" dirpath = fixpath(dirpath) filepath = os.path.join(dirpath, "__coconut__.py") with openfile(filepath, "w") as opened: writefile(opened, self.comp.getheader("__coconut__")) def has_hash_of(self, destpath, code, package): """Determine if a file has the hash of the code.""" if destpath is not None and os.path.isfile(destpath): with openfile(destpath, "r") as opened: compiled = readfile(opened) hashash = gethash(compiled) if hashash is not None and hashash == self.comp.genhash(package, code): return compiled return None def get_input(self, more=False): """Prompt for code input.""" received = None try: received = self.prompt.input(more) except KeyboardInterrupt: print() printerr("KeyboardInterrupt") except EOFError: print() self.exit_runner() else: if received.startswith(exit_chars): self.exit_runner() received = None return received def start_running(self): """Start running the Runner.""" self.comp.warm_up() self.check_runner() self.running = True def start_prompt(self): """Start the interpreter.""" logger.show("Coconut Interpreter:") logger.show("(type 'exit()' or press Ctrl-D to end)") self.start_running() while self.running: try: code = self.get_input() if code: compiled = self.handle_input(code) if compiled: self.execute(compiled, use_eval=None) except KeyboardInterrupt: printerr("\nKeyboardInterrupt") def exit_runner(self, exit_code=0): """Exit the interpreter.""" self.register_error(exit_code) self.running = False def handle_input(self, code): """Compile Coconut interpreter input.""" if not self.prompt.multiline: if not should_indent(code): try: return self.comp.parse_block(code) except CoconutException: pass while True: line = self.get_input(more=True) if line is None: return None elif line: code += "\n" + line else: break try: return self.comp.parse_block(code) except CoconutException: logger.display_exc() return None def execute(self, compiled=None, path=None, use_eval=False, allow_show=True): """Execute compiled code.""" self.check_runner() if compiled is not None: if allow_show and self.show: print(compiled) if path is not None: # path means header is included, and thus encoding must be removed compiled = rem_encoding(compiled) self.runner.run(compiled, use_eval=use_eval, path=path, all_errors_exit=(path is not None)) self.run_mypy(code=self.runner.was_run_code()) def execute_file(self, destpath): """Execute compiled file.""" self.check_runner() self.runner.run_file(destpath) def check_runner(self): """Make sure there is a runner.""" if os.getcwd() not in sys.path: sys.path.append(os.getcwd()) if self.runner is None: self.runner = Runner(self.comp, exit=self.exit_runner, store=self.mypy) @property def mypy(self): """Whether using MyPy or not.""" return self.mypy_args is not None def set_mypy_args(self, mypy_args=None): """Set MyPy arguments.""" if mypy_args is None: self.mypy_args = None else: self.mypy_errs = [] self.mypy_args = list(mypy_args) if not any(arg.startswith("--python-version") for arg in mypy_args): self.mypy_args += [ "--python-version", ".".join(str(v) for v in get_target_info_len2(self.comp.target, mode="nearest")), ] if logger.verbose: for arg in verbose_mypy_args: if arg not in self.mypy_args: self.mypy_args.append(arg) logger.log("MyPy args:", self.mypy_args) def run_mypy(self, paths=(), code=None): """Run MyPy with arguments.""" if self.mypy: set_mypy_path(stub_dir) from coconut.command.mypy import mypy_run args = list(paths) + self.mypy_args if code is not None: args += ["-c", code] for line, is_err in mypy_run(args): if code is None or line not in self.mypy_errs: printerr(line) if line not in self.mypy_errs: self.mypy_errs.append(line) self.register_error(errmsg="MyPy error") def start_jupyter(self, args): """Start Jupyter with the Coconut kernel.""" install_func = partial(run_cmd, show_output=logger.verbose) try: install_func(["jupyter", "--version"]) except CalledProcessError: jupyter = "ipython" else: jupyter = "jupyter" # always install kernels if given no args, otherwise only if there's a kernel missing do_install = not args if not do_install: kernel_list = run_cmd([jupyter, "kernelspec", "list"], show_output=False, raise_errs=False) do_install = any(ker not in kernel_list for ker in icoconut_kernel_names) if do_install: success = True for icoconut_kernel_dir in icoconut_kernel_dirs: install_args = [jupyter, "kernelspec", "install", icoconut_kernel_dir, "--replace"] try: install_func(install_args) except CalledProcessError: user_install_args = install_args + ["--user"] try: install_func(user_install_args) except CalledProcessError: logger.warn("kernel install failed on command'", " ".join(install_args)) self.register_error(errmsg="Jupyter error") success = False if success: logger.show_sig("Successfully installed Coconut Jupyter kernel.") if args: if args[0] == "console": ver = "2" if PY2 else "3" try: install_func(["python" + ver, "-m", "coconut.main", "--version"]) except CalledProcessError: kernel_name = "coconut" else: kernel_name = "coconut" + ver run_args = [jupyter, "console", "--kernel", kernel_name] + args[1:] else: run_args = [jupyter] + args self.register_error(run_cmd(run_args, raise_errs=False), errmsg="Jupyter error") def watch(self, source, write=True, package=None, run=False, force=False): """Watch a source and recompiles on change.""" from coconut.command.watch import Observer, RecompilationWatcher source = fixpath(source) logger.show() logger.show_tabulated("Watching", showpath(source), "(press Ctrl-C to end)...") def recompile(path): path = fixpath(path) if os.path.isfile(path) and os.path.splitext(path)[1] in code_exts: with self.handling_exceptions(): if write is True or write is None: writedir = write else: # correct the compilation path based on the relative position of path to source dirpath = os.path.dirname(path) writedir = os.path.join(write, os.path.relpath(dirpath, source)) filepaths = self.compile_path(path, writedir, package, run, force, show_unchanged=False) self.run_mypy(filepaths) watcher = RecompilationWatcher(recompile) observer = Observer() observer.schedule(watcher, source, recursive=True) with self.running_jobs(): observer.start() try: while True: time.sleep(watch_interval) watcher.keep_watching() except KeyboardInterrupt: logger.show_sig("Got KeyboardInterrupt; stopping watcher.") finally: observer.stop() observer.join()
def __init__(self, pyfun): self.pyfun = pyfun self.c = Compiler() self.c.compile(pyfun.__code__) self.ircfg = self.c.ircfg
def compile_coconut(cls, source): """Compile the given Coconut source text.""" if cls.coconut_compiler is None: cls.coconut_compiler = Compiler(**coconut_encoding_kwargs) return cls.coconut_compiler.parse_sys(source)
def __init__(self, pyfun): self.pyfun = pyfun self.c = Compiler() self.c.compile(pyfun.__code__) self.ircfg = self.c.ircfg
class Command(object): """Coconut command-line interface.""" comp = None # current coconut.compiler.Compiler show = False # corresponds to --display flag runner = None # the current Runner jobs = 0 # corresponds to --jobs flag executor = None # runs --jobs exit_code = 0 # exit status to return errmsg = None # error message to display mypy_args = None # corresponds to --mypy flag argv_args = None # corresponds to --argv flag def __init__(self): """Create the CLI.""" self.prompt = Prompt() def start(self, run=False): """Endpoint for coconut and coconut-run.""" if run: args, argv = [], [] # for coconut-run, all args beyond the source file should be wrapped in an --argv for i in range(1, len(sys.argv)): arg = sys.argv[i] args.append(arg) # if arg is source file, put everything else in argv if not arg.startswith("-") and can_parse(arguments, args[:-1]): argv = sys.argv[i + 1:] break for run_arg in (coconut_run_verbose_args if "--verbose" in args else coconut_run_args): if run_arg not in args: args.append(run_arg) self.cmd(args, argv=argv) else: self.cmd() def cmd(self, args=None, argv=None, interact=True): """Process command-line arguments.""" if args is None: parsed_args = arguments.parse_args() else: parsed_args = arguments.parse_args(args) if argv is not None: if parsed_args.argv is not None: raise CoconutException("cannot pass --argv/--args when using coconut-run (coconut-run interprets any arguments after the source file as --argv/--args)") parsed_args.argv = argv self.exit_code = 0 with self.handling_exceptions(): self.use_args(parsed_args, interact, original_args=args) self.exit_on_error() def setup(self, *args, **kwargs): """Set parameters for the compiler.""" if self.comp is None: self.comp = Compiler(*args, **kwargs) else: self.comp.setup(*args, **kwargs) def exit_on_error(self): """Exit if exit_code is abnormal.""" if self.exit_code: if self.errmsg is not None: logger.show("Exiting with error: " + self.errmsg) self.errmsg = None if self.using_jobs: kill_children() sys.exit(self.exit_code) def use_args(self, args, interact=True, original_args=None): """Handle command-line arguments.""" # fix args if not DEVELOP: args.trace = args.profile = False # set up logger logger.quiet, logger.verbose, logger.tracing = args.quiet, args.verbose, args.trace if args.trace or args.profile: unset_fast_pyparsing_reprs() if args.profile: collect_timing_info() logger.log(cli_version) if original_args is not None: logger.log("Directly passed args:", original_args) logger.log("Parsed args:", args) # validate general command args if args.mypy is not None and args.line_numbers: logger.warn("extraneous --line-numbers argument passed; --mypy implies --line-numbers") if args.site_install and args.site_uninstall: raise CoconutException("cannot --site-install and --site-uninstall simultaneously") # process general command args if args.recursion_limit is not None: set_recursion_limit(args.recursion_limit) if args.jobs is not None: self.set_jobs(args.jobs) if args.display: self.show = True if args.style is not None: self.prompt.set_style(args.style) if args.history_file is not None: self.prompt.set_history_file(args.history_file) if args.vi_mode: self.prompt.vi_mode = True if args.docs: launch_documentation() if args.tutorial: launch_tutorial() if args.site_uninstall: self.site_uninstall() if args.site_install: self.site_install() if args.argv is not None: self.argv_args = list(args.argv) # additional validation after processing if args.profile and self.jobs != 0: raise CoconutException("--profile incompatible with --jobs {jobs}".format(jobs=args.jobs)) # process general compiler args self.setup( target=args.target, strict=args.strict, minify=args.minify, line_numbers=args.line_numbers or args.mypy is not None, keep_lines=args.keep_lines, no_tco=args.no_tco, no_wrap=args.no_wrap, ) # process mypy args (must come after compiler setup) if args.mypy is not None: self.set_mypy_args(args.mypy) if args.source is not None: # warnings if source is given if args.interact and args.run: logger.warn("extraneous --run argument passed; --interact implies --run") if args.package and self.mypy: logger.warn("extraneous --package argument passed; --mypy implies --package") # errors if source is given if args.standalone and args.package: raise CoconutException("cannot compile as both --package and --standalone") if args.standalone and self.mypy: raise CoconutException("cannot compile as both --package (implied by --mypy) and --standalone") if args.no_write and self.mypy: raise CoconutException("cannot compile with --no-write when using --mypy") # process all source, dest pairs src_dest_package_triples = [ self.process_source_dest(src, dst, args) for src, dst in ( [(args.source, args.dest)] + (getattr(args, "and") or []) ) ] # do compilation with self.running_jobs(exit_on_error=not args.watch): filepaths = [] for source, dest, package in src_dest_package_triples: filepaths += self.compile_path(source, dest, package, run=args.run or args.interact, force=args.force) self.run_mypy(filepaths) # validate args if no source is given elif ( args.run or args.no_write or args.force or args.package or args.standalone or args.watch ): raise CoconutException("a source file/folder must be specified when options that depend on the source are enabled") elif getattr(args, "and"): raise CoconutException("--and should only be used for extra source/dest pairs, not the first source/dest pair") # handle extra cli tasks if args.code is not None: self.execute(self.comp.parse_block(args.code)) got_stdin = False if args.jupyter is not None: self.start_jupyter(args.jupyter) elif stdin_readable(): logger.log("Reading piped input from stdin...") self.execute(self.comp.parse_block(sys.stdin.read())) got_stdin = True if args.interact or ( interact and not ( got_stdin or args.source or args.code or args.tutorial or args.docs or args.watch or args.site_uninstall or args.site_install or args.jupyter is not None or args.mypy == [mypy_install_arg] ) ): self.start_prompt() if args.watch: # src_dest_package_triples is always available here self.watch(src_dest_package_triples, args.run, args.force) if args.profile: print_timing_info() def process_source_dest(self, source, dest, args): """Determine the correct source, dest, package mode to use for the given source, dest, and args.""" # determine source processed_source = fixpath(source) # validate args if (args.run or args.interact) and os.path.isdir(processed_source): if args.run: raise CoconutException("source path %r must point to file not directory when --run is enabled" % (source,)) if args.interact: raise CoconutException("source path %r must point to file not directory when --run (implied by --interact) is enabled" % (source,)) if args.watch and os.path.isfile(processed_source): raise CoconutException("source path %r must point to directory not file when --watch is enabled" % (source,)) # determine dest if dest is None: if args.no_write: processed_dest = False # no dest else: processed_dest = True # auto-generate dest elif args.no_write: raise CoconutException("destination path cannot be given when --no-write is enabled") else: processed_dest = dest # determine package mode if args.package or self.mypy: package = True elif args.standalone: package = False else: # auto-decide package if os.path.isfile(source): package = False elif os.path.isdir(source): package = True else: raise CoconutException("could not find source path", source) return processed_source, processed_dest, package def register_error(self, code=1, errmsg=None): """Update the exit code.""" if errmsg is not None: if self.errmsg is None: self.errmsg = errmsg elif errmsg not in self.errmsg: self.errmsg += ", " + errmsg if code is not None: self.exit_code = code or self.exit_code @contextmanager def handling_exceptions(self): """Perform proper exception handling.""" try: if self.using_jobs: with handling_broken_process_pool(): yield else: yield except SystemExit as err: self.register_error(err.code) except BaseException as err: if isinstance(err, CoconutException): logger.print_exc() elif not isinstance(err, KeyboardInterrupt): logger.print_exc() printerr(report_this_text) self.register_error(errmsg=err.__class__.__name__) def compile_path(self, path, write=True, package=True, **kwargs): """Compile a path and returns paths to compiled files.""" if not isinstance(write, bool): write = fixpath(write) if os.path.isfile(path): destpath = self.compile_file(path, write, package, **kwargs) return [destpath] if destpath is not None else [] elif os.path.isdir(path): return self.compile_folder(path, write, package, **kwargs) else: raise CoconutException("could not find source path", path) def compile_folder(self, directory, write=True, package=True, **kwargs): """Compile a directory and returns paths to compiled files.""" if not isinstance(write, bool) and os.path.isfile(write): raise CoconutException("destination path cannot point to a file when compiling a directory") filepaths = [] for dirpath, dirnames, filenames in os.walk(directory): if isinstance(write, bool): writedir = write else: writedir = os.path.join(write, os.path.relpath(dirpath, directory)) for filename in filenames: if os.path.splitext(filename)[1] in code_exts: with self.handling_exceptions(): destpath = self.compile_file(os.path.join(dirpath, filename), writedir, package, **kwargs) if destpath is not None: filepaths.append(destpath) for name in dirnames[:]: if not is_special_dir(name) and name.startswith("."): if logger.verbose: logger.show_tabulated("Skipped directory", name, "(explicitly pass as source to override).") dirnames.remove(name) # directories removed from dirnames won't appear in further os.walk iterations return filepaths def compile_file(self, filepath, write=True, package=False, force=False, **kwargs): """Compile a file and returns the compiled file's path.""" set_ext = False if write is False: destpath = None elif write is True: destpath = filepath set_ext = True elif os.path.splitext(write)[1]: # write is a file; it is the destination filepath destpath = write else: # write is a dir; make the destination filepath by adding the filename destpath = os.path.join(write, os.path.basename(filepath)) set_ext = True if set_ext: base, ext = os.path.splitext(os.path.splitext(destpath)[0]) if not ext: ext = comp_ext destpath = fixpath(base + ext) if filepath == destpath: raise CoconutException("cannot compile " + showpath(filepath) + " to itself", extra="incorrect file extension") if destpath is not None: dest_ext = os.path.splitext(destpath)[1] if dest_ext in code_exts: if force: logger.warn("found destination path with " + dest_ext + " extension; compiling anyway due to --force") else: raise CoconutException( "found destination path with " + dest_ext + " extension; aborting compilation", extra="pass --force to override", ) self.compile(filepath, destpath, package, force=force, **kwargs) return destpath def compile(self, codepath, destpath=None, package=False, run=False, force=False, show_unchanged=True): """Compile a source Coconut file to a destination Python file.""" with univ_open(codepath, "r") as opened: code = readfile(opened) package_level = -1 if destpath is not None: destpath = fixpath(destpath) destdir = os.path.dirname(destpath) if not os.path.exists(destdir): os.makedirs(destdir) if package is True: package_level = self.get_package_level(codepath) if package_level == 0: self.create_package(destdir) foundhash = None if force else self.has_hash_of(destpath, code, package_level) if foundhash: if show_unchanged: logger.show_tabulated("Left unchanged", showpath(destpath), "(pass --force to override).") if self.show: print(foundhash) if run: self.execute_file(destpath, argv_source_path=codepath) else: logger.show_tabulated("Compiling", showpath(codepath), "...") def callback(compiled): if destpath is None: logger.show_tabulated("Compiled", showpath(codepath), "without writing to file.") else: with univ_open(destpath, "w") as opened: writefile(opened, compiled) logger.show_tabulated("Compiled to", showpath(destpath), ".") if self.show: print(compiled) if run: if destpath is None: self.execute(compiled, path=codepath, allow_show=False) else: self.execute_file(destpath, argv_source_path=codepath) if package is True: self.submit_comp_job(codepath, callback, "parse_package", code, package_level=package_level) elif package is False: self.submit_comp_job(codepath, callback, "parse_file", code) else: raise CoconutInternalException("invalid value for package", package) def get_package_level(self, codepath): """Get the relative level to the base directory of the package.""" package_level = -1 check_dir = os.path.dirname(os.path.abspath(codepath)) while check_dir: has_init = False for ext in code_exts: init_file = os.path.join(check_dir, "__init__" + ext) if os.path.exists(init_file): has_init = True break if has_init: package_level += 1 check_dir = os.path.dirname(check_dir) else: break if package_level < 0: if self.comp.strict: logger.warn("missing __init__" + code_exts[0] + " in package", check_dir, extra="remove --strict to dismiss") package_level = 0 return package_level return 0 def create_package(self, dirpath): """Set up a package directory.""" filepath = os.path.join(dirpath, "__coconut__.py") with univ_open(filepath, "w") as opened: writefile(opened, self.comp.getheader("__coconut__")) def submit_comp_job(self, path, callback, method, *args, **kwargs): """Submits a job on self.comp to be run in parallel.""" if self.executor is None: with self.handling_exceptions(): callback(getattr(self.comp, method)(*args, **kwargs)) else: path = showpath(path) with logger.in_path(path): # pickle the compiler in the path context future = self.executor.submit(multiprocess_wrapper(self.comp, method), *args, **kwargs) def callback_wrapper(completed_future): """Ensures that all errors are always caught, since errors raised in a callback won't be propagated.""" with logger.in_path(path): # handle errors in the path context with self.handling_exceptions(): result = completed_future.result() callback(result) future.add_done_callback(callback_wrapper) def set_jobs(self, jobs): """Set --jobs.""" if jobs == "sys": self.jobs = None else: try: jobs = int(jobs) except ValueError: jobs = -1 # will raise error below if jobs < 0: raise CoconutException("--jobs must be an integer >= 0 or 'sys'") self.jobs = jobs @property def using_jobs(self): """Determine whether or not multiprocessing is being used.""" return self.jobs is None or self.jobs > 1 @contextmanager def running_jobs(self, exit_on_error=True): """Initialize multiprocessing.""" with self.handling_exceptions(): if self.using_jobs: from concurrent.futures import ProcessPoolExecutor try: with ProcessPoolExecutor(self.jobs) as self.executor: yield finally: self.executor = None else: yield if exit_on_error: self.exit_on_error() def has_hash_of(self, destpath, code, package_level): """Determine if a file has the hash of the code.""" if destpath is not None and os.path.isfile(destpath): with univ_open(destpath, "r") as opened: compiled = readfile(opened) hashash = gethash(compiled) if hashash is not None and hashash == self.comp.genhash(code, package_level): return True return False def get_input(self, more=False): """Prompt for code input.""" received = None try: received = self.prompt.input(more) except KeyboardInterrupt: print() printerr("KeyboardInterrupt") except EOFError: print() self.exit_runner() else: if received.startswith(exit_chars): self.exit_runner() received = None return received def start_running(self): """Start running the Runner.""" self.comp.warm_up() self.check_runner() self.running = True def start_prompt(self): """Start the interpreter.""" logger.show( "Coconut Interpreter v{co_ver}:".format( co_ver=VERSION, ), ) logger.show("(enter 'exit()' or press Ctrl-D to end)") self.start_running() while self.running: try: code = self.get_input() if code: compiled = self.handle_input(code) if compiled: self.execute(compiled, use_eval=None) except KeyboardInterrupt: printerr("\nKeyboardInterrupt") def exit_runner(self, exit_code=0): """Exit the interpreter.""" self.register_error(exit_code) self.running = False def handle_input(self, code): """Compile Coconut interpreter input.""" if not self.prompt.multiline: if not should_indent(code): try: return self.comp.parse_block(code) except CoconutException: pass while True: line = self.get_input(more=True) if line is None: return None elif line: code += "\n" + line else: break try: return self.comp.parse_block(code) except CoconutException: logger.print_exc() return None def execute(self, compiled=None, path=None, use_eval=False, allow_show=True): """Execute compiled code.""" self.check_runner() if compiled is not None: if allow_show and self.show: print(compiled) if path is None: # header is not included if not self.mypy: no_str_code = self.comp.remove_strs(compiled) result = mypy_builtin_regex.search(no_str_code) if result: logger.warn("found mypy-only built-in " + repr(result.group(0)) + "; pass --mypy to use mypy-only built-ins at the interpreter") else: # header is included compiled = rem_encoding(compiled) self.runner.run(compiled, use_eval=use_eval, path=path, all_errors_exit=path is not None) self.run_mypy(code=self.runner.was_run_code()) def execute_file(self, destpath, **kwargs): """Execute compiled file.""" self.check_runner(**kwargs) self.runner.run_file(destpath) def check_runner(self, set_sys_vars=True, argv_source_path=""): """Make sure there is a runner.""" if set_sys_vars: # set sys.path if os.getcwd() not in sys.path: sys.path.append(os.getcwd()) # set sys.argv if self.argv_args is not None: sys.argv = [argv_source_path] + self.argv_args # set up runner if self.runner is None: self.runner = Runner(self.comp, exit=self.exit_runner, store=self.mypy) @property def mypy(self): """Whether using MyPy or not.""" return self.mypy_args is not None def set_mypy_args(self, mypy_args=None): """Set MyPy arguments.""" if mypy_args is None: self.mypy_args = None elif mypy_install_arg in mypy_args: if mypy_args != [mypy_install_arg]: raise CoconutException("'--mypy install' cannot be used alongside other --mypy arguments") stub_dir = set_mypy_path() logger.show_sig("Successfully installed MyPy stubs into " + repr(stub_dir)) self.mypy_args = None else: self.mypy_args = list(mypy_args) if not any(arg.startswith("--python-version") for arg in self.mypy_args): self.mypy_args += [ "--python-version", ver_tuple_to_str(get_target_info_smart(self.comp.target, mode="mypy")), ] if not any(arg.startswith("--python-executable") for arg in self.mypy_args): self.mypy_args += [ "--python-executable", sys.executable, ] add_mypy_args = default_mypy_args + (verbose_mypy_args if logger.verbose else ()) for arg in add_mypy_args: no_arg = invert_mypy_arg(arg) arg_prefixes = (arg,) + ((no_arg,) if no_arg is not None else ()) if not any(arg.startswith(arg_prefixes) for arg in self.mypy_args): self.mypy_args.append(arg) logger.log("MyPy args:", self.mypy_args) self.mypy_errs = [] def run_mypy(self, paths=(), code=None): """Run MyPy with arguments.""" if self.mypy: set_mypy_path() from coconut.command.mypy import mypy_run args = list(paths) + self.mypy_args if code is not None: # interpreter args += ["-c", code] for line, is_err in mypy_run(args): logger.log("[MyPy]", line) if line.startswith(mypy_silent_err_prefixes): if code is None: # file printerr(line) self.register_error(errmsg="MyPy error") elif not line.startswith(mypy_silent_non_err_prefixes): if code is None: # file printerr(line) if any(infix in line for infix in mypy_err_infixes): self.register_error(errmsg="MyPy error") if line not in self.mypy_errs: if code is not None: # interpreter printerr(line) self.mypy_errs.append(line) def run_silent_cmd(self, *args): """Same as run_cmd$(show_output=logger.verbose).""" return run_cmd(*args, show_output=logger.verbose) def install_jupyter_kernel(self, jupyter, kernel_dir): """Install the given kernel via the command line and return whether successful.""" install_args = jupyter + ["kernelspec", "install", kernel_dir, "--replace"] try: self.run_silent_cmd(install_args) except CalledProcessError: user_install_args = install_args + ["--user"] try: self.run_silent_cmd(user_install_args) except CalledProcessError: logger.warn("kernel install failed on command", " ".join(install_args)) self.register_error(errmsg="Jupyter kernel error") return False return True def remove_jupyter_kernel(self, jupyter, kernel_name): """Remove the given kernel via the command line and return whether successful.""" remove_args = jupyter + ["kernelspec", "remove", kernel_name, "-f"] try: self.run_silent_cmd(remove_args) except CalledProcessError: logger.warn("kernel removal failed on command", " ".join(remove_args)) self.register_error(errmsg="Jupyter kernel error") return False return True def install_default_jupyter_kernels(self, jupyter, kernel_list): """Install icoconut default kernels.""" logger.show_sig("Installing Jupyter kernels '" + "', '".join(icoconut_default_kernel_names) + "'...") overall_success = True for old_kernel_name in icoconut_old_kernel_names: if old_kernel_name in kernel_list: success = self.remove_jupyter_kernel(jupyter, old_kernel_name) overall_success = overall_success and success for kernel_dir in icoconut_default_kernel_dirs: success = self.install_jupyter_kernel(jupyter, kernel_dir) overall_success = overall_success and success if overall_success: return icoconut_default_kernel_names else: return [] def get_jupyter_kernels(self, jupyter): """Get the currently installed Jupyter kernels.""" raw_kernel_list = run_cmd(jupyter + ["kernelspec", "list"], show_output=False, raise_errs=False) kernel_list = [] for line in raw_kernel_list.splitlines(): kernel_list.append(line.split()[0]) return kernel_list def start_jupyter(self, args): """Start Jupyter with the Coconut kernel.""" # get the correct jupyter command for jupyter in ( [sys.executable, "-m", "jupyter"], [sys.executable, "-m", "ipython"], ["jupyter"], ): try: self.run_silent_cmd(jupyter + ["--help"]) # --help is much faster than --version except CalledProcessError: logger.warn("failed to find Jupyter command at " + repr(" ".join(jupyter))) else: break # get a list of installed kernels kernel_list = self.get_jupyter_kernels(jupyter) newly_installed_kernels = [] # always update the custom kernel, but only reinstall it if it isn't already there or given no args custom_kernel_dir = install_custom_kernel(logger=logger) if custom_kernel_dir is not None and (icoconut_custom_kernel_name not in kernel_list or not args): logger.show_sig("Installing Jupyter kernel {name!r}...".format(name=icoconut_custom_kernel_name)) if self.install_jupyter_kernel(jupyter, custom_kernel_dir): newly_installed_kernels.append(icoconut_custom_kernel_name) if not args: # install default kernels if given no args newly_installed_kernels += self.install_default_jupyter_kernels(jupyter, kernel_list) run_args = None else: # use the custom kernel if it exists if icoconut_custom_kernel_name in kernel_list or icoconut_custom_kernel_name in newly_installed_kernels: kernel = icoconut_custom_kernel_name # otherwise determine which default kernel to use and install them if necessary else: ver = "2" if PY2 else "3" try: self.run_silent_cmd(["python" + ver, "-m", "coconut.main", "--version"]) except CalledProcessError: kernel = "coconut_py" else: kernel = "coconut_py" + ver if kernel not in kernel_list: newly_installed_kernels += self.install_default_jupyter_kernels(jupyter, kernel_list) logger.warn("could not find {name!r} kernel; using {kernel!r} kernel instead".format(name=icoconut_custom_kernel_name, kernel=kernel)) # pass the kernel to the console or otherwise just launch Jupyter now that we know our kernel is available if args[0] == "console": run_args = jupyter + ["console", "--kernel", kernel] + args[1:] else: run_args = jupyter + args if newly_installed_kernels: logger.show_sig("Successfully installed Jupyter kernels: '" + "', '".join(newly_installed_kernels) + "'") # run the Jupyter command if run_args is not None: self.register_error(run_cmd(run_args, raise_errs=False), errmsg="Jupyter error") def watch(self, src_dest_package_triples, run=False, force=False): """Watch a source and recompile on change.""" from coconut.command.watch import Observer, RecompilationWatcher for src, _, _ in src_dest_package_triples: logger.show() logger.show_tabulated("Watching", showpath(src), "(press Ctrl-C to end)...") def recompile(path, src, dest, package): path = fixpath(path) if os.path.isfile(path) and os.path.splitext(path)[1] in code_exts: with self.handling_exceptions(): if dest is True or dest is None: writedir = dest else: # correct the compilation path based on the relative position of path to src dirpath = os.path.dirname(path) writedir = os.path.join(dest, os.path.relpath(dirpath, src)) filepaths = self.compile_path(path, writedir, package, run=run, force=force, show_unchanged=False) self.run_mypy(filepaths) observer = Observer() watchers = [] for src, dest, package in src_dest_package_triples: watcher = RecompilationWatcher(recompile, src, dest, package) observer.schedule(watcher, src, recursive=True) watchers.append(watcher) with self.running_jobs(): observer.start() try: while True: time.sleep(watch_interval) for wcher in watchers: wcher.keep_watching() except KeyboardInterrupt: logger.show_sig("Got KeyboardInterrupt; stopping watcher.") finally: observer.stop() observer.join() def get_python_lib(self): """Get current Python lib location.""" from distutils import sysconfig # expensive, so should only be imported here return fixpath(sysconfig.get_python_lib()) def site_install(self): """Add Coconut's pth file to site-packages.""" python_lib = self.get_python_lib() shutil.copy(coconut_pth_file, python_lib) logger.show_sig("Added %s to %s." % (os.path.basename(coconut_pth_file), python_lib)) def site_uninstall(self): """Remove Coconut's pth file from site-packages.""" python_lib = self.get_python_lib() pth_file = os.path.join(python_lib, os.path.basename(coconut_pth_file)) if os.path.isfile(pth_file): os.remove(pth_file) logger.show_sig("Removed %s from %s." % (os.path.basename(coconut_pth_file), python_lib)) else: raise CoconutException("failed to find %s file to remove" % (os.path.basename(coconut_pth_file),))