def transform(grammar, text): """Transform text by replacing matches to grammar.""" results = [] intervals = [] for result, start, stop in all_matches(grammar, text): if result is not ignore_transform: internal_assert(isinstance(result, str), "got non-string transform result", result) if start == 0 and stop == len(text): return result results.append(result) intervals.append((start, stop)) if not results: return None split_indices = [0] split_indices.extend(start for start, _ in intervals) split_indices.extend(stop for _, stop in intervals) split_indices.sort() split_indices.append(None) out = [] for i in range(len(split_indices) - 1): if i % 2 == 0: start, stop = split_indices[i], split_indices[i + 1] out.append(text[start:stop]) else: out.append(results[i // 2]) if i // 2 < len(results) - 1: raise CoconutInternalException("unused transform results", results[i // 2 + 1:]) if stop is not None: raise CoconutInternalException("failed to properly split text to be transformed") return "".join(out)
def match_star(self, tokens, item): """Matches starred assignment.""" internal_assert(1 <= len(tokens) <= 3, "invalid star match tokens", tokens) head_matches, last_matches = None, None if len(tokens) == 1: middle = tokens[0] elif len(tokens) == 2: if isinstance(tokens[0], str): middle, last_matches = tokens else: head_matches, middle = tokens else: head_matches, middle, last_matches = tokens self.add_check("_coconut.isinstance(" + item + ", _coconut.abc.Iterable)") if head_matches is None and last_matches is None: if middle != wildcard: self.add_def(middle + " = _coconut.list(" + item + ")") else: itervar = self.get_temp_var() self.add_def(itervar + " = _coconut.list(" + item + ")") with self.down_a_level(): req_length = (len(head_matches) if head_matches is not None else 0) + (len(last_matches) if last_matches is not None else 0) self.add_check("_coconut.len(" + itervar + ") >= " + str(req_length)) if middle != wildcard: head_splice = str(len(head_matches)) if head_matches is not None else "" last_splice = "-" + str(len(last_matches)) if last_matches is not None else "" self.add_def(middle + " = " + itervar + "[" + head_splice + ":" + last_splice + "]") if head_matches is not None: self.match_all_in(head_matches, itervar) if last_matches is not None: for x in range(1, len(last_matches) + 1): self.match(last_matches[-x], itervar + "[-" + str(x) + "]")
def match_iterator(self, tokens, item): """Matches a lazy list or a chain.""" internal_assert(2 <= len(tokens) <= 3, "invalid iterator match tokens", tokens) tail = None if len(tokens) == 2: _, matches = tokens else: _, matches, tail = tokens self.add_check("_coconut.isinstance(" + item + ", _coconut.abc.Iterable)") if tail is None: itervar = self.get_temp_var() self.add_def(itervar + " = _coconut.tuple(" + item + ")") elif matches: itervar = self.get_temp_var() if tail == wildcard: tail = item else: self.add_def(tail + " = _coconut.iter(" + item + ")") self.add_def(itervar + " = _coconut.tuple(_coconut_igetitem(" + tail + ", _coconut.slice(None, " + str(len(matches)) + ")))") else: itervar = None if tail != wildcard: self.add_def(tail + " = " + item) if itervar is not None: with self.down_a_level(): self.add_check("_coconut.len(" + itervar + ") == " + str(len(matches))) self.match_all_in(matches, itervar)
def disable_inside(item, *elems, **kwargs): """Prevent elems from matching inside of item. Returns (item with elem disabled, *new versions of elems). """ _invert = kwargs.pop("_invert", False) internal_assert(not kwargs, "excess keyword arguments passed to disable_inside") level = [0] # number of wrapped items deep we are; in a list to allow modification @contextmanager def manage_item(self, instring, loc): level[0] += 1 try: yield finally: level[0] -= 1 yield Wrap(item, manage_item) @contextmanager def manage_elem(self, instring, loc): if level[0] == 0 if not _invert else level[0] > 0: yield else: raise ParseException(instring, loc, self.errmsg, self) for elem in elems: yield Wrap(elem, manage_elem)
def longest(*args): """Match the longest of the given grammar elements.""" internal_assert(len(args) >= 2, "longest expects at least two args") matcher = args[0] + skip_whitespace for elem in args[1:]: matcher ^= elem + skip_whitespace return matcher
def add_action(item, action): """Add a parse action to the given item.""" item_ref_count = sys.getrefcount(item) if CPYTHON else float("inf") internal_assert(item_ref_count >= temp_grammar_item_ref_count, "add_action got item with too low ref count", (item, type(item), item_ref_count)) if item_ref_count > temp_grammar_item_ref_count: item = item.copy() return item.addParseAction(action)
def split_data_or_class_match(self, tokens): """Split data/class match tokens into cls_name, pos_matches, name_matches, star_match.""" cls_name, matches = tokens pos_matches = [] name_matches = {} star_match = None for match_arg in matches: if len(match_arg) == 1: match, = match_arg if star_match is not None: raise CoconutDeferredSyntaxError("positional arg after starred arg in data/class match", self.loc) if name_matches: raise CoconutDeferredSyntaxError("positional arg after named arg in data/class match", self.loc) pos_matches.append(match) elif len(match_arg) == 2: internal_assert(match_arg[0] == "*", "invalid starred data/class match arg tokens", match_arg) _, match = match_arg if star_match is not None: raise CoconutDeferredSyntaxError("duplicate starred arg in data/class match", self.loc) if name_matches: raise CoconutDeferredSyntaxError("both starred arg and named arg in data/class match", self.loc) star_match = match elif len(match_arg) == 3: internal_assert(match_arg[1] == "=", "invalid named data/class match arg tokens", match_arg) name, _, match = match_arg if star_match is not None: raise CoconutDeferredSyntaxError("both named arg and starred arg in data/class match", self.loc) if name in name_matches: raise CoconutDeferredSyntaxError("duplicate named arg {name!r} in data/class match".format(name=name), self.loc) name_matches[name] = match else: raise CoconutInternalException("invalid data/class match arg", match_arg) return cls_name, pos_matches, name_matches, star_match
def minify(compiled): """Perform basic minification of the header. Fails on non-tabideal indentation, strings with #s, or multi-line strings. (So don't do those things in the header.) """ compiled = compiled.strip() if compiled: out = [] for line in compiled.splitlines(): new_line, comment = split_comment(line) new_line = new_line.rstrip() if new_line: ind = 0 while new_line.startswith(" "): new_line = new_line[1:] ind += 1 internal_assert(ind % tabideal == 0, "invalid indentation in", line) new_line = " " * (ind // tabideal) + new_line comment = comment.strip() if comment: new_line += "#" + comment if new_line: out.append(new_line) compiled = "\n".join(out) + "\n" return compiled
def run_cmd(cmd, show_output=True, raise_errs=True, **kwargs): """Run a console command. When show_output=True, prints output and returns exit code, otherwise returns output. When raise_errs=True, raises a subprocess.CalledProcessError if the command fails. """ internal_assert(cmd and isinstance(cmd, list), "console commands must be passed as non-empty lists") if hasattr(shutil, "which"): cmd[0] = shutil.which(cmd[0]) or cmd[0] logger.log_cmd(cmd) try: if show_output and raise_errs: return subprocess.check_call(cmd, **kwargs) elif show_output: return subprocess.call(cmd, **kwargs) else: stdout, stderr, retcode = call_output(cmd, **kwargs) output = "".join(stdout + stderr) if retcode and raise_errs: raise subprocess.CalledProcessError(retcode, cmd, output=output) return output except OSError: logger.log_exc() if raise_errs: raise subprocess.CalledProcessError(oserror_retcode, cmd) elif show_output: return oserror_retcode else: return ""
def match_infix(self, tokens, item): """Matches infix patterns.""" internal_assert(len(tokens) > 1 and len(tokens) % 2 == 1, "invalid infix match tokens", tokens) match = tokens[0] for i in range(1, len(tokens), 2): op, arg = tokens[i], tokens[i + 1] self.add_check("(" + op + ")(" + item + ", " + arg + ")") self.match(match, item)
def base_pycondition(target, ver, if_lt=None, if_ge=None, indent=None, newline=False, fallback=""): """Produce code that depends on the Python version for the given target.""" internal_assert(isinstance(ver, tuple), "invalid pycondition version") internal_assert(if_lt or if_ge, "either if_lt or if_ge must be specified") if if_lt: if_lt = if_lt.strip() if if_ge: if_ge = if_ge.strip() target_supported_vers = get_vers_for_target(target) if all(tar_ver < ver for tar_ver in target_supported_vers): if not if_lt: return fallback out = if_lt elif all(tar_ver >= ver for tar_ver in target_supported_vers): if not if_ge: return fallback out = if_ge else: if if_lt and if_ge: out = """if _coconut_sys.version_info < {ver}: {lt_block} else: {ge_block}""".format( ver=repr(ver), lt_block=_indent(if_lt, by=1), ge_block=_indent(if_ge, by=1), ) elif if_lt: out = """if _coconut_sys.version_info < {ver}: {lt_block}""".format( ver=repr(ver), lt_block=_indent(if_lt, by=1), ) else: out = """if _coconut_sys.version_info >= {ver}: {ge_block}""".format( ver=repr(ver), ge_block=_indent(if_ge, by=1), ) if indent is not None: out = _indent(out, by=indent) if newline: out += "\n" return out
def interleaved_join(first_list, second_list): """Interleaves two lists of strings and joins the result. Example: interleaved_join(['1', '3'], ['2']) == '123' The first list must be 1 longer than the second list. """ internal_assert(len(first_list) == len(second_list) + 1, "invalid list lengths to interleaved_join", (first_list, second_list)) interleaved = [] for first_second in zip(first_list, second_list): interleaved.extend(first_second) interleaved.append(first_list[-1]) return "".join(interleaved)
def match_trailer(self, tokens, item): """Matches typedefs and as patterns.""" internal_assert(len(tokens) > 1 and len(tokens) % 2 == 1, "invalid trailer match tokens", tokens) match, trailers = tokens[0], tokens[1:] for i in range(0, len(trailers), 2): op, arg = trailers[i], trailers[i + 1] if op == "as": self.match_var([arg], item, bind_wildcard=True) elif op == "is": self.add_check("_coconut.isinstance(" + item + ", " + arg + ")") else: raise CoconutInternalException("invalid trailer match operation", op) self.match(match, item)
def keyword(name, explicit_prefix=None): """Construct a grammar which matches name as a Python keyword.""" if explicit_prefix is not False: internal_assert( (name in reserved_vars) is (explicit_prefix is not None), "invalid keyword call for", name, extra="pass explicit_prefix to keyword for all reserved_vars and only reserved_vars", ) base_kwd = regex_item(name + r"\b") if explicit_prefix in (None, False): return base_kwd else: return Optional(explicit_prefix.suppress()) + base_kwd
def memoized_parse_block(code): """Memoized version of parse_block.""" internal_assert(lambda: code not in parse_block_memo.values(), "attempted recompilation of", code) success, result = parse_block_memo.get(code, (None, None)) if success is None: try: parsed = COMPILER.parse_block(code) except Exception as err: success, result = False, err else: success, result = True, parsed parse_block_memo[code] = (success, result) if success: return result else: raise result
def match_dict(self, tokens, item): """Matches a dictionary.""" internal_assert(1 <= len(tokens) <= 2, "invalid dict match tokens", tokens) if len(tokens) == 1: matches, rest = tokens[0], None else: matches, rest = tokens self.add_check("_coconut.isinstance(" + item + ", _coconut.abc.Mapping)") if rest is None: self.rule_conflict_warn( "ambiguous pattern; could be Coconut-style len-checking dict match or Python-style len-ignoring dict match", if_coconut='resolving to Coconut-style len-checking dict match by default', if_python='resolving to Python-style len-ignoring dict match due to Python-style "match: case" block', extra="use explicit '{..., **_}' or '{..., **{}}' syntax to dismiss", ) check_len = not self.using_python_rules elif rest == "{}": check_len = True rest = None else: check_len = False if check_len: self.add_check("_coconut.len(" + item + ") == " + str(len(matches))) seen_keys = set() for k, v in matches: if k in seen_keys: raise CoconutDeferredSyntaxError("duplicate key {k!r} in dictionary pattern".format(k=k), self.loc) seen_keys.add(k) key_var = self.get_temp_var() self.add_def(key_var + " = " + item + ".get(" + k + ", _coconut_sentinel)") with self.down_a_level(): self.add_check(key_var + " is not _coconut_sentinel") self.match(v, key_var) if rest is not None and rest != wildcard: match_keys = [k for k, v in matches] with self.down_a_level(): self.add_def( rest + " = dict((k, v) for k, v in " + item + ".items() if k not in set((" + ", ".join(match_keys) + ("," if len(match_keys) == 1 else "") + ")))", )
def match_sequence(self, tokens, item): """Matches a sequence.""" internal_assert(2 <= len(tokens) <= 3, "invalid sequence match tokens", tokens) tail = None if len(tokens) == 2: series_type, matches = tokens else: series_type, matches, tail = tokens self.add_check("_coconut.isinstance(" + item + ", _coconut.abc.Sequence)") if tail is None: self.add_check("_coconut.len(" + item + ") == " + str(len(matches))) else: self.add_check("_coconut.len(" + item + ") >= " + str(len(matches))) if tail != wildcard: if len(matches) > 0: splice = "[" + str(len(matches)) + ":]" else: splice = "" self.assign_to_series(tail, series_type, item + splice) self.match_all_in(matches, item)
def input(self, more=False): """Prompt for code input.""" sys.stdout.flush() if more: msg = more_prompt else: msg = main_prompt if self.style is not None: internal_assert(prompt_toolkit is not None, "without prompt_toolkit cannot highlight style", self.style) try: return self.prompt(msg) except EOFError: raise # issubclass(EOFError, Exception), so we have to do this except (Exception, AssertionError): logger.print_exc() logger.show_sig( "Syntax highlighting failed; switching to --style none.") self.style = None return input(msg)
def handle_indentation(inputstr, add_newline=False): """Replace tabideal indentation with openindent and closeindent. Ignores whitespace-only lines.""" out_lines = [] prev_ind = None for line in inputstr.splitlines(): line = line.rstrip() if line: new_ind_str, _ = split_leading_indent(line) internal_assert(new_ind_str.strip(" ") == "", "invalid indentation characters for handle_indentation", new_ind_str) internal_assert(len(new_ind_str) % tabideal == 0, "invalid indentation level for handle_indentation", len(new_ind_str)) new_ind = len(new_ind_str) // tabideal if prev_ind is None: # first line indent = "" elif new_ind > prev_ind: # indent indent = openindent * (new_ind - prev_ind) elif new_ind < prev_ind: # dedent indent = closeindent * (prev_ind - new_ind) else: indent = "" out_lines.append(indent + line) prev_ind = new_ind if add_newline: out_lines.append("") if prev_ind > 0: out_lines[-1] += closeindent * prev_ind out = "\n".join(out_lines) internal_assert(lambda: out.count(openindent) == out.count(closeindent), "failed to properly handle indentation in", out) return out
def __init__(self, comp, original, loc, check_var, style="coconut", name_list=None, checkdefs=None, parent_names={}, var_index_obj=None): """Creates the matcher.""" self.comp = comp self.original = original self.loc = loc self.check_var = check_var internal_assert(style in self.valid_styles, "invalid Matcher style", style) self.style = style self.name_list = name_list self.position = 0 self.checkdefs = [] if checkdefs is None: self.increment() else: for checks, defs in checkdefs: self.checkdefs.append((checks[:], defs[:])) self.set_position(-1) self.parent_names = parent_names self.names = OrderedDict() # ensures deterministic ordering of name setting code self.var_index_obj = [0] if var_index_obj is None else var_index_obj self.guards = [] self.child_groups = []
def evaluate(self): """Get the result of evaluating the computation graph at this node.""" if DEVELOP: internal_assert(not self.been_called, "inefficient reevaluation of action " + self.name + " with tokens", self.tokens) self.been_called = True evaluated_toks = evaluate_tokens(self.tokens) if logger.tracing: # avoid the overhead of the call if not tracing logger.log_trace(self.name, self.original, self.loc, evaluated_toks, self.tokens) try: return _trim_arity(self.action)( self.original, self.loc, evaluated_toks, ) except CoconutException: raise except (Exception, AssertionError): logger.print_exc() error = CoconutInternalException("error computing action " + self.name + " of evaluated tokens", evaluated_toks) if embed_on_internal_exc: logger.warn_err(error) embed(depth=2) else: raise error
def add_timing_to_method(cls, method_name, method): """Add timing collection to the given method. It's a monstrosity, but it's only used for profiling.""" from coconut.terminal import internal_assert # hide to avoid circular import args, varargs, keywords, defaults = inspect.getargspec(method) internal_assert(args[:1] == ["self"], "cannot add timing to method", method_name) if not defaults: defaults = [] num_undefaulted_args = len(args) - len(defaults) def_args = [] call_args = [] fix_arg_defaults = [] defaults_dict = {} for i, arg in enumerate(args): if i >= num_undefaulted_args: default = defaults[i - num_undefaulted_args] def_args.append(arg + "=_timing_sentinel") defaults_dict[arg] = default fix_arg_defaults.append( """ if {arg} is _timing_sentinel: {arg} = _exec_dict["defaults_dict"]["{arg}"] """.strip("\n").format(arg=arg, ), ) else: def_args.append(arg) call_args.append(arg) if varargs: def_args.append("*" + varargs) call_args.append("*" + varargs) if keywords: def_args.append("**" + keywords) call_args.append("**" + keywords) new_method_name = "new_" + method_name + "_func" _exec_dict = globals().copy() _exec_dict.update(locals()) new_method_code = """ def {new_method_name}({def_args}): {fix_arg_defaults} _all_args = (lambda *args, **kwargs: args + tuple(kwargs.values()))({call_args}) _exec_dict["internal_assert"](not any(_arg is _timing_sentinel for _arg in _all_args), "error handling arguments in timed method {new_method_name}({def_args}); got", _all_args) _start_time = _exec_dict["get_clock_time"]() try: return _exec_dict["method"]({call_args}) finally: _timing_info[0][str(self)] += _exec_dict["get_clock_time"]() - _start_time {new_method_name}._timed = True """.format( fix_arg_defaults="\n".join(fix_arg_defaults), new_method_name=new_method_name, def_args=", ".join(def_args), call_args=", ".join(call_args), ) exec(new_method_code, _exec_dict) setattr(cls, method_name, _exec_dict[new_method_name]) return True
def _combine(self, original, loc, tokens): """Implement the parse action for Combine.""" combined_tokens = super(CombineNode, self).postParse(original, loc, tokens) if DEVELOP: # avoid the overhead of the call if not develop internal_assert(len(combined_tokens) == 1, "Combine produced multiple tokens", combined_tokens) return combined_tokens[0]
def stores_loc_action(loc, tokens): """Action that just parses to loc.""" internal_assert(len(tokens) == 0, "invalid store loc tokens", tokens) return str(loc)
def increment(self, by=1): """Advances the if-statement position.""" new_pos = self.position + by internal_assert(new_pos > 0, "invalid increment/decrement call to set pos to", new_pos) self.set_position(new_pos)
def getheader(which, target, use_hash, no_tco, strict, no_wrap): """Generate the specified header.""" internal_assert( which.startswith("package") or which in ( "none", "initial", "__coconut__", "sys", "code", "file", ), "invalid header type", which, ) if which == "none": return "" target_startswith = one_num_ver(target) target_info = get_target_info(target) # initial, __coconut__, package:n, sys, code, file format_dict = process_header_args(which, target, use_hash, no_tco, strict, no_wrap) if which == "initial" or which == "__coconut__": header = '''#!/usr/bin/env python{target_startswith} # -*- coding: {default_encoding} -*- {hash_line}{typing_line} # Compiled with Coconut version {VERSION_STR} {module_docstring}'''.format(**format_dict) elif use_hash is not None: raise CoconutInternalException( "can only add a hash to an initial or __coconut__ header, not", which) else: header = "" if which == "initial": return header # __coconut__, package:n, sys, code, file header += section("Coconut Header", newline_before=False) if target_startswith != "3": header += "from __future__ import print_function, absolute_import, unicode_literals, division\n" elif target_info >= (3, 7): if no_wrap: header += "from __future__ import generator_stop\n" else: header += "from __future__ import generator_stop, annotations\n" elif target_info >= (3, 5): header += "from __future__ import generator_stop\n" if which.startswith("package"): levels_up = int(which[len("package:"):]) coconut_file_dir = "_coconut_os.path.dirname(_coconut_os.path.abspath(__file__))" for _ in range(levels_up): coconut_file_dir = "_coconut_os.path.dirname(" + coconut_file_dir + ")" return header + '''import sys as _coconut_sys, os as _coconut_os _coconut_file_dir = {coconut_file_dir} _coconut_cached_module = _coconut_sys.modules.get({__coconut__}) if _coconut_cached_module is not None and _coconut_os.path.dirname(_coconut_cached_module.__file__) != _coconut_file_dir: del _coconut_sys.modules[{__coconut__}] _coconut_sys.path.insert(0, _coconut_file_dir) _coconut_module_name = _coconut_os.path.splitext(_coconut_os.path.basename(_coconut_file_dir))[0] if _coconut_module_name and _coconut_module_name[0].isalpha() and all(c.isalpha() or c.isdigit() for c in _coconut_module_name) and "__init__.py" in _coconut_os.listdir(_coconut_file_dir): _coconut_full_module_name = str(_coconut_module_name + ".__coconut__") import __coconut__ as _coconut__coconut__ _coconut__coconut__.__name__ = _coconut_full_module_name for _coconut_v in vars(_coconut__coconut__).values(): if getattr(_coconut_v, "__module__", None) == {__coconut__}: try: _coconut_v.__module__ = _coconut_full_module_name except AttributeError: _coconut_v_type = type(_coconut_v) if getattr(_coconut_v_type, "__module__", None) == {__coconut__}: _coconut_v_type.__module__ = _coconut_full_module_name _coconut_sys.modules[_coconut_full_module_name] = _coconut__coconut__ from __coconut__ import * from __coconut__ import {underscore_imports} _coconut_sys.path.pop(0) '''.format(coconut_file_dir=coconut_file_dir, __coconut__=('"__coconut__"' if target_startswith == "3" else 'b"__coconut__"' if target_startswith == "2" else 'str("__coconut__")'), **format_dict) + section("Compiled Coconut") if which == "sys": return header + '''import sys as _coconut_sys from coconut.__coconut__ import * from coconut.__coconut__ import {underscore_imports} '''.format(**format_dict) # __coconut__, code, file header += "import sys as _coconut_sys\n" if target_info >= (3, 7): header += PY37_HEADER elif target_startswith == "3": header += PY3_HEADER elif target_info >= (2, 7): header += PY27_HEADER elif target_startswith == "2": header += PY2_HEADER else: header += PYCHECK_HEADER header += get_template("header").format(**format_dict) if which == "file": header += section("Compiled Coconut") return header
def evaluate_tokens(tokens, **kwargs): """Evaluate the given tokens in the computation graph.""" # can't have this be a normal kwarg to make evaluate_tokens a valid parse action evaluated_toklists = kwargs.pop("evaluated_toklists", ()) internal_assert(not kwargs, "invalid keyword arguments to evaluate_tokens", kwargs) if isinstance(tokens, ParseResults): # evaluate the list portion of the ParseResults old_toklist, name, asList, modal = tokens.__getnewargs__() new_toklist = None for eval_old_toklist, eval_new_toklist in evaluated_toklists: if old_toklist == eval_old_toklist: new_toklist = eval_new_toklist break if new_toklist is None: new_toklist = [evaluate_tokens(toks, evaluated_toklists=evaluated_toklists) for toks in old_toklist] # overwrite evaluated toklists rather than appending, since this # should be all the information we need for evaluating the dictionary evaluated_toklists = ((old_toklist, new_toklist),) new_tokens = ParseResults(new_toklist, name, asList, modal) new_tokens._ParseResults__accumNames.update(tokens._ParseResults__accumNames) # evaluate the dictionary portion of the ParseResults new_tokdict = {} for name, occurrences in tokens._ParseResults__tokdict.items(): new_occurrences = [] for value, position in occurrences: new_value = evaluate_tokens(value, evaluated_toklists=evaluated_toklists) new_occurrences.append(_ParseResultsWithOffset(new_value, position)) new_tokdict[name] = new_occurrences new_tokens._ParseResults__tokdict.update(new_tokdict) return new_tokens else: if evaluated_toklists: for eval_old_toklist, eval_new_toklist in evaluated_toklists: indices = multi_index_lookup(eval_old_toklist, tokens, indexable_types=indexable_evaluated_tokens_types) if indices is not None: new_tokens = eval_new_toklist for ind in indices: new_tokens = new_tokens[ind] return new_tokens complain( lambda: CoconutInternalException( "inefficient reevaluation of tokens: {tokens} not in:\n{toklists}".format( tokens=tokens, toklists=pformat([eval_old_toklist for eval_old_toklist, eval_new_toklist in evaluated_toklists]), ), ), ) if isinstance(tokens, str): return tokens elif isinstance(tokens, ComputationNode): return tokens.evaluate() elif isinstance(tokens, list): return [evaluate_tokens(inner_toks, evaluated_toklists=evaluated_toklists) for inner_toks in tokens] elif isinstance(tokens, tuple): return tuple(evaluate_tokens(inner_toks, evaluated_toklists=evaluated_toklists) for inner_toks in tokens) else: raise CoconutInternalException("invalid computation graph tokens", tokens)