# Parsing

args_list = []

tokens = lex(code)

while tokens != []:

to_print = add_print(tokens)

parsed, tokens = parse(tokens, safe_mode)

if to_print:

parsed = 'imp_print(' + parsed + ')'

# Finish semicolon parsing

if tokens and tokens[0] == ';':

tokens = tokens[1:]

args_list.append(parsed)

# Build the output string.

args_list = add_preps(preps_used, safe_mode) + args_list

py_code = '\n'.join(args_list)

assert isinstance(tokens, list)

# If we've reached the end of the string, finish up.

if not tokens:

if lambda_stack:

return lambda_stack[-1], []

else:

preps_used.add('Q')

return 'Q', []

# Separate active character from the rest of the code.

active_token = tokens[0]

rest_tokens = tokens[1:]

assert isinstance(active_token, str)

# Deal with numbers

if active_token == '.':

raise PythParseError(active_token, rest_tokens)

if active_token[0] in "0123456789"\

or active_token[0] == '.' and active_token[1] in "0123456789":

return active_token, rest_tokens

# String literals

if active_token[0] == '"':

return str_parse_next(active_token), rest_tokens

if active_token[:2] == '."':

string = str_parse_next(active_token[1:])

return "%s(%s)" % (c_to_f['."'][0], string), rest_tokens

# Python code literals

if active_token[0] == '$':

if safe_mode:

raise UnsafeInputError(active_token, rest_tokens)

else:

return active_token.strip('$'), rest_tokens

# End paren is magic (early-end current function/statement).

if active_token == ')':

return '', rest_tokens

if active_token == ';':

# Inside a lambda, return the innermost lambdas leading variable.

if lambda_stack or state_maintaining_depth:

return 'env_lookup({!r})'.format((lambda_stack or ['Q'])[-1]), rest_tokens

# Semicolon is more magic (early-end all active functions/statements).

if not rest_tokens:

return '', [';']

else:

return '', [';'] + rest_tokens

# Designated variables

if active_token in variables:

if active_token in prepend:

preps_used.add(active_token)

return active_token, rest_tokens

if active_token[0] == '\\':

if active_token[1] in '"\\':

return '"\\%s"' % active_token[1], rest_tokens

else:

return '"%s"' % active_token[1], rest_tokens

# Replace replaements

if active_token in replacements:

return replace_parse(active_token, rest_tokens, safe_mode, spacing)

# Syntactic sugar handling.

if rest_tokens and (active_token in c_to_f or active_token in c_to_i):

sugar_char = rest_tokens[0]

remainder = rest_tokens[1:]

if active_token in c_to_f:

arity = c_to_f[active_token][1]

else:

arity = c_to_i[active_token][1]

# Sugar Chaining

sugar_chars = 'FMLBRID#VW'

sugar_active_tokens = [active_token]

while sugar_char in sugar_chars and remainder and remainder[0] in sugar_chars:

sugar_active_tokens += sugar_char

sugar_char = remainder[0]

remainder = remainder[1:]

if arity > 0:

if sugar_char == 'F':

if arity == 1:

# Unary: Repeated application

rep_arg1, post1 = parse(remainder, safe_mode)

rep_arg2, post2 = parse(post1, safe_mode)

func, rest = parse(sugar_active_tokens + ['b'], safe_mode)

assert not rest, "Sugar parse F repeat failed"

func = "lambda b:" + func

return "repeat({}, {}, {})".format(func, rep_arg1, rep_arg2), post2

if arity == 2:

# <binary function/infix>F: Fold operator

fold_list, post_fold = next_seg(remainder, safe_mode)

if len(sugar_active_tokens) == 1\

and sugar_active_tokens[0] in c_to_f\

and not sugar_active_tokens[0] in lambda_f:

parsed_list, rest = parse(fold_list, safe_mode)

assert not rest, "Sugar parse F fold simple failed"

func = c_to_f[sugar_active_tokens[0]][0]

return "fold({}, {})".format(func, parsed_list), post_fold

reduce_arg1 = lambda_vars['.U'][0][0]

reduce_arg2 = lambda_vars['.U'][0][-1]

full_fold, rest = parse([".U"] + sugar_active_tokens +

[reduce_arg1, reduce_arg2] + fold_list, safe_mode)

assert not rest, "Sugar parse F fold failed"

return full_fold, post_fold

if arity > 2:

# Just splat it - it's a common use case.

splat_list, post_splat = next_seg(remainder, safe_mode)

full_splat, rest = parse(sugar_active_tokens + ['.*'] + splat_list, safe_mode)

assert not rest, "Sugar parse F splat failed"

return full_splat, post_splat

# <function>M: Map operator

if sugar_char == 'M':

m_arg = lambda_vars['m'][0][0]

if arity == 1:

map_target, post_map = next_seg(remainder, safe_mode)

full_map, rest = parse(['m'] + sugar_active_tokens + [m_arg] + map_target, safe_mode)

assert not rest, "Sugar parse M 1 arg failed"

return full_map, post_map

else:

map_target, post_map = next_seg(remainder, safe_mode)

full_map, rest = parse(['m'] + sugar_active_tokens + ['F', m_arg] + map_target, safe_mode)

assert not rest, "Sugar parse M 2+ args failed"

return full_map, post_map

# <binary function>L<any><seq>: Left Map operator

# >LG[1 2 3 4 -> 'm>Gd[1 2 3 4'.

if sugar_char == 'L':

if arity >= 2:

m_arg = lambda_vars['m'][0][0]

lmap_lambda_args, remainder = next_n_segs(arity - 1, remainder, safe_mode)

lmap_target, post_lmap = next_seg(remainder, safe_mode)

full_lmap, rest = parse(['m'] + sugar_active_tokens +

lmap_lambda_args + [m_arg] + lmap_target, safe_mode)

assert not rest, "Sugar parse L failed"

return full_lmap, post_lmap

# <function>V<seq><seq> Vectorize operator.

# Equivalent to <func>MC,<seq><seq>.

if sugar_char == 'V':

vmap_target, post_vmap = next_n_segs(2, remainder, safe_mode)

full_vmap, rest = parse(sugar_active_tokens + ['M', 'C', ','] + vmap_target, safe_mode)

assert not rest, "Sugar parse V failed"

return full_vmap, post_vmap

# <function>W<condition><arg><rgs> Condition application operator.

# Equivalent to ?<condition><function><arg><args><arg>

if sugar_char == 'W':

condition, rest_tokens1 = parse(remainder, safe_mode)

arg1, _ = state_maintaining_parse(rest_tokens1, safe_mode)

func, rest_tokens2b = parse(sugar_active_tokens + rest_tokens1, safe_mode)

return ('(%s if %s else %s)' % (func, condition, arg1), rest_tokens2b)

# <function>B<arg><args> -> ,<arg><function><arg><args>

# <unary function>I<any> Invariant operator.

# Equivalent to q<func><any><any>

if sugar_char in 'BI':

dup_dict = {

'B': '[{},{}]',

'I': '{}=={}',

}

dup_format = dup_dict[sugar_char]

dup_parsed, _ = state_maintaining_parse(remainder, safe_mode)

non_dup_parsed, post_dup = parse(sugar_active_tokens + remainder, safe_mode)

return dup_format.format(dup_parsed, non_dup_parsed), post_dup

# Right operators

# R is Map operator

# D is Sort operator

# # is Filter operator - it looks like a strainer.

if sugar_char in 'RD#':

func_dict = {

'R': 'm',

'D': 'o',

'#': 'f',

}

func_char = func_dict[sugar_char]

lambda_arg = lambda_vars[func_char][0][0]

rop_args, post_rop = next_n_segs(arity, remainder, safe_mode)

full_rop, rest = parse([func_char] + sugar_active_tokens + [lambda_arg] + rop_args, safe_mode)

assert not rest, 'Sugar parse %s failed' % sugar_char

return full_rop, post_rop

# =<function/infix>, ~<function/infix>: augmented assignment.

if active_token in ('=', '~'):

if augment_assignment_test(rest_tokens):

return augment_assignment_parse(active_token, rest_tokens, safe_mode)

# And for general functions

if active_token in c_to_f:

if active_token in lambda_f:

return lambda_function_parse(active_token, rest_tokens, safe_mode)

else:

return function_parse(active_token, rest_tokens, safe_mode)

# General format functions/operators

if active_token in c_to_i:

return infix_parse(active_token, rest_tokens, safe_mode)

# Statements:

if active_token in c_to_s:

return statement_parse(active_token, rest_tokens, safe_mode, spacing)

# If we get here, the character has not been implemented.

# There is no non-ASCII support.

if not isinstance(n, int):

assert n == float('inf'), "arities must be either ints or infinity"

raise RuntimeError # Can't use unbounded arity function in this context.

global c_to_i

global state_maintaining_depth

saved_c_to_i = c.deepcopy(c_to_i)

state_maintaining_depth += 1

remainder = code

for _ in range(n):

_, remainder = parse(remainder, safe_mode)

state_maintaining_depth -= 1

c_to_i = saved_c_to_i

global c_to_i

global state_maintaining_depth

saved_c_to_i = c.deepcopy(c_to_i)

state_maintaining_depth += 1

py_code, rest_tokens = parse(code, safe_mode)

state_maintaining_depth -= 1

c_to_i = saved_c_to_i

# Recurse until terminated by end paren or EOF

# or received enough arguments

args_list = []

while (len(args_list) != arity

and not (not rest_tokens

and (arity == float('inf')

or (active_token in optional_final_args

and len(args_list) >= arity - optional_final_args[active_token])))):

parsed, rest_tokens = parse(rest_tokens, safe_mode)

if not parsed:

break

args_list.append(parsed)

func_name, arity = c_to_f[active_token]

var = lambda_vars[active_token][0]

# Swap what variables are used in lambda functions.

saved_lambda_vars = lambda_vars[active_token]

lambda_vars[active_token] = lambda_vars[active_token][1:] + [var]

lambda_stack.append(var[0])

# Take one argument, the lambda.

lambda_parsed, rest_tokens = parse(rest_tokens, safe_mode)

# Rotate back.

lambda_vars[active_token] = saved_lambda_vars

lambda_stack.pop()

partial_args_list, rest_tokens = gather_args(active_token, rest_tokens, arity-1, safe_mode)

args_list = [lambda_parsed] + partial_args_list

py_code = '%s(lambda %s:%s)' % (func_name, var, ','.join(args_list))

func_name, arity = c_to_f[active_token]

args_list, rest_tokens = gather_args(active_token, rest_tokens, arity, safe_mode)

py_code = '%s(%s)' % (func_name, ','.join(args_list))

infixes, arity = c_to_i[active_token]

args_list = []

# Lambda infix(es)

if active_token == '.W':

lambda_stack.extend(['Z', 'H'])

while len(args_list) != arity:

if (not rest_tokens

and active_token in optional_final_args

and len(args_list) >= arity - optional_final_args[active_token]):

args_list.append('')

break

parsed, rest_tokens = parse(rest_tokens, safe_mode)

if not parsed:

break

args_list.append(parsed)

if active_token == '.W' and len(args_list) <= 2:

lambda_stack.pop()

# Statements that cannot have anything after them

if active_token in end_statement:

rest_tokens = [")"] + rest_tokens

py_code = infixes.format(*args_list)

# Advance infixes.

if active_token in next_c_to_i:

c_to_i[active_token] = next_c_to_i[active_token]

# Handle the initial portion (head)

# addl_spaces denotes the amount of extra spacing needed.

if len(c_to_s[active_token]) == 2:

infixes, arity = c_to_s[active_token]

addl_spaces = ''

else:

infixes, arity, num_spaces = c_to_s[active_token]

addl_spaces = ' ' * num_spaces

# Handle newlines in infix segments

infixes = infixes.replace("\n", spacing[:-1])

args_list, rest_tokens = gather_args(active_token, rest_tokens, arity, safe_mode)

# Handle the body - ends object as well.

body_lines = []

while rest_tokens:

to_print = add_print(rest_tokens)

parsed, rest_tokens = parse(rest_tokens, safe_mode, spacing + addl_spaces + ' ')

if not parsed:

break

if to_print:

parsed = 'imp_print(%s)' % parsed

body_lines.append(parsed)

if body_lines == []:

body_lines = ['pass']

# Combine pieces - intro, statement, conclusion.

total_spacing = spacing + addl_spaces

body = total_spacing + total_spacing.join(body_lines)

args_list.append(body)

# Rotate replacements.

repl_list = replacements[active_token][0]

saved_replacements = replacements[active_token]

replacements[active_token] = replacements[active_token][1:] + [repl_list]

parsed, remainder = parse(repl_list + rest_tokens, safe_mode, spacing)

# Rotate back in some cases.

if active_token in rotate_back_replacements:

replacements[active_token] = saved_replacements

if len(code) > 0:

if code[0] in c_to_s or code[0] in replacements or\

code[0] in ('=', '~', 'B', 'R', 'p', ' ', '\n', ')', ';'):

return False

if code[0] in next_c_to_i:

return c_to_i[code[0]] == next_c_to_i[code[0]]

# Reading a file keeps trailing newlines, remove them.

cleaned_code_lines = [line.rstrip("\n") for line in code_lines_with_newlines]

# Deal with comments starting with ; and metacommands.

indent = 2

i = 0

end_found = False

while i < len(cleaned_code_lines):

code_line = cleaned_code_lines[i].lstrip()

if code_line.startswith(";"):

meta_line = code_line[1:].strip()

cleaned_code_lines.pop(i)

if meta_line.startswith("indent"):

try:

indent = int(meta_line.split()[1])

except ValueError:

print("Error: expected number after indent meta-command")

sys.exit(1)

elif meta_line.startswith("end"):

cleaned_code_lines = cleaned_code_lines[:i]

end_found = True

elif end_found:

cleaned_code_lines.pop(i)

else:

i += 1

indent_level = 0

for linenr, line in enumerate(cleaned_code_lines):

new_indent_level = 0

# Deal with indentation.

for _ in range(indent_level + 1):

# Allow an increase of at lost one indent level per line.

if line.startswith("\t"):

line = line[1:]

elif line.startswith(" " * indent):

line = line[indent:]

else:

break

new_indent_level += 1

# Detect in-line comments.

in_string = False

consecutive_spaces = 0

i = 0

while i < len(line):

char = line[i]

if in_string:

if char == "\"":

in_string = False

elif char == "\\":

i += 1 # Nothing after a backslash can close the string.

elif char == " ":

consecutive_spaces += 1

elif char == "\"":

consecutive_spaces = 0

in_string = True

elif char == "\\":

consecutive_spaces = 0

i += 1 # Skip one-character string.

else:

consecutive_spaces = 0

if consecutive_spaces == 2:

line = line[:i - 1]

break

i += 1

# If this line was non-empty after stripping inline comments, set the

# new indent level to this line, otherwise keep the old indent level.

if line.strip():

indent_level = new_indent_level

# Strip trailing whitespace, unless the line ends with

# an uneven amount of backslashes, then

# keep one trailing whitespace if present.

stripped_line = line.rstrip()

if (len(stripped_line) - len(stripped_line.rstrip("\\"))) % 2 == 1:

stripped_line = line[:len(stripped_line) + 1]

cleaned_code_lines[linenr] = stripped_line

global c_to_i

global replacements

global preps_used

old_stdout, old_stdin = sys.stdout, sys.stdin

sys.stdout = io.StringIO()

sys.stdin = io.StringIO(inp)

error = None

saved_env = c.deepcopy(environment)

saved_c_to_i = c.deepcopy(c_to_i)

saved_replacements = c.deepcopy(replacements)

preps_used = set()

try:

safe_mode_setting = False

exec(general_parse(code, safe_mode_setting), environment)

except SystemExit:

pass

except Exception as e:

error = e

for key in list(environment):

del environment[key]

for key in saved_env:

environment[key] = saved_env[key]

c_to_i = saved_c_to_i

replacements = saved_replacements

result = sys.stdout.getvalue()

sys.stdout = old_stdout

sys.stdin = old_stdin

output = ""

prompt = ">>> "

intro = """Welcome to the Pyth REPL.

Each input line will be compiled and executed, and the results of

each one will be passed into the next one's input stream.

"""

def __init__(self, debug_flag_on):

self.debug_on = debug_flag_on

cmd.Cmd.__init__(self)

def default(self, code):

global preps_used

old_stdout, old_stdin = sys.stdout, sys.stdin

sys.stdout = io.StringIO()

sys.stdin = io.StringIO(self.output)

preps_used = set()

pyth_code_gen = general_parse(code, False)

if self.debug_on:

print(pyth_code_gen, file=sys.stderr)

print('=' * 50, file=sys.stderr)

try:

exec(pyth_code_gen, environment)

except Exception:

traceback.print_exc()

self.output = sys.stdout.getvalue()

sys.stdout = old_stdout

sys.stdin = old_stdin

print(self.output, end="")

def do_EOF(self, line):

# Shut up, linter

return True or line or self

@property

@memoized

def docs(self): #Cache the docs so don't read multiple times

with open("rev-doc.txt") as doc_file:

docs_dict = {}

for line in doc_file.read().split("Tokens:\n")[1].split("\n")[:-1]:

token = (line.split(" ")[0] if not line.startswith(" ") else "space")

lines = docs_dict.get(token, [])

lines.append(line)

docs_dict[token] = lines

string_docs_dict = {}

for token, lines in docs_dict.items():

string_docs_dict[token] = '\n'.join(lines)

return string_docs_dict

def do_help(self, line):

if line:

print(self.docs.get(line, "%s is not a valid token" % line) if not

all(i in "123456789." for i in line) else self.docs["0123456789."])

else:

print("""This is the REPL for Pyth, an extremely concise language.

Use "help [token]" to get information about that token, or read rev-doc.txt""")

def postloop(self):

print()

def complete(self):