def get_part_chapters(title):
"""If [title] is the title of a part, returns a list of pairs of chapter
numbers and names."""
chapters = []
for part in book.TOC:
if title == part['name']:
for chapter in part['chapters']:
chapter_number = book.chapter_number(chapter['name'])
chapters.append([chapter_number, chapter['name']])
break
return chapters
def split_file(chapter_name, path, snippet=None, index=None):
chapter_number = book.chapter_number(chapter_name)
source_dir = book.get_language(chapter_name)
relative = os.path.relpath(path, source_dir)
directory = os.path.dirname(relative)
# Don't split the generated files.
if relative == "com/craftinginterpreters/lox/Expr.java": return
if relative == "com/craftinginterpreters/lox/Stmt.java": return
package = book.get_short_name(chapter_name)
if snippet:
package = os.path.join("snippets", package,
"{:02}-{}".format(index, snippet))
output_path = os.path.join("gen", package, relative)
# If we're generating the split for an entire chapter, include all its
# snippets.
if not snippet:
snippet = source_code.last_snippet_for_chapter(chapter_name).name
output = source_code.split_chapter(relative, chapter_name, snippet)
if output:
# Don't overwrite it if it didn't change, so the makefile doesn't think it
# was touched.
if os.path.exists(output_path):
with open(output_path, 'r') as file:
previous = file.read()
if output == previous: return
# Write the changed output.
ensure_dir(os.path.join("gen", package, directory))
with codecs.open(output_path, "w", encoding="utf-8") as out:
print(output_path)
out.write(output)
else:
# Remove it if it's supposed to be nonexistent.
if os.path.exists(output_path):
os.remove(output_path)
def load_file(source_code, source_dir, path):
relative = os.path.relpath(path, source_dir)
# Don't process the generated files. We only worry about GenerateAst.java.
if relative == "com/craftinginterpreters/lox/Expr.java": return
if relative == "com/craftinginterpreters/lox/Stmt.java": return
file = SourceFile(relative)
source_code.files.append(file)
line_num = 1
state = ParseState(None, None)
handled = False
function_before_block = None
current_function = None
current_class = None
def error(message):
print("Error: {} line {}: {}".format(relative, line_num, message),
file=sys.stderr)
source_code.errors[state.start.chapter].append("{} line {}: {}".format(
relative, line_num, message))
def push(chapter, name, end_chapter=None, end_name=None):
nonlocal state
nonlocal handled
start = source_code.find_snippet_tag(chapter, name)
end = None
if end_chapter:
end = source_code.find_snippet_tag(end_chapter, end_name)
state = ParseState(state, start, end)
handled = True
def pop():
nonlocal state
nonlocal handled
state = state.parent
handled = True
# Split the source file into chunks.
with open(path, 'r') as input:
for line in input:
line = line.rstrip()
handled = False
# See if we reached a new function or method declaration.
match = FUNCTION_PATTERN.search(line)
if match and match.group(1) not in KEYWORDS:
# Hack. Don't get caught by comments or string literals.
if '//' not in line and '"' not in line:
current_function = match.group(2)
match = CONSTRUCTOR_PATTERN.match(line)
if match:
current_function = match.group(1)
match = CLASS_PATTERN.match(line)
if match:
current_class = match.group(2)
match = BLOCK_PATTERN.match(line)
if match:
push(match.group(1), match.group(2), match.group(3),
match.group(4))
function_before_block = current_function
match = BLOCK_SNIPPET_PATTERN.match(line)
if match:
name = match.group(1)
push(state.start.chapter, state.start.name,
state.start.chapter, name)
function_before_block = current_function
if line.strip() == '*/' and state.end:
current_function = function_before_block
pop()
match = BEGIN_SNIPPET_PATTERN.match(line)
if match:
name = match.group(1)
tag = source_code.find_snippet_tag(state.start.chapter, name)
if tag < state.start:
error("Can't push earlier snippet {} from {}.".format(
name, state.start.name))
elif tag == state.start:
error("Can't push to same snippet {}.".format(name))
push(state.start.chapter, name)
match = END_SNIPPET_PATTERN.match(line)
if match:
name = match.group(1)
if name != state.start.name:
error("Expecting to pop {} but got {}.".format(
state.start.name, name))
if state.parent.start.chapter == None:
error('Cannot pop last state {}.'.format(state.start))
pop()
match = BEGIN_CHAPTER_PATTERN.match(line)
if match:
chapter = match.group(1)
name = match.group(2)
if state.start != None:
old_chapter = book.chapter_number(state.start.chapter)
new_chapter = book.chapter_number(chapter)
if chapter == state.start.chapter and name == state.start.name:
error('Pushing same snippet "{} {}"'.format(
chapter, name))
if chapter == state.start.chapter:
error(
'Pushing same chapter, just use "//>> {}"'.format(
name))
if new_chapter < old_chapter:
error('Can\'t push earlier chapter "{}" from "{}".'.
format(chapter, state.start.chapter))
push(chapter, name)
match = END_CHAPTER_PATTERN.match(line)
if match:
chapter = match.group(1)
name = match.group(2)
if chapter != state.start.chapter or name != state.start.name:
error('Expecting to pop "{} {}" but got "{} {}".'.format(
state.start.chapter, state.start.name, chapter, name))
if state.parent.start.chapter == None:
error('Cannot pop last state "{}".'.format(state.start))
pop()
if not handled:
if not state.start:
error("No snippet in effect.".format(relative))
source_line = SourceLine(line, current_function, current_class,
state.start, state.end)
file.lines.append(source_line)
# Hacky. Detect the end of the function or class. Assumes everything is
# nicely indented.
if path.endswith('.java') and line == ' }':
current_function = None
elif (path.endswith('.c') or path.endswith('.h')) and line == '}':
current_function = None
if path.endswith('.java') and line == '}':
current_class = None
line_num += 1
# ".parent.parent" because there is always the top "null" state.
if state.parent != None and state.parent.parent != None:
print("{}: Ended with more than one state on the stack.".format(
relative),
file=sys.stderr)
s = state
while s.parent != None:
print(" {}".format(s.start), file=sys.stderr)
s = s.parent
sys.exit(1)
def format_file(path, skip_up_to_date, dependencies_mod):
basename = os.path.basename(path)
basename = basename.split('.')[0]
output_path = "site/" + basename + ".html"
# See if the HTML is up to date.
if skip_up_to_date:
source_mod = max(os.path.getmtime(path), dependencies_mod)
dest_mod = os.path.getmtime(output_path)
if source_mod < dest_mod:
return
title = ''
title_html = ''
part = None
template_file = 'page'
errors = []
sections = []
header_index = 0
subheader_index = 0
has_challenges = False
design_note = None
snippets = None
# Read the markdown file and preprocess it.
contents = ''
with open(path, 'r') as input:
# Read each line, preprocessing the special codes.
for line in input:
stripped = line.lstrip()
indentation = line[:len(line) - len(stripped)]
if line.startswith('^'):
command, _, arg = stripped.rstrip('\n').lstrip('^').partition(
' ')
arg = arg.strip()
if command == 'title':
title = arg
title_html = title
# Remove any discretionary hyphens from the title.
title = title.replace('­', '')
# Load the code snippets now that we know the title.
snippets = source_code.find_all(title)
# If there were any errors loading the code, include them.
if title in book.CODE_CHAPTERS:
errors.extend(source_code.errors[title])
elif command == 'part':
part = arg
elif command == 'template':
template_file = arg
elif command == 'code':
contents = insert_snippet(snippets, arg, contents, errors)
else:
raise Exception('Unknown command "^{} {}"'.format(
command, arg))
elif stripped.startswith('## Challenges'):
has_challenges = True
contents += '<h2><a href="#challenges" name="challenges">Challenges</a></h2>\n'
elif stripped.startswith('## Design Note:'):
has_design_note = True
design_note = stripped[len('## Design Note:') + 1:]
contents += '<h2><a href="#design-note" name="design-note">Design Note: {}</a></h2>\n'.format(
design_note)
elif stripped.startswith('# ') or stripped.startswith(
'## ') or stripped.startswith('### '):
# Build the section navigation from the headers.
index = stripped.find(" ")
header_type = stripped[:index]
header = pretty(stripped[index:].strip())
anchor = book.get_file_name(header)
anchor = re.sub(r'[.?!:/"]', '', anchor)
# Add an anchor to the header.
contents += indentation + header_type
if len(header_type) == 2:
header_index += 1
subheader_index = 0
page_number = book.chapter_number(title)
number = '{0} . {1}'.format(
page_number, header_index)
elif len(header_type) == 3:
subheader_index += 1
page_number = book.chapter_number(title)
number = '{0} . {1} . {2}'.format(
page_number, header_index, subheader_index)
header_line = '<a href="#{0}" name="{0}"><small>{1}</small> {2}</a>\n'.format(
anchor, number, header)
contents += header_line
# Build the section navigation.
if len(header_type) == 2:
sections.append([header_index, header])
else:
contents += pretty(line)
# Validate that every snippet for the chapter is included.
for name, snippet in snippets.items():
if name != 'not-yet' and name != 'omit' and snippet != False:
errors.append("Unused snippet {}".format(name))
# Show any errors at the top of the file.
if errors:
error_markdown = ""
for error in errors:
error_markdown += "**Error: {}**\n\n".format(error)
contents = error_markdown + contents
# Fix up em dashes. We do this on the entire contents instead of in pretty()
# so that we can handle surrounding whitespace even when the "--" is at the
# beginning of end of a line in Markdown.
contents = EM_DASH_PATTERN.sub('<span class="em">—</span>', contents)
# Allow processing markdown inside some tags.
contents = contents.replace('<aside', '<aside markdown="1"')
contents = contents.replace('<div class="challenges">',
'<div class="challenges" markdown="1">')
contents = contents.replace('<div class="design-note">',
'<div class="design-note" markdown="1">')
body = markdown.markdown(contents, ['extra', 'codehilite', 'smarty'])
# Turn aside markers in code into spans.
# <span class="c1">// [repl]</span>
body = ASIDE_COMMENT_PATTERN.sub(r'<span name="\1"></span>', body)
body = ASIDE_WITH_COMMENT_PATTERN.sub(
r'<span class="c1" name="\2">// \1</span>', body)
up = 'Table of Contents'
if part:
up = part
elif title == 'Table of Contents':
up = 'Crafting Interpreters'
data = {
'title': title,
'part': part,
'body': body,
'sections': sections,
'chapters': get_part_chapters(title),
'design_note': design_note,
'has_challenges': has_challenges,
'number': book.chapter_number(title),
'prev': book.adjacent_page(title, -1),
'prev_type': book.adjacent_type(title, -1),
'next': book.adjacent_page(title, 1),
'next_type': book.adjacent_type(title, 1),
'up': up,
'toc': book.TOC
}
template = environment.get_template(template_file + '.html')
output = template.render(data)
# Write the output.
with codecs.open(output_path, "w", encoding="utf-8") as out:
out.write(output)
global total_words
global num_chapters
global empty_chapters
word_count = len(contents.split(None))
num = book.chapter_number(title)
if num:
num = '{}. '.format(num)
# Non-chapter pages aren't counted like regular chapters.
if part:
num_chapters += 1
if word_count < 50:
empty_chapters += 1
print(" {}{}{}{}".format(GRAY, num, title, DEFAULT))
elif word_count < 2000:
empty_chapters += 1
print(" {}-{} {}{} ({} words)".format(YELLOW, DEFAULT, num, title,
word_count))
else:
total_words += word_count
print(" {}✓{} {}{} ({} words)".format(GREEN, DEFAULT, num, title,
word_count))
elif title in ["Crafting Interpreters", "Table of Contents"]:
print("{}•{} {}{}".format(GREEN, DEFAULT, num, title))
else:
if word_count < 50:
print(" {}{}{}{}".format(GRAY, num, title, DEFAULT))
else:
print("{}✓{} {}{} ({} words)".format(GREEN, DEFAULT, num, title,
word_count))
def load_file(source_code, source_dir, path):
relative = os.path.relpath(path, source_dir)
file = SourceFile(relative)
source_code.files.append(file)
line_num = 1
state = ParseState(None, None)
handled = False
current_location = Location(None, 'file', file.nice_path())
location_before_block = None
def error(message):
print("Error: {} line {}: {}".format(relative, line_num, message),
file=sys.stderr)
source_code.errors[state.start.chapter].append("{} line {}: {}".format(
relative, line_num, message))
def push(chapter, name, end_chapter=None, end_name=None):
nonlocal state
nonlocal handled
start = source_code.find_snippet_tag(chapter, name)
end = None
if end_chapter:
end = source_code.find_snippet_tag(end_chapter, end_name)
state = ParseState(state, start, end)
handled = True
def pop():
nonlocal state
nonlocal handled
state = state.parent
handled = True
# Split the source file into chunks.
with open(path, 'r') as input:
lines = input.read().splitlines()
printed_file = False
line_num = 1
for line in lines:
line = line.rstrip()
handled = False
# Report any lines that are too long.
trimmed = re.sub(r'// \[([-a-z0-9]+)\]', '', line)
if len(trimmed) > 72 and not '/*' in trimmed:
if not printed_file:
print("Long line in {}:".format(file.path))
printed_file = True
print("{0:4} ({1:2} chars): {2}".format(
line_num, len(trimmed), trimmed))
# See if we reached a new function or method declaration.
match = FUNCTION_PATTERN.search(line)
is_function_declaration = False
if match and "#define" not in line and match.group(
1) not in KEYWORDS:
# Hack. Don't get caught by comments or string literals.
if '//' not in line and '"' not in line:
current_location = Location(
current_location, 'method'
if file.path.endswith('.java') else 'function',
match.group(2))
# TODO: What about declarations with aside comments:
# void foo(); // [wat]
is_function_declaration = line.endswith(';')
match = CONSTRUCTOR_PATTERN.match(line)
if match:
current_location = Location(current_location, 'constructor',
match.group(1))
match = TYPE_PATTERN.search(line)
if match:
# Hack. Don't get caught by comments or string literals.
if '//' not in line and '"' not in line:
current_location = Location(current_location,
match.group(3), match.group(4))
match = TYPEDEF_PATTERN.match(line)
if match:
# We don't know the name of the typedef.
current_location = Location(current_location, match.group(1),
'???')
match = BLOCK_PATTERN.match(line)
if match:
push(match.group(1), match.group(2), match.group(3),
match.group(4))
location_before_block = current_location
match = BLOCK_SNIPPET_PATTERN.match(line)
if match:
name = match.group(1)
push(state.start.chapter, state.start.name,
state.start.chapter, name)
location_before_block = current_location
if line.strip() == '*/' and state.end:
current_location = location_before_block
pop()
match = BEGIN_SNIPPET_PATTERN.match(line)
if match:
name = match.group(1)
tag = source_code.find_snippet_tag(state.start.chapter, name)
if tag < state.start:
error("Can't push earlier snippet {} from {}.".format(
name, state.start.name))
elif tag == state.start:
error("Can't push to same snippet {}.".format(name))
push(state.start.chapter, name)
match = END_SNIPPET_PATTERN.match(line)
if match:
name = match.group(1)
if name != state.start.name:
error("Expecting to pop {} but got {}.".format(
state.start.name, name))
if state.parent.start.chapter == None:
error('Cannot pop last state {}.'.format(state.start))
pop()
match = BEGIN_CHAPTER_PATTERN.match(line)
if match:
chapter = match.group(1)
name = match.group(2)
if state.start != None:
old_chapter = book.chapter_number(state.start.chapter)
new_chapter = book.chapter_number(chapter)
if chapter == state.start.chapter and name == state.start.name:
error('Pushing same snippet "{} {}"'.format(
chapter, name))
if chapter == state.start.chapter:
error(
'Pushing same chapter, just use "//>> {}"'.format(
name))
if new_chapter < old_chapter:
error('Can\'t push earlier chapter "{}" from "{}".'.
format(chapter, state.start.chapter))
push(chapter, name)
match = END_CHAPTER_PATTERN.match(line)
if match:
chapter = match.group(1)
name = match.group(2)
if chapter != state.start.chapter or name != state.start.name:
error('Expecting to pop "{} {}" but got "{} {}".'.format(
state.start.chapter, state.start.name, chapter, name))
if state.start.chapter == None:
error('Cannot pop last state "{}".'.format(state.start))
pop()
if not handled:
if not state.start:
error("No snippet in effect.".format(relative))
source_line = SourceLine(line, current_location, state.start,
state.end)
file.lines.append(source_line)
match = TYPEDEF_NAME_PATTERN.match(line)
if match:
# Now we know the typedef name.
current_location.name = match.group(1)
current_location = current_location.parent
# Use "startswith" to include lines like "} [aside-marker]".
# TODO: Hacky. Generalize?
if line.startswith('}'):
current_location = current_location.pop_to_depth(0)
elif line.startswith(' }'):
current_location = current_location.pop_to_depth(1)
elif line.startswith(' }'):
current_location = current_location.pop_to_depth(2)
# If we reached a function declaration, not a definition, then it's done
# after one line.
if is_function_declaration:
current_location = current_location.parent
# Hack. There is a one-line class in Parser.java.
if 'class ParseError' in line:
current_location = current_location.parent
line_num += 1
# ".parent.parent" because there is always the top "null" state.
if state.parent != None and state.parent.parent != None:
print("{}: Ended with more than one state on the stack.".format(
relative),
file=sys.stderr)
s = state
while s.parent != None:
print(" {}".format(s.start), file=sys.stderr)
s = s.parent
sys.exit(1)
def load_file(source_code, source_dir, path):
relative = os.path.relpath(path, source_dir)
file = SourceFile(relative)
source_code.files.append(file)
line_num = 1
state = ParseState(None, None)
handled = False
# The name of a typedef appears after its body, but we need to know it while
# we're creating SourceLines for the body. So we do a separate pass to find
# the name for each typedef and store them here.
typedef_starts = {}
function_before_block = None
current_function = None
current_kind = None
current_type = None
nested_class = None
def error(message):
print("Error: {} line {}: {}".format(relative, line_num, message),
file=sys.stderr)
source_code.errors[state.start.chapter].append(
"{} line {}: {}".format(relative, line_num, message))
def push(chapter, name, end_chapter=None, end_name=None):
nonlocal state
nonlocal handled
start = source_code.find_snippet_tag(chapter, name)
end = None
if end_chapter:
end = source_code.find_snippet_tag(end_chapter, end_name)
state = ParseState(state, start, end)
handled = True
def pop():
nonlocal state
nonlocal handled
state = state.parent
handled = True
# Split the source file into chunks.
with open(path, 'r') as input:
lines = input.read().splitlines()
# Find the names for each struct typedef.
typedef_start_line = None
typedef_kind = None
for line in lines:
line = line.rstrip()
match = TYPEDEF_PATTERN.match(line)
if match:
typedef_kind = match.group(1)
typedef_start_line = line_num
match = TYPEDEF_NAME_PATTERN.match(line)
if match:
typedef_starts[typedef_start_line] = [typedef_kind, match.group(1)]
line_num += 1
line_num = 1
for line in lines:
line = line.rstrip()
handled = False
# See if we reached a new function or method declaration.
match = FUNCTION_PATTERN.search(line)
if match and "#define" not in line and match.group(1) not in KEYWORDS:
# Hack. Don't get caught by comments or string literals.
if '//' not in line and '"' not in line:
current_function = match.group(2)
match = CONSTRUCTOR_PATTERN.match(line)
if match:
current_function = match.group(1)
match = CLASS_PATTERN.match(line)
if match:
current_kind = match.group(3)
current_type = match.group(4)
match = TYPEDEF_PATTERN.match(line)
if match:
typedef = typedef_starts[line_num]
current_kind = typedef[0]
current_type = typedef[1]
match = TYPEDEF_NAME_PATTERN.match(line)
if match:
current_kind = None
current_type = None
match = NESTED_CLASS_PATTERN.match(line)
if match:
nested_class = match.group(1)
match = BLOCK_PATTERN.match(line)
if match:
push(match.group(1), match.group(2), match.group(3), match.group(4))
function_before_block = current_function
match = BLOCK_SNIPPET_PATTERN.match(line)
if match:
name = match.group(1)
push(state.start.chapter, state.start.name, state.start.chapter, name)
function_before_block = current_function
if line.strip() == '*/' and state.end:
current_function = function_before_block
pop()
match = BEGIN_SNIPPET_PATTERN.match(line)
if match:
name = match.group(1)
tag = source_code.find_snippet_tag(state.start.chapter, name)
if tag < state.start:
error("Can't push earlier snippet {} from {}.".format(name, state.start.name))
elif tag == state.start:
error("Can't push to same snippet {}.".format(name))
push(state.start.chapter, name)
match = END_SNIPPET_PATTERN.match(line)
if match:
name = match.group(1)
if name != state.start.name:
error("Expecting to pop {} but got {}.".format(state.start.name, name))
if state.parent.start.chapter == None:
error('Cannot pop last state {}.'.format(state.start))
pop()
match = BEGIN_CHAPTER_PATTERN.match(line)
if match:
chapter = match.group(1)
name = match.group(2)
if state.start != None:
old_chapter = book.chapter_number(state.start.chapter)
new_chapter = book.chapter_number(chapter)
if chapter == state.start.chapter and name == state.start.name:
error('Pushing same snippet "{} {}"'.format(chapter, name))
if chapter == state.start.chapter:
error('Pushing same chapter, just use "//>> {}"'.format(name))
if new_chapter < old_chapter:
error('Can\'t push earlier chapter "{}" from "{}".'.format(
chapter, state.start.chapter))
push(chapter, name)
match = END_CHAPTER_PATTERN.match(line)
if match:
chapter = match.group(1)
name = match.group(2)
if chapter != state.start.chapter or name != state.start.name:
error('Expecting to pop "{} {}" but got "{} {}".'.format(
state.start.chapter, state.start.name, chapter, name))
if state.start.chapter == None:
error('Cannot pop last state "{}".'.format(state.start))
pop()
if not handled:
if not state.start:
error("No snippet in effect.".format(relative))
source_line = SourceLine(line, current_function, current_type,
current_kind, nested_class, state.start, state.end)
file.lines.append(source_line)
# Hacky. Detect the end of the function or class. Assumes everything is
# nicely indented.
if path.endswith('.java') and line == ' }':
if nested_class:
nested_class = None
else:
current_function = None
elif (path.endswith('.c') or path.endswith('.h')) and line == '}':
current_function = None
if path.endswith('.java') and line == '}':
current_kind = None
current_type = None
line_num += 1
# ".parent.parent" because there is always the top "null" state.
if state.parent != None and state.parent.parent != None:
print("{}: Ended with more than one state on the stack.".format(relative),
file=sys.stderr)
s = state
while s.parent != None:
print(" {}".format(s.start), file=sys.stderr)
s = s.parent
sys.exit(1)
