def get_part_chapters(title): """If [title] is the title of a part, returns a list of pairs of chapter numbers and names.""" chapters = [] for part in book.TOC: if title == part['name']: for chapter in part['chapters']: chapter_number = book.chapter_number(chapter['name']) chapters.append([chapter_number, chapter['name']]) break return chapters
def split_file(chapter_name, path, snippet=None, index=None): chapter_number = book.chapter_number(chapter_name) source_dir = book.get_language(chapter_name) relative = os.path.relpath(path, source_dir) directory = os.path.dirname(relative) # Don't split the generated files. if relative == "com/craftinginterpreters/lox/Expr.java": return if relative == "com/craftinginterpreters/lox/Stmt.java": return package = book.get_short_name(chapter_name) if snippet: package = os.path.join("snippets", package, "{:02}-{}".format(index, snippet)) output_path = os.path.join("gen", package, relative) # If we're generating the split for an entire chapter, include all its # snippets. if not snippet: snippet = source_code.last_snippet_for_chapter(chapter_name).name output = source_code.split_chapter(relative, chapter_name, snippet) if output: # Don't overwrite it if it didn't change, so the makefile doesn't think it # was touched. if os.path.exists(output_path): with open(output_path, 'r') as file: previous = file.read() if output == previous: return # Write the changed output. ensure_dir(os.path.join("gen", package, directory)) with codecs.open(output_path, "w", encoding="utf-8") as out: print(output_path) out.write(output) else: # Remove it if it's supposed to be nonexistent. if os.path.exists(output_path): os.remove(output_path)
def load_file(source_code, source_dir, path): relative = os.path.relpath(path, source_dir) # Don't process the generated files. We only worry about GenerateAst.java. if relative == "com/craftinginterpreters/lox/Expr.java": return if relative == "com/craftinginterpreters/lox/Stmt.java": return file = SourceFile(relative) source_code.files.append(file) line_num = 1 state = ParseState(None, None) handled = False function_before_block = None current_function = None current_class = None def error(message): print("Error: {} line {}: {}".format(relative, line_num, message), file=sys.stderr) source_code.errors[state.start.chapter].append("{} line {}: {}".format( relative, line_num, message)) def push(chapter, name, end_chapter=None, end_name=None): nonlocal state nonlocal handled start = source_code.find_snippet_tag(chapter, name) end = None if end_chapter: end = source_code.find_snippet_tag(end_chapter, end_name) state = ParseState(state, start, end) handled = True def pop(): nonlocal state nonlocal handled state = state.parent handled = True # Split the source file into chunks. with open(path, 'r') as input: for line in input: line = line.rstrip() handled = False # See if we reached a new function or method declaration. match = FUNCTION_PATTERN.search(line) if match and match.group(1) not in KEYWORDS: # Hack. Don't get caught by comments or string literals. if '//' not in line and '"' not in line: current_function = match.group(2) match = CONSTRUCTOR_PATTERN.match(line) if match: current_function = match.group(1) match = CLASS_PATTERN.match(line) if match: current_class = match.group(2) match = BLOCK_PATTERN.match(line) if match: push(match.group(1), match.group(2), match.group(3), match.group(4)) function_before_block = current_function match = BLOCK_SNIPPET_PATTERN.match(line) if match: name = match.group(1) push(state.start.chapter, state.start.name, state.start.chapter, name) function_before_block = current_function if line.strip() == '*/' and state.end: current_function = function_before_block pop() match = BEGIN_SNIPPET_PATTERN.match(line) if match: name = match.group(1) tag = source_code.find_snippet_tag(state.start.chapter, name) if tag < state.start: error("Can't push earlier snippet {} from {}.".format( name, state.start.name)) elif tag == state.start: error("Can't push to same snippet {}.".format(name)) push(state.start.chapter, name) match = END_SNIPPET_PATTERN.match(line) if match: name = match.group(1) if name != state.start.name: error("Expecting to pop {} but got {}.".format( state.start.name, name)) if state.parent.start.chapter == None: error('Cannot pop last state {}.'.format(state.start)) pop() match = BEGIN_CHAPTER_PATTERN.match(line) if match: chapter = match.group(1) name = match.group(2) if state.start != None: old_chapter = book.chapter_number(state.start.chapter) new_chapter = book.chapter_number(chapter) if chapter == state.start.chapter and name == state.start.name: error('Pushing same snippet "{} {}"'.format( chapter, name)) if chapter == state.start.chapter: error( 'Pushing same chapter, just use "//>> {}"'.format( name)) if new_chapter < old_chapter: error('Can\'t push earlier chapter "{}" from "{}".'. format(chapter, state.start.chapter)) push(chapter, name) match = END_CHAPTER_PATTERN.match(line) if match: chapter = match.group(1) name = match.group(2) if chapter != state.start.chapter or name != state.start.name: error('Expecting to pop "{} {}" but got "{} {}".'.format( state.start.chapter, state.start.name, chapter, name)) if state.parent.start.chapter == None: error('Cannot pop last state "{}".'.format(state.start)) pop() if not handled: if not state.start: error("No snippet in effect.".format(relative)) source_line = SourceLine(line, current_function, current_class, state.start, state.end) file.lines.append(source_line) # Hacky. Detect the end of the function or class. Assumes everything is # nicely indented. if path.endswith('.java') and line == ' }': current_function = None elif (path.endswith('.c') or path.endswith('.h')) and line == '}': current_function = None if path.endswith('.java') and line == '}': current_class = None line_num += 1 # ".parent.parent" because there is always the top "null" state. if state.parent != None and state.parent.parent != None: print("{}: Ended with more than one state on the stack.".format( relative), file=sys.stderr) s = state while s.parent != None: print(" {}".format(s.start), file=sys.stderr) s = s.parent sys.exit(1)
def format_file(path, skip_up_to_date, dependencies_mod): basename = os.path.basename(path) basename = basename.split('.')[0] output_path = "site/" + basename + ".html" # See if the HTML is up to date. if skip_up_to_date: source_mod = max(os.path.getmtime(path), dependencies_mod) dest_mod = os.path.getmtime(output_path) if source_mod < dest_mod: return title = '' title_html = '' part = None template_file = 'page' errors = [] sections = [] header_index = 0 subheader_index = 0 has_challenges = False design_note = None snippets = None # Read the markdown file and preprocess it. contents = '' with open(path, 'r') as input: # Read each line, preprocessing the special codes. for line in input: stripped = line.lstrip() indentation = line[:len(line) - len(stripped)] if line.startswith('^'): command, _, arg = stripped.rstrip('\n').lstrip('^').partition( ' ') arg = arg.strip() if command == 'title': title = arg title_html = title # Remove any discretionary hyphens from the title. title = title.replace('­', '') # Load the code snippets now that we know the title. snippets = source_code.find_all(title) # If there were any errors loading the code, include them. if title in book.CODE_CHAPTERS: errors.extend(source_code.errors[title]) elif command == 'part': part = arg elif command == 'template': template_file = arg elif command == 'code': contents = insert_snippet(snippets, arg, contents, errors) else: raise Exception('Unknown command "^{} {}"'.format( command, arg)) elif stripped.startswith('## Challenges'): has_challenges = True contents += '<h2><a href="#challenges" name="challenges">Challenges</a></h2>\n' elif stripped.startswith('## Design Note:'): has_design_note = True design_note = stripped[len('## Design Note:') + 1:] contents += '<h2><a href="#design-note" name="design-note">Design Note: {}</a></h2>\n'.format( design_note) elif stripped.startswith('# ') or stripped.startswith( '## ') or stripped.startswith('### '): # Build the section navigation from the headers. index = stripped.find(" ") header_type = stripped[:index] header = pretty(stripped[index:].strip()) anchor = book.get_file_name(header) anchor = re.sub(r'[.?!:/"]', '', anchor) # Add an anchor to the header. contents += indentation + header_type if len(header_type) == 2: header_index += 1 subheader_index = 0 page_number = book.chapter_number(title) number = '{0} . {1}'.format( page_number, header_index) elif len(header_type) == 3: subheader_index += 1 page_number = book.chapter_number(title) number = '{0} . {1} . {2}'.format( page_number, header_index, subheader_index) header_line = '<a href="#{0}" name="{0}"><small>{1}</small> {2}</a>\n'.format( anchor, number, header) contents += header_line # Build the section navigation. if len(header_type) == 2: sections.append([header_index, header]) else: contents += pretty(line) # Validate that every snippet for the chapter is included. for name, snippet in snippets.items(): if name != 'not-yet' and name != 'omit' and snippet != False: errors.append("Unused snippet {}".format(name)) # Show any errors at the top of the file. if errors: error_markdown = "" for error in errors: error_markdown += "**Error: {}**\n\n".format(error) contents = error_markdown + contents # Fix up em dashes. We do this on the entire contents instead of in pretty() # so that we can handle surrounding whitespace even when the "--" is at the # beginning of end of a line in Markdown. contents = EM_DASH_PATTERN.sub('<span class="em">—</span>', contents) # Allow processing markdown inside some tags. contents = contents.replace('<aside', '<aside markdown="1"') contents = contents.replace('<div class="challenges">', '<div class="challenges" markdown="1">') contents = contents.replace('<div class="design-note">', '<div class="design-note" markdown="1">') body = markdown.markdown(contents, ['extra', 'codehilite', 'smarty']) # Turn aside markers in code into spans. # <span class="c1">// [repl]</span> body = ASIDE_COMMENT_PATTERN.sub(r'<span name="\1"></span>', body) body = ASIDE_WITH_COMMENT_PATTERN.sub( r'<span class="c1" name="\2">// \1</span>', body) up = 'Table of Contents' if part: up = part elif title == 'Table of Contents': up = 'Crafting Interpreters' data = { 'title': title, 'part': part, 'body': body, 'sections': sections, 'chapters': get_part_chapters(title), 'design_note': design_note, 'has_challenges': has_challenges, 'number': book.chapter_number(title), 'prev': book.adjacent_page(title, -1), 'prev_type': book.adjacent_type(title, -1), 'next': book.adjacent_page(title, 1), 'next_type': book.adjacent_type(title, 1), 'up': up, 'toc': book.TOC } template = environment.get_template(template_file + '.html') output = template.render(data) # Write the output. with codecs.open(output_path, "w", encoding="utf-8") as out: out.write(output) global total_words global num_chapters global empty_chapters word_count = len(contents.split(None)) num = book.chapter_number(title) if num: num = '{}. '.format(num) # Non-chapter pages aren't counted like regular chapters. if part: num_chapters += 1 if word_count < 50: empty_chapters += 1 print(" {}{}{}{}".format(GRAY, num, title, DEFAULT)) elif word_count < 2000: empty_chapters += 1 print(" {}-{} {}{} ({} words)".format(YELLOW, DEFAULT, num, title, word_count)) else: total_words += word_count print(" {}✓{} {}{} ({} words)".format(GREEN, DEFAULT, num, title, word_count)) elif title in ["Crafting Interpreters", "Table of Contents"]: print("{}•{} {}{}".format(GREEN, DEFAULT, num, title)) else: if word_count < 50: print(" {}{}{}{}".format(GRAY, num, title, DEFAULT)) else: print("{}✓{} {}{} ({} words)".format(GREEN, DEFAULT, num, title, word_count))
def load_file(source_code, source_dir, path): relative = os.path.relpath(path, source_dir) file = SourceFile(relative) source_code.files.append(file) line_num = 1 state = ParseState(None, None) handled = False current_location = Location(None, 'file', file.nice_path()) location_before_block = None def error(message): print("Error: {} line {}: {}".format(relative, line_num, message), file=sys.stderr) source_code.errors[state.start.chapter].append("{} line {}: {}".format( relative, line_num, message)) def push(chapter, name, end_chapter=None, end_name=None): nonlocal state nonlocal handled start = source_code.find_snippet_tag(chapter, name) end = None if end_chapter: end = source_code.find_snippet_tag(end_chapter, end_name) state = ParseState(state, start, end) handled = True def pop(): nonlocal state nonlocal handled state = state.parent handled = True # Split the source file into chunks. with open(path, 'r') as input: lines = input.read().splitlines() printed_file = False line_num = 1 for line in lines: line = line.rstrip() handled = False # Report any lines that are too long. trimmed = re.sub(r'// \[([-a-z0-9]+)\]', '', line) if len(trimmed) > 72 and not '/*' in trimmed: if not printed_file: print("Long line in {}:".format(file.path)) printed_file = True print("{0:4} ({1:2} chars): {2}".format( line_num, len(trimmed), trimmed)) # See if we reached a new function or method declaration. match = FUNCTION_PATTERN.search(line) is_function_declaration = False if match and "#define" not in line and match.group( 1) not in KEYWORDS: # Hack. Don't get caught by comments or string literals. if '//' not in line and '"' not in line: current_location = Location( current_location, 'method' if file.path.endswith('.java') else 'function', match.group(2)) # TODO: What about declarations with aside comments: # void foo(); // [wat] is_function_declaration = line.endswith(';') match = CONSTRUCTOR_PATTERN.match(line) if match: current_location = Location(current_location, 'constructor', match.group(1)) match = TYPE_PATTERN.search(line) if match: # Hack. Don't get caught by comments or string literals. if '//' not in line and '"' not in line: current_location = Location(current_location, match.group(3), match.group(4)) match = TYPEDEF_PATTERN.match(line) if match: # We don't know the name of the typedef. current_location = Location(current_location, match.group(1), '???') match = BLOCK_PATTERN.match(line) if match: push(match.group(1), match.group(2), match.group(3), match.group(4)) location_before_block = current_location match = BLOCK_SNIPPET_PATTERN.match(line) if match: name = match.group(1) push(state.start.chapter, state.start.name, state.start.chapter, name) location_before_block = current_location if line.strip() == '*/' and state.end: current_location = location_before_block pop() match = BEGIN_SNIPPET_PATTERN.match(line) if match: name = match.group(1) tag = source_code.find_snippet_tag(state.start.chapter, name) if tag < state.start: error("Can't push earlier snippet {} from {}.".format( name, state.start.name)) elif tag == state.start: error("Can't push to same snippet {}.".format(name)) push(state.start.chapter, name) match = END_SNIPPET_PATTERN.match(line) if match: name = match.group(1) if name != state.start.name: error("Expecting to pop {} but got {}.".format( state.start.name, name)) if state.parent.start.chapter == None: error('Cannot pop last state {}.'.format(state.start)) pop() match = BEGIN_CHAPTER_PATTERN.match(line) if match: chapter = match.group(1) name = match.group(2) if state.start != None: old_chapter = book.chapter_number(state.start.chapter) new_chapter = book.chapter_number(chapter) if chapter == state.start.chapter and name == state.start.name: error('Pushing same snippet "{} {}"'.format( chapter, name)) if chapter == state.start.chapter: error( 'Pushing same chapter, just use "//>> {}"'.format( name)) if new_chapter < old_chapter: error('Can\'t push earlier chapter "{}" from "{}".'. format(chapter, state.start.chapter)) push(chapter, name) match = END_CHAPTER_PATTERN.match(line) if match: chapter = match.group(1) name = match.group(2) if chapter != state.start.chapter or name != state.start.name: error('Expecting to pop "{} {}" but got "{} {}".'.format( state.start.chapter, state.start.name, chapter, name)) if state.start.chapter == None: error('Cannot pop last state "{}".'.format(state.start)) pop() if not handled: if not state.start: error("No snippet in effect.".format(relative)) source_line = SourceLine(line, current_location, state.start, state.end) file.lines.append(source_line) match = TYPEDEF_NAME_PATTERN.match(line) if match: # Now we know the typedef name. current_location.name = match.group(1) current_location = current_location.parent # Use "startswith" to include lines like "} [aside-marker]". # TODO: Hacky. Generalize? if line.startswith('}'): current_location = current_location.pop_to_depth(0) elif line.startswith(' }'): current_location = current_location.pop_to_depth(1) elif line.startswith(' }'): current_location = current_location.pop_to_depth(2) # If we reached a function declaration, not a definition, then it's done # after one line. if is_function_declaration: current_location = current_location.parent # Hack. There is a one-line class in Parser.java. if 'class ParseError' in line: current_location = current_location.parent line_num += 1 # ".parent.parent" because there is always the top "null" state. if state.parent != None and state.parent.parent != None: print("{}: Ended with more than one state on the stack.".format( relative), file=sys.stderr) s = state while s.parent != None: print(" {}".format(s.start), file=sys.stderr) s = s.parent sys.exit(1)
def load_file(source_code, source_dir, path): relative = os.path.relpath(path, source_dir) file = SourceFile(relative) source_code.files.append(file) line_num = 1 state = ParseState(None, None) handled = False # The name of a typedef appears after its body, but we need to know it while # we're creating SourceLines for the body. So we do a separate pass to find # the name for each typedef and store them here. typedef_starts = {} function_before_block = None current_function = None current_kind = None current_type = None nested_class = None def error(message): print("Error: {} line {}: {}".format(relative, line_num, message), file=sys.stderr) source_code.errors[state.start.chapter].append( "{} line {}: {}".format(relative, line_num, message)) def push(chapter, name, end_chapter=None, end_name=None): nonlocal state nonlocal handled start = source_code.find_snippet_tag(chapter, name) end = None if end_chapter: end = source_code.find_snippet_tag(end_chapter, end_name) state = ParseState(state, start, end) handled = True def pop(): nonlocal state nonlocal handled state = state.parent handled = True # Split the source file into chunks. with open(path, 'r') as input: lines = input.read().splitlines() # Find the names for each struct typedef. typedef_start_line = None typedef_kind = None for line in lines: line = line.rstrip() match = TYPEDEF_PATTERN.match(line) if match: typedef_kind = match.group(1) typedef_start_line = line_num match = TYPEDEF_NAME_PATTERN.match(line) if match: typedef_starts[typedef_start_line] = [typedef_kind, match.group(1)] line_num += 1 line_num = 1 for line in lines: line = line.rstrip() handled = False # See if we reached a new function or method declaration. match = FUNCTION_PATTERN.search(line) if match and "#define" not in line and match.group(1) not in KEYWORDS: # Hack. Don't get caught by comments or string literals. if '//' not in line and '"' not in line: current_function = match.group(2) match = CONSTRUCTOR_PATTERN.match(line) if match: current_function = match.group(1) match = CLASS_PATTERN.match(line) if match: current_kind = match.group(3) current_type = match.group(4) match = TYPEDEF_PATTERN.match(line) if match: typedef = typedef_starts[line_num] current_kind = typedef[0] current_type = typedef[1] match = TYPEDEF_NAME_PATTERN.match(line) if match: current_kind = None current_type = None match = NESTED_CLASS_PATTERN.match(line) if match: nested_class = match.group(1) match = BLOCK_PATTERN.match(line) if match: push(match.group(1), match.group(2), match.group(3), match.group(4)) function_before_block = current_function match = BLOCK_SNIPPET_PATTERN.match(line) if match: name = match.group(1) push(state.start.chapter, state.start.name, state.start.chapter, name) function_before_block = current_function if line.strip() == '*/' and state.end: current_function = function_before_block pop() match = BEGIN_SNIPPET_PATTERN.match(line) if match: name = match.group(1) tag = source_code.find_snippet_tag(state.start.chapter, name) if tag < state.start: error("Can't push earlier snippet {} from {}.".format(name, state.start.name)) elif tag == state.start: error("Can't push to same snippet {}.".format(name)) push(state.start.chapter, name) match = END_SNIPPET_PATTERN.match(line) if match: name = match.group(1) if name != state.start.name: error("Expecting to pop {} but got {}.".format(state.start.name, name)) if state.parent.start.chapter == None: error('Cannot pop last state {}.'.format(state.start)) pop() match = BEGIN_CHAPTER_PATTERN.match(line) if match: chapter = match.group(1) name = match.group(2) if state.start != None: old_chapter = book.chapter_number(state.start.chapter) new_chapter = book.chapter_number(chapter) if chapter == state.start.chapter and name == state.start.name: error('Pushing same snippet "{} {}"'.format(chapter, name)) if chapter == state.start.chapter: error('Pushing same chapter, just use "//>> {}"'.format(name)) if new_chapter < old_chapter: error('Can\'t push earlier chapter "{}" from "{}".'.format( chapter, state.start.chapter)) push(chapter, name) match = END_CHAPTER_PATTERN.match(line) if match: chapter = match.group(1) name = match.group(2) if chapter != state.start.chapter or name != state.start.name: error('Expecting to pop "{} {}" but got "{} {}".'.format( state.start.chapter, state.start.name, chapter, name)) if state.start.chapter == None: error('Cannot pop last state "{}".'.format(state.start)) pop() if not handled: if not state.start: error("No snippet in effect.".format(relative)) source_line = SourceLine(line, current_function, current_type, current_kind, nested_class, state.start, state.end) file.lines.append(source_line) # Hacky. Detect the end of the function or class. Assumes everything is # nicely indented. if path.endswith('.java') and line == ' }': if nested_class: nested_class = None else: current_function = None elif (path.endswith('.c') or path.endswith('.h')) and line == '}': current_function = None if path.endswith('.java') and line == '}': current_kind = None current_type = None line_num += 1 # ".parent.parent" because there is always the top "null" state. if state.parent != None and state.parent.parent != None: print("{}: Ended with more than one state on the stack.".format(relative), file=sys.stderr) s = state while s.parent != None: print(" {}".format(s.start), file=sys.stderr) s = s.parent sys.exit(1)