def pandoc_author(authors_and_institutions, auth2index, inst2index, index2inst, auth2email): # List authors on multiple lines authors = [] for author, i, e in authors_and_institutions: author = '**%s**' % author # set in boldface if i is None: authors.append(author) else: authors.append(author + ' at ' + ' and '.join(i)) plain_text = '### Author' if len(authors) > 1: plain_text += 's' plain_text += '\n\n' + '\n\n'.join(authors) + '\n\n' if option('strapdown'): return plain_text elif option('strict_markdown_output'): return plain_text elif option('multimarkdown_output'): return 'Author: ' + ', '.join(authors) + '\n' else: # pandoc-extended markdown syntax return '% ' + '; '.join(authors) + '\n'
def __init__(self): self.tracks = os.listdir(option("music-dir")) if option("music-shuffle"): random.shuffle(self.tracks) self.track = 0 pygame.mixer.music.set_endevent(Music.MUSIC_FINISHED)
def pandoc_date(m): date = m.group('subst') if option('strapdown'): return '#### Date: ' + date elif option('strict_markdown_output'): return '#### Date: ' + date elif option('multimarkdown_output'): return 'Date: ' + date + '\n' else: # pandoc-extended markdown syntax return '% ' + date + '\n'
def pandoc_date(m): date = m.group("subst") if option("strapdown"): return "#### Date: " + date elif option("strict_markdown_output"): return "#### Date: " + date elif option("multimarkdown_output"): return "Date: " + date + "\n" else: # pandoc-extended markdown syntax return "% " + date + "\n"
def pandoc_title(m): title = m.group('subst') if option('strapdown'): # title is in <title> tag in INTRO for the header of the HTML output return '' elif option('strict_markdown_output'): return '# ' + title elif option('multimarkdown_output'): return 'Title: ' + title else: return '% ' + title
def pandoc_title(m): title = m.group('subst') if option('strapdown'): # title is in <title> tag in INTRO for the header of the HTML output return '' elif option('strict_markdown_output'): return '# ' + title elif option('multimarkdown_output'): return 'Title: ' + title else: # pandoc-extended markdown syntax return '% ' + title
def pandoc_title(m): title = m.group("subst") if option("strapdown"): # title is in <title> tag in INTRO for the header of the HTML output return "" elif option("strict_markdown_output"): return "# " + title elif option("multimarkdown_output"): return "Title: " + title else: # pandoc-extended markdown syntax return "% " + title
def matlabnb_quiz(quiz): # Simple typesetting of a quiz import string question_prefix = quiz.get('question prefix', option('quiz_question_prefix=', 'Question:')) common_choice_prefix = option('quiz_choice_prefix=', 'Choice') quiz_expl = option('quiz_explanations=', 'on') text = '\n\n' if 'new page' in quiz: text += '======= %s =======\n\n' % (quiz['new page']) # Don't write Question: ... if inside an exercise section if quiz.get('embedding', 'None') in [ 'exercise', ]: pass else: text += '\n' if question_prefix: text += '%s ' % (question_prefix) text += quiz['question'] + '\n\n' # List choices as paragraphs for i, choice in enumerate(quiz['choices']): #choice_no = i+1 choice_no = string.ascii_uppercase[i] answer = choice[0].capitalize() + '!' choice_prefix = common_choice_prefix if 'choice prefix' in quiz: if isinstance(quiz['choice prefix'][i], basestring): choice_prefix = quiz['choice prefix'][i] if choice_prefix == '' or choice_prefix[-1] in ['.', ':', '?']: pass # don't add choice number/letter else: choice_prefix += ' %s:' % choice_no # Let choice start with a newline if pure code starts the choice # (test for different code block types so this function can work # for other formats too...) choice = choice[1].lstrip() code_starters = 'Code::', '~~~', '```', '{{{' for code_starter in code_starters: if choice.startswith(code_starter): choice = '\n' + choice # Cannot treat explanations text += '%s %s\n\n' % (choice_prefix, choice) return text
def xml_movie(m): filename = m.group('filename').srip() opts = m.group('options').strip() caption = m.group('caption').strip() if opts: opts = [s.split('=') for s in opts.split()] opts = ['%s="%s"' % (key, value) for key, value in opts] opts = ' ' + ' '.join(opts) else: opts = '' autoplay = option('html_video_autoplay=', 'False') if autoplay in ('on', 'off', 'True', 'true'): autoplay = True else: autoplay = False if opts: opts.append('autoplay="%s"' % str(autoplay)) text = '<figure filename="%s%s>' % (filename, opts) if caption: text += '\n<caption>\n%s\n</caption>' % caption text += '\n</figure>\n' return text
def rst_abstract(m): # r'\n*\g<type>.* \g<text>\n\g<rest>' name = m.group("type").strip() text = m.group("text").strip() rest = m.group("rest").strip() if option("rst_uio"): s = """ .. uio-introduction:: %s .. contents:: .. section-numbering:: %s """ % ( indent_lines(text, "rst"), rest, ) return s else: return "\n*%(name)s.* %(text)s\n\n%(rest)s" % vars()
def rst_abstract(m): # r'\n*\g<type>.* \g<text>\n\g<rest>' name = m.group('type').strip() text = m.group('text').strip() rest = m.group('rest').strip() if option('rst_uio'): s = """ .. uio-introduction:: %s .. contents:: .. section-numbering:: %s """ % (indent_lines(text, 'rst'), rest) return s else: if name.lower() == 'preface': # Drop heading (short abstract for books) return '\n%(text)s\n\n%(rest)s' % vars() else: return '\n*%(name)s.* %(text)s\n\n%(rest)s' % vars()
def rst_author(authors_and_institutions, auth2index, inst2index, index2inst, auth2email): if option("rst_uio"): if authors_and_institutions: # Use first author and email responsible = authors_and_institutions[0][0] email = authors_and_institutions[0][2] text = ( """ .. uio-meta:: :responsible-name: %s """ % responsible ) if email: text += " :responsible-email: %s\n\n" % email else: print "*** error: with --rst_uio there must be an AUTHOR:" print " field with (at least) one author w/email who will be" print " listed as the resposible under uio-meta::" _abort() else: authors = [] for author, i, email in authors_and_institutions: if email: email = email.replace("@", " at ") authors.append(author + " (%s)" % email) else: authors.append(author) text = ":Authors: " + ", ".join(authors) # (text is already r-stripped in typeset_authors) # we skip institutions in rst return text
def pandoc_linebreak(m): # orig: r'\g<text>\\n', text = m.group('text') if option('github_md'): return text + '<br />' else: return text + r'\\n'
def rst_author(authors_and_institutions, auth2index, inst2index, index2inst, auth2email): if option('rst_uio'): if authors_and_institutions: # Use first author and email responsible = authors_and_institutions[0][0] email = authors_and_institutions[0][2] text = """ .. uio-meta:: :responsible-name: %s """ % responsible if email: text += ' :responsible-email: %s\n\n' % email else: print '*** error: with --rst_uio there must be an AUTHOR:' print ' field with (at least) one author w/email who will be' print ' listed as the resposible under uio-meta::' _abort() else: authors = [] for author, i, email in authors_and_institutions: if email: email = email.replace('@', ' at ') authors.append(author + ' (%s)' % email) else: authors.append(author) text = ':Authors: ' + ', '.join( authors) # (text is already r-stripped in typeset_authors) # we skip institutions in rst return text
def rst_author(authors_and_institutions, auth2index, inst2index, index2inst, auth2email): if option('rst_uio'): if authors_and_institutions: # Use first author and email responsible = authors_and_institutions[0][0] email = authors_and_institutions[0][2] text = """ .. uio-meta:: :responsible-name: %s """ % responsible if email: text += ' :responsible-email: %s\n\n' % email else: errwarn('*** error: with --rst_uio there must be an AUTHOR:') errwarn(' field with (at least) one author w/email who will be') errwarn(' listed as the resposible under uio-meta::') _abort() else: authors = [] for author, i, email in authors_and_institutions: if email: email = email.replace('@', ' at ') authors.append(author + ' (%s)' % email) else: authors.append(author) text = ':Authors: ' + ', '.join(authors) # (text is already r-stripped in typeset_authors) # we skip institutions in rst return text
def sphinx_movie(m): filename = m.group("filename") special_movie = ( "*" in filename or "->" in filename or "youtu.be" in filename or "youtube.com" in filename or "vimeo.com" in filename ) if option("runestone") and not special_movie: # Use RunestoneInteractive video environment global video_counter video_counter += 1 text = """ .. video:: video_%d :controls: %s """ % ( video_counter, filename, ) return text else: # Use plain html code return rst_movie(m)
def pandoc_linebreak(m): # orig: r'\g<text>\\n', text = m.group('text') if option('github_md'): return text + '<br />' else: return text + r'\n'
def pandoc_linebreak(m): # orig: r'\g<text>\\n', text = m.group("text") if option("github_md"): return text + "<br />" else: return text + r"\n"
def plain_quiz(quiz): # Simple typesetting of a quiz import string question_prefix = quiz.get('question prefix', option('quiz_question_prefix=', 'Question:')) common_choice_prefix = option('quiz_choice_prefix=', 'Choice') quiz_expl = option('quiz_explanations=', 'on') text = '\n\n' if 'new page' in quiz: text += '======= %s =======\n\n' % (quiz['new page']) # Don't write Question: ... if inside an exercise section if quiz.get('embedding', 'None') in ['exercise',]: pass else: text += '\n' if question_prefix: text += '%s ' % (question_prefix) text += quiz['question'] + '\n\n' # List choices as paragraphs for i, choice in enumerate(quiz['choices']): #choice_no = i+1 choice_no = string.ascii_uppercase[i] answer = choice[0].capitalize() + '!' choice_prefix = common_choice_prefix if 'choice prefix' in quiz: if isinstance(quiz['choice prefix'][i], basestring): choice_prefix = quiz['choice prefix'][i] if choice_prefix == '' or choice_prefix[-1] in ['.', ':', '?']: pass # don't add choice number/letter else: choice_prefix += ' %s:' % choice_no # Let choice start with a newline if pure code starts the choice # (test for different code block types so this function can work # for other formats too...) choice = choice[1].lstrip() code_starters = 'Code::', '~~~', '```', '{{{' for code_starter in code_starters: if choice.startswith(code_starter): choice = '\n' + choice # Cannot treat explanations text += '%s %s\n\n' % (choice_prefix, choice) return text
def plain_quiz(quiz): # Simple typesetting of a quiz import string question_prefix = quiz.get("question prefix", option("quiz_question_prefix=", "Question:")) common_choice_prefix = option("quiz_choice_prefix=", "Choice") quiz_expl = option("quiz_explanations=", "on") text = "\n\n" if "new page" in quiz: text += "======= %s =======\n\n" % (quiz["new page"]) # Don't write Question: ... if inside an exercise section if quiz.get("embedding", "None") in ["exercise"]: pass else: text += "\n" if question_prefix: text += "%s " % (question_prefix) text += quiz["question"] + "\n\n" # List choices as paragraphs for i, choice in enumerate(quiz["choices"]): # choice_no = i+1 choice_no = string.ascii_uppercase[i] answer = choice[0].capitalize() + "!" choice_prefix = common_choice_prefix if "choice prefix" in quiz: if isinstance(quiz["choice prefix"][i], basestring): choice_prefix = quiz["choice prefix"][i] if choice_prefix == "" or choice_prefix[-1] in [".", ":", "?"]: pass # don't add choice number/letter else: choice_prefix += " %s:" % choice_no # Let choice start with a newline if pure code starts the choice # (test for different code block types so this function can work # for other formats too...) choice = choice[1].lstrip() code_starters = "Code::", "~~~", "```", "{{{" for code_starter in code_starters: if choice.startswith(code_starter): choice = "\n" + choice # Cannot treat explanations text += "%s %s\n\n" % (choice_prefix, choice) return text
def pandoc_quiz(quiz): # Simple typesetting of a quiz import string question_prefix = quiz.get('question prefix', option('quiz_question_prefix=', 'Question:')) common_choice_prefix = option('quiz_choice_prefix=', 'Choice') quiz_expl = option('quiz_explanations=', 'on') text = '\n\n' if 'new page' in quiz: text += '## %s\n\n' % (quiz['new page']) # Don't write Question: ... if inside an exercise section if quiz.get('embedding', 'None') in [ 'exercise', ]: pass else: text += '\n' if question_prefix: text += '**%s** ' % (question_prefix) text += quiz['question'] + '\n\n' # List choices as paragraphs for i, choice in enumerate(quiz['choices']): #choice_no = i+1 choice_no = string.ascii_uppercase[i] answer = choice[0].capitalize() + '!' choice_prefix = common_choice_prefix if 'choice prefix' in quiz: if isinstance(quiz['choice prefix'][i], basestring): choice_prefix = quiz['choice prefix'][i] if choice_prefix == '' or choice_prefix[-1] in ['.', ':', '?']: pass # don't add choice number/letter else: choice_prefix += ' %s:' % choice_no if choice_prefix: choice_prefix = '**%s**' % choice_prefix # Always have a newline after choice in case code or tex # blocks appear first choice_prefix = choice_prefix + '\n' # Cannot treat explanations and answers text += '%s %s\n\n' % (choice_prefix, choice[1]) return text
def sphinx_ref_and_label(section_label2title, format, filestr): # Special fix early in the process: # Deal with !split - by default we place splits before # the topmost sections # (This must be done before labels are put above section # headings) if not option('sphinx_keep_splits'): print '*** warning: new !split inserted (override all existing !split)' # Note: the title is at this stage translated to a chapter heading! # This title/heading must be removed for the algorithm below to work # (remove it, then insert afterwards) pattern = r'^.. Document title:\n\n={3,9}.+?={3,9}' m = re.search(pattern, filestr, flags=re.MULTILINE) title_replacement = '<<<<<<<DOCUMENT TITLE>>>>>>>>>>>>' # "unlikely" str if m: title = m.group() filestr = filestr.replace(title, title_replacement) else: title = '' topmost_section = 0 for i in [9, 7, 5]: if re.search(r'^%s' % ('=' * i), filestr, flags=re.MULTILINE): topmost_section = i print ' before every %s heading %s' % \ ('='*topmost_section, '='*topmost_section) print ' because this strategy gives a well-functioning' print ' table of contents in Sphinx' print ' (use --sphinx_keep_splits to enforce your own !split commands)' break if topmost_section: # First remove all !split filestr = re.sub(r'^!split *\n', '', filestr, flags=re.MULTILINE) # Insert new splits before all topmost sections pattern = r'^%s (.+?) %s' % \ ('='*topmost_section, '='*topmost_section) lines = filestr.splitlines() for i in range(len(lines)): if re.search(pattern, lines[i]): lines[i] = '!split\n' + lines[i] filestr = '\n'.join(lines) filestr = filestr.replace(title_replacement, title) filestr = ref_and_label_commoncode(section_label2title, format, filestr) # replace all references to sections: for label in section_label2title: filestr = filestr.replace('ref{%s}' % label, ':ref:`%s`' % label) # Not of interest after sphinx got equation references: #from common import ref2equations #filestr = ref2equations(filestr) # Replace remaining ref{x} as :ref:`x` filestr = re.sub(r'ref\{(.+?)\}', ':ref:`\g<1>`', filestr) return filestr
def sphinx_ref_and_label(section_label2title, format, filestr): # Special fix early in the process: # Deal with !split - by default we place splits before # the all the topmost sections # (This must be done before labels are put above section # headings) if '!split' in filestr and not option('sphinx_keep_splits'): print '*** warning: new !split inserted (override all existing !split)' # Note: the title is at this stage translated to a chapter heading! # This title/heading must be removed for the algorithm below to work # (remove it, then insert afterwards) pattern = r'^.. Document title:\n\n={3,9}.+?={3,9}' m = re.search(pattern, filestr, flags=re.MULTILINE) title_replacement = '<<<<<<<DOCUMENT TITLE>>>>>>>>>>>>' # "unlikely" str if m: title = m.group() filestr = filestr.replace(title, title_replacement) else: title = '' topmost_section = 0 for i in [9, 7, 5]: if re.search(r'^%s' % ('='*i), filestr, flags=re.MULTILINE): topmost_section = i print ' before every %s heading %s' % \ ('='*topmost_section, '='*topmost_section) print ' because this strategy gives a well-functioning' print ' table of contents in Sphinx' print ' (use --sphinx_keep_splits to enforce your own !split commands)' break if topmost_section: # First remove all !split filestr = re.sub(r'^!split *\n', '', filestr, flags=re.MULTILINE) # Insert new splits before all topmost sections pattern = r'^%s (.+?) %s' % \ ('='*topmost_section, '='*topmost_section) lines = filestr.splitlines() for i in range(len(lines)): if re.search(pattern, lines[i]): lines[i] = '!split\n' + lines[i] filestr = '\n'.join(lines) filestr = filestr.replace(title_replacement, title) filestr = ref_and_label_commoncode(section_label2title, format, filestr) # replace all references to sections: for label in section_label2title: filestr = filestr.replace('ref{%s}' % label, ':ref:`%s`' % label) # Not of interest after sphinx got equation references: #from common import ref2equations #filestr = ref2equations(filestr) # Replace remaining ref{x} as :ref:`x` filestr = re.sub(r'ref\{(.+?)\}', ':ref:`\g<1>`', filestr) return filestr
def pandoc_quiz(quiz): # Simple typesetting of a quiz import string question_prefix = quiz.get('question prefix', option('quiz_question_prefix=', 'Question:')) common_choice_prefix = option('quiz_choice_prefix=', 'Choice') quiz_expl = option('quiz_explanations=', 'on') text = '\n\n' if 'new page' in quiz: text += '## %s\n\n' % (quiz['new page']) # Don't write Question: ... if inside an exercise section if quiz.get('embedding', 'None') in ['exercise',]: pass else: text += '\n' if question_prefix: text += '**%s** ' % (question_prefix) text += quiz['question'] + '\n\n' # List choices as paragraphs for i, choice in enumerate(quiz['choices']): #choice_no = i+1 choice_no = string.ascii_uppercase[i] answer = choice[0].capitalize() + '!' choice_prefix = common_choice_prefix if 'choice prefix' in quiz: if isinstance(quiz['choice prefix'][i], basestring): choice_prefix = quiz['choice prefix'][i] if choice_prefix == '' or choice_prefix[-1] in ['.', ':', '?']: pass # don't add choice number/letter else: choice_prefix += ' %s:' % choice_no if choice_prefix: choice_prefix = '**%s**' % choice_prefix # Always have a newline after choice in case code or tex # blocks appear first choice_prefix = choice_prefix + '\n' # Cannot treat explanations and answers text += '%s %s\n\n' % (choice_prefix, choice[1]) return text
def pandoc_quiz(quiz): # Simple typesetting of a quiz import string question_prefix = quiz.get("question prefix", option("quiz_question_prefix=", "Question:")) common_choice_prefix = option("quiz_choice_prefix=", "Choice") quiz_expl = option("quiz_explanations=", "on") text = "\n\n" if "new page" in quiz: text += "## %s\n\n" % (quiz["new page"]) # Don't write Question: ... if inside an exercise section if quiz.get("embedding", "None") in ["exercise"]: pass else: text += "\n" if question_prefix: text += "**%s** " % (question_prefix) text += quiz["question"] + "\n\n" # List choices as paragraphs for i, choice in enumerate(quiz["choices"]): # choice_no = i+1 choice_no = string.ascii_uppercase[i] answer = choice[0].capitalize() + "!" choice_prefix = common_choice_prefix if "choice prefix" in quiz: if isinstance(quiz["choice prefix"][i], basestring): choice_prefix = quiz["choice prefix"][i] if choice_prefix == "" or choice_prefix[-1] in [".", ":", "?"]: pass # don't add choice number/letter else: choice_prefix += " %s:" % choice_no if choice_prefix: choice_prefix = "**%s**" % choice_prefix # Always have a newline after choice in case code or tex # blocks appear first choice_prefix = choice_prefix + "\n" # Cannot treat explanations and answers text += "%s %s\n\n" % (choice_prefix, choice[1]) return text
def sphinx_ref_and_label(section_label2title, format, filestr): # Special fix early in the process: # Deal with !split - by default we place splits before # the all the topmost sections # (This must be done before labels are put above section # headings) if "!split" in filestr and not option("sphinx_keep_splits"): errwarn("*** warning: new !split inserted (override all existing !split)") # Note: the title is at this stage translated to a chapter heading! # This title/heading must be removed for the algorithm below to work # (remove it, then insert afterwards) pattern = r"^.. Document title:\n\n={3,9}.+?={3,9}" m = re.search(pattern, filestr, flags=re.MULTILINE) title_replacement = "<<<<<<<DOCUMENT TITLE>>>>>>>>>>>>" # "unlikely" str if m: title = m.group() filestr = filestr.replace(title, title_replacement) else: title = "" topmost_section = 0 for i in [9, 7, 5]: if re.search(r"^%s" % ("=" * i), filestr, flags=re.MULTILINE): topmost_section = i errwarn(" before every %s heading %s" % ("=" * topmost_section, "=" * topmost_section)) errwarn(" because this strategy gives a well-functioning") errwarn(" table of contents in Sphinx") errwarn(" (use --sphinx_keep_splits to enforce your own !split commands)") break if topmost_section: # First remove all !split filestr = re.sub(r"^!split *\n", "", filestr, flags=re.MULTILINE) # Insert new splits before all topmost sections pattern = r"^%s (.+?) %s" % ("=" * topmost_section, "=" * topmost_section) lines = filestr.splitlines() for i in range(len(lines)): if re.search(pattern, lines[i]): lines[i] = "!split\n" + lines[i] filestr = "\n".join(lines) filestr = filestr.replace(title_replacement, title) filestr = ref_and_label_commoncode(section_label2title, format, filestr) # replace all references to sections: for label in section_label2title: filestr = filestr.replace("ref{%s}" % label, ":ref:`%s`" % label) # Not of interest after sphinx got equation references: # from common import ref2equations # filestr = ref2equations(filestr) # Replace remaining ref{x} as :ref:`x` filestr = re.sub(r"ref\{(.+?)\}", ":ref:`\g<1>`", filestr) return filestr
def pandoc_table(table): if option('github_md'): text = html_table(table) # Fix the problem that `verbatim` inside the table is not # typeset as verbatim (according to the pandoc translator rules) # in the GitHub Issue Tracker text = re.sub(r'`([^`]+?)`', '<code>\g<1></code>', text) return text # else: Pandoc-extended Markdown syntax """ Simple markdown tables look like this:: Left Right Center Default ------- ------ ---------- ------- 12 12 12 12 123 123 123 123 1 1 1 1 """ # Slight modification of rst_table column_width = table_analysis(table['rows']) ncolumns = len(column_width) column_spec = table.get('columns_align', 'c'*ncolumns).replace('|', '') heading_spec = table.get('headings_align', 'c'*ncolumns).replace('|', '') a2py = {'r': 'rjust', 'l': 'ljust', 'c': 'center'} s = '' # '\n' for i, row in enumerate(table['rows']): #s += ' ' # indentation of tables if row == ['horizontal rule'] and i > 0 and i < len(table['rows'])-1: # No horizontal rule at the top and bottom, just after heading for w in column_width: s += '-'*w + ' ' else: # check if this is a headline between two horizontal rules: if i == 1 and \ table['rows'][i-1] == ['horizontal rule'] and \ table['rows'][i+1] == ['horizontal rule']: headline = True else: headline = False for w, c, ha, ca in \ zip(column_width, row, heading_spec, column_spec): if headline: s += getattr(c, a2py[ha])(w) + ' ' elif row != ['horizontal rule']: s += getattr(c, a2py[ca])(w) + ' ' s += '\n' s += '\n' return s
def pandoc_table(table): if option('github_md'): text = html_table(table) # Fix the problem that `verbatim` inside the table is not # typeset as verbatim (according to the pandoc translator rules) # in the GitHub Issue Tracker text = re.sub(r'`([^`]+?)`', '<code>\g<1></code>', text) return text # else: Pandoc-extended Markdown syntax """ Simple markdown tables look like this:: Left Right Center Default ------- ------ ---------- ------- 12 12 12 12 123 123 123 123 1 1 1 1 """ # Slight modification of rst_table column_width = table_analysis(table['rows']) ncolumns = len(column_width) column_spec = table.get('columns_align', 'c' * ncolumns).replace('|', '') heading_spec = table.get('headings_align', 'c' * ncolumns).replace('|', '') a2py = {'r': 'rjust', 'l': 'ljust', 'c': 'center'} s = '' # '\n' for i, row in enumerate(table['rows']): #s += ' ' # indentation of tables if row == ['horizontal rule'] and i > 0 and i < len(table['rows']) - 1: # No horizontal rule at the top and bottom, just after heading for w in column_width: s += '-' * w + ' ' else: # check if this is a headline between two horizontal rules: if i == 1 and \ table['rows'][i-1] == ['horizontal rule'] and \ table['rows'][i+1] == ['horizontal rule']: headline = True else: headline = False for w, c, ha, ca in \ zip(column_width, row, heading_spec, column_spec): if headline: s += getattr(c, a2py[ha])(w) + ' ' elif row != ['horizontal rule']: s += getattr(c, a2py[ca])(w) + ' ' s += '\n' s += '\n' return s
def ipynb_index_bib(filestr, index, citations, pubfile, pubdata): # ipynb has support for latex-style bibliography. # Quite some code here is copy from latex_index_bib # http://nbviewer.ipython.org/github/ipython/nbconvert-examples/blob/master/citations/Tutorial.ipynb if citations: from common import cite_with_multiple_args2multiple_cites filestr = cite_with_multiple_args2multiple_cites(filestr) for label in citations: filestr = filestr.replace( 'cite{%s}' % label, '<cite data-cite="%s">[%d]</cite>' % (label, citations[label])) if pubfile is not None: # Always produce a new bibtex file bibtexfile = pubfile[:-3] + 'bib' errwarn('\nexporting publish database %s to %s:' % (pubfile, bibtexfile)) publish_cmd = 'publish export %s' % os.path.basename(bibtexfile) # Note: we have to run publish in the directory where pubfile resides this_dir = os.getcwd() pubfile_dir = os.path.dirname(pubfile) if not pubfile_dir: pubfile_dir = os.curdir os.chdir(pubfile_dir) os.system(publish_cmd) os.chdir(this_dir) bibstyle = option('latex_bibstyle=', 'plain') from latex import fix_latex_command_regex bibtext = fix_latex_command_regex(r""" ((*- extends 'latex_article.tplx' -*)) ((* block bibliography *)) \bibliographystyle{%s} \bibliography{%s} ((* endblock bibliography *)) """ % (bibstyle, bibtexfile[:-4]), application='replacement') filestr = re.sub(r'^BIBFILE:.+$', bibtext, filestr, flags=re.MULTILINE) # Save idx{} and label{} as metadata, also have labels as div tags filestr = re.sub(r'((idx\{.+?\})', r'<!-- dom:\g<1> -->', filestr) filestr = re.sub(r'(label\{(.+?)\})', r'<!-- dom:\g<1> --><div id="\g<2>"></div>', filestr) # Also treat special cell delimiter comments that might appear from # doconce ipynb2doconce conversions filestr = re.sub(r'^# ---------- (markdown|code) cell$', '', filestr, flags=re.MULTILINE) return filestr
def pandoc_table(table): if option("github_md"): text = html_table(table) # Fix the problem that `verbatim` inside the table is not # typeset as verbatim (according to the pandoc translator rules) # in the GitHub Issue Tracker text = re.sub(r"`([^`]+?)`", "<code>\g<1></code>", text) return text # else: Pandoc-extended Markdown syntax """ Simple markdown tables look like this:: Left Right Center Default ------- ------ ---------- ------- 12 12 12 12 123 123 123 123 1 1 1 1 """ # Slight modification of rst_table column_width = table_analysis(table["rows"]) ncolumns = len(column_width) column_spec = table.get("columns_align", "c" * ncolumns).replace("|", "") heading_spec = table.get("headings_align", "c" * ncolumns).replace("|", "") a2py = {"r": "rjust", "l": "ljust", "c": "center"} s = "" # '\n' for i, row in enumerate(table["rows"]): # s += ' ' # indentation of tables if row == ["horizontal rule"] and i > 0 and i < len(table["rows"]) - 1: # No horizontal rule at the top and bottom, just after heading for w in column_width: s += "-" * w + " " else: # check if this is a headline between two horizontal rules: if i == 1 and table["rows"][i - 1] == ["horizontal rule"] and table["rows"][i + 1] == ["horizontal rule"]: headline = True else: headline = False for w, c, ha, ca in zip(column_width, row, heading_spec, column_spec): if headline: s += getattr(c, a2py[ha])(w) + " " elif row != ["horizontal rule"]: s += getattr(c, a2py[ca])(w) + " " s += "\n" s += "\n" return s
def pandoc_author(authors_and_institutions, auth2index, inst2index, index2inst, auth2email): # List authors on multiple lines authors = [] for author, i, e in authors_and_institutions: author = "**%s**" % author # set in boldface if i is None: authors.append(author) else: authors.append(author + " at " + " and ".join(i)) plain_text = "### Author" if len(authors) > 1: plain_text += "s" plain_text += "\n\n" + "\n\n".join(authors) + "\n\n" if option("strapdown"): return plain_text elif option("strict_markdown_output"): return plain_text elif option("multimarkdown_output"): return "Author: " + ", ".join(authors) + "\n" else: # pandoc-extended markdown syntax return "% " + "; ".join(authors) + "\n"
def has_copyright(filestr): copyright_ = False # We use the copyright field for citing doconce if option('cite_doconce'): copyright_ = True # Check each author for explicit copyright authors = re.findall(r'^AUTHOR:(.+)', filestr, flags=re.MULTILINE) symbol = False # We do not need a (c)-symbol if we're only citing doconce for author in authors: if '{copyright' in author: copyright_ = True symbol = True break return copyright_, symbol
def ipynb_index_bib(filestr, index, citations, pubfile, pubdata): # ipynb has support for latex-style bibliography. # Quite some code here is copy from latex_index_bib # http://nbviewer.ipython.org/github/ipython/nbconvert-examples/blob/master/citations/Tutorial.ipynb if citations: from common import cite_with_multiple_args2multiple_cites filestr = cite_with_multiple_args2multiple_cites(filestr) for label in citations: filestr = filestr.replace('cite{%s}' % label, '<cite data-cite="%s">[%d]</cite>' % (label, citations[label])) if pubfile is not None: # Always produce a new bibtex file bibtexfile = pubfile[:-3] + 'bib' errwarn('\nexporting publish database %s to %s:' % (pubfile, bibtexfile)) publish_cmd = 'publish export %s' % os.path.basename(bibtexfile) # Note: we have to run publish in the directory where pubfile resides this_dir = os.getcwd() pubfile_dir = os.path.dirname(pubfile) if not pubfile_dir: pubfile_dir = os.curdir os.chdir(pubfile_dir) os.system(publish_cmd) os.chdir(this_dir) bibstyle = option('latex_bibstyle=', 'plain') from latex import fix_latex_command_regex bibtext = fix_latex_command_regex(r""" ((*- extends 'latex_article.tplx' -*)) ((* block bibliography *)) \bibliographystyle{%s} \bibliography{%s} ((* endblock bibliography *)) """ % (bibstyle, bibtexfile[:-4]), application='replacement') filestr = re.sub(r'^BIBFILE:.+$', bibtext, filestr, flags=re.MULTILINE) # Save idx{} and label{} as metadata, also have labels as div tags filestr = re.sub(r'((idx\{.+?\})', r'<!-- dom:\g<1> -->', filestr) filestr = re.sub(r'(label\{(.+?)\})', r'<!-- dom:\g<1> --><div id="\g<2>"></div>', filestr) # Also treat special cell delimiter comments that might appear from # doconce ipynb2doconce conversions filestr = re.sub(r'^# ---------- (markdown|code) cell$', '', filestr, flags=re.MULTILINE) return filestr
def sphinx_movie(m): filename = m.group('filename') special_movie = '*' in filename or '->' in filename or 'youtu.be' in filename or 'youtube.com' in filename or 'vimeo.com' in filename if option('runestone') and not special_movie: # Use RunestoneInteractive video environment global video_counter video_counter += 1 text = """ .. video:: video_%d :controls: %s """ % (video_counter, filename) return text else: # Use plain html code return rst_movie(m)
def rst_code(filestr, code_blocks, code_block_types, tex_blocks, format): # In rst syntax, code blocks are typeset with :: (verbatim) # followed by intended blocks. This function indents everything # inside code (or TeX) blocks. for i in range(len(code_blocks)): code_blocks[i] = indent_lines(code_blocks[i], format) for i in range(len(tex_blocks)): tex_blocks[i] = indent_lines(tex_blocks[i], format) # Fix labels if option('rst_mathjax'): for i in range(len(tex_blocks)): tex_blocks[i] = tex_blocks[i].replace(' label{', ' \\label{') filestr = insert_code_and_tex(filestr, code_blocks, tex_blocks, 'rst') # substitute !bc and !ec appropriately: # the line before the !bc block must end in [a-zA-z0-9)"...] # followed by [\n:.?!,] see the bc_regex_pattern global variable above # (problems with substituting !bc and !bt may be caused by # missing characters in these two families) filestr = re.sub(bc_regex_pattern, r'\g<1>::\n\n', filestr, flags=re.MULTILINE | re.DOTALL) # Need a fix for :: appended to special comment lines (---:: -> ---\nCode::) filestr = re.sub(r' ---::\n\n', ' ---\nCode::\n\n', filestr) filestr = re.sub(r'^!ec\n', '\n', filestr, flags=re.MULTILINE) #filestr = re.sub(r'^!ec\n', '', filestr, flags=re.MULTILINE) #c = re.compile(r'([a-zA-Z0-9)"])[:.]?\s*?!bt\n', re.DOTALL) #filestr = c.sub(r'\g<1>:\n\n', filestr) #filestr = re.sub(r'^!bt\n', '.. latex-math::\n\n', filestr, re.MULTILINE) #filestr = re.sub(r'^!bt\n', '.. latex::\n\n', filestr, re.MULTILINE) if option('rst_mathjax') and (re.search( r'^!bt', filestr, flags=re.MULTILINE) or re.search(r'\\\( .+ \\\)', filestr)): # First add MathJax script in the very beginning of the file from html import mathjax_header latex = indent_lines(mathjax_header(filestr).lstrip(), 'rst') filestr = '\n.. raw:: html\n\n' + latex + '\n\n' + filestr # Replace all the !bt parts by raw html directive (make sure # the coming block is sufficiently indented, we used 8 chars above)[[[ filestr = re.sub(bt_regex_pattern, r'\g<1>\n\n.. raw:: html\n\n $$', filestr, flags=re.MULTILINE) filestr = re.sub(r'^!et *\n', ' $$\n\n', filestr, flags=re.MULTILINE) # Remove inner \[..\] from equations $$ \[ ... \] $$ filestr = re.sub(r'\$\$\s*\\\[', '$$', filestr) filestr = re.sub(r'\\\]\s*\$\$', '$$', filestr) # Equation references (ref{...}) must be \eqref{...} in MathJax # (note: this affects also (ref{...}) syntax in verbatim blocks...) filestr = re.sub(r'\(ref\{(.+?)\}\)', r'\eqref{\g<1>}', filestr) else: # just use the same substitution for tex blocks as for code blocks: filestr = re.sub(bt_regex_pattern, r'\g<1>::\n', filestr, flags=re.MULTILINE) #filestr = re.sub(r'^!et *\n', '\n\n', filestr, flags=re.MULTILINE) filestr = re.sub(r'^!et *\n', '\n', filestr, flags=re.MULTILINE) # Fix: if there are !bc-!ec or other environments after each # other without text in between, there is a difficulty with the # :: symbol before the code block. In these cases, we get # !ec::, !et::, !bbox:: etc. from the above substitutions. # We just replace these by empty text. filestr = re.sub(r'^(!(b|e)[a-z]+)::', r'\g<1>', filestr, flags=re.MULTILINE) # Check for pattern in '^!bt', '^!et': c = re.compile(pattern, re.MULTILINE) m = c.search(filestr) if m: print """ Still %s left after handling of code and tex blocks. Problem is probably that %s is not preceded by text which can be extended with :: (required). """ % (pattern, pattern) _abort() # Final fixes filestr = fix_underlines_in_headings(filestr) # Ensure blank line before and after comments filestr = re.sub(r'([.:;?!])\n^\.\. ', r'\g<1>\n\n.. ', filestr, flags=re.MULTILINE) filestr = re.sub(r'(^\.\. .+)\n([^ \n]+)', r'\g<1>\n\n\g<2>', filestr, flags=re.MULTILINE) # Line breaks interfer with tables and needs a final blank line too lines = filestr.splitlines() inside_block = False for i in range(len(lines)): if lines[i].startswith('<linebreakpipe>') and not inside_block: inside_block = True lines[i] = lines[i].replace('<linebreakpipe> ', '') + '\n' continue if lines[i].startswith('<linebreakpipe>') and inside_block: lines[i] = '|' + lines[i].replace('<linebreakpipe>', '') continue if inside_block and not lines[i].startswith('<linebreakpipe>'): inside_block = False lines[i] = '| ' + lines[i] + '\n' filestr = '\n'.join(lines) # Remove too much vertical space filestr = re.sub(r'\n\n\n+', '\n\n', filestr) return filestr
def rst_quiz(quiz): import string question_prefix = quiz.get('question prefix', option('quiz_question_prefix=', 'Question:')) common_choice_prefix = option('quiz_choice_prefix=', 'Choice') quiz_expl = option('quiz_explanations=', 'on') # Sphinx tooltop: :abbr:`TERM (explanation in tooltip)` # Can e.g. just have the right answer number as tooltip! text = '\n\n' if 'new page' in quiz: text += '.. !split\n%s\n%s' % (quiz['new page'], '-' * len(quiz['new page'])) text += '.. begin quiz\n\n' # Don't write Question: ... if inside an exercise section if quiz.get('embedding', 'None') in [ 'exercise', ]: pass else: text += '\n\n' if question_prefix: text += '**%s** ' % (question_prefix) if quiz['question'].lstrip().startswith('..'): # block, add extra \n text += '\n\n' text += quiz['question'] + '\n\n\n' # List choices as paragraphs for i, choice in enumerate(quiz['choices']): #choice_no = i+1 choice_no = string.ascii_uppercase[i] answer = choice[0].capitalize() + '!' choice_prefix = common_choice_prefix if 'choice prefix' in quiz: if isinstance(quiz['choice prefix'][i], basestring): choice_prefix = quiz['choice prefix'][i] if choice_prefix == '' or choice_prefix[-1] in ['.', ':', '?']: pass # don't add choice number/letter else: choice_prefix += ' %s:' % choice_no expl = '' if len(choice) == 3 and quiz_expl == 'on': expl = choice[2] if '.. figure::' in expl or 'math::' in expl or '.. code-block::' in expl: print '*** warning: quiz explanation contains block (fig/code/math)' print ' and is therefore skipped' print expl, '\n' expl = '' # drop explanation when it needs blocks # Should remove markup pattern = r'`(.+?) (<https?.+?)>`__' # URL expl = re.sub(pattern, '\g<1> (\g<2>)', expl) pattern = r'``(.+?)``' # verbatim expl = re.sub(pattern, '\g<1>', expl) pattern = r':math:`(.+?)`' # inline math expl = re.sub(pattern, '\g<1>', expl) # mimic italic.... pattern = r':\*\*(.+?)\*\*' # bold expl = re.sub(pattern, '\g<1>', expl, flags=re.DOTALL) pattern = r':\*(.+?)\*' # emphasize expl = re.sub(pattern, '\g<1>', expl, flags=re.DOTALL) tooltip = ' '.join(expl.splitlines()) if expl: text += '**%s** %s\n\n:abbr:`? (%s)` :abbr:`# (%s)`\n\n' % ( choice_prefix, choice[1], answer, tooltip) else: # no explanation text += '**%s** %s\n\n:abbr:`? (%s)`\n\n' % (choice_prefix, choice[1], answer) text += '.. end quiz\n\n' return text
def define(FILENAME_EXTENSION, BLANKLINE, INLINE_TAGS_SUBST, CODE, LIST, ARGLIST, TABLE, EXERCISE, FIGURE_EXT, CROSS_REFS, INDEX_BIB, TOC, ENVIRS, QUIZ, INTRO, OUTRO, filestr): # all arguments are dicts and accept in-place modifications (extensions) FILENAME_EXTENSION['rst'] = '.rst' BLANKLINE['rst'] = '\n' encoding = 'utf-8' # 'latin-1' INLINE_TAGS_SUBST['rst'] = { 'math': r'\g<begin>\g<subst>\g<end>', 'math2': r'\g<begin>\g<puretext>\g<end>', # math and math2 are redefined below if --rst_mathjax #'math': r'\g<begin>:math:`\g<subst>`\g<end>', # sphinx #'math2': r'\g<begin>:math:`\g<latexmath>`\g<end>', 'emphasize': None, # => just use doconce markup (*emphasized words*) 'bold': r'\g<begin>**\g<subst>**\g<end>', 'verbatim': r'\g<begin>``\g<subst>``\g<end>', 'label': r'\g<subst>', # should be improved, rst has cross ref 'reference': r'\g<subst>', #colortext cannot employ pure HTML code. Recipe: http://stackoverflow.com/questions/4669689/how-to-use-color-in-text-with-restructured-text-rst2html-py-or-how-to-insert-h (this is too comprehensive). Use bold instead. #'colortext': r'<font color="\g<color>">\g<text></font>', 'colortext': r'**\g<text>**', # Use anonymous hyperlink references to avoid warnings if the link # name appears twice #'linkURL': r'\g<begin>`\g<link> <\g<url>>`__\g<end>', #'linkURL': r'\g<begin>`\g<link>`_\g<end>' + '\n\n.. ' + r'__\g<link>: \g<url>' + '\n\n', # better (?): make function instead that stacks up the URLs and dumps them at the end; can be used for citations as well 'linkURL2': r'`\g<link> <\g<url>>`__', 'linkURL3': r'`\g<link> <\g<url>>`__', 'linkURL2v': r'`\g<link> <\g<url>>`__', # no verbatim, does not work well 'linkURL3v': r'`\g<link> <\g<url>>`__', # same 'plainURL': r'`<\g<url>>`_', 'inlinecomment': r'color{red}{(**\g<name>**: \g<comment>})', # the replacement string differs, depending on the match object m: # (note len(m.group('subst')) gives wrong length for latin-1 strings, # seems to work for utf-8, if problems: replace lambda function # with an ordinary function where you can debug and test! #'chapter': lambda m: '%s\n%s' % (m.group('subst'), '%'*len(m.group('subst').decode(encoding))), 'chapter': lambda m: '%s\n%s' % (m.group('subst'), '%' * len(m.group('subst'))), 'section': lambda m: '%s\n%s' % (m.group('subst'), '=' * len(m.group('subst'))), 'subsection': lambda m: '%s\n%s' % (m.group('subst'), '-' * len(m.group('subst'))), 'subsubsection': lambda m: '%s\n%s\n' % (m.group('subst'), '~' * len(m.group('subst'))), 'paragraph': r'**\g<subst>**\n', # extra newline 'abstract': rst_abstract, #'title': r'======= \g<subst> =======\n', # doconce top section, must be the highest section level (but no higher than others, need more code) 'title': None, # taken care of in ref_and_label_commoncode 'date': r':Date: \g<subst>\n', 'author': rst_author, 'figure': rst_figure, 'movie': rst_movie, #'comment': '.. %s', # rst does not like empty comment lines: # so therefore we introduce a function to remove empty comment lines # (we insert an extra blank first to be safe) 'comment': lambda c: '' if c.isspace() or c == '' else '\n.. %s\n' % c, #'linebreak': r'| \g<text>', # does not work: interfers with tables and requires a final blank line after block 'linebreak': r'<linebreakpipe> \g<text>', # fixed in rst_code/sphinx_code as a hack 'footnote': rst_footnotes, 'non-breaking-space': ' |nbsp| ', 'horizontal-rule': '---------', 'ampersand2': r' \g<1>&\g<2>', } if option('rst_mathjax'): # rst2html conversion requires four backslashes here for one of them # to survive INLINE_TAGS_SUBST['rst'][ 'math'] = r'\g<begin>\\\\( \g<subst> \\\\)\g<end>' INLINE_TAGS_SUBST['rst'][ 'math2'] = r'\g<begin>\\\\( \g<latexmath> \\\\)\g<end>' ENVIRS['rst'] = { 'quote': rst_quote, 'warning': rst_warning, 'question': rst_question, 'notice': rst_notice, 'summary': rst_summary, 'block': rst_block, 'box': rst_box, } CODE['rst'] = rst_code # function for typesetting code LIST['rst'] = { 'itemize': { 'begin': '', 'item': '*', 'end': '\n' }, # lists must end with a blank line - we insert one extra, 'enumerate': { 'begin': '', 'item': '%d.', 'end': '\n' }, 'description': { 'begin': '', 'item': '%s', 'end': '\n' }, 'separator': '\n', } from common import DEFAULT_ARGLIST ARGLIST['rst'] = DEFAULT_ARGLIST FIGURE_EXT['rst'] = { 'search': ('.png', '.gif', '.jpg', '.jpeg', '.pdf', '.eps', '.ps'), 'convert': ('.png', '.gif', '.jpg') } CROSS_REFS['rst'] = rst_ref_and_label INDEX_BIB['rst'] = rst_index_bib TABLE['rst'] = rst_table EXERCISE['rst'] = plain_exercise TOC['rst'] = lambda s: '.. contents:: Table of Contents\n :depth: 2' QUIZ['rst'] = rst_quiz INTRO['rst'] = """\ .. Automatically generated reStructuredText file from DocOnce source (https://github.com/hplgit/doconce/) """ # http://stackoverflow.com/questions/11830242/non-breaking-space from common import INLINE_TAGS if re.search(INLINE_TAGS['non-breaking-space'], filestr): nbsp = """ .. |nbsp| unicode:: 0xA0 :trim: """ if 'TITLE:' not in filestr: import common if common.format in ('rst', 'sphinx'): print '*** error: non-breaking space character ~ is used,' print ' but this will give an error when the document does' print ' not have a title.' _abort() else: INTRO['rst'] += nbsp
def pandoc_code(filestr, code_blocks, code_block_types, tex_blocks, format): """ # We expand all newcommands now from html import embed_newcommands newcommands = embed_newcommands(filestr) if newcommands: filestr = newcommands + filestr """ # Note: the tex code require the MathJax fix of doconce md2html # to insert right MathJax extensions to interpret align and labels # correctly. # (Also, doconce.py runs align2equations so there are no align/align* # environments in tex blocks.) for i in range(len(tex_blocks)): # Remove latex envir in single equations tex_blocks[i] = tex_blocks[i].replace(r'\[', '') tex_blocks[i] = tex_blocks[i].replace(r'\]', '') tex_blocks[i] = tex_blocks[i].replace(r'\begin{equation*}', '') tex_blocks[i] = tex_blocks[i].replace(r'\end{equation*}', '') #tex_blocks[i] = tex_blocks[i].replace(r'\[', '$$') #tex_blocks[i] = tex_blocks[i].replace(r'\]', '$$') # Check for illegal environments m = re.search(r'\\begin\{(.+?)\}', tex_blocks[i]) if m: envir = m.group(1) if envir not in ('equation', 'equation*', 'align*', 'align', 'array'): errwarn("""\ *** warning: latex envir \\begin{%s} does not work well. """ % envir) # Add $$ on each side of the equation tex_blocks[i] = '$$\n' + tex_blocks[i] + '$$\n' # Note: HTML output from pandoc requires $$ while latex cannot have # them if begin-end inside ($$\begin{...} \end{...}$$) if option('strict_markdown_output'): # Code blocks are just indented for i in range(len(code_blocks)): code_blocks[i] = indent_lines(code_blocks[i], format) filestr = insert_code_and_tex(filestr, code_blocks, tex_blocks, format) github_md = option('github_md') if not option('strict_markdown_output'): pass if github_md: for key in language2pandoc: language2pandoc[key] = language2pandoc[key].lower() # Code blocks apply the ~~~~~ delimiter, with blank lines before # and after for key in language2pandoc: language = language2pandoc[key] if github_md: replacement = '\n```%s\n' % language2pandoc[key] else: # pandoc-extended Markdown replacement = '\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.%s}\n' % language2pandoc[ key] #replacement = '\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.%s ,numberLines}\n' % language2pandoc[key] # enable line numbering filestr = re.sub(r'^!bc\s+%s\s*\n' % key, replacement, filestr, flags=re.MULTILINE) # any !bc with/without argument becomes an unspecified block if github_md: replacement = '\n```' else: replacement = '\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' filestr = re.sub(r'^!bc.*$', replacement, filestr, flags=re.MULTILINE) if github_md: replacement = '```\n' else: replacement = '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' filestr = re.sub(r'^!ec\s*$', replacement, filestr, flags=re.MULTILINE) else: # Strict Markdown: just indented blocks filestr = re.sub(r'^!bc.*$', '', filestr, flags=re.MULTILINE) filestr = re.sub(r'^!ec\s*$', '', filestr, flags=re.MULTILINE) filestr = re.sub(r'^!bt *\n', '', filestr, flags=re.MULTILINE) filestr = re.sub(r'^!et *\n', '', filestr, flags=re.MULTILINE) # \eqref and labels will not work, but labels do no harm filestr = filestr.replace(' label{', ' \\label{') pattern = r'^label\{' filestr = re.sub(pattern, '\\label{', filestr, flags=re.MULTILINE) filestr = re.sub(r'\(ref\{(.+?)\}\)', r'\eqref{\g<1>}', filestr) # Final fixes # Seems that title and author must appear on the very first lines filestr = filestr.lstrip() # Enable tasks lists: # - [x] task 1 done # - [ ] task 2 not yet done if github_md: pattern = '^(\s+)\*\s+(\[[x ]\])\s+' filestr = re.sub(pattern, '\g<1>- \g<2> ', filestr, flags=re.MULTILINE) return filestr
def ipynb_movie(m): # m.group() must be called before m.group('name') text = '<!-- dom:%s -->' % m.group() global html_encountered, movie_encountered, movie_files filename = m.group('filename') caption = m.group('caption').strip() youtube = False if 'youtu.be' in filename or 'youtube.com' in filename: youtube = True if '*' in filename or '->' in filename: print '*** warning: * or -> in movie filenames is not supported in ipynb' return text def YouTubeVideo(filename): # Use YouTubeVideo object if 'watch?v=' in filename: name = filename.split('watch?v=')[1] elif 'youtu.be/' in filename: name = filename.split('youtu.be/')[1] else: print '*** error: youtube movie name "%s" could not be interpreted' % filename _abort() text = '' global movie_encountered if not movie_encountered: text += 'from IPython.display import YouTubeVideo\n' movie_encountered = True text += 'YouTubeVideo("%s")\n' % name return text text += '\n<!-- begin movie -->\n' display_method = option('ipynb_movie=', 'HTML') if display_method == 'md': text += html_movie(m) elif display_method.startswith('HTML'): text += '\n!bc pycod\n' if youtube and 'YouTube' in display_method: text += YouTubeVideo(filename) if caption: text += '\nprint "%s"' % caption else: # Use HTML formatting if not html_encountered: text += 'from IPython.display import HTML\n' html_encountered = True text += '_s = """' + html_movie(m) + '"""\n' text += 'HTML(_s)\n' if not filename.startswith('http'): movie_files.append(filename) text += '!ec\n' elif display_method == 'ipynb': text += '!bc pycod\n' if youtube: text += YouTubeVideo(filename) if caption: text += '\nprint "%s"' % caption else: # see http://nbviewer.ipython.org/github/ipython/ipython/blob/1.x/examples/notebooks/Part%205%20-%20Rich%20Display%20System.ipynb # http://stackoverflow.com/questions/18019477/how-can-i-play-a-local-video-in-my-ipython-notebook # http://python.6.x6.nabble.com/IPython-User-embedding-non-YouTube-movies-in-the-IPython-notebook-td5024035.html # Just support .mp4, .ogg, and.webm stem, ext = os.path.splitext(filename) if ext not in ('.mp4', '.ogg', '.webm'): print '*** error: movie "%s" in format %s is not supported for --ipynb_movie=%s' % ( filename, ext, display_method) print ' use --ipynb_movie=HTML instead' _abort() height = 365 width = 640 if filename.startswith('http'): file_open = 'import urllib\nvideo = urllib.urlopen("%s").read()' % filename else: file_open = 'video = open("%s", "rb").read()' % filename text += """ %s from base64 import b64encode video_encoded = b64encode(video) video_tag = '<video controls loop alt="%s" height="%s" width="%s" src="data:video/%s;base64,{0}">'.format(video_encoded) """ % (file_open, filename, height, width, ext[1:]) if not filename.startswith('http'): movie_files.append(filename) if not html_encountered: text += 'from IPython.display import HTML\n' html_encountered = True text += 'HTML(data=video_tag)\n' if caption: text += '\nprint "%s"' % caption text += '!ec\n' else: print '*** error: --ipynb_movie=%s is not supported' % display_method _abort() text += '<!-- end movie -->\n' return text
def define(FILENAME_EXTENSION, BLANKLINE, INLINE_TAGS_SUBST, CODE, LIST, ARGLIST, TABLE, EXERCISE, FIGURE_EXT, CROSS_REFS, INDEX_BIB, TOC, ENVIRS, QUIZ, INTRO, OUTRO, filestr): # all arguments are dicts and accept in-place modifications (extensions) FILENAME_EXTENSION['pandoc'] = '.md' BLANKLINE['pandoc'] = '\n' # replacement patterns for substitutions of inline tags INLINE_TAGS_SUBST['pandoc'] = { 'math': None, # indicates no substitution, leave as is 'math2': r'\g<begin>$\g<latexmath>$\g<end>', 'emphasize': None, 'bold': r'\g<begin>**\g<subst>**\g<end>', 'figure': pandoc_figure, #'movie': default_movie, 'movie': html_movie, 'verbatim': None, #'linkURL': r'\g<begin>\g<link> (\g<url>)\g<end>', 'linkURL2': r'[\g<link>](\g<url>)', 'linkURL3': r'[\g<link>](\g<url>)', 'linkURL2v': r'[`\g<link>`](\g<url>)', 'linkURL3v': r'[`\g<link>`](\g<url>)', 'plainURL': r'<\g<url>>', 'colortext': r'<font color="\g<color>">\g<text></font>', # HTML # "Reference links" in pandoc are not yet supported 'title': pandoc_title, 'author': pandoc_author, 'date': pandoc_date, 'chapter': lambda m: '# ' + m.group('subst'), 'section': lambda m: '## ' + m.group('subst'), 'subsection': lambda m: '### ' + m.group('subst'), 'subsubsection': lambda m: '#### ' + m.group('subst') + '\n', 'paragraph': r'*\g<subst>*\g<space>', 'abstract': r'*\g<type>.* \g<text>\n\n\g<rest>', 'comment': '<!-- %s -->', 'linebreak': pandoc_linebreak, 'non-breaking-space': '\\ ', 'ampersand2': r' \g<1>&\g<2>', } CODE['pandoc'] = pandoc_code ENVIRS['pandoc'] = { 'quote': pandoc_quote, } if option('slate_md'): ENVIRS['pandoc'] = { 'warning': slate_warning, 'notice': slate_notice, 'block': slate_success, 'summary': functools.partial(slate_success, title='Summary'), 'question': functools.partial(slate_success, title='Question'), 'box': slate_success, } from common import DEFAULT_ARGLIST ARGLIST['pandoc'] = DEFAULT_ARGLIST LIST['pandoc'] = { 'itemize': { 'begin': '', 'item': '*', 'end': '\n' }, 'enumerate': { 'begin': '', 'item': '%d.', 'end': '\n' }, 'description': { 'begin': '', 'item': '%s\n : ', 'end': '\n' }, #'separator': '\n', 'separator': '', } CROSS_REFS['pandoc'] = pandoc_ref_and_label TABLE['pandoc'] = pandoc_table INDEX_BIB['pandoc'] = pandoc_index_bib EXERCISE['pandoc'] = plain_exercise TOC['pandoc'] = lambda s, f: '# Table of contents: Run pandoc with --toc option' QUIZ['pandoc'] = pandoc_quiz FIGURE_EXT['pandoc'] = { 'search': ('.png', '.gif', '.jpg', '.jpeg', '.tif', '.tiff', '.pdf'), 'convert': ('.png', '.gif', '.jpg') } # Wrap markdown output in strapdown HTML code for quick auto rendering # with Bootstrap themes? if option('strapdown'): # Themes boostrap_bootwatch_theme = option('bootswatch_theme=', 'spacelab') # Grab title title = '' if 'TITLE:' in filestr: for line in filestr.splitlines(): if line.startswith('TITLE:'): title = line.split('TITLE:')[-1].strip() break INTRO['pandoc'] = """<!DOCTYPE html> <html> <title>%(title)s</title> <xmp theme="%(boostrap_bootwatch_theme)s" style="display:none;"> # Markdown text goes in here """ % vars() OUTRO['pandoc'] = """
def ipynb_code(filestr, code_blocks, code_block_types, tex_blocks, format): """ # We expand all newcommands now from html import embed_newcommands newcommands = embed_newcommands(filestr) if newcommands: filestr = newcommands + filestr """ # Fix pandoc citations to normal internal links: [[key]](#key) filestr = re.sub(r'\[@(.+?)\]', r'[[\g<1>]](#\g<1>)', filestr) # filestr becomes json list after this function so we must typeset # envirs here. All envirs are typeset as pandoc_quote. from common import _CODE_BLOCK, _MATH_BLOCK envir_format = option('ipynb_admon=', 'paragraph') # Remove all !bpop-!epop environments (they cause only problens and # have no use) for envir in 'pop', 'slidecell': filestr = re.sub('^<!-- !b%s .*\n' % envir, '', filestr, flags=re.MULTILINE) filestr = re.sub('^<!-- !e%s .*\n' % envir, '', filestr, flags=re.MULTILINE) filestr = re.sub('^<!-- !bnotes.*?<!-- !enotes -->\n', '', filestr, flags=re.DOTALL|re.MULTILINE) filestr = re.sub('^<!-- !split -->\n', '', filestr, flags=re.MULTILINE) from doconce import doconce_envirs envirs = doconce_envirs()[8:-2] for envir in envirs: pattern = r'^!b%s(.*?)\n(.+?)\s*^!e%s' % (envir, envir) if envir_format in ('quote', 'paragraph', 'hrule'): def subst(m): title = m.group(1).strip() # Text size specified in parenthesis? m2 = re.search('^\s*\((.+?)\)', title) if title == '' and envir not in ('block', 'quote'): title = envir.capitalize() + '.' elif title.lower() == 'none': title == '' elif m2: text_size = m2.group(1).lower() title = title.replace('(%s)' % text_size, '').strip() elif title and title[-1] not in ('.', ':', '!', '?'): # Make sure the title ends with puncuation title += '.' # Recall that this formatting is called very late # so native format must be used! if title: title = '**' + title + '**\n' # Could also consider subsubsection formatting block = m.group(2) # Always use quote typesetting for quotes if envir_format == 'quote' or envir == 'quote': # Make Markdown quote of the block: lines start with > lines = [] for line in block.splitlines(): # Just quote plain text if not (_MATH_BLOCK in line or _CODE_BLOCK in line or line.startswith('FIGURE:') or line.startswith('MOVIE:') or line.startswith('|')): lines.append('> ' + line) else: lines.append('\n' + line + '\n') block = '\n'.join(lines) + '\n\n' # Add quote and a blank line after title if title: title = '> ' + title + '>\n' else: # Add a blank line after title if title: title += '\n' if envir_format == 'hrule': # Native ------ does not work, use <hr/> #text = '\n\n----------\n' + title + '----------\n' + \ # block + '\n----------\n\n' text = '\n\n<hr/>\n' + title + \ block + '\n<hr/>\n\n' else: text = title + block + '\n\n' return text else: errwarn('*** error: --ipynb_admon=%s is not supported' % envir_format) filestr = re.sub(pattern, subst, filestr, flags=re.DOTALL | re.MULTILINE) # Fix pyshell and ipy interactive sessions: remove prompt and output. # or split in multiple cells such that output comes out at the end of a cell # Fix sys environments and use run prog.py so programs can be run in cell # Insert %matplotlib inline in the first block using matplotlib # Only typeset Python code as blocks, otherwise !bc environmens # become plain indented Markdown. from doconce import dofile_basename from sets import Set ipynb_tarfile = 'ipynb-%s-src.tar.gz' % dofile_basename src_paths = Set() mpl_inline = False split_pyshell = option('ipynb_split_pyshell=', 'on') if split_pyshell is None: split_pyshell = False elif split_pyshell in ('no', 'False', 'off'): split_pyshell = False else: split_pyshell = True ipynb_code_tp = [None]*len(code_blocks) for i in range(len(code_blocks)): # Check if continuation lines are in the code block, because # doconce.py inserts a blank after the backslash if '\\ \n' in code_blocks[i]: code_blocks[i] = code_blocks[i].replace('\\ \n', '\\\n') if not mpl_inline and ( re.search(r'import +matplotlib', code_blocks[i]) or \ re.search(r'from +matplotlib', code_blocks[i]) or \ re.search(r'import +scitools', code_blocks[i]) or \ re.search(r'from +scitools', code_blocks[i])): code_blocks[i] = '%matplotlib inline\n\n' + code_blocks[i] mpl_inline = True tp = code_block_types[i] if tp.endswith('-t'): # Standard Markdown code with pandoc/github extension language = tp[:-2] language_spec = language2pandoc.get(language, '') #code_blocks[i] = '\n' + indent_lines(code_blocks[i], format) + '\n' code_blocks[i] = "```%s\n" % language_spec + \ indent_lines(code_blocks[i].strip(), format) + \ "```" ipynb_code_tp[i] = 'markdown' elif tp.startswith('pyshell') or tp.startswith('ipy'): lines = code_blocks[i].splitlines() last_cell_end = -1 if split_pyshell: new_code_blocks = [] # Split for each output an put in separate cell for j in range(len(lines)): if lines[j].startswith('>>>') or lines[j].startswith('... '): lines[j] = lines[j][4:] elif lines[j].startswith('In ['): # IPython lines[j] = ':'.join(lines[j].split(':')[1:]).strip() elif lines[j].startswith(' ...: '): # IPython lines[j] = lines[j][8:] else: # output (no prefix or Out) lines[j] = '' new_code_blocks.append( '\n'.join(lines[last_cell_end+1:j+1])) last_cell_end = j code_blocks[i] = new_code_blocks ipynb_code_tp[i] = 'cell' else: # Remove prompt and output lines; leave code executable in cell for j in range(len(lines)): if lines[j].startswith('>>> ') or lines[j].startswith('... '): lines[j] = lines[j][4:] elif lines[j].startswith('In ['): lines[j] = ':'.join(lines[j].split(':')[1:]).strip() else: # output lines[j] = '' for j in range(lines.count('')): lines.remove('') code_blocks[i] = '\n'.join(lines) ipynb_code_tp[i] = 'cell' elif tp.startswith('sys'): # Do we find execution of python file? If so, copy the file # to separate subdir and make a run file command in a cell. # Otherwise, it is just a plain verbatim Markdown block. found_unix_lines = False lines = code_blocks[i].splitlines() for j in range(len(lines)): m = re.search(r'(.+?>|\$) *python +([A-Za-z_0-9]+?\.py)', lines[j]) if m: name = m.group(2).strip() if os.path.isfile(name): src_paths.add(os.path.dirname(name)) lines[j] = '%%run "%s"' % fullpath else: found_unix_lines = True src_paths = list(src_paths) if src_paths and not found_unix_lines: # This is a sys block with run commands only code_blocks[i] = '\n'.join(lines) ipynb_code_tp[i] = 'cell' else: # Standard Markdown code code_blocks[i] = '\n'.join(lines) code_blocks[i] = indent_lines(code_blocks[i], format) ipynb_code_tp[i] = 'markdown' elif tp.endswith('hid'): ipynb_code_tp[i] = 'cell_hidden' elif tp.startswith('py'): ipynb_code_tp[i] = 'cell' else: # Should support other languages as well, but not for now code_blocks[i] = indent_lines(code_blocks[i], format) ipynb_code_tp[i] = 'markdown' # figure_files and movie_files are global variables and contain # all figures and movies referred to src_paths = list(src_paths) if figure_files: src_paths += figure_files if movie_files: src_paths += movie_files if src_paths: # Make tar file with all the source dirs with files # that need to be executed os.system('tar cfz %s %s' % (ipynb_tarfile, ' '.join(src_paths))) errwarn('collected all required additional files in ' + ipynb_tarfile + ' which must be distributed with the notebook') elif os.path.isfile(ipynb_tarfile): os.remove(ipynb_tarfile) # Parse document into markdown text, code blocks, and tex blocks. # Store in nested list notebook_blocks. notebook_blocks = [[]] authors = '' for line in filestr.splitlines(): if line.startswith('authors = [new_author(name='): # old author method authors = line[10:] elif _CODE_BLOCK in line: code_block_tp = line.split()[-1] if code_block_tp in ('pyhid',) or not code_block_tp.endswith('hid'): notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip() notebook_blocks.append(line) # else: hidden block to be dropped (may include more languages # with time in the above tuple) elif _MATH_BLOCK in line: notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip() notebook_blocks.append(line) else: if not isinstance(notebook_blocks[-1], list): notebook_blocks.append([]) notebook_blocks[-1].append(line) if isinstance(notebook_blocks[-1], list): notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip() # Add block type info pattern = r'(\d+) +%s' for i in range(len(notebook_blocks)): if re.match(pattern % _CODE_BLOCK, notebook_blocks[i]): m = re.match(pattern % _CODE_BLOCK, notebook_blocks[i]) idx = int(m.group(1)) if ipynb_code_tp[idx] == 'cell': notebook_blocks[i] = ['cell', notebook_blocks[i]] elif ipynb_code_tp[idx] == 'cell_hidden': notebook_blocks[i] = ['cell_hidden', notebook_blocks[i]] else: notebook_blocks[i] = ['text', notebook_blocks[i]] elif re.match(pattern % _MATH_BLOCK, notebook_blocks[i]): notebook_blocks[i] = ['math', notebook_blocks[i]] else: notebook_blocks[i] = ['text', notebook_blocks[i]] # Go through tex_blocks and wrap in $$ # (doconce.py runs align2equations so there are no align/align* # environments in tex blocks) label2tag = {} tag_counter = 1 for i in range(len(tex_blocks)): # Extract labels and add tags labels = re.findall(r'label\{(.+?)\}', tex_blocks[i]) for label in labels: label2tag[label] = tag_counter # Insert tag to get labeled equation tex_blocks[i] = tex_blocks[i].replace( 'label{%s}' % label, 'label{%s} \\tag{%s}' % (label, tag_counter)) tag_counter += 1 # Remove \[ and \] or \begin/end{equation*} in single equations tex_blocks[i] = tex_blocks[i].replace(r'\[', '') tex_blocks[i] = tex_blocks[i].replace(r'\]', '') tex_blocks[i] = tex_blocks[i].replace(r'\begin{equation*}', '') tex_blocks[i] = tex_blocks[i].replace(r'\end{equation*}', '') # Check for illegal environments m = re.search(r'\\begin\{(.+?)\}', tex_blocks[i]) if m: envir = m.group(1) if envir not in ('equation', 'equation*', 'align*', 'align', 'array'): errwarn("""\ *** warning: latex envir \\begin{%s} does not work well in Markdown. Stick to \\[ ... \\], equation, equation*, align, or align* environments in math environments. """ % envir) eq_type = 'heading' # or '$$' eq_type = '$$' # Markdown: add $$ on each side of the equation if eq_type == '$$': # Make sure there are no newline after equation tex_blocks[i] = '$$\n' + tex_blocks[i].strip() + '\n$$' # Here: use heading (###) and simple formula (remove newline # in math expressions to keep everything within a heading) as # the equation then looks bigger elif eq_type == 'heading': tex_blocks[i] = '### $ ' + ' '.join(tex_blocks[i].splitlines()) + ' $' # Add labels for the eqs above the block (for reference) if labels: #label_tp = '<a name="%s"></a>' label_tp = '<div id="%s"></div>' tex_blocks[i] = '<!-- Equation labels as ordinary links -->\n' + \ ' '.join([label_tp % label for label in labels]) + '\n\n' + \ tex_blocks[i] # blocks is now a list of text chunks in markdown and math/code line # instructions. Insert code and tex blocks for i in range(len(notebook_blocks)): if _CODE_BLOCK in notebook_blocks[i][1] or _MATH_BLOCK in notebook_blocks[i][1]: words = notebook_blocks[i][1].split() # start of notebook_blocks[i]: number block-indicator code-type n = int(words[0]) if _CODE_BLOCK in notebook_blocks[i][1]: notebook_blocks[i][1] = code_blocks[n] # can be list! if _MATH_BLOCK in notebook_blocks[i][1]: notebook_blocks[i][1] = tex_blocks[n] # Make IPython structures nb_version = int(option('ipynb_version=', '4')) if nb_version == 3: try: from IPython.nbformat.v3 import ( new_code_cell, new_text_cell, new_worksheet, new_notebook, new_metadata, new_author) nb = new_worksheet() except ImportError: errwarn('*** error: could not import IPython.nbformat.v3!') errwarn(' set --ipynb_version=4 or leave out --ipynb_version=3') _abort() elif nb_version == 4: try: from nbformat.v4 import ( new_code_cell, new_markdown_cell, new_notebook) except ImportError: # Try old style try: from IPython.nbformat.v4 import ( new_code_cell, new_markdown_cell, new_notebook) except ImportError: errwarn('*** error: cannot do import nbformat.v4 or IPython.nbformat.v4') errwarn(' make sure IPython notebook or Jupyter is installed correctly') _abort() cells = [] mdstr = [] # plain md format of the notebook prompt_number = 1 for block_tp, block in notebook_blocks: if (block_tp == 'text' or block_tp == 'math') and block != '': # Pure comments between math/code and math/code come # out as empty blocks, should detect that situation # (challenging - can have multiple lines of comments, # or begin and end comment lines with important things between) if nb_version == 3: nb.cells.append(new_text_cell(u'markdown', source=block)) elif nb_version == 4: cells.append(new_markdown_cell(source=block)) mdstr.append(('markdown', block)) elif block_tp == 'cell' and block != '' and block != []: if isinstance(block, list): for block_ in block: block_ = block_.rstrip() if block_ != '': if nb_version == 3: nb.cells.append(new_code_cell( input=block_, prompt_number=prompt_number, collapsed=False)) elif nb_version == 4: cells.append(new_code_cell( source=block_, execution_count=prompt_number, metadata=dict(collapsed=False))) prompt_number += 1 mdstr.append(('codecell', block_)) else: block = block.rstrip() if block != '': if nb_version == 3: nb.cells.append(new_code_cell( input=block, prompt_number=prompt_number, collapsed=False)) elif nb_version == 4: cells.append(new_code_cell( source=block, execution_count=prompt_number, metadata=dict(collapsed=False))) prompt_number += 1 mdstr.append(('codecell', block)) elif block_tp == 'cell_hidden' and block != '': block = block.rstrip() if nb_version == 3: nb.cells.append(new_code_cell( input=block, prompt_number=prompt_number, collapsed=True)) elif nb_version == 4: cells.append(new_code_cell( source=block, execution_count=prompt_number, metadata=dict(collapsed=True))) prompt_number += 1 mdstr.append(('codecell', block)) """ # Dump the notebook cells in a simple ASCII format # (doc/src/ipynb/ipynb_generator.py can translate it back to .ipynb file) f = open(dofile_basename + '.md-ipynb', 'w') for cell_tp, block in mdstr: if cell_tp == 'markdown': f.write('\n-----\n\n') elif cell_tp == 'codecell': f.write('\n-----py\n\n') f.write(block) f.close() """ if nb_version == 3: # Catch the title as the first heading m = re.search(r'^#+\s*(.+)$', filestr, flags=re.MULTILINE) title = m.group(1).strip() if m else '' # md below is not used for anything if authors: authors = eval(authors) md = new_metadata(name=title, authors=authors) else: md = new_metadata(name=title) nb = new_notebook(worksheets=[nb], metadata=new_metadata()) # Let us make v4 notebook here by upgrading from IPython.nbformat.v4 import upgrade nb = upgrade(nb) import IPython.nbformat.v4.nbjson as nbjson # Convert nb to json format filestr = nbjson.writes(nb) elif nb_version == 4: nb = new_notebook(cells=cells) from IPython.nbformat import writes filestr = writes(nb, version=4) # Check that there are no empty cells: if '"input": []' in filestr: errwarn('*** error: empty cells in notebook - report bug in DocOnce') _abort() # must do the replacements here at the very end when json is written out # \eqref and labels will not work, but labels (only in math) do no harm filestr = re.sub(r'([^\\])label\{', r'\g<1>\\\\label{', filestr, flags=re.MULTILINE) # \\eqref{} just gives (???) link at this stage - future versions # will probably support labels #filestr = re.sub(r'\(ref\{(.+?)\}\)', r'\\eqref{\g<1>}', filestr) # Now we use explicit references to tags def subst(m): label = m.group(1) try: return r'[(%s)](#%s)' % (label2tag[label], label) except KeyError as e: errwarn('*** error: label "%s" is not defined' % str(e)) filestr = re.sub(r'\(ref\{(.+?)\}\)', subst, filestr) """ # MathJax reference to tag (recall that the equations have both label # and tag (know that tag only works well in HTML, but this mjx-eqn-no # label does not work in ipynb) filestr = re.sub(r'\(ref\{(.+?)\}\)', lambda m: r'[(%s)](#mjx-eqn-%s)' % (label2tag[m.group(1)], label2tag[m.group(1)]), filestr) """ #filestr = re.sub(r'\(ref\{(.+?)\}\)', r'Eq (\g<1>)', filestr) ''' # Final fixes: replace all text between cells by markdown code cells # Note: the patterns are overlapping so a plain re.sub will not work, # here we run through all blocks found and subsitute the first remaining # one, one by one. pattern = r' \},\n(.+?)\{\n "cell_type":' begin_pattern = r'^(.+?)\{\n "cell_type":' remaining_block_begin = re.findall(begin_pattern, filestr, flags=re.DOTALL) remaining_blocks = re.findall(pattern, filestr, flags=re.DOTALL) import string for block in remaining_block_begin + remaining_blocks: filestr = string.replace(filestr, block, json_markdown(block) + ' ', maxreplace=1) filestr_end = re.sub(r' \{\n "cell_type": .+?\n \},\n', '', filestr, flags=re.DOTALL) filestr = filestr.replace(filestr_end, json_markdown(filestr_end)) filestr = """{ "metadata": { "name": "SOME NAME" }, "nbformat": 3, "nbformat_minor": 0, "worksheets": [ { "cells": [ """ + filestr.rstrip() + '\n'+ \ json_pycode('', final_prompt_no+1, 'python').rstrip()[:-1] + """ ], "metadata": {} } ] }""" ''' return filestr
def remove_code_and_tex(filestr, format): """ Remove verbatim and latex (math) code blocks from the file and store separately in lists (code_blocks and tex_blocks). The function insert_code_and_tex will insert these blocks again. """ # Method: # store code and tex blocks in lists and substitute these blocks # by the contents of _CODE_BLOCK and _MATH_BLOCK (arguments after # !bc must be copied after _CODE_BLOCK). # later we replace _CODE_BLOCK by !bc and !ec and the code block again # (similarly for the tex/math block). # ipynb (and future interactive executable documents) needs to # see if a code is to be executed or just displayed as text. # !bc *cod-t and !bc *pro-t is used to indicate pure text. if format not in ('ipynb', 'matlabnb'): filestr = re.sub(r'^!bc +([a-z0-9]+)-t', r'!bc \g<1>', filestr, flags=re.MULTILINE) # (recall that !bc can be followed by extra information that we must keep:) code = re.compile(r'^!bc(.*?)\n(.*?)^!ec *\n', re.DOTALL|re.MULTILINE) # Note: final \n is required and may be missing if there is a block # at the end of the file, so let us ensure that a blank final # line is appended to the text: if filestr[-1] != '\n': filestr = filestr + '\n' result = code.findall(filestr) code_blocks = [c for opt, c in result] code_block_types = [opt.strip() for opt, c in result] tex = re.compile(r'^!bt *\n(.*?)^!et *\n', re.DOTALL|re.MULTILINE) tex_blocks = tex.findall(filestr) # Remove blocks and substitute by a one-line sign filestr = code.sub('%s \g<1>\n' % _CODE_BLOCK, filestr) filestr = tex.sub('%s\n' % _MATH_BLOCK, filestr) # Number the blocks lines = filestr.splitlines() code_block_counter = 0 math_block_counter = 0 for i in range(len(lines)): if lines[i].startswith(_CODE_BLOCK): lines[i] = '%d ' % code_block_counter + lines[i] code_block_counter += 1 if lines[i].startswith(_MATH_BLOCK): lines[i] = '%d ' % math_block_counter + lines[i] math_block_counter += 1 filestr = safe_join(lines, '\n') # Number all equations? if option('number_all_equations'): subst = [('\\begin{equation*}', '\\begin{equation}'), ('\\end{equation*}', '\\end{equation}'), ('\\[', '\\begin{equation} '), ('\\]', '\\end{equation} '), ('\\begin{align*}', '\\begin{align}'), ('\\end{align*}', '\\end{align}'), ] if option('denumber_all_equations'): # Remove equation numbers and also labels in those equations subst = [('\\begin{equation}', '\\begin{equation*}'), ('\\end{equation}', '\\end{equation*}'), ('\\begin{align}', '\\begin{align*}'), ('\\end{align}', '\\end{align*}'), ] removed_labels = [] for i in range(len(tex_blocks)): found = False for construction, dummy in subst: if construction in tex_blocks[i]: found = True break if found: for from_, to_ in subst: tex_blocks[i] = tex_blocks[i].replace(from_, to_) removed_labels += re.findall(r'label\{(.+?)\}', tex_blocks[i]) tex_blocks[i] = re.sub(r'label\{.+?\}\n', '', tex_blocks[i]) tex_blocks[i] = re.sub(r'label\{.+?\}', '', tex_blocks[i]) all_refs = re.findall(r'ref\{(.+?)\}', filestr) problematic_refs = [] for ref in all_refs: if ref in removed_labels: problematic_refs.append(ref) if problematic_refs: print '*** error: removed all equation labels from the DocOnce source,' print ' but there are still references (ref{...}) to equation labels:' print '\n ', ', '.join(problematic_refs) print '\n remove all these references!' _abort() # Give error if blocks contain !bt for i in range(len(tex_blocks)): if '!bt' in tex_blocks[i] or '!et' in tex_blocks[i]: print '*** error: double !bt or !et in latex block:' print tex_blocks[i] _abort() # Check that math blocks do not contain edit markup or comments for block in tex_blocks: m = re.search(INLINE_TAGS['inlinecomment'], block, flags=re.DOTALL) if m: print '*** error: tex block with mathematics cannot contain' print ' inline comment or edit markup!' if m.group('name') in ('del', 'add') or '->' in m.group('comment'): # edit markup print ' Place info about editing after the block.' print block _abort() # Remove |\pause| in code blocks if not latex if format not in ('latex', 'pdflatex'): for i in range(len(code_blocks)): if r'|\pause|' in code_blocks[i]: code_blocks[i] = re.sub(r'^\|\\pause\|\n', '', code_blocks[i], flags=re.MULTILINE) return filestr, code_blocks, code_block_types, tex_blocks
def doconce_exercise_output( exer, solution_header = '__Solution.__', answer_header = '__Answer.__', hint_header = '__Hint.__', include_numbering=True, include_type=True, ): """ Write exercise in DocOnce format. This output can be reused in most formats. """ # Note: answers, solutions, and hints must be written out and not # removed here, because if they contain math or code blocks, # there will be fewer blocks in the end that what was extracted # at the beginning of the translation process. latex_style = option('latex_style=', 'std') # Store solutions in a separate string has_solutions = False if exer['solution']: has_solutions = True if exer['answer']: has_solutions = True for subex in exer['subex']: if subex['solution']: has_solutions = True if subex['answer']: has_solutions = True sol = '' # Solutions # s holds the formatted exercise in doconce format s = '\n\n# ' + envir_delimiter_lines['exercise'][0] + '\n\n' s += exer['heading'] # result string if has_solutions: sol += '\n\n# ' + envir_delimiter_lines['exercise'][0] + ' solution\n\n' if latex_style == 'Springer_sv': sol += r""" \begin{sol}{%s} \textbf{%s}\\ """ % (exer['label'], exer['title']) else: sol += exer['heading'] comments = '' # collect comments at the end of the exercises if include_numbering and not include_type: include_type = True if not exer['type_visible']: include_type = False if include_type: s += ' ' + exer['type'] if sol: sol += ' Solution to ' + exer['type'] if include_numbering: exer_numbering = option('exercise_numbering=', 'absolute') if exer_numbering == 'chapter' and exer['chapter_type'] is not None: s += ' %s.%s' % (exer['chapter_no'], exer['chapter_exercise']) else: s += ' ' + str(exer['no']) if sol: sol += ' ' + str(exer['no']) s += ':' if sol: sol += ':' s += ' ' + exer['title'] + ' ' + exer['heading'] + '\n' if sol: sol += ' ' + exer['title'] + ' ' + exer['heading'] + '\n' if exer['label']: s += 'label{%s}' % exer['label'] + '\n' if sol: sol += '# Solution to Exercise ref{%s}' % exer['label'] + '\n' if exer['keywords']: s += '# keywords = %s' % '; '.join(exer['keywords']) + '\n' if exer['text']: # Let comments at the end of the text come very last, if there # are no subexercises. Just outputting comments at the end # makes Filename: ... on a separate line, which does not look good. # We extract the final comments and print them after anything else. # Final comments often contain fruitful comments about the solution. if (not exer['subex']) and '\n#' in exer['text']: lines = exer['text'].splitlines() newlines = [] comments = [] for i, line in enumerate(reversed(lines)): if (line.startswith('#') or line.isspace() or line == '') \ and not line.startswith('# ---'): # (do not touch # --- begin/end type of comments!) comments.append(line) else: break comments = '\n'.join(reversed(comments)) if i == 0: exer['text'] = '\n'.join(lines) elif i > 0: exer['text'] = '\n'.join(lines[:-i]) s += '\n' + exer['text'] + '\n' if exer['hints']: for i, hint in enumerate(exer['hints']): if len(exer['hints']) == 1 and i == 0: hint_header_ = hint_header else: hint_header_ = hint_header.replace('Hint.', 'Hint %d.' % (i+1)) if exer['type'] != 'Example': s += '\n# ' + envir_delimiter_lines['hint'][0] + '\n' s += '\n' + hint_header_ + '\n' + hint + '\n' if exer['type'] != 'Example': s += '\n# ' + envir_delimiter_lines['hint'][1] + '\n' if exer['subex']: s += '\n' if sol: sol += '\n' import string for i, subex in enumerate(exer['subex']): letter = string.ascii_lowercase[i] s += '\n__%s)__\n' % letter if subex['solution'] or (subex['answer'] and not option('without_answers')): sol += '\n__%s)__\n' % letter if subex['text']: s += subex['text'] + '\n' for i, hint in enumerate(subex['hints']): if len(subex['hints']) == 1 and i == 0: hint_header_ = hint_header else: hint_header_ = hint_header.replace( 'Hint.', 'Hint %d.' % (i+1)) if exer['type'] != 'Example': s += '\n# ' + envir_delimiter_lines['hint'][0] + '\n' s += '\n' + hint_header_ + '\n' + hint + '\n' if exer['type'] != 'Example': s += '\n# ' + envir_delimiter_lines['hint'][1] + '\n' if subex['file']: if len(subex['file']) == 1: s += 'Filename: `%s`' % subex['file'][0] + '.\n' else: s += 'Filenames: %s' % \ ', '.join(['`%s`' % f for f in subex['file']]) + '.\n' if subex['answer']: s += '\n' if exer['type'] != 'Example': s += '\n# ' + envir_delimiter_lines['ans'][0] + '\n' sol += '\n# ' + envir_delimiter_lines['ans'][0] + '\n' s += answer_header + '\n' + subex['answer'] + '\n' sol += answer_header + '\n' + subex['answer'] + '\n' if exer['type'] != 'Example': s += '\n# ' + envir_delimiter_lines['ans'][1] + '\n' sol += '\n# ' + envir_delimiter_lines['ans'][1] + '\n' if subex['solution']: s += '\n' if exer['type'] != 'Example': s += '\n# ' + envir_delimiter_lines['sol'][0] + '\n' s += solution_header + '\n' # Make sure we have a sentence after the heading if solution_header.endswith('===') and \ re.search(r'^\d+ %s' % _CODE_BLOCK, subex['solution'].lstrip()): print '\nwarning: open the solution in exercise "%s" with a line of\ntext before the code! (Now "Code:" is inserted)' % exer['title'] + '\n' s += 'Code:\n' sol += '\nCode:\n' s += subex['solution'] + '\n' sol += '\n' + subex['solution'] + '\n' if exer['type'] != 'Example': s += '\n# ' + envir_delimiter_lines['sol'][1] + '\n' if 'aftertext' in subex: s += subex['aftertext'] if exer['answer']: s += '\n' # Leave out begin-end answer comments if example since we want to # avoid marking such sections for deletion (--without_answers) if exer['type'] != 'Example': s += '\n# ' + envir_delimiter_lines['ans'][0] + '\n' sol += '\n# ' + envir_delimiter_lines['ans'][0] + '\n' s += answer_header + '\n' + exer['answer'] + '\n' sol += answer_header + '\n' + exer['answer'] + '\n' if exer['type'] != 'Example': s += '\n# ' + envir_delimiter_lines['ans'][1] + '\n' sol += '\n# ' + envir_delimiter_lines['ans'][1] + '\n' if exer['solution']: s += '\n' # Leave out begin-end solution comments if example since we want to # avoid marking such sections for deletion (--without_solutions) if exer['type'] != 'Example': s += '\n# ' + envir_delimiter_lines['sol'][0] + '\n' s += solution_header + '\n' # Make sure we have a sentence after the heading if real heading if solution_header.endswith('===') and \ re.search(r'^\d+ %s' % _CODE_BLOCK, exer['solution'].lstrip()): print '\nwarning: open the solution in exercise "%s" with a line of\ntext before the code! (Now "Code:" is inserted)' % exer['title'] + '\n' s += 'Code:\n' sol += '\nCode:\n' s += exer['solution'] + '\n' sol += '\n'+ exer['solution'] + '\n' if exer['type'] != 'Example': s += '\n# ' + envir_delimiter_lines['sol'][1] + '\n' if exer['file']: if exer['subex']: # Place Filename: ... as a list paragraph if subexercises, # otherwise let it proceed at the end of the exercise text. s += '\n' if len(exer['file']) == 1: s += 'Filename: `%s`' % exer['file'][0] + '.\n' else: s += 'Filenames: %s' % \ ', '.join(['`%s`' % f for f in exer['file']]) + '.\n' #s += '*Filename*: `%s`' % exer['file'] + '.\n' #s += '\n' + '*Filename*: `%s`' % exer['file'] + '.\n' if exer['closing_remarks']: s += '\n# Closing remarks for this %s\n\n=== Remarks ===\n\n' % \ exer['type'] + exer['closing_remarks'] + '\n\n' if exer['solution_file']: if len(exer['solution_file']) == 1: s += '# solution file: %s\n' % exer['solution_file'][0] else: s += '# solution files: %s\n' % ', '.join(exer['solution_file']) if comments: s += '\n' + comments s += '\n# ' + envir_delimiter_lines['exercise'][1] + '\n\n' if sol: if latex_style == 'Springer_sv': sol += r'\end{sol}' + '\n' else: sol += '\n# ' + envir_delimiter_lines['exercise'][1] + ' solution\n\n' return s, sol
def sphinx_code(filestr, code_blocks, code_block_types, tex_blocks, format): # In rst syntax, code blocks are typeset with :: (verbatim) # followed by intended blocks. This function indents everything # inside code (or TeX) blocks. # default mappings of !bc environments and pygments languages: envir2pygments = dict( cod='python', pro='python', pycod='python', cycod='cython', pypro='python', cypro='cython', fcod='fortran', fpro='fortran', ccod='c', cppcod='c++', cpro='c', cpppro='c++', mcod='matlab', mpro='matlab', plcod='perl', plpro='perl', shcod='bash', shpro='bash', rbcod='ruby', rbpro='ruby', #sys='console', sys='text', rst='rst', css='css', csspro='css', csscod='css', dat='text', csv='text', txt='text', cc='text', ccq='text', # not possible with extra indent for ccq ipy='ipy', xmlcod='xml', xmlpro='xml', xml='xml', htmlcod='html', htmlpro='html', html='html', texcod='latex', texpro='latex', tex='latex', latexcod='latex', latexpro='latex', latex='latex', do='doconce', pyshell='python', pyoptpro='python', pyscpro='python', ) # grab line with: # sphinx code-blocks: cod=python cpp=c++ etc # (do this before code is inserted in case verbatim blocks contain # such specifications for illustration) m = re.search(r'.. *[Ss]phinx +code-blocks?:(.+)', filestr) if m: defs_line = m.group(1) # turn specifications into a dictionary: for definition in defs_line.split(): key, value = definition.split('=') envir2pygments[key] = value # First indent all code blocks for i in range(len(code_blocks)): if code_block_types[i].startswith( 'pyoptpro') and not option('runestone'): code_blocks[i] = online_python_tutor(code_blocks[i], return_tp='iframe') if code_block_types[i].endswith('-h'): indentation = ' ' * 8 else: indentation = ' ' * 4 code_blocks[i] = indent_lines(code_blocks[i], format, indentation) # After transforming align environments to separate equations # the problem with math labels in multiple eqs has disappeared. # (doconce.py applies align2equations, which takes all align # envirs and translates them to separate equations, but align* # environments are allowed. # Any output of labels in align means an error in the # align -> equation transformation...) math_labels = [] multiple_math_labels = [] # sphinx has problems with multiple math labels for i in range(len(tex_blocks)): tex_blocks[i] = indent_lines(tex_blocks[i], format) # extract all \label{}s inside tex blocks and typeset them # with :label: tags label_regex = fix_latex(r'label\{(.+?)\}', application='match') labels = re.findall(label_regex, tex_blocks[i]) if len(labels) == 1: tex_blocks[i] = ' :label: %s\n' % labels[0] + tex_blocks[i] elif len(labels) > 1: multiple_math_labels.append(labels) if len(labels) > 0: math_labels.extend(labels) tex_blocks[i] = re.sub(label_regex, '', tex_blocks[i]) # fix latex constructions that do not work with sphinx math # (just remove them) commands = [ r'\begin{equation}', r'\end{equation}', r'\begin{equation*}', r'\end{equation*}', #r'\begin{eqnarray}', #r'\end{eqnarray}', #r'\begin{eqnarray*}', #r'\end{eqnarray*}', #r'\begin{align}', #r'\end{align}', #r'\begin{align*}', #r'\end{align*}', r'\begin{multline}', r'\end{multline}', r'\begin{multline*}', r'\end{multline*}', #r'\begin{split}', #r'\end{split}', #r'\begin{gather}', #r'\end{gather}', #r'\begin{gather*}', #r'\end{gather*}', r'\[', r'\]', # some common abbreviations (newcommands): r'\beqan', r'\eeqan', r'\beqa', r'\eeqa', r'\balnn', r'\ealnn', r'\baln', r'\ealn', r'\beq', r'\eeq', # the simplest name, contained in others, must come last! ] for command in commands: tex_blocks[i] = tex_blocks[i].replace(command, '') # &=& -> &= tex_blocks[i] = re.sub('&\s*=\s*&', ' &= ', tex_blocks[i]) # provide warnings for problematic environments # Replace all references to equations that have labels in math environments: for label in math_labels: filestr = filestr.replace('(:ref:`%s`)' % label, ':eq:`%s`' % label) multiple_math_labels_with_refs = [] # collect the labels with references for labels in multiple_math_labels: for label in labels: ref = ':eq:`%s`' % label # ref{} is translated to eq:`` if ref in filestr: multiple_math_labels_with_refs.append(label) if multiple_math_labels_with_refs: errwarn(""" *** warning: detected non-align math environment with multiple labels (Sphinx cannot handle this equation system - labels will be removed and references to them will be empty):""") for label in multiple_math_labels_with_refs: errwarn(' label{%s}' % label) print filestr = insert_code_and_tex(filestr, code_blocks, tex_blocks, 'sphinx') # Remove all !bc ipy and !bc pyshell since interactive sessions # are automatically handled by sphinx without indentation # (just a blank line before and after) filestr = re.sub(r'^!bc +d?ipy *\n(.*?)^!ec *\n', '\n\g<1>\n', filestr, re.DOTALL | re.MULTILINE) filestr = re.sub(r'^!bc +d?pyshell *\n(.*?)^!ec *\n', '\n\g<1>\n', filestr, re.DOTALL | re.MULTILINE) # Check if we have custom pygments lexers if 'ipy' in code_block_types: if not has_custom_pygments_lexer('ipy'): envir2pygments['ipy'] = 'python' if 'do' in code_block_types: if not has_custom_pygments_lexer('doconce'): envir2pygments['do'] = 'text' # Make correct code-block:: language constructions legal_pygments_languages = get_legal_pygments_lexers() for key in set(code_block_types): if key in envir2pygments: if not envir2pygments[key] in legal_pygments_languages: errwarn( """*** warning: %s is not a legal Pygments language (lexer) found in line: %s The 'text' lexer will be used instead. """ % (envir2pygments[key], defs_line)) envir2pygments[key] = 'text' #filestr = re.sub(r'^!bc\s+%s\s*\n' % key, # '\n.. code-block:: %s\n\n' % envir2pygments[key], filestr, # flags=re.MULTILINE) # Check that we have code installed to handle pyscpro if 'pyscpro' in filestr and key == 'pyscpro': try: import icsecontrib.sagecellserver except ImportError: errwarn(""" *** warning: pyscpro for computer code (sage cells) is requested, but' icsecontrib.sagecellserver from https://github.com/kriskda/sphinx-sagecell is not installed. Using plain Python typesetting instead.""") key = 'pypro' if key == 'pyoptpro': if option('runestone'): filestr = re.sub( r'^!bc\s+%s\s*\n' % key, '\n.. codelens:: codelens_\n :showoutput:\n\n', filestr, flags=re.MULTILINE) else: filestr = re.sub(r'^!bc\s+%s\s*\n' % key, '\n.. raw:: html\n\n', filestr, flags=re.MULTILINE) elif key == 'pyscpro': if option('runestone'): filestr = re.sub(r'^!bc\s+%s\s*\n' % key, """ .. activecode:: activecode_ :language: python """, filestr, flags=re.MULTILINE) else: filestr = re.sub(r'^!bc\s+%s\s*\n' % key, '\n.. sagecellserver::\n\n', filestr, flags=re.MULTILINE) elif key == 'pysccod': if option('runestone'): # Include (i.e., run) all previous code segments... # NOTE: this is most likely not what we want include = ', '.join([i for i in range(1, activecode_counter)]) filestr = re.sub(r'^!bc\s+%s\s*\n' % key, """ .. activecode:: activecode_ :language: python "include: %s """ % include, filestr, flags=re.MULTILINE) else: errwarn( '*** error: pysccod for sphinx is not supported without the --runestone flag\n (but pyscpro is via Sage Cell Server)' ) _abort() elif key == '': # any !bc with/without argument becomes a text block: filestr = re.sub(r'^!bc$', '\n.. code-block:: text\n\n', filestr, flags=re.MULTILINE) elif key.endswith('hid'): if key in ('pyhid', 'jshid', 'htmlhid') and option('runestone'): # Allow runestone books to run hidden code blocks # (replace pyhid by pycod, then remove all !bc *hid) for i in range(len(code_block_types)): if code_block_types[i] == key: code_block_types[i] = key.replace('hid', 'cod') key2language = dict(py='python', js='javascript', html='html') language = key2language[key.replace('hid', '')] include = ', '.join([i for i in range(1, activecode_counter)]) filestr = re.sub(r'^!bc +%s\s*\n' % key, """ .. activecode:: activecode_ :language: %s :include: %s :hidecode: """ % (language, include), filestr, flags=re.MULTILINE) else: # Remove hidden code block pattern = r'^!bc +%s\n.+?^!ec' % key filestr = re.sub(pattern, '', filestr, flags=re.MULTILINE | re.DOTALL) else: show_hide = False if key.endswith('-h'): key_orig = key key = key[:-2] show_hide = True # Use the standard sphinx code-block directive if key in envir2pygments: pygments_language = envir2pygments[key] elif key in legal_pygments_languages: pygments_language = key else: errwarn('*** error: detected code environment "%s"' % key) errwarn( ' which is not registered in sphinx.py (sphinx_code)') errwarn(' or not a language registered in pygments') _abort() if show_hide: filestr = re.sub(r'^!bc +%s\s*\n' % key_orig, '\n.. container:: toggle\n\n .. container:: header\n\n **Show/Hide Code**\n\n .. code-block:: %s\n\n' % \ pygments_language, filestr, flags=re.MULTILINE) # Must add 4 indent in corresponding code_blocks[i], done above else: filestr = re.sub(r'^!bc +%s\s*\n' % key, '\n.. code-block:: %s\n\n' % \ pygments_language, filestr, flags=re.MULTILINE) # any !bc with/without argument becomes a text block: filestr = re.sub(r'^!bc.*$', '\n.. code-block:: text\n\n', filestr, flags=re.MULTILINE) filestr = re.sub(r'^!ec *\n', '\n', filestr, flags=re.MULTILINE) #filestr = re.sub(r'^!ec\n', '\n', filestr, flags=re.MULTILINE) #filestr = re.sub(r'^!ec\n', '', filestr, flags=re.MULTILINE) filestr = re.sub(r'^!bt *\n', '\n.. math::\n', filestr, flags=re.MULTILINE) filestr = re.sub(r'^!et *\n', '\n', filestr, flags=re.MULTILINE) # Fix lacking blank line after :label: filestr = re.sub(r'^( :label: .+?)(\n *[^ ]+)', r'\g<1>\n\n\g<2>', filestr, flags=re.MULTILINE) # Insert counters for runestone blocks if option('runestone'): codelens_counter = 0 activecode_counter = 0 lines = filestr.splitlines() for i in range(len(lines)): if '.. codelens:: codelens_' in lines[i]: codelens_counter += 1 lines[i] = lines[i].replace('codelens_', 'codelens_%d' % codelens_counter) if '.. activecode:: activecode_' in lines[i]: activecode_counter += 1 lines[i] = lines[i].replace( 'activecode_', 'activecode_%d' % activecode_counter) filestr = '\n'.join(lines) # Final fixes filestr = fix_underlines_in_headings(filestr) # Ensure blank line before and after comments filestr = re.sub(r'([.:;?!])\n^\.\. ', r'\g<1>\n\n.. ', filestr, flags=re.MULTILINE) filestr = re.sub(r'(^\.\. .+)\n([^ \n]+)', r'\g<1>\n\n\g<2>', filestr, flags=re.MULTILINE) # Line breaks interfer with tables and needs a final blank line too lines = filestr.splitlines() inside_block = False for i in range(len(lines)): if lines[i].startswith('<linebreakpipe>') and not inside_block: inside_block = True lines[i] = lines[i].replace('<linebreakpipe> ', '') + '\n' continue if lines[i].startswith('<linebreakpipe>') and inside_block: lines[i] = '|' + lines[i].replace('<linebreakpipe>', '') continue if inside_block and not lines[i].startswith('<linebreakpipe>'): inside_block = False lines[i] = '| ' + lines[i] + '\n' filestr = '\n'.join(lines) # Remove double !split (TOC with a prefix !split gives two !splits) pattern = '^.. !split\s+.. !split' filestr = re.sub(pattern, '.. !split', filestr, flags=re.MULTILINE) if option('html_links_in_new_window'): # Insert a comment to be recognized by automake_sphinx.py such that it # can replace the default links by proper modified target= option. #filestr = '\n\n.. NOTE: Open external links in new windows.\n\n' + filestr # Use JavaScript instead filestr = """.. raw:: html <script type="text/javascript"> $(document).ready(function() { $("a[href^='http']").attr('target','_blank'); }); </script> """ + filestr # Remove too much vertical space filestr = re.sub(r'\n{3,}', '\n\n', filestr) return filestr
def sphinx_quiz(quiz): if option('runestone'): return sphinx_quiz_runestone(quiz) else: return rst_quiz(quiz)
def sphinx_figure(m): result = '' # m is a MatchObject filename = m.group('filename') caption = m.group('caption').strip() # Stubstitute DocOnce label by rst label in caption # (also, remove final period in caption since caption is used as hyperlink # text to figures). m_label = re.search(r'label\{(.+?)\}', caption) if m_label: label = m_label.group(1) result += '\n.. _%s:\n' % label # remove . at the end of the caption text parts = caption.split('label') parts[0] = parts[0].rstrip() if parts[0] and parts[0][-1] == '.': parts[0] = parts[0][:-1] parts[0] = parts[0].strip() # insert emphasize marks if not latex $ at the # beginning or end (math subst does not work for *$I=1$*) # or if not boldface or emphasize already in the caption caption_font = option('sphinx_figure_captions=', 'emphasize') if parts[0] and \ caption_font == 'emphasize' and \ not parts[0].startswith('$') and \ not parts[0].endswith('$') and \ not '*' in parts[0] and \ not '_' in parts[0]: parts[0] = '*' + parts[0] + '*' #caption = ' label'.join(parts) caption = parts[0] # contrary to rst_figure, we do not write label into caption # since we just want to remove the whole label as part of # the caption (otherwise done when handling ref and label) else: if caption and caption[-1] == '.': caption = caption[:-1] # math is ignored in references to figures, test for math only if caption.startswith('$') and caption.endswith('$'): errwarn( '*** warning: math only in sphinx figure caption (it will be ignored by sphinx, resulting in empty caption)\n %s\n FIGURE: [%s' % (caption, filename)) #stem = os.path.splitext(filename)[0] #result += '\n.. figure:: ' + stem + '.*\n' # utilize flexibility # does not work yet result += '\n.. figure:: ' + filename + '\n' opts = m.group('options') if opts: # opts: width=600 frac=0.5 align=center # opts: width=600, frac=0.5, align=center info = [s.split('=') for s in opts.split()] fig_info = [ ' :%s: %s' % (opt, value.replace(',', '')) for opt, value in info if opt not in ['frac', 'sidecap'] ] result += '\n'.join(fig_info) if caption: result += '\n\n ' + caption + '\n' else: result += '\n\n' #errwarn('sphinx figure: caption=\n', caption, '\nresult:\n', result) return result
def sphinx_quiz_runestone(quiz): quiz_feedback = option('quiz_explanations=', 'on') text = '' if 'new page' in quiz: text += '.. !split\n%s\n%s' % (quiz['new page'], '-' * len(quiz['new page'])) text += '.. begin quiz\n\n' global question_counter question_counter += 1 # Multiple correct answers? if sum([1 for choice in quiz['choices'] if choice[0] == 'right']) > 1: text += '.. mchoicema:: question_%d' % question_counter + '\n' else: text += '.. mchoicemf:: question_%d' % question_counter + '\n' def fix_text(s, tp='answer'): """ Answers and feedback in RunestoneInteractive book quizzes cannot contain math, figure and rst markup. Perform fixes. """ drop = False if 'math::' in s: errwarn('\n*** warning: quiz %s with math block not supported:' % tp) errwarn(s) drop = True if '.. code-block::' in s: errwarn('\n*** warning: quiz %s with code block not supported:' % tp) errwarn(s) drop = True if '.. figure::' in s: errwarn('\n*** warning: quiz %s with figure not supported:' % tp) errwarn(s) drop = True if drop: return '' # Make multi-line paragraph a one-liner s = ' '.join(s.splitlines()).rstrip() # Fixes pattern = r'`(.+?) (<https?.+?)>`__' # URL s = re.sub(pattern, '<a href="\g<2>"> \g<1> </a>', s) pattern = r'``(.+?)``' # verbatim s = re.sub(pattern, '<tt>\g<1></tt>', s) pattern = r':math:`(.+?)`' # inline math s = re.sub(pattern, '<em>\g<1></em>', s) # mimic italic.... pattern = r':\*(.+?)\*' # emphasize s = re.sub(pattern, '\g<1>', s, flags=re.DOTALL) return s import string correct = [] for i, choice in enumerate(quiz['choices']): if i > 4: # not supported errwarn( '*** warning: quiz with %d choices gets truncated (first 5)' % len(quiz['choices'])) break letter = string.ascii_lowercase[i] text += ' :answer_%s: ' % letter answer = fix_text(choice[1], tp='answer') if not answer: answer = 'Too advanced typesetting prevents the text from being rendered' text += answer + '\n' if choice[0] == 'right': correct.append(letter) if correct: text += ' :correct: ' + ', '.join(correct) + '\n' else: errwarn( '*** error: correct choice in quiz has index > 5 (max 5 allowed for RunestoneInteractive books)' ) errwarn(quiz['question']) _abort() for i, choice in enumerate(quiz['choices']): if i > 4: # not supported break letter = string.ascii_lowercase[i] text += ' :feedback_%s: ' % letter # must be present if len(choice) == 3 and quiz_feedback == 'on': feedback = fix_text(choice[2], tp='explanation') if not feedback: feedback = '(Too advanced typesetting prevents the text from being rendered)' text += feedback text += '\n' text += '\n' + indent_lines(quiz['question'], 'sphinx', ' ' * 3) + '\n\n\n' return text
def insert_code_and_tex(filestr, code_blocks, tex_blocks, format, complete_doc=True): # Consistency check (only for complete documents): # find no of distinct code and math blocks # (can be duplicates when solutions are copied at the end) pattern = r'^\d+ ' + _CODE_BLOCK code_lines = re.findall(pattern, filestr, flags=re.MULTILINE) n = len(set(code_lines)) if complete_doc and len(code_blocks) != n: print '*** error: found %d code block markers for %d initial code blocks' % (n, len(code_blocks)) print """ Possible causes: - mismatch of !bt and !et within one file, such that a !bt swallows code - mismatch of !bt and !et across files in multi-file documents - !bc and !ec inside code blocks - replace by |bc and |ec (run doconce on each individual file to locate the problem, then on smaller and smaller parts of each file)""" numbers = range(len(code_blocks)) # expected numbers in code blocks for e in code_lines: # remove number number = int(e.split()[0]) if number not in numbers: print ' Problem: found %s, but the number %d was unexpected' % (e, number) else: numbers.remove(number) if numbers: print ' Problem: did not find XX <<<!!CODE_BLOCK for XX in', numbers _abort() pattern = r'^\d+ ' + _MATH_BLOCK n = len(set(re.findall(pattern, filestr, flags=re.MULTILINE))) if complete_doc and len(tex_blocks) != n: print '*** error: found %d tex block markers for %d initial tex blocks\nAbort!' % (n, len(tex_blocks)) print """ Possible causes: - mismatch of !bc and !ec within one file, such that a !bc swallows tex blocks - mismatch of !bc and !ec across files in multi-file documents - !bt and !et inside code blocks - replace by |bt and |et (run doconce on each file to locate the problem, then on smaller and smaller parts of each file)""" _abort() from misc import option max_linelength = option('max_bc_linelength=', None) if max_linelength is not None: max_linelength = int(max_linelength) for i in range(len(code_blocks)): lines = code_blocks[i].splitlines() truncated = False for j in range(len(lines)): if len(lines[j]) > max_linelength: lines[j] = lines[j][:max_linelength] + '...' truncated = True if truncated: code_blocks[i] = '\n'.join(lines) + '\n' lines = filestr.splitlines() # Note: re.sub cannot be used because newlines, \nabla, etc # are not handled correctly. Need str.replace. for i in range(len(lines)): if _CODE_BLOCK in lines[i] or _MATH_BLOCK in lines[i]: words = lines[i].split() # on a line: number block-indicator code-type n = int(words[0]) if _CODE_BLOCK in lines[i]: words[1] = '!bc' code = code_blocks[n] lines[i] = ' '.join(words[1:]) + '\n' + code + '!ec' if _MATH_BLOCK in lines[i]: words[1] = '!bc' math = tex_blocks[n] lines[i] = '!bt\n' + math + '!et' filestr = safe_join(lines, '\n') # All formats except sphinx and ipynb must remove !bc *hid blocks # (maybe html will get the possibility to run hidden blocks) if format not in ('sphinx', 'ipynb'): filestr = remove_hidden_code_blocks(filestr, format) return filestr
def define(FILENAME_EXTENSION, BLANKLINE, INLINE_TAGS_SUBST, CODE, LIST, ARGLIST, TABLE, EXERCISE, FIGURE_EXT, CROSS_REFS, INDEX_BIB, TOC, ENVIRS, QUIZ, INTRO, OUTRO, filestr): # all arguments are dicts and accept in-place modifications (extensions) FILENAME_EXTENSION['xml'] = '.xml' # output file extension BLANKLINE['xml'] = '\n<blankline />\n' # blank input line => new paragraph BLANKLINE['xml'] = '\n' INLINE_TAGS_SUBST['xml'] = { # from inline tags to HTML tags # keep math as is: 'math': r'\g<begin><inlinemath>\g<subst></inlinemath>\g<end>', 'math2': r'\g<begin><inlinemath>\g<latexmath></inlinemath>\g<end>', 'emphasize': r'\g<begin><em>\g<subst></em>\g<end>', 'bold': r'\g<begin><b>\g<subst></b>\g<end>', 'verbatim': r'\g<begin><inlinecode>\g<subst></inlinecode>\g<end>', 'colortext': r'<color type="\g<color>">\g<text></color>', #'linkURL': r'\g<begin><a href="\g<url>">\g<link></a>\g<end>', 'linkURL2': r'<link url="\g<url>">\g<link></link>', 'linkURL3': r'<link url="\g<url>">\g<link></link>', 'linkURL2v': r'<link url="\g<url>"><inlinecode>\g<link></inlinecode></link>', 'linkURL3v': r'<link url="\g<url>"><inlinecode>\g<link></inlinecode></link>', 'plainURL': r'<link url="\g<url>"><inlinecode>\g<url></inlinecode></link>', 'inlinecomment': r'<inlinecomment name="\g<name>">\g<comment></inlinecomment>', 'chapter': r'\n<h1>\g<subst></h1>', 'section': r'\n<h2>\g<subst></h2>', 'subsection': r'\n<h3>\g<subst></h3>', 'subsubsection': r'\n<h4>\g<subst></h4>\n', 'paragraph': r'<b>\g<subst></b>\g<space>', 'abstract': r'<abstract type="\g<type>">\n\g<text>\n</abstract>\n\g<rest>', 'title': r'\n<title>\g<subst></title>', 'date': r'<date>\n\g<subst>\n</date>', 'author': xml_author, 'figure': xml_figure, 'movie': xml_movie, 'comment': '<comment>%s</comment>', 'footnote': xml_footnotes, 'non-breaking-space': '<nonbreakingspace />', 'horizontal-rule': '<horizontalrule />', 'linebreak': r'\g<text><linebreak />', 'ampersand2': r' \g<1><ampersand />\g<2>', 'ampersand1': r'\g<1> <ampersand /> \g<2>', } if option('wordpress'): INLINE_TAGS_SUBST['xml'].update({ 'math': r'\g<begin>$latex \g<subst>$\g<end>', 'math2': r'\g<begin>$latex \g<latexmath>$\g<end>' }) ENVIRS['xml'] = {} CODE['xml'] = xml_code # how to typeset lists and their items in html: LIST['xml'] = { 'itemize': {'begin': '\n<ul>\n', 'item': '<li>', 'end': '</ul>\n\n'}, 'enumerate': {'begin': '\n<ol>\n', 'item': '<li>', 'end': '</ol>\n\n'}, 'description': {'begin': '\n<dl>\n', 'item': '<dt>%s<dd>', 'end': '</dl>\n\n'}, 'separator': '<newline/>', } # how to typeset description lists for function arguments, return # values, and module/class variables: ARGLIST['xml'] = { 'parameter': '<b>argument</b>', 'keyword': '<b>keyword argument</b>', 'return': '<b>return value(s)</b>', 'instance variable': '<b>instance variable</b>', 'class variable': '<b>class variable</b>', 'module variable': '<b>module variable</b>', } FIGURE_EXT['xml'] = { 'search': ('.png', '.gif', '.jpg', '.jpeg'), 'convert': ('.png', '.gif', '.jpg')} CROSS_REFS['xml'] = xml_ref_and_label TABLE['xml'] = xml_table INDEX_BIB['xml'] = xml_index_bib EXERCISE['xml'] = plain_exercise TOC['xml'] = xml_toc QUIZ['xml'] = xml_quiz INTRO['xml'] = """\ <?xml version="1.0" encoding="utf-8"?> """ keywords = re.findall(r'idx\{(.+?)\}', filestr) # idx with verbatim is usually too specialized - remove them keywords = [keyword for keyword in keywords if not '`' in keyword] # keyword!subkeyword -> keyword subkeyword keywords = ','.join(keywords).replace('!', ' ') if keywords: meta_tags = '<meta name="keywords" content="%s">\n' % keywords INTRO['xml'] += """\ <keywords>%s</keywords> """ % (meta_tags) # document ending: OUTRO['xml'] = ""
def ipynb_movie(m): # m.group() must be called before m.group('name') text = '<!-- dom:%s -->' % m.group() global html_encountered, movie_encountered, movie_files filename = m.group('filename') caption = m.group('caption').strip() youtube = False if 'youtu.be' in filename or 'youtube.com' in filename: youtube = True if '*' in filename or '->' in filename: errwarn('*** warning: * or -> in movie filenames is not supported in ipynb') return text def YouTubeVideo(filename): # Use YouTubeVideo object if 'watch?v=' in filename: name = filename.split('watch?v=')[1] elif 'youtu.be/' in filename: name = filename.split('youtu.be/')[1] else: errwarn('*** error: youtube movie name "%s" could not be interpreted' % filename) _abort() text = '' global movie_encountered if not movie_encountered: text += 'from IPython.display import YouTubeVideo\n' movie_encountered = True text += 'YouTubeVideo("%s")\n' % name return text text += '\n<!-- begin movie -->\n' display_method = option('ipynb_movie=', 'HTML') if display_method == 'md': text += html_movie(m) elif display_method.startswith('HTML'): text += '\n!bc pycod\n' if youtube and 'YouTube' in display_method: text += YouTubeVideo(filename) if caption: text += '\nprint "%s"' % caption else: # Use HTML formatting if not html_encountered: text += 'from IPython.display import HTML\n' html_encountered = True text += '_s = """' + html_movie(m) + '"""\n' text += 'HTML(_s)\n' if not filename.startswith('http'): movie_files.append(filename) text += '!ec\n' elif display_method == 'ipynb': text += '!bc pycod\n' if youtube: text += YouTubeVideo(filename) if caption: text += '\nprint "%s"' % caption else: # see http://nbviewer.ipython.org/github/ipython/ipython/blob/1.x/examples/notebooks/Part%205%20-%20Rich%20Display%20System.ipynb # http://stackoverflow.com/questions/18019477/how-can-i-play-a-local-video-in-my-ipython-notebook # http://python.6.x6.nabble.com/IPython-User-embedding-non-YouTube-movies-in-the-IPython-notebook-td5024035.html # Just support .mp4, .ogg, and.webm stem, ext = os.path.splitext(filename) if ext not in ('.mp4', '.ogg', '.webm'): errwarn('*** error: movie "%s" in format %s is not supported for --ipynb_movie=%s' % (filename, ext, display_method)) errwarn(' use --ipynb_movie=HTML instead') _abort() height = 365 width = 640 if filename.startswith('http'): file_open = 'import urllib\nvideo = urllib.urlopen("%s").read()' % filename else: file_open = 'video = open("%s", "rb").read()' % filename text += """ %s from base64 import b64encode video_encoded = b64encode(video) video_tag = '<video controls loop alt="%s" height="%s" width="%s" src="data:video/%s;base64,{0}">'.format(video_encoded) """ % (file_open, filename, height, width, ext[1:]) if not filename.startswith('http'): movie_files.append(filename) if not html_encountered: text += 'from IPython.display import HTML\n' html_encountered = True text += 'HTML(data=video_tag)\n' if caption: text += '\nprint "%s"' % caption text += '!ec\n' else: errwarn('*** error: --ipynb_movie=%s is not supported' % display_method) _abort() text += '<!-- end movie -->\n' return text
def define(FILENAME_EXTENSION, BLANKLINE, INLINE_TAGS_SUBST, CODE, LIST, ARGLIST, TABLE, EXERCISE, FIGURE_EXT, CROSS_REFS, INDEX_BIB, TOC, ENVIRS, QUIZ, INTRO, OUTRO, filestr): # all arguments are dicts and accept in-place modifications (extensions) FILENAME_EXTENSION['ipynb'] = '.ipynb' BLANKLINE['ipynb'] = '\n' # replacement patterns for substitutions of inline tags INLINE_TAGS_SUBST['ipynb'] = { 'math': None, # indicates no substitution, leave as is 'math2': r'\g<begin>$\g<latexmath>$\g<end>', 'emphasize': None, 'bold': r'\g<begin>**\g<subst>**\g<end>', 'figure': ipynb_figure, 'movie': ipynb_movie, 'verbatim': None, #'linkURL': r'\g<begin>\g<link> (\g<url>)\g<end>', 'linkURL2': r'[\g<link>](\g<url>)', 'linkURL3': r'[\g<link>](\g<url>)', 'linkURL2v': r'[`\g<link>`](\g<url>)', 'linkURL3v': r'[`\g<link>`](\g<url>)', 'plainURL': r'<\g<url>>', 'colortext': r'<font color="\g<color>">\g<text></font>', 'title': r'<!-- dom:TITLE: \g<subst> -->\n# \g<subst>', 'author': ipynb_author, 'date': '\nDate: _\g<subst>_\n', 'chapter': lambda m: '# ' + m.group('subst'), # seldom used in notebooks 'section': lambda m: '# ' + m.group('subst'), 'subsection': lambda m: '## ' + m.group('subst'), 'subsubsection': lambda m: '### ' + m.group('subst') + '\n', 'paragraph': r'**\g<subst>**\g<space>', 'abstract': r'\n**\g<type>.** \g<text>\n\n\g<rest>', 'comment': '<!-- %s -->', 'linebreak': r'\g<text>', # Not sure how this is supported; Markdown applies <br> but that cannot be used for latex output with ipynb... 'non-breaking-space': ' ', 'ampersand2': r' \g<1>&\g<2>', } CODE['ipynb'] = ipynb_code # Envirs are in doconce.py treated after code is inserted, which # means that the ipynb format is json. Therefore, we need special # treatment of envirs in ipynb_code and ENVIRS can be empty. ENVIRS['ipynb'] = {} from common import DEFAULT_ARGLIST ARGLIST['ipynb'] = DEFAULT_ARGLIST LIST['ipynb'] = { 'itemize': { 'begin': '', 'item': '*', 'end': '\n' }, 'enumerate': { 'begin': '', 'item': '%d.', 'end': '\n' }, 'description': { 'begin': '', 'item': '%s\n : ', 'end': '\n' }, 'separator': '\n', } CROSS_REFS['ipynb'] = pandoc_ref_and_label TABLE['ipynb'] = ipynb_table cite = option('ipynb_cite=', 'plain') if cite == 'latex': INDEX_BIB['ipynb'] = ipynb_index_bib else: INDEX_BIB['ipynb'] = pandoc_index_bib EXERCISE['ipynb'] = plain_exercise TOC['ipynb'] = lambda s: '' FIGURE_EXT['ipynb'] = { 'search': ('.png', '.gif', '.jpg', '.jpeg', '.tif', '.tiff', '.pdf'), 'convert': ('.png', '.gif', '.jpg') } QUIZ['ipynb'] = pandoc_quiz
def ipynb_figure(m): # m.group() must be called before m.group('name') text = '<!-- dom:%s -->\n<!-- begin figure -->\n' % m.group() filename = m.group('filename') caption = m.group('caption').strip() opts = m.group('options').strip() if opts: info = [s.split('=') for s in opts.split()] opts = ' ' .join(['%s=%s' % (opt, value) for opt, value in info if opt not in ['frac', 'sidecap']]) global figure_files if not filename.startswith('http'): figure_files.append(filename) # Extract optional label in caption label = None pattern = r' *label\{(.+?)\}' m = re.search(pattern, caption) if m: label = m.group(1).strip() caption = re.sub(pattern, '', caption) display_method = option('ipynb_figure=', 'imgtag') if display_method == 'md': # Markdown image syntax for embedded image in text # (no control of size, then one must use HTML syntax) if label is not None: #text += '<a name="%s"></a>\n' % label text += '<div id="%s"></div>\n' % label text += '![%s](%s)' % (caption, filename) elif display_method == 'imgtag': # Plain <img tag, allows specifying the image size if label is not None: #text += '<a name="%s"></a>' % label text += '<div id="%s"></div>\n' % label # Fix caption markup so it becomes html from doconce import INLINE_TAGS_SUBST, INLINE_TAGS for tag in 'bold', 'emphasize', 'verbatim': caption = re.sub(INLINE_TAGS[tag], INLINE_TAGS_SUBST['html'][tag], caption, flags=re.MULTILINE) text += """ <p>%s</p> <img src="%s" %s> """ % (caption, filename, opts) elif display_method == 'Image': # Image object # NOTE: This code will normally not work because it inserts a verbatim # block in the file *after* all such blocks have been removed and # numbered. doconce.py makes a test prior to removal of blocks and # runs the handle_figures and movie substitution if ipynb format # and Image or movie object display. text += '\n' if label is not None: text += '<div id="%s"></div>' % label text += '<!-- options: %s -->\n' % opts text = '!bc pycod\n' global figure_encountered if not figure_encountered: # First time we have a figure, we must import Image text += 'from IPython.display import Image\n' figure_encountered = True if caption: text += '# ' + caption if filename.startswith('http'): keyword = 'url' else: keyword = 'filename' text += 'Image(%s="%s")\n' % (keyword, filename) text += '!ec\n' else: errwarn('*** error: --ipynb_figure=%s is illegal, must be md, imgtag or Image' % display_method) _abort() text += '<!-- end figure -->\n' return text
def ipynb_figure(m): # m.group() must be called before m.group('name') text = '<!-- dom:%s -->\n<!-- begin figure -->\n' % m.group() filename = m.group('filename') caption = m.group('caption').strip() opts = m.group('options').strip() if opts: info = [s.split('=') for s in opts.split()] opts = ' '.join([ '%s=%s' % (opt, value) for opt, value in info if opt not in ['frac', 'sidecap'] ]) global figure_files if not filename.startswith('http'): figure_files.append(filename) # Extract optional label in caption label = None pattern = r' *label\{(.+?)\}' m = re.search(pattern, caption) if m: label = m.group(1).strip() caption = re.sub(pattern, '', caption) display_method = option('ipynb_figure=', 'imgtag') if display_method == 'md': # Markdown image syntax for embedded image in text # (no control of size, then one must use HTML syntax) if label is not None: #text += '<a name="%s"></a>\n' % label text += '<div id="%s"></div>\n' % label text += '![%s](%s)' % (caption, filename) elif display_method == 'imgtag': # Plain <img tag, allows specifying the image size if label is not None: #text += '<a name="%s"></a>' % label text += '<div id="%s"></div>\n' % label text += """ <p>%s</p> <img src="%s" %s> """ % (caption, filename, opts) elif display_method == 'Image': # Image object # NOTE: This code will normally not work because it inserts a verbatim # block in the file *after* all such blocks have been removed and # numbered. doconce.py makes a test prior to removal of blocks and # runs the handle_figures and movie substitution if ipynb format # and Image or movie object display. text += '\n' if label is not None: text += '<div id="%s"></div>' % label text += '<!-- options: %s -->\n' % opts text = '!bc pycod\n' global figure_encountered if not figure_encountered: # First time we have a figure, we must import Image text += 'from IPython.display import Image\n' figure_encountered = True if caption: text += '# ' + caption if filename.startswith('http'): keyword = 'url' else: keyword = 'filename' text += 'Image(%s="%s")\n' % (keyword, filename) text += '!ec\n' else: print '*** error: --ipynb_figure=%s is illegal, must be md, imgtag or Image' % display_method _abort() text += '<!-- end figure -->\n' return text
def define(FILENAME_EXTENSION, BLANKLINE, INLINE_TAGS_SUBST, CODE, LIST, ARGLIST, TABLE, EXERCISE, FIGURE_EXT, CROSS_REFS, INDEX_BIB, TOC, ENVIRS, QUIZ, INTRO, OUTRO, filestr): # all arguments are dicts and accept in-place modifications (extensions) FILENAME_EXTENSION['ipynb'] = '.ipynb' BLANKLINE['ipynb'] = '\n' # replacement patterns for substitutions of inline tags INLINE_TAGS_SUBST['ipynb'] = { 'math': None, # indicates no substitution, leave as is 'math2': r'\g<begin>$\g<latexmath>$\g<end>', 'emphasize': None, 'bold': r'\g<begin>**\g<subst>**\g<end>', 'figure': ipynb_figure, 'movie': ipynb_movie, 'verbatim': None, #'linkURL': r'\g<begin>\g<link> (\g<url>)\g<end>', 'linkURL2': r'[\g<link>](\g<url>)', 'linkURL3': r'[\g<link>](\g<url>)', 'linkURL2v': r'[`\g<link>`](\g<url>)', 'linkURL3v': r'[`\g<link>`](\g<url>)', 'plainURL': r'<\g<url>>', 'colortext': r'<font color="\g<color>">\g<text></font>', 'title': r'<!-- dom:TITLE: \g<subst> -->\n# \g<subst>', 'author': ipynb_author, 'date': '\nDate: _\g<subst>_\n', 'chapter': lambda m: '# ' + m.group('subst'), # seldom used in notebooks 'section': lambda m: '# ' + m.group('subst'), 'subsection': lambda m: '## ' + m.group('subst'), 'subsubsection': lambda m: '### ' + m.group('subst') + '\n', 'paragraph': r'**\g<subst>**\g<space>', 'abstract': r'\n**\g<type>.** \g<text>\n\n\g<rest>', 'comment': '<!-- %s -->', 'linebreak': r'\g<text>', # Not sure how this is supported; Markdown applies <br> but that cannot be used for latex output with ipynb... 'non-breaking-space': ' ', 'ampersand2': r' \g<1>&\g<2>', } CODE['ipynb'] = ipynb_code # Envirs are in doconce.py treated after code is inserted, which # means that the ipynb format is json. Therefore, we need special # treatment of envirs in ipynb_code and ENVIRS can be empty. ENVIRS['ipynb'] = {} from common import DEFAULT_ARGLIST ARGLIST['ipynb'] = DEFAULT_ARGLIST LIST['ipynb'] = { 'itemize': {'begin': '', 'item': '*', 'end': '\n'}, 'enumerate': {'begin': '', 'item': '%d.', 'end': '\n'}, 'description': {'begin': '', 'item': '%s\n : ', 'end': '\n'}, 'separator': '\n', } CROSS_REFS['ipynb'] = pandoc_ref_and_label TABLE['ipynb'] = ipynb_table cite = option('ipynb_cite=', 'plain') if cite == 'latex': INDEX_BIB['ipynb'] = ipynb_index_bib else: INDEX_BIB['ipynb'] = pandoc_index_bib EXERCISE['ipynb'] = plain_exercise TOC['ipynb'] = lambda s: '' FIGURE_EXT['ipynb'] = { 'search': ('.png', '.gif', '.jpg', '.jpeg', '.tif', '.tiff', '.pdf'), 'convert': ('.png', '.gif', '.jpg')} QUIZ['ipynb'] = pandoc_quiz
def ipynb_code(filestr, code_blocks, code_block_types, tex_blocks, format): """ # We expand all newcommands now from html import embed_newcommands newcommands = embed_newcommands(filestr) if newcommands: filestr = newcommands + filestr """ # Fix pandoc citations to normal internal links: [[key]](#key) filestr = re.sub(r'\[@(.+?)\]', r'[[\g<1>]](#\g<1>)', filestr) # filestr becomes json list after this function so we must typeset # envirs here. All envirs are typeset as pandoc_quote. from common import _CODE_BLOCK, _MATH_BLOCK envir_format = option('ipynb_admon=', 'paragraph') # Remove all !bpop-!epop environments (they cause only problens and # have no use) for envir in 'pop', 'slidecell': filestr = re.sub('^<!-- !b%s .*\n' % envir, '', filestr, flags=re.MULTILINE) filestr = re.sub('^<!-- !e%s .*\n' % envir, '', filestr, flags=re.MULTILINE) filestr = re.sub('^<!-- !bnotes.*?<!-- !enotes -->\n', '', filestr, flags=re.DOTALL | re.MULTILINE) filestr = re.sub('^<!-- !split -->\n', '', filestr, flags=re.MULTILINE) from doconce import doconce_envirs envirs = doconce_envirs()[8:-2] for envir in envirs: pattern = r'^!b%s(.*?)\n(.+?)\s*^!e%s' % (envir, envir) if envir_format in ('quote', 'paragraph', 'hrule'): def subst(m): title = m.group(1).strip() # Text size specified in parenthesis? m2 = re.search('^\s*\((.+?)\)', title) if title == '' and envir not in ('block', 'quote'): title = envir.capitalize() + '.' elif title.lower() == 'none': title == '' elif m2: text_size = m2.group(1).lower() title = title.replace('(%s)' % text_size, '').strip() elif title and title[-1] not in ('.', ':', '!', '?'): # Make sure the title ends with puncuation title += '.' # Recall that this formatting is called very late # so native format must be used! if title: title = '**' + title + '**\n' # Could also consider subsubsection formatting block = m.group(2) # Always use quote typesetting for quotes if envir_format == 'quote' or envir == 'quote': # Make Markdown quote of the block: lines start with > lines = [] for line in block.splitlines(): # Just quote plain text if not (_MATH_BLOCK in line or _CODE_BLOCK in line or line.startswith('FIGURE:') or line.startswith('MOVIE:') or line.startswith('|')): lines.append('> ' + line) else: lines.append('\n' + line + '\n') block = '\n'.join(lines) + '\n\n' # Add quote and a blank line after title if title: title = '> ' + title + '>\n' else: # Add a blank line after title if title: title += '\n' if envir_format == 'hrule': # Native ------ does not work, use <hr/> #text = '\n\n----------\n' + title + '----------\n' + \ # block + '\n----------\n\n' text = '\n\n<hr/>\n' + title + \ block + '\n<hr/>\n\n' else: text = title + block + '\n\n' return text else: print '*** error: --ipynb_admon=%s is not supported' % envir_format filestr = re.sub(pattern, subst, filestr, flags=re.DOTALL | re.MULTILINE) # Fix pyshell and ipy interactive sessions: remove prompt and output. # or split in multiple cells such that output comes out at the end of a cell # Fix sys environments and use run prog.py so programs can be run in cell # Insert %matplotlib inline in the first block using matplotlib # Only typeset Python code as blocks, otherwise !bc environmens # become plain indented Markdown. from doconce import dofile_basename from sets import Set ipynb_tarfile = 'ipynb-%s-src.tar.gz' % dofile_basename src_paths = Set() mpl_inline = False split_pyshell = option('ipynb_split_pyshell=', 'on') if split_pyshell is None: split_pyshell = False elif split_pyshell in ('no', 'False', 'off'): split_pyshell = False else: split_pyshell = True ipynb_code_tp = [None] * len(code_blocks) for i in range(len(code_blocks)): # Check if continuation lines are in the code block, because # doconce.py inserts a blank after the backslash if '\\ \n' in code_blocks[i]: code_blocks[i] = code_blocks[i].replace('\\ \n', '\\\n') if not mpl_inline and ( re.search(r'import +matplotlib', code_blocks[i]) or \ re.search(r'from +matplotlib', code_blocks[i]) or \ re.search(r'import +scitools', code_blocks[i]) or \ re.search(r'from +scitools', code_blocks[i])): code_blocks[i] = '%matplotlib inline\n\n' + code_blocks[i] mpl_inline = True tp = code_block_types[i] if tp.endswith('-t'): # Standard Markdown code with pandoc/github extension language = tp[:-2] language_spec = language2pandoc.get(language, '') #code_blocks[i] = '\n' + indent_lines(code_blocks[i], format) + '\n' code_blocks[i] = "```%s\n" % language_spec + \ indent_lines(code_blocks[i].strip(), format) + \ "```" ipynb_code_tp[i] = 'markdown' elif tp.startswith('pyshell') or tp.startswith('ipy'): lines = code_blocks[i].splitlines() last_cell_end = -1 if split_pyshell: new_code_blocks = [] # Split for each output an put in separate cell for j in range(len(lines)): if lines[j].startswith('>>>') or lines[j].startswith( '... '): lines[j] = lines[j][4:] elif lines[j].startswith('In ['): # IPython lines[j] = ':'.join(lines[j].split(':')[1:]).strip() elif lines[j].startswith(' ...: '): # IPython lines[j] = lines[j][8:] else: # output (no prefix or Out) lines[j] = '' new_code_blocks.append('\n'.join(lines[last_cell_end + 1:j + 1])) last_cell_end = j code_blocks[i] = new_code_blocks ipynb_code_tp[i] = 'cell' else: # Remove prompt and output lines; leave code executable in cell for j in range(len(lines)): if lines[j].startswith('>>> ') or lines[j].startswith( '... '): lines[j] = lines[j][4:] elif lines[j].startswith('In ['): lines[j] = ':'.join(lines[j].split(':')[1:]).strip() else: # output lines[j] = '' for j in range(lines.count('')): lines.remove('') code_blocks[i] = '\n'.join(lines) ipynb_code_tp[i] = 'cell' elif tp.startswith('sys'): # Do we find execution of python file? If so, copy the file # to separate subdir and make a run file command in a cell. # Otherwise, it is just a plain verbatim Markdown block. found_unix_lines = False lines = code_blocks[i].splitlines() for j in range(len(lines)): m = re.search(r'(.+?>|\$) *python +([A-Za-z_0-9]+?\.py)', lines[j]) if m: name = m.group(2).strip() if os.path.isfile(name): src_paths.add(os.path.dirname(name)) lines[j] = '%%run "%s"' % fullpath else: found_unix_lines = True src_paths = list(src_paths) if src_paths and not found_unix_lines: # This is a sys block with run commands only code_blocks[i] = '\n'.join(lines) ipynb_code_tp[i] = 'cell' else: # Standard Markdown code code_blocks[i] = '\n'.join(lines) code_blocks[i] = indent_lines(code_blocks[i], format) ipynb_code_tp[i] = 'markdown' elif tp.endswith('hid'): ipynb_code_tp[i] = 'cell_hidden' elif tp.startswith('py'): ipynb_code_tp[i] = 'cell' else: # Should support other languages as well, but not for now code_blocks[i] = indent_lines(code_blocks[i], format) ipynb_code_tp[i] = 'markdown' # figure_files and movie_files are global variables and contain # all figures and movies referred to src_paths = list(src_paths) if figure_files: src_paths += figure_files if movie_files: src_paths += movie_files if src_paths: # Make tar file with all the source dirs with files # that need to be executed os.system('tar cfz %s %s' % (ipynb_tarfile, ' '.join(src_paths))) print 'collected all required additional files in', ipynb_tarfile, 'which must be distributed with the notebook' elif os.path.isfile(ipynb_tarfile): os.remove(ipynb_tarfile) # Parse document into markdown text, code blocks, and tex blocks. # Store in nested list notebook_blocks. notebook_blocks = [[]] authors = '' for line in filestr.splitlines(): if line.startswith('authors = [new_author(name='): # old author method authors = line[10:] elif _CODE_BLOCK in line: code_block_tp = line.split()[-1] if code_block_tp in ( 'pyhid', ) or not code_block_tp.endswith('hid'): notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip() notebook_blocks.append(line) # else: hidden block to be dropped (may include more languages # with time in the above tuple) elif _MATH_BLOCK in line: notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip() notebook_blocks.append(line) else: if not isinstance(notebook_blocks[-1], list): notebook_blocks.append([]) notebook_blocks[-1].append(line) if isinstance(notebook_blocks[-1], list): notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip() # Add block type info pattern = r'(\d+) +%s' for i in range(len(notebook_blocks)): if re.match(pattern % _CODE_BLOCK, notebook_blocks[i]): m = re.match(pattern % _CODE_BLOCK, notebook_blocks[i]) idx = int(m.group(1)) if ipynb_code_tp[idx] == 'cell': notebook_blocks[i] = ['cell', notebook_blocks[i]] elif ipynb_code_tp[idx] == 'cell_hidden': notebook_blocks[i] = ['cell_hidden', notebook_blocks[i]] else: notebook_blocks[i] = ['text', notebook_blocks[i]] elif re.match(pattern % _MATH_BLOCK, notebook_blocks[i]): notebook_blocks[i] = ['math', notebook_blocks[i]] else: notebook_blocks[i] = ['text', notebook_blocks[i]] # Go through tex_blocks and wrap in $$ # (doconce.py runs align2equations so there are no align/align* # environments in tex blocks) label2tag = {} tag_counter = 1 for i in range(len(tex_blocks)): # Extract labels and add tags labels = re.findall(r'label\{(.+?)\}', tex_blocks[i]) for label in labels: label2tag[label] = tag_counter # Insert tag to get labeled equation tex_blocks[i] = tex_blocks[i].replace( 'label{%s}' % label, 'label{%s} \\tag{%s}' % (label, tag_counter)) tag_counter += 1 # Remove \[ and \] or \begin/end{equation*} in single equations tex_blocks[i] = tex_blocks[i].replace(r'\[', '') tex_blocks[i] = tex_blocks[i].replace(r'\]', '') tex_blocks[i] = tex_blocks[i].replace(r'\begin{equation*}', '') tex_blocks[i] = tex_blocks[i].replace(r'\end{equation*}', '') # Check for illegal environments m = re.search(r'\\begin\{(.+?)\}', tex_blocks[i]) if m: envir = m.group(1) if envir not in ('equation', 'equation*', 'align*', 'align', 'array'): print """\ *** warning: latex envir \\begin{%s} does not work well in Markdown. Stick to \\[ ... \\], equation, equation*, align, or align* environments in math environments. """ % envir eq_type = 'heading' # or '$$' eq_type = '$$' # Markdown: add $$ on each side of the equation if eq_type == '$$': # Make sure there are no newline after equation tex_blocks[i] = '$$\n' + tex_blocks[i].strip() + '\n$$' # Here: use heading (###) and simple formula (remove newline # in math expressions to keep everything within a heading) as # the equation then looks bigger elif eq_type == 'heading': tex_blocks[i] = '### $ ' + ' '.join( tex_blocks[i].splitlines()) + ' $' # Add labels for the eqs above the block (for reference) if labels: #label_tp = '<a name="%s"></a>' label_tp = '<div id="%s"></div>' tex_blocks[i] = '<!-- Equation labels as ordinary links -->\n' + \ ' '.join([label_tp % label for label in labels]) + '\n\n' + \ tex_blocks[i] # blocks is now a list of text chunks in markdown and math/code line # instructions. Insert code and tex blocks for i in range(len(notebook_blocks)): if _CODE_BLOCK in notebook_blocks[i][ 1] or _MATH_BLOCK in notebook_blocks[i][1]: words = notebook_blocks[i][1].split() # start of notebook_blocks[i]: number block-indicator code-type n = int(words[0]) if _CODE_BLOCK in notebook_blocks[i][1]: notebook_blocks[i][1] = code_blocks[n] # can be list! if _MATH_BLOCK in notebook_blocks[i][1]: notebook_blocks[i][1] = tex_blocks[n] # Make IPython structures nb_version = int(option('ipynb_version=', '3')) if nb_version == 3: from IPython.nbformat.v3 import (new_code_cell, new_text_cell, new_worksheet, new_notebook, new_metadata, new_author) nb = new_worksheet() elif nb_version == 4: from IPython.nbformat.v4 import (new_code_cell, new_markdown_cell, new_notebook) cells = [] mdstr = [] # plain md format of the notebook prompt_number = 1 for block_tp, block in notebook_blocks: if (block_tp == 'text' or block_tp == 'math') and block != '': # Pure comments between math/code and math/code come # out as empty blocks, should detect that situation # (challenging - can have multiple lines of comments, # or begin and end comment lines with important things between) if nb_version == 3: nb.cells.append(new_text_cell(u'markdown', source=block)) elif nb_version == 4: cells.append(new_markdown_cell(source=block)) mdstr.append(('markdown', block)) elif block_tp == 'cell' and block != '' and block != []: if isinstance(block, list): for block_ in block: block_ = block_.rstrip() if block_ != '': if nb_version == 3: nb.cells.append( new_code_cell(input=block_, prompt_number=prompt_number, collapsed=False)) elif nb_version == 4: cells.append( new_code_cell(source=block_, execution_count=prompt_number)) prompt_number += 1 mdstr.append(('codecell', block_)) else: block = block.rstrip() if block != '': if nb_version == 3: nb.cells.append( new_code_cell(input=block, prompt_number=prompt_number, collapsed=False)) elif nb_version == 4: cells.append( new_code_cell(source=block, execution_count=prompt_number)) prompt_number += 1 mdstr.append(('codecell', block)) elif block_tp == 'cell_hidden' and block != '': block = block.rstrip() if nb_version == 3: nb.cells.append( new_code_cell(input=block, prompt_number=prompt_number, collapsed=True)) elif nb_version == 4: cells.append( new_code_cell(source=block, execution_count=prompt_number)) prompt_number += 1 mdstr.append(('codecell', block)) """ # Dump the notebook cells in a simple ASCII format # (doc/src/ipynb/ipynb_generator.py can translate it back to .ipynb file) f = open(dofile_basename + '.md-ipynb', 'w') for cell_tp, block in mdstr: if cell_tp == 'markdown': f.write('\n-----\n\n') elif cell_tp == 'codecell': f.write('\n-----py\n\n') f.write(block) f.close() """ if nb_version == 3: # Catch the title as the first heading m = re.search(r'^#+\s*(.+)$', filestr, flags=re.MULTILINE) title = m.group(1).strip() if m else '' # md below is not used for anything if authors: authors = eval(authors) md = new_metadata(name=title, authors=authors) else: md = new_metadata(name=title) nb = new_notebook(worksheets=[nb], metadata=new_metadata()) # Let us make v4 notebook here by upgrading from IPython.nbformat.v4 import upgrade nb = upgrade(nb) import IPython.nbformat.v4.nbjson as nbjson # Convert nb to json format filestr = nbjson.writes(nb) elif nb_version == 4: nb = new_notebook(cells=cells) from IPython.nbformat import writes filestr = writes(nb, version=4) # Check that there are no empty cells: if '"input": []' in filestr: print '*** error: empty cells in notebook - report bug in DocOnce' _abort() # must do the replacements here at the very end when json is written out # \eqref and labels will not work, but labels (only in math) do no harm filestr = re.sub(r'([^\\])label\{', r'\g<1>\\\\label{', filestr, flags=re.MULTILINE) # \\eqref{} just gives (???) link at this stage - future versions # will probably support labels #filestr = re.sub(r'\(ref\{(.+?)\}\)', r'\\eqref{\g<1>}', filestr) # Now we use explicit references to tags def subst(m): label = m.group(1) try: return r'[(%s)](#%s)' % (label2tag[label], label) except KeyError as e: print '*** error: label "%s" is not defined' % str(e) filestr = re.sub(r'\(ref\{(.+?)\}\)', subst, filestr) """ # MathJax reference to tag (recall that the equations have both label # and tag (know that tag only works well in HTML, but this mjx-eqn-no # label does not work in ipynb) filestr = re.sub(r'\(ref\{(.+?)\}\)', lambda m: r'[(%s)](#mjx-eqn-%s)' % (label2tag[m.group(1)], label2tag[m.group(1)]), filestr) """ #filestr = re.sub(r'\(ref\{(.+?)\}\)', r'Eq (\g<1>)', filestr) ''' # Final fixes: replace all text between cells by markdown code cells # Note: the patterns are overlapping so a plain re.sub will not work, # here we run through all blocks found and subsitute the first remaining # one, one by one. pattern = r' \},\n(.+?)\{\n "cell_type":' begin_pattern = r'^(.+?)\{\n "cell_type":' remaining_block_begin = re.findall(begin_pattern, filestr, flags=re.DOTALL) remaining_blocks = re.findall(pattern, filestr, flags=re.DOTALL) import string for block in remaining_block_begin + remaining_blocks: filestr = string.replace(filestr, block, json_markdown(block) + ' ', maxreplace=1) filestr_end = re.sub(r' \{\n "cell_type": .+?\n \},\n', '', filestr, flags=re.DOTALL) filestr = filestr.replace(filestr_end, json_markdown(filestr_end)) filestr = """{ "metadata": { "name": "SOME NAME" }, "nbformat": 3, "nbformat_minor": 0, "worksheets": [ { "cells": [ """ + filestr.rstrip() + '\n'+ \ json_pycode('', final_prompt_no+1, 'python').rstrip()[:-1] + """ ], "metadata": {} } ] }""" ''' return filestr