def cwiki_figure(m):
filename = m.group('filename')
link = filename if filename.startswith('http') else None
if not link and not os.path.isfile(filename):
raise IOError('no figure file %s' % filename)
basename = os.path.basename(filename)
stem, ext = os.path.splitext(basename)
root, ext = os.path.splitext(filename)
if link is None:
if not ext in '.png .gif .jpg .jpeg'.split():
# try to convert image file to PNG, using
# convert from ImageMagick:
cmd = 'convert %s png:%s' % (filename, root + '.png')
try:
output = subprocess.check_output(cmd,
shell=True,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
errwarn('\n**** Warning: could not run ' + cmd)
errwarn('Convert %s to PNG format manually' % filename)
_abort()
filename = root + '.png'
caption = m.group('caption')
# keep label if it's there:
caption = re.sub(r'label\{(.+?)\}', '(\g<1>)', caption)
result = r"""{{%s|%s}}""" % (filename, caption)
return result
def cwiki_figure(m):
filename = m.group('filename')
link = filename if filename.startswith('http') else None
if not link and not os.path.isfile(filename):
raise IOError('no figure file %s' % filename)
basename = os.path.basename(filename)
stem, ext = os.path.splitext(basename)
root, ext = os.path.splitext(filename)
if link is None:
if not ext in '.png .gif .jpg .jpeg'.split():
# try to convert image file to PNG, using
# convert from ImageMagick:
cmd = 'convert %s png:%s' % (filename, root + '.png')
failure, output = commands.getstatusoutput(cmd)
if failure:
print '\n**** Warning: could not run', cmd
print 'Convert %s to PNG format manually' % filename
_abort()
filename = root + '.png'
caption = m.group('caption')
# keep label if it's there:
caption = re.sub(r'label\{(.+?)\}', '(\g<1>)', caption)
result = r"""{{%s|%s}}""" % (filename, caption)
return result
def cwiki_figure(m):
filename = m.group('filename')
link = filename if filename.startswith('http') else None
if not link and not os.path.isfile(filename):
raise IOError('no figure file %s' % filename)
basename = os.path.basename(filename)
stem, ext = os.path.splitext(basename)
root, ext = os.path.splitext(filename)
if link is None:
if not ext in '.png .gif .jpg .jpeg'.split():
# try to convert image file to PNG, using
# convert from ImageMagick:
cmd = 'convert %s png:%s' % (filename, root+'.png')
try:
output = subprocess.check_output(cmd, shell=True,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
errwarn('\n**** Warning: could not run ' + cmd)
errwarn('Convert %s to PNG format manually' % filename)
_abort()
filename = root + '.png'
caption = m.group('caption')
# keep label if it's there:
caption = re.sub(r'label\{(.+?)\}', '(\g<1>)', caption)
result = r"""{{%s|%s}}""" % (filename, caption)
return result
def begin_end_consistency_checks(filestr, envirs):
"""Perform consistency checks: no of !bc must equal no of !ec, etc."""
for envir in envirs:
begin = '!b' + envir
end = '!e' + envir
nb = len(re.findall(r'^%s' % begin, filestr, flags=re.MULTILINE))
ne = len(re.findall(r'^%s' % end, filestr, flags=re.MULTILINE))
lines = []
if nb != ne:
print 'ERROR: %d %s do not match %d %s directives' % \
(nb, begin, ne, end)
if not lines:
lines = filestr.splitlines()
begin_ends = []
for i, line in enumerate(lines):
if line.startswith(begin):
begin_ends.append((begin, i))
if line.startswith(end):
begin_ends.append((end, i))
for k in range(1, len(begin_ends)):
pattern, i = begin_ends[k]
if pattern == begin_ends[k-1][0]:
print '\n\nTwo', pattern, 'after each other!\n'
for j in range(begin_ends[k-1][1], begin_ends[k][1]+1):
print lines[j]
_abort()
if begin_ends[-1][0].startswith('!b'):
print 'Missing %s after final %s' % \
(begin_ends[-1][0].replace('!b', '!e'),
begin_ends[-1][0])
_abort()
def begin_end_consistency_checks(filestr, envirs):
"""Perform consistency checks: no of !bc must equal no of !ec, etc."""
for envir in envirs:
begin = '!b' + envir
end = '!e' + envir
nb = len(re.findall(r'^%s' % begin, filestr, flags=re.MULTILINE))
ne = len(re.findall(r'^%s' % end, filestr, flags=re.MULTILINE))
lines = []
if nb != ne:
print 'ERROR: %d %s do not match %d %s directives' % \
(nb, begin, ne, end)
if not lines:
lines = filestr.splitlines()
begin_ends = []
for i, line in enumerate(lines):
if line.startswith(begin):
begin_ends.append((begin, i))
if line.startswith(end):
begin_ends.append((end, i))
for k in range(1, len(begin_ends)):
pattern, i = begin_ends[k]
if pattern == begin_ends[k-1][0]:
print '\n\nTwo', pattern, 'after each other!\n'
for j in range(begin_ends[k-1][1], begin_ends[k][1]+1):
print lines[j]
_abort()
if begin_ends[-1][0].startswith('!b'):
print 'Missing %s after final %s' % \
(begin_ends[-1][0].replace('!b', '!e'),
begin_ends[-1][0])
_abort()
def cwiki_figure(m):
filename = m.group('filename')
link = filename if filename.startswith('http') else None
if not link and not os.path.isfile(filename):
raise IOError('no figure file %s' % filename)
basename = os.path.basename(filename)
stem, ext = os.path.splitext(basename)
root, ext = os.path.splitext(filename)
if link is None:
if not ext in '.png .gif .jpg .jpeg'.split():
# try to convert image file to PNG, using
# convert from ImageMagick:
cmd = 'convert %s png:%s' % (filename, root+'.png')
failure, output = commands.getstatusoutput(cmd)
if failure:
print '\n**** Warning: could not run', cmd
print 'Convert %s to PNG format manually' % filename
_abort()
filename = root + '.png'
caption = m.group('caption')
# keep label if it's there:
caption = re.sub(r'label\{(.+?)\}', '(\g<1>)', caption)
result = r"""{{%s|%s}}""" % (filename, caption)
return result
def subst_footnote(m):
name = m.group('name').strip()
if name in name2index:
i = name2index[m.group('name')]
else:
errwarn('*** error: found footnote with name "%s", but this one is not defined' % name)
_abort()
xml = r'<footnote id="%s">%s<footnote>' % (i, name)
return xml
def subst_footnote(m):
name = m.group('name').strip()
if name in name2index:
i = name2index[m.group('name')]
else:
print '*** error: found footnote with name "%s", but this one is not defined' % name
_abort()
xml = r'<footnote id="%s">%s<footnote>' % (i, name)
return xml
def safe_join(lines, delimiter):
try:
filestr = delimiter.join(lines) + '\n' # will fail if ord(char) > 127
return filestr
except UnicodeDecodeError, e:
if "'ascii' codec can't decode":
print '*** error: non-ascii character - rerun with --encoding=utf-8'
_abort()
else:
print e
_abort()
def safe_join(lines, delimiter):
try:
filestr = delimiter.join(lines) + '\n' # will fail if ord(char) > 127
return filestr
except UnicodeDecodeError, e:
if "'ascii' codec can't decode":
print '*** error: non-ascii character - rerun with --encoding=utf-8'
_abort()
else:
print e
_abort()
def gwiki_figure(m):
filename = m.group("filename")
link = filename if filename.startswith("http") else None
if not link and not os.path.isfile(filename):
raise IOError("no figure file %s" % filename)
basename = os.path.basename(filename)
stem, ext = os.path.splitext(basename)
root, ext = os.path.splitext(filename)
if link is None:
if not ext in ".png .gif .jpg .jpeg".split():
# try to convert image file to PNG, using
# convert from ImageMagick:
cmd = "convert %s png:%s" % (filename, root + ".png")
failure, output = commands.getstatusoutput(cmd)
if failure:
print "\n**** Warning: could not run", cmd
print "Convert %s to PNG format manually" % filename
_abort()
filename = root + ".png"
caption = m.group("caption")
# keep label if it's there:
caption = re.sub(r"label\{(.+?)\}", "(\g<1>)", caption)
print """
NOTE: Place %s at some place on the web and edit the
.gwiki page, either manually (seach for 'Figure: ')
or use the doconce script:
doconce gwiki_figsubst.py mydoc.gwiki URL
""" % filename
result = r"""
---------------------------------------------------------------
Figure: %s
(the URL of the image file %s must be inserted here)
<wiki:comment>
Put the figure file %s on the web (e.g., as part of the
googlecode repository) and substitute the line above with the URL.
</wiki:comment>
---------------------------------------------------------------
""" % (
caption,
filename,
filename,
)
return result
def gwiki_figure(m):
filename = m.group('filename')
link = filename if filename.startswith('http') else None
if not link and not os.path.isfile(filename):
raise IOError('no figure file %s' % filename)
basename = os.path.basename(filename)
stem, ext = os.path.splitext(basename)
root, ext = os.path.splitext(filename)
if link is None:
if not ext in '.png .gif .jpg .jpeg'.split():
# try to convert image file to PNG, using
# convert from ImageMagick:
cmd = 'convert %s png:%s' % (filename, root + '.png')
failure, output = commands.getstatusoutput(cmd)
if failure:
errwarn('\n**** Warning: could not run ' + cmd)
errwarn('Convert %s to PNG format manually' % filename)
_abort()
filename = root + '.png'
caption = m.group('caption')
# keep label if it's there:
caption = re.sub(r'label\{(.+?)\}', '(\g<1>)', caption)
errwarn("""
NOTE: Place %s at some place on the web and edit the
.gwiki page, either manually (seach for 'Figure: ')
or use the doconce script:
doconce gwiki_figsubst.py mydoc.gwiki URL
""" % filename)
result = r"""
---------------------------------------------------------------
Figure: %s
(the URL of the image file %s must be inserted here)
<wiki:comment>
Put the figure file %s on the web (e.g., as part of the
googlecode repository) and substitute the line above with the URL.
</wiki:comment>
---------------------------------------------------------------
""" % (caption, filename, filename)
return result
def YouTubeVideo(filename):
# Use YouTubeVideo object
if 'watch?v=' in filename:
name = filename.split('watch?v=')[1]
elif 'youtu.be/' in filename:
name = filename.split('youtu.be/')[1]
else:
print '*** error: youtube movie name "%s" could not be interpreted' % filename
_abort()
text = ''
global movie_encountered
if not movie_encountered:
text += 'from IPython.display import YouTubeVideo\n'
movie_encountered = True
text += 'YouTubeVideo("%s")\n' % name
return text
def YouTubeVideo(filename):
# Use YouTubeVideo object
if 'watch?v=' in filename:
name = filename.split('watch?v=')[1]
elif 'youtu.be/' in filename:
name = filename.split('youtu.be/')[1]
else:
errwarn('*** error: youtube movie name "%s" could not be interpreted' % filename)
_abort()
text = ''
global movie_encountered
if not movie_encountered:
text += 'from IPython.display import YouTubeVideo\n'
movie_encountered = True
text += 'YouTubeVideo("%s")\n' % name
return text
def sphinx_inline_comment(m):
# Explicit HTML typesetting does not work, we just use bold
name = m.group('name').strip()
comment = m.group('comment').strip()
global edit_markup_warning
if (not edit_markup_warning) and \
(name[:3] in ('add', 'del', 'edi') or '->' in comment):
errwarn('*** warning: sphinx/rst is a suboptimal format for')
errwarn(' typesetting edit markup such as')
errwarn(' ' + m.group())
errwarn(' Use HTML or LaTeX output instead, implement the')
errwarn(' edits (doconce apply_edit_comments) and then use sphinx.')
edit_markup_warning = True
chars = {',': 'comma', ';': 'semicolon', '.': 'period'}
if name[:4] == 'del ':
for char in chars:
if comment == char:
return r' (**edit %s**: delete %s)' % (name[4:], chars[char])
return r'(**edit %s**: **delete** %s)' % (name[4:], comment)
elif name[:4] == 'add ':
for char in chars:
if comment == char:
return r'%s (**edit %s: add %s**)' % (comment, name[4:],
chars[char])
return r' (**edit %s: add**) %s (**end add**)' % (name[4:], comment)
else:
# Ordinary name
comment = ' '.join(comment.splitlines()) # '\s->\s' -> ' -> '
if ' -> ' in comment:
# Replacement
if comment.count(' -> ') != 1:
errwarn('*** wrong syntax in inline comment:')
errwarn(comment)
errwarn('(more than two ->)')
_abort()
orig, new = comment.split(' -> ')
return r'(**%s: remove** %s) (**insert:**)%s (**end insert**)' % (
name, orig, new)
else:
# Ordinary comment
return r'[**%s**: %s]' % (name, comment)
def align2equations(filestr, format):
"""Turn align environments into separate equation environments."""
if not '{align}' in filestr:
return filestr
# sphinx: just replace align, pandoc/ipynb: replace align and align*
# technique: add } if sphinx
postfixes = ['}'] if format == 'sphinx' else ['}', '*}']
lines = filestr.splitlines()
inside_align = False
inside_code = False
for postfix in postfixes:
for i in range(len(lines)):
if lines[i].startswith('!bc'):
inside_code = True
if lines[i].startswith('!ec'):
inside_code = False
if inside_code:
continue
if r'\begin{align%s' % postfix in lines[i]:
inside_align = True
lines[i] = lines[i].replace(
r'\begin{align%s' % postfix, r'\begin{equation%s' % postfix)
if inside_align and '\\\\' in lines[i]:
lines[i] = lines[i].replace(
'\\\\', '\n' + r'\end{equation%s' % postfix + '\n!et\n\n!bt\n' + r'\begin{equation%s ' % postfix)
if inside_align and ('begin{array}' in lines[i] or
'begin{bmatrix}' in lines[i]):
print '*** error: with %s output, align environments' % format
print ' cannot have arrays/matrices with & and \\\\'
print ' rewrite with single equations!'
print '\n'.join(lines[i-4:i+5]).replace('{equation', '{align')
_abort()
if inside_align and '&' in lines[i]:
lines[i] = lines[i].replace('&', '')
if r'\end{align%s' % postfix in lines[i]:
inside_align = False
lines[i] = lines[i].replace(
r'\end{align%s' % postfix, r'\end{equation%s' % postfix)
filestr = '\n'.join(lines)
return filestr
def sphinx_inline_comment(m):
# Explicit HTML typesetting does not work, we just use bold
name = m.group('name').strip()
comment = m.group('comment').strip()
global edit_markup_warning
if (not edit_markup_warning) and \
(name[:3] in ('add', 'del', 'edi') or '->' in comment):
print '*** warning: sphinx/rst is a suboptimal format for'
print ' typesetting edit markup such as'
print ' ', m.group()
print ' Use HTML or LaTeX output instead, implement the'
print ' edits (doconce apply_edit_comments) and then use sphinx.'
edit_markup_warning = True
chars = {',': 'comma', ';': 'semicolon', '.': 'period'}
if name[:4] == 'del ':
for char in chars:
if comment == char:
return r' (**edit %s**: delete %s)' % (name[4:], chars[char])
return r'(**edit %s**: **delete** %s)' % (name[4:], comment)
elif name[:4] == 'add ':
for char in chars:
if comment == char:
return r'%s (**edit %s: add %s**)' % (comment, name[4:], chars[char])
return r' (**edit %s: add**) %s (**end add**)' % (name[4:], comment)
else:
# Ordinary name
comment = ' '.join(comment.splitlines()) # '\s->\s' -> ' -> '
if ' -> ' in comment:
# Replacement
if comment.count(' -> ') != 1:
print '*** wrong syntax in inline comment:'
print comment
print '(more than two ->)'
_abort()
orig, new = comment.split(' -> ')
return r'(**%s: remove** %s) (**insert:**)%s (**end insert**)' % (name, orig, new)
else:
# Ordinary comment
return r'[**%s**: %s]' % (name, comment)
def sphinx_inline_comment(m):
# Explicit HTML typesetting does not work, we just use bold
name = m.group("name").strip()
comment = m.group("comment").strip()
global edit_markup_warning
if (not edit_markup_warning) and (name[:3] in ("add", "del", "edi") or "->" in comment):
errwarn("*** warning: sphinx/rst is a suboptimal format for")
errwarn(" typesetting edit markup such as")
errwarn(" " + m.group())
errwarn(" Use HTML or LaTeX output instead, implement the")
errwarn(" edits (doconce apply_edit_comments) and then use sphinx.")
edit_markup_warning = True
chars = {",": "comma", ";": "semicolon", ".": "period"}
if name[:4] == "del ":
for char in chars:
if comment == char:
return r" (**edit %s**: delete %s)" % (name[4:], chars[char])
return r"(**edit %s**: **delete** %s)" % (name[4:], comment)
elif name[:4] == "add ":
for char in chars:
if comment == char:
return r"%s (**edit %s: add %s**)" % (comment, name[4:], chars[char])
return r" (**edit %s: add**) %s (**end add**)" % (name[4:], comment)
else:
# Ordinary name
comment = " ".join(comment.splitlines()) # '\s->\s' -> ' -> '
if " -> " in comment:
# Replacement
if comment.count(" -> ") != 1:
errwarn("*** wrong syntax in inline comment:")
errwarn(comment)
errwarn("(more than two ->)")
_abort()
orig, new = comment.split(" -> ")
return r"(**%s: remove** %s) (**insert:**)%s (**end insert**)" % (name, orig, new)
else:
# Ordinary comment
return r"[**%s**: %s]" % (name, comment)
def insert_code_and_tex(filestr, code_blocks, tex_blocks, format):
# Consistency check: find no of distinct code and math blocks
# (can be duplicates when solutions are copied at the end)
import sets
pattern = r'^\d+ ' + _CODE_BLOCK
n = len(sets.Set(re.findall(pattern, filestr, flags=re.MULTILINE)))
if len(code_blocks) != n:
print '*** error: found %d code block markers for %d initial code blocks' % (n, len(code_blocks))
print """ Possible causes:
- mismatch of !bt and !et within one file, such that a !bt
swallows code
- mismatch of !bt and !et across files in multi-file documents
- !bc and !ec inside code blocks - replace by |bc and |ec
(run doconce on each file to locate the problem, then on
smaller and smaller parts of each file)"""
_abort()
pattern = r'^\d+ ' + _MATH_BLOCK
n = len(sets.Set(re.findall(pattern, filestr, flags=re.MULTILINE)))
if len(tex_blocks) != n:
print '*** error: found %d tex block markers for %d initial tex blocks\nAbort!' % (n, len(tex_blocks))
print """ Possible causes:
- mismatch of !bc and !ec within one file, such that a !bc
swallows tex blocks
- mismatch of !bc and !ec across files in multi-file documents
- !bt and !et inside code blocks - replace by |bt and |et
(run doconce on each file to locate the problem, then on
smaller and smaller parts of each file)"""
_abort()
from misc import option
max_linelength = option('max_bc_linelength=', None)
if max_linelength is not None:
max_linelength = int(max_linelength)
for i in range(len(code_blocks)):
lines = code_blocks[i].splitlines()
truncated = False
for j in range(len(lines)):
if len(lines[j]) > max_linelength:
lines[j] = lines[j][:max_linelength] + '...'
truncated = True
if truncated:
code_blocks[i] = '\n'.join(lines) + '\n'
lines = filestr.splitlines()
# Note: re.sub cannot be used because newlines, \nabla, etc
# are not handled correctly. Need str.replace.
for i in range(len(lines)):
if _CODE_BLOCK in lines[i] or _MATH_BLOCK in lines[i]:
words = lines[i].split()
# on a line: number block-indicator code-type
n = int(words[0])
if _CODE_BLOCK in lines[i]:
words[1] = '!bc'
code = code_blocks[n]
lines[i] = ' '.join(words[1:]) + '\n' + code + '!ec'
if _MATH_BLOCK in lines[i]:
words[1] = '!bc'
math = tex_blocks[n]
lines[i] = '!bt\n' + math + '!et'
filestr = safe_join(lines, '\n')
# All formats except sphinx and ipynb must remove !bc *hid blocks
# (maybe html will get the possibility to run hidden blocks)
if format not in ('sphinx', 'ipynb'):
filestr = remove_hidden_code_blocks(filestr, format)
return filestr
def remove_code_and_tex(filestr, format):
"""
Remove verbatim and latex (math) code blocks from the file and
store separately in lists (code_blocks and tex_blocks).
The function insert_code_and_tex will insert these blocks again.
"""
# Method:
# store code and tex blocks in lists and substitute these blocks
# by the contents of _CODE_BLOCK and _MATH_BLOCK (arguments after
# !bc must be copied after _CODE_BLOCK).
# later we replace _CODE_BLOCK by !bc and !ec and the code block again
# (similarly for the tex/math block).
# ipynb (and future interactive executable documents) needs to
# see if a code is to be executed or just displayed as text.
# !bc *cod-t and !bc *pro-t is used to indicate pure text.
if format not in ('ipynb', 'matlabnb'):
filestr = re.sub(r'^!bc +([a-z0-9]+)-t', r'!bc \g<1>',
filestr, flags=re.MULTILINE)
# (recall that !bc can be followed by extra information that we must keep:)
code = re.compile(r'^!bc(.*?)\n(.*?)^!ec *\n', re.DOTALL|re.MULTILINE)
# Note: final \n is required and may be missing if there is a block
# at the end of the file, so let us ensure that a blank final
# line is appended to the text:
if filestr[-1] != '\n':
filestr = filestr + '\n'
result = code.findall(filestr)
code_blocks = [c for opt, c in result]
code_block_types = [opt.strip() for opt, c in result]
tex = re.compile(r'^!bt *\n(.*?)^!et *\n', re.DOTALL|re.MULTILINE)
tex_blocks = tex.findall(filestr)
# Remove blocks and substitute by a one-line sign
filestr = code.sub('%s \g<1>\n' % _CODE_BLOCK, filestr)
filestr = tex.sub('%s\n' % _MATH_BLOCK, filestr)
# Number the blocks
lines = filestr.splitlines()
code_block_counter = 0
math_block_counter = 0
for i in range(len(lines)):
if lines[i].startswith(_CODE_BLOCK):
lines[i] = '%d ' % code_block_counter + lines[i]
code_block_counter += 1
if lines[i].startswith(_MATH_BLOCK):
lines[i] = '%d ' % math_block_counter + lines[i]
math_block_counter += 1
filestr = safe_join(lines, '\n')
# Number all equations?
if option('number_all_equations'):
subst = [('\\begin{equation*}', '\\begin{equation}'),
('\\end{equation*}', '\\end{equation}'),
('\\[', '\\begin{equation} '),
('\\]', '\\end{equation} '),
('\\begin{align*}', '\\begin{align}'),
('\\end{align*}', '\\end{align}'),
]
if option('denumber_all_equations'):
# Remove equation numbers and also labels in those equations
subst = [('\\begin{equation}', '\\begin{equation*}'),
('\\end{equation}', '\\end{equation*}'),
('\\begin{align}', '\\begin{align*}'),
('\\end{align}', '\\end{align*}'),
]
removed_labels = []
for i in range(len(tex_blocks)):
found = False
for construction, dummy in subst:
if construction in tex_blocks[i]:
found = True
break
if found:
for from_, to_ in subst:
tex_blocks[i] = tex_blocks[i].replace(from_, to_)
removed_labels += re.findall(r'label\{(.+?)\}', tex_blocks[i])
tex_blocks[i] = re.sub(r'label\{.+?\}\n', '', tex_blocks[i])
tex_blocks[i] = re.sub(r'label\{.+?\}', '', tex_blocks[i])
all_refs = re.findall(r'ref\{(.+?)\}', filestr)
problematic_refs = []
for ref in all_refs:
if ref in removed_labels:
problematic_refs.append(ref)
if problematic_refs:
print '*** error: removed all equation labels from the DocOnce source,'
print ' but there are still references (ref{...}) to equation labels:'
print '\n ', ', '.join(problematic_refs)
print '\n remove all these references!'
_abort()
# Give error if blocks contain !bt
for i in range(len(tex_blocks)):
if '!bt' in tex_blocks[i] or '!et' in tex_blocks[i]:
print '*** error: double !bt or !et in latex block:'
print tex_blocks[i]
_abort()
# Check that math blocks do not contain edit markup or comments
for block in tex_blocks:
m = re.search(INLINE_TAGS['inlinecomment'], block, flags=re.DOTALL)
if m:
print '*** error: tex block with mathematics cannot contain'
print ' inline comment or edit markup!'
if m.group('name') in ('del', 'add') or '->' in m.group('comment'):
# edit markup
print ' Place info about editing after the block.'
print block
_abort()
# Remove |\pause| in code blocks if not latex
if format not in ('latex', 'pdflatex'):
for i in range(len(code_blocks)):
if r'|\pause|' in code_blocks[i]:
code_blocks[i] = re.sub(r'^\|\\pause\|\n', '', code_blocks[i], flags=re.MULTILINE)
return filestr, code_blocks, code_block_types, tex_blocks
def remove_code_and_tex(filestr, format):
"""
Remove verbatim and latex (math) code blocks from the file and
store separately in lists (code_blocks and tex_blocks).
The function insert_code_and_tex will insert these blocks again.
"""
# Method:
# store code and tex blocks in lists and substitute these blocks
# by the contents of _CODE_BLOCK and _MATH_BLOCK (arguments after
# !bc must be copied after _CODE_BLOCK).
# later we replace _CODE_BLOCK by !bc and !ec and the code block again
# (similarly for the tex/math block).
# ipynb (and future interactive executable documents) needs to
# see if a code is to be executed or just displayed as text.
# !bc *cod-t and !bc *pro-t is used to indicate pure text.
if format not in ('ipynb',):
filestr = re.sub(r'^!bc +([a-z0-9]+)-t', r'!bc \g<1>',
filestr, flags=re.MULTILINE)
# (recall that !bc can be followed by extra information that we must keep:)
code = re.compile(r'^!bc(.*?)\n(.*?)^!ec *\n', re.DOTALL|re.MULTILINE)
# Note: final \n is required and may be missing if there is a block
# at the end of the file, so let us ensure that a blank final
# line is appended to the text:
if filestr[-1] != '\n':
filestr = filestr + '\n'
result = code.findall(filestr)
code_blocks = [c for opt, c in result]
code_block_types = [opt.strip() for opt, c in result]
tex = re.compile(r'^!bt *\n(.*?)^!et *\n', re.DOTALL|re.MULTILINE)
tex_blocks = tex.findall(filestr)
# Remove blocks and substitute by a one-line sign
filestr = code.sub('%s \g<1>\n' % _CODE_BLOCK, filestr)
filestr = tex.sub('%s\n' % _MATH_BLOCK, filestr)
# Number the blocks
lines = filestr.splitlines()
code_block_counter = 0
math_block_counter = 0
for i in range(len(lines)):
if lines[i].startswith(_CODE_BLOCK):
lines[i] = '%d ' % code_block_counter + lines[i]
code_block_counter += 1
if lines[i].startswith(_MATH_BLOCK):
lines[i] = '%d ' % math_block_counter + lines[i]
math_block_counter += 1
filestr = safe_join(lines, '\n')
# Number all equations?
if option('number_all_equations'):
subst = [('\\begin{equation*}', '\\begin{equation}'),
('\\end{equation*}', '\\end{equation}'),
('\\[', '\\begin{equation} '),
('\\]', '\\end{equation} '),
('\\begin{align*}', '\\begin{align}'),
('\\end{align*}', '\\end{align}'),
]
if option('denumber_all_equations'):
# Remove equation numbers and also labels in those equations
subst = [('\\begin{equation}', '\\begin{equation*}'),
('\\end{equation}', '\\end{equation*}'),
('\\begin{align}', '\\begin{align*}'),
('\\end{align}', '\\end{align*}'),
]
removed_labels = []
for i in range(len(tex_blocks)):
found = False
for construction, dummy in subst:
if construction in tex_blocks[i]:
found = True
break
if found:
for from_, to_ in subst:
tex_blocks[i] = tex_blocks[i].replace(from_, to_)
removed_labels += re.findall(r'label\{(.+?)\}', tex_blocks[i])
tex_blocks[i] = re.sub(r'label\{.+?\}\n', '', tex_blocks[i])
tex_blocks[i] = re.sub(r'label\{.+?\}', '', tex_blocks[i])
all_refs = re.findall(r'ref\{(.+?)\}', filestr)
problematic_refs = []
for ref in all_refs:
if ref in removed_labels:
problematic_refs.append(ref)
if problematic_refs:
print '*** error: removed all equation labels from the DocOnce source,'
print ' but there are still references (ref{...}) to equation labels:'
print '\n ', ', '.join(problematic_refs)
print '\n remove all these references!'
_abort()
# Give error if blocks contain !bt
for i in range(len(tex_blocks)):
if '!bt' in tex_blocks[i] or '!et' in tex_blocks[i]:
print '*** error: double !bt or !et in latex block:'
print tex_blocks[i]
_abort()
# Check that math blocks do not contain edit markup or comments
for block in tex_blocks:
m = re.search(INLINE_TAGS['inlinecomment'], block, flags=re.DOTALL)
if m:
print '*** error: tex block with mathematics cannot contain'
print ' inline comment or edit markup!'
if m.group('name') in ('del', 'add') or '->' in m.group('comment'):
# edit markup
print ' Place info about editing after the block.'
print block
_abort()
# Remove |\pause| in code blocks if not latex
if format not in ('latex', 'pdflatex'):
for i in range(len(code_blocks)):
if r'|\pause|' in code_blocks[i]:
code_blocks[i] = re.sub(r'^\|\\pause\|\n', '', code_blocks[i], flags=re.MULTILINE)
return filestr, code_blocks, code_block_types, tex_blocks
def insert_code_and_tex(filestr, code_blocks, tex_blocks, format,
complete_doc=True):
# Consistency check (only for complete documents):
# find no of distinct code and math blocks
# (can be duplicates when solutions are copied at the end)
pattern = r'^\d+ ' + _CODE_BLOCK
code_lines = re.findall(pattern, filestr, flags=re.MULTILINE)
n = len(set(code_lines))
if complete_doc and len(code_blocks) != n:
print '*** error: found %d code block markers for %d initial code blocks' % (n, len(code_blocks))
print """ Possible causes:
- mismatch of !bt and !et within one file, such that a !bt
swallows code
- mismatch of !bt and !et across files in multi-file documents
- !bc and !ec inside code blocks - replace by |bc and |ec
(run doconce on each individual file to locate the problem, then on
smaller and smaller parts of each file)"""
numbers = range(len(code_blocks)) # expected numbers in code blocks
for e in code_lines:
# remove number
number = int(e.split()[0])
if number not in numbers:
print ' Problem: found %s, but the number %d was unexpected' % (e, number)
else:
numbers.remove(number)
if numbers:
print ' Problem: did not find XX <<<!!CODE_BLOCK for XX in', numbers
_abort()
pattern = r'^\d+ ' + _MATH_BLOCK
n = len(set(re.findall(pattern, filestr, flags=re.MULTILINE)))
if complete_doc and len(tex_blocks) != n:
print '*** error: found %d tex block markers for %d initial tex blocks\nAbort!' % (n, len(tex_blocks))
print """ Possible causes:
- mismatch of !bc and !ec within one file, such that a !bc
swallows tex blocks
- mismatch of !bc and !ec across files in multi-file documents
- !bt and !et inside code blocks - replace by |bt and |et
(run doconce on each file to locate the problem, then on
smaller and smaller parts of each file)"""
_abort()
from misc import option
max_linelength = option('max_bc_linelength=', None)
if max_linelength is not None:
max_linelength = int(max_linelength)
for i in range(len(code_blocks)):
lines = code_blocks[i].splitlines()
truncated = False
for j in range(len(lines)):
if len(lines[j]) > max_linelength:
lines[j] = lines[j][:max_linelength] + '...'
truncated = True
if truncated:
code_blocks[i] = '\n'.join(lines) + '\n'
lines = filestr.splitlines()
# Note: re.sub cannot be used because newlines, \nabla, etc
# are not handled correctly. Need str.replace.
for i in range(len(lines)):
if _CODE_BLOCK in lines[i] or _MATH_BLOCK in lines[i]:
words = lines[i].split()
# on a line: number block-indicator code-type
n = int(words[0])
if _CODE_BLOCK in lines[i]:
words[1] = '!bc'
code = code_blocks[n]
lines[i] = ' '.join(words[1:]) + '\n' + code + '!ec'
if _MATH_BLOCK in lines[i]:
words[1] = '!bc'
math = tex_blocks[n]
lines[i] = '!bt\n' + math + '!et'
filestr = safe_join(lines, '\n')
# All formats except sphinx and ipynb must remove !bc *hid blocks
# (maybe html will get the possibility to run hidden blocks)
if format not in ('sphinx', 'ipynb'):
filestr = remove_hidden_code_blocks(filestr, format)
return filestr
def mwiki_figure(m):
filename = m.group('filename')
link = filename if filename.startswith('http') else None
if not link and not os.path.isfile(filename):
raise IOError('no figure file %s' % filename)
basename = os.path.basename(filename)
stem, ext = os.path.splitext(basename)
root, ext = os.path.splitext(filename)
if link is None:
if not ext in '.png .gif .jpg .jpeg'.split():
# try to convert image file to PNG, using
# convert from ImageMagick:
cmd = 'convert %s png:%s' % (filename, root+'.png')
failure, output = commands.getstatusoutput(cmd)
if failure:
print '\n**** warning: could not run ', cmd
print ' convert %s to PNG format manually' % filename
_abort()
filename = root + '.png'
caption = m.group('caption').strip()
if caption != '':
caption = '|' + caption # add | for non-empty caption
else:
# Avoid filename as caption when caption is empty
# see http://www.mediawiki.org/wiki/Help:Images
caption = '|<span title=""></span>'
# keep label if it's there:
caption = re.sub(r'label\{(.+?)\}', '(\g<1>)', caption)
size = ''
opts = m.group('options').strip()
if opts:
info = dict([s.split('=') for s in opts.split()])
if 'width' in info and 'height' in info:
size = '|%sx%spx' % (info['width'], info['height'])
elif 'width' in info:
size = '|%spx' % info['width']
elif 'height' in info:
size = '|x%spx' % info['height']
if link:
# We link to some image on the web
filename = os.path.basename(filename)
link = os.path.dirname(link)
result = r"""
[[File:%s|frame%s|link=%s|alt=%s%s]]
""" % (filename, size, link, filename, caption)
else:
# We try to link to a file at wikimedia.org.
found_wikimedia = False
orig_filename = filename
# Check if the file exists and find the appropriate wikimedia name.
# http://en.wikipedia.org/w/api.php?action=query&titles=Image:filename&prop=imageinfo&format=xml
# Skip directories - get the basename
filename = os.path.basename(filename)
import urllib
prms = urllib.urlencode({
'action': 'query', 'titles': 'Image:' + filename,
'prop': 'imageinfo', 'format': 'xml'})
url = 'http://en.wikipedia.org/w/api.php?' + prms
try:
print ' ...checking if %s is stored at en.wikipedia.org/w/api.php...' % filename
f = urllib.urlopen(url)
imageinfo = f.read()
f.close()
def get_data(name, text):
pattern = '%s="(.*?)"' % name
m = re.search(pattern, text)
if m:
match = m.group(1)
if 'Image:' in match:
return match.split('Image:')[1]
if 'File:' in match:
return match.split('File:')[1]
else:
return match
else:
return None
data = ['from', 'to', 'title', 'missing', 'imagerepository',
'timestamp', 'user']
orig_filename = filename
filename = get_data('title', imageinfo)
user = get_data('user', imageinfo)
timestamp = get_data('timestamp', imageinfo)
if user:
found_wikimedia = True
print ' ...found %s at wikimedia' % filename
result = r"""
[[File:%s|frame%s|alt=%s%s]] <!-- user: %s, filename: %s, timestamp: %s -->
""" % (filename, size, filename, caption, user, orig_filename, timestamp)
except IOError:
print ' ...no Internet connection...'
if not found_wikimedia:
print ' ...for wikipedia/wikibooks you must upload image file %s to\n common.wikimedia.org' % orig_filename
# see http://commons.wikimedia.org/wiki/Commons:Upload
# and http://commons.wikimedia.org/wiki/Special:UploadWizard
print ' ...for now we use local file %s' % filename
# This is fine if we use github wiki
result = r"""
[[File:%s|frame%s|alt=%s%s]] <!-- not yet uploaded to common.wikimedia.org -->
""" % (filename, size, filename, caption)
return result
def ipynb_code(filestr, code_blocks, code_block_types, tex_blocks, format):
"""
# We expand all newcommands now
from html import embed_newcommands
newcommands = embed_newcommands(filestr)
if newcommands:
filestr = newcommands + filestr
"""
# Fix pandoc citations to normal internal links: [[key]](#key)
filestr = re.sub(r'\[@(.+?)\]', r'[[\g<1>]](#\g<1>)', filestr)
# filestr becomes json list after this function so we must typeset
# envirs here. All envirs are typeset as pandoc_quote.
from common import _CODE_BLOCK, _MATH_BLOCK
envir_format = option('ipynb_admon=', 'paragraph')
# Remove all !bpop-!epop environments (they cause only problens and
# have no use)
for envir in 'pop', 'slidecell':
filestr = re.sub('^<!-- !b%s .*\n' % envir,
'',
filestr,
flags=re.MULTILINE)
filestr = re.sub('^<!-- !e%s .*\n' % envir,
'',
filestr,
flags=re.MULTILINE)
filestr = re.sub('^<!-- !bnotes.*?<!-- !enotes -->\n',
'',
filestr,
flags=re.DOTALL | re.MULTILINE)
filestr = re.sub('^<!-- !split -->\n', '', filestr, flags=re.MULTILINE)
from doconce import doconce_envirs
envirs = doconce_envirs()[8:-2]
for envir in envirs:
pattern = r'^!b%s(.*?)\n(.+?)\s*^!e%s' % (envir, envir)
if envir_format in ('quote', 'paragraph', 'hrule'):
def subst(m):
title = m.group(1).strip()
# Text size specified in parenthesis?
m2 = re.search('^\s*\((.+?)\)', title)
if title == '' and envir not in ('block', 'quote'):
title = envir.capitalize() + '.'
elif title.lower() == 'none':
title == ''
elif m2:
text_size = m2.group(1).lower()
title = title.replace('(%s)' % text_size, '').strip()
elif title and title[-1] not in ('.', ':', '!', '?'):
# Make sure the title ends with puncuation
title += '.'
# Recall that this formatting is called very late
# so native format must be used!
if title:
title = '**' + title + '**\n'
# Could also consider subsubsection formatting
block = m.group(2)
# Always use quote typesetting for quotes
if envir_format == 'quote' or envir == 'quote':
# Make Markdown quote of the block: lines start with >
lines = []
for line in block.splitlines():
# Just quote plain text
if not (_MATH_BLOCK in line or _CODE_BLOCK in line
or line.startswith('FIGURE:')
or line.startswith('MOVIE:')
or line.startswith('|')):
lines.append('> ' + line)
else:
lines.append('\n' + line + '\n')
block = '\n'.join(lines) + '\n\n'
# Add quote and a blank line after title
if title:
title = '> ' + title + '>\n'
else:
# Add a blank line after title
if title:
title += '\n'
if envir_format == 'hrule':
# Native ------ does not work, use <hr/>
#text = '\n\n----------\n' + title + '----------\n' + \
# block + '\n----------\n\n'
text = '\n\n<hr/>\n' + title + \
block + '\n<hr/>\n\n'
else:
text = title + block + '\n\n'
return text
else:
print '*** error: --ipynb_admon=%s is not supported' % envir_format
filestr = re.sub(pattern,
subst,
filestr,
flags=re.DOTALL | re.MULTILINE)
# Fix pyshell and ipy interactive sessions: remove prompt and output.
# or split in multiple cells such that output comes out at the end of a cell
# Fix sys environments and use run prog.py so programs can be run in cell
# Insert %matplotlib inline in the first block using matplotlib
# Only typeset Python code as blocks, otherwise !bc environmens
# become plain indented Markdown.
from doconce import dofile_basename
from sets import Set
ipynb_tarfile = 'ipynb-%s-src.tar.gz' % dofile_basename
src_paths = Set()
mpl_inline = False
split_pyshell = option('ipynb_split_pyshell=', 'on')
if split_pyshell is None:
split_pyshell = False
elif split_pyshell in ('no', 'False', 'off'):
split_pyshell = False
else:
split_pyshell = True
ipynb_code_tp = [None] * len(code_blocks)
for i in range(len(code_blocks)):
# Check if continuation lines are in the code block, because
# doconce.py inserts a blank after the backslash
if '\\ \n' in code_blocks[i]:
code_blocks[i] = code_blocks[i].replace('\\ \n', '\\\n')
if not mpl_inline and (
re.search(r'import +matplotlib', code_blocks[i]) or \
re.search(r'from +matplotlib', code_blocks[i]) or \
re.search(r'import +scitools', code_blocks[i]) or \
re.search(r'from +scitools', code_blocks[i])):
code_blocks[i] = '%matplotlib inline\n\n' + code_blocks[i]
mpl_inline = True
tp = code_block_types[i]
if tp.endswith('-t'):
# Standard Markdown code with pandoc/github extension
language = tp[:-2]
language_spec = language2pandoc.get(language, '')
#code_blocks[i] = '\n' + indent_lines(code_blocks[i], format) + '\n'
code_blocks[i] = "```%s\n" % language_spec + \
indent_lines(code_blocks[i].strip(), format) + \
"```"
ipynb_code_tp[i] = 'markdown'
elif tp.startswith('pyshell') or tp.startswith('ipy'):
lines = code_blocks[i].splitlines()
last_cell_end = -1
if split_pyshell:
new_code_blocks = []
# Split for each output an put in separate cell
for j in range(len(lines)):
if lines[j].startswith('>>>') or lines[j].startswith(
'... '):
lines[j] = lines[j][4:]
elif lines[j].startswith('In ['): # IPython
lines[j] = ':'.join(lines[j].split(':')[1:]).strip()
elif lines[j].startswith(' ...: '): # IPython
lines[j] = lines[j][8:]
else:
# output (no prefix or Out)
lines[j] = ''
new_code_blocks.append('\n'.join(lines[last_cell_end +
1:j + 1]))
last_cell_end = j
code_blocks[i] = new_code_blocks
ipynb_code_tp[i] = 'cell'
else:
# Remove prompt and output lines; leave code executable in cell
for j in range(len(lines)):
if lines[j].startswith('>>> ') or lines[j].startswith(
'... '):
lines[j] = lines[j][4:]
elif lines[j].startswith('In ['):
lines[j] = ':'.join(lines[j].split(':')[1:]).strip()
else:
# output
lines[j] = ''
for j in range(lines.count('')):
lines.remove('')
code_blocks[i] = '\n'.join(lines)
ipynb_code_tp[i] = 'cell'
elif tp.startswith('sys'):
# Do we find execution of python file? If so, copy the file
# to separate subdir and make a run file command in a cell.
# Otherwise, it is just a plain verbatim Markdown block.
found_unix_lines = False
lines = code_blocks[i].splitlines()
for j in range(len(lines)):
m = re.search(r'(.+?>|\$) *python +([A-Za-z_0-9]+?\.py)',
lines[j])
if m:
name = m.group(2).strip()
if os.path.isfile(name):
src_paths.add(os.path.dirname(name))
lines[j] = '%%run "%s"' % fullpath
else:
found_unix_lines = True
src_paths = list(src_paths)
if src_paths and not found_unix_lines:
# This is a sys block with run commands only
code_blocks[i] = '\n'.join(lines)
ipynb_code_tp[i] = 'cell'
else:
# Standard Markdown code
code_blocks[i] = '\n'.join(lines)
code_blocks[i] = indent_lines(code_blocks[i], format)
ipynb_code_tp[i] = 'markdown'
elif tp.endswith('hid'):
ipynb_code_tp[i] = 'cell_hidden'
elif tp.startswith('py'):
ipynb_code_tp[i] = 'cell'
else:
# Should support other languages as well, but not for now
code_blocks[i] = indent_lines(code_blocks[i], format)
ipynb_code_tp[i] = 'markdown'
# figure_files and movie_files are global variables and contain
# all figures and movies referred to
src_paths = list(src_paths)
if figure_files:
src_paths += figure_files
if movie_files:
src_paths += movie_files
if src_paths:
# Make tar file with all the source dirs with files
# that need to be executed
os.system('tar cfz %s %s' % (ipynb_tarfile, ' '.join(src_paths)))
print 'collected all required additional files in', ipynb_tarfile, 'which must be distributed with the notebook'
elif os.path.isfile(ipynb_tarfile):
os.remove(ipynb_tarfile)
# Parse document into markdown text, code blocks, and tex blocks.
# Store in nested list notebook_blocks.
notebook_blocks = [[]]
authors = ''
for line in filestr.splitlines():
if line.startswith('authors = [new_author(name='): # old author method
authors = line[10:]
elif _CODE_BLOCK in line:
code_block_tp = line.split()[-1]
if code_block_tp in (
'pyhid', ) or not code_block_tp.endswith('hid'):
notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
notebook_blocks.append(line)
# else: hidden block to be dropped (may include more languages
# with time in the above tuple)
elif _MATH_BLOCK in line:
notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
notebook_blocks.append(line)
else:
if not isinstance(notebook_blocks[-1], list):
notebook_blocks.append([])
notebook_blocks[-1].append(line)
if isinstance(notebook_blocks[-1], list):
notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
# Add block type info
pattern = r'(\d+) +%s'
for i in range(len(notebook_blocks)):
if re.match(pattern % _CODE_BLOCK, notebook_blocks[i]):
m = re.match(pattern % _CODE_BLOCK, notebook_blocks[i])
idx = int(m.group(1))
if ipynb_code_tp[idx] == 'cell':
notebook_blocks[i] = ['cell', notebook_blocks[i]]
elif ipynb_code_tp[idx] == 'cell_hidden':
notebook_blocks[i] = ['cell_hidden', notebook_blocks[i]]
else:
notebook_blocks[i] = ['text', notebook_blocks[i]]
elif re.match(pattern % _MATH_BLOCK, notebook_blocks[i]):
notebook_blocks[i] = ['math', notebook_blocks[i]]
else:
notebook_blocks[i] = ['text', notebook_blocks[i]]
# Go through tex_blocks and wrap in $$
# (doconce.py runs align2equations so there are no align/align*
# environments in tex blocks)
label2tag = {}
tag_counter = 1
for i in range(len(tex_blocks)):
# Extract labels and add tags
labels = re.findall(r'label\{(.+?)\}', tex_blocks[i])
for label in labels:
label2tag[label] = tag_counter
# Insert tag to get labeled equation
tex_blocks[i] = tex_blocks[i].replace(
'label{%s}' % label,
'label{%s} \\tag{%s}' % (label, tag_counter))
tag_counter += 1
# Remove \[ and \] or \begin/end{equation*} in single equations
tex_blocks[i] = tex_blocks[i].replace(r'\[', '')
tex_blocks[i] = tex_blocks[i].replace(r'\]', '')
tex_blocks[i] = tex_blocks[i].replace(r'\begin{equation*}', '')
tex_blocks[i] = tex_blocks[i].replace(r'\end{equation*}', '')
# Check for illegal environments
m = re.search(r'\\begin\{(.+?)\}', tex_blocks[i])
if m:
envir = m.group(1)
if envir not in ('equation', 'equation*', 'align*', 'align',
'array'):
print """\
*** warning: latex envir \\begin{%s} does not work well in Markdown.
Stick to \\[ ... \\], equation, equation*, align, or align*
environments in math environments.
""" % envir
eq_type = 'heading' # or '$$'
eq_type = '$$'
# Markdown: add $$ on each side of the equation
if eq_type == '$$':
# Make sure there are no newline after equation
tex_blocks[i] = '$$\n' + tex_blocks[i].strip() + '\n$$'
# Here: use heading (###) and simple formula (remove newline
# in math expressions to keep everything within a heading) as
# the equation then looks bigger
elif eq_type == 'heading':
tex_blocks[i] = '### $ ' + ' '.join(
tex_blocks[i].splitlines()) + ' $'
# Add labels for the eqs above the block (for reference)
if labels:
#label_tp = '<a name="%s"></a>'
label_tp = '<div id="%s"></div>'
tex_blocks[i] = '<!-- Equation labels as ordinary links -->\n' + \
' '.join([label_tp % label
for label in labels]) + '\n\n' + \
tex_blocks[i]
# blocks is now a list of text chunks in markdown and math/code line
# instructions. Insert code and tex blocks
for i in range(len(notebook_blocks)):
if _CODE_BLOCK in notebook_blocks[i][
1] or _MATH_BLOCK in notebook_blocks[i][1]:
words = notebook_blocks[i][1].split()
# start of notebook_blocks[i]: number block-indicator code-type
n = int(words[0])
if _CODE_BLOCK in notebook_blocks[i][1]:
notebook_blocks[i][1] = code_blocks[n] # can be list!
if _MATH_BLOCK in notebook_blocks[i][1]:
notebook_blocks[i][1] = tex_blocks[n]
# Make IPython structures
nb_version = int(option('ipynb_version=', '3'))
if nb_version == 3:
from IPython.nbformat.v3 import (new_code_cell, new_text_cell,
new_worksheet, new_notebook,
new_metadata, new_author)
nb = new_worksheet()
elif nb_version == 4:
from IPython.nbformat.v4 import (new_code_cell, new_markdown_cell,
new_notebook)
cells = []
mdstr = [] # plain md format of the notebook
prompt_number = 1
for block_tp, block in notebook_blocks:
if (block_tp == 'text' or block_tp == 'math') and block != '':
# Pure comments between math/code and math/code come
# out as empty blocks, should detect that situation
# (challenging - can have multiple lines of comments,
# or begin and end comment lines with important things between)
if nb_version == 3:
nb.cells.append(new_text_cell(u'markdown', source=block))
elif nb_version == 4:
cells.append(new_markdown_cell(source=block))
mdstr.append(('markdown', block))
elif block_tp == 'cell' and block != '' and block != []:
if isinstance(block, list):
for block_ in block:
block_ = block_.rstrip()
if block_ != '':
if nb_version == 3:
nb.cells.append(
new_code_cell(input=block_,
prompt_number=prompt_number,
collapsed=False))
elif nb_version == 4:
cells.append(
new_code_cell(source=block_,
execution_count=prompt_number))
prompt_number += 1
mdstr.append(('codecell', block_))
else:
block = block.rstrip()
if block != '':
if nb_version == 3:
nb.cells.append(
new_code_cell(input=block,
prompt_number=prompt_number,
collapsed=False))
elif nb_version == 4:
cells.append(
new_code_cell(source=block,
execution_count=prompt_number))
prompt_number += 1
mdstr.append(('codecell', block))
elif block_tp == 'cell_hidden' and block != '':
block = block.rstrip()
if nb_version == 3:
nb.cells.append(
new_code_cell(input=block,
prompt_number=prompt_number,
collapsed=True))
elif nb_version == 4:
cells.append(
new_code_cell(source=block, execution_count=prompt_number))
prompt_number += 1
mdstr.append(('codecell', block))
"""
# Dump the notebook cells in a simple ASCII format
# (doc/src/ipynb/ipynb_generator.py can translate it back to .ipynb file)
f = open(dofile_basename + '.md-ipynb', 'w')
for cell_tp, block in mdstr:
if cell_tp == 'markdown':
f.write('\n-----\n\n')
elif cell_tp == 'codecell':
f.write('\n-----py\n\n')
f.write(block)
f.close()
"""
if nb_version == 3:
# Catch the title as the first heading
m = re.search(r'^#+\s*(.+)$', filestr, flags=re.MULTILINE)
title = m.group(1).strip() if m else ''
# md below is not used for anything
if authors:
authors = eval(authors)
md = new_metadata(name=title, authors=authors)
else:
md = new_metadata(name=title)
nb = new_notebook(worksheets=[nb], metadata=new_metadata())
# Let us make v4 notebook here by upgrading
from IPython.nbformat.v4 import upgrade
nb = upgrade(nb)
import IPython.nbformat.v4.nbjson as nbjson
# Convert nb to json format
filestr = nbjson.writes(nb)
elif nb_version == 4:
nb = new_notebook(cells=cells)
from IPython.nbformat import writes
filestr = writes(nb, version=4)
# Check that there are no empty cells:
if '"input": []' in filestr:
print '*** error: empty cells in notebook - report bug in DocOnce'
_abort()
# must do the replacements here at the very end when json is written out
# \eqref and labels will not work, but labels (only in math) do no harm
filestr = re.sub(r'([^\\])label\{',
r'\g<1>\\\\label{',
filestr,
flags=re.MULTILINE)
# \\eqref{} just gives (???) link at this stage - future versions
# will probably support labels
#filestr = re.sub(r'\(ref\{(.+?)\}\)', r'\\eqref{\g<1>}', filestr)
# Now we use explicit references to tags
def subst(m):
label = m.group(1)
try:
return r'[(%s)](#%s)' % (label2tag[label], label)
except KeyError as e:
print '*** error: label "%s" is not defined' % str(e)
filestr = re.sub(r'\(ref\{(.+?)\}\)', subst, filestr)
"""
# MathJax reference to tag (recall that the equations have both label
# and tag (know that tag only works well in HTML, but this mjx-eqn-no
# label does not work in ipynb)
filestr = re.sub(r'\(ref\{(.+?)\}\)',
lambda m: r'[(%s)](#mjx-eqn-%s)' % (label2tag[m.group(1)], label2tag[m.group(1)]), filestr)
"""
#filestr = re.sub(r'\(ref\{(.+?)\}\)', r'Eq (\g<1>)', filestr)
'''
# Final fixes: replace all text between cells by markdown code cells
# Note: the patterns are overlapping so a plain re.sub will not work,
# here we run through all blocks found and subsitute the first remaining
# one, one by one.
pattern = r' \},\n(.+?)\{\n "cell_type":'
begin_pattern = r'^(.+?)\{\n "cell_type":'
remaining_block_begin = re.findall(begin_pattern, filestr, flags=re.DOTALL)
remaining_blocks = re.findall(pattern, filestr, flags=re.DOTALL)
import string
for block in remaining_block_begin + remaining_blocks:
filestr = string.replace(filestr, block, json_markdown(block) + ' ',
maxreplace=1)
filestr_end = re.sub(r' \{\n "cell_type": .+?\n \},\n', '', filestr,
flags=re.DOTALL)
filestr = filestr.replace(filestr_end, json_markdown(filestr_end))
filestr = """{
"metadata": {
"name": "SOME NAME"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
""" + filestr.rstrip() + '\n'+ \
json_pycode('', final_prompt_no+1, 'python').rstrip()[:-1] + """
],
"metadata": {}
}
]
}"""
'''
return filestr
def sphinx_code(filestr, code_blocks, code_block_types, tex_blocks, format):
# In rst syntax, code blocks are typeset with :: (verbatim)
# followed by intended blocks. This function indents everything
# inside code (or TeX) blocks.
# default mappings of !bc environments and pygments languages:
envir2pygments = dict(
cod="python",
pro="python",
pycod="python",
cycod="cython",
pypro="python",
cypro="cython",
fcod="fortran",
fpro="fortran",
ccod="c",
cppcod="c++",
cpro="c",
cpppro="c++",
mcod="matlab",
mpro="matlab",
plcod="perl",
plpro="perl",
shcod="bash",
shpro="bash",
rbcod="ruby",
rbpro="ruby",
# sys='console',
sys="text",
rst="rst",
css="css",
csspro="css",
csscod="css",
dat="text",
csv="text",
txt="text",
cc="text",
ccq="text", # not possible with extra indent for ccq
ipy="ipy",
xmlcod="xml",
xmlpro="xml",
xml="xml",
htmlcod="html",
htmlpro="html",
html="html",
texcod="latex",
texpro="latex",
tex="latex",
latexcod="latex",
latexpro="latex",
latex="latex",
do="doconce",
pyshell="python",
pyoptpro="python",
pyscpro="python",
)
# grab line with: # sphinx code-blocks: cod=python cpp=c++ etc
# (do this before code is inserted in case verbatim blocks contain
# such specifications for illustration)
m = re.search(r".. *[Ss]phinx +code-blocks?:(.+)", filestr)
if m:
defs_line = m.group(1)
# turn specifications into a dictionary:
for definition in defs_line.split():
key, value = definition.split("=")
envir2pygments[key] = value
# First indent all code blocks
for i in range(len(code_blocks)):
if code_block_types[i].startswith("pyoptpro") and not option("runestone"):
code_blocks[i] = online_python_tutor(code_blocks[i], return_tp="iframe")
if code_block_types[i].endswith("-h"):
indentation = " " * 8
else:
indentation = " " * 4
code_blocks[i] = indent_lines(code_blocks[i], format, indentation)
# After transforming align environments to separate equations
# the problem with math labels in multiple eqs has disappeared.
# (doconce.py applies align2equations, which takes all align
# envirs and translates them to separate equations, but align*
# environments are allowed.
# Any output of labels in align means an error in the
# align -> equation transformation...)
math_labels = []
multiple_math_labels = [] # sphinx has problems with multiple math labels
for i in range(len(tex_blocks)):
tex_blocks[i] = indent_lines(tex_blocks[i], format)
# extract all \label{}s inside tex blocks and typeset them
# with :label: tags
label_regex = fix_latex(r"label\{(.+?)\}", application="match")
labels = re.findall(label_regex, tex_blocks[i])
if len(labels) == 1:
tex_blocks[i] = " :label: %s\n" % labels[0] + tex_blocks[i]
elif len(labels) > 1:
multiple_math_labels.append(labels)
if len(labels) > 0:
math_labels.extend(labels)
tex_blocks[i] = re.sub(label_regex, "", tex_blocks[i])
# fix latex constructions that do not work with sphinx math
# (just remove them)
commands = [
r"\begin{equation}",
r"\end{equation}",
r"\begin{equation*}",
r"\end{equation*}",
# r'\begin{eqnarray}',
# r'\end{eqnarray}',
# r'\begin{eqnarray*}',
# r'\end{eqnarray*}',
# r'\begin{align}',
# r'\end{align}',
# r'\begin{align*}',
# r'\end{align*}',
r"\begin{multline}",
r"\end{multline}",
r"\begin{multline*}",
r"\end{multline*}",
# r'\begin{split}',
# r'\end{split}',
# r'\begin{gather}',
# r'\end{gather}',
# r'\begin{gather*}',
# r'\end{gather*}',
r"\[",
r"\]",
# some common abbreviations (newcommands):
r"\beqan",
r"\eeqan",
r"\beqa",
r"\eeqa",
r"\balnn",
r"\ealnn",
r"\baln",
r"\ealn",
r"\beq",
r"\eeq", # the simplest name, contained in others, must come last!
]
for command in commands:
tex_blocks[i] = tex_blocks[i].replace(command, "")
# &=& -> &=
tex_blocks[i] = re.sub("&\s*=\s*&", " &= ", tex_blocks[i])
# provide warnings for problematic environments
# Replace all references to equations that have labels in math environments:
for label in math_labels:
filestr = filestr.replace("(:ref:`%s`)" % label, ":eq:`%s`" % label)
multiple_math_labels_with_refs = [] # collect the labels with references
for labels in multiple_math_labels:
for label in labels:
ref = ":eq:`%s`" % label # ref{} is translated to eq:``
if ref in filestr:
multiple_math_labels_with_refs.append(label)
if multiple_math_labels_with_refs:
errwarn(
"""
*** warning: detected non-align math environment with multiple labels
(Sphinx cannot handle this equation system - labels will be removed
and references to them will be empty):"""
)
for label in multiple_math_labels_with_refs:
errwarn(" label{%s}" % label)
print
filestr = insert_code_and_tex(filestr, code_blocks, tex_blocks, "sphinx")
# Remove all !bc ipy and !bc pyshell since interactive sessions
# are automatically handled by sphinx without indentation
# (just a blank line before and after)
filestr = re.sub(r"^!bc +d?ipy *\n(.*?)^!ec *\n", "\n\g<1>\n", filestr, re.DOTALL | re.MULTILINE)
filestr = re.sub(r"^!bc +d?pyshell *\n(.*?)^!ec *\n", "\n\g<1>\n", filestr, re.DOTALL | re.MULTILINE)
# Check if we have custom pygments lexers
if "ipy" in code_block_types:
if not has_custom_pygments_lexer("ipy"):
envir2pygments["ipy"] = "python"
if "do" in code_block_types:
if not has_custom_pygments_lexer("doconce"):
envir2pygments["do"] = "text"
# Make correct code-block:: language constructions
legal_pygments_languages = get_legal_pygments_lexers()
for key in set(code_block_types):
if key in envir2pygments:
if not envir2pygments[key] in legal_pygments_languages:
errwarn(
"""*** warning: %s is not a legal Pygments language (lexer)
found in line:
%s
The 'text' lexer will be used instead.
"""
% (envir2pygments[key], defs_line)
)
envir2pygments[key] = "text"
# filestr = re.sub(r'^!bc\s+%s\s*\n' % key,
# '\n.. code-block:: %s\n\n' % envir2pygments[key], filestr,
# flags=re.MULTILINE)
# Check that we have code installed to handle pyscpro
if "pyscpro" in filestr and key == "pyscpro":
try:
import icsecontrib.sagecellserver
except ImportError:
errwarn(
"""
*** warning: pyscpro for computer code (sage cells) is requested, but'
icsecontrib.sagecellserver from https://github.com/kriskda/sphinx-sagecell
is not installed. Using plain Python typesetting instead."""
)
key = "pypro"
if key == "pyoptpro":
if option("runestone"):
filestr = re.sub(
r"^!bc\s+%s\s*\n" % key,
"\n.. codelens:: codelens_\n :showoutput:\n\n",
filestr,
flags=re.MULTILINE,
)
else:
filestr = re.sub(r"^!bc\s+%s\s*\n" % key, "\n.. raw:: html\n\n", filestr, flags=re.MULTILINE)
elif key == "pyscpro":
if option("runestone"):
filestr = re.sub(
r"^!bc\s+%s\s*\n" % key,
"""
.. activecode:: activecode_
:language: python
""",
filestr,
flags=re.MULTILINE,
)
else:
filestr = re.sub(r"^!bc\s+%s\s*\n" % key, "\n.. sagecellserver::\n\n", filestr, flags=re.MULTILINE)
elif key == "pysccod":
if option("runestone"):
# Include (i.e., run) all previous code segments...
# NOTE: this is most likely not what we want
include = ", ".join([i for i in range(1, activecode_counter)])
filestr = re.sub(
r"^!bc\s+%s\s*\n" % key,
"""
.. activecode:: activecode_
:language: python
"include: %s
"""
% include,
filestr,
flags=re.MULTILINE,
)
else:
errwarn(
"*** error: pysccod for sphinx is not supported without the --runestone flag\n (but pyscpro is via Sage Cell Server)"
)
_abort()
elif key == "":
# any !bc with/without argument becomes a text block:
filestr = re.sub(r"^!bc$", "\n.. code-block:: text\n\n", filestr, flags=re.MULTILINE)
elif key.endswith("hid"):
if key in ("pyhid", "jshid", "htmlhid") and option("runestone"):
# Allow runestone books to run hidden code blocks
# (replace pyhid by pycod, then remove all !bc *hid)
for i in range(len(code_block_types)):
if code_block_types[i] == key:
code_block_types[i] = key.replace("hid", "cod")
key2language = dict(py="python", js="javascript", html="html")
language = key2language[key.replace("hid", "")]
include = ", ".join([i for i in range(1, activecode_counter)])
filestr = re.sub(
r"^!bc +%s\s*\n" % key,
"""
.. activecode:: activecode_
:language: %s
:include: %s
:hidecode:
"""
% (language, include),
filestr,
flags=re.MULTILINE,
)
else:
# Remove hidden code block
pattern = r"^!bc +%s\n.+?^!ec" % key
filestr = re.sub(pattern, "", filestr, flags=re.MULTILINE | re.DOTALL)
else:
show_hide = False
if key.endswith("-h"):
key_orig = key
key = key[:-2]
show_hide = True
# Use the standard sphinx code-block directive
if key in envir2pygments:
pygments_language = envir2pygments[key]
elif key in legal_pygments_languages:
pygments_language = key
else:
errwarn('*** error: detected code environment "%s"' % key)
errwarn(" which is not registered in sphinx.py (sphinx_code)")
errwarn(" or not a language registered in pygments")
_abort()
if show_hide:
filestr = re.sub(
r"^!bc +%s\s*\n" % key_orig,
"\n.. container:: toggle\n\n .. container:: header\n\n **Show/Hide Code**\n\n .. code-block:: %s\n\n"
% pygments_language,
filestr,
flags=re.MULTILINE,
)
# Must add 4 indent in corresponding code_blocks[i], done above
else:
filestr = re.sub(
r"^!bc +%s\s*\n" % key, "\n.. code-block:: %s\n\n" % pygments_language, filestr, flags=re.MULTILINE
)
# any !bc with/without argument becomes a text block:
filestr = re.sub(r"^!bc.*$", "\n.. code-block:: text\n\n", filestr, flags=re.MULTILINE)
filestr = re.sub(r"^!ec *\n", "\n", filestr, flags=re.MULTILINE)
# filestr = re.sub(r'^!ec\n', '\n', filestr, flags=re.MULTILINE)
# filestr = re.sub(r'^!ec\n', '', filestr, flags=re.MULTILINE)
filestr = re.sub(r"^!bt *\n", "\n.. math::\n", filestr, flags=re.MULTILINE)
filestr = re.sub(r"^!et *\n", "\n", filestr, flags=re.MULTILINE)
# Fix lacking blank line after :label:
filestr = re.sub(r"^( :label: .+?)(\n *[^ ]+)", r"\g<1>\n\n\g<2>", filestr, flags=re.MULTILINE)
# Insert counters for runestone blocks
if option("runestone"):
codelens_counter = 0
activecode_counter = 0
lines = filestr.splitlines()
for i in range(len(lines)):
if ".. codelens:: codelens_" in lines[i]:
codelens_counter += 1
lines[i] = lines[i].replace("codelens_", "codelens_%d" % codelens_counter)
if ".. activecode:: activecode_" in lines[i]:
activecode_counter += 1
lines[i] = lines[i].replace("activecode_", "activecode_%d" % activecode_counter)
filestr = "\n".join(lines)
# Final fixes
filestr = fix_underlines_in_headings(filestr)
# Ensure blank line before and after comments
filestr = re.sub(r"([.:;?!])\n^\.\. ", r"\g<1>\n\n.. ", filestr, flags=re.MULTILINE)
filestr = re.sub(r"(^\.\. .+)\n([^ \n]+)", r"\g<1>\n\n\g<2>", filestr, flags=re.MULTILINE)
# Line breaks interfer with tables and needs a final blank line too
lines = filestr.splitlines()
inside_block = False
for i in range(len(lines)):
if lines[i].startswith("<linebreakpipe>") and not inside_block:
inside_block = True
lines[i] = lines[i].replace("<linebreakpipe> ", "") + "\n"
continue
if lines[i].startswith("<linebreakpipe>") and inside_block:
lines[i] = "|" + lines[i].replace("<linebreakpipe>", "")
continue
if inside_block and not lines[i].startswith("<linebreakpipe>"):
inside_block = False
lines[i] = "| " + lines[i] + "\n"
filestr = "\n".join(lines)
# Remove double !split (TOC with a prefix !split gives two !splits)
pattern = "^.. !split\s+.. !split"
filestr = re.sub(pattern, ".. !split", filestr, flags=re.MULTILINE)
if option("html_links_in_new_window"):
# Insert a comment to be recognized by automake_sphinx.py such that it
# can replace the default links by proper modified target= option.
# filestr = '\n\n.. NOTE: Open external links in new windows.\n\n' + filestr
# Use JavaScript instead
filestr = (
""".. raw:: html
<script type="text/javascript">
$(document).ready(function() {
$("a[href^='http']").attr('target','_blank');
});
</script>
"""
+ filestr
)
# Remove too much vertical space
filestr = re.sub(r"\n{3,}", "\n\n", filestr)
return filestr
def sphinx_code(filestr, code_blocks, code_block_types, tex_blocks, format):
# In rst syntax, code blocks are typeset with :: (verbatim)
# followed by intended blocks. This function indents everything
# inside code (or TeX) blocks.
# default mappings of !bc environments and pygments languages:
envir2pygments = dict(
cod='python',
pro='python',
pycod='python',
cycod='cython',
pypro='python',
cypro='cython',
fcod='fortran',
fpro='fortran',
ccod='c',
cppcod='c++',
cpro='c',
cpppro='c++',
mcod='matlab',
mpro='matlab',
plcod='perl',
plpro='perl',
shcod='bash',
shpro='bash',
rbcod='ruby',
rbpro='ruby',
#sys='console',
sys='text',
rst='rst',
css='css',
csspro='css',
csscod='css',
dat='text',
csv='text',
txt='text',
cc='text',
ccq='text', # not possible with extra indent for ccq
ipy='ipy',
xmlcod='xml',
xmlpro='xml',
xml='xml',
htmlcod='html',
htmlpro='html',
html='html',
texcod='latex',
texpro='latex',
tex='latex',
latexcod='latex',
latexpro='latex',
latex='latex',
do='doconce',
pyshell='python',
pyoptpro='python',
pyscpro='python',
)
# grab line with: # sphinx code-blocks: cod=python cpp=c++ etc
# (do this before code is inserted in case verbatim blocks contain
# such specifications for illustration)
m = re.search(r'.. *[Ss]phinx +code-blocks?:(.+)', filestr)
if m:
defs_line = m.group(1)
# turn specifications into a dictionary:
for definition in defs_line.split():
key, value = definition.split('=')
envir2pygments[key] = value
# First indent all code blocks
for i in range(len(code_blocks)):
if code_block_types[i].startswith(
'pyoptpro') and not option('runestone'):
code_blocks[i] = online_python_tutor(code_blocks[i],
return_tp='iframe')
if code_block_types[i].endswith('-h'):
indentation = ' ' * 8
else:
indentation = ' ' * 4
code_blocks[i] = indent_lines(code_blocks[i], format, indentation)
# After transforming align environments to separate equations
# the problem with math labels in multiple eqs has disappeared.
# (doconce.py applies align2equations, which takes all align
# envirs and translates them to separate equations, but align*
# environments are allowed.
# Any output of labels in align means an error in the
# align -> equation transformation...)
math_labels = []
multiple_math_labels = [] # sphinx has problems with multiple math labels
for i in range(len(tex_blocks)):
tex_blocks[i] = indent_lines(tex_blocks[i], format)
# extract all \label{}s inside tex blocks and typeset them
# with :label: tags
label_regex = fix_latex(r'label\{(.+?)\}', application='match')
labels = re.findall(label_regex, tex_blocks[i])
if len(labels) == 1:
tex_blocks[i] = ' :label: %s\n' % labels[0] + tex_blocks[i]
elif len(labels) > 1:
multiple_math_labels.append(labels)
if len(labels) > 0:
math_labels.extend(labels)
tex_blocks[i] = re.sub(label_regex, '', tex_blocks[i])
# fix latex constructions that do not work with sphinx math
# (just remove them)
commands = [
r'\begin{equation}',
r'\end{equation}',
r'\begin{equation*}',
r'\end{equation*}',
#r'\begin{eqnarray}',
#r'\end{eqnarray}',
#r'\begin{eqnarray*}',
#r'\end{eqnarray*}',
#r'\begin{align}',
#r'\end{align}',
#r'\begin{align*}',
#r'\end{align*}',
r'\begin{multline}',
r'\end{multline}',
r'\begin{multline*}',
r'\end{multline*}',
#r'\begin{split}',
#r'\end{split}',
#r'\begin{gather}',
#r'\end{gather}',
#r'\begin{gather*}',
#r'\end{gather*}',
r'\[',
r'\]',
# some common abbreviations (newcommands):
r'\beqan',
r'\eeqan',
r'\beqa',
r'\eeqa',
r'\balnn',
r'\ealnn',
r'\baln',
r'\ealn',
r'\beq',
r'\eeq', # the simplest name, contained in others, must come last!
]
for command in commands:
tex_blocks[i] = tex_blocks[i].replace(command, '')
# &=& -> &=
tex_blocks[i] = re.sub('&\s*=\s*&', ' &= ', tex_blocks[i])
# provide warnings for problematic environments
# Replace all references to equations that have labels in math environments:
for label in math_labels:
filestr = filestr.replace('(:ref:`%s`)' % label, ':eq:`%s`' % label)
multiple_math_labels_with_refs = [] # collect the labels with references
for labels in multiple_math_labels:
for label in labels:
ref = ':eq:`%s`' % label # ref{} is translated to eq:``
if ref in filestr:
multiple_math_labels_with_refs.append(label)
if multiple_math_labels_with_refs:
errwarn("""
*** warning: detected non-align math environment with multiple labels
(Sphinx cannot handle this equation system - labels will be removed
and references to them will be empty):""")
for label in multiple_math_labels_with_refs:
errwarn(' label{%s}' % label)
print
filestr = insert_code_and_tex(filestr, code_blocks, tex_blocks, 'sphinx')
# Remove all !bc ipy and !bc pyshell since interactive sessions
# are automatically handled by sphinx without indentation
# (just a blank line before and after)
filestr = re.sub(r'^!bc +d?ipy *\n(.*?)^!ec *\n', '\n\g<1>\n', filestr,
re.DOTALL | re.MULTILINE)
filestr = re.sub(r'^!bc +d?pyshell *\n(.*?)^!ec *\n', '\n\g<1>\n', filestr,
re.DOTALL | re.MULTILINE)
# Check if we have custom pygments lexers
if 'ipy' in code_block_types:
if not has_custom_pygments_lexer('ipy'):
envir2pygments['ipy'] = 'python'
if 'do' in code_block_types:
if not has_custom_pygments_lexer('doconce'):
envir2pygments['do'] = 'text'
# Make correct code-block:: language constructions
legal_pygments_languages = get_legal_pygments_lexers()
for key in set(code_block_types):
if key in envir2pygments:
if not envir2pygments[key] in legal_pygments_languages:
errwarn(
"""*** warning: %s is not a legal Pygments language (lexer)
found in line:
%s
The 'text' lexer will be used instead.
""" % (envir2pygments[key], defs_line))
envir2pygments[key] = 'text'
#filestr = re.sub(r'^!bc\s+%s\s*\n' % key,
# '\n.. code-block:: %s\n\n' % envir2pygments[key], filestr,
# flags=re.MULTILINE)
# Check that we have code installed to handle pyscpro
if 'pyscpro' in filestr and key == 'pyscpro':
try:
import icsecontrib.sagecellserver
except ImportError:
errwarn("""
*** warning: pyscpro for computer code (sage cells) is requested, but'
icsecontrib.sagecellserver from https://github.com/kriskda/sphinx-sagecell
is not installed. Using plain Python typesetting instead.""")
key = 'pypro'
if key == 'pyoptpro':
if option('runestone'):
filestr = re.sub(
r'^!bc\s+%s\s*\n' % key,
'\n.. codelens:: codelens_\n :showoutput:\n\n',
filestr,
flags=re.MULTILINE)
else:
filestr = re.sub(r'^!bc\s+%s\s*\n' % key,
'\n.. raw:: html\n\n',
filestr,
flags=re.MULTILINE)
elif key == 'pyscpro':
if option('runestone'):
filestr = re.sub(r'^!bc\s+%s\s*\n' % key,
"""
.. activecode:: activecode_
:language: python
""",
filestr,
flags=re.MULTILINE)
else:
filestr = re.sub(r'^!bc\s+%s\s*\n' % key,
'\n.. sagecellserver::\n\n',
filestr,
flags=re.MULTILINE)
elif key == 'pysccod':
if option('runestone'):
# Include (i.e., run) all previous code segments...
# NOTE: this is most likely not what we want
include = ', '.join([i for i in range(1, activecode_counter)])
filestr = re.sub(r'^!bc\s+%s\s*\n' % key,
"""
.. activecode:: activecode_
:language: python
"include: %s
""" % include,
filestr,
flags=re.MULTILINE)
else:
errwarn(
'*** error: pysccod for sphinx is not supported without the --runestone flag\n (but pyscpro is via Sage Cell Server)'
)
_abort()
elif key == '':
# any !bc with/without argument becomes a text block:
filestr = re.sub(r'^!bc$',
'\n.. code-block:: text\n\n',
filestr,
flags=re.MULTILINE)
elif key.endswith('hid'):
if key in ('pyhid', 'jshid', 'htmlhid') and option('runestone'):
# Allow runestone books to run hidden code blocks
# (replace pyhid by pycod, then remove all !bc *hid)
for i in range(len(code_block_types)):
if code_block_types[i] == key:
code_block_types[i] = key.replace('hid', 'cod')
key2language = dict(py='python', js='javascript', html='html')
language = key2language[key.replace('hid', '')]
include = ', '.join([i for i in range(1, activecode_counter)])
filestr = re.sub(r'^!bc +%s\s*\n' % key,
"""
.. activecode:: activecode_
:language: %s
:include: %s
:hidecode:
""" % (language, include),
filestr,
flags=re.MULTILINE)
else:
# Remove hidden code block
pattern = r'^!bc +%s\n.+?^!ec' % key
filestr = re.sub(pattern,
'',
filestr,
flags=re.MULTILINE | re.DOTALL)
else:
show_hide = False
if key.endswith('-h'):
key_orig = key
key = key[:-2]
show_hide = True
# Use the standard sphinx code-block directive
if key in envir2pygments:
pygments_language = envir2pygments[key]
elif key in legal_pygments_languages:
pygments_language = key
else:
errwarn('*** error: detected code environment "%s"' % key)
errwarn(
' which is not registered in sphinx.py (sphinx_code)')
errwarn(' or not a language registered in pygments')
_abort()
if show_hide:
filestr = re.sub(r'^!bc +%s\s*\n' % key_orig,
'\n.. container:: toggle\n\n .. container:: header\n\n **Show/Hide Code**\n\n .. code-block:: %s\n\n' % \
pygments_language, filestr, flags=re.MULTILINE)
# Must add 4 indent in corresponding code_blocks[i], done above
else:
filestr = re.sub(r'^!bc +%s\s*\n' % key,
'\n.. code-block:: %s\n\n' % \
pygments_language, filestr, flags=re.MULTILINE)
# any !bc with/without argument becomes a text block:
filestr = re.sub(r'^!bc.*$',
'\n.. code-block:: text\n\n',
filestr,
flags=re.MULTILINE)
filestr = re.sub(r'^!ec *\n', '\n', filestr, flags=re.MULTILINE)
#filestr = re.sub(r'^!ec\n', '\n', filestr, flags=re.MULTILINE)
#filestr = re.sub(r'^!ec\n', '', filestr, flags=re.MULTILINE)
filestr = re.sub(r'^!bt *\n', '\n.. math::\n', filestr, flags=re.MULTILINE)
filestr = re.sub(r'^!et *\n', '\n', filestr, flags=re.MULTILINE)
# Fix lacking blank line after :label:
filestr = re.sub(r'^( :label: .+?)(\n *[^ ]+)',
r'\g<1>\n\n\g<2>',
filestr,
flags=re.MULTILINE)
# Insert counters for runestone blocks
if option('runestone'):
codelens_counter = 0
activecode_counter = 0
lines = filestr.splitlines()
for i in range(len(lines)):
if '.. codelens:: codelens_' in lines[i]:
codelens_counter += 1
lines[i] = lines[i].replace('codelens_',
'codelens_%d' % codelens_counter)
if '.. activecode:: activecode_' in lines[i]:
activecode_counter += 1
lines[i] = lines[i].replace(
'activecode_', 'activecode_%d' % activecode_counter)
filestr = '\n'.join(lines)
# Final fixes
filestr = fix_underlines_in_headings(filestr)
# Ensure blank line before and after comments
filestr = re.sub(r'([.:;?!])\n^\.\. ',
r'\g<1>\n\n.. ',
filestr,
flags=re.MULTILINE)
filestr = re.sub(r'(^\.\. .+)\n([^ \n]+)',
r'\g<1>\n\n\g<2>',
filestr,
flags=re.MULTILINE)
# Line breaks interfer with tables and needs a final blank line too
lines = filestr.splitlines()
inside_block = False
for i in range(len(lines)):
if lines[i].startswith('<linebreakpipe>') and not inside_block:
inside_block = True
lines[i] = lines[i].replace('<linebreakpipe> ', '') + '\n'
continue
if lines[i].startswith('<linebreakpipe>') and inside_block:
lines[i] = '|' + lines[i].replace('<linebreakpipe>', '')
continue
if inside_block and not lines[i].startswith('<linebreakpipe>'):
inside_block = False
lines[i] = '| ' + lines[i] + '\n'
filestr = '\n'.join(lines)
# Remove double !split (TOC with a prefix !split gives two !splits)
pattern = '^.. !split\s+.. !split'
filestr = re.sub(pattern, '.. !split', filestr, flags=re.MULTILINE)
if option('html_links_in_new_window'):
# Insert a comment to be recognized by automake_sphinx.py such that it
# can replace the default links by proper modified target= option.
#filestr = '\n\n.. NOTE: Open external links in new windows.\n\n' + filestr
# Use JavaScript instead
filestr = """.. raw:: html
<script type="text/javascript">
$(document).ready(function() {
$("a[href^='http']").attr('target','_blank');
});
</script>
""" + filestr
# Remove too much vertical space
filestr = re.sub(r'\n{3,}', '\n\n', filestr)
return filestr
def ipynb_figure(m):
# m.group() must be called before m.group('name')
text = '<!-- dom:%s -->\n<!-- begin figure -->\n' % m.group()
filename = m.group('filename')
caption = m.group('caption').strip()
opts = m.group('options').strip()
if opts:
info = [s.split('=') for s in opts.split()]
opts = ' '.join([
'%s=%s' % (opt, value) for opt, value in info
if opt not in ['frac', 'sidecap']
])
global figure_files
if not filename.startswith('http'):
figure_files.append(filename)
# Extract optional label in caption
label = None
pattern = r' *label\{(.+?)\}'
m = re.search(pattern, caption)
if m:
label = m.group(1).strip()
caption = re.sub(pattern, '', caption)
display_method = option('ipynb_figure=', 'imgtag')
if display_method == 'md':
# Markdown image syntax for embedded image in text
# (no control of size, then one must use HTML syntax)
if label is not None:
#text += '<a name="%s"></a>\n' % label
text += '<div id="%s"></div>\n' % label
text += '' % (caption, filename)
elif display_method == 'imgtag':
# Plain <img tag, allows specifying the image size
if label is not None:
#text += '<a name="%s"></a>' % label
text += '<div id="%s"></div>\n' % label
text += """
<p>%s</p>
<img src="%s" %s>
""" % (caption, filename, opts)
elif display_method == 'Image':
# Image object
# NOTE: This code will normally not work because it inserts a verbatim
# block in the file *after* all such blocks have been removed and
# numbered. doconce.py makes a test prior to removal of blocks and
# runs the handle_figures and movie substitution if ipynb format
# and Image or movie object display.
text += '\n'
if label is not None:
text += '<div id="%s"></div>' % label
text += '<!-- options: %s -->\n' % opts
text = '!bc pycod\n'
global figure_encountered
if not figure_encountered:
# First time we have a figure, we must import Image
text += 'from IPython.display import Image\n'
figure_encountered = True
if caption:
text += '# ' + caption
if filename.startswith('http'):
keyword = 'url'
else:
keyword = 'filename'
text += 'Image(%s="%s")\n' % (keyword, filename)
text += '!ec\n'
else:
print '*** error: --ipynb_figure=%s is illegal, must be md, imgtag or Image' % display_method
_abort()
text += '<!-- end figure -->\n'
return text
def sphinx_quiz_runestone(quiz):
quiz_feedback = option('quiz_explanations=', 'on')
text = ''
if 'new page' in quiz:
text += '.. !split\n%s\n%s' % (quiz['new page'],
'-' * len(quiz['new page']))
text += '.. begin quiz\n\n'
global question_counter
question_counter += 1
# Multiple correct answers?
if sum([1 for choice in quiz['choices'] if choice[0] == 'right']) > 1:
text += '.. mchoicema:: question_%d' % question_counter + '\n'
else:
text += '.. mchoicemf:: question_%d' % question_counter + '\n'
def fix_text(s, tp='answer'):
"""
Answers and feedback in RunestoneInteractive book quizzes
cannot contain math, figure and rst markup. Perform fixes.
"""
drop = False
if 'math::' in s:
errwarn('\n*** warning: quiz %s with math block not supported:' %
tp)
errwarn(s)
drop = True
if '.. code-block::' in s:
errwarn('\n*** warning: quiz %s with code block not supported:' %
tp)
errwarn(s)
drop = True
if '.. figure::' in s:
errwarn('\n*** warning: quiz %s with figure not supported:' % tp)
errwarn(s)
drop = True
if drop:
return ''
# Make multi-line paragraph a one-liner
s = ' '.join(s.splitlines()).rstrip()
# Fixes
pattern = r'`(.+?) (<https?.+?)>`__' # URL
s = re.sub(pattern, '<a href="\g<2>"> \g<1> </a>', s)
pattern = r'``(.+?)``' # verbatim
s = re.sub(pattern, '<tt>\g<1></tt>', s)
pattern = r':math:`(.+?)`' # inline math
s = re.sub(pattern, '<em>\g<1></em>', s) # mimic italic....
pattern = r':\*(.+?)\*' # emphasize
s = re.sub(pattern, '\g<1>', s, flags=re.DOTALL)
return s
import string
correct = []
for i, choice in enumerate(quiz['choices']):
if i > 4: # not supported
errwarn(
'*** warning: quiz with %d choices gets truncated (first 5)' %
len(quiz['choices']))
break
letter = string.ascii_lowercase[i]
text += ' :answer_%s: ' % letter
answer = fix_text(choice[1], tp='answer')
if not answer:
answer = 'Too advanced typesetting prevents the text from being rendered'
text += answer + '\n'
if choice[0] == 'right':
correct.append(letter)
if correct:
text += ' :correct: ' + ', '.join(correct) + '\n'
else:
errwarn(
'*** error: correct choice in quiz has index > 5 (max 5 allowed for RunestoneInteractive books)'
)
errwarn(quiz['question'])
_abort()
for i, choice in enumerate(quiz['choices']):
if i > 4: # not supported
break
letter = string.ascii_lowercase[i]
text += ' :feedback_%s: ' % letter # must be present
if len(choice) == 3 and quiz_feedback == 'on':
feedback = fix_text(choice[2], tp='explanation')
if not feedback:
feedback = '(Too advanced typesetting prevents the text from being rendered)'
text += feedback
text += '\n'
text += '\n' + indent_lines(quiz['question'], 'sphinx', ' ' * 3) + '\n\n\n'
return text
def mwiki_figure(m):
filename = m.group('filename')
link = filename if filename.startswith('http') else None
if not link and not os.path.isfile(filename):
raise IOError('no figure file %s' % filename)
basename = os.path.basename(filename)
stem, ext = os.path.splitext(basename)
root, ext = os.path.splitext(filename)
if link is None:
if not ext in '.png .gif .jpg .jpeg'.split():
# try to convert image file to PNG, using
# convert from ImageMagick:
cmd = 'convert %s png:%s' % (filename, root + '.png')
try:
output = subprocess.check_output(cmd,
shell=True,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
errwarn('\n**** warning: could not run ' + cmd)
errwarn(' convert %s to PNG format manually' % filename)
_abort()
filename = root + '.png'
caption = m.group('caption').strip()
if caption != '':
caption = '|' + caption # add | for non-empty caption
else:
# Avoid filename as caption when caption is empty
# see http://www.mediawiki.org/wiki/Help:Images
caption = '|<span title=""></span>'
# keep label if it's there:
caption = re.sub(r'label\{(.+?)\}', '(\g<1>)', caption)
size = ''
opts = m.group('options').strip()
if opts:
info = dict([s.split('=') for s in opts.split()])
if 'width' in info and 'height' in info:
size = '|%sx%spx' % (info['width'], info['height'])
elif 'width' in info:
size = '|%spx' % info['width']
elif 'height' in info:
size = '|x%spx' % info['height']
if link:
# We link to some image on the web
filename = os.path.basename(filename)
link = os.path.dirname(link)
result = r"""
[[File:%s|frame%s|link=%s|alt=%s%s]]
""" % (filename, size, link, filename, caption)
else:
# We try to link to a file at wikimedia.org.
found_wikimedia = False
orig_filename = filename
# Check if the file exists and find the appropriate wikimedia name.
# http://en.wikipedia.org/w/api.php?action=query&titles=Image:filename&prop=imageinfo&format=xml
# Skip directories - get the basename
filename = os.path.basename(filename)
import urllib
prms = urllib.urlencode({
'action': 'query',
'titles': 'Image:' + filename,
'prop': 'imageinfo',
'format': 'xml'
})
url = 'http://en.wikipedia.org/w/api.php?' + prms
try:
errwarn(
' ...checking if %s is stored at en.wikipedia.org/w/api.php...'
% filename)
f = urllib.urlopen(url)
imageinfo = f.read()
f.close()
def get_data(name, text):
pattern = '%s="(.*?)"' % name
m = re.search(pattern, text)
if m:
match = m.group(1)
if 'Image:' in match:
return match.split('Image:')[1]
if 'File:' in match:
return match.split('File:')[1]
else:
return match
else:
return None
data = [
'from', 'to', 'title', 'missing', 'imagerepository',
'timestamp', 'user'
]
orig_filename = filename
filename = get_data('title', imageinfo)
user = get_data('user', imageinfo)
timestamp = get_data('timestamp', imageinfo)
if user:
found_wikimedia = True
errwarn(' ...found %s at wikimedia' % filename)
result = r"""
[[File:%s|frame%s|alt=%s%s]] <!-- user: %s, filename: %s, timestamp: %s -->
""" % (filename, size, filename, caption, user, orig_filename, timestamp)
except IOError:
errwarn(' ...no Internet connection...')
if not found_wikimedia:
errwarn(
' ...for wikipedia/wikibooks you must upload image file %s to\n common.wikimedia.org'
% orig_filename)
# see http://commons.wikimedia.org/wiki/Commons:Upload
# and http://commons.wikimedia.org/wiki/Special:UploadWizard
errwarn(' ...for now we use local file %s' % filename)
# This is fine if we use github wiki
result = r"""
[[File:%s|frame%s|alt=%s%s]] <!-- not yet uploaded to common.wikimedia.org -->
""" % (filename, size, filename, caption)
return result
def sphinx_code(filestr, code_blocks, code_block_types,
tex_blocks, format):
# In rst syntax, code blocks are typeset with :: (verbatim)
# followed by intended blocks. This function indents everything
# inside code (or TeX) blocks.
# default mappings of !bc environments and pygments languages:
envir2pygments = dict(
cod='python', pro='python',
pycod='python', cycod='cython',
pypro='python', cypro='cython',
fcod='fortran', fpro='fortran',
ccod='c', cppcod='c++',
cpro='c', cpppro='c++',
mcod='matlab', mpro='matlab',
plcod='perl', plpro='perl',
shcod='bash', shpro='bash',
rbcod='ruby', rbpro='ruby',
#sys='console',
sys='text',
rst='rst',
css='css', csspro='css', csscod='css',
dat='text', csv='text', txt='text',
cc='text', ccq='text', # not possible with extra indent for ccq
ipy='ipy',
xmlcod='xml', xmlpro='xml', xml='xml',
htmlcod='html', htmlpro='html', html='html',
texcod='latex', texpro='latex', tex='latex',
latexcod='latex', latexpro='latex', latex='latex',
do='doconce',
pyshell='python',
pyoptpro='python', pyscpro='python',
)
# grab line with: # Sphinx code-blocks: cod=python cpp=c++ etc
# (do this before code is inserted in case verbatim blocks contain
# such specifications for illustration)
m = re.search(r'.. *[Ss]phinx +code-blocks?:(.+)', filestr)
if m:
defs_line = m.group(1)
# turn specifications into a dictionary:
for definition in defs_line.split():
key, value = definition.split('=')
envir2pygments[key] = value
# First indent all code blocks
for i in range(len(code_blocks)):
if code_block_types[i].startswith('pyoptpro') and not option('runestone'):
code_blocks[i] = online_python_tutor(code_blocks[i],
return_tp='iframe')
code_blocks[i] = indent_lines(code_blocks[i], format)
# After transforming align environments to separate equations
# the problem with math labels in multiple eqs has disappeared.
# (doconce.py applies align2equations, which takes all align
# envirs and translates them to separate equations, but align*
# environments are allowed.
# Any output of labels in align means an error in the
# align -> equation transformation...)
math_labels = []
multiple_math_labels = [] # sphinx has problems with multiple math labels
for i in range(len(tex_blocks)):
tex_blocks[i] = indent_lines(tex_blocks[i], format)
# extract all \label{}s inside tex blocks and typeset them
# with :label: tags
label_regex = fix_latex( r'label\{(.+?)\}', application='match')
labels = re.findall(label_regex, tex_blocks[i])
if len(labels) == 1:
tex_blocks[i] = ' :label: %s\n' % labels[0] + tex_blocks[i]
elif len(labels) > 1:
multiple_math_labels.append(labels)
if len(labels) > 0:
math_labels.extend(labels)
tex_blocks[i] = re.sub(label_regex, '', tex_blocks[i])
# fix latex constructions that do not work with sphinx math
commands = [r'\begin{equation}',
r'\end{equation}',
r'\begin{equation*}',
r'\end{equation*}',
r'\begin{eqnarray}',
r'\end{eqnarray}',
r'\begin{eqnarray*}',
r'\end{eqnarray*}',
r'\begin{align}',
r'\end{align}',
r'\begin{align*}',
r'\end{align*}',
r'\begin{multline}',
r'\end{multline}',
r'\begin{multline*}',
r'\end{multline*}',
r'\begin{split}',
r'\end{split}',
r'\begin{gather}',
r'\end{gather}',
r'\begin{gather*}',
r'\end{gather*}',
r'\[',
r'\]',
# some common abbreviations (newcommands):
r'\beqan',
r'\eeqan',
r'\beqa',
r'\eeqa',
r'\balnn',
r'\ealnn',
r'\baln',
r'\ealn',
r'\beq',
r'\eeq', # the simplest, contained in others, must come last!
]
for command in commands:
tex_blocks[i] = tex_blocks[i].replace(command, '')
tex_blocks[i] = re.sub('&\s*=\s*&', ' &= ', tex_blocks[i])
# provide warnings for problematic environments
if '{alignat' in tex_blocks[i]:
print '*** warning: the "alignat" environment will give errors in Sphinx:\n\n', tex_blocks[i], '\n'
# Replace all references to equations that have labels in math environments:
for label in math_labels:
filestr = filestr.replace('(:ref:`%s`)' % label, ':eq:`%s`' % label)
multiple_math_labels_with_refs = [] # collect the labels with references
for labels in multiple_math_labels:
for label in labels:
ref = ':eq:`%s`' % label # ref{} is translated to eq:``
if ref in filestr:
multiple_math_labels_with_refs.append(label)
if multiple_math_labels_with_refs:
print """
*** warning: detected non-align math environment with multiple labels
(Sphinx cannot handle this equation system - labels will be removed
and references to them will be empty):"""
for label in multiple_math_labels_with_refs:
print ' label{%s}' % label
print
filestr = insert_code_and_tex(filestr, code_blocks, tex_blocks, 'sphinx')
# Remove all !bc ipy and !bc pyshell since interactive sessions
# are automatically handled by sphinx without indentation
# (just a blank line before and after)
filestr = re.sub(r'^!bc +d?ipy *\n(.*?)^!ec *\n',
'\n\g<1>\n', filestr, re.DOTALL|re.MULTILINE)
filestr = re.sub(r'^!bc +d?pyshell *\n(.*?)^!ec *\n',
'\n\g<1>\n', filestr, re.DOTALL|re.MULTILINE)
# Check if we have custom pygments lexers
if 'ipy' in code_block_types:
if not has_custom_pygments_lexer('ipy'):
envir2pygments['ipy'] = 'python'
if 'do' in code_block_types:
if not has_custom_pygments_lexer('doconce'):
envir2pygments['do'] = 'text'
# Make correct code-block:: language constructions
legal_pygments_languages = get_legal_pygments_lexers()
for key in set(code_block_types):
if key in envir2pygments:
if not envir2pygments[key] in legal_pygments_languages:
print """*** warning: %s is not a legal Pygments language (lexer)
found in line:
%s
The 'text' lexer will be used instead.
""" % (envir2pygments[key], defs_line)
envir2pygments[key] = 'text'
#filestr = re.sub(r'^!bc\s+%s\s*\n' % key,
# '\n.. code-block:: %s\n\n' % envir2pygments[key], filestr,
# flags=re.MULTILINE)
# Check that we have code installed to handle pyscpro
if 'pyscpro' in filestr and key == 'pyscpro':
try:
import icsecontrib.sagecellserver
except ImportError:
print """
*** warning: pyscpro for computer code (sage cells) is requested, but'
icsecontrib.sagecellserver from https://github.com/kriskda/sphinx-sagecell
is not installed. Using plain Python typesetting instead."""
key = 'pypro'
if key == 'pyoptpro':
if option('runestone'):
filestr = re.sub(r'^!bc\s+%s\s*\n' % key,
'\n.. codelens:: codelens_\n :showoutput:\n\n',
filestr, flags=re.MULTILINE)
else:
filestr = re.sub(r'^!bc\s+%s\s*\n' % key,
'\n.. raw:: html\n\n',
filestr, flags=re.MULTILINE)
elif key == 'pyscpro':
if option('runestone'):
filestr = re.sub(r'^!bc\s+%s\s*\n' % key,
"""
.. activecode:: activecode_
:language: python
""", filestr, flags=re.MULTILINE)
else:
filestr = re.sub(r'^!bc\s+%s\s*\n' % key,
'\n.. sagecellserver::\n\n',
filestr, flags=re.MULTILINE)
elif key == 'pysccod':
if option('runestone'):
# Include (i.e., run) all previous code segments...
# NOTE: this is most likely not what we want
include = ', '.join([i for i in range(1, activecode_counter)])
filestr = re.sub(r'^!bc\s+%s\s*\n' % key,
"""
.. activecode:: activecode_
:language: python
"include: %s
""" % include, filestr, flags=re.MULTILINE)
else:
print '*** error: pysccod for sphinx is not supported without the --runestone flag\n (but pyscpro is via Sage Cell Server)'
_abort()
elif key == '':
# any !bc with/without argument becomes a text block:
filestr = re.sub(r'^!bc$', '\n.. code-block:: text\n\n', filestr,
flags=re.MULTILINE)
elif key.endswith('hid'):
if key in ('pyhid', 'jshid', 'htmlhid') and option('runestone'):
# Allow runestone books to run hidden code blocks
# (replace pyhid by pycod, then remove all !bc *hid)
for i in range(len(code_block_types)):
if code_block_types[i] == key:
code_block_types[i] = key.replace('hid', 'cod')
key2language = dict(py='python', js='javascript', html='html')
language = key2language[key.replace('hid', '')]
include = ', '.join([i for i in range(1, activecode_counter)])
filestr = re.sub(r'^!bc +%s\s*\n' % key,
"""
.. activecode:: activecode_
:language: %s
:include: %s
:hidecode:
""" % (language, include), filestr, flags=re.MULTILINE)
else:
# Remove hidden code block
pattern = r'^!bc +%s\n.+?^!ec' % key
filestr = re.sub(pattern, '', filestr,
flags=re.MULTILINE|re.DOTALL)
else:
# Use the standard sphinx code-block directive
if key in envir2pygments:
pygments_language = envir2pygments[key]
elif key in legal_pygments_languages:
pygments_language = key
else:
print '*** error: detected code environment "%s"' % key
print ' which is not registered in sphinx.py (sphinx_code)'
print ' or not a language registered in pygments'
_abort()
filestr = re.sub(r'^!bc +%s\s*\n' % key,
'\n.. code-block:: %s\n\n' % \
pygments_language, filestr, flags=re.MULTILINE)
# any !bc with/without argument becomes a text block:
filestr = re.sub(r'^!bc.*$', '\n.. code-block:: text\n\n', filestr,
flags=re.MULTILINE)
filestr = re.sub(r'^!ec *\n', '\n', filestr, flags=re.MULTILINE)
#filestr = re.sub(r'^!ec\n', '\n', filestr, flags=re.MULTILINE)
#filestr = re.sub(r'^!ec\n', '', filestr, flags=re.MULTILINE)
filestr = re.sub(r'^!bt *\n', '\n.. math::\n', filestr, flags=re.MULTILINE)
filestr = re.sub(r'^!et *\n', '\n', filestr, flags=re.MULTILINE)
# Insert counters for runestone blocks
if option('runestone'):
codelens_counter = 0
activecode_counter = 0
lines = filestr.splitlines()
for i in range(len(lines)):
if '.. codelens:: codelens_' in lines[i]:
codelens_counter += 1
lines[i] = lines[i].replace('codelens_', 'codelens_%d' %
codelens_counter)
if '.. activecode:: activecode_' in lines[i]:
activecode_counter += 1
lines[i] = lines[i].replace('activecode_', 'activecode_%d' %
activecode_counter)
filestr = '\n'.join(lines)
# Final fixes
filestr = fix_underlines_in_headings(filestr)
# Ensure blank line before and after comments
filestr = re.sub(r'([.:;?!])\n^\.\. ', r'\g<1>\n\n.. ',
filestr, flags=re.MULTILINE)
filestr = re.sub(r'(^\.\. .+)\n([^ \n]+)', r'\g<1>\n\n\g<2>',
filestr, flags=re.MULTILINE)
# Line breaks interfer with tables and needs a final blank line too
lines = filestr.splitlines()
inside_block = False
for i in range(len(lines)):
if lines[i].startswith('<linebreakpipe>') and not inside_block:
inside_block = True
lines[i] = lines[i].replace('<linebreakpipe> ', '') + '\n'
continue
if lines[i].startswith('<linebreakpipe>') and inside_block:
lines[i] = '|' + lines[i].replace('<linebreakpipe>', '')
continue
if inside_block and not lines[i].startswith('<linebreakpipe>'):
inside_block = False
lines[i] = '| ' + lines[i] + '\n'
filestr = '\n'.join(lines)
# Remove double !split (TOC with a prefix !split gives two !splits)
pattern = '^.. !split\s+.. !split'
filestr = re.sub(pattern, '.. !split', filestr, flags=re.MULTILINE)
if option('html_links_in_new_window'):
# Insert a comment to be recognized by automake_sphinx.py such that it
# can replace the default links by proper modified target= option.
#filestr = '\n\n.. NOTE: Open external links in new windows.\n\n' + filestr
# Use JavaScript instead
filestr = """.. raw:: html
<script type="text/javascript">
$(document).ready(function() {
$("a[href^='http']").attr('target','_blank');
});
</script>
""" + filestr
# Remove too much vertical space
filestr = re.sub(r'\n{3,}', '\n\n', filestr)
return filestr
def ipynb_movie(m):
# m.group() must be called before m.group('name')
text = '<!-- dom:%s -->' % m.group()
global html_encountered, movie_encountered, movie_files
filename = m.group('filename')
caption = m.group('caption').strip()
youtube = False
if 'youtu.be' in filename or 'youtube.com' in filename:
youtube = True
if '*' in filename or '->' in filename:
errwarn('*** warning: * or -> in movie filenames is not supported in ipynb')
return text
def YouTubeVideo(filename):
# Use YouTubeVideo object
if 'watch?v=' in filename:
name = filename.split('watch?v=')[1]
elif 'youtu.be/' in filename:
name = filename.split('youtu.be/')[1]
else:
errwarn('*** error: youtube movie name "%s" could not be interpreted' % filename)
_abort()
text = ''
global movie_encountered
if not movie_encountered:
text += 'from IPython.display import YouTubeVideo\n'
movie_encountered = True
text += 'YouTubeVideo("%s")\n' % name
return text
text += '\n<!-- begin movie -->\n'
display_method = option('ipynb_movie=', 'HTML')
if display_method == 'md':
text += html_movie(m)
elif display_method.startswith('HTML'):
text += '\n!bc pycod\n'
if youtube and 'YouTube' in display_method:
text += YouTubeVideo(filename)
if caption:
text += '\nprint "%s"' % caption
else:
# Use HTML formatting
if not html_encountered:
text += 'from IPython.display import HTML\n'
html_encountered = True
text += '_s = """' + html_movie(m) + '"""\n'
text += 'HTML(_s)\n'
if not filename.startswith('http'):
movie_files.append(filename)
text += '!ec\n'
elif display_method == 'ipynb':
text += '!bc pycod\n'
if youtube:
text += YouTubeVideo(filename)
if caption:
text += '\nprint "%s"' % caption
else:
# see http://nbviewer.ipython.org/github/ipython/ipython/blob/1.x/examples/notebooks/Part%205%20-%20Rich%20Display%20System.ipynb
# http://stackoverflow.com/questions/18019477/how-can-i-play-a-local-video-in-my-ipython-notebook
# http://python.6.x6.nabble.com/IPython-User-embedding-non-YouTube-movies-in-the-IPython-notebook-td5024035.html
# Just support .mp4, .ogg, and.webm
stem, ext = os.path.splitext(filename)
if ext not in ('.mp4', '.ogg', '.webm'):
errwarn('*** error: movie "%s" in format %s is not supported for --ipynb_movie=%s' % (filename, ext, display_method))
errwarn(' use --ipynb_movie=HTML instead')
_abort()
height = 365
width = 640
if filename.startswith('http'):
file_open = 'import urllib\nvideo = urllib.urlopen("%s").read()' % filename
else:
file_open = 'video = open("%s", "rb").read()' % filename
text += """
%s
from base64 import b64encode
video_encoded = b64encode(video)
video_tag = '<video controls loop alt="%s" height="%s" width="%s" src="data:video/%s;base64,{0}">'.format(video_encoded)
""" % (file_open, filename, height, width, ext[1:])
if not filename.startswith('http'):
movie_files.append(filename)
if not html_encountered:
text += 'from IPython.display import HTML\n'
html_encountered = True
text += 'HTML(data=video_tag)\n'
if caption:
text += '\nprint "%s"' % caption
text += '!ec\n'
else:
errwarn('*** error: --ipynb_movie=%s is not supported' % display_method)
_abort()
text += '<!-- end movie -->\n'
return text
def ipynb_movie(m):
# m.group() must be called before m.group('name')
text = '<!-- dom:%s -->' % m.group()
global html_encountered, movie_encountered, movie_files
filename = m.group('filename')
caption = m.group('caption').strip()
youtube = False
if 'youtu.be' in filename or 'youtube.com' in filename:
youtube = True
if '*' in filename or '->' in filename:
print '*** warning: * or -> in movie filenames is not supported in ipynb'
return text
def YouTubeVideo(filename):
# Use YouTubeVideo object
if 'watch?v=' in filename:
name = filename.split('watch?v=')[1]
elif 'youtu.be/' in filename:
name = filename.split('youtu.be/')[1]
else:
print '*** error: youtube movie name "%s" could not be interpreted' % filename
_abort()
text = ''
global movie_encountered
if not movie_encountered:
text += 'from IPython.display import YouTubeVideo\n'
movie_encountered = True
text += 'YouTubeVideo("%s")\n' % name
return text
text += '\n<!-- begin movie -->\n'
display_method = option('ipynb_movie=', 'HTML')
if display_method == 'md':
text += html_movie(m)
elif display_method.startswith('HTML'):
text += '\n!bc pycod\n'
if youtube and 'YouTube' in display_method:
text += YouTubeVideo(filename)
if caption:
text += '\nprint "%s"' % caption
else:
# Use HTML formatting
if not html_encountered:
text += 'from IPython.display import HTML\n'
html_encountered = True
text += '_s = """' + html_movie(m) + '"""\n'
text += 'HTML(_s)\n'
if not filename.startswith('http'):
movie_files.append(filename)
text += '!ec\n'
elif display_method == 'ipynb':
text += '!bc pycod\n'
if youtube:
text += YouTubeVideo(filename)
if caption:
text += '\nprint "%s"' % caption
else:
# see http://nbviewer.ipython.org/github/ipython/ipython/blob/1.x/examples/notebooks/Part%205%20-%20Rich%20Display%20System.ipynb
# http://stackoverflow.com/questions/18019477/how-can-i-play-a-local-video-in-my-ipython-notebook
# http://python.6.x6.nabble.com/IPython-User-embedding-non-YouTube-movies-in-the-IPython-notebook-td5024035.html
# Just support .mp4, .ogg, and.webm
stem, ext = os.path.splitext(filename)
if ext not in ('.mp4', '.ogg', '.webm'):
print '*** error: movie "%s" in format %s is not supported for --ipynb_movie=%s' % (
filename, ext, display_method)
print ' use --ipynb_movie=HTML instead'
_abort()
height = 365
width = 640
if filename.startswith('http'):
file_open = 'import urllib\nvideo = urllib.urlopen("%s").read()' % filename
else:
file_open = 'video = open("%s", "rb").read()' % filename
text += """
%s
from base64 import b64encode
video_encoded = b64encode(video)
video_tag = '<video controls loop alt="%s" height="%s" width="%s" src="data:video/%s;base64,{0}">'.format(video_encoded)
""" % (file_open, filename, height, width, ext[1:])
if not filename.startswith('http'):
movie_files.append(filename)
if not html_encountered:
text += 'from IPython.display import HTML\n'
html_encountered = True
text += 'HTML(data=video_tag)\n'
if caption:
text += '\nprint "%s"' % caption
text += '!ec\n'
else:
print '*** error: --ipynb_movie=%s is not supported' % display_method
_abort()
text += '<!-- end movie -->\n'
return text
def sphinx_quiz_runestone(quiz):
quiz_feedback = option('quiz_explanations=', 'on')
text = ''
if 'new page' in quiz:
text += '.. !split\n%s\n%s' % (quiz['new page'], '-'*len(quiz['new page']))
text += '.. begin quiz\n\n'
global question_counter
question_counter += 1
# Multiple correct answers?
if sum([1 for choice in quiz['choices'] if choice[0] == 'right']) > 1:
text += '.. mchoicema:: question_%d' % question_counter + '\n'
else:
text += '.. mchoicemf:: question_%d' % question_counter + '\n'
def fix_text(s, tp='answer'):
"""
Answers and feedback in RunestoneInteractive book quizzes
cannot contain math, figure and rst markup. Perform fixes.
"""
drop = False
if 'math::' in s:
print '\n*** warning: quiz %s with math block not supported:' % tp
print s
drop = True
if '.. code-block::' in s:
print '\n*** warning: quiz %s with code block not supported:' % tp
print s
drop = True
if '.. figure::' in s:
print '\n*** warning: quiz %s with figure not supported:' % tp
print s
drop = True
if drop:
return ''
# Make multi-line paragraph a one-liner
s = ' '.join(s.splitlines()).rstrip()
# Fixes
pattern = r'`(.+?) (<https?.+?)>`__' # URL
s = re.sub(pattern, '<a href="\g<2>"> \g<1> </a>', s)
pattern = r'``(.+?)``' # verbatim
s = re.sub(pattern, '<tt>\g<1></tt>', s)
pattern = r':math:`(.+?)`' # inline math
s = re.sub(pattern, '<em>\g<1></em>', s) # mimic italic....
pattern = r':\*(.+?)\*' # emphasize
s = re.sub(pattern, '\g<1>', s, flags=re.DOTALL)
return s
import string
correct = []
for i, choice in enumerate(quiz['choices']):
if i > 4: # not supported
print '*** warning: quiz with %d choices gets truncated (first 5)' % len(quiz['choices'])
break
letter = string.ascii_lowercase[i]
text += ' :answer_%s: ' % letter
answer = fix_text(choice[1], tp='answer')
if not answer:
answer = 'Too advanced typesetting prevents the text from being rendered'
text += answer + '\n'
if choice[0] == 'right':
correct.append(letter)
if correct:
text += ' :correct: ' + ', '.join(correct) + '\n'
else:
print '*** error: correct choice in quiz has index > 5 (max 5 allowed for RunestoneInteractive books)'
print quiz['question']
_abort()
for i, choice in enumerate(quiz['choices']):
if i > 4: # not supported
break
letter = string.ascii_lowercase[i]
text += ' :feedback_%s: ' % letter # must be present
if len(choice) == 3 and quiz_feedback == 'on':
feedback = fix_text(choice[2], tp='explanation')
if not feedback:
feedback = '(Too advanced typesetting prevents the text from being rendered)'
text += feedback
text += '\n'
text += '\n' + indent_lines(quiz['question'], 'sphinx', ' '*3) + '\n\n\n'
return text
def ipynb_code(filestr, code_blocks, code_block_types,
tex_blocks, format):
"""
# We expand all newcommands now
from html import embed_newcommands
newcommands = embed_newcommands(filestr)
if newcommands:
filestr = newcommands + filestr
"""
# Fix pandoc citations to normal internal links: [[key]](#key)
filestr = re.sub(r'\[@(.+?)\]', r'[[\g<1>]](#\g<1>)', filestr)
# filestr becomes json list after this function so we must typeset
# envirs here. All envirs are typeset as pandoc_quote.
from common import _CODE_BLOCK, _MATH_BLOCK
envir_format = option('ipynb_admon=', 'paragraph')
# Remove all !bpop-!epop environments (they cause only problens and
# have no use)
for envir in 'pop', 'slidecell':
filestr = re.sub('^<!-- !b%s .*\n' % envir, '', filestr,
flags=re.MULTILINE)
filestr = re.sub('^<!-- !e%s .*\n' % envir, '', filestr,
flags=re.MULTILINE)
filestr = re.sub('^<!-- !bnotes.*?<!-- !enotes -->\n', '', filestr,
flags=re.DOTALL|re.MULTILINE)
filestr = re.sub('^<!-- !split -->\n', '', filestr, flags=re.MULTILINE)
from doconce import doconce_envirs
envirs = doconce_envirs()[8:-2]
for envir in envirs:
pattern = r'^!b%s(.*?)\n(.+?)\s*^!e%s' % (envir, envir)
if envir_format in ('quote', 'paragraph', 'hrule'):
def subst(m):
title = m.group(1).strip()
# Text size specified in parenthesis?
m2 = re.search('^\s*\((.+?)\)', title)
if title == '' and envir not in ('block', 'quote'):
title = envir.capitalize() + '.'
elif title.lower() == 'none':
title == ''
elif m2:
text_size = m2.group(1).lower()
title = title.replace('(%s)' % text_size, '').strip()
elif title and title[-1] not in ('.', ':', '!', '?'):
# Make sure the title ends with puncuation
title += '.'
# Recall that this formatting is called very late
# so native format must be used!
if title:
title = '**' + title + '**\n'
# Could also consider subsubsection formatting
block = m.group(2)
# Always use quote typesetting for quotes
if envir_format == 'quote' or envir == 'quote':
# Make Markdown quote of the block: lines start with >
lines = []
for line in block.splitlines():
# Just quote plain text
if not (_MATH_BLOCK in line or
_CODE_BLOCK in line or
line.startswith('FIGURE:') or
line.startswith('MOVIE:') or
line.startswith('|')):
lines.append('> ' + line)
else:
lines.append('\n' + line + '\n')
block = '\n'.join(lines) + '\n\n'
# Add quote and a blank line after title
if title:
title = '> ' + title + '>\n'
else:
# Add a blank line after title
if title:
title += '\n'
if envir_format == 'hrule':
# Native ------ does not work, use <hr/>
#text = '\n\n----------\n' + title + '----------\n' + \
# block + '\n----------\n\n'
text = '\n\n<hr/>\n' + title + \
block + '\n<hr/>\n\n'
else:
text = title + block + '\n\n'
return text
else:
errwarn('*** error: --ipynb_admon=%s is not supported' % envir_format)
filestr = re.sub(pattern, subst, filestr,
flags=re.DOTALL | re.MULTILINE)
# Fix pyshell and ipy interactive sessions: remove prompt and output.
# or split in multiple cells such that output comes out at the end of a cell
# Fix sys environments and use run prog.py so programs can be run in cell
# Insert %matplotlib inline in the first block using matplotlib
# Only typeset Python code as blocks, otherwise !bc environmens
# become plain indented Markdown.
from doconce import dofile_basename
from sets import Set
ipynb_tarfile = 'ipynb-%s-src.tar.gz' % dofile_basename
src_paths = Set()
mpl_inline = False
split_pyshell = option('ipynb_split_pyshell=', 'on')
if split_pyshell is None:
split_pyshell = False
elif split_pyshell in ('no', 'False', 'off'):
split_pyshell = False
else:
split_pyshell = True
ipynb_code_tp = [None]*len(code_blocks)
for i in range(len(code_blocks)):
# Check if continuation lines are in the code block, because
# doconce.py inserts a blank after the backslash
if '\\ \n' in code_blocks[i]:
code_blocks[i] = code_blocks[i].replace('\\ \n', '\\\n')
if not mpl_inline and (
re.search(r'import +matplotlib', code_blocks[i]) or \
re.search(r'from +matplotlib', code_blocks[i]) or \
re.search(r'import +scitools', code_blocks[i]) or \
re.search(r'from +scitools', code_blocks[i])):
code_blocks[i] = '%matplotlib inline\n\n' + code_blocks[i]
mpl_inline = True
tp = code_block_types[i]
if tp.endswith('-t'):
# Standard Markdown code with pandoc/github extension
language = tp[:-2]
language_spec = language2pandoc.get(language, '')
#code_blocks[i] = '\n' + indent_lines(code_blocks[i], format) + '\n'
code_blocks[i] = "```%s\n" % language_spec + \
indent_lines(code_blocks[i].strip(), format) + \
"```"
ipynb_code_tp[i] = 'markdown'
elif tp.startswith('pyshell') or tp.startswith('ipy'):
lines = code_blocks[i].splitlines()
last_cell_end = -1
if split_pyshell:
new_code_blocks = []
# Split for each output an put in separate cell
for j in range(len(lines)):
if lines[j].startswith('>>>') or lines[j].startswith('... '):
lines[j] = lines[j][4:]
elif lines[j].startswith('In ['): # IPython
lines[j] = ':'.join(lines[j].split(':')[1:]).strip()
elif lines[j].startswith(' ...: '): # IPython
lines[j] = lines[j][8:]
else:
# output (no prefix or Out)
lines[j] = ''
new_code_blocks.append(
'\n'.join(lines[last_cell_end+1:j+1]))
last_cell_end = j
code_blocks[i] = new_code_blocks
ipynb_code_tp[i] = 'cell'
else:
# Remove prompt and output lines; leave code executable in cell
for j in range(len(lines)):
if lines[j].startswith('>>> ') or lines[j].startswith('... '):
lines[j] = lines[j][4:]
elif lines[j].startswith('In ['):
lines[j] = ':'.join(lines[j].split(':')[1:]).strip()
else:
# output
lines[j] = ''
for j in range(lines.count('')):
lines.remove('')
code_blocks[i] = '\n'.join(lines)
ipynb_code_tp[i] = 'cell'
elif tp.startswith('sys'):
# Do we find execution of python file? If so, copy the file
# to separate subdir and make a run file command in a cell.
# Otherwise, it is just a plain verbatim Markdown block.
found_unix_lines = False
lines = code_blocks[i].splitlines()
for j in range(len(lines)):
m = re.search(r'(.+?>|\$) *python +([A-Za-z_0-9]+?\.py)',
lines[j])
if m:
name = m.group(2).strip()
if os.path.isfile(name):
src_paths.add(os.path.dirname(name))
lines[j] = '%%run "%s"' % fullpath
else:
found_unix_lines = True
src_paths = list(src_paths)
if src_paths and not found_unix_lines:
# This is a sys block with run commands only
code_blocks[i] = '\n'.join(lines)
ipynb_code_tp[i] = 'cell'
else:
# Standard Markdown code
code_blocks[i] = '\n'.join(lines)
code_blocks[i] = indent_lines(code_blocks[i], format)
ipynb_code_tp[i] = 'markdown'
elif tp.endswith('hid'):
ipynb_code_tp[i] = 'cell_hidden'
elif tp.startswith('py'):
ipynb_code_tp[i] = 'cell'
else:
# Should support other languages as well, but not for now
code_blocks[i] = indent_lines(code_blocks[i], format)
ipynb_code_tp[i] = 'markdown'
# figure_files and movie_files are global variables and contain
# all figures and movies referred to
src_paths = list(src_paths)
if figure_files:
src_paths += figure_files
if movie_files:
src_paths += movie_files
if src_paths:
# Make tar file with all the source dirs with files
# that need to be executed
os.system('tar cfz %s %s' % (ipynb_tarfile, ' '.join(src_paths)))
errwarn('collected all required additional files in ' + ipynb_tarfile + ' which must be distributed with the notebook')
elif os.path.isfile(ipynb_tarfile):
os.remove(ipynb_tarfile)
# Parse document into markdown text, code blocks, and tex blocks.
# Store in nested list notebook_blocks.
notebook_blocks = [[]]
authors = ''
for line in filestr.splitlines():
if line.startswith('authors = [new_author(name='): # old author method
authors = line[10:]
elif _CODE_BLOCK in line:
code_block_tp = line.split()[-1]
if code_block_tp in ('pyhid',) or not code_block_tp.endswith('hid'):
notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
notebook_blocks.append(line)
# else: hidden block to be dropped (may include more languages
# with time in the above tuple)
elif _MATH_BLOCK in line:
notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
notebook_blocks.append(line)
else:
if not isinstance(notebook_blocks[-1], list):
notebook_blocks.append([])
notebook_blocks[-1].append(line)
if isinstance(notebook_blocks[-1], list):
notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
# Add block type info
pattern = r'(\d+) +%s'
for i in range(len(notebook_blocks)):
if re.match(pattern % _CODE_BLOCK, notebook_blocks[i]):
m = re.match(pattern % _CODE_BLOCK, notebook_blocks[i])
idx = int(m.group(1))
if ipynb_code_tp[idx] == 'cell':
notebook_blocks[i] = ['cell', notebook_blocks[i]]
elif ipynb_code_tp[idx] == 'cell_hidden':
notebook_blocks[i] = ['cell_hidden', notebook_blocks[i]]
else:
notebook_blocks[i] = ['text', notebook_blocks[i]]
elif re.match(pattern % _MATH_BLOCK, notebook_blocks[i]):
notebook_blocks[i] = ['math', notebook_blocks[i]]
else:
notebook_blocks[i] = ['text', notebook_blocks[i]]
# Go through tex_blocks and wrap in $$
# (doconce.py runs align2equations so there are no align/align*
# environments in tex blocks)
label2tag = {}
tag_counter = 1
for i in range(len(tex_blocks)):
# Extract labels and add tags
labels = re.findall(r'label\{(.+?)\}', tex_blocks[i])
for label in labels:
label2tag[label] = tag_counter
# Insert tag to get labeled equation
tex_blocks[i] = tex_blocks[i].replace(
'label{%s}' % label, 'label{%s} \\tag{%s}' % (label, tag_counter))
tag_counter += 1
# Remove \[ and \] or \begin/end{equation*} in single equations
tex_blocks[i] = tex_blocks[i].replace(r'\[', '')
tex_blocks[i] = tex_blocks[i].replace(r'\]', '')
tex_blocks[i] = tex_blocks[i].replace(r'\begin{equation*}', '')
tex_blocks[i] = tex_blocks[i].replace(r'\end{equation*}', '')
# Check for illegal environments
m = re.search(r'\\begin\{(.+?)\}', tex_blocks[i])
if m:
envir = m.group(1)
if envir not in ('equation', 'equation*', 'align*', 'align',
'array'):
errwarn("""\
*** warning: latex envir \\begin{%s} does not work well in Markdown.
Stick to \\[ ... \\], equation, equation*, align, or align*
environments in math environments.
""" % envir)
eq_type = 'heading' # or '$$'
eq_type = '$$'
# Markdown: add $$ on each side of the equation
if eq_type == '$$':
# Make sure there are no newline after equation
tex_blocks[i] = '$$\n' + tex_blocks[i].strip() + '\n$$'
# Here: use heading (###) and simple formula (remove newline
# in math expressions to keep everything within a heading) as
# the equation then looks bigger
elif eq_type == 'heading':
tex_blocks[i] = '### $ ' + ' '.join(tex_blocks[i].splitlines()) + ' $'
# Add labels for the eqs above the block (for reference)
if labels:
#label_tp = '<a name="%s"></a>'
label_tp = '<div id="%s"></div>'
tex_blocks[i] = '<!-- Equation labels as ordinary links -->\n' + \
' '.join([label_tp % label
for label in labels]) + '\n\n' + \
tex_blocks[i]
# blocks is now a list of text chunks in markdown and math/code line
# instructions. Insert code and tex blocks
for i in range(len(notebook_blocks)):
if _CODE_BLOCK in notebook_blocks[i][1] or _MATH_BLOCK in notebook_blocks[i][1]:
words = notebook_blocks[i][1].split()
# start of notebook_blocks[i]: number block-indicator code-type
n = int(words[0])
if _CODE_BLOCK in notebook_blocks[i][1]:
notebook_blocks[i][1] = code_blocks[n] # can be list!
if _MATH_BLOCK in notebook_blocks[i][1]:
notebook_blocks[i][1] = tex_blocks[n]
# Make IPython structures
nb_version = int(option('ipynb_version=', '4'))
if nb_version == 3:
try:
from IPython.nbformat.v3 import (
new_code_cell, new_text_cell, new_worksheet,
new_notebook, new_metadata, new_author)
nb = new_worksheet()
except ImportError:
errwarn('*** error: could not import IPython.nbformat.v3!')
errwarn(' set --ipynb_version=4 or leave out --ipynb_version=3')
_abort()
elif nb_version == 4:
try:
from nbformat.v4 import (
new_code_cell, new_markdown_cell, new_notebook)
except ImportError:
# Try old style
try:
from IPython.nbformat.v4 import (
new_code_cell, new_markdown_cell, new_notebook)
except ImportError:
errwarn('*** error: cannot do import nbformat.v4 or IPython.nbformat.v4')
errwarn(' make sure IPython notebook or Jupyter is installed correctly')
_abort()
cells = []
mdstr = [] # plain md format of the notebook
prompt_number = 1
for block_tp, block in notebook_blocks:
if (block_tp == 'text' or block_tp == 'math') and block != '':
# Pure comments between math/code and math/code come
# out as empty blocks, should detect that situation
# (challenging - can have multiple lines of comments,
# or begin and end comment lines with important things between)
if nb_version == 3:
nb.cells.append(new_text_cell(u'markdown', source=block))
elif nb_version == 4:
cells.append(new_markdown_cell(source=block))
mdstr.append(('markdown', block))
elif block_tp == 'cell' and block != '' and block != []:
if isinstance(block, list):
for block_ in block:
block_ = block_.rstrip()
if block_ != '':
if nb_version == 3:
nb.cells.append(new_code_cell(
input=block_,
prompt_number=prompt_number,
collapsed=False))
elif nb_version == 4:
cells.append(new_code_cell(
source=block_,
execution_count=prompt_number,
metadata=dict(collapsed=False)))
prompt_number += 1
mdstr.append(('codecell', block_))
else:
block = block.rstrip()
if block != '':
if nb_version == 3:
nb.cells.append(new_code_cell(
input=block,
prompt_number=prompt_number,
collapsed=False))
elif nb_version == 4:
cells.append(new_code_cell(
source=block,
execution_count=prompt_number,
metadata=dict(collapsed=False)))
prompt_number += 1
mdstr.append(('codecell', block))
elif block_tp == 'cell_hidden' and block != '':
block = block.rstrip()
if nb_version == 3:
nb.cells.append(new_code_cell(
input=block, prompt_number=prompt_number, collapsed=True))
elif nb_version == 4:
cells.append(new_code_cell(
source=block,
execution_count=prompt_number,
metadata=dict(collapsed=True)))
prompt_number += 1
mdstr.append(('codecell', block))
"""
# Dump the notebook cells in a simple ASCII format
# (doc/src/ipynb/ipynb_generator.py can translate it back to .ipynb file)
f = open(dofile_basename + '.md-ipynb', 'w')
for cell_tp, block in mdstr:
if cell_tp == 'markdown':
f.write('\n-----\n\n')
elif cell_tp == 'codecell':
f.write('\n-----py\n\n')
f.write(block)
f.close()
"""
if nb_version == 3:
# Catch the title as the first heading
m = re.search(r'^#+\s*(.+)$', filestr, flags=re.MULTILINE)
title = m.group(1).strip() if m else ''
# md below is not used for anything
if authors:
authors = eval(authors)
md = new_metadata(name=title, authors=authors)
else:
md = new_metadata(name=title)
nb = new_notebook(worksheets=[nb], metadata=new_metadata())
# Let us make v4 notebook here by upgrading
from IPython.nbformat.v4 import upgrade
nb = upgrade(nb)
import IPython.nbformat.v4.nbjson as nbjson
# Convert nb to json format
filestr = nbjson.writes(nb)
elif nb_version == 4:
nb = new_notebook(cells=cells)
from IPython.nbformat import writes
filestr = writes(nb, version=4)
# Check that there are no empty cells:
if '"input": []' in filestr:
errwarn('*** error: empty cells in notebook - report bug in DocOnce')
_abort()
# must do the replacements here at the very end when json is written out
# \eqref and labels will not work, but labels (only in math) do no harm
filestr = re.sub(r'([^\\])label\{', r'\g<1>\\\\label{', filestr,
flags=re.MULTILINE)
# \\eqref{} just gives (???) link at this stage - future versions
# will probably support labels
#filestr = re.sub(r'\(ref\{(.+?)\}\)', r'\\eqref{\g<1>}', filestr)
# Now we use explicit references to tags
def subst(m):
label = m.group(1)
try:
return r'[(%s)](#%s)' % (label2tag[label], label)
except KeyError as e:
errwarn('*** error: label "%s" is not defined' % str(e))
filestr = re.sub(r'\(ref\{(.+?)\}\)', subst, filestr)
"""
# MathJax reference to tag (recall that the equations have both label
# and tag (know that tag only works well in HTML, but this mjx-eqn-no
# label does not work in ipynb)
filestr = re.sub(r'\(ref\{(.+?)\}\)',
lambda m: r'[(%s)](#mjx-eqn-%s)' % (label2tag[m.group(1)], label2tag[m.group(1)]), filestr)
"""
#filestr = re.sub(r'\(ref\{(.+?)\}\)', r'Eq (\g<1>)', filestr)
'''
# Final fixes: replace all text between cells by markdown code cells
# Note: the patterns are overlapping so a plain re.sub will not work,
# here we run through all blocks found and subsitute the first remaining
# one, one by one.
pattern = r' \},\n(.+?)\{\n "cell_type":'
begin_pattern = r'^(.+?)\{\n "cell_type":'
remaining_block_begin = re.findall(begin_pattern, filestr, flags=re.DOTALL)
remaining_blocks = re.findall(pattern, filestr, flags=re.DOTALL)
import string
for block in remaining_block_begin + remaining_blocks:
filestr = string.replace(filestr, block, json_markdown(block) + ' ',
maxreplace=1)
filestr_end = re.sub(r' \{\n "cell_type": .+?\n \},\n', '', filestr,
flags=re.DOTALL)
filestr = filestr.replace(filestr_end, json_markdown(filestr_end))
filestr = """{
"metadata": {
"name": "SOME NAME"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
""" + filestr.rstrip() + '\n'+ \
json_pycode('', final_prompt_no+1, 'python').rstrip()[:-1] + """
],
"metadata": {}
}
]
}"""
'''
return filestr
def ipynb_figure(m):
# m.group() must be called before m.group('name')
text = '<!-- dom:%s -->\n<!-- begin figure -->\n' % m.group()
filename = m.group('filename')
caption = m.group('caption').strip()
opts = m.group('options').strip()
if opts:
info = [s.split('=') for s in opts.split()]
opts = ' ' .join(['%s=%s' % (opt, value)
for opt, value in info
if opt not in ['frac', 'sidecap']])
global figure_files
if not filename.startswith('http'):
figure_files.append(filename)
# Extract optional label in caption
label = None
pattern = r' *label\{(.+?)\}'
m = re.search(pattern, caption)
if m:
label = m.group(1).strip()
caption = re.sub(pattern, '', caption)
display_method = option('ipynb_figure=', 'imgtag')
if display_method == 'md':
# Markdown image syntax for embedded image in text
# (no control of size, then one must use HTML syntax)
if label is not None:
#text += '<a name="%s"></a>\n' % label
text += '<div id="%s"></div>\n' % label
text += '' % (caption, filename)
elif display_method == 'imgtag':
# Plain <img tag, allows specifying the image size
if label is not None:
#text += '<a name="%s"></a>' % label
text += '<div id="%s"></div>\n' % label
# Fix caption markup so it becomes html
from doconce import INLINE_TAGS_SUBST, INLINE_TAGS
for tag in 'bold', 'emphasize', 'verbatim':
caption = re.sub(INLINE_TAGS[tag], INLINE_TAGS_SUBST['html'][tag],
caption, flags=re.MULTILINE)
text += """
<p>%s</p>
<img src="%s" %s>
""" % (caption, filename, opts)
elif display_method == 'Image':
# Image object
# NOTE: This code will normally not work because it inserts a verbatim
# block in the file *after* all such blocks have been removed and
# numbered. doconce.py makes a test prior to removal of blocks and
# runs the handle_figures and movie substitution if ipynb format
# and Image or movie object display.
text += '\n'
if label is not None:
text += '<div id="%s"></div>' % label
text += '<!-- options: %s -->\n' % opts
text = '!bc pycod\n'
global figure_encountered
if not figure_encountered:
# First time we have a figure, we must import Image
text += 'from IPython.display import Image\n'
figure_encountered = True
if caption:
text += '# ' + caption
if filename.startswith('http'):
keyword = 'url'
else:
keyword = 'filename'
text += 'Image(%s="%s")\n' % (keyword, filename)
text += '!ec\n'
else:
errwarn('*** error: --ipynb_figure=%s is illegal, must be md, imgtag or Image' % display_method)
_abort()
text += '<!-- end figure -->\n'
return text
def sphinx_quiz_runestone(quiz):
quiz_feedback = option("quiz_explanations=", "on")
text = ""
if "new page" in quiz:
text += ".. !split\n%s\n%s" % (quiz["new page"], "-" * len(quiz["new page"]))
text += ".. begin quiz\n\n"
global question_counter
question_counter += 1
# Multiple correct answers?
if sum([1 for choice in quiz["choices"] if choice[0] == "right"]) > 1:
text += ".. mchoicema:: question_%d" % question_counter + "\n"
else:
text += ".. mchoicemf:: question_%d" % question_counter + "\n"
def fix_text(s, tp="answer"):
"""
Answers and feedback in RunestoneInteractive book quizzes
cannot contain math, figure and rst markup. Perform fixes.
"""
drop = False
if "math::" in s:
errwarn("\n*** warning: quiz %s with math block not supported:" % tp)
errwarn(s)
drop = True
if ".. code-block::" in s:
errwarn("\n*** warning: quiz %s with code block not supported:" % tp)
errwarn(s)
drop = True
if ".. figure::" in s:
errwarn("\n*** warning: quiz %s with figure not supported:" % tp)
errwarn(s)
drop = True
if drop:
return ""
# Make multi-line paragraph a one-liner
s = " ".join(s.splitlines()).rstrip()
# Fixes
pattern = r"`(.+?) (<https?.+?)>`__" # URL
s = re.sub(pattern, '<a href="\g<2>"> \g<1> </a>', s)
pattern = r"``(.+?)``" # verbatim
s = re.sub(pattern, "<tt>\g<1></tt>", s)
pattern = r":math:`(.+?)`" # inline math
s = re.sub(pattern, "<em>\g<1></em>", s) # mimic italic....
pattern = r":\*(.+?)\*" # emphasize
s = re.sub(pattern, "\g<1>", s, flags=re.DOTALL)
return s
import string
correct = []
for i, choice in enumerate(quiz["choices"]):
if i > 4: # not supported
errwarn("*** warning: quiz with %d choices gets truncated (first 5)" % len(quiz["choices"]))
break
letter = string.ascii_lowercase[i]
text += " :answer_%s: " % letter
answer = fix_text(choice[1], tp="answer")
if not answer:
answer = "Too advanced typesetting prevents the text from being rendered"
text += answer + "\n"
if choice[0] == "right":
correct.append(letter)
if correct:
text += " :correct: " + ", ".join(correct) + "\n"
else:
errwarn("*** error: correct choice in quiz has index > 5 (max 5 allowed for RunestoneInteractive books)")
errwarn(quiz["question"])
_abort()
for i, choice in enumerate(quiz["choices"]):
if i > 4: # not supported
break
letter = string.ascii_lowercase[i]
text += " :feedback_%s: " % letter # must be present
if len(choice) == 3 and quiz_feedback == "on":
feedback = fix_text(choice[2], tp="explanation")
if not feedback:
feedback = "(Too advanced typesetting prevents the text from being rendered)"
text += feedback
text += "\n"
text += "\n" + indent_lines(quiz["question"], "sphinx", " " * 3) + "\n\n\n"
return text