def target(state: StateBlock, startLine: int, endLine: int, silent: bool):
pos = state.bMarks[startLine] + state.tShift[startLine]
maximum = state.eMarks[startLine]
# if it's indented more than 3 spaces, it should be a code block
if state.sCount[startLine] - state.blkIndent >= 4:
return False
text = state.src[pos:maximum].strip()
if not text.startswith("("):
return False
if not text.endswith(")="):
return False
if not text[1:-2]:
return False
if silent:
return True
state.line = startLine + 1
token = state.push("myst_target", "", 0)
token.attrSet("class", "myst-target")
token.content = text[1:-2]
token.map = [startLine, state.line]
return True
def amsmath_block(state: StateBlock, startLine: int, endLine: int,
silent: bool):
# if it's indented more than 3 spaces, it should be a code block
if state.sCount[startLine] - state.blkIndent >= 4:
return False
begin = state.bMarks[startLine] + state.tShift[startLine]
outcome = match_environment(state.src[begin:])
if not outcome:
return False
environment, numbered, endpos = outcome
endpos += begin
line = startLine
while line < endLine:
if endpos >= state.bMarks[line] and endpos <= state.eMarks[line]:
# line for end of block math found ...
state.line = line + 1
break
line += 1
if not silent:
token = state.push("amsmath", "math", 0)
token.block = True
token.content = state.src[begin:endpos]
token.meta = {"environment": environment, "numbered": numbered}
token.map = [startLine, line]
return True
def line_comment(state: StateBlock, startLine: int, endLine: int,
silent: bool):
pos = state.bMarks[startLine] + state.tShift[startLine]
maximum = state.eMarks[startLine]
# if it's indented more than 3 spaces, it should be a code block
if state.sCount[startLine] - state.blkIndent >= 4:
return False
marker = state.srcCharCode[pos]
pos += 1
# Check block marker /* % */
if marker != 0x25:
return False
if silent:
return True
state.line = startLine + 1
token = state.push("myst_line_comment", "", 0)
token.attrSet("class", "myst-line-comment")
token.content = state.src[pos:maximum].strip()
token.map = [startLine, state.line]
token.markup = chr(marker)
return True
def skipMarker(state: StateBlock, line: int):
"""Search `[:~][\n ]`, returns next pos after marker on success or -1 on fail."""
start = state.bMarks[line] + state.tShift[line]
maximum = state.eMarks[line]
if start >= maximum:
return -1
# Check bullet
marker = state.srcCharCode[start]
start += 1
if marker != 0x7E and marker != 0x3A: # ~ :
return -1
pos = state.skipSpaces(start)
# require space after ":"
if start == pos:
return -1
# no empty definitions, e.g. " : "
if pos >= maximum:
return -1
return start
def block_break(state: StateBlock, startLine: int, endLine: int, silent: bool):
pos = state.bMarks[startLine] + state.tShift[startLine]
maximum = state.eMarks[startLine]
# if it's indented more than 3 spaces, it should be a code block
if state.sCount[startLine] - state.blkIndent >= 4:
return False
marker = state.srcCharCode[pos]
pos += 1
# Check block marker /* + */
if marker != 0x2B:
return False
# markers can be mixed with spaces, but there should be at least 3 of them
cnt = 1
while pos < maximum:
ch = state.srcCharCode[pos]
if ch != marker and not isSpace(ch):
break
if ch == marker:
cnt += 1
pos += 1
if cnt < 3:
return False
if silent:
return True
state.line = startLine + 1
token = state.push("myst_block_break", "hr", 0)
token.attrSet("class", "myst-block")
token.content = state.src[pos:maximum].strip()
token.map = [startLine, state.line]
token.markup = chr(marker) * cnt
return True
def _substitution_block(state: StateBlock, startLine: int, endLine: int,
silent: bool):
startPos = state.bMarks[startLine] + state.tShift[startLine]
end = state.eMarks[startLine]
# if it's indented more than 3 spaces, it should be a code block
if state.sCount[startLine] - state.blkIndent >= 4:
return False
lineText = state.src[startPos:end].strip()
try:
if (lineText[0] != start_delimiter
or lineText[1] != start_delimiter
or lineText[-1] != end_delimiter
or lineText[-2] != end_delimiter or len(lineText) < 5):
return False
except IndexError:
return False
text = lineText[2:-2].strip()
# special case if multiple on same line, e.g. {{a}}{{b}}
if (end_delimiter * 2) in text:
return False
state.line = startLine + 1
if silent:
return True
token = state.push("substitution_block", "div", 0)
token.block = True
token.content = text
token.attrSet("class", "substitution")
token.attrSet("text", text)
token.markup = f"{start_delimiter}{end_delimiter}"
token.map = [startLine, state.line]
return True
def target(state: StateBlock, startLine: int, endLine: int, silent: bool):
pos = state.bMarks[startLine] + state.tShift[startLine]
maximum = state.eMarks[startLine]
# if it's indented more than 3 spaces, it should be a code block
if state.sCount[startLine] - state.blkIndent >= 4:
return False
match = TARGET_PATTERN.match(state.src[pos:maximum])
if not match:
return False
if silent:
return True
state.line = startLine + 1
token = state.push("myst_target", "", 0)
token.attrSet("class", "myst-target")
token.content = match.group(1)
token.map = [startLine, state.line]
return True
def line_comment(state: StateBlock, startLine: int, endLine: int,
silent: bool):
pos = state.bMarks[startLine] + state.tShift[startLine]
maximum = state.eMarks[startLine]
# if it's indented more than 3 spaces, it should be a code block
if state.sCount[startLine] - state.blkIndent >= 4:
return False
if state.src[pos] != "%":
return False
if silent:
return True
token = state.push("myst_line_comment", "", 0)
token.attrSet("class", "myst-line-comment")
token.content = state.src[pos + 1:maximum].rstrip()
token.markup = "%"
# search end of block while appending lines to `token.content`
for nextLine in itertools.count(startLine + 1):
if nextLine >= endLine:
break
pos = state.bMarks[nextLine] + state.tShift[nextLine]
maximum = state.eMarks[nextLine]
if state.src[pos] != "%":
break
token.content += "\n" + state.src[pos + 1:maximum].rstrip()
state.line = nextLine
token.map = [startLine, nextLine]
return True
def front_matter(state: StateBlock, start_line: int, end_line: int,
silent: bool):
# grab initial data if it's : separated
if start_line != 0:
return False
# Since start is found, we can report success here in validation mode
# if silent:
# return True
# Search for the end of the block
next_line = start_line
start_content = 0
meta = {}
while True:
next_line += 1
if next_line >= end_line:
# unclosed block should be autoclosed by end of document.
return False
start = state.bMarks[next_line]
maximum = state.eMarks[next_line]
if start == maximum:
# empty line is terminator
break
key_value = state.src[start:maximum].split(":", 1)
if len(key_value) != 2:
# Error here, we have no k/v separator
return False
meta[key_value[0].lower()] = key_value[1]
old_parent = state.parentType
old_line_max = state.lineMax
state.parentType = "container"
# this will prevent lazy continuations from ever going past our end marker
state.lineMax = next_line
token = state.push("pelican_frontmatter", "", 0)
# token.hidden = True
token.content = state.src[state.bMarks[start_content]:state.
eMarks[next_line]]
token.block = True
token.meta = meta
state.parentType = old_parent
state.lineMax = old_line_max
state.line = next_line
token.map = [start_line, state.line]
# consider taking the content into a dictionary and checking for Title: .+\n
return True
def test_block_func():
block_func = main.make_block_func(main.rules.dollars.block[0])
md = MarkdownIt()
src = r"$$\na=1\n\nc\nb=2$$ (abc)"
tokens = []
state = StateBlock(src, md, {}, tokens)
block_func(state, 0, 10, False)
assert tokens[0].as_dict() == {
"type": "math_block_eqno",
"tag": "math",
"nesting": 0,
"attrs": None,
"map": None,
"level": 0,
"children": None,
"content": "\\na=1\\n\\nc\\nb=2",
"markup": "$$",
"info": "abc",
"meta": {},
"block": True,
"hidden": False,
}
def container_func(state: StateBlock, startLine: int, endLine: int, silent: bool):
auto_closed = False
start = state.bMarks[startLine] + state.tShift[startLine]
maximum = state.eMarks[startLine]
# Check out the first character quickly,
# this should filter out most of non-containers
if marker_char != charCodeAt(state.src, start):
return False
# Check out the rest of the marker string
pos = start + 1
while pos <= maximum:
if marker_str[(pos - start) % marker_len] != state.src[pos]:
break
pos += 1
marker_count = floor((pos - start) / marker_len)
if marker_count < min_markers:
return False
pos -= (pos - start) % marker_len
markup = state.src[start:pos]
params = state.src[pos:maximum]
if not validate(params, markup):
return False
# Since start is found, we can report success here in validation mode
if silent:
return True
# Search for the end of the block
nextLine = startLine
while True:
nextLine += 1
if nextLine >= endLine:
# unclosed block should be autoclosed by end of document.
# also block seems to be autoclosed by end of parent
break
start = state.bMarks[nextLine] + state.tShift[nextLine]
maximum = state.eMarks[nextLine]
if start < maximum and state.sCount[nextLine] < state.blkIndent:
# non-empty line with negative indent should stop the list:
# - ```
# test
break
if marker_char != charCodeAt(state.src, start):
continue
if state.sCount[nextLine] - state.blkIndent >= 4:
# closing fence should be indented less than 4 spaces
continue
pos = start + 1
while pos <= maximum:
if marker_str[(pos - start) % marker_len] != state.src[pos]:
break
pos += 1
# closing code fence must be at least as long as the opening one
if floor((pos - start) / marker_len) < marker_count:
continue
# make sure tail has spaces only
pos -= (pos - start) % marker_len
pos = state.skipSpaces(pos)
if pos < maximum:
continue
# found!
auto_closed = True
break
old_parent = state.parentType
old_line_max = state.lineMax
state.parentType = "container"
# this will prevent lazy continuations from ever going past our end marker
state.lineMax = nextLine
token = state.push(f"container_{name}_open", "div", 1)
token.markup = markup
token.block = True
token.info = params
token.map = [startLine, nextLine]
state.md.block.tokenize(state, startLine + 1, nextLine)
token = state.push(f"container_{name}_close", "div", -1)
token.markup = state.src[start:pos]
token.block = True
state.parentType = old_parent
state.lineMax = old_line_max
state.line = nextLine + (1 if auto_closed else 0)
return True
def _math_block_dollar(state: StateBlock, startLine: int, endLine: int,
silent: bool) -> bool:
# TODO internal backslash escaping
haveEndMarker = False
startPos = state.bMarks[startLine] + state.tShift[startLine]
end = state.eMarks[startLine]
# if it's indented more than 3 spaces, it should be a code block
if state.sCount[startLine] - state.blkIndent >= 4:
return False
if startPos + 2 > end:
return False
if (state.srcCharCode[startPos] != 0x24
or state.srcCharCode[startPos + 1] != 0x24): # /* $ */
return False
# search for end of block
nextLine = startLine
label = None
# search for end of block on same line
lineText = state.src[startPos:end]
if len(lineText.strip()) > 3:
if lineText.strip().endswith("$$"):
haveEndMarker = True
end = end - 2 - (len(lineText) - len(lineText.strip()))
elif allow_labels:
# reverse the line and match
eqnoMatch = DOLLAR_EQNO_REV.match(lineText[::-1])
if eqnoMatch:
haveEndMarker = True
label = eqnoMatch.group(1)[::-1]
end = end - eqnoMatch.end()
# search for end of block on subsequent line
if not haveEndMarker:
while True:
nextLine += 1
if nextLine >= endLine:
break
start = state.bMarks[nextLine] + state.tShift[nextLine]
end = state.eMarks[nextLine]
if end - start < 2:
continue
lineText = state.src[start:end]
if lineText.strip().endswith("$$"):
haveEndMarker = True
end = end - 2 - (len(lineText) - len(lineText.strip()))
break
# reverse the line and match
if allow_labels:
eqnoMatch = DOLLAR_EQNO_REV.match(lineText[::-1])
if eqnoMatch:
haveEndMarker = True
label = eqnoMatch.group(1)[::-1]
end = end - eqnoMatch.end()
break
if not haveEndMarker:
return False
state.line = nextLine + (1 if haveEndMarker else 0)
token = state.push("math_block_label" if label else "math_block",
"math", 0)
token.block = True
token.content = state.src[startPos + 2:end]
token.markup = "$$"
token.map = [startLine, state.line]
if label:
token.info = label if label_normalizer is None else label_normalizer(
label)
return True
def test_footnote_tail():
md = MarkdownIt()
tokens = [
Token(
**{
"type": "footnote_reference_open",
"tag": "",
"nesting": 1,
"attrs": None,
"map": None,
"level": 0,
"children": None,
"content": "",
"markup": "",
"info": "",
"meta": {
"label": "a"
},
"block": False,
"hidden": False,
}),
Token(
**{
"type": "paragraph_open",
"tag": "p",
"nesting": 1,
"attrs": None,
"map": [0, 1],
"level": 1,
"children": None,
"content": "",
"markup": "",
"info": "",
"meta": {},
"block": True,
"hidden": False,
}),
Token(
**{
"type": "inline",
"tag": "",
"nesting": 0,
"attrs": None,
"map": [0, 1],
"level": 2,
"children": [],
"content": "xyz",
"markup": "",
"info": "",
"meta": {},
"block": True,
"hidden": False,
}),
Token(
**{
"type": "paragraph_close",
"tag": "p",
"nesting": -1,
"attrs": None,
"map": None,
"level": 1,
"children": None,
"content": "",
"markup": "",
"info": "",
"meta": {},
"block": True,
"hidden": False,
}),
Token(
**{
"type": "footnote_reference_close",
"tag": "",
"nesting": -1,
"attrs": None,
"map": None,
"level": 0,
"children": None,
"content": "",
"markup": "",
"info": "",
"meta": {},
"block": False,
"hidden": False,
}),
Token("other", "", 0),
]
env = {
"footnotes": {
"refs": {
":a": 0
},
"list": {
0: {
"label": "a",
"count": 1
}
}
}
}
state = StateBlock("", md, env, tokens)
index.footnote_tail(state)
assert state.tokens == [
Token(
type="other",
tag="",
nesting=0,
attrs=None,
map=None,
level=0,
children=None,
content="",
markup="",
info="",
meta={},
block=False,
hidden=False,
),
Token(
type="footnote_block_open",
tag="",
nesting=1,
attrs=None,
map=None,
level=0,
children=None,
content="",
markup="",
info="",
meta={},
block=False,
hidden=False,
),
Token(
type="footnote_open",
tag="",
nesting=1,
attrs=None,
map=None,
level=0,
children=None,
content="",
markup="",
info="",
meta={
"id": 0,
"label": "a"
},
block=False,
hidden=False,
),
Token(
type="paragraph_open",
tag="p",
nesting=1,
attrs=None,
map=[0, 1],
level=1,
children=None,
content="",
markup="",
info="",
meta={},
block=True,
hidden=False,
),
Token(
type="inline",
tag="",
nesting=0,
attrs=None,
map=[0, 1],
level=2,
children=[],
content="xyz",
markup="",
info="",
meta={},
block=True,
hidden=False,
),
Token(
type="footnote_anchor",
tag="",
nesting=0,
attrs=None,
map=None,
level=0,
children=None,
content="",
markup="",
info="",
meta={
"id": 0,
"subId": 0,
"label": "a"
},
block=False,
hidden=False,
),
Token(
type="paragraph_close",
tag="p",
nesting=-1,
attrs=None,
map=None,
level=1,
children=None,
content="",
markup="",
info="",
meta={},
block=True,
hidden=False,
),
Token(
type="footnote_close",
tag="",
nesting=-1,
attrs=None,
map=None,
level=0,
children=None,
content="",
markup="",
info="",
meta={},
block=False,
hidden=False,
),
Token(
type="footnote_block_close",
tag="",
nesting=-1,
attrs=None,
map=None,
level=0,
children=None,
content="",
markup="",
info="",
meta={},
block=False,
hidden=False,
),
]
def _rule(state: StateBlock, startLine: int, endLine: int, silent: bool):
haveEndMarker = False
pos = state.bMarks[startLine] + state.tShift[startLine]
maximum = state.eMarks[startLine]
# if it's indented more than 3 spaces, it should be a code block
if state.sCount[startLine] - state.blkIndent >= 4:
return False
if pos + 3 > maximum:
return False
marker = state.srcCharCode[pos]
# /* : */
if marker != 0x3A:
return False
# scan marker length
mem = pos
pos = state.skipChars(pos, marker)
length = pos - mem
if length < 3:
return False
markup = state.src[mem:pos]
params = state.src[pos:maximum]
# Since start is found, we can report success here in validation mode
if silent:
return True
# search end of block
nextLine = startLine
while True:
nextLine += 1
if nextLine >= endLine:
# unclosed block should be autoclosed by end of document.
# also block seems to be autoclosed by end of parent
break
pos = mem = state.bMarks[nextLine] + state.tShift[nextLine]
maximum = state.eMarks[nextLine]
if pos < maximum and state.sCount[nextLine] < state.blkIndent:
# non-empty line with negative indent should stop the list:
# - ```
# test
break
if state.srcCharCode[pos] != marker:
continue
if state.sCount[nextLine] - state.blkIndent >= 4:
# closing fence should be indented less than 4 spaces
continue
pos = state.skipChars(pos, marker)
# closing code fence must be at least as long as the opening one
if pos - mem < length:
continue
# make sure tail has spaces only
pos = state.skipSpaces(pos)
if pos < maximum:
continue
haveEndMarker = True
# found!
break
# If a fence has heading spaces, they should be removed from its inner block
length = state.sCount[startLine]
state.line = nextLine + (1 if haveEndMarker else 0)
token = state.push("colon_fence", "code", 0)
token.info = stripEscape(params)
token.content = state.getLines(startLine + 1, nextLine, length, True)
token.markup = markup
token.map = [startLine, state.line]
return True
def footnote_tail(state: StateBlock, *args, **kwargs):
"""Post-processing step, to move footnote tokens to end of the token stream.
Also removes un-referenced tokens.
"""
insideRef = False
refTokens = {}
if "footnotes" not in state.env:
return
current = []
tok_filter = []
for tok in state.tokens:
if tok.type == "footnote_reference_open":
insideRef = True
current = []
currentLabel = tok.meta["label"]
tok_filter.append(False)
continue
if tok.type == "footnote_reference_close":
insideRef = False
# prepend ':' to avoid conflict with Object.prototype members
refTokens[":" + currentLabel] = current
tok_filter.append(False)
continue
if insideRef:
current.append(tok)
tok_filter.append((not insideRef))
state.tokens = [t for t, f in zip(state.tokens, tok_filter) if f]
if "list" not in state.env.get("footnotes", {}):
return
foot_list = state.env["footnotes"]["list"]
token = Token("footnote_block_open", "", 1)
state.tokens.append(token)
for i, foot_note in foot_list.items():
token = Token("footnote_open", "", 1)
token.meta = {"id": i, "label": foot_note.get("label", None)}
# TODO propagate line positions of original foot note
# (but don't store in token.map, because this is used for scroll syncing)
state.tokens.append(token)
if "tokens" in foot_note:
tokens = []
token = Token("paragraph_open", "p", 1)
token.block = True
tokens.append(token)
token = Token("inline", "", 0)
token.children = foot_note["tokens"]
token.content = foot_note["content"]
tokens.append(token)
token = Token("paragraph_close", "p", -1)
token.block = True
tokens.append(token)
elif "label" in foot_note:
tokens = refTokens[":" + foot_note["label"]]
state.tokens.extend(tokens)
if state.tokens[len(state.tokens) - 1].type == "paragraph_close":
lastParagraph = state.tokens.pop()
else:
lastParagraph = None
t = (foot_note["count"] if
(("count" in foot_note) and (foot_note["count"] > 0)) else 1)
j = 0
while j < t:
token = Token("footnote_anchor", "", 0)
token.meta = {
"id": i,
"subId": j,
"label": foot_note.get("label", None)
}
state.tokens.append(token)
j += 1
if lastParagraph:
state.tokens.append(lastParagraph)
token = Token("footnote_close", "", -1)
state.tokens.append(token)
token = Token("footnote_block_close", "", -1)
state.tokens.append(token)
def test_footnote_def():
md = MarkdownIt()
src = r"[^a]: xyz"
tokens = []
state = StateBlock(src, md, {}, tokens)
index.footnote_def(state, 0, 1, False)
assert [t.as_dict() for t in tokens] == [
{
"type": "footnote_reference_open",
"tag": "",
"nesting": 1,
"attrs": None,
"map": [0, 1],
"level": 0,
"children": None,
"content": "",
"markup": "",
"info": "",
"meta": {
"label": "a"
},
"block": False,
"hidden": False,
},
{
"type": "paragraph_open",
"tag": "p",
"nesting": 1,
"attrs": None,
"map": [0, 1],
"level": 1,
"children": None,
"content": "",
"markup": "",
"info": "",
"meta": {},
"block": True,
"hidden": False,
},
{
"type": "inline",
"tag": "",
"nesting": 0,
"attrs": None,
"map": [0, 1],
"level": 2,
"children": [],
"content": "xyz",
"markup": "",
"info": "",
"meta": {},
"block": True,
"hidden": False,
},
{
"type": "paragraph_close",
"tag": "p",
"nesting": -1,
"attrs": None,
"map": None,
"level": 1,
"children": None,
"content": "",
"markup": "",
"info": "",
"meta": {},
"block": True,
"hidden": False,
},
{
"type": "footnote_reference_close",
"tag": "",
"nesting": -1,
"attrs": None,
"map": None,
"level": 0,
"children": None,
"content": "",
"markup": "",
"info": "",
"meta": {},
"block": False,
"hidden": False,
},
]
assert state.env == {"footnotes": {"refs": {":a": -1}}}
def frontMatter(state: StateBlock, startLine: int, endLine: int,
silent: bool):
auto_closed = False
start = state.bMarks[startLine] + state.tShift[startLine]
maximum = state.eMarks[startLine]
# Check out the first character of the first line quickly,
# this should filter out non-front matter
if startLine != 0 or marker_char != state.srcCharCode[0]:
return False
# Check out the rest of the marker string
# while pos <= 3
pos = start + 1
while pos <= maximum:
if marker_str[(pos - start) % marker_len] != state.src[pos]:
start_content = pos + 1
break
pos += 1
marker_count = floor((pos - start) / marker_len)
if marker_count < min_markers:
return False
pos -= (pos - start) % marker_len
# Since start is found, we can report success here in validation mode
if silent:
return True
# Search for the end of the block
nextLine = startLine
while True:
nextLine += 1
if nextLine >= endLine:
# unclosed block should be autoclosed by end of document.
return False
if state.src[start:maximum] == "...":
break
start = state.bMarks[nextLine] + state.tShift[nextLine]
maximum = state.eMarks[nextLine]
if start < maximum and state.sCount[nextLine] < state.blkIndent:
# non-empty line with negative indent should stop the list:
# - ```
# test
break
if marker_char != state.srcCharCode[start]:
continue
if state.sCount[nextLine] - state.blkIndent >= 4:
# closing fence should be indented less than 4 spaces
continue
pos = start + 1
while pos < maximum:
if marker_str[(pos - start) % marker_len] != state.src[pos]:
break
pos += 1
# closing code fence must be at least as long as the opening one
if floor((pos - start) / marker_len) < marker_count:
continue
# make sure tail has spaces only
pos -= (pos - start) % marker_len
pos = state.skipSpaces(pos)
if pos < maximum:
continue
# found!
auto_closed = True
break
old_parent = state.parentType
old_line_max = state.lineMax
state.parentType = "container"
# this will prevent lazy continuations from ever going past our end marker
state.lineMax = nextLine
token = state.push("front_matter", "", 0)
token.hidden = True
token.markup = marker_str * min_markers
token.content = state.src[state.bMarks[startLine +
1]:state.eMarks[nextLine - 1]]
token.block = True
token.meta = state.src[start_content:start - 1]
state.parentType = old_parent
state.lineMax = old_line_max
state.line = nextLine + (1 if auto_closed else 0)
token.map = [startLine, state.line]
return True
def deflist(state: StateBlock, startLine: int, endLine: int, silent: bool):
if silent:
# quirk: validation mode validates a dd block only, not a whole deflist
if state.ddIndent < 0:
return False
return skipMarker(state, startLine) >= 0
nextLine = startLine + 1
if nextLine >= endLine:
return False
if state.isEmpty(nextLine):
nextLine += 1
if nextLine >= endLine:
return False
if state.sCount[nextLine] < state.blkIndent:
return False
contentStart = skipMarker(state, nextLine)
if contentStart < 0:
return False
# Start list
listTokIdx = len(state.tokens)
tight = True
token = state.push("dl_open", "dl", 1)
token.map = listLines = [startLine, 0]
# Iterate list items
dtLine = startLine
ddLine = nextLine
# One definition list can contain multiple DTs,
# and one DT can be followed by multiple DDs.
#
# Thus, there is two loops here, and label is
# needed to break out of the second one
#
break_outer = False
while True:
prevEmptyEnd = False
token = state.push("dt_open", "dt", 1)
token.map = [dtLine, dtLine]
token = state.push("inline", "", 0)
token.map = [dtLine, dtLine]
token.content = state.getLines(dtLine, dtLine + 1, state.blkIndent,
False).strip()
token.children = []
token = state.push("dt_close", "dt", -1)
while True:
token = state.push("dd_open", "dd", 1)
token.map = itemLines = [nextLine, 0]
pos = contentStart
maximum = state.eMarks[ddLine]
offset = (state.sCount[ddLine] + contentStart -
(state.bMarks[ddLine] + state.tShift[ddLine]))
while pos < maximum:
ch = state.srcCharCode[pos]
if isSpace(ch):
if ch == 0x09:
offset += 4 - offset % 4
else:
offset += 1
else:
break
pos += 1
contentStart = pos
oldTight = state.tight
oldDDIndent = state.ddIndent
oldIndent = state.blkIndent
oldTShift = state.tShift[ddLine]
oldSCount = state.sCount[ddLine]
oldParentType = state.parentType
state.blkIndent = state.ddIndent = state.sCount[ddLine] + 2
state.tShift[ddLine] = contentStart - state.bMarks[ddLine]
state.sCount[ddLine] = offset
state.tight = True
state.parentType = "deflist"
state.md.block.tokenize(state, ddLine, endLine, True)
# If any of list item is tight, mark list as tight
if not state.tight or prevEmptyEnd:
tight = False
# Item become loose if finish with empty line,
# but we should filter last element, because it means list finish
prevEmptyEnd = (state.line -
ddLine) > 1 and state.isEmpty(state.line - 1)
state.tShift[ddLine] = oldTShift
state.sCount[ddLine] = oldSCount
state.tight = oldTight
state.parentType = oldParentType
state.blkIndent = oldIndent
state.ddIndent = oldDDIndent
token = state.push("dd_close", "dd", -1)
itemLines[1] = nextLine = state.line
if nextLine >= endLine:
break_outer = True
break
if state.sCount[nextLine] < state.blkIndent:
break_outer = True
break
contentStart = skipMarker(state, nextLine)
if contentStart < 0:
break
ddLine = nextLine
# go to the next loop iteration:
# insert DD tag and repeat checking
if break_outer:
break_outer = False
break
if nextLine >= endLine:
break
dtLine = nextLine
if state.isEmpty(dtLine):
break
if state.sCount[dtLine] < state.blkIndent:
break
ddLine = dtLine + 1
if ddLine >= endLine:
break
if state.isEmpty(ddLine):
ddLine += 1
if ddLine >= endLine:
break
if state.sCount[ddLine] < state.blkIndent:
break
contentStart = skipMarker(state, ddLine)
if contentStart < 0:
break
# go to the next loop iteration:
# insert DT and DD tags and repeat checking
# Finalise list
token = state.push("dl_close", "dl", -1)
listLines[1] = nextLine
state.line = nextLine
# mark paragraphs tight if needed
if tight:
markTightParagraphs(state, listTokIdx)
return True
def footnote_def(state: StateBlock, startLine: int, endLine: int,
silent: bool):
"""Process footnote block definition"""
start = state.bMarks[startLine] + state.tShift[startLine]
maximum = state.eMarks[startLine]
# line should be at least 5 chars - "[^x]:"
if start + 4 > maximum:
return False
if state.srcCharCode[start] != 0x5B: # /* [ */
return False
if state.srcCharCode[start + 1] != 0x5E: # /* ^ */
return False
pos = start + 2
while pos < maximum:
if state.srcCharCode[pos] == 0x20:
return False
if state.srcCharCode[pos] == 0x5D: # /* ] */
break
pos += 1
if pos == start + 2: # no empty footnote labels
return False
pos += 1
if pos + 1 >= maximum or state.srcCharCode[pos] != 0x3A: # /* : */
return False
if silent:
return True
pos += 1
label = state.src[start + 2:pos - 2]
state.env.setdefault("footnotes", {}).setdefault("refs",
{})[":" + label] = -1
open_token = Token("footnote_reference_open", "", 1)
open_token.meta = {"label": label}
open_token.level = state.level
state.level += 1
state.tokens.append(open_token)
oldBMark = state.bMarks[startLine]
oldTShift = state.tShift[startLine]
oldSCount = state.sCount[startLine]
oldParentType = state.parentType
posAfterColon = pos
initial = offset = (state.sCount[startLine] + pos -
(state.bMarks[startLine] + state.tShift[startLine]))
while pos < maximum:
ch = state.srcCharCode[pos]
if isSpace(ch):
if ch == 0x09:
offset += 4 - offset % 4
else:
offset += 1
else:
break
pos += 1
state.tShift[startLine] = pos - posAfterColon
state.sCount[startLine] = offset - initial
state.bMarks[startLine] = posAfterColon
state.blkIndent += 4
state.parentType = "footnote"
if state.sCount[startLine] < state.blkIndent:
state.sCount[startLine] += state.blkIndent
state.md.block.tokenize(state, startLine, endLine, True)
state.parentType = oldParentType
state.blkIndent -= 4
state.tShift[startLine] = oldTShift
state.sCount[startLine] = oldSCount
state.bMarks[startLine] = oldBMark
open_token.map = [startLine, state.line]
token = Token("footnote_reference_close", "", -1)
state.level -= 1
token.level = state.level
state.tokens.append(token)
return True
def set_parent_type(state: StateBlock, name: str):
"""Temporarily set parent type to `name`"""
oldParentType = state.parentType
state.parentType = name
yield
state.parentType = oldParentType
def _fieldlist_rule(state: StateBlock, startLine: int, endLine: int, silent: bool):
# adapted from markdown_it/rules_block/list.py::list_block
# if it's indented more than 3 spaces, it should be a code block
if state.sCount[startLine] - state.blkIndent >= 4:
return False
posAfterName, name_text = parseNameMarker(state, startLine)
if posAfterName < 0:
return False
# For validation mode we can terminate immediately
if silent:
return True
# start field list
token = state.push("field_list_open", "dl", 1)
token.attrSet("class", "field-list")
token.map = listLines = [startLine, 0]
# iterate list items
nextLine = startLine
with set_parent_type(state, "fieldlist"):
while nextLine < endLine:
# create name tokens
token = state.push("fieldlist_name_open", "dt", 1)
token.map = [startLine, startLine]
token = state.push("inline", "", 0)
token.map = [startLine, startLine]
token.content = name_text
token.children = []
token = state.push("fieldlist_name_close", "dt", -1)
# set indent positions
pos = posAfterName
maximum = state.eMarks[nextLine]
offset = (
state.sCount[nextLine]
+ posAfterName
- (state.bMarks[startLine] + state.tShift[startLine])
)
# find indent to start of body on first line
while pos < maximum:
ch = state.srcCharCode[pos]
if ch == 0x09: # \t
offset += 4 - (offset + state.bsCount[nextLine]) % 4
elif ch == 0x20: # \s
offset += 1
else:
break
pos += 1
contentStart = pos
# set indent for body text
if contentStart >= maximum:
# no body on first line, so use constant indentation
# TODO adapt to indentation of subsequent lines?
indent = 2
else:
indent = offset
# Run subparser on the field body
token = state.push("fieldlist_body_open", "dd", 1)
token.map = itemLines = [startLine, 0]
# change current state, then restore it after parser subcall
oldTShift = state.tShift[startLine]
oldSCount = state.sCount[startLine]
oldBlkIndent = state.blkIndent
state.tShift[startLine] = contentStart - state.bMarks[startLine]
state.sCount[startLine] = offset
state.blkIndent = indent
state.md.block.tokenize(state, startLine, endLine)
state.blkIndent = oldBlkIndent
state.tShift[startLine] = oldTShift
state.sCount[startLine] = oldSCount
token = state.push("fieldlist_body_close", "dd", -1)
nextLine = startLine = state.line
itemLines[1] = nextLine
if nextLine >= endLine:
break
contentStart = state.bMarks[startLine]
# Try to check if list is terminated or continued.
if state.sCount[nextLine] < state.blkIndent:
break
# if it's indented more than 3 spaces, it should be a code block
if state.sCount[startLine] - state.blkIndent >= 4:
break
# get next field item
posAfterName, name_text = parseNameMarker(state, startLine)
if posAfterName < 0:
break
# Finalize list
token = state.push("field_list_close", "dl", -1)
listLines[1] = nextLine
state.line = nextLine
return True
