def test_token():
token = Token("name", "tag", 0)
assert token.as_dict() == {
"type": "name",
"tag": "tag",
"nesting": 0,
"attrs": None,
"map": None,
"level": 0,
"children": None,
"content": "",
"markup": "",
"info": "",
"meta": {},
"block": False,
"hidden": False,
}
token.attrSet("a", "b")
assert token.attrGet("a") == "b"
token.attrJoin("a", "c")
assert token.attrGet("a") == "b c"
token.attrPush(["x", "y"])
assert token.attrGet("x") == "y"
assert token.attrIndex("a") == 0
assert token.attrIndex("x") == 1
assert token.attrIndex("j") == -1
def test_comment_token():
md = MarkdownIt("commonmark").use(myst_block_plugin)
tokens = md.parse("\n\n% abc \n%def")
expected_token = Token(
type="myst_line_comment",
tag="",
nesting=0,
map=[2, 4],
level=0,
children=None,
content=" abc\ndef",
markup="%",
info="",
meta={},
block=True,
hidden=False,
)
expected_token.attrSet("class", "myst-line-comment")
assert tokens == [expected_token]
def test_block_token():
md = MarkdownIt("commonmark").use(myst_block_plugin)
tokens = md.parse("+++")
expected_token = Token(
type="myst_block_break",
tag="hr",
nesting=0,
map=[0, 1],
level=0,
children=None,
content="",
markup="+++",
info="",
meta={},
block=True,
hidden=False,
)
expected_token.attrSet("class", "myst-block")
assert tokens == [expected_token]
tokens = md.parse("\n+ + + abc")
expected_token = Token(
type="myst_block_break",
tag="hr",
nesting=0,
map=[1, 2],
level=0,
children=None,
content="abc",
markup="+++",
info="",
meta={},
block=True,
hidden=False,
)
expected_token.attrSet("class", "myst-block")
assert tokens == [expected_token]
def _anchor_func(state: StateCore):
for (idx, token) in enumerate(state.tokens):
if token.type != "heading_open":
continue
level = int(token.tag[1])
if level not in selected_levels:
continue
inline_token = state.tokens[idx + 1]
assert inline_token.children is not None
title = "".join(child.content for child in inline_token.children
if child.type in ["text", "code_inline"])
slug = unique_slug(slug_func(title), slugs)
token.attrSet("id", slug)
if permalink:
link_open = Token(
"link_open",
"a",
1,
)
link_open.attrSet("class", "header-anchor")
link_open.attrSet("href", f"#{slug}")
link_tokens = [
link_open,
Token("html_block", "", 0, content=permalinkSymbol),
Token("link_close", "a", -1),
]
if permalinkBefore:
inline_token.children = (
link_tokens + ([Token("text", "", 0, content=" ")]
if permalinkSpace else []) +
inline_token.children)
else:
inline_token.children.extend(
([Token("text", "", 0, content=" "
)] if permalinkSpace else []) + link_tokens)
