def test_tokenize_files():
contents = [
['# First file\n', 'Hey\n'],
['# Second file\n', 'Ho\n'],
]
assert list(tokenize_files(contents)) == [
Token.FILE(),
Token.TITLE('# First file\n', level=1),
Token.AFTER_TITLE(level=1),
Token.LINE('Hey\n'),
Token.AFTER_FILE(),
Token.FILE(),
Token.TITLE('# Second file\n', level=1),
Token.AFTER_TITLE(level=1),
Token.LINE('Ho\n'),
Token.AFTER_FILE(),
]
Token.LINE('\n'),
Token.START_CODE('```python\n', language='python', skip=False),
Token.LINE('def random():\n'),
Token.LINE(' return 4\n'),
Token.END_CODE('```\n'),
Token.LINE('\n'),
Token.SPLIT('---\n'),
Token.LINE('\n'),
Token.START_CODE('```python\n', language='python', skip=False),
Token.LINE('print(random())\n'),
Token.END_CODE('```\n'),
Token.LINE('\n'),
Token.SPLIT('---\n'),
Token.LINE('\n'),
Token.LINE('Text.\n'),
Token.AFTER_FILE(),
Token.FILE(),
Token.AFTER_FILE(),
Token.FILE(),
Token.TITLE('## 222\n', level=2),
Token.AFTER_TITLE(level=2),
Token.LINE('\n'),
Token.START_CODE('```python skip\n', language='python', skip=True),
Token.LINE('import itertools\n'),
Token.END_CODE('```\n'),
Token.LINE('\n'),
Token.LINE('Hello world.\n'),
Token.AFTER_FILE(),
]