예제 #1
0
def testCoreToken_ProcessHeaders(mockGUI):
    """Test the header and page parser of the Tokenizer class.
    """
    theProject = NWProject(mockGUI)
    theProject.projLang = "en"
    theProject._loadProjectLocalisation()
    theToken = BareTokenizer(theProject)

    # Nothing
    theToken._theText = "Some text ...\n"
    assert theToken.doHeaders() is False
    theToken._isNone = True
    assert theToken.doHeaders() is False
    theToken._isNone = False
    assert theToken.doHeaders() is False
    theToken._isNote = True
    assert theToken.doHeaders() is False
    theToken._isNote = False

    ##
    #  Story FIles
    ##

    theToken._isNone = False
    theToken._isNote = False
    theToken._isNovel = True

    # Titles
    # ======

    # H1: Title, First Page
    assert theToken._isFirst is True
    theToken._theText = "# Part One\n"
    theToken.setTitleFormat(r"T: %title%")
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_HEAD1, 1, "T: Part One", None, Tokenizer.A_CENTRE),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H1: Title, Not First Page
    assert theToken._isFirst is False
    theToken._theText = "# Part One\n"
    theToken.setTitleFormat(r"T: %title%")
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_HEAD1, 1, "T: Part One", None,
         Tokenizer.A_PBB | Tokenizer.A_CENTRE),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # Chapters
    # ========

    # H2: Chapter
    theToken._theText = "## Chapter One\n"
    theToken.setChapterFormat(r"C: %title%")
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_HEAD2, 1, "C: Chapter One", None, Tokenizer.A_PBB),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H2: Unnumbered Chapter
    theToken._theText = "##! Prologue\n"
    theToken.setUnNumberedFormat(r"U: %title%")
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_UNNUM, 1, "U: Prologue", None, Tokenizer.A_PBB),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H2: Chapter Word Number
    theToken._theText = "## Chapter\n"
    theToken.setChapterFormat(r"Chapter %chw%")
    theToken._numChapter = 0
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_HEAD2, 1, "Chapter One", None, Tokenizer.A_PBB),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H2: Chapter Roman Number Upper Case
    theToken._theText = "## Chapter\n"
    theToken.setChapterFormat(r"Chapter %chI%")
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_HEAD2, 1, "Chapter II", None, Tokenizer.A_PBB),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H2: Chapter Roman Number Lower Case
    theToken._theText = "## Chapter\n"
    theToken.setChapterFormat(r"Chapter %chi%")
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_HEAD2, 1, "Chapter iii", None, Tokenizer.A_PBB),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # Scenes
    # ======

    # H3: Scene w/Title
    theToken._theText = "### Scene One\n"
    theToken.setSceneFormat(r"S: %title%", False)
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_HEAD3, 1, "S: Scene One", None, Tokenizer.A_NONE),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H3: Scene Hidden wo/Format
    theToken._theText = "### Scene One\n"
    theToken.setSceneFormat(r"", True)
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H3: Scene wo/Format, first
    theToken._theText = "### Scene One\n"
    theToken.setSceneFormat(r"", False)
    theToken._firstScene = True
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H3: Scene wo/Format, not first
    theToken._theText = "### Scene One\n"
    theToken.setSceneFormat(r"", False)
    theToken._firstScene = False
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H3: Scene Separator, first
    theToken._theText = "### Scene One\n"
    theToken.setSceneFormat(r"* * *", False)
    theToken._firstScene = True
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H3: Scene Separator, not first
    theToken._theText = "### Scene One\n"
    theToken.setSceneFormat(r"* * *", False)
    theToken._firstScene = False
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_SEP, 1, "* * *", None, Tokenizer.A_CENTRE),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H3: Scene w/Absolute Number
    theToken._theText = "### A Scene\n"
    theToken.setSceneFormat(r"Scene %sca%", False)
    theToken._numAbsScene = 0
    theToken._numChScene = 0
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_HEAD3, 1, "Scene 1", None, Tokenizer.A_NONE),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H3: Scene w/Chapter Number
    theToken._theText = "### A Scene\n"
    theToken.setSceneFormat(r"Scene %ch%.%sc%", False)
    theToken._numAbsScene = 0
    theToken._numChScene = 1
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_HEAD3, 1, "Scene 3.2", None, Tokenizer.A_NONE),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # Sections
    # ========

    # H4: Section Hidden wo/Format
    theToken._theText = "#### A Section\n"
    theToken.setSectionFormat(r"", True)
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H4: Section Visible wo/Format
    theToken._theText = "#### A Section\n"
    theToken.setSectionFormat(r"", False)
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_SKIP, 1, "", None, Tokenizer.A_NONE),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H4: Section w/Format
    theToken._theText = "#### A Section\n"
    theToken.setSectionFormat(r"X: %title%", False)
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_HEAD4, 1, "X: A Section", None, Tokenizer.A_NONE),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # H4: Section Separator
    theToken._theText = "#### A Section\n"
    theToken.setSectionFormat(r"* * *", False)
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._theTokens == [
        (Tokenizer.T_SEP, 1, "* * *", None, Tokenizer.A_CENTRE),
        (Tokenizer.T_EMPTY, 1, "", None, Tokenizer.A_NONE),
    ]

    # Check the first scene detector
    assert theToken._firstScene is False
    theToken._firstScene = True
    theToken._theText = "Some text ...\n"
    theToken.tokenizeText()
    theToken.doHeaders()
    assert theToken._firstScene is False
예제 #2
0
def testCoreToken_TextOps(monkeypatch, nwMinimal, mockGUI):
    """Test handling files and text in the Tokenizer class.
    """
    theProject = NWProject(mockGUI)
    theProject.projTree.setSeed(42)
    theProject.projLang = "en"
    theProject._loadProjectLocalisation()

    theToken = BareTokenizer(theProject)
    theToken.setKeepMarkdown(True)

    assert theProject.openProject(nwMinimal)
    sHandle = "8c659a11cd429"

    # Set some content to work with

    docText = (
        "### Scene Six\n\n"
        "This is text with _italic text_, some **bold text**, some ~~deleted text~~, "
        "and some **_mixed text_** and **some _nested_ text**.\n\n"
        "#### Replace\n\n"
        "Also, replace <A> and <B>.\n\n")
    docTextR = docText.replace("<A>", "this").replace("<B>", "that")

    nDoc = NWDoc(theProject, sHandle)
    assert nDoc.writeDocument(docText)

    theProject.setAutoReplace({"A": "this", "B": "that"})

    assert theProject.saveProject()

    # Root Heading
    assert theToken.addRootHeading("stuff") is False
    assert theToken.addRootHeading(sHandle) is False

    # First Page
    assert theToken.addRootHeading("7695ce551d265") is True
    assert theToken.theMarkdown[-1] == "# Notes: Plot\n\n"
    assert theToken._theTokens[-1] == (Tokenizer.T_TITLE, 0, "Notes: Plot",
                                       None, Tokenizer.A_CENTRE)

    # Not First Page
    assert theToken.addRootHeading("7695ce551d265") is True
    assert theToken.theMarkdown[-1] == "# Notes: Plot\n\n"
    assert theToken._theTokens[-1] == (Tokenizer.T_TITLE, 0, "Notes: Plot",
                                       None,
                                       Tokenizer.A_CENTRE | Tokenizer.A_PBB)

    # Set Text
    assert theToken.setText("stuff") is False
    assert theToken.setText(sHandle) is True
    assert theToken._theText == docText

    with monkeypatch.context() as mp:
        mp.setattr("novelwriter.constants.nwConst.MAX_DOCSIZE", 100)
        assert theToken.setText(sHandle, docText) is True
        assert theToken._theText == (
            "# ERROR\n\n"
            "Document 'New Scene' is too big (0.00 MB). Skipping.\n\n")

    assert theToken.setText(sHandle, docText) is True
    assert theToken._theText == docText

    assert theToken._isNone is False
    assert theToken._isNovel is True
    assert theToken._isNote is False

    # Pre Processing
    theToken.doPreProcessing()
    assert theToken._theText == docTextR

    # Post Processing
    theToken._theResult = r"This is text with escapes: \** \~~ \__"
    theToken.doPostProcessing()
    assert theToken.theResult == "This is text with escapes: ** ~~ __"

    # Save File
    savePath = os.path.join(nwMinimal, "dump.nwd")
    theToken.saveRawMarkdown(savePath)
    assert readFile(savePath) == ("# Notes: Plot\n\n" "# Notes: Plot\n\n")

    # Ckeck abstract method
    with pytest.raises(NotImplementedError):
        theToken.doConvert()