def doWikiWideSearch(wikiDocument, regexpString):
    sarOp = SearchReplaceOperation()

    sarOp.wikiWide = True
    sarOp.wildCard = 'regex'
    sarOp.caseSensitive = False

    sarOp.searchStr = regexpString

    return wikiDocument.searchWiki(sarOp)
def doWikiWideSearch(wikiDocument, regexpString):
    sarOp = SearchReplaceOperation()

    sarOp.wikiWide = True
    sarOp.wildCard = 'regex'
    sarOp.caseSensitive = False

    sarOp.searchStr = regexpString

    return wikiDocument.searchWiki(sarOp)
Beispiel #3
0
def updateDatabase(connwrap, dataDir):
    """
    Update a database from an older version to current (checkDatabaseFormat()
    should have returned 1 before calling this function)
    """
    connwrap.syncCommit()
    
    # Always remember that there is no automatic unicode<->utf-8 conversion
    # during this function!
    indices = connwrap.execSqlQuerySingleColumn(
            "select name from sqlite_master where type='index'")
    tables = connwrap.execSqlQuerySingleColumn(
            "select name from sqlite_master where type='table'")

    indices = [s.upper() for s in indices]
    tables = [s.upper() for s in tables]
    
    # updatedTables = []
    
    if not "SETTINGS" in tables:
        # We are prior WikidPadCompact 1.3pre (which writes format version 0)
        
        # From WikidPad original
        if "WIKIWORDPROPS_PKEY" in indices:
            connwrap.execSql("drop index wikiwordprops_pkey")
        if "REGISTRATION" in tables:
            connwrap.execSql("drop table registration")
            
        # Here we have reached WikidPadCompact 1.0/1.1 format            
            
        # For database format changes
        rt = "real not null default 0.0"
        it = "integer not null default 0"
        tt = "text not null default ''"
        bt = "blob not null default x''"
    
        # From WikiPadCompact 1.1 to 1.2:            
        # if "WIKIWORDS" in tables:
        connwrap.execSqlNoError("drop table wikiwords")
            
        # Remove column "created" from "wikirelations"
        changed = changeTableSchema(connwrap, "wikirelations", 
                TABLE_DEFINITIONS["wikirelations"])

        # --- WikiPadCompact 1.2 reached ---
    
        # From WikiPadCompact 1.2 to 1.3:
        # Update all text items from standard encoding to UTF8
        # TODO: "Please wait"

        ## wikidata.dbConn.createFunction("stdToUtf8", 1, sqlite_stdToUtf8)
        
        connwrap.execSql("update wikiwordcontent set word=latin1ToUtf8(word), "+
                "content=textToBlob(content)")
        connwrap.execSql("update wikiwordprops set word=latin1ToUtf8(word), "+
                "key=latin1ToUtf8(key), value=latin1ToUtf8(value)")
        connwrap.execSql("update todos set word=latin1ToUtf8(word), "+
                "todo=latin1ToUtf8(todo)")
        connwrap.execSql("update search_views set search=latin1ToUtf8(search)")
        connwrap.execSql("update wikirelations set word=latin1ToUtf8(word), "+
                "relation=latin1ToUtf8(relation)")
                
        
        if hasVersioningData(connwrap):
            connwrap.execSql("update changelog set word=latin1ToUtf8(word)")
            connwrap.execSql("update headversion set word=latin1ToUtf8(word)")
            connwrap.execSql("update versions set description=latin1ToUtf8(description)")


        # Create the settings table:
        changeTableSchema(connwrap, "settings", 
                TABLE_DEFINITIONS["settings"])

        connwrap.executemany("insert or replace into settings(key, value) "+
                "values (?, ?)", (
            ("formatver", "0"),  # Version of database format the data was written in
            ("writecompatver", "0"),  # Lowest format version which is write compatible
            ("readcompatver", "0"),  # Lowest format version which is read compatible
            ("branchtag", "WikidPadCompact")  # Tag of the WikidPad branch
            )   )
                
                
        # --- WikiPadCompact 1.3pre reached (formatver=0, writecompatver=0,
        #         readcompatver=0) ---
                
                
    formatver = getSettingsInt(connwrap, "formatver")
    
    if formatver == 0:
        # Update wikiwordcontent (add "created" column):
        changeTableSchema(connwrap, "wikiwordcontent", 
                TABLE_DEFINITIONS["wikiwordcontent"])
                
        formatver = 1
        
    # --- WikiPadCompact 1.3 reached (formatver=1, writecompatver=1,
    #         readcompatver=0) ---


    if formatver == 1:
        # Repair bugs in versioning data (missing "created" column, wrong content type)
        if hasVersioningData(connwrap):
            # Update headversion (add "created" column):
            changeTableSchema(connwrap, "headversion", 
                    TABLE_DEFINITIONS["headversion"])
                    
            # Convert content values to blobs                    
            # Will be incorporated in transition 2 -> 3:
#             connwrap.execSql("update headversion set content=textToBlob(content)")
#             connwrap.execSql("update changelog set content=textToBlob(content)")
  
        formatver = 2

    # --- WikiPadCompact 1.3.1 reached (formatver=2, writecompatver=1,
    #         readcompatver=0) ---

    if formatver == 2:
        # Switch everything to utf-8, repair faked utf-8
        
        connwrap.execSql("update wikiwordcontent set "+
                "word=mbcsToUtf8(utf8ToLatin1(word)), "+
                "content=textToBlob(mbcsToUtf8(content))")
        connwrap.execSql("update wikiwordprops set "+
                "word=mbcsToUtf8(utf8ToLatin1(word)), "+
                "key=mbcsToUtf8(utf8ToLatin1(key)), "+
                "value=mbcsToUtf8(utf8ToLatin1(value))")
        connwrap.execSql("update todos set "+
                "word=mbcsToUtf8(utf8ToLatin1(word)), "+
                "todo=mbcsToUtf8(utf8ToLatin1(todo))")
        connwrap.execSql("update search_views set "+
                "search=mbcsToUtf8(utf8ToLatin1(search))")
        connwrap.execSql("update wikirelations set "+
                "word=mbcsToUtf8(utf8ToLatin1(word)), "+
                "relation=mbcsToUtf8(utf8ToLatin1(relation))")


#         if hasVersioningData(connwrap):   #TODO !!!
#             connwrap.execSql("update headversion set "+
#                     "word=mbcsToUtf8(utf8ToLatin1(word)), "+
#                     "content=textToBlob(mbcsToUtf8(content))")
#             connwrap.execSql("update versions set "+
#                     "description=mbcsToUtf8(utf8ToLatin1(description))")
#                     
#             # Updating changelog
#             # 1. word column for all rows
#             connwrap.execSql("update changelog set "+
#                     "word=mbcsToUtf8(utf8ToLatin1(word))")
# 
#             # 2. Update modify operation
#             #    Create temporary copy of headversion table
#             sqlcreate = "create temp table tempverupd ("
#             sqlcreate += ", ".join(map(lambda sc: "%s %s" % sc,
#                     TABLE_DEFINITIONS["headversion"]))
#             sqlcreate += ")"
#             connwrap.execSql(sqlcreate)
#             
#             connwrap.execSql("insert into tempverupd select * from headversion")
#             
#             
#             connwrap.execSqlQuery
#             changes = connWrap.execSqlQuery("select word, op, content, moddate "+
#                     "from changelog order by id desc")
# 
#             for word, op, content, moddate in changes:
#                 if op == 0 or op == 2:
#                     connwrap.execSql("insert or replace into tempverupd"+
#                             "(content) values (?)", (content,))
#                 elif op == 1:
#                     fromContent = connwrap.execSqlSingleItem("select content "+
#                             "from tempverupd where word=?", (word,))
#                     oldDiff = content
#                     toContent = applyBinCompact(fromContent, diff)
#                     newDiff = getBinCompactForDiff(fromContent, toContent)
#                     self.setContentRaw(word, applyBinCompact(self.getContent(word), content), moddate)
#                 elif op == 3:
#                     connwrap.execSql("delete from tempverupd where word=?",
#                             (word,)
# 
#             # 3. changelog content column for all but modify operation (op code 1)
#             connwrap.execSql("update changelog set "+
#                     "content=textToBlob(mbcsToUtf8(word)) where op != 1")
#                 
#             # !!!!!!!!!!!


        formatver = 3

    # --- WikiPadCompact 1.3.2uni reached (formatver=3, writecompatver=3,
    #         readcompatver=3) ---

    if formatver == 3:
        # Update search_views
        searches = connwrap.execSqlQuerySingleColumn(
                "select search from search_views")
        
        changeTableSchema(connwrap, "search_views", 
                TABLE_DEFINITIONS["search_views"])
        
        for search in searches:
            searchOp = SearchReplaceOperation()
            searchOp.searchStr = search
            searchOp.wikiWide = True
            searchOp.booleanOp = True

            try:
                # Raises exception if search is invalid
                searchOp.rebuildSearchOpTree()
            except:
                continue

            datablock = searchOp.getPackedSettings()

            connwrap.execSql(
                "insert or replace into search_views(title, datablock) "+\
                "values (?, ?)", (searchOp.getTitle(), sqlite.Binary(datablock)))

        formatver = 4

    # --- WikiPadCompact 1.5u reached (formatver=4, writecompatver=4,
    #         readcompatver=4) ---
    
    if formatver == 4:
        # Remove brackets from all wikiword references in database
        connwrap.execSql("update or ignore wikiwordcontent set "
                "word=nakedWord(word)")
        connwrap.execSql("update or ignore wikiwordprops set "
                "word=nakedWord(word)")
        connwrap.execSql("update or ignore todos set "
                "word=nakedWord(word)")
        connwrap.execSql("update or ignore wikirelations set "
                "word=nakedWord(word), "
                "relation=nakedWord(relation)")
                
        formatver = 5
    # --- WikiPad(Compact) 1.6beta2 reached (formatver=5, writecompatver=5,
    #         readcompatver=5) ---

    if formatver == 5:
        # Add column "firstcharpos" to some tables

        changeTableSchema(connwrap, "wikirelations",
                TABLE_DEFINITIONS["wikirelations"])
        changeTableSchema(connwrap, "wikiwordprops", 
                TABLE_DEFINITIONS["wikiwordattrs"])
        changeTableSchema(connwrap, "todos", 
                TABLE_DEFINITIONS["todos_PRE2_1alpha01"])
        
        formatver = 6

    # --- WikiPad 1.7beta1 reached (formatver=6, writecompatver=5,
    #         readcompatver=5) ---

    if formatver == 6:
        # Add columns "presentationdatablock" and "wordnormcase" to wikiwordcontent

        changeTableSchema(connwrap, "wikiwordcontent",
                TABLE_DEFINITIONS["wikiwordcontent"])

        formatver = 7

    # --- WikiPad 1.8beta1 reached (formatver=7, writecompatver=7,
    #         readcompatver=7) ---

    if formatver == 7:
        # Update "wikiwordcontent" schema and create new tables
        for tn in ("wikiwordcontent", "wikiwordmatchterms", "datablocks"):
            changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn])

        # Transfer "search_views" data to "datablocks" table
        searches = connwrap.execSqlQuery(
                "select title, datablock from search_views")

        for title, data in searches:
            connwrap.execSql(
                "insert into datablocks(unifiedname, data) "+\
                "values (?, ?)", ("savedsearch/" + title, sqlite.Binary(data)))

        connwrap.execSql("drop table search_views")

        funcWords = connwrap.execSqlQuery(
                "select word, content from wikiwordcontent where word glob '[[]*'")

        # Move functional pages to new table "datablocks" and rename them
        for funcWord, content in funcWords:
            if funcWord not in ("[TextBlocks]", "[PWL]", "[CCBlacklist]"):
                continue # Error ?!

            unifName = "wiki/" + funcWord[1:-1]
            connwrap.execSql(
                "insert into datablocks(unifiedname, data) values (?, ?)",
                (unifName.encode("utf-8"), sqlite.Binary(content)))
            connwrap.execSql("delete from wikiwordcontent where word = ?",
                    (funcWord.encode("utf-8"),))


        formatver = 8
        
    # --- WikiPad 2.0alpha1 reached (formatver=8, writecompatver=8,
    #         readcompatver=8) ---

    if formatver == 8:
        # Recreate table "todos"
        connwrap.execSql("drop table todos;")
        changeTableSchema(connwrap, "todos", TABLE_DEFINITIONS["todos"])
        connwrap.execSql("update todos set firstcharpos=-1;")

        # Rename table "wikiwordprops" to "wikiwordattrs"
        changeTableSchema(connwrap, "wikiwordattrs", TABLE_DEFINITIONS["wikiwordattrs"])
        connwrap.execSql("insert into wikiwordattrs(word, key, value, "
                "firstcharpos, charlength) select word, key, value, "
                "firstcharpos, -1 from wikiwordprops;")
        connwrap.execSql("drop table wikiwordprops;")

        for tn in ("wikirelations", "wikiwordmatchterms"):
            changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn])
            connwrap.execSql("update %s set firstcharpos=-1;" % tn)

        # Mark all wikiwords to need a rebuild
        connwrap.execSql("update wikiwordcontent set metadataprocessed=0;")

        formatver = 9

    # --- WikiPad 2.1alpha.1 reached (formatver=9, writecompatver=9,
    #         readcompatver=9) ---


    # Write format information

    connwrap.executemany("insert or replace into settings(key, value) "+
            "values (?, ?)", (
        ("formatver", str(VERSION_DB)),  # Version of database format the data was written
        ("writecompatver", str(VERSION_WRITECOMPAT)),  # Lowest format version which is write compatible
        ("readcompatver", str(VERSION_READCOMPAT)),  # Lowest format version which is read compatible
        ("branchtag", "WikidPadCompact")  # Tag of the WikidPad branch
        )   )

    rebuildIndices(connwrap)
    
    connwrap.syncCommit()
Beispiel #4
0
def updateDatabase(connwrap, dataDir, pagefileSuffix):
    """
    Update a database from an older version to current (checkDatabaseFormat()
    should have returned 1 before calling this function)
    """
    connwrap.commit()

    indices = connwrap.execSqlQuerySingleColumn(
        "select INDEX_NAME from __indices__")
    tables = connwrap.execSqlQuerySingleColumn(
        "select TABLE_NAME from __table_names__")

    indices = map(string.upper, indices)
    tables = map(string.upper, tables)

    # updatedTables = []

    if not "SETTINGS" in tables:
        # We are prior WikidPad 1.2beta2 (which writes format version 0)

        if "WIKIWORDPROPS_PKEY" in indices:
            #             print "dropping index wikiwordprops_pkey"
            connwrap.execSql("drop index wikiwordprops_pkey")
#         if "WIKIWORDPROPS_WORD" not in indices:
#             print "creating index wikiwordprops_word"
#             connwrap.execSql("create index wikiwordprops_word on wikiwordprops(word)")
#         if "WIKIRELATIONS_WORD" not in indices:
#             print "creating index wikirelations_word"
#             connwrap.execSql("create index wikirelations_word on wikirelations(word)")
        if "REGISTRATION" in tables:
            connwrap.execSql("drop table registration")

        # Update search_views
        searches = connwrap.execSqlQuerySingleColumn(
            "select search from search_views")

        changeTableSchema(connwrap, "search_views",
                          TABLE_DEFINITIONS["search_views"])

        for search in searches:
            searchOp = SearchReplaceOperation()
            searchOp.searchStr = search
            searchOp.wikiWide = True
            searchOp.booleanOp = True

            try:
                # Raises exception if search is invalid
                searchOp.rebuildSearchOpTree()
            except:
                continue

            datablock = searchOp.getPackedSettings()

            connwrap.execSql(
                "insert into search_views(title, datablock) "+\
                "values (?, ?)", (searchOp.getTitle(), datablock))

        formatver = 0

        changeTableSchema(connwrap, "settings", TABLE_DEFINITIONS["settings"])

        # Write initial format versions
        for key, value in (
            ("formatver",
             "0"),  # Version of database format the data was written
            ("writecompatver",
             "0"),  # Lowest format version which is write compatible
            ("readcompatver",
             "0"),  # Lowest format version which is read compatible
            ("branchtag", "WikidPad")  # Tag of the WikidPad branch
        ):
            setSettingsValue(connwrap, key, value)

        # --- WikiPad 1.20beta2 reached (formatver=0, writecompatver=0,
        #         readcompatver=0) ---

    formatver = getSettingsInt(connwrap, "formatver")

    if formatver == 0:
        # From formatver 0 to 1, all filenames with brackets are renamed
        # to have no brackets
        filenames = glob.glob(join(mbcsEnc(dataDir, "replace")[0], '*.wiki'))
        for fn in filenames:
            fn = mbcsDec(fn, "replace")[0]
            bn = basename(fn)
            newbname = removeBracketsFilename(bn)
            if bn == newbname:
                continue

            newname = mbcsEnc(join(dataDir, newbname), "replace")[0]
            if exists(pathEnc(newname)):
                # A file with the designated new name of fn already exists
                # -> do nothing
                continue

            try:
                rename(fn, newname)
            except (IOError, OSError):
                pass

        formatver = 1

        # --- WikiPad 1.20beta3 reached (formatver=1, writecompatver=1,
        #         readcompatver=1) ---

    if formatver == 1:
        # remove brackets from all wikiwords in database

        # table wikiwords
        dataIn = connwrap.execSqlQuery(
            "select word, created, modified from wikiwords")
        connwrap.execSql("drop table wikiwords")
        connwrap.commit()
        changeTableSchema(connwrap, "wikiwords",
                          TABLE_DEFINITIONS["wikiwords"])
        rebuildIndices(connwrap)

        uniqueCtl = {}
        for w, c, m in dataIn:
            w = oldWikiWordToLabel(w)
            if not uniqueCtl.has_key(w):
                connwrap.execSqlInsert(
                    "wikiwords",
                    ("word", "created", "modified", "presentationdatablock"),
                    (w, c, m, ""))
                #                 connwrap.execSql("insert into wikiwords(word, created, modified) "
                #                         "values (?, ?, ?)", (w, c, m))
                uniqueCtl[w] = None

        # table wikirelations
        dataIn = connwrap.execSqlQuery(
            "select word, relation, created from wikirelations")
        connwrap.execSql("drop table wikirelations")
        connwrap.commit()
        changeTableSchema(connwrap, "wikirelations",
                          TABLE_DEFINITIONS["wikirelations"])
        rebuildIndices(connwrap)

        uniqueCtl = {}
        for w, r, c in dataIn:
            w, r = oldWikiWordToLabel(w), oldWikiWordToLabel(r)
            if not uniqueCtl.has_key((w, r)):
                connwrap.execSqlInsert("wikirelations",
                                       ("word", "relation", "created"),
                                       (w, r, c))
                #                 connwrap.execSql("insert into wikirelations(word, relation, created) "
                #                         "values (?, ?, ?)", (w, r, c))
                uniqueCtl[(w, r)] = None

        # table wikiwordprops
        dataIn = connwrap.execSqlQuery(
            "select word, key, value from wikiwordprops")
        connwrap.execSql("drop table wikiwordprops")
        connwrap.commit()
        changeTableSchema(connwrap, "wikiwordprops",
                          TABLE_DEFINITIONS["wikiwordprops_PRE2_1alpha01"])
        rebuildIndices(connwrap)

        for w, k, v in dataIn:
            connwrap.execSqlInsert("wikiwordprops", ("word", "key", "value"),
                                   (oldWikiWordToLabel(w), k, v),
                                   tableDefault="wikiwordprops_PRE2_1alpha01")
#             connwrap.execSql("insert into wikiwordprops(word, key, value) "
#                     "values (?, ?, ?)", (oldWikiWordToLabel(w), k, v))

# table todos
        dataIn = connwrap.execSqlQuery("select word, todo from todos")
        connwrap.execSql("drop table todos")
        connwrap.commit()
        changeTableSchema(connwrap, "todos",
                          TABLE_DEFINITIONS["todos_PRE2_1alpha01"])
        rebuildIndices(connwrap)

        for w, t in dataIn:
            connwrap.execSqlInsert("todos", ("word", "todo"),
                                   (oldWikiWordToLabel(w), t),
                                   tableDefault="todos_PRE2_1alpha01")


#             connwrap.execSql("insert into todos(word, todo) "
#                     "values (?, ?)", (oldWikiWordToLabel(w), t))

        formatver = 2

        # --- WikiPad 1.6beta2 reached (formatver=2, writecompatver=2,
        #         readcompatver=2) ---

    if formatver == 2:
        changeTableSchema(connwrap, "wikiwords",
                          TABLE_DEFINITIONS["wikiwords"])

        # --- WikiPad 1.8beta1 reached (formatver=3, writecompatver=3,
        #         readcompatver=2) ---

        formatver = 3

    if formatver == 3:

        # Update "wikiwords" schema and create new tables
        for tn in ("wikiwords", "wikiwordmatchterms", "datablocks",
                   "datablocksexternal", "defaultvalues"):
            changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn])

        # (Re)fill "defaultvalues" and read them into connection wrapper
        connwrap.fillDefaultValues()
        connwrap.readDefaultValues()

        # Transfer "search_views" data to "datablocks" table
        searches = connwrap.execSqlQuery(
            "select title, datablock from search_views", strConv=(True, False))

        for title, data in searches:
            connwrap.execSql(
                "insert into datablocks(unifiedname, data) "+\
                "values (?, ?)", (u"savedsearch/" + title, data))

        connwrap.execSql("drop table search_views")

        allWords = connwrap.execSqlQuerySingleColumn(
            "select word from wikiwords")

        # Divide into functional and wiki pages
        wikiWords = []
        funcWords = []
        for w in allWords:
            if w.startswith('['):
                funcWords.append(w)
            else:
                wikiWords.append(w)

        # Fill the new fields in table "wikiwords"
        for wikiWord in wikiWords:
            filename = wikiWord + pagefileSuffix
            fullPath = join(dataDir, filename)
            try:
                # We don't use coarsening here for the FSB because a different
                # coarsening setting can't exist for the old wiki format
                filesig = getFileSignatureBlock(fullPath)
            except (IOError, WindowsError):
                traceback.print_exc()
                continue

            connwrap.execSql(
                "update wikiwords set filepath = ?, "
                "filenamelowercase = ?, filesignature = ? "
                "where word = ?",
                (filename, filename.lower(), filesig, wikiWord))

        # Move functional pages to new table "datablocksexternal" and rename them
        for funcWord in funcWords:
            if funcWord not in (u"[TextBlocks]", u"[PWL]", u"[CCBlacklist]"):
                continue  # Error ?!

            unifName = u"wiki/" + funcWord[1:-1]
            fullPath = join(dataDir, funcWord + pagefileSuffix)

            icf = iterCompatibleFilename(unifName, u".data")

            for i in range(
                    10):  # Actual "while True", but that's too dangerous
                newFilename = icf.next()
                newPath = join(dataDir, newFilename)

                if exists(pathEnc(newPath)):
                    # A file with the designated new name of fn already exists
                    # -> do nothing
                    continue

                try:
                    rename(pathEnc(fullPath), pathEnc(newPath))

                    # We don't use coarsening here for the FSB because a different
                    # coarsening setting can't exist for the old wiki format
                    connwrap.execSqlInsert(
                        "datablocksexternal",
                        ("unifiedname", "filepath", "filenamelowercase",
                         "filesignature"),
                        (unifName, newFilename, newFilename.lower(),
                         getFileSignatureBlock(newPath)))
                    connwrap.execSql("delete from wikiwords where word = ?",
                                     (funcWord, ))
                    break
                except (IOError, OSError):
                    traceback.print_exc()
                    continue

        # --- WikiPad 2.0alpha1 reached (formatver=4, writecompatver=4,
        #         readcompatver=4) ---

        formatver = 4

    if formatver == 4:
        # (Re)fill "defaultvalues" and read them into connection wrapper
        connwrap.fillDefaultValues()
        connwrap.readDefaultValues()

        # Recreate table "todos" with new schema
        connwrap.execSql("drop table todos")
        changeTableSchema(connwrap, "todos", TABLE_DEFINITIONS["todos"])

        # Rename table "wikiwordprops" to "wikiwordattrs"
        changeTableSchema(connwrap, "wikiwordattrs",
                          TABLE_DEFINITIONS["wikiwordattrs"])
        connwrap.execSql("insert into wikiwordattrs(word, key, value) "
                         "select word, key, value from wikiwordprops")
        connwrap.execSql("drop table wikiwordprops")

        for tn in ("wikirelations", "wikiwordmatchterms"):
            changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn])

        # Mark all wikiwords to need a rebuild
        connwrap.execSql("update wikiwords set metadataprocessed=0")

        formatver = 5

    # --- WikiPad 2.1alpha.1 reached (formatver=5, writecompatver=5,
    #         readcompatver=5) ---

    # Write format information
    for key, value in (
        ("formatver",
         str(VERSION_DB)),  # Version of database format the data was written
        ("writecompatver", str(VERSION_WRITECOMPAT)
         ),  # Lowest format version which is write compatible
        ("readcompatver", str(VERSION_READCOMPAT)
         ),  # Lowest format version which is read compatible
        ("branchtag", "WikidPad"),  # Tag of the WikidPad branch
        ("locale", "-"
         )  # Locale for cached wordnormcase column. '-': column invalid
    ):
        setSettingsValue(connwrap, key, value)

    rebuildIndices(connwrap)

    connwrap.commit()
Beispiel #5
0
def updateDatabase(connwrap, dataDir, pagefileSuffix):
    """
    Update a database from an older version to current (checkDatabaseFormat()
    should have returned 1 before calling this function)
    """
    connwrap.commit()

    indices = connwrap.execSqlQuerySingleColumn("select INDEX_NAME from __indices__")
    tables = connwrap.execSqlQuerySingleColumn("select TABLE_NAME from __table_names__")

    indices = map(string.upper, indices)
    tables = map(string.upper, tables)
    
    # updatedTables = []
    
    if not "SETTINGS" in tables:
        # We are prior WikidPad 1.2beta2 (which writes format version 0)
        
        if "WIKIWORDPROPS_PKEY" in indices:
#             print "dropping index wikiwordprops_pkey"
            connwrap.execSql("drop index wikiwordprops_pkey")
#         if "WIKIWORDPROPS_WORD" not in indices:
#             print "creating index wikiwordprops_word"
#             connwrap.execSql("create index wikiwordprops_word on wikiwordprops(word)")
#         if "WIKIRELATIONS_WORD" not in indices:
#             print "creating index wikirelations_word"
#             connwrap.execSql("create index wikirelations_word on wikirelations(word)")
        if "REGISTRATION" in tables:
            connwrap.execSql("drop table registration")

        # Update search_views
        searches = connwrap.execSqlQuerySingleColumn(
                "select search from search_views")
                
        changeTableSchema(connwrap, "search_views", 
                TABLE_DEFINITIONS["search_views"])
        
        for search in searches:
            searchOp = SearchReplaceOperation()
            searchOp.searchStr = search
            searchOp.wikiWide = True
            searchOp.booleanOp = True

            try:
                # Raises exception if search is invalid
                searchOp.rebuildSearchOpTree()
            except:
                continue

            datablock = searchOp.getPackedSettings()

            connwrap.execSql(
                "insert into search_views(title, datablock) "+\
                "values (?, ?)", (searchOp.getTitle(), datablock))

        formatver = 0
        
        changeTableSchema(connwrap, "settings", 
                TABLE_DEFINITIONS["settings"])
        
        # Write initial format versions
        for key, value in (
                ("formatver", "0"),  # Version of database format the data was written
                ("writecompatver", "0"),  # Lowest format version which is write compatible
                ("readcompatver", "0"),  # Lowest format version which is read compatible
                ("branchtag", "WikidPad")  # Tag of the WikidPad branch
                ):
            setSettingsValue(connwrap, key, value)


        # --- WikiPad 1.20beta2 reached (formatver=0, writecompatver=0,
        #         readcompatver=0) ---


    formatver = getSettingsInt(connwrap, "formatver")
    
    if formatver == 0:
        # From formatver 0 to 1, all filenames with brackets are renamed
        # to have no brackets
        filenames = glob.glob(join(mbcsEnc(dataDir, "replace")[0], '*.wiki'))
        for fn in filenames:
            fn = mbcsDec(fn, "replace")[0]
            bn = basename(fn)
            newbname = removeBracketsFilename(bn)
            if bn == newbname:
                continue
                    
            newname = mbcsEnc(join(dataDir, newbname), "replace")[0]
            if exists(pathEnc(newname)):
                # A file with the designated new name of fn already exists
                # -> do nothing
                continue
            
            try:
                rename(fn, newname)
            except (IOError, OSError):
                pass
        
        formatver = 1
        
        # --- WikiPad 1.20beta3 reached (formatver=1, writecompatver=1,
        #         readcompatver=1) ---

    if formatver == 1:
        # remove brackets from all wikiwords in database
        
        # table wikiwords
        dataIn = connwrap.execSqlQuery(
                "select word, created, modified from wikiwords")
        connwrap.execSql("drop table wikiwords")
        connwrap.commit()
        changeTableSchema(connwrap, "wikiwords", 
                TABLE_DEFINITIONS["wikiwords"])
        rebuildIndices(connwrap)
        
        uniqueCtl = {}
        for w, c, m in dataIn:
            w = oldWikiWordToLabel(w)
            if not uniqueCtl.has_key(w):
                connwrap.execSqlInsert("wikiwords", ("word", "created", 
                        "modified", "presentationdatablock"),
                        (w, c, m, ""))
#                 connwrap.execSql("insert into wikiwords(word, created, modified) "
#                         "values (?, ?, ?)", (w, c, m))
                uniqueCtl[w] = None

        # table wikirelations
        dataIn = connwrap.execSqlQuery(
                "select word, relation, created from wikirelations")
        connwrap.execSql("drop table wikirelations")
        connwrap.commit()
        changeTableSchema(connwrap, "wikirelations", 
                TABLE_DEFINITIONS["wikirelations"])
        rebuildIndices(connwrap)

        uniqueCtl = {}
        for w, r, c in dataIn:
            w, r = oldWikiWordToLabel(w), oldWikiWordToLabel(r)
            if not uniqueCtl.has_key((w, r)):
                connwrap.execSqlInsert("wikirelations", ("word", "relation", 
                        "created"), (w, r, c))
#                 connwrap.execSql("insert into wikirelations(word, relation, created) "
#                         "values (?, ?, ?)", (w, r, c))
                uniqueCtl[(w, r)] = None

        # table wikiwordprops
        dataIn = connwrap.execSqlQuery(
                "select word, key, value from wikiwordprops")
        connwrap.execSql("drop table wikiwordprops")
        connwrap.commit()
        changeTableSchema(connwrap, "wikiwordprops", 
                TABLE_DEFINITIONS["wikiwordprops_PRE2_1alpha01"])
        rebuildIndices(connwrap)

        for w, k, v in dataIn:
            connwrap.execSqlInsert("wikiwordprops", ("word", "key", 
                    "value"), (oldWikiWordToLabel(w), k, v),
                    tableDefault="wikiwordprops_PRE2_1alpha01")
#             connwrap.execSql("insert into wikiwordprops(word, key, value) "
#                     "values (?, ?, ?)", (oldWikiWordToLabel(w), k, v))

        # table todos
        dataIn = connwrap.execSqlQuery(
                "select word, todo from todos")
        connwrap.execSql("drop table todos")
        connwrap.commit()
        changeTableSchema(connwrap, "todos", 
                TABLE_DEFINITIONS["todos_PRE2_1alpha01"])
        rebuildIndices(connwrap)

        for w, t in dataIn:
            connwrap.execSqlInsert("todos", ("word", "todo"),
                    (oldWikiWordToLabel(w), t),
                    tableDefault="todos_PRE2_1alpha01")
#             connwrap.execSql("insert into todos(word, todo) "
#                     "values (?, ?)", (oldWikiWordToLabel(w), t))

        formatver = 2

        # --- WikiPad 1.6beta2 reached (formatver=2, writecompatver=2,
        #         readcompatver=2) ---

    if formatver == 2:
        changeTableSchema(connwrap, "wikiwords", 
                TABLE_DEFINITIONS["wikiwords"])
                
        # --- WikiPad 1.8beta1 reached (formatver=3, writecompatver=3,
        #         readcompatver=2) ---

        formatver = 3
        

    if formatver == 3:

        # Update "wikiwords" schema and create new tables
        for tn in ("wikiwords", "wikiwordmatchterms", "datablocks",
                "datablocksexternal", "defaultvalues"):
            changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn])
        
        # (Re)fill "defaultvalues" and read them into connection wrapper
        connwrap.fillDefaultValues()
        connwrap.readDefaultValues()


        # Transfer "search_views" data to "datablocks" table
        searches = connwrap.execSqlQuery(
                "select title, datablock from search_views",
                strConv=(True, False))

        for title, data in searches:
            connwrap.execSql(
                "insert into datablocks(unifiedname, data) "+\
                "values (?, ?)", (u"savedsearch/" + title, data))

        connwrap.execSql("drop table search_views")

        allWords = connwrap.execSqlQuerySingleColumn("select word from wikiwords")
        
        # Divide into functional and wiki pages
        wikiWords = []
        funcWords = []
        for w in allWords:
            if w.startswith('['):
                funcWords.append(w)
            else:
                wikiWords.append(w)

        # Fill the new fields in table "wikiwords"
        for wikiWord in wikiWords:
            filename = wikiWord + pagefileSuffix
            fullPath = join(dataDir, filename)
            try:
                # We don't use coarsening here for the FSB because a different
                # coarsening setting can't exist for the old wiki format
                filesig = getFileSignatureBlock(fullPath)
            except (IOError, WindowsError):
                traceback.print_exc()
                continue

            connwrap.execSql("update wikiwords set filepath = ?, "
                    "filenamelowercase = ?, filesignature = ? "
                    "where word = ?", (filename, filename.lower(), filesig,
                    wikiWord))

        # Move functional pages to new table "datablocksexternal" and rename them
        for funcWord in funcWords:
            if funcWord not in (u"[TextBlocks]", u"[PWL]", u"[CCBlacklist]"):
                continue # Error ?!
            
            unifName = u"wiki/" + funcWord[1:-1]
            fullPath = join(dataDir, funcWord + pagefileSuffix)
            
            icf = iterCompatibleFilename(unifName, u".data")
            
            for i in range(10):  # Actual "while True", but that's too dangerous
                newFilename = icf.next()
                newPath = join(dataDir, newFilename)

                if exists(pathEnc(newPath)):
                    # A file with the designated new name of fn already exists
                    # -> do nothing
                    continue

                try:
                    rename(pathEnc(fullPath), pathEnc(newPath))

                    # We don't use coarsening here for the FSB because a different
                    # coarsening setting can't exist for the old wiki format
                    connwrap.execSqlInsert("datablocksexternal", ("unifiedname",
                            "filepath", "filenamelowercase", "filesignature"),
                            (unifName, newFilename, newFilename.lower(),
                            getFileSignatureBlock(newPath)))
                    connwrap.execSql("delete from wikiwords where word = ?",
                            (funcWord,))
                    break
                except (IOError, OSError):
                    traceback.print_exc()
                    continue


        # --- WikiPad 2.0alpha1 reached (formatver=4, writecompatver=4,
        #         readcompatver=4) ---

        formatver = 4
        
    if formatver == 4:
        # (Re)fill "defaultvalues" and read them into connection wrapper
        connwrap.fillDefaultValues()
        connwrap.readDefaultValues()

        # Recreate table "todos" with new schema
        connwrap.execSql("drop table todos")
        changeTableSchema(connwrap, "todos", TABLE_DEFINITIONS["todos"])

        # Rename table "wikiwordprops" to "wikiwordattrs"
        changeTableSchema(connwrap, "wikiwordattrs", TABLE_DEFINITIONS["wikiwordattrs"])
        connwrap.execSql("insert into wikiwordattrs(word, key, value) "
                "select word, key, value from wikiwordprops")
        connwrap.execSql("drop table wikiwordprops")

        for tn in ("wikirelations", "wikiwordmatchterms"):
            changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn])

        # Mark all wikiwords to need a rebuild
        connwrap.execSql("update wikiwords set metadataprocessed=0")

        formatver = 5

    # --- WikiPad 2.1alpha.1 reached (formatver=5, writecompatver=5,
    #         readcompatver=5) ---



    # Write format information
    for key, value in (
            ("formatver", str(VERSION_DB)),  # Version of database format the data was written
            ("writecompatver", str(VERSION_WRITECOMPAT)),  # Lowest format version which is write compatible
            ("readcompatver", str(VERSION_READCOMPAT)),  # Lowest format version which is read compatible
            ("branchtag", "WikidPad"),  # Tag of the WikidPad branch
            ("locale", "-") # Locale for cached wordnormcase column. '-': column invalid
            ):
        setSettingsValue(connwrap, key, value)

    rebuildIndices(connwrap)
    
    connwrap.commit()