def doWikiWideSearch(wikiDocument, regexpString):
sarOp = SearchReplaceOperation()
sarOp.wikiWide = True
sarOp.wildCard = 'regex'
sarOp.caseSensitive = False
sarOp.searchStr = regexpString
return wikiDocument.searchWiki(sarOp)
def doWikiWideSearch(wikiDocument, regexpString):
sarOp = SearchReplaceOperation()
sarOp.wikiWide = True
sarOp.wildCard = 'regex'
sarOp.caseSensitive = False
sarOp.searchStr = regexpString
return wikiDocument.searchWiki(sarOp)
def updateDatabase(connwrap, dataDir):
"""
Update a database from an older version to current (checkDatabaseFormat()
should have returned 1 before calling this function)
"""
connwrap.syncCommit()
# Always remember that there is no automatic unicode<->utf-8 conversion
# during this function!
indices = connwrap.execSqlQuerySingleColumn(
"select name from sqlite_master where type='index'")
tables = connwrap.execSqlQuerySingleColumn(
"select name from sqlite_master where type='table'")
indices = [s.upper() for s in indices]
tables = [s.upper() for s in tables]
# updatedTables = []
if not "SETTINGS" in tables:
# We are prior WikidPadCompact 1.3pre (which writes format version 0)
# From WikidPad original
if "WIKIWORDPROPS_PKEY" in indices:
connwrap.execSql("drop index wikiwordprops_pkey")
if "REGISTRATION" in tables:
connwrap.execSql("drop table registration")
# Here we have reached WikidPadCompact 1.0/1.1 format
# For database format changes
rt = "real not null default 0.0"
it = "integer not null default 0"
tt = "text not null default ''"
bt = "blob not null default x''"
# From WikiPadCompact 1.1 to 1.2:
# if "WIKIWORDS" in tables:
connwrap.execSqlNoError("drop table wikiwords")
# Remove column "created" from "wikirelations"
changed = changeTableSchema(connwrap, "wikirelations",
TABLE_DEFINITIONS["wikirelations"])
# --- WikiPadCompact 1.2 reached ---
# From WikiPadCompact 1.2 to 1.3:
# Update all text items from standard encoding to UTF8
# TODO: "Please wait"
## wikidata.dbConn.createFunction("stdToUtf8", 1, sqlite_stdToUtf8)
connwrap.execSql("update wikiwordcontent set word=latin1ToUtf8(word), "+
"content=textToBlob(content)")
connwrap.execSql("update wikiwordprops set word=latin1ToUtf8(word), "+
"key=latin1ToUtf8(key), value=latin1ToUtf8(value)")
connwrap.execSql("update todos set word=latin1ToUtf8(word), "+
"todo=latin1ToUtf8(todo)")
connwrap.execSql("update search_views set search=latin1ToUtf8(search)")
connwrap.execSql("update wikirelations set word=latin1ToUtf8(word), "+
"relation=latin1ToUtf8(relation)")
if hasVersioningData(connwrap):
connwrap.execSql("update changelog set word=latin1ToUtf8(word)")
connwrap.execSql("update headversion set word=latin1ToUtf8(word)")
connwrap.execSql("update versions set description=latin1ToUtf8(description)")
# Create the settings table:
changeTableSchema(connwrap, "settings",
TABLE_DEFINITIONS["settings"])
connwrap.executemany("insert or replace into settings(key, value) "+
"values (?, ?)", (
("formatver", "0"), # Version of database format the data was written in
("writecompatver", "0"), # Lowest format version which is write compatible
("readcompatver", "0"), # Lowest format version which is read compatible
("branchtag", "WikidPadCompact") # Tag of the WikidPad branch
) )
# --- WikiPadCompact 1.3pre reached (formatver=0, writecompatver=0,
# readcompatver=0) ---
formatver = getSettingsInt(connwrap, "formatver")
if formatver == 0:
# Update wikiwordcontent (add "created" column):
changeTableSchema(connwrap, "wikiwordcontent",
TABLE_DEFINITIONS["wikiwordcontent"])
formatver = 1
# --- WikiPadCompact 1.3 reached (formatver=1, writecompatver=1,
# readcompatver=0) ---
if formatver == 1:
# Repair bugs in versioning data (missing "created" column, wrong content type)
if hasVersioningData(connwrap):
# Update headversion (add "created" column):
changeTableSchema(connwrap, "headversion",
TABLE_DEFINITIONS["headversion"])
# Convert content values to blobs
# Will be incorporated in transition 2 -> 3:
# connwrap.execSql("update headversion set content=textToBlob(content)")
# connwrap.execSql("update changelog set content=textToBlob(content)")
formatver = 2
# --- WikiPadCompact 1.3.1 reached (formatver=2, writecompatver=1,
# readcompatver=0) ---
if formatver == 2:
# Switch everything to utf-8, repair faked utf-8
connwrap.execSql("update wikiwordcontent set "+
"word=mbcsToUtf8(utf8ToLatin1(word)), "+
"content=textToBlob(mbcsToUtf8(content))")
connwrap.execSql("update wikiwordprops set "+
"word=mbcsToUtf8(utf8ToLatin1(word)), "+
"key=mbcsToUtf8(utf8ToLatin1(key)), "+
"value=mbcsToUtf8(utf8ToLatin1(value))")
connwrap.execSql("update todos set "+
"word=mbcsToUtf8(utf8ToLatin1(word)), "+
"todo=mbcsToUtf8(utf8ToLatin1(todo))")
connwrap.execSql("update search_views set "+
"search=mbcsToUtf8(utf8ToLatin1(search))")
connwrap.execSql("update wikirelations set "+
"word=mbcsToUtf8(utf8ToLatin1(word)), "+
"relation=mbcsToUtf8(utf8ToLatin1(relation))")
# if hasVersioningData(connwrap): #TODO !!!
# connwrap.execSql("update headversion set "+
# "word=mbcsToUtf8(utf8ToLatin1(word)), "+
# "content=textToBlob(mbcsToUtf8(content))")
# connwrap.execSql("update versions set "+
# "description=mbcsToUtf8(utf8ToLatin1(description))")
#
# # Updating changelog
# # 1. word column for all rows
# connwrap.execSql("update changelog set "+
# "word=mbcsToUtf8(utf8ToLatin1(word))")
#
# # 2. Update modify operation
# # Create temporary copy of headversion table
# sqlcreate = "create temp table tempverupd ("
# sqlcreate += ", ".join(map(lambda sc: "%s %s" % sc,
# TABLE_DEFINITIONS["headversion"]))
# sqlcreate += ")"
# connwrap.execSql(sqlcreate)
#
# connwrap.execSql("insert into tempverupd select * from headversion")
#
#
# connwrap.execSqlQuery
# changes = connWrap.execSqlQuery("select word, op, content, moddate "+
# "from changelog order by id desc")
#
# for word, op, content, moddate in changes:
# if op == 0 or op == 2:
# connwrap.execSql("insert or replace into tempverupd"+
# "(content) values (?)", (content,))
# elif op == 1:
# fromContent = connwrap.execSqlSingleItem("select content "+
# "from tempverupd where word=?", (word,))
# oldDiff = content
# toContent = applyBinCompact(fromContent, diff)
# newDiff = getBinCompactForDiff(fromContent, toContent)
# self.setContentRaw(word, applyBinCompact(self.getContent(word), content), moddate)
# elif op == 3:
# connwrap.execSql("delete from tempverupd where word=?",
# (word,)
#
# # 3. changelog content column for all but modify operation (op code 1)
# connwrap.execSql("update changelog set "+
# "content=textToBlob(mbcsToUtf8(word)) where op != 1")
#
# # !!!!!!!!!!!
formatver = 3
# --- WikiPadCompact 1.3.2uni reached (formatver=3, writecompatver=3,
# readcompatver=3) ---
if formatver == 3:
# Update search_views
searches = connwrap.execSqlQuerySingleColumn(
"select search from search_views")
changeTableSchema(connwrap, "search_views",
TABLE_DEFINITIONS["search_views"])
for search in searches:
searchOp = SearchReplaceOperation()
searchOp.searchStr = search
searchOp.wikiWide = True
searchOp.booleanOp = True
try:
# Raises exception if search is invalid
searchOp.rebuildSearchOpTree()
except:
continue
datablock = searchOp.getPackedSettings()
connwrap.execSql(
"insert or replace into search_views(title, datablock) "+\
"values (?, ?)", (searchOp.getTitle(), sqlite.Binary(datablock)))
formatver = 4
# --- WikiPadCompact 1.5u reached (formatver=4, writecompatver=4,
# readcompatver=4) ---
if formatver == 4:
# Remove brackets from all wikiword references in database
connwrap.execSql("update or ignore wikiwordcontent set "
"word=nakedWord(word)")
connwrap.execSql("update or ignore wikiwordprops set "
"word=nakedWord(word)")
connwrap.execSql("update or ignore todos set "
"word=nakedWord(word)")
connwrap.execSql("update or ignore wikirelations set "
"word=nakedWord(word), "
"relation=nakedWord(relation)")
formatver = 5
# --- WikiPad(Compact) 1.6beta2 reached (formatver=5, writecompatver=5,
# readcompatver=5) ---
if formatver == 5:
# Add column "firstcharpos" to some tables
changeTableSchema(connwrap, "wikirelations",
TABLE_DEFINITIONS["wikirelations"])
changeTableSchema(connwrap, "wikiwordprops",
TABLE_DEFINITIONS["wikiwordattrs"])
changeTableSchema(connwrap, "todos",
TABLE_DEFINITIONS["todos_PRE2_1alpha01"])
formatver = 6
# --- WikiPad 1.7beta1 reached (formatver=6, writecompatver=5,
# readcompatver=5) ---
if formatver == 6:
# Add columns "presentationdatablock" and "wordnormcase" to wikiwordcontent
changeTableSchema(connwrap, "wikiwordcontent",
TABLE_DEFINITIONS["wikiwordcontent"])
formatver = 7
# --- WikiPad 1.8beta1 reached (formatver=7, writecompatver=7,
# readcompatver=7) ---
if formatver == 7:
# Update "wikiwordcontent" schema and create new tables
for tn in ("wikiwordcontent", "wikiwordmatchterms", "datablocks"):
changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn])
# Transfer "search_views" data to "datablocks" table
searches = connwrap.execSqlQuery(
"select title, datablock from search_views")
for title, data in searches:
connwrap.execSql(
"insert into datablocks(unifiedname, data) "+\
"values (?, ?)", ("savedsearch/" + title, sqlite.Binary(data)))
connwrap.execSql("drop table search_views")
funcWords = connwrap.execSqlQuery(
"select word, content from wikiwordcontent where word glob '[[]*'")
# Move functional pages to new table "datablocks" and rename them
for funcWord, content in funcWords:
if funcWord not in ("[TextBlocks]", "[PWL]", "[CCBlacklist]"):
continue # Error ?!
unifName = "wiki/" + funcWord[1:-1]
connwrap.execSql(
"insert into datablocks(unifiedname, data) values (?, ?)",
(unifName.encode("utf-8"), sqlite.Binary(content)))
connwrap.execSql("delete from wikiwordcontent where word = ?",
(funcWord.encode("utf-8"),))
formatver = 8
# --- WikiPad 2.0alpha1 reached (formatver=8, writecompatver=8,
# readcompatver=8) ---
if formatver == 8:
# Recreate table "todos"
connwrap.execSql("drop table todos;")
changeTableSchema(connwrap, "todos", TABLE_DEFINITIONS["todos"])
connwrap.execSql("update todos set firstcharpos=-1;")
# Rename table "wikiwordprops" to "wikiwordattrs"
changeTableSchema(connwrap, "wikiwordattrs", TABLE_DEFINITIONS["wikiwordattrs"])
connwrap.execSql("insert into wikiwordattrs(word, key, value, "
"firstcharpos, charlength) select word, key, value, "
"firstcharpos, -1 from wikiwordprops;")
connwrap.execSql("drop table wikiwordprops;")
for tn in ("wikirelations", "wikiwordmatchterms"):
changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn])
connwrap.execSql("update %s set firstcharpos=-1;" % tn)
# Mark all wikiwords to need a rebuild
connwrap.execSql("update wikiwordcontent set metadataprocessed=0;")
formatver = 9
# --- WikiPad 2.1alpha.1 reached (formatver=9, writecompatver=9,
# readcompatver=9) ---
# Write format information
connwrap.executemany("insert or replace into settings(key, value) "+
"values (?, ?)", (
("formatver", str(VERSION_DB)), # Version of database format the data was written
("writecompatver", str(VERSION_WRITECOMPAT)), # Lowest format version which is write compatible
("readcompatver", str(VERSION_READCOMPAT)), # Lowest format version which is read compatible
("branchtag", "WikidPadCompact") # Tag of the WikidPad branch
) )
rebuildIndices(connwrap)
connwrap.syncCommit()
def updateDatabase(connwrap, dataDir, pagefileSuffix):
"""
Update a database from an older version to current (checkDatabaseFormat()
should have returned 1 before calling this function)
"""
connwrap.commit()
indices = connwrap.execSqlQuerySingleColumn(
"select INDEX_NAME from __indices__")
tables = connwrap.execSqlQuerySingleColumn(
"select TABLE_NAME from __table_names__")
indices = map(string.upper, indices)
tables = map(string.upper, tables)
# updatedTables = []
if not "SETTINGS" in tables:
# We are prior WikidPad 1.2beta2 (which writes format version 0)
if "WIKIWORDPROPS_PKEY" in indices:
# print "dropping index wikiwordprops_pkey"
connwrap.execSql("drop index wikiwordprops_pkey")
# if "WIKIWORDPROPS_WORD" not in indices:
# print "creating index wikiwordprops_word"
# connwrap.execSql("create index wikiwordprops_word on wikiwordprops(word)")
# if "WIKIRELATIONS_WORD" not in indices:
# print "creating index wikirelations_word"
# connwrap.execSql("create index wikirelations_word on wikirelations(word)")
if "REGISTRATION" in tables:
connwrap.execSql("drop table registration")
# Update search_views
searches = connwrap.execSqlQuerySingleColumn(
"select search from search_views")
changeTableSchema(connwrap, "search_views",
TABLE_DEFINITIONS["search_views"])
for search in searches:
searchOp = SearchReplaceOperation()
searchOp.searchStr = search
searchOp.wikiWide = True
searchOp.booleanOp = True
try:
# Raises exception if search is invalid
searchOp.rebuildSearchOpTree()
except:
continue
datablock = searchOp.getPackedSettings()
connwrap.execSql(
"insert into search_views(title, datablock) "+\
"values (?, ?)", (searchOp.getTitle(), datablock))
formatver = 0
changeTableSchema(connwrap, "settings", TABLE_DEFINITIONS["settings"])
# Write initial format versions
for key, value in (
("formatver",
"0"), # Version of database format the data was written
("writecompatver",
"0"), # Lowest format version which is write compatible
("readcompatver",
"0"), # Lowest format version which is read compatible
("branchtag", "WikidPad") # Tag of the WikidPad branch
):
setSettingsValue(connwrap, key, value)
# --- WikiPad 1.20beta2 reached (formatver=0, writecompatver=0,
# readcompatver=0) ---
formatver = getSettingsInt(connwrap, "formatver")
if formatver == 0:
# From formatver 0 to 1, all filenames with brackets are renamed
# to have no brackets
filenames = glob.glob(join(mbcsEnc(dataDir, "replace")[0], '*.wiki'))
for fn in filenames:
fn = mbcsDec(fn, "replace")[0]
bn = basename(fn)
newbname = removeBracketsFilename(bn)
if bn == newbname:
continue
newname = mbcsEnc(join(dataDir, newbname), "replace")[0]
if exists(pathEnc(newname)):
# A file with the designated new name of fn already exists
# -> do nothing
continue
try:
rename(fn, newname)
except (IOError, OSError):
pass
formatver = 1
# --- WikiPad 1.20beta3 reached (formatver=1, writecompatver=1,
# readcompatver=1) ---
if formatver == 1:
# remove brackets from all wikiwords in database
# table wikiwords
dataIn = connwrap.execSqlQuery(
"select word, created, modified from wikiwords")
connwrap.execSql("drop table wikiwords")
connwrap.commit()
changeTableSchema(connwrap, "wikiwords",
TABLE_DEFINITIONS["wikiwords"])
rebuildIndices(connwrap)
uniqueCtl = {}
for w, c, m in dataIn:
w = oldWikiWordToLabel(w)
if not uniqueCtl.has_key(w):
connwrap.execSqlInsert(
"wikiwords",
("word", "created", "modified", "presentationdatablock"),
(w, c, m, ""))
# connwrap.execSql("insert into wikiwords(word, created, modified) "
# "values (?, ?, ?)", (w, c, m))
uniqueCtl[w] = None
# table wikirelations
dataIn = connwrap.execSqlQuery(
"select word, relation, created from wikirelations")
connwrap.execSql("drop table wikirelations")
connwrap.commit()
changeTableSchema(connwrap, "wikirelations",
TABLE_DEFINITIONS["wikirelations"])
rebuildIndices(connwrap)
uniqueCtl = {}
for w, r, c in dataIn:
w, r = oldWikiWordToLabel(w), oldWikiWordToLabel(r)
if not uniqueCtl.has_key((w, r)):
connwrap.execSqlInsert("wikirelations",
("word", "relation", "created"),
(w, r, c))
# connwrap.execSql("insert into wikirelations(word, relation, created) "
# "values (?, ?, ?)", (w, r, c))
uniqueCtl[(w, r)] = None
# table wikiwordprops
dataIn = connwrap.execSqlQuery(
"select word, key, value from wikiwordprops")
connwrap.execSql("drop table wikiwordprops")
connwrap.commit()
changeTableSchema(connwrap, "wikiwordprops",
TABLE_DEFINITIONS["wikiwordprops_PRE2_1alpha01"])
rebuildIndices(connwrap)
for w, k, v in dataIn:
connwrap.execSqlInsert("wikiwordprops", ("word", "key", "value"),
(oldWikiWordToLabel(w), k, v),
tableDefault="wikiwordprops_PRE2_1alpha01")
# connwrap.execSql("insert into wikiwordprops(word, key, value) "
# "values (?, ?, ?)", (oldWikiWordToLabel(w), k, v))
# table todos
dataIn = connwrap.execSqlQuery("select word, todo from todos")
connwrap.execSql("drop table todos")
connwrap.commit()
changeTableSchema(connwrap, "todos",
TABLE_DEFINITIONS["todos_PRE2_1alpha01"])
rebuildIndices(connwrap)
for w, t in dataIn:
connwrap.execSqlInsert("todos", ("word", "todo"),
(oldWikiWordToLabel(w), t),
tableDefault="todos_PRE2_1alpha01")
# connwrap.execSql("insert into todos(word, todo) "
# "values (?, ?)", (oldWikiWordToLabel(w), t))
formatver = 2
# --- WikiPad 1.6beta2 reached (formatver=2, writecompatver=2,
# readcompatver=2) ---
if formatver == 2:
changeTableSchema(connwrap, "wikiwords",
TABLE_DEFINITIONS["wikiwords"])
# --- WikiPad 1.8beta1 reached (formatver=3, writecompatver=3,
# readcompatver=2) ---
formatver = 3
if formatver == 3:
# Update "wikiwords" schema and create new tables
for tn in ("wikiwords", "wikiwordmatchterms", "datablocks",
"datablocksexternal", "defaultvalues"):
changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn])
# (Re)fill "defaultvalues" and read them into connection wrapper
connwrap.fillDefaultValues()
connwrap.readDefaultValues()
# Transfer "search_views" data to "datablocks" table
searches = connwrap.execSqlQuery(
"select title, datablock from search_views", strConv=(True, False))
for title, data in searches:
connwrap.execSql(
"insert into datablocks(unifiedname, data) "+\
"values (?, ?)", (u"savedsearch/" + title, data))
connwrap.execSql("drop table search_views")
allWords = connwrap.execSqlQuerySingleColumn(
"select word from wikiwords")
# Divide into functional and wiki pages
wikiWords = []
funcWords = []
for w in allWords:
if w.startswith('['):
funcWords.append(w)
else:
wikiWords.append(w)
# Fill the new fields in table "wikiwords"
for wikiWord in wikiWords:
filename = wikiWord + pagefileSuffix
fullPath = join(dataDir, filename)
try:
# We don't use coarsening here for the FSB because a different
# coarsening setting can't exist for the old wiki format
filesig = getFileSignatureBlock(fullPath)
except (IOError, WindowsError):
traceback.print_exc()
continue
connwrap.execSql(
"update wikiwords set filepath = ?, "
"filenamelowercase = ?, filesignature = ? "
"where word = ?",
(filename, filename.lower(), filesig, wikiWord))
# Move functional pages to new table "datablocksexternal" and rename them
for funcWord in funcWords:
if funcWord not in (u"[TextBlocks]", u"[PWL]", u"[CCBlacklist]"):
continue # Error ?!
unifName = u"wiki/" + funcWord[1:-1]
fullPath = join(dataDir, funcWord + pagefileSuffix)
icf = iterCompatibleFilename(unifName, u".data")
for i in range(
10): # Actual "while True", but that's too dangerous
newFilename = icf.next()
newPath = join(dataDir, newFilename)
if exists(pathEnc(newPath)):
# A file with the designated new name of fn already exists
# -> do nothing
continue
try:
rename(pathEnc(fullPath), pathEnc(newPath))
# We don't use coarsening here for the FSB because a different
# coarsening setting can't exist for the old wiki format
connwrap.execSqlInsert(
"datablocksexternal",
("unifiedname", "filepath", "filenamelowercase",
"filesignature"),
(unifName, newFilename, newFilename.lower(),
getFileSignatureBlock(newPath)))
connwrap.execSql("delete from wikiwords where word = ?",
(funcWord, ))
break
except (IOError, OSError):
traceback.print_exc()
continue
# --- WikiPad 2.0alpha1 reached (formatver=4, writecompatver=4,
# readcompatver=4) ---
formatver = 4
if formatver == 4:
# (Re)fill "defaultvalues" and read them into connection wrapper
connwrap.fillDefaultValues()
connwrap.readDefaultValues()
# Recreate table "todos" with new schema
connwrap.execSql("drop table todos")
changeTableSchema(connwrap, "todos", TABLE_DEFINITIONS["todos"])
# Rename table "wikiwordprops" to "wikiwordattrs"
changeTableSchema(connwrap, "wikiwordattrs",
TABLE_DEFINITIONS["wikiwordattrs"])
connwrap.execSql("insert into wikiwordattrs(word, key, value) "
"select word, key, value from wikiwordprops")
connwrap.execSql("drop table wikiwordprops")
for tn in ("wikirelations", "wikiwordmatchterms"):
changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn])
# Mark all wikiwords to need a rebuild
connwrap.execSql("update wikiwords set metadataprocessed=0")
formatver = 5
# --- WikiPad 2.1alpha.1 reached (formatver=5, writecompatver=5,
# readcompatver=5) ---
# Write format information
for key, value in (
("formatver",
str(VERSION_DB)), # Version of database format the data was written
("writecompatver", str(VERSION_WRITECOMPAT)
), # Lowest format version which is write compatible
("readcompatver", str(VERSION_READCOMPAT)
), # Lowest format version which is read compatible
("branchtag", "WikidPad"), # Tag of the WikidPad branch
("locale", "-"
) # Locale for cached wordnormcase column. '-': column invalid
):
setSettingsValue(connwrap, key, value)
rebuildIndices(connwrap)
connwrap.commit()
def updateDatabase(connwrap, dataDir, pagefileSuffix):
"""
Update a database from an older version to current (checkDatabaseFormat()
should have returned 1 before calling this function)
"""
connwrap.commit()
indices = connwrap.execSqlQuerySingleColumn("select INDEX_NAME from __indices__")
tables = connwrap.execSqlQuerySingleColumn("select TABLE_NAME from __table_names__")
indices = map(string.upper, indices)
tables = map(string.upper, tables)
# updatedTables = []
if not "SETTINGS" in tables:
# We are prior WikidPad 1.2beta2 (which writes format version 0)
if "WIKIWORDPROPS_PKEY" in indices:
# print "dropping index wikiwordprops_pkey"
connwrap.execSql("drop index wikiwordprops_pkey")
# if "WIKIWORDPROPS_WORD" not in indices:
# print "creating index wikiwordprops_word"
# connwrap.execSql("create index wikiwordprops_word on wikiwordprops(word)")
# if "WIKIRELATIONS_WORD" not in indices:
# print "creating index wikirelations_word"
# connwrap.execSql("create index wikirelations_word on wikirelations(word)")
if "REGISTRATION" in tables:
connwrap.execSql("drop table registration")
# Update search_views
searches = connwrap.execSqlQuerySingleColumn(
"select search from search_views")
changeTableSchema(connwrap, "search_views",
TABLE_DEFINITIONS["search_views"])
for search in searches:
searchOp = SearchReplaceOperation()
searchOp.searchStr = search
searchOp.wikiWide = True
searchOp.booleanOp = True
try:
# Raises exception if search is invalid
searchOp.rebuildSearchOpTree()
except:
continue
datablock = searchOp.getPackedSettings()
connwrap.execSql(
"insert into search_views(title, datablock) "+\
"values (?, ?)", (searchOp.getTitle(), datablock))
formatver = 0
changeTableSchema(connwrap, "settings",
TABLE_DEFINITIONS["settings"])
# Write initial format versions
for key, value in (
("formatver", "0"), # Version of database format the data was written
("writecompatver", "0"), # Lowest format version which is write compatible
("readcompatver", "0"), # Lowest format version which is read compatible
("branchtag", "WikidPad") # Tag of the WikidPad branch
):
setSettingsValue(connwrap, key, value)
# --- WikiPad 1.20beta2 reached (formatver=0, writecompatver=0,
# readcompatver=0) ---
formatver = getSettingsInt(connwrap, "formatver")
if formatver == 0:
# From formatver 0 to 1, all filenames with brackets are renamed
# to have no brackets
filenames = glob.glob(join(mbcsEnc(dataDir, "replace")[0], '*.wiki'))
for fn in filenames:
fn = mbcsDec(fn, "replace")[0]
bn = basename(fn)
newbname = removeBracketsFilename(bn)
if bn == newbname:
continue
newname = mbcsEnc(join(dataDir, newbname), "replace")[0]
if exists(pathEnc(newname)):
# A file with the designated new name of fn already exists
# -> do nothing
continue
try:
rename(fn, newname)
except (IOError, OSError):
pass
formatver = 1
# --- WikiPad 1.20beta3 reached (formatver=1, writecompatver=1,
# readcompatver=1) ---
if formatver == 1:
# remove brackets from all wikiwords in database
# table wikiwords
dataIn = connwrap.execSqlQuery(
"select word, created, modified from wikiwords")
connwrap.execSql("drop table wikiwords")
connwrap.commit()
changeTableSchema(connwrap, "wikiwords",
TABLE_DEFINITIONS["wikiwords"])
rebuildIndices(connwrap)
uniqueCtl = {}
for w, c, m in dataIn:
w = oldWikiWordToLabel(w)
if not uniqueCtl.has_key(w):
connwrap.execSqlInsert("wikiwords", ("word", "created",
"modified", "presentationdatablock"),
(w, c, m, ""))
# connwrap.execSql("insert into wikiwords(word, created, modified) "
# "values (?, ?, ?)", (w, c, m))
uniqueCtl[w] = None
# table wikirelations
dataIn = connwrap.execSqlQuery(
"select word, relation, created from wikirelations")
connwrap.execSql("drop table wikirelations")
connwrap.commit()
changeTableSchema(connwrap, "wikirelations",
TABLE_DEFINITIONS["wikirelations"])
rebuildIndices(connwrap)
uniqueCtl = {}
for w, r, c in dataIn:
w, r = oldWikiWordToLabel(w), oldWikiWordToLabel(r)
if not uniqueCtl.has_key((w, r)):
connwrap.execSqlInsert("wikirelations", ("word", "relation",
"created"), (w, r, c))
# connwrap.execSql("insert into wikirelations(word, relation, created) "
# "values (?, ?, ?)", (w, r, c))
uniqueCtl[(w, r)] = None
# table wikiwordprops
dataIn = connwrap.execSqlQuery(
"select word, key, value from wikiwordprops")
connwrap.execSql("drop table wikiwordprops")
connwrap.commit()
changeTableSchema(connwrap, "wikiwordprops",
TABLE_DEFINITIONS["wikiwordprops_PRE2_1alpha01"])
rebuildIndices(connwrap)
for w, k, v in dataIn:
connwrap.execSqlInsert("wikiwordprops", ("word", "key",
"value"), (oldWikiWordToLabel(w), k, v),
tableDefault="wikiwordprops_PRE2_1alpha01")
# connwrap.execSql("insert into wikiwordprops(word, key, value) "
# "values (?, ?, ?)", (oldWikiWordToLabel(w), k, v))
# table todos
dataIn = connwrap.execSqlQuery(
"select word, todo from todos")
connwrap.execSql("drop table todos")
connwrap.commit()
changeTableSchema(connwrap, "todos",
TABLE_DEFINITIONS["todos_PRE2_1alpha01"])
rebuildIndices(connwrap)
for w, t in dataIn:
connwrap.execSqlInsert("todos", ("word", "todo"),
(oldWikiWordToLabel(w), t),
tableDefault="todos_PRE2_1alpha01")
# connwrap.execSql("insert into todos(word, todo) "
# "values (?, ?)", (oldWikiWordToLabel(w), t))
formatver = 2
# --- WikiPad 1.6beta2 reached (formatver=2, writecompatver=2,
# readcompatver=2) ---
if formatver == 2:
changeTableSchema(connwrap, "wikiwords",
TABLE_DEFINITIONS["wikiwords"])
# --- WikiPad 1.8beta1 reached (formatver=3, writecompatver=3,
# readcompatver=2) ---
formatver = 3
if formatver == 3:
# Update "wikiwords" schema and create new tables
for tn in ("wikiwords", "wikiwordmatchterms", "datablocks",
"datablocksexternal", "defaultvalues"):
changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn])
# (Re)fill "defaultvalues" and read them into connection wrapper
connwrap.fillDefaultValues()
connwrap.readDefaultValues()
# Transfer "search_views" data to "datablocks" table
searches = connwrap.execSqlQuery(
"select title, datablock from search_views",
strConv=(True, False))
for title, data in searches:
connwrap.execSql(
"insert into datablocks(unifiedname, data) "+\
"values (?, ?)", (u"savedsearch/" + title, data))
connwrap.execSql("drop table search_views")
allWords = connwrap.execSqlQuerySingleColumn("select word from wikiwords")
# Divide into functional and wiki pages
wikiWords = []
funcWords = []
for w in allWords:
if w.startswith('['):
funcWords.append(w)
else:
wikiWords.append(w)
# Fill the new fields in table "wikiwords"
for wikiWord in wikiWords:
filename = wikiWord + pagefileSuffix
fullPath = join(dataDir, filename)
try:
# We don't use coarsening here for the FSB because a different
# coarsening setting can't exist for the old wiki format
filesig = getFileSignatureBlock(fullPath)
except (IOError, WindowsError):
traceback.print_exc()
continue
connwrap.execSql("update wikiwords set filepath = ?, "
"filenamelowercase = ?, filesignature = ? "
"where word = ?", (filename, filename.lower(), filesig,
wikiWord))
# Move functional pages to new table "datablocksexternal" and rename them
for funcWord in funcWords:
if funcWord not in (u"[TextBlocks]", u"[PWL]", u"[CCBlacklist]"):
continue # Error ?!
unifName = u"wiki/" + funcWord[1:-1]
fullPath = join(dataDir, funcWord + pagefileSuffix)
icf = iterCompatibleFilename(unifName, u".data")
for i in range(10): # Actual "while True", but that's too dangerous
newFilename = icf.next()
newPath = join(dataDir, newFilename)
if exists(pathEnc(newPath)):
# A file with the designated new name of fn already exists
# -> do nothing
continue
try:
rename(pathEnc(fullPath), pathEnc(newPath))
# We don't use coarsening here for the FSB because a different
# coarsening setting can't exist for the old wiki format
connwrap.execSqlInsert("datablocksexternal", ("unifiedname",
"filepath", "filenamelowercase", "filesignature"),
(unifName, newFilename, newFilename.lower(),
getFileSignatureBlock(newPath)))
connwrap.execSql("delete from wikiwords where word = ?",
(funcWord,))
break
except (IOError, OSError):
traceback.print_exc()
continue
# --- WikiPad 2.0alpha1 reached (formatver=4, writecompatver=4,
# readcompatver=4) ---
formatver = 4
if formatver == 4:
# (Re)fill "defaultvalues" and read them into connection wrapper
connwrap.fillDefaultValues()
connwrap.readDefaultValues()
# Recreate table "todos" with new schema
connwrap.execSql("drop table todos")
changeTableSchema(connwrap, "todos", TABLE_DEFINITIONS["todos"])
# Rename table "wikiwordprops" to "wikiwordattrs"
changeTableSchema(connwrap, "wikiwordattrs", TABLE_DEFINITIONS["wikiwordattrs"])
connwrap.execSql("insert into wikiwordattrs(word, key, value) "
"select word, key, value from wikiwordprops")
connwrap.execSql("drop table wikiwordprops")
for tn in ("wikirelations", "wikiwordmatchterms"):
changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn])
# Mark all wikiwords to need a rebuild
connwrap.execSql("update wikiwords set metadataprocessed=0")
formatver = 5
# --- WikiPad 2.1alpha.1 reached (formatver=5, writecompatver=5,
# readcompatver=5) ---
# Write format information
for key, value in (
("formatver", str(VERSION_DB)), # Version of database format the data was written
("writecompatver", str(VERSION_WRITECOMPAT)), # Lowest format version which is write compatible
("readcompatver", str(VERSION_READCOMPAT)), # Lowest format version which is read compatible
("branchtag", "WikidPad"), # Tag of the WikidPad branch
("locale", "-") # Locale for cached wordnormcase column. '-': column invalid
):
setSettingsValue(connwrap, key, value)
rebuildIndices(connwrap)
connwrap.commit()
