def _isIdentical(self, path1, path2): """ Checks if the files denoted by path1 and path2 are identical according to the settings of the object. The files must have passed the preliminary test by _preTestIdentity(). """ stat1 = os.stat(pathEnc(path1)) stat2 = os.stat(pathEnc(path2)) if self.modDateIsEnough and stat1.st_mtime == stat2.st_mtime: return True # End of fast tests, now the whole content must be compared file1 = file(path1, "rb") file2 = file(path2, "rb") try: while True: block1 = file1.read(1024 * 1024) block2 = file2.read(1024 * 1024) if len(block1) == 0 and len(block2) == 0: # EOF return True if len(block1) != len(block2): raise FSException(_(u"File compare error, file not readable or " u"changed during compare")) if block1 != block2: return False finally: file2.close() file1.close()
def _isIdentical(self, path1, path2): """ Checks if the files denoted by path1 and path2 are identical according to the settings of the object. The files must have passed the preliminary test by _preTestIdentity(). """ stat1 = os.stat(pathEnc(path1)) stat2 = os.stat(pathEnc(path2)) if self.modDateIsEnough and stat1.st_mtime == stat2.st_mtime: return True # End of fast tests, now the whole content must be compared file1 = open(path1, "rb") file2 = open(path2, "rb") try: while True: block1 = file1.read(1024 * 1024) block2 = file2.read(1024 * 1024) if len(block1) == 0 and len(block2) == 0: # EOF return True if len(block1) != len(block2): raise FSException(_("File compare error, file not readable or " "changed during compare")) if block1 != block2: return False finally: file2.close() file1.close()
def findDestPathNoSource(self, suffix, prefix=u""): """ Find a path to a destination. """ self._ensureStorage() if prefix: fname = prefix + suffix destPath = os.path.join(self.storagePath, fname) if not os.path.exists(pathEnc(destPath)): return destPath else: prefix = u"" # mat = _FILESPLITPAT.match(fname) # if mat is None: # raise FSException("Internal error: Bad source file name") # # coreName = mat.group("name") # suffix = mat.group("suffix") for t in xrange(20): # Number of tries newName = u"%s_%s%s" % (prefix, createRandomString(20), suffix) destPath = os.path.join(self.storagePath, newName) if not os.path.exists(pathEnc(destPath)): return destPath # Give up return None
def findDestPathNoSource(self, suffix, prefix=""): """ Find a path to a destination. """ self._ensureStorage() if prefix: fname = prefix + suffix destPath = os.path.join(self.storagePath, fname) if not os.path.exists(pathEnc(destPath)): return destPath else: prefix = "" # mat = _FILESPLITPAT.match(fname) # if mat is None: # raise FSException("Internal error: Bad source file name") # # coreName = mat.group("name") # suffix = mat.group("suffix") for t in range(20): # Number of tries newName = "%s_%s%s" % (prefix, createRandomString(20), suffix) destPath = os.path.join(self.storagePath, newName) if not os.path.exists(pathEnc(destPath)): return destPath # Give up return None
def createWikiDB(wikiName, dataDir, overwrite=False): """ creates the initial db Warning: If overwrite is True, a previous file will be deleted! """ dbfile = join(dataDir, "wiki.sli") if (not exists(pathEnc(dbfile)) or overwrite): if (not exists(pathEnc(dataDir))): mkdir(pathEnc(dataDir)) else: if exists(pathEnc(dbfile)) and overwrite: unlink(pathEnc(dbfile)) # if (wikiDocument is not None): # dbName = self.wikiDocument.getWikiConfig().get("wiki_db", "db_filename", # u"").strip() # # if (dbName == u""): # dbName = u"wikidb" # else: dbName = u"wikidb" # create the database connection = gadfly.gadfly() connection.startup(dbName, dataDir) connwrap = ConnectWrap(connection) try: for tn in MAIN_TABLES: changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn]) for key, value in ( ("formatver", str(VERSION_DB)), # Version of database format the data was written ("writecompatver", str(VERSION_WRITECOMPAT)), # Lowest format version which is write compatible ("readcompatver", str(VERSION_READCOMPAT)), # Lowest format version which is read compatible ("branchtag", "WikidPad"), # Tag of the WikidPad branch ("locale", "-") # Locale for cached wordnormcase column. '-': column invalid ): setSettingsValue(connwrap, key, value) # connwrap.executemany("insert or replace into settings(key, value) "+ # "values (?, ?)", ( # ("formatver", "0"), # Version of database format the data was written # ("writecompatver", "0"), # Lowest format version which is write compatible # ("readcompatver", "0"), # Lowest format version which is read compatible # ("branchtag", "WikidPad") # Tag of the WikidPad branch # ) ) rebuildIndices(connwrap) connwrap.commit() finally: # close the connection connwrap.close() else: raise WikiDBExistsException( _(u"database already exists at location: %s") % dataDir)
def createWikiDB(wikiName, dataDir, overwrite=False): """ creates the initial db Warning: If overwrite is True, a previous file will be deleted! """ dbfile = join(dataDir, "wiki.sli") if (not exists(pathEnc(dbfile)) or overwrite): if (not exists(pathEnc(dataDir))): mkdir(pathEnc(dataDir)) else: if exists(pathEnc(dbfile)) and overwrite: unlink(pathEnc(dbfile)) # create the database connection = gadfly.gadfly() connection.startup("wikidb", dataDir) connwrap = ConnectWrap(connection) try: for tn in MAIN_TABLES: changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn]) for key, value in ( ("formatver", str(VERSION_DB) ), # Version of database format the data was written ("writecompatver", str(VERSION_WRITECOMPAT) ), # Lowest format version which is write compatible ("readcompatver", str(VERSION_READCOMPAT) ), # Lowest format version which is read compatible ("branchtag", "WikidPad"), # Tag of the WikidPad branch ( "locale", "-" ) # Locale for cached wordnormcase column. '-': column invalid ): setSettingsValue(connwrap, key, value) # connwrap.executemany("insert or replace into settings(key, value) "+ # "values (?, ?)", ( # ("formatver", "0"), # Version of database format the data was written # ("writecompatver", "0"), # Lowest format version which is write compatible # ("readcompatver", "0"), # Lowest format version which is read compatible # ("branchtag", "WikidPad") # Tag of the WikidPad branch # ) ) rebuildIndices(connwrap) connwrap.commit() finally: # close the connection connwrap.close() else: raise WikiDBExistsException( _(u"database already exists at location: %s") % dataDir)
def createWikiDB(wikiName, dataDir, overwrite=False, wikiDocument=None): """ creates the initial db Warning: If overwrite is True, a previous file will be deleted! """ if (wikiDocument is not None): dbPath = wikiDocument.getWikiConfig().get("wiki_db", "db_filename", "").strip() if (dbPath == ""): dbPath = "wikiovw.sli" else: dbPath = "wikiovw.sli" dbfile = join(dataDir, dbPath) if (not exists(pathEnc(dbfile)) or overwrite): if (not exists(pathEnc(dataDir))): mkdir(pathEnc(dataDir)) else: if exists(pathEnc(dbfile)) and overwrite: unlink(pathEnc(dbfile)) # create the database connwrap = ConnectWrapSyncCommit(sqlite.connect(dbfile)) try: for tn in MAIN_TABLES: changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn]) connwrap.executemany( "insert or replace into settings(key, value) " + "values (?, ?)", ( ("formatver", str(VERSION_DB) ), # Version of database format the data was written ("writecompatver", str(VERSION_WRITECOMPAT) ), # Lowest format version which is write compatible ("readcompatver", str(VERSION_READCOMPAT) ), # Lowest format version which is read compatible ("branchtag", "WikidPad") # Tag of the WikidPad branch # ("locale", "-") # Locale for cached wordnormcase column. '-': column invalid )) rebuildIndices(connwrap) connwrap.syncCommit() finally: # close the connection connwrap.close() else: raise WikiDBExistsException( _("database already exists at location: %s") % dataDir)
def createWikiDB(wikiName, dataDir, overwrite=False, wikiDocument=None): """ creates the initial db Warning: If overwrite is True, a previous file will be deleted! """ if (wikiDocument is not None): dbPath = wikiDocument.getWikiConfig().get("wiki_db", "db_filename", "").strip() if (dbPath == ""): dbPath = "wikiovw.sli" else: dbPath = "wikiovw.sli" dbfile = join(dataDir, dbPath) if (not exists(pathEnc(dbfile)) or overwrite): if (not exists(pathEnc(dataDir))): mkdir(pathEnc(dataDir)) else: if exists(pathEnc(dbfile)) and overwrite: unlink(pathEnc(dbfile)) # create the database connwrap = ConnectWrapSyncCommit(sqlite.connect(dbfile)) try: for tn in MAIN_TABLES: changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn]) connwrap.executemany("insert or replace into settings(key, value) "+ "values (?, ?)", ( ("formatver", str(VERSION_DB)), # Version of database format the data was written ("writecompatver", str(VERSION_WRITECOMPAT)), # Lowest format version which is write compatible ("readcompatver", str(VERSION_READCOMPAT)), # Lowest format version which is read compatible ("branchtag", "WikidPad") # Tag of the WikidPad branch # ("locale", "-") # Locale for cached wordnormcase column. '-': column invalid ) ) rebuildIndices(connwrap) connwrap.syncCommit() finally: # close the connection connwrap.close() else: raise WikiDBExistsException( _("database already exists at location: %s") % dataDir)
def _preTestIdentity(self, destpath, srcfname, srcstat): """ Preliminary test if the destination file denoted by destpath and the source described by its name srcfnam without path and its stats COULD BE identical. If the function returns False, they are definitely different, if it returns True, further tests are necessary. """ if not os.path.isfile(destpath): # Must be existing file return False destfname = os.path.basename(destpath) if os.path.splitext(srcfname)[1] != os.path.splitext(destfname)[1]: # file suffix must match always return False if self.filenameMustMatch and srcfname != destfname: return False deststat = os.stat(pathEnc(destpath)) if deststat.st_size != srcstat.st_size: # Size must match always return False if self.modDateMustMatch and deststat.st_mtime != srcstat.st_mtime: return False # This means it COULD BE identical according to the quick tests return True
def findDestPath(self, srcPath): """ Find a path to a new destination of the source file denoted by srcPath. Some settings of the object determine how this is done exactly. Returns a tuple (path, exists) where path is the destination path and exists is True if an identical file exists already at the destination. If path is None, a new filename couldn't be found. """ if not (os.path.isfile(srcPath) or os.path.isdir(srcPath)): raise FSException(_("Path '%s' must point to an existing file") % srcPath) self._ensureStorage() for c in self._getCandidates(srcPath): if self._isIdentical(srcPath, c): return (c, True) # No identical file found, so find a not yet used name for the new file. fname = os.path.basename(srcPath) if not os.path.exists(pathEnc(os.path.join(self.storagePath, fname))): return (os.path.join(self.storagePath, fname), False) mat = _FILESPLITPAT.match(fname) if mat is None: raise FSException(_("Internal error: Bad source file name")) coreName = mat.group("name") suffix = mat.group("suffix") for t in range(10): # Number of tries newName = "%s_%s%s" % (coreName, createRandomString(10), suffix) if not os.path.exists(pathEnc(os.path.join( self.storagePath, newName))): return (os.path.join(self.storagePath, newName), False) # Give up return (None, False)
def findDestPath(self, srcPath): """ Find a path to a new destination of the source file denoted by srcPath. Some settings of the object determine how this is done exactly. Returns a tuple (path, exists) where path is the destination path and exists is True if an identical file exists already at the destination. If path is None, a new filename couldn't be found. """ if not (os.path.isfile(srcPath) or os.path.isdir(srcPath)): raise FSException(_(u"Path '%s' must point to an existing file") % srcPath) self._ensureStorage() for c in self._getCandidates(srcPath): if self._isIdentical(srcPath, c): return (c, True) # No identical file found, so find a not yet used name for the new file. fname = os.path.basename(srcPath) if not os.path.exists(pathEnc(os.path.join(self.storagePath, fname))): return (os.path.join(self.storagePath, fname), False) mat = _FILESPLITPAT.match(fname) if mat is None: raise FSException(_(u"Internal error: Bad source file name")) coreName = mat.group("name") suffix = mat.group("suffix") for t in xrange(10): # Number of tries newName = u"%s_%s%s" % (coreName, createRandomString(10), suffix) if not os.path.exists(pathEnc(os.path.join( self.storagePath, newName))): return (os.path.join(self.storagePath, newName), False) # Give up return (None, False)
def _getCandidates(self, srcPath): """ Find possible candidates for detailed identity check. The file storage must exist already. srcPath -- Must be a path to an existing file """ srcfname = os.path.basename(srcPath) srcstat = os.stat(pathEnc(srcPath)) ccSameName = set() # candidate category: same filename # (only one possible entry) ccSameMod = set() # candidate category: same mod. date ccElse = set() # candidate category: everything else samenamepath = os.path.join(self.storagePath, srcfname) if self._preTestIdentity(samenamepath, srcfname, srcstat): ccSameName.add(samenamepath) if self.filenameMustMatch: # No other candidates possible return list(ccSameName) ext = os.path.splitext(srcPath)[1] for p in glob.glob(os.path.join(self.storagePath, "*" + ext)): if p == samenamepath: # Already tested above continue if self._preTestIdentity(p, srcfname, srcstat): deststat = os.stat(pathEnc(p)) if deststat.st_mtime == srcstat.st_mtime: ccSameMod.add(p) else: ccElse.add(p) return list(ccSameName) + list(ccSameMod) + list(ccElse)
def updateDatabase(connwrap, dataDir, pagefileSuffix, wikiData): """ Update a database from an older version to current (checkDatabaseFormat() should have returned 1 before calling this function) """ connwrap.syncCommit() # Always remember that there is no automatic unicode<->utf-8 conversion # during this function! formatver = getSettingsInt(connwrap, "formatver") if formatver == 0: # Insert in table wikiwords column wordnormcase changeTableSchema(connwrap, "wikiwords", TABLE_DEFINITIONS["wikiwords"]) formatver = 1 # --- WikiPad 1.8beta1 reached (formatver=1, writecompatver=1, # readcompatver=1) --- if formatver == 1: # Update "wikiwords" schema and create new tables for tn in ("wikiwords", "wikiwordmatchterms", "datablocks", "datablocksexternal"): changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn]) # Transfer "search_views" data to "datablocks" table searches = connwrap.execSqlQuery( "select title, datablock from search_views") for title, data in searches: connwrap.execSql( "insert into datablocks(unifiedname, data) "+\ "values (?, ?)", ("savedsearch/" + title, sqlite.Binary(data))) connwrap.execSql("drop table search_views") allWords = connwrap.execSqlQuerySingleColumn("select word from wikiwords") # Divide into functional and wiki pages wikiWords = [] funcWords = [] for w in allWords: w = w.decode("utf-8") if w.startswith(u'['): funcWords.append(w) else: wikiWords.append(w) # Fill the new fields in table "wikiwords" for wikiWord in wikiWords: filename = wikiWord + pagefileSuffix fullPath = join(dataDir, filename) try: # We don't use coarsening here for the FSB because a different # coarsening setting can't exist for the old wiki format filesig = getFileSignatureBlock(fullPath) except (IOError, WindowsError): traceback.print_exc() continue connwrap.execSql("update wikiwords set filepath = ?, " "filenamelowercase = ?, filesignature = ? " "where word = ?", (filename.encode("utf-8"), filename.lower().encode("utf-8"), sqlite.Binary(filesig), wikiWord.encode("utf-8"))) # Move functional pages to new table "datablocksexternal" and rename them for funcWord in funcWords: if funcWord not in (u"[TextBlocks]", u"[PWL]", u"[CCBlacklist]"): continue # Error ?! unifName = u"wiki/" + funcWord[1:-1] fullPath = join(dataDir, funcWord + pagefileSuffix) icf = iterCompatibleFilename(unifName, u".data") for i in range(10): # Actual "while True", but that's too dangerous newFilename = icf.next() newPath = join(dataDir, newFilename) if exists(pathEnc(newPath)): # A file with the designated new name of fn already exists # -> do nothing continue try: rename(pathEnc(fullPath), pathEnc(newPath)) # We don't use coarsening here for the FSB because a different # coarsening setting can't exist for the old wiki format connwrap.execSql( "insert into datablocksexternal(unifiedname, filepath, " "filenamelowercase, filesignature) values (?, ?, ?, ?)", (unifName.encode("utf-8"), newFilename.encode("utf-8"), newFilename.lower().encode("utf-8"), sqlite.Binary(getFileSignatureBlock(newPath)))) connwrap.execSql("delete from wikiwords where word = ?", (funcWord.encode("utf-8"),)) break except (IOError, OSError): traceback.print_exc() continue formatver = 2 # --- WikiPad 2.0alpha1 reached (formatver=2, writecompatver=2, # readcompatver=2) --- if formatver == 2: # Recreate table "todos" connwrap.execSql("drop table todos;") changeTableSchema(connwrap, "todos", TABLE_DEFINITIONS["todos"]) connwrap.execSql("update todos set firstcharpos=-1;") # Rename table "wikiwordprops" to "wikiwordattrs" changeTableSchema(connwrap, "wikiwordattrs", TABLE_DEFINITIONS["wikiwordattrs"]) connwrap.execSql("insert into wikiwordattrs(word, key, value, " "firstcharpos, charlength) select word, key, value, " "firstcharpos, -1 from wikiwordprops;") connwrap.execSql("drop table wikiwordprops;") for tn in ("wikirelations", "wikiwordmatchterms"): changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn]) connwrap.execSql("update %s set firstcharpos=-1;" % tn) # Mark all wikiwords to need a rebuild connwrap.execSql("update wikiwords set metadataprocessed=0;") formatver = 3 # --- WikiPad 2.1alpha1 reached (formatver=3, writecompatver=3, # readcompatver=3) --- connwrap.executemany("insert or replace into settings(key, value) "+ "values (?, ?)", ( ("formatver", str(VERSION_DB)), # Version of database format the data was written ("writecompatver", str(VERSION_WRITECOMPAT)), # Lowest format version which is write compatible ("readcompatver", str(VERSION_READCOMPAT)), # Lowest format version which is read compatible ("branchtag", "WikidPad") # Tag of the WikidPad branch ) ) rebuildIndices(connwrap) connwrap.syncCommit()
def _storageExists(self): """ Test if file storage (=directory) exists. """ return os.path.exists(pathEnc(self.storagePath))
def updateDatabase(connwrap, dataDir, pagefileSuffix): """ Update a database from an older version to current (checkDatabaseFormat() should have returned 1 before calling this function) """ connwrap.commit() indices = connwrap.execSqlQuerySingleColumn( "select INDEX_NAME from __indices__") tables = connwrap.execSqlQuerySingleColumn( "select TABLE_NAME from __table_names__") indices = map(string.upper, indices) tables = map(string.upper, tables) # updatedTables = [] if not "SETTINGS" in tables: # We are prior WikidPad 1.2beta2 (which writes format version 0) if "WIKIWORDPROPS_PKEY" in indices: # print "dropping index wikiwordprops_pkey" connwrap.execSql("drop index wikiwordprops_pkey") # if "WIKIWORDPROPS_WORD" not in indices: # print "creating index wikiwordprops_word" # connwrap.execSql("create index wikiwordprops_word on wikiwordprops(word)") # if "WIKIRELATIONS_WORD" not in indices: # print "creating index wikirelations_word" # connwrap.execSql("create index wikirelations_word on wikirelations(word)") if "REGISTRATION" in tables: connwrap.execSql("drop table registration") # Update search_views searches = connwrap.execSqlQuerySingleColumn( "select search from search_views") changeTableSchema(connwrap, "search_views", TABLE_DEFINITIONS["search_views"]) for search in searches: searchOp = SearchReplaceOperation() searchOp.searchStr = search searchOp.wikiWide = True searchOp.booleanOp = True try: # Raises exception if search is invalid searchOp.rebuildSearchOpTree() except: continue datablock = searchOp.getPackedSettings() connwrap.execSql( "insert into search_views(title, datablock) "+\ "values (?, ?)", (searchOp.getTitle(), datablock)) formatver = 0 changeTableSchema(connwrap, "settings", TABLE_DEFINITIONS["settings"]) # Write initial format versions for key, value in ( ("formatver", "0"), # Version of database format the data was written ("writecompatver", "0"), # Lowest format version which is write compatible ("readcompatver", "0"), # Lowest format version which is read compatible ("branchtag", "WikidPad") # Tag of the WikidPad branch ): setSettingsValue(connwrap, key, value) # --- WikiPad 1.20beta2 reached (formatver=0, writecompatver=0, # readcompatver=0) --- formatver = getSettingsInt(connwrap, "formatver") if formatver == 0: # From formatver 0 to 1, all filenames with brackets are renamed # to have no brackets filenames = glob.glob(join(mbcsEnc(dataDir, "replace")[0], '*.wiki')) for fn in filenames: fn = mbcsDec(fn, "replace")[0] bn = basename(fn) newbname = removeBracketsFilename(bn) if bn == newbname: continue newname = mbcsEnc(join(dataDir, newbname), "replace")[0] if exists(pathEnc(newname)): # A file with the designated new name of fn already exists # -> do nothing continue try: rename(fn, newname) except (IOError, OSError): pass formatver = 1 # --- WikiPad 1.20beta3 reached (formatver=1, writecompatver=1, # readcompatver=1) --- if formatver == 1: # remove brackets from all wikiwords in database # table wikiwords dataIn = connwrap.execSqlQuery( "select word, created, modified from wikiwords") connwrap.execSql("drop table wikiwords") connwrap.commit() changeTableSchema(connwrap, "wikiwords", TABLE_DEFINITIONS["wikiwords"]) rebuildIndices(connwrap) uniqueCtl = {} for w, c, m in dataIn: w = oldWikiWordToLabel(w) if not uniqueCtl.has_key(w): connwrap.execSqlInsert( "wikiwords", ("word", "created", "modified", "presentationdatablock"), (w, c, m, "")) # connwrap.execSql("insert into wikiwords(word, created, modified) " # "values (?, ?, ?)", (w, c, m)) uniqueCtl[w] = None # table wikirelations dataIn = connwrap.execSqlQuery( "select word, relation, created from wikirelations") connwrap.execSql("drop table wikirelations") connwrap.commit() changeTableSchema(connwrap, "wikirelations", TABLE_DEFINITIONS["wikirelations"]) rebuildIndices(connwrap) uniqueCtl = {} for w, r, c in dataIn: w, r = oldWikiWordToLabel(w), oldWikiWordToLabel(r) if not uniqueCtl.has_key((w, r)): connwrap.execSqlInsert("wikirelations", ("word", "relation", "created"), (w, r, c)) # connwrap.execSql("insert into wikirelations(word, relation, created) " # "values (?, ?, ?)", (w, r, c)) uniqueCtl[(w, r)] = None # table wikiwordprops dataIn = connwrap.execSqlQuery( "select word, key, value from wikiwordprops") connwrap.execSql("drop table wikiwordprops") connwrap.commit() changeTableSchema(connwrap, "wikiwordprops", TABLE_DEFINITIONS["wikiwordprops_PRE2_1alpha01"]) rebuildIndices(connwrap) for w, k, v in dataIn: connwrap.execSqlInsert("wikiwordprops", ("word", "key", "value"), (oldWikiWordToLabel(w), k, v), tableDefault="wikiwordprops_PRE2_1alpha01") # connwrap.execSql("insert into wikiwordprops(word, key, value) " # "values (?, ?, ?)", (oldWikiWordToLabel(w), k, v)) # table todos dataIn = connwrap.execSqlQuery("select word, todo from todos") connwrap.execSql("drop table todos") connwrap.commit() changeTableSchema(connwrap, "todos", TABLE_DEFINITIONS["todos_PRE2_1alpha01"]) rebuildIndices(connwrap) for w, t in dataIn: connwrap.execSqlInsert("todos", ("word", "todo"), (oldWikiWordToLabel(w), t), tableDefault="todos_PRE2_1alpha01") # connwrap.execSql("insert into todos(word, todo) " # "values (?, ?)", (oldWikiWordToLabel(w), t)) formatver = 2 # --- WikiPad 1.6beta2 reached (formatver=2, writecompatver=2, # readcompatver=2) --- if formatver == 2: changeTableSchema(connwrap, "wikiwords", TABLE_DEFINITIONS["wikiwords"]) # --- WikiPad 1.8beta1 reached (formatver=3, writecompatver=3, # readcompatver=2) --- formatver = 3 if formatver == 3: # Update "wikiwords" schema and create new tables for tn in ("wikiwords", "wikiwordmatchterms", "datablocks", "datablocksexternal", "defaultvalues"): changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn]) # (Re)fill "defaultvalues" and read them into connection wrapper connwrap.fillDefaultValues() connwrap.readDefaultValues() # Transfer "search_views" data to "datablocks" table searches = connwrap.execSqlQuery( "select title, datablock from search_views", strConv=(True, False)) for title, data in searches: connwrap.execSql( "insert into datablocks(unifiedname, data) "+\ "values (?, ?)", (u"savedsearch/" + title, data)) connwrap.execSql("drop table search_views") allWords = connwrap.execSqlQuerySingleColumn( "select word from wikiwords") # Divide into functional and wiki pages wikiWords = [] funcWords = [] for w in allWords: if w.startswith('['): funcWords.append(w) else: wikiWords.append(w) # Fill the new fields in table "wikiwords" for wikiWord in wikiWords: filename = wikiWord + pagefileSuffix fullPath = join(dataDir, filename) try: # We don't use coarsening here for the FSB because a different # coarsening setting can't exist for the old wiki format filesig = getFileSignatureBlock(fullPath) except (IOError, WindowsError): traceback.print_exc() continue connwrap.execSql( "update wikiwords set filepath = ?, " "filenamelowercase = ?, filesignature = ? " "where word = ?", (filename, filename.lower(), filesig, wikiWord)) # Move functional pages to new table "datablocksexternal" and rename them for funcWord in funcWords: if funcWord not in (u"[TextBlocks]", u"[PWL]", u"[CCBlacklist]"): continue # Error ?! unifName = u"wiki/" + funcWord[1:-1] fullPath = join(dataDir, funcWord + pagefileSuffix) icf = iterCompatibleFilename(unifName, u".data") for i in range( 10): # Actual "while True", but that's too dangerous newFilename = icf.next() newPath = join(dataDir, newFilename) if exists(pathEnc(newPath)): # A file with the designated new name of fn already exists # -> do nothing continue try: rename(pathEnc(fullPath), pathEnc(newPath)) # We don't use coarsening here for the FSB because a different # coarsening setting can't exist for the old wiki format connwrap.execSqlInsert( "datablocksexternal", ("unifiedname", "filepath", "filenamelowercase", "filesignature"), (unifName, newFilename, newFilename.lower(), getFileSignatureBlock(newPath))) connwrap.execSql("delete from wikiwords where word = ?", (funcWord, )) break except (IOError, OSError): traceback.print_exc() continue # --- WikiPad 2.0alpha1 reached (formatver=4, writecompatver=4, # readcompatver=4) --- formatver = 4 if formatver == 4: # (Re)fill "defaultvalues" and read them into connection wrapper connwrap.fillDefaultValues() connwrap.readDefaultValues() # Recreate table "todos" with new schema connwrap.execSql("drop table todos") changeTableSchema(connwrap, "todos", TABLE_DEFINITIONS["todos"]) # Rename table "wikiwordprops" to "wikiwordattrs" changeTableSchema(connwrap, "wikiwordattrs", TABLE_DEFINITIONS["wikiwordattrs"]) connwrap.execSql("insert into wikiwordattrs(word, key, value) " "select word, key, value from wikiwordprops") connwrap.execSql("drop table wikiwordprops") for tn in ("wikirelations", "wikiwordmatchterms"): changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn]) # Mark all wikiwords to need a rebuild connwrap.execSql("update wikiwords set metadataprocessed=0") formatver = 5 # --- WikiPad 2.1alpha.1 reached (formatver=5, writecompatver=5, # readcompatver=5) --- # Write format information for key, value in ( ("formatver", str(VERSION_DB)), # Version of database format the data was written ("writecompatver", str(VERSION_WRITECOMPAT) ), # Lowest format version which is write compatible ("readcompatver", str(VERSION_READCOMPAT) ), # Lowest format version which is read compatible ("branchtag", "WikidPad"), # Tag of the WikidPad branch ("locale", "-" ) # Locale for cached wordnormcase column. '-': column invalid ): setSettingsValue(connwrap, key, value) rebuildIndices(connwrap) connwrap.commit()
def updateDatabase(connwrap, dataDir, pagefileSuffix): """ Update a database from an older version to current (checkDatabaseFormat() should have returned 1 before calling this function) """ connwrap.syncCommit() # Always remember that there is no automatic unicode<->utf-8 conversion # during this function! formatver = getSettingsInt(connwrap, "formatver") if formatver == 0: # Insert in table wikiwords column wordnormcase changeTableSchema(connwrap, "wikiwords", TABLE_DEFINITIONS["wikiwords"]) formatver = 1 # --- WikiPad 1.8beta1 reached (formatver=1, writecompatver=1, # readcompatver=1) --- if formatver == 1: # Update "wikiwords" schema and create new tables for tn in ("wikiwords", "wikiwordmatchterms", "datablocks", "datablocksexternal"): changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn]) # Transfer "search_views" data to "datablocks" table searches = connwrap.execSqlQuery( "select title, datablock from search_views") for title, data in searches: connwrap.execSql( "insert into datablocks(unifiedname, data) "+\ "values (?, ?)", ("savedsearch/" + title, sqlite.Binary(data))) connwrap.execSql("drop table search_views") allWords = connwrap.execSqlQuerySingleColumn( "select word from wikiwords") # Divide into functional and wiki pages wikiWords = [] funcWords = [] for w in allWords: w = w.decode("utf-8") if w.startswith('['): funcWords.append(w) else: wikiWords.append(w) # Fill the new fields in table "wikiwords" for wikiWord in wikiWords: filename = wikiWord + pagefileSuffix fullPath = join(dataDir, filename) try: # We don't use coarsening here for the FSB because a different # coarsening setting can't exist for the old wiki format filesig = getFileSignatureBlock(fullPath) except (IOError, WindowsError): traceback.print_exc() continue connwrap.execSql( "update wikiwords set filepath = ?, " "filenamelowercase = ?, filesignature = ? " "where word = ?", (filename.encode("utf-8"), filename.lower().encode("utf-8"), sqlite.Binary(filesig), wikiWord.encode("utf-8"))) # Move functional pages to new table "datablocksexternal" and rename them for funcWord in funcWords: if funcWord not in ("[TextBlocks]", "[PWL]", "[CCBlacklist]"): continue # Error ?! unifName = "wiki/" + funcWord[1:-1] fullPath = join(dataDir, funcWord + pagefileSuffix) icf = iterCompatibleFilename(unifName, ".data") for i in range( 10): # Actual "while True", but that's too dangerous newFilename = next(icf) newPath = join(dataDir, newFilename) if exists(pathEnc(newPath)): # A file with the designated new name of fn already exists # -> do nothing continue try: rename(pathEnc(fullPath), pathEnc(newPath)) # We don't use coarsening here for the FSB because a different # coarsening setting can't exist for the old wiki format connwrap.execSql( "insert into datablocksexternal(unifiedname, filepath, " "filenamelowercase, filesignature) values (?, ?, ?, ?)", (unifName.encode("utf-8"), newFilename.encode("utf-8"), newFilename.lower().encode("utf-8"), sqlite.Binary(getFileSignatureBlock(newPath)))) connwrap.execSql("delete from wikiwords where word = ?", (funcWord.encode("utf-8"), )) break except (IOError, OSError): traceback.print_exc() continue formatver = 2 # --- WikiPad 2.0alpha1 reached (formatver=2, writecompatver=2, # readcompatver=2) --- if formatver == 2: # Recreate table "todos" connwrap.execSql("drop table todos;") changeTableSchema(connwrap, "todos", TABLE_DEFINITIONS["todos"]) connwrap.execSql("update todos set firstcharpos=-1;") # Rename table "wikiwordprops" to "wikiwordattrs" changeTableSchema(connwrap, "wikiwordattrs", TABLE_DEFINITIONS["wikiwordattrs"]) connwrap.execSql("insert into wikiwordattrs(word, key, value, " "firstcharpos, charlength) select word, key, value, " "firstcharpos, -1 from wikiwordprops;") connwrap.execSql("drop table wikiwordprops;") for tn in ("wikirelations", "wikiwordmatchterms"): changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn]) connwrap.execSql("update %s set firstcharpos=-1;" % tn) # Mark all wikiwords to need a rebuild connwrap.execSql("update wikiwords set metadataprocessed=0;") formatver = 3 # --- WikiPad 2.1alpha1 reached (formatver=3, writecompatver=3, # readcompatver=3) --- connwrap.executemany( "insert or replace into settings(key, value) " + "values (?, ?)", ( ("formatver", str(VERSION_DB) ), # Version of database format the data was written ("writecompatver", str(VERSION_WRITECOMPAT) ), # Lowest format version which is write compatible ("readcompatver", str(VERSION_READCOMPAT) ), # Lowest format version which is read compatible ("branchtag", "WikidPad") # Tag of the WikidPad branch )) rebuildIndices(connwrap) connwrap.syncCommit()
def updateDatabase(connwrap, dataDir, pagefileSuffix): """ Update a database from an older version to current (checkDatabaseFormat() should have returned 1 before calling this function) """ connwrap.commit() indices = connwrap.execSqlQuerySingleColumn("select INDEX_NAME from __indices__") tables = connwrap.execSqlQuerySingleColumn("select TABLE_NAME from __table_names__") indices = map(string.upper, indices) tables = map(string.upper, tables) # updatedTables = [] if not "SETTINGS" in tables: # We are prior WikidPad 1.2beta2 (which writes format version 0) if "WIKIWORDPROPS_PKEY" in indices: # print "dropping index wikiwordprops_pkey" connwrap.execSql("drop index wikiwordprops_pkey") # if "WIKIWORDPROPS_WORD" not in indices: # print "creating index wikiwordprops_word" # connwrap.execSql("create index wikiwordprops_word on wikiwordprops(word)") # if "WIKIRELATIONS_WORD" not in indices: # print "creating index wikirelations_word" # connwrap.execSql("create index wikirelations_word on wikirelations(word)") if "REGISTRATION" in tables: connwrap.execSql("drop table registration") # Update search_views searches = connwrap.execSqlQuerySingleColumn( "select search from search_views") changeTableSchema(connwrap, "search_views", TABLE_DEFINITIONS["search_views"]) for search in searches: searchOp = SearchReplaceOperation() searchOp.searchStr = search searchOp.wikiWide = True searchOp.booleanOp = True try: # Raises exception if search is invalid searchOp.rebuildSearchOpTree() except: continue datablock = searchOp.getPackedSettings() connwrap.execSql( "insert into search_views(title, datablock) "+\ "values (?, ?)", (searchOp.getTitle(), datablock)) formatver = 0 changeTableSchema(connwrap, "settings", TABLE_DEFINITIONS["settings"]) # Write initial format versions for key, value in ( ("formatver", "0"), # Version of database format the data was written ("writecompatver", "0"), # Lowest format version which is write compatible ("readcompatver", "0"), # Lowest format version which is read compatible ("branchtag", "WikidPad") # Tag of the WikidPad branch ): setSettingsValue(connwrap, key, value) # --- WikiPad 1.20beta2 reached (formatver=0, writecompatver=0, # readcompatver=0) --- formatver = getSettingsInt(connwrap, "formatver") if formatver == 0: # From formatver 0 to 1, all filenames with brackets are renamed # to have no brackets filenames = glob.glob(join(mbcsEnc(dataDir, "replace")[0], '*.wiki')) for fn in filenames: fn = mbcsDec(fn, "replace")[0] bn = basename(fn) newbname = removeBracketsFilename(bn) if bn == newbname: continue newname = mbcsEnc(join(dataDir, newbname), "replace")[0] if exists(pathEnc(newname)): # A file with the designated new name of fn already exists # -> do nothing continue try: rename(fn, newname) except (IOError, OSError): pass formatver = 1 # --- WikiPad 1.20beta3 reached (formatver=1, writecompatver=1, # readcompatver=1) --- if formatver == 1: # remove brackets from all wikiwords in database # table wikiwords dataIn = connwrap.execSqlQuery( "select word, created, modified from wikiwords") connwrap.execSql("drop table wikiwords") connwrap.commit() changeTableSchema(connwrap, "wikiwords", TABLE_DEFINITIONS["wikiwords"]) rebuildIndices(connwrap) uniqueCtl = {} for w, c, m in dataIn: w = oldWikiWordToLabel(w) if not uniqueCtl.has_key(w): connwrap.execSqlInsert("wikiwords", ("word", "created", "modified", "presentationdatablock"), (w, c, m, "")) # connwrap.execSql("insert into wikiwords(word, created, modified) " # "values (?, ?, ?)", (w, c, m)) uniqueCtl[w] = None # table wikirelations dataIn = connwrap.execSqlQuery( "select word, relation, created from wikirelations") connwrap.execSql("drop table wikirelations") connwrap.commit() changeTableSchema(connwrap, "wikirelations", TABLE_DEFINITIONS["wikirelations"]) rebuildIndices(connwrap) uniqueCtl = {} for w, r, c in dataIn: w, r = oldWikiWordToLabel(w), oldWikiWordToLabel(r) if not uniqueCtl.has_key((w, r)): connwrap.execSqlInsert("wikirelations", ("word", "relation", "created"), (w, r, c)) # connwrap.execSql("insert into wikirelations(word, relation, created) " # "values (?, ?, ?)", (w, r, c)) uniqueCtl[(w, r)] = None # table wikiwordprops dataIn = connwrap.execSqlQuery( "select word, key, value from wikiwordprops") connwrap.execSql("drop table wikiwordprops") connwrap.commit() changeTableSchema(connwrap, "wikiwordprops", TABLE_DEFINITIONS["wikiwordprops_PRE2_1alpha01"]) rebuildIndices(connwrap) for w, k, v in dataIn: connwrap.execSqlInsert("wikiwordprops", ("word", "key", "value"), (oldWikiWordToLabel(w), k, v), tableDefault="wikiwordprops_PRE2_1alpha01") # connwrap.execSql("insert into wikiwordprops(word, key, value) " # "values (?, ?, ?)", (oldWikiWordToLabel(w), k, v)) # table todos dataIn = connwrap.execSqlQuery( "select word, todo from todos") connwrap.execSql("drop table todos") connwrap.commit() changeTableSchema(connwrap, "todos", TABLE_DEFINITIONS["todos_PRE2_1alpha01"]) rebuildIndices(connwrap) for w, t in dataIn: connwrap.execSqlInsert("todos", ("word", "todo"), (oldWikiWordToLabel(w), t), tableDefault="todos_PRE2_1alpha01") # connwrap.execSql("insert into todos(word, todo) " # "values (?, ?)", (oldWikiWordToLabel(w), t)) formatver = 2 # --- WikiPad 1.6beta2 reached (formatver=2, writecompatver=2, # readcompatver=2) --- if formatver == 2: changeTableSchema(connwrap, "wikiwords", TABLE_DEFINITIONS["wikiwords"]) # --- WikiPad 1.8beta1 reached (formatver=3, writecompatver=3, # readcompatver=2) --- formatver = 3 if formatver == 3: # Update "wikiwords" schema and create new tables for tn in ("wikiwords", "wikiwordmatchterms", "datablocks", "datablocksexternal", "defaultvalues"): changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn]) # (Re)fill "defaultvalues" and read them into connection wrapper connwrap.fillDefaultValues() connwrap.readDefaultValues() # Transfer "search_views" data to "datablocks" table searches = connwrap.execSqlQuery( "select title, datablock from search_views", strConv=(True, False)) for title, data in searches: connwrap.execSql( "insert into datablocks(unifiedname, data) "+\ "values (?, ?)", (u"savedsearch/" + title, data)) connwrap.execSql("drop table search_views") allWords = connwrap.execSqlQuerySingleColumn("select word from wikiwords") # Divide into functional and wiki pages wikiWords = [] funcWords = [] for w in allWords: if w.startswith('['): funcWords.append(w) else: wikiWords.append(w) # Fill the new fields in table "wikiwords" for wikiWord in wikiWords: filename = wikiWord + pagefileSuffix fullPath = join(dataDir, filename) try: # We don't use coarsening here for the FSB because a different # coarsening setting can't exist for the old wiki format filesig = getFileSignatureBlock(fullPath) except (IOError, WindowsError): traceback.print_exc() continue connwrap.execSql("update wikiwords set filepath = ?, " "filenamelowercase = ?, filesignature = ? " "where word = ?", (filename, filename.lower(), filesig, wikiWord)) # Move functional pages to new table "datablocksexternal" and rename them for funcWord in funcWords: if funcWord not in (u"[TextBlocks]", u"[PWL]", u"[CCBlacklist]"): continue # Error ?! unifName = u"wiki/" + funcWord[1:-1] fullPath = join(dataDir, funcWord + pagefileSuffix) icf = iterCompatibleFilename(unifName, u".data") for i in range(10): # Actual "while True", but that's too dangerous newFilename = icf.next() newPath = join(dataDir, newFilename) if exists(pathEnc(newPath)): # A file with the designated new name of fn already exists # -> do nothing continue try: rename(pathEnc(fullPath), pathEnc(newPath)) # We don't use coarsening here for the FSB because a different # coarsening setting can't exist for the old wiki format connwrap.execSqlInsert("datablocksexternal", ("unifiedname", "filepath", "filenamelowercase", "filesignature"), (unifName, newFilename, newFilename.lower(), getFileSignatureBlock(newPath))) connwrap.execSql("delete from wikiwords where word = ?", (funcWord,)) break except (IOError, OSError): traceback.print_exc() continue # --- WikiPad 2.0alpha1 reached (formatver=4, writecompatver=4, # readcompatver=4) --- formatver = 4 if formatver == 4: # (Re)fill "defaultvalues" and read them into connection wrapper connwrap.fillDefaultValues() connwrap.readDefaultValues() # Recreate table "todos" with new schema connwrap.execSql("drop table todos") changeTableSchema(connwrap, "todos", TABLE_DEFINITIONS["todos"]) # Rename table "wikiwordprops" to "wikiwordattrs" changeTableSchema(connwrap, "wikiwordattrs", TABLE_DEFINITIONS["wikiwordattrs"]) connwrap.execSql("insert into wikiwordattrs(word, key, value) " "select word, key, value from wikiwordprops") connwrap.execSql("drop table wikiwordprops") for tn in ("wikirelations", "wikiwordmatchterms"): changeTableSchema(connwrap, tn, TABLE_DEFINITIONS[tn]) # Mark all wikiwords to need a rebuild connwrap.execSql("update wikiwords set metadataprocessed=0") formatver = 5 # --- WikiPad 2.1alpha.1 reached (formatver=5, writecompatver=5, # readcompatver=5) --- # Write format information for key, value in ( ("formatver", str(VERSION_DB)), # Version of database format the data was written ("writecompatver", str(VERSION_WRITECOMPAT)), # Lowest format version which is write compatible ("readcompatver", str(VERSION_READCOMPAT)), # Lowest format version which is read compatible ("branchtag", "WikidPad"), # Tag of the WikidPad branch ("locale", "-") # Locale for cached wordnormcase column. '-': column invalid ): setSettingsValue(connwrap, key, value) rebuildIndices(connwrap) connwrap.commit()