def convertUsers(srcdir, dstdir):
""" Convert users files
@param srcdir: old users dir
@param dstdir: new users dir
"""
charset = 'utf-8'
# Create dstdir
if not os.path.exists(dstdir):
try:
os.mkdir(dstdir)
except OSError:
migutil.fatalError("can't create user directory at '%s'" % dstdir)
if not os.path.isdir(srcdir):
migutil.fatalError("can't find user directory at '%s'" % srcdir)
for name in migutil.listdir(srcdir):
if name == 'README' or name.endswith('.trail'):
# Copy as is
migutil.copy_file(join(srcdir, name), join(dstdir, name))
else:
srcfile = join(srcdir, name)
f = codecs.open(srcfile, 'rb', charset)
text = f.read()
f.close()
text = convertUserData(text)
dstfile = join(dstdir, name)
f = codecs.open(dstfile, 'wb', charset)
f.write(text)
f.close()
print "Converted '%s' to '%s'" % (srcfile, dstfile)
def convertUsers(srcdir, dstdir):
""" Convert users files
@param srcdir: old users dir
@param dstdir: new users dir
"""
charset = 'utf-8'
# Create dstdir
if not os.path.exists(dstdir):
try:
os.mkdir(dstdir)
except OSError:
migutil.fatalError("can't create user directory at '%s'" % dstdir)
if not os.path.isdir(srcdir):
migutil.fatalError("can't find user directory at '%s'" % srcdir)
for name in migutil.listdir(srcdir):
if name == 'README' or name.endswith('.trail'):
# Copy as is
migutil.copy_file(join(srcdir, name), join(dstdir, name))
else:
srcfile = join(srcdir, name)
f = codecs.open(srcfile, 'rb', charset)
text = f.read()
f.close()
text = convertUserData(text)
dstfile = join(dstdir, name)
f = codecs.open(dstfile, 'wb', charset)
f.write(text)
f.close()
print "Converted '%s' to '%s'" % (srcfile, dstfile)
def gather_pagedirs(dir_from):
""" this gathers edit-log information from the pagedirs, just to make sure
"""
pagedir = opj(dir_from, 'pages')
pagelist = listdir(pagedir)
for pagename in pagelist:
editlog_from = opj(pagedir, pagename, 'edit-log')
gather_editlog(editlog_from, pagename)
def convert_pagedir(dir_from, dir_to):
os.mkdir(dir_to)
for dname_from in listdir(dir_from):
print "%s" % (dname_from, )
dname_to = dname_from
shutil.copytree(opj(dir_from, dname_from), opj(dir_to, dname_to), 1)
try:
os.remove(opj(dir_to, dname_to, 'last-edited'))
except: pass
def process_pagedirs(dir_from, dir_to):
pagelist = listdir(dir_from)
for pagename in pagelist:
pagedir_from = opj(dir_from, pagename)
pagedir_to = opj(dir_to, pagename)
# first we copy all, even the stuff we convert later:
copy_dir(pagedir_from, pagedir_to)
rev_from = opj(pagedir_from, 'revisions')
rev_to = opj(pagedir_to, 'revisions')
if os.path.exists(rev_from):
revlist = listdir(rev_from)
for rfile in revlist:
rev = int(rfile)
r_from = opj(rev_from, rfile)
r_to = opj(rev_to, rfile)
tocrlf(r_from, r_to)
def process_pagedirs(dir_from, dir_to):
pagelist = listdir(dir_from)
for pagename in pagelist:
pagedir_from = opj(dir_from, pagename)
pagedir_to = opj(dir_to, pagename)
# first we copy all, even the stuff we convert later:
copy_dir(pagedir_from, pagedir_to)
rev_from = opj(pagedir_from, 'revisions')
rev_to = opj(pagedir_to, 'revisions')
if os.path.exists(rev_from):
revlist = listdir(rev_from)
for rfile in revlist:
rev = int(rfile)
r_from = opj(rev_from, rfile)
r_to = opj(rev_to, rfile)
tocrlf(r_from, r_to)
def convert_pagedir(dir_from, dir_to):
os.mkdir(dir_to)
for dname_from in listdir(dir_from):
print "%s" % (dname_from, )
dname_to = dname_from
shutil.copytree(opj(dir_from, dname_from), opj(dir_to, dname_to), 1)
try:
os.remove(opj(dir_to, dname_to, 'last-edited'))
except:
pass
def convert_userdir(dir_from, dir_to):
os.mkdir(dir_to)
for fname in listdir(dir_from):
if fname.endswith('.bookmark'):
bm = open(opj(dir_from, fname)).read().strip()
bm = str(wikiutil.timestamp2version(float(bm)))
f = open(opj(dir_to, fname), 'w')
f.write(bm)
f.close()
else:
copy_file(opj(dir_from, fname), opj(dir_to, fname))
def convert_userdir(dir_from, dir_to):
os.mkdir(dir_to)
for fname in listdir(dir_from):
if fname.endswith('.bookmark'):
bm = open(opj(dir_from, fname)).read().strip()
bm = str(wikiutil.timestamp2version(float(bm)))
f = open(opj(dir_to, fname), 'w')
f.write(bm)
f.close()
else:
copy_file(opj(dir_from, fname), opj(dir_to, fname))
def read_src(self):
# create Page objects in memory
pages_dir = opj(self.sdata, 'pages')
pagelist = listdir(pages_dir)
for qpagename in pagelist:
p = Page(self.request, pages_dir, qpagename)
self.pages[p.name] = p
# create User objects in memory
users_dir = opj(self.sdata, 'user')
user_re = re.compile(r'^\d+\.\d+(\.\d+)?$')
userlist = listdir(users_dir)
userlist = [f for f in userlist if user_re.match(f)]
for userid in userlist:
u = User(self.request, users_dir, userid)
self.users[u.uid] = u
# create log objects in memory
self.editlog = EditLog(self.request, opj(self.sdata, 'edit-log'))
self.eventlog = EventLog(self.request, opj(self.sdata, 'event-log'))
def read_src(self):
# create Page objects in memory
pages_dir = opj(self.sdata, 'pages')
pagelist = listdir(pages_dir)
for qpagename in pagelist:
p = Page(self.request, pages_dir, qpagename)
self.pages[p.name] = p
# create User objects in memory
users_dir = opj(self.sdata, 'user')
user_re = re.compile(r'^\d+\.\d+(\.\d+)?$')
userlist = listdir(users_dir)
userlist = [f for f in userlist if user_re.match(f)]
for userid in userlist:
u = User(self.request, users_dir, userid)
self.users[u.uid] = u
# create log objects in memory
self.editlog = EditLog(self.request, opj(self.sdata, 'edit-log'))
self.eventlog = EventLog(self.request, opj(self.sdata, 'event-log'))
def convert_pagedir(dir_from, dir_to, enc_from, enc_to):
os.mkdir(dir_to)
for dname_from in listdir(dir_from):
dname_to = qf_convert_string(dname_from, enc_from, enc_to)
print "%s -> %s" % (dname_from, dname_to)
shutil.copytree(opj(dir_from, dname_from), opj(dir_to, dname_to), 1)
try:
convert_editlog(opj(dir_from, dname_from, 'last-edited'),
opj(dir_to, dname_to, 'last-edited'),
enc_from, enc_to)
except IOError:
pass # we ignore if it doesnt exist
def convert_pagedir(dir_from, dir_to, is_backupdir=0):
os.mkdir(dir_to)
for pagedir in listdir(dir_from):
text_from = opj(dir_from, pagedir, 'text')
text_to = opj(dir_to, pagedir, 'text')
os.mkdir(opj(dir_to, pagedir))
copy_file(text_from, text_to)
backupdir_from = opj(dir_from, pagedir, 'backup')
backupdir_to = opj(dir_to, pagedir, 'backup')
if os.path.exists(backupdir_from):
os.mkdir(backupdir_to)
for ts in listdir(backupdir_from):
ts_usec = str(convert_ts(float(ts)))
backup_from = opj(backupdir_from, ts)
backup_to = opj(backupdir_to, ts_usec)
copy_file(backup_from, backup_to)
editlog_from = opj(dir_from, pagedir, 'edit-log')
editlog_to = opj(dir_to, pagedir, 'edit-log')
convert_editlog(editlog_from, editlog_to)
#cachedir_from = opj(dir_from, pagedir, 'cache')
#cachedir_to = opj(dir_to, pagedir, 'cache')
#if os.path.exists(cachedir_from):
# os.mkdir(cachedir_to)
# try:
# copy_file(
# opj(cachedir_from, 'hitcounts'),
# opj(cachedir_to, 'hitcounts'))
# except: pass
attachdir_from = opj(dir_from, pagedir, 'attachments')
attachdir_to = opj(dir_to, pagedir, 'attachments')
if os.path.exists(attachdir_from):
try:
copy_dir(attachdir_from, attachdir_to)
except:
pass
def read(self):
""" read a page, including revisions, log, attachments from disk """
page_dir = self.page_dir
# read current file
current_fname = opj(page_dir, 'current')
if os.path.exists(current_fname):
current_file = file(current_fname, "r")
current_rev = current_file.read()
current_file.close()
try:
self.current = int(current_rev)
except ValueError:
print "Error: invalid current file %s, SKIPPING THIS PAGE!" % current_fname
return
# read edit-log
editlog_fname = opj(page_dir, 'edit-log')
if os.path.exists(editlog_fname):
self.editlog = EditLog(self.request, editlog_fname)
# read page revisions
rev_dir = opj(page_dir, 'revisions')
if os.path.exists(rev_dir):
revlist = listdir(rev_dir)
revlist = [int(rev) for rev in revlist]
revlist.sort()
self.revlist = revlist
self.revisions = {}
for rev in revlist:
self.revisions[rev] = PageRev(self.request, self.name_old,
rev_dir, rev)
# set deleted status
self.is_deleted = not self.revisions or self.current not in self.revisions
# read attachment filenames
attach_dir = opj(page_dir, 'attachments')
if os.path.exists(attach_dir):
self.attachments = {}
attlist = listdir(attach_dir)
for attfile in attlist:
a = Attachment(self.request, attach_dir, attfile)
self.attachments[a.name] = a
def convert_pagedir(dir_from, dir_to, is_backupdir=0):
os.mkdir(dir_to)
for pagedir in listdir(dir_from):
text_from = opj(dir_from, pagedir, 'text')
text_to = opj(dir_to, pagedir, 'text')
os.mkdir(opj(dir_to, pagedir))
copy_file(text_from, text_to)
backupdir_from = opj(dir_from, pagedir, 'backup')
backupdir_to = opj(dir_to, pagedir, 'backup')
if os.path.exists(backupdir_from):
os.mkdir(backupdir_to)
for ts in listdir(backupdir_from):
ts_usec = str(convert_ts(float(ts)))
backup_from = opj(backupdir_from, ts)
backup_to = opj(backupdir_to, ts_usec)
copy_file(backup_from, backup_to)
editlog_from = opj(dir_from, pagedir, 'edit-log')
editlog_to = opj(dir_to, pagedir, 'edit-log')
convert_editlog(editlog_from, editlog_to)
#cachedir_from = opj(dir_from, pagedir, 'cache')
#cachedir_to = opj(dir_to, pagedir, 'cache')
#if os.path.exists(cachedir_from):
# os.mkdir(cachedir_to)
# try:
# copy_file(
# opj(cachedir_from, 'hitcounts'),
# opj(cachedir_to, 'hitcounts'))
# except: pass
attachdir_from = opj(dir_from, pagedir, 'attachments')
attachdir_to = opj(dir_to, pagedir, 'attachments')
if os.path.exists(attachdir_from):
try:
copy_dir(attachdir_from, attachdir_to)
except: pass
def read(self):
""" read a page, including revisions, log, attachments from disk """
page_dir = self.page_dir
# read current file
current_fname = opj(page_dir, 'current')
if os.path.exists(current_fname):
current_file = file(current_fname, "r")
current_rev = current_file.read()
current_file.close()
try:
self.current = int(current_rev)
except ValueError:
print "Error: invalid current file %s, SKIPPING THIS PAGE!" % current_fname
return
# read edit-log
editlog_fname = opj(page_dir, 'edit-log')
if os.path.exists(editlog_fname):
self.editlog = EditLog(self.request, editlog_fname)
# read page revisions
rev_dir = opj(page_dir, 'revisions')
if os.path.exists(rev_dir):
revlist = listdir(rev_dir)
revlist = [int(rev) for rev in revlist]
revlist.sort()
self.revlist = revlist
self.revisions = {}
for rev in revlist:
self.revisions[rev] = PageRev(self.request, self.name_old, rev_dir, rev)
# set deleted status
self.is_deleted = not self.revisions or self.current not in self.revisions
# read attachment filenames
attach_dir = opj(page_dir, 'attachments')
if os.path.exists(attach_dir):
self.attachments = {}
attlist = listdir(attach_dir)
for attfile in attlist:
a = Attachment(self.request, attach_dir, attfile)
self.attachments[a.name] = a
def convert_textdir(dir_from, dir_to, enc_from, enc_to, is_backupdir=0):
os.mkdir(dir_to)
for fname_from in listdir(dir_from):
if is_backupdir:
fname, timestamp = fname_from.split('.')
else:
fname = fname_from
fname = qf_convert_string(fname, enc_from, enc_to)
if is_backupdir:
fname_to = '.'.join([fname, timestamp])
else:
fname_to = fname
convert_file(opj(dir_from, fname_from), opj(dir_to, fname_to),
enc_from, enc_to)
def convert_textdir(dir_from, dir_to, is_backupdir=0):
for fname_from in listdir(dir_from):
if is_backupdir:
fname, timestamp = fname_from.split('.')
else:
fname = fname_from
try:
os.mkdir(opj(dir_to, 'pages', fname))
except: pass
try:
os.mkdir(opj(dir_to, 'pages', fname, 'backup'))
except: pass
try:
os.mkdir(opj(dir_to, 'pages', fname, 'cache'))
except: pass
if is_backupdir:
fname_to = opj('pages', fname, 'backup', timestamp)
else:
fname_to = opj('pages', fname, 'text')
copy_file(opj(dir_from, fname_from), opj(dir_to, fname_to))
def migrate(dir_to):
""" this removes edit-lock files from the pagedirs and
converts attachment filenames
"""
pagesdir = opj(dir_to, 'pages')
pagelist = listdir(pagesdir)
for pagename in pagelist:
pagedir = opj(pagesdir, pagename)
editlock = opj(pagedir, 'edit-lock')
try:
os.remove(editlock)
except:
pass
attachdir = os.path.join(pagedir, 'attachments')
for root, dirs, files in os.walk(attachdir):
for f in files:
try:
f.decode(to_encoding)
except UnicodeDecodeError:
fnew = f.decode(from_encoding).encode(to_encoding)
os.rename(os.path.join(root, f), os.path.join(root, fnew))
print 'renamed', f, '\n ->', fnew, ' in dir:', root
def convert_textdir(dir_from, dir_to, is_backupdir=0):
for fname_from in listdir(dir_from):
if is_backupdir:
fname, timestamp = fname_from.split('.')
else:
fname = fname_from
try:
os.mkdir(opj(dir_to, 'pages', fname))
except:
pass
try:
os.mkdir(opj(dir_to, 'pages', fname, 'backup'))
except:
pass
try:
os.mkdir(opj(dir_to, 'pages', fname, 'cache'))
except:
pass
if is_backupdir:
fname_to = opj('pages', fname, 'backup', timestamp)
else:
fname_to = opj('pages', fname, 'text')
copy_file(opj(dir_from, fname_from), opj(dir_to, fname_to))
def gather_pagedirs(dir_from, is_backupdir=0):
""" this gathers information from the pagedirs, i.e. text and backup
files (and also the local editlog) and tries to merge/synchronize
with the informations gathered from editlog
"""
global pagelist
pagelist = listdir(dir_from)
for pagename in pagelist:
editlog_from = opj(dir_from, pagename, 'edit-log')
gather_editlog(dir_from, editlog_from)
entry = info.get(pagename, {})
loglist = [] # editlog timestamps of page revisions
for ts,data in entry.items():
if data[1][2] in ['SAVE','SAVENEW','SAVE/REVERT',]:
loglist.append(ts)
loglist.sort()
lleftover = loglist[:]
# remember the latest log entry
if lleftover:
llatest = lleftover[-1]
else:
llatest = None
backupdir_from = opj(dir_from, pagename, 'backup')
if os.path.exists(backupdir_from):
backuplist = listdir(backupdir_from)
bleftover = backuplist[:]
for bfile in backuplist:
backup_from = opj(backupdir_from, bfile)
ts = long(bfile)
if ts in loglist: # we have an editlog entry, exact match
entry[ts][0] = backup_from
lleftover.remove(ts)
bleftover.remove(bfile)
text_from = opj(dir_from, pagename, 'text')
found_text = False
if os.path.exists(text_from): # we have a text file, it should match latest log entry
exists[pagename] = True
mtime = os.path.getmtime(text_from)
if llatest and llatest in lleftover:
ts = llatest
if abs(wikiutil.timestamp2version(mtime) - ts) < 2000000: # less than a second diff
entry[ts][0] = text_from
lleftover.remove(ts)
found_text = True
else: # we have no log entries left 8(
ts = wikiutil.timestamp2version(mtime)
data = [ts,'','SAVE', pagename,'','','','','missing editlog entry for this page version']
entry[ts] = [text_from, data]
else:
# this page was maybe deleted, so we remember for later:
exists[pagename] = False
if llatest in lleftover: # if a page is deleted, the last log entry has no file
entry[llatest][0] = None
lleftover.remove(llatest)
if os.path.exists(backupdir_from):
backuplist = listdir(backupdir_from)
for bfile in backuplist:
if not bfile in bleftover: continue
backup_from = opj(backupdir_from, bfile)
bts = long(bfile) # must be long for py 2.2.x
for ts in lleftover:
tdiff = abs(bts-ts)
if tdiff < 2000000: # editlog, inexact match
entry[ts][0] = backup_from
lleftover.remove(ts)
bleftover.remove(bfile)
elif 3599000000 <= tdiff <= 3601000000: # editlog, win32 daylight saving bug
entry[ts][0] = backup_from
lleftover.remove(ts)
bleftover.remove(bfile)
print "Warning: Win32 daylight saving bug encountered & fixed!"
if len(bleftover) == 1 and len(lleftover) == 1: # only 1 left, must be this
backup_from = opj(backupdir_from, bleftover[0])
entry[lleftover[0]][0] = backup_from
lleftover = []
bleftover = []
# fake some log entries
for bfile in bleftover:
backup_from = opj(backupdir_from, bfile)
bts = long(bfile) # must be long py 2.2.x
data = [ts,'','SAVE',pagename,'','','','','missing editlog entry for this page version']
entry[bts] = [backup_from, data]
# check if we still haven't matched the "text" file
if not found_text and os.path.exists(text_from):
if llatest in lleftover: # latest log entry still free
entry[llatest][0] = text_from # take it. do not care about mtime of file.
lleftover.remove(llatest)
else: # log for "text" file is missing or latest was taken by other rev 8(
mtime = os.path.getmtime(text_from)
ts = wikiutil.timestamp2version(mtime) # take mtime, we have nothing better
data = [ts,'','SAVE', pagename,'','','','','missing editlog entry for this page version']
entry[ts] = [text_from, data]
# delete unmatching log entries
for ts in lleftover:
#print "XXX Deleting leftover log entry: %r" % entry[ts]
del entry[ts]
info[pagename] = entry
def gather_pagedirs(dir_from, is_backupdir=0):
""" this gathers information from the pagedirs, i.e. text and backup
files (and also the local editlog) and tries to merge/synchronize
with the informations gathered from editlog
"""
global pagelist
pagelist = listdir(dir_from)
for pagename in pagelist:
editlog_from = opj(dir_from, pagename, 'edit-log')
gather_editlog(dir_from, editlog_from)
entry = info.get(pagename, {})
loglist = [] # editlog timestamps of page revisions
for ts, data in entry.items():
if data[1][2] in [
'SAVE',
'SAVENEW',
'SAVE/REVERT',
]:
loglist.append(ts)
loglist.sort()
lleftover = loglist[:]
# remember the latest log entry
if lleftover:
llatest = lleftover[-1]
else:
llatest = None
backupdir_from = opj(dir_from, pagename, 'backup')
if os.path.exists(backupdir_from):
backuplist = listdir(backupdir_from)
bleftover = backuplist[:]
for bfile in backuplist:
backup_from = opj(backupdir_from, bfile)
ts = long(bfile)
if ts in loglist: # we have an editlog entry, exact match
entry[ts][0] = backup_from
lleftover.remove(ts)
bleftover.remove(bfile)
text_from = opj(dir_from, pagename, 'text')
found_text = False
if os.path.exists(
text_from
): # we have a text file, it should match latest log entry
exists[pagename] = True
mtime = os.path.getmtime(text_from)
if llatest and llatest in lleftover:
ts = llatest
if abs(wikiutil.timestamp2version(mtime) -
ts) < 2000000: # less than a second diff
entry[ts][0] = text_from
lleftover.remove(ts)
found_text = True
else: # we have no log entries left 8(
ts = wikiutil.timestamp2version(mtime)
data = [
ts, '', 'SAVE', pagename, '', '', '', '',
'missing editlog entry for this page version'
]
entry[ts] = [text_from, data]
else:
# this page was maybe deleted, so we remember for later:
exists[pagename] = False
if llatest in lleftover: # if a page is deleted, the last log entry has no file
entry[llatest][0] = None
lleftover.remove(llatest)
if os.path.exists(backupdir_from):
backuplist = listdir(backupdir_from)
for bfile in backuplist:
if not bfile in bleftover: continue
backup_from = opj(backupdir_from, bfile)
bts = long(bfile) # must be long for py 2.2.x
for ts in lleftover:
tdiff = abs(bts - ts)
if tdiff < 2000000: # editlog, inexact match
entry[ts][0] = backup_from
lleftover.remove(ts)
bleftover.remove(bfile)
elif 3599000000 <= tdiff <= 3601000000: # editlog, win32 daylight saving bug
entry[ts][0] = backup_from
lleftover.remove(ts)
bleftover.remove(bfile)
print "Warning: Win32 daylight saving bug encountered & fixed!"
if len(bleftover) == 1 and len(
lleftover) == 1: # only 1 left, must be this
backup_from = opj(backupdir_from, bleftover[0])
entry[lleftover[0]][0] = backup_from
lleftover = []
bleftover = []
# fake some log entries
for bfile in bleftover:
backup_from = opj(backupdir_from, bfile)
bts = long(bfile) # must be long py 2.2.x
data = [
ts, '', 'SAVE', pagename, '', '', '', '',
'missing editlog entry for this page version'
]
entry[bts] = [backup_from, data]
# check if we still haven't matched the "text" file
if not found_text and os.path.exists(text_from):
if llatest in lleftover: # latest log entry still free
entry[llatest][
0] = text_from # take it. do not care about mtime of file.
lleftover.remove(llatest)
else: # log for "text" file is missing or latest was taken by other rev 8(
mtime = os.path.getmtime(text_from)
ts = wikiutil.timestamp2version(
mtime) # take mtime, we have nothing better
data = [
ts, '', 'SAVE', pagename, '', '', '', '',
'missing editlog entry for this page version'
]
entry[ts] = [text_from, data]
# delete unmatching log entries
for ts in lleftover:
#print "XXX Deleting leftover log entry: %r" % entry[ts]
del entry[ts]
info[pagename] = entry
def convert_userdir(dir_from, dir_to, enc_from, enc_to):
os.mkdir(dir_to)
for fname in listdir(dir_from):
convert_file(opj(dir_from, fname), opj(dir_to, fname),
enc_from, enc_to)