def remove_trash(dir_from):
for pagename in info:
# omit dead pages and MoinEditorBackup
if pagename in pagelist and (
os.path.exists(opj(dir_from, pagename, 'text')) or
os.path.exists(opj(dir_from, pagename, 'backup'))
) and not pagename.endswith('MoinEditorBackup'):
info2[pagename] = info[pagename]
def gather_pagedirs(dir_from):
""" this gathers edit-log information from the pagedirs, just to make sure
"""
pagedir = opj(dir_from, 'pages')
pagelist = listdir(pagedir)
for pagename in pagelist:
editlog_from = opj(pagedir, pagename, 'edit-log')
gather_editlog(editlog_from, pagename)
def convert_pagedir(dir_from, dir_to):
os.mkdir(dir_to)
for dname_from in listdir(dir_from):
print "%s" % (dname_from,)
dname_to = dname_from
shutil.copytree(opj(dir_from, dname_from), opj(dir_to, dname_to), 1)
try:
os.remove(opj(dir_to, dname_to, "last-edited"))
except:
pass
def convert_userdir(dir_from, dir_to):
os.mkdir(dir_to)
for fname in listdir(dir_from):
if fname.endswith('.bookmark'):
bm = open(opj(dir_from, fname)).read().strip()
bm = str(wikiutil.timestamp2version(float(bm)))
f = open(opj(dir_to, fname), 'w')
f.write(bm)
f.close()
else:
copy_file(opj(dir_from, fname), opj(dir_to, fname))
def convert_pagedir(dir_from, dir_to, enc_from, enc_to):
os.mkdir(dir_to)
for dname_from in listdir(dir_from):
dname_to = qf_convert_string(dname_from, enc_from, enc_to)
print "%s -> %s" % (dname_from, dname_to)
shutil.copytree(opj(dir_from, dname_from), opj(dir_to, dname_to), 1)
try:
convert_editlog(opj(dir_from, dname_from, 'last-edited'),
opj(dir_to, dname_to, 'last-edited'),
enc_from, enc_to)
except IOError:
pass # we ignore if it doesnt exist
def convert_textdir(dir_from, dir_to, enc_from, enc_to, is_backupdir=0):
os.mkdir(dir_to)
for fname_from in listdir(dir_from):
if is_backupdir:
fname, timestamp = fname_from.split('.')
else:
fname = fname_from
fname = qf_convert_string(fname, enc_from, enc_to)
if is_backupdir:
fname_to = '.'.join([fname, timestamp])
else:
fname_to = fname
convert_file(opj(dir_from, fname_from), opj(dir_to, fname_to),
enc_from, enc_to)
def migrate(dir_to):
""" this removes edit-lock files from the pagedirs and
converts attachment filenames
"""
pagesdir = opj(dir_to, 'pages')
pagelist = listdir(pagesdir)
for pagename in pagelist:
pagedir = opj(pagesdir, pagename)
editlock = opj(pagedir, 'edit-lock')
try:
os.remove(editlock)
except:
pass
attachdir = os.path.join(pagedir, 'attachments')
for root, dirs, files in walk(attachdir):
for f in files:
try:
f.decode(to_encoding)
except UnicodeDecodeError:
fnew = f.decode(from_encoding).encode(to_encoding)
os.rename(os.path.join(root,f), os.path.join(root, fnew))
print 'renamed', f, '\n ->', fnew, ' in dir:', root
def migrate(destdir):
plugindir = opj(destdir, 'plugin')
makedir(plugindir)
fname = opj(plugindir, '__init__.py')
f = open(fname, 'w')
f.write('''\
# *** Do not remove this! ***
# Although being empty, the presence of this file is important for plugins
# working correctly.
''')
f.close()
for d in ['action', 'formatter', 'macro', 'parser', 'processor', 'theme', 'xmlrpc', ]:
thisdir = opj(plugindir, d)
makedir(thisdir)
fname = opj(thisdir, '__init__.py')
f = open(fname, 'w')
f.write('''\
# -*- coding: iso-8859-1 -*-
from MoinMoin.util import pysupport
modules = pysupport.getPackageModules(__file__)
''')
f.close()
def generate_editlog(dir_from, dir_to):
editlog = {}
for pagename in info2:
entry = info2.get(pagename, {})
for ts in entry:
file_from, data = entry[ts]
editlog[ts] = data
tslist = editlog.keys()
tslist.sort()
editlog_file = opj(dir_to, 'edit-log')
f = open(editlog_file, 'w')
for ts in tslist:
data = editlog[ts]
f.write('\t'.join(data)+'\n')
f.close()
def convert_editlog(file_from, file_to, dir_to):
for l in open(file_from):
data = l.split("\t")
pagename = data[0]
timestamp = data[2]
data[2] = str(long(float(timestamp))) # we only want integer (must be long for py 2.2.x)
data = "\t".join(data)
f = open(file_to, "a")
f.write(data)
f.close()
try:
file_to2 = opj(dir_to, pagename, "edit-log")
f = open(file_to2, "a")
f.write(data)
f.close()
except:
pass
def generate_pages(dir_from, dir_to):
for pagename in info2:
entry = info2.get(pagename, {})
tslist = entry.keys()
if tslist:
pagedir = opj(dir_to, 'pages', pagename)
os.makedirs(opj(pagedir, 'revisions'))
editlog_file = opj(pagedir, 'edit-log')
f = open(editlog_file, 'w')
rev = 0
tslist.sort()
for ts in tslist:
rev += 1
revstr = '%08d' % rev
file_from, data = entry[ts]
data[0] = str(ts)
data[1] = revstr
if data[2].endswith('/REVERT'):
# replace the timestamp with the revision number
revertts = long(data[7]) # must be long for py 2.2.x
try:
revertrev = int(entry[revertts][1][1])
except KeyError:
# never should trigger...
print "********* KeyError %s entry[%d][1][1] **********" % (pagename, revertts)
revertrev = 0
data[7] = '%08d' % revertrev
f.write('\t'.join(data)+'\n')
if file_from is not None:
file_to = opj(pagedir, 'revisions', revstr)
copy_file(file_from, file_to)
f.close()
curr_file = opj(pagedir, 'current')
f = open(curr_file, 'w')
f.write(revstr)
f.close()
att_from = opj(dir_from, 'pages', pagename, 'attachments')
if os.path.exists(att_from):
att_to = opj(pagedir, 'attachments')
copy_dir(att_from, att_to)
def process_pagedirs(dir_from, dir_to):
pagelist = listdir(dir_from)
for pagename in pagelist:
pagedir_from = opj(dir_from, pagename)
pagedir_to = opj(dir_to, pagename)
# first we copy all, even the stuff we convert later:
copy_dir(pagedir_from, pagedir_to)
rev_from = opj(pagedir_from, 'revisions')
rev_to = opj(pagedir_to, 'revisions')
if os.path.exists(rev_from):
revlist = listdir(rev_from)
for rfile in revlist:
rev = int(rfile)
r_from = opj(rev_from, rfile)
r_to = opj(rev_to, rfile)
tocrlf(r_from, r_to)
def convert_textdir(dir_from, dir_to, is_backupdir=0):
for fname_from in listdir(dir_from):
if is_backupdir:
fname, timestamp = fname_from.split(".")
else:
fname = fname_from
try:
os.mkdir(opj(dir_to, "pages", fname))
except:
pass
try:
os.mkdir(opj(dir_to, "pages", fname, "backup"))
except:
pass
try:
os.mkdir(opj(dir_to, "pages", fname, "cache"))
except:
pass
if is_backupdir:
fname_to = opj("pages", fname, "backup", timestamp)
else:
fname_to = opj("pages", fname, "text")
copy_file(opj(dir_from, fname_from), opj(dir_to, fname_to))
data = editlog[ts]
f.write('\t'.join(data)+'\n')
f.close()
origdir = 'data.pre-mig5'
# Backup original dir and create new empty dir
try:
os.rename('data', origdir)
os.mkdir('data')
except OSError:
print "You need to be in the directory where your copy of the 'data' directory is located."
sys.exit(1)
gather_editlog(origdir, opj(origdir, 'edit-log'))
gather_pagedirs(opj(origdir, 'pages'))
remove_trash(opj(origdir, 'pages'))
generate_pages(origdir, 'data')
generate_editlog(origdir, 'data')
copy_dir(opj(origdir, 'plugin'), opj('data', 'plugin'))
copy_dir(opj(origdir, 'user'), opj('data', 'user'))
copy_file(opj(origdir, 'event-log'), opj('data', 'event-log'))
copy_file(opj(origdir, 'intermap.txt'), opj('data', 'intermap.txt'))
revlist = listdir(rev_from)
for rfile in revlist:
rev = int(rfile)
r_from = opj(rev_from, rfile)
r_to = opj(rev_to, rfile)
tocrlf(r_from, r_to)
origdir = 'data.pre-mig7'
try:
os.rename('data', origdir)
except OSError:
print "You need to be in the directory where your copy of the 'data' directory is located."
sys.exit(1)
os.makedirs(opj('data','pages'))
process_pagedirs(opj(origdir, 'pages'), opj('data', 'pages'))
copy_dir(opj(origdir, 'plugin'), opj('data', 'plugin'))
copy_dir(opj(origdir, 'user'), opj('data', 'user'))
copy_file(opj(origdir, 'edit-log'), opj('data', 'edit-log'))
copy_file(opj(origdir, 'event-log'), opj('data', 'event-log'))
copy_file(opj(origdir, 'intermap.txt'), opj('data', 'intermap.txt'))
def convert_userdir(dir_from, dir_to, enc_from, enc_to):
os.mkdir(dir_to)
for fname in listdir(dir_from):
convert_file(opj(dir_from, fname), opj(dir_to, fname),
enc_from, enc_to)
datatmp[1] = '%08d' % rev
f.write('\t'.join(datatmp)+'\n')
f.close()
origdir = 'data.pre-mig8'
# Backup original dir and create new empty dir
try:
os.rename('data', origdir)
os.mkdir('data')
except OSError:
print "You need to be in the directory where your copy of the 'data' directory is located."
sys.exit(1)
#gather_editlog(opj(origdir, 'edit-log'))
gather_pagedirs(origdir)
generate_editlog(origdir, 'data')
generate_pages(origdir, 'data')
copy_dir(opj(origdir, 'plugin'), opj('data', 'plugin'))
copy_dir(opj(origdir, 'user'), opj('data', 'user'))
copy_file(opj(origdir, 'event-log'), opj('data', 'event-log'))
copy_file(opj(origdir, 'intermap.txt'), opj('data', 'intermap.txt'))
def generate_pages(dir_from, dir_to):
revactions = ['SAVE','SAVENEW','SAVE/REVERT',] # these actions create revisions
for pn in info:
entry = info.get(pn, {})
tslist = entry.keys()
if tslist:
pagedir = opj(dir_to, 'pages', pn)
revdir = opj(pagedir, 'revisions')
os.makedirs(revdir)
editlog_file = opj(pagedir, 'edit-log')
f = open(editlog_file, 'w')
revnew = 0
tslist.sort()
for ts in tslist:
data = entry[ts][1]
datanew = data[:]
(timestamp,rev,action,pagename,ip,host,id,extra,comment) = data
revstr = '%08d' % rev
if action in revactions:
revnew += 1
revnewstr = '%08d' % revnew
entry[ts][0] = revnew # remember what new revno we chose
else: # ATTNEW,ATTDRW,ATTDEL
revnewstr = '99999999'
if action.endswith('/REVERT'):
# replace the old revno with the correct new revno
revertrevold = int(extra)
revertrevnew = 0
for ts2 in tslist:
data2 = entry[ts2][1]
(timestamp2,rev2,action2,pagename2,ip2,host2,id2,extra2,comment2) = data2
if rev2 == revertrevold:
revertrevnew = entry[ts2][0]
datanew[7] = '%08d' % revertrevnew
datanew[1] = revnewstr
f.write('\t'.join(datanew)+'\n') # does make a CRLF on win32 in the file
if action in revactions: # we DO have a page rev for this one
file_from = opj(dir_from, 'pages', pn, 'revisions', revstr)
file_to = opj(revdir, revnewstr)
copy_file(file_from, file_to)
f.close()
# check if page exists or is deleted in orig dir
pagedir_from = opj(dir_from, 'pages', pn)
revdir_from = opj(pagedir_from, 'revisions')
try:
curr_file_from = opj(pagedir_from, 'current')
currentfrom = open(curr_file_from).read().strip() # try to access it
page_exists = 1
except:
page_exists = 0
# re-make correct DELETED status!
if page_exists:
curr_file = opj(pagedir, 'current')
f = open(curr_file, 'w')
f.write("%08d\n" % revnew) # we add a \n, so it is easier to hack in there manually
f.close()
att_from = opj(dir_from, 'pages', pn, 'attachments')
if os.path.exists(att_from):
att_to = opj(pagedir, 'attachments')
copy_dir(att_from, att_to)
fields[0] = qf_convert_string(fields[0], enc_from, enc_to)
fields[5] = convert_string(fields[5], enc_from, enc_to)
line = '\t'.join(fields)
file_to.write(line)
origdir = 'data.pre-mig2'
# Backup original dir and create new empty dir
try:
os.rename('data', origdir)
os.mkdir('data')
except OSError:
print "You need to be in the directory where your copy of the 'data' directory is located."
sys.exit(1)
convert_textdir(opj(origdir, 'text'), opj('data', 'text'), from_encoding, to_encoding)
convert_textdir(opj(origdir, 'backup'), opj('data', 'backup'), from_encoding, to_encoding, 1)
convert_pagedir(opj(origdir, 'pages'), opj('data', 'pages'), from_encoding, to_encoding)
convert_userdir(opj(origdir, 'user'), opj('data', 'user'), from_encoding, to_encoding)
convert_editlog(opj(origdir, 'editlog'), opj('data', 'editlog'), from_encoding, to_encoding)
copy_file(opj(origdir, 'event.log'), opj('data', 'event.log'))
copy_dir(opj(origdir, 'plugin'), opj('data', 'plugin'))
copy_file(opj(origdir, 'intermap.txt'), opj('data', 'intermap.txt'))
def convert_pagedir(dir_from, dir_to, is_backupdir=0):
os.mkdir(dir_to)
for pagedir in listdir(dir_from):
text_from = opj(dir_from, pagedir, 'text')
text_to = opj(dir_to, pagedir, 'text')
os.mkdir(opj(dir_to, pagedir))
copy_file(text_from, text_to)
backupdir_from = opj(dir_from, pagedir, 'backup')
backupdir_to = opj(dir_to, pagedir, 'backup')
if os.path.exists(backupdir_from):
os.mkdir(backupdir_to)
for ts in listdir(backupdir_from):
ts_usec = str(convert_ts(float(ts)))
backup_from = opj(backupdir_from, ts)
backup_to = opj(backupdir_to, ts_usec)
copy_file(backup_from, backup_to)
editlog_from = opj(dir_from, pagedir, 'edit-log')
editlog_to = opj(dir_to, pagedir, 'edit-log')
convert_editlog(editlog_from, editlog_to)
#cachedir_from = opj(dir_from, pagedir, 'cache')
#cachedir_to = opj(dir_to, pagedir, 'cache')
#if os.path.exists(cachedir_from):
# os.mkdir(cachedir_to)
# try:
# copy_file(
# opj(cachedir_from, 'hitcounts'),
# opj(cachedir_to, 'hitcounts'))
# except: pass
attachdir_from = opj(dir_from, pagedir, 'attachments')
attachdir_to = opj(dir_to, pagedir, 'attachments')
if os.path.exists(attachdir_from):
try:
copy_dir(attachdir_from, attachdir_to)
except: pass
f = open(file_to2, "a")
f.write(data)
f.close()
except:
pass
# Backup original dir and create new empty dir
try:
os.rename("data", origdir)
os.mkdir("data")
except OSError:
print "You need to be in the directory where your copy of the 'data' directory is located."
sys.exit(1)
convert_pagedir(opj(origdir, "pages"), opj("data", "pages"))
convert_textdir(opj(origdir, "text"), "data")
convert_textdir(opj(origdir, "backup"), "data", 1)
convert_editlog(opj(origdir, "editlog"), opj("data", "edit-log"), opj("data", "pages"))
copy_file(opj(origdir, "event.log"), opj("data", "event.log"))
copy_dir(opj(origdir, "plugin"), opj("data", "plugin"))
copy_dir(opj(origdir, "user"), opj("data", "user"))
copy_file(opj(origdir, "intermap.txt"), opj("data", "intermap.txt"))
bm = str(wikiutil.timestamp2version(float(bm)))
f = open(opj(dir_to, fname), 'w')
f.write(bm)
f.close()
else:
copy_file(opj(dir_from, fname), opj(dir_to, fname))
origdir = 'data.pre-mig4'
# Backup original dir and create new empty dir
try:
os.rename('data', origdir)
os.mkdir('data')
except OSError:
print "You need to be in the directory where your copy of the 'data' directory is located."
sys.exit(1)
convert_pagedir(opj(origdir, 'pages'), opj('data', 'pages'))
convert_editlog(opj(origdir, 'edit-log'), opj('data', 'edit-log'))
convert_eventlog(opj(origdir, 'event.log'), opj('data', 'event-log'))
convert_userdir(opj(origdir, 'user'), opj('data', 'user'))
copy_dir(opj(origdir, 'plugin'), opj('data', 'plugin'))
copy_file(opj(origdir, 'intermap.txt'), opj('data', 'intermap.txt'))
key, val = kvpair.split('=')
key = urllib.unquote(key)
val = urllib.unquote(val)
key = convert_string(key, enc_from, enc_to)
val = convert_string(val, enc_from, enc_to)
key = urllib.quote(key)
val = urllib.quote(val)
kvlist.append("%s=%s" % (key,val))
fields[2] = '&'.join(kvlist)
line = '\t'.join(fields) + '\n'
file_to.write(line)
file_to.close()
file_from.close()
st=os.stat(fname_from)
os.utime(fname_to, (st.st_atime,st.st_mtime))
origdir = 'data.pre-mig6'
try:
os.rename('data', origdir)
except OSError:
print "You need to be in the directory where your copy of the 'data' directory is located."
sys.exit(1)
copy_dir(origdir, 'data')
os.remove(opj('data','event-log')) # old format
convert_eventlog(opj(origdir, 'event-log'), opj('data', 'event-log'), from_encoding, to_encoding)
def gather_pagedirs(dir_from, is_backupdir=0):
""" this gathers information from the pagedirs, i.e. text and backup
files (and also the local editlog) and tries to merge/synchronize
with the informations gathered from editlog
"""
global pagelist
pagelist = listdir(dir_from)
for pagename in pagelist:
editlog_from = opj(dir_from, pagename, 'edit-log')
gather_editlog(dir_from, editlog_from)
entry = info.get(pagename, {})
loglist = [] # editlog timestamps of page revisions
for ts,data in entry.items():
if data[1][2] in ['SAVE','SAVENEW','SAVE/REVERT',]:
loglist.append(ts)
loglist.sort()
lleftover = loglist[:]
# remember the latest log entry
if lleftover:
llatest = lleftover[-1]
else:
llatest = None
backupdir_from = opj(dir_from, pagename, 'backup')
if os.path.exists(backupdir_from):
backuplist = listdir(backupdir_from)
bleftover = backuplist[:]
for bfile in backuplist:
backup_from = opj(backupdir_from, bfile)
ts = long(bfile)
if ts in loglist: # we have an editlog entry, exact match
entry[ts][0] = backup_from
lleftover.remove(ts)
bleftover.remove(bfile)
text_from = opj(dir_from, pagename, 'text')
found_text = False
if os.path.exists(text_from): # we have a text file, it should match latest log entry
exists[pagename] = True
mtime = os.path.getmtime(text_from)
if llatest and llatest in lleftover:
ts = llatest
if abs(wikiutil.timestamp2version(mtime) - ts) < 2000000: # less than a second diff
entry[ts][0] = text_from
lleftover.remove(ts)
found_text = True
else: # we have no log entries left 8(
ts = wikiutil.timestamp2version(mtime)
data = [ts,'','SAVE', pagename,'','','','','missing editlog entry for this page version']
entry[ts] = [text_from, data]
else:
# this page was maybe deleted, so we remember for later:
exists[pagename] = False
if llatest in lleftover: # if a page is deleted, the last log entry has no file
entry[llatest][0] = None
lleftover.remove(llatest)
if os.path.exists(backupdir_from):
backuplist = listdir(backupdir_from)
for bfile in backuplist:
if not bfile in bleftover: continue
backup_from = opj(backupdir_from, bfile)
bts = long(bfile) # must be long for py 2.2.x
for ts in lleftover:
if abs(bts-ts) < 2000000: # editlog, inexact match
entry[ts][0] = backup_from
lleftover.remove(ts)
bleftover.remove(bfile)
if len(bleftover) == 1 and len(lleftover) == 1: # only 1 left, must be this
backup_from = opj(backupdir_from, bleftover[0])
entry[lleftover[0]][0] = backup_from
lleftover = []
bleftover = []
# fake some log entries
for bfile in bleftover:
backup_from = opj(backupdir_from, bfile)
bts = long(bfile) # must be long py 2.2.x
data = [ts,'','SAVE',pagename,'','','','','missing editlog entry for this page version']
entry[bts] = [backup_from, data]
# check if we still haven't matched the "text" file
if not found_text and os.path.exists(text_from):
if llatest in lleftover: # latest log entry still free
entry[llatest][0] = text_from # take it. do not care about mtime of file.
lleftover.remove(llatest)
else: # log for "text" file is missing or latest was taken by other rev 8(
mtime = os.path.getmtime(text_from)
ts = wikiutil.timestamp2version(mtime) # take mtime, we have nothing better
data = [ts,'','SAVE', pagename,'','','','','missing editlog entry for this page version']
entry[ts] = [text_from, data]
# delete unmatching log entries
for ts in lleftover:
del entry[ts]
info[pagename] = entry
