class User:
""" represents a user with all related data """
def __init__(self, request, users_dir, uid):
self.request = request
self.uid = uid
self.users_dir = users_dir
self.profile = None
self.bookmarks = None
def read(self):
""" read profile and bookmarks data from disk """
self.profile = {}
fname = opj(self.users_dir, self.uid)
# read user profile
f = codecs.open(fname, 'r', config.charset)
for line in f:
line = line.replace(u'\r', '').replace(u'\n', '')
if not line.strip() or line.startswith(u'#'): # skip empty or comment lines
continue
try:
key, value = line.split(u'=', 1)
except Exception, err:
print "Error: User reader can not parse line %r from profile %r (%s)" % (line, fname, str(err))
continue
self.profile[key] = value
f.close()
# read bookmarks
self.bookmarks = {}
fname_pattern = opj(self.users_dir, "%s.*.bookmark" % self.uid)
for fname in glob.glob(fname_pattern):
f = file(fname, "r")
bookmark = f.read()
f.close()
wiki = fname.replace('.bookmark', '').replace(opj(self.users_dir, self.uid+'.'), '')
self.bookmarks[wiki] = int(bookmark)
def migrate(destdir):
plugindir = opj(destdir, 'plugin')
makedir(plugindir)
fname = opj(plugindir, '__init__.py')
f = open(fname, 'w')
f.write('''\
# *** Do not remove this! ***
# Although being empty, the presence of this file is important for plugins
# working correctly.
''')
f.close()
for d in [
'action',
'formatter',
'macro',
'parser',
'processor',
'theme',
'xmlrpc',
]:
thisdir = opj(plugindir, d)
makedir(thisdir)
fname = opj(thisdir, '__init__.py')
f = open(fname, 'w')
f.write('''\
# -*- coding: iso-8859-1 -*-
from MoinMoin.util import pysupport
modules = pysupport.getPackageModules(__file__)
''')
f.close()
def write(self, pages_dir):
""" write a page, including revisions, log, attachments to disk """
if ('PAGE', self.name) in self.renames:
name_new = self.renames[('PAGE', self.name)]
if name_new != self.name:
print "Renaming page %r -> %r" % (self.name, name_new)
self.name_old = self.name
self.name = name_new
qpagename = wikiutil.quoteWikinameFS(self.name)
page_dir = opj(pages_dir, qpagename)
os.makedirs(page_dir)
# write current file
current = self.current
if current is not None:
if create_rev and not self.is_deleted:
current += 1
current_fname = opj(page_dir, 'current')
current_file = file(current_fname, "w")
current_str = '%08d\n' % current
current_file.write(current_str)
current_file.close()
# copy edit-log
if self.editlog is not None:
editlog_fname = opj(page_dir, 'edit-log')
self.editlog.copy(editlog_fname,
self.renames,
deleted=self.is_deleted)
# copy page revisions
if self.revisions is not None:
rev_dir = opj(page_dir, 'revisions')
os.makedirs(rev_dir)
for rev in self.revlist:
if create_rev:
self.revisions[rev].copy(rev_dir, self.renames)
else:
if int(rev) == self.current:
self.revisions[rev].copy(rev_dir,
self.renames,
convert=True)
else:
self.revisions[rev].copy(rev_dir, self.renames)
if create_rev and not self.is_deleted:
self.revisions[rev].copy(rev_dir,
self.renames,
convert=True,
new_rev=rev + 1)
# copy attachments
if self.attachments is not None:
attach_dir = opj(page_dir, 'attachments')
os.makedirs(attach_dir)
for fn, att in self.attachments.items():
# we have to check for renames here because we need the (old) pagename, too:
if ('FILE', self.name_old, fn) in self.renames:
fn_new = self.renames[('FILE', self.name_old, fn)]
if fn_new != fn:
print "Renaming file %r %r -> %r" % (self.name_old, fn,
fn_new)
att.name = fn_new
att.copy(attach_dir)
def remove_trash(dir_from):
for pagename in info:
# omit dead pages and MoinEditorBackup
if pagename in pagelist and (
os.path.exists(opj(dir_from, pagename, 'text')) or
os.path.exists(opj(dir_from, pagename, 'backup'))
) and not pagename.endswith('MoinEditorBackup'):
info2[pagename] = info[pagename]
def gather_pagedirs(dir_from):
""" this gathers edit-log information from the pagedirs, just to make sure
"""
pagedir = opj(dir_from, 'pages')
pagelist = listdir(pagedir)
for pagename in pagelist:
editlog_from = opj(pagedir, pagename, 'edit-log')
gather_editlog(editlog_from, pagename)
def remove_trash(dir_from):
for pagename in info:
# omit dead pages and MoinEditorBackup
if pagename in pagelist and (
os.path.exists(opj(dir_from, pagename, 'text'))
or os.path.exists(opj(dir_from, pagename, 'backup'))
) and not pagename.endswith('MoinEditorBackup'):
info2[pagename] = info[pagename]
def convert_pagedir(dir_from, dir_to):
os.mkdir(dir_to)
for dname_from in listdir(dir_from):
print "%s" % (dname_from, )
dname_to = dname_from
shutil.copytree(opj(dir_from, dname_from), opj(dir_to, dname_to), 1)
try:
os.remove(opj(dir_to, dname_to, 'last-edited'))
except: pass
def convert_pagedir(dir_from, dir_to):
os.mkdir(dir_to)
for dname_from in listdir(dir_from):
print "%s" % (dname_from, )
dname_to = dname_from
shutil.copytree(opj(dir_from, dname_from), opj(dir_to, dname_to), 1)
try:
os.remove(opj(dir_to, dname_to, 'last-edited'))
except:
pass
def convert_userdir(dir_from, dir_to):
os.mkdir(dir_to)
for fname in listdir(dir_from):
if fname.endswith('.bookmark'):
bm = open(opj(dir_from, fname)).read().strip()
bm = str(wikiutil.timestamp2version(float(bm)))
f = open(opj(dir_to, fname), 'w')
f.write(bm)
f.close()
else:
copy_file(opj(dir_from, fname), opj(dir_to, fname))
def convert_userdir(dir_from, dir_to):
os.mkdir(dir_to)
for fname in listdir(dir_from):
if fname.endswith('.bookmark'):
bm = open(opj(dir_from, fname)).read().strip()
bm = str(wikiutil.timestamp2version(float(bm)))
f = open(opj(dir_to, fname), 'w')
f.write(bm)
f.close()
else:
copy_file(opj(dir_from, fname), opj(dir_to, fname))
def __init__(self, request, src_data_dir, dest_data_dir):
self.request = request
self.sdata = src_data_dir
self.ddata = dest_data_dir
self.pages = {}
self.users = {}
self.complete = {}
self.renames = {}
self.complete_fname = opj(self.sdata, 'complete.txt')
self.rename_fname1 = opj(self.sdata, 'rename1.txt')
self.rename_fname2 = opj(self.sdata, 'rename2.txt')
def __init__(self, request, src_data_dir, dest_data_dir):
self.request = request
self.sdata = src_data_dir
self.ddata = dest_data_dir
self.pages = {}
self.users = {}
self.complete = {}
self.renames = {}
self.complete_fname = opj(self.sdata, 'complete.txt')
self.rename_fname1 = opj(self.sdata, 'rename1.txt')
self.rename_fname2 = opj(self.sdata, 'rename2.txt')
def convert_pagedir(dir_from, dir_to, enc_from, enc_to):
os.mkdir(dir_to)
for dname_from in listdir(dir_from):
dname_to = qf_convert_string(dname_from, enc_from, enc_to)
print "%s -> %s" % (dname_from, dname_to)
shutil.copytree(opj(dir_from, dname_from), opj(dir_to, dname_to), 1)
try:
convert_editlog(opj(dir_from, dname_from, 'last-edited'),
opj(dir_to, dname_to, 'last-edited'),
enc_from, enc_to)
except IOError:
pass # we ignore if it doesnt exist
def write(self, pages_dir):
""" write a page, including revisions, log, attachments to disk """
if ('PAGE', self.name) in self.renames:
name_new = self.renames[('PAGE', self.name)]
if name_new != self.name:
print "Renaming page %r -> %r" % (self.name, name_new)
self.name_old = self.name
self.name = name_new
qpagename = wikiutil.quoteWikinameFS(self.name)
page_dir = opj(pages_dir, qpagename)
os.makedirs(page_dir)
# write current file
current = self.current
if current is not None:
if create_rev and not self.is_deleted:
current += 1
current_fname = opj(page_dir, 'current')
current_file = file(current_fname, "w")
current_str = '%08d\n' % current
current_file.write(current_str)
current_file.close()
# copy edit-log
if self.editlog is not None:
editlog_fname = opj(page_dir, 'edit-log')
self.editlog.copy(editlog_fname, self.renames, deleted=self.is_deleted)
# copy page revisions
if self.revisions is not None:
rev_dir = opj(page_dir, 'revisions')
os.makedirs(rev_dir)
for rev in self.revlist:
if create_rev:
self.revisions[rev].copy(rev_dir, self.renames)
else:
if int(rev) == self.current:
self.revisions[rev].copy(rev_dir, self.renames, convert=True)
else:
self.revisions[rev].copy(rev_dir, self.renames)
if create_rev and not self.is_deleted:
self.revisions[rev].copy(rev_dir, self.renames, convert=True, new_rev=rev+1)
# copy attachments
if self.attachments is not None:
attach_dir = opj(page_dir, 'attachments')
os.makedirs(attach_dir)
for fn, att in self.attachments.items():
# we have to check for renames here because we need the (old) pagename, too:
if ('FILE', self.name_old, fn) in self.renames:
fn_new = self.renames[('FILE', self.name_old, fn)]
if fn_new != fn:
print "Renaming file %r %r -> %r" % (self.name_old, fn, fn_new)
att.name = fn_new
att.copy(attach_dir)
def convert_textdir(dir_from, dir_to, enc_from, enc_to, is_backupdir=0):
os.mkdir(dir_to)
for fname_from in listdir(dir_from):
if is_backupdir:
fname, timestamp = fname_from.split('.')
else:
fname = fname_from
fname = qf_convert_string(fname, enc_from, enc_to)
if is_backupdir:
fname_to = '.'.join([fname, timestamp])
else:
fname_to = fname
convert_file(opj(dir_from, fname_from), opj(dir_to, fname_to),
enc_from, enc_to)
def read(self):
fname = opj(self.rev_dir, '%08d' % self.rev)
f = file(fname, "rb")
data = f.read()
f.close()
data = data.decode(config.charset)
return data
def write(self, users_dir):
""" write profile and bookmarks data to disk """
fname = opj(users_dir, self.uid)
f = codecs.open(fname, 'w', config.charset)
for key, value in self.profile.items():
if key in (u'subscribed_pages', u'quicklinks'):
pages = value.split(u'\t')
for i in range(len(pages)):
pagename = pages[i]
try:
interwiki, pagename = pagename.split(u':', 1)
except:
interwiki, pagename = u'Self', pagename
if interwiki == u'Self' or interwiki == self.request.cfg.interwikiname:
if ('PAGE', pagename) in self.renames:
pagename = self.renames[('PAGE', pagename)]
pages[i] = u'%s:%s' % (interwiki, pagename)
key += '[]' # we have lists here
value = u'\t'.join(pages)
f.write(u"%s=%s\n" % (key, value))
else:
f.write(u"%s=%s\n" % (key, value))
bookmark_entries = [u'%s:%s' % item for item in self.bookmarks.items()]
key = u"bookmarks{}"
value = u'\t'.join(bookmark_entries)
f.write(u"%s=%s\n" % (key, value))
f.close()
def write(self, users_dir):
""" write profile and bookmarks data to disk """
fname = opj(users_dir, self.uid)
f = codecs.open(fname, "w", config.charset)
for key, value in self.profile.items():
if key in (u"subscribed_pages", u"quicklinks"):
pages = value.split(u"\t")
for i in range(len(pages)):
pagename = pages[i]
try:
interwiki, pagename = pagename.split(u":", 1)
except:
interwiki, pagename = u"Self", pagename
if interwiki == u"Self" or interwiki == self.request.cfg.interwikiname:
if ("PAGE", pagename) in self.renames:
pagename = self.renames[("PAGE", pagename)]
pages[i] = u"%s:%s" % (interwiki, pagename)
key += "[]" # we have lists here
value = u"\t".join(pages)
f.write(u"%s=%s\n" % (key, value))
else:
f.write(u"%s=%s\n" % (key, value))
bookmark_entries = [u"%s:%s" % item for item in self.bookmarks.items()]
key = u"bookmarks{}"
value = u"\t".join(bookmark_entries)
f.write(u"%s=%s\n" % (key, value))
f.close()
def write(self, users_dir):
""" write profile and bookmarks data to disk """
fname = opj(users_dir, self.uid)
f = codecs.open(fname, 'w', config.charset)
for key, value in self.profile.items():
if key in (u'subscribed_pages', u'quicklinks'):
pages = value.split(u'\t')
for i in range(len(pages)):
pagename = pages[i]
try:
interwiki, pagename = pagename.split(u':', 1)
except:
interwiki, pagename = u'Self', pagename
if interwiki == u'Self' or interwiki == self.request.cfg.interwikiname:
if ('PAGE', pagename) in self.renames:
pagename = self.renames[('PAGE', pagename)]
pages[i] = u'%s:%s' % (interwiki, pagename)
key += '[]' # we have lists here
value = u'\t'.join(pages)
f.write(u"%s=%s\n" % (key, value))
else:
f.write(u"%s=%s\n" % (key, value))
bookmark_entries = [u'%s:%s' % item for item in self.bookmarks.items()]
key = u"bookmarks{}"
value = u'\t'.join(bookmark_entries)
f.write(u"%s=%s\n" % (key, value))
f.close()
def read(self):
fname = opj(self.rev_dir, '%08d' % self.rev)
f = file(fname, "rb")
data = f.read()
f.close()
data = data.decode(config.charset)
return data
def execute(script, data_dir, rev):
plugindir = opj(data_dir, 'plugin')
for d in ['filter', ]:
thisdir = opj(plugindir, d)
makedir(thisdir)
fname = opj(thisdir, '__init__.py')
f = open(fname, 'w')
f.write("""\
# -*- coding: iso-8859-1 -*-
from MoinMoin.util import pysupport
modules = pysupport.getPackageModules(__file__)
""")
f.close()
return rev + 1
def write_dest(self):
self.init_dest()
# copy pages
pages_dir = opj(self.ddata, 'pages')
for pn, page in self.pages.items():
if pn.endswith('/MoinEditorBackup'):
continue # we don't care for old editor backups
page.copy(pages_dir, self.renames)
# copy users
users_dir = opj(self.ddata, 'user')
for user in self.users.values():
user.copy(users_dir, self.renames)
# copy logs
self.editlog.copy(opj(self.ddata, 'edit-log'), self.renames)
self.eventlog.copy(opj(self.ddata, 'event-log'), self.renames)
def write_dest(self):
self.init_dest()
# copy pages
pages_dir = opj(self.ddata, 'pages')
for pn, page in self.pages.items():
if pn.endswith('/MoinEditorBackup'):
continue # we don't care for old editor backups
page.copy(pages_dir, self.renames)
# copy users
users_dir = opj(self.ddata, 'user')
for user in self.users.values():
user.copy(users_dir, self.renames)
# copy logs
self.editlog.copy(opj(self.ddata, 'edit-log'), self.renames)
self.eventlog.copy(opj(self.ddata, 'event-log'), self.renames)
def execute(script, data_dir, rev):
plugindir = opj(data_dir, 'plugin')
for d in [
'filter',
]:
thisdir = opj(plugindir, d)
makedir(thisdir)
fname = opj(thisdir, '__init__.py')
f = open(fname, 'w')
f.write("""\
# -*- coding: iso-8859-1 -*-
from MoinMoin.util import pysupport
modules = pysupport.getPackageModules(__file__)
""")
f.close()
return rev + 1
def write(self, data, rev_dir, convert, rev=None):
if rev is None:
rev = self.rev
if convert:
data = markup_converter(self.request, self.pagename, data, self.renames)
fname = opj(rev_dir, '%08d' % rev)
data = data.encode(config.charset)
f = file(fname, "wb")
f.write(data)
f.close()
def read_src(self):
# create Page objects in memory
pages_dir = opj(self.sdata, 'pages')
pagelist = listdir(pages_dir)
for qpagename in pagelist:
p = Page(self.request, pages_dir, qpagename)
self.pages[p.name] = p
# create User objects in memory
users_dir = opj(self.sdata, 'user')
user_re = re.compile(r'^\d+\.\d+(\.\d+)?$')
userlist = listdir(users_dir)
userlist = [f for f in userlist if user_re.match(f)]
for userid in userlist:
u = User(self.request, users_dir, userid)
self.users[u.uid] = u
# create log objects in memory
self.editlog = EditLog(self.request, opj(self.sdata, 'edit-log'))
self.eventlog = EventLog(self.request, opj(self.sdata, 'event-log'))
def __init__(self, request, pages_dir, qpagename):
self.request = request
self.name = wikiutil.unquoteWikiname(qpagename)
self.name_old = self.name # renaming: still original name when self.name has the new name
self.page_dir = opj(pages_dir, qpagename)
self.current = None # int current
self.editlog = None # dict (see read_editlog)
self.revlist = None # list of ints (page text revisions)
self.revisions = None # dict int: pagerev obj
self.attachments = None # dict of unicode fname: full path
self.renames = {} # info for renaming pages/attachments
def __init__(self, request, pages_dir, qpagename):
self.request = request
self.name = wikiutil.unquoteWikiname(qpagename)
self.name_old = self.name # renaming: still original name when self.name has the new name
self.page_dir = opj(pages_dir, qpagename)
self.current = None # int current
self.editlog = None # dict (see read_editlog)
self.revlist = None # list of ints (page text revisions)
self.revisions = None # dict int: pagerev obj
self.attachments = None # dict of unicode fname: full path
self.renames = {} # info for renaming pages/attachments
def read_src(self):
# create Page objects in memory
pages_dir = opj(self.sdata, 'pages')
pagelist = listdir(pages_dir)
for qpagename in pagelist:
p = Page(self.request, pages_dir, qpagename)
self.pages[p.name] = p
# create User objects in memory
users_dir = opj(self.sdata, 'user')
user_re = re.compile(r'^\d+\.\d+(\.\d+)?$')
userlist = listdir(users_dir)
userlist = [f for f in userlist if user_re.match(f)]
for userid in userlist:
u = User(self.request, users_dir, userid)
self.users[u.uid] = u
# create log objects in memory
self.editlog = EditLog(self.request, opj(self.sdata, 'edit-log'))
self.eventlog = EventLog(self.request, opj(self.sdata, 'event-log'))
def read(self):
""" read a page, including revisions, log, attachments from disk """
page_dir = self.page_dir
# read current file
current_fname = opj(page_dir, 'current')
if os.path.exists(current_fname):
current_file = file(current_fname, "r")
current_rev = current_file.read()
current_file.close()
try:
self.current = int(current_rev)
except ValueError:
print "Error: invalid current file %s, SKIPPING THIS PAGE!" % current_fname
return
# read edit-log
editlog_fname = opj(page_dir, 'edit-log')
if os.path.exists(editlog_fname):
self.editlog = EditLog(self.request, editlog_fname)
# read page revisions
rev_dir = opj(page_dir, 'revisions')
if os.path.exists(rev_dir):
revlist = listdir(rev_dir)
revlist = [int(rev) for rev in revlist]
revlist.sort()
self.revlist = revlist
self.revisions = {}
for rev in revlist:
self.revisions[rev] = PageRev(self.request, self.name_old,
rev_dir, rev)
# set deleted status
self.is_deleted = not self.revisions or self.current not in self.revisions
# read attachment filenames
attach_dir = opj(page_dir, 'attachments')
if os.path.exists(attach_dir):
self.attachments = {}
attlist = listdir(attach_dir)
for attfile in attlist:
a = Attachment(self.request, attach_dir, attfile)
self.attachments[a.name] = a
def read(self):
""" read a page, including revisions, log, attachments from disk """
page_dir = self.page_dir
# read current file
current_fname = opj(page_dir, 'current')
if os.path.exists(current_fname):
current_file = file(current_fname, "r")
current_rev = current_file.read()
current_file.close()
try:
self.current = int(current_rev)
except ValueError:
print "Error: invalid current file %s, SKIPPING THIS PAGE!" % current_fname
return
# read edit-log
editlog_fname = opj(page_dir, 'edit-log')
if os.path.exists(editlog_fname):
self.editlog = EditLog(self.request, editlog_fname)
# read page revisions
rev_dir = opj(page_dir, 'revisions')
if os.path.exists(rev_dir):
revlist = listdir(rev_dir)
revlist = [int(rev) for rev in revlist]
revlist.sort()
self.revlist = revlist
self.revisions = {}
for rev in revlist:
self.revisions[rev] = PageRev(self.request, self.name_old, rev_dir, rev)
# set deleted status
self.is_deleted = not self.revisions or self.current not in self.revisions
# read attachment filenames
attach_dir = opj(page_dir, 'attachments')
if os.path.exists(attach_dir):
self.attachments = {}
attlist = listdir(attach_dir)
for attfile in attlist:
a = Attachment(self.request, attach_dir, attfile)
self.attachments[a.name] = a
def migrate(dir_to):
""" this removes edit-lock files from the pagedirs and
converts attachment filenames
"""
pagesdir = opj(dir_to, 'pages')
pagelist = listdir(pagesdir)
for pagename in pagelist:
pagedir = opj(pagesdir, pagename)
editlock = opj(pagedir, 'edit-lock')
try:
os.remove(editlock)
except:
pass
attachdir = os.path.join(pagedir, 'attachments')
for root, dirs, files in os.walk(attachdir):
for f in files:
try:
f.decode(to_encoding)
except UnicodeDecodeError:
fnew = f.decode(from_encoding).encode(to_encoding)
os.rename(os.path.join(root, f), os.path.join(root, fnew))
print 'renamed', f, '\n ->', fnew, ' in dir:', root
def init_dest(self):
try:
os.makedirs(self.ddata)
except:
pass
os.makedirs(opj(self.ddata, "pages"))
os.makedirs(opj(self.ddata, "user"))
copy_dir(opj(self.sdata, "plugin"), opj(self.ddata, "plugin"))
copy_file(opj(self.sdata, "intermap.txt"), opj(self.ddata, "intermap.txt"))
def init_dest(self):
try:
os.makedirs(self.ddata)
except:
pass
os.makedirs(opj(self.ddata, 'pages'))
os.makedirs(opj(self.ddata, 'user'))
copy_dir(opj(self.sdata, 'plugin'), opj(self.ddata, 'plugin'))
copy_file(opj(self.sdata, 'intermap.txt'), opj(self.ddata, 'intermap.txt'))
def migrate(destdir):
plugindir = opj(destdir, 'plugin')
makedir(plugindir)
fname = opj(plugindir, '__init__.py')
f = open(fname, 'w')
f.write('''\
# *** Do not remove this! ***
# Although being empty, the presence of this file is important for plugins
# working correctly.
''')
f.close()
for d in ['action', 'formatter', 'macro', 'parser', 'processor', 'theme', 'xmlrpc', ]:
thisdir = opj(plugindir, d)
makedir(thisdir)
fname = opj(thisdir, '__init__.py')
f = open(fname, 'w')
f.write('''\
# -*- coding: iso-8859-1 -*-
from MoinMoin.util import pysupport
modules = pysupport.getPackageModules(__file__)
''')
f.close()
def read(self):
""" read profile and bookmarks data from disk """
self.profile = {}
fname = opj(self.users_dir, self.uid)
# read user profile
f = codecs.open(fname, 'r', config.charset)
for line in f:
line = line.replace(u'\r', '').replace(u'\n', '')
if not line.strip() or line.startswith(u'#'): # skip empty or comment lines
continue
try:
key, value = line.split(u'=', 1)
except Exception, err:
print "Error: User reader can not parse line %r from profile %r (%s)" % (line, fname, str(err))
continue
self.profile[key] = value
def generate_editlog(dir_from, dir_to):
editlog = {}
for pagename in info2:
entry = info2.get(pagename, {})
for ts in entry:
file_from, data = entry[ts]
editlog[ts] = data
tslist = editlog.keys()
tslist.sort()
editlog_file = opj(dir_to, 'edit-log')
f = open(editlog_file, 'w')
for ts in tslist:
data = editlog[ts]
f.write('\t'.join(data) + '\n')
f.close()
def generate_editlog(dir_from, dir_to):
editlog = {}
for pagename in info2:
entry = info2.get(pagename, {})
for ts in entry:
file_from, data = entry[ts]
editlog[ts] = data
tslist = editlog.keys()
tslist.sort()
editlog_file = opj(dir_to, 'edit-log')
f = open(editlog_file, 'w')
for ts in tslist:
data = editlog[ts]
f.write('\t'.join(data)+'\n')
f.close()
def convert_editlog(file_from, file_to, dir_to):
for l in open(file_from):
data = l.split('\t')
pagename = data[0]
timestamp = data[2]
data[2] = str(long(float(timestamp))) # we only want integer (must be long for py 2.2.x)
data = '\t'.join(data)
f = open(file_to, 'a')
f.write(data)
f.close()
try:
file_to2 = opj(dir_to, pagename, 'edit-log')
f = open(file_to2, 'a')
f.write(data)
f.close()
except: pass
def convert_editlog(file_from, file_to, dir_to):
for l in open(file_from):
data = l.split('\t')
pagename = data[0]
timestamp = data[2]
data[2] = str(long(float(
timestamp))) # we only want integer (must be long for py 2.2.x)
data = '\t'.join(data)
f = open(file_to, 'a')
f.write(data)
f.close()
try:
file_to2 = opj(dir_to, pagename, 'edit-log')
f = open(file_to2, 'a')
f.write(data)
f.close()
except:
pass
def generate_pages(dir_from, dir_to):
for pagename in info2:
entry = info2.get(pagename, {})
tslist = entry.keys()
if tslist:
pagedir = opj(dir_to, 'pages', pagename)
os.makedirs(opj(pagedir, 'revisions'))
editlog_file = opj(pagedir, 'edit-log')
f = open(editlog_file, 'w')
rev = 0
tslist.sort()
for ts in tslist:
rev += 1
revstr = '%08d' % rev
file_from, data = entry[ts]
data[0] = str(ts)
data[1] = revstr
if data[2].endswith('/REVERT'):
# replace the timestamp with the revision number
revertts = long(data[7]) # must be long for py 2.2.x
try:
revertrev = int(entry[revertts][1][1])
except KeyError:
# never should trigger...
print "********* KeyError %s entry[%d][1][1] **********" % (
pagename, revertts)
revertrev = 0
data[7] = '%08d' % revertrev
f.write('\t'.join(data) + '\n')
if file_from is not None:
file_to = opj(pagedir, 'revisions', revstr)
copy_file(file_from, file_to)
f.close()
curr_file = opj(pagedir, 'current')
f = open(curr_file, 'w')
f.write(revstr)
f.close()
att_from = opj(dir_from, 'pages', pagename, 'attachments')
if os.path.exists(att_from):
att_to = opj(pagedir, 'attachments')
copy_dir(att_from, att_to)
def generate_pages(dir_from, dir_to):
for pagename in info2:
entry = info2.get(pagename, {})
tslist = entry.keys()
if tslist:
pagedir = opj(dir_to, 'pages', pagename)
os.makedirs(opj(pagedir, 'revisions'))
editlog_file = opj(pagedir, 'edit-log')
f = open(editlog_file, 'w')
rev = 0
tslist.sort()
for ts in tslist:
rev += 1
revstr = '%08d' % rev
file_from, data = entry[ts]
data[0] = str(ts)
data[1] = revstr
if data[2].endswith('/REVERT'):
# replace the timestamp with the revision number
revertts = long(data[7]) # must be long for py 2.2.x
try:
revertrev = int(entry[revertts][1][1])
except KeyError:
# never should trigger...
print "********* KeyError %s entry[%d][1][1] **********" % (pagename, revertts)
revertrev = 0
data[7] = '%08d' % revertrev
f.write('\t'.join(data)+'\n')
if file_from is not None:
file_to = opj(pagedir, 'revisions', revstr)
copy_file(file_from, file_to)
f.close()
curr_file = opj(pagedir, 'current')
f = open(curr_file, 'w')
f.write(revstr)
f.close()
att_from = opj(dir_from, 'pages', pagename, 'attachments')
if os.path.exists(att_from):
att_to = opj(pagedir, 'attachments')
copy_dir(att_from, att_to)
def convert_textdir(dir_from, dir_to, is_backupdir=0):
for fname_from in listdir(dir_from):
if is_backupdir:
fname, timestamp = fname_from.split('.')
else:
fname = fname_from
try:
os.mkdir(opj(dir_to, 'pages', fname))
except: pass
try:
os.mkdir(opj(dir_to, 'pages', fname, 'backup'))
except: pass
try:
os.mkdir(opj(dir_to, 'pages', fname, 'cache'))
except: pass
if is_backupdir:
fname_to = opj('pages', fname, 'backup', timestamp)
else:
fname_to = opj('pages', fname, 'text')
copy_file(opj(dir_from, fname_from), opj(dir_to, fname_to))
def process_pagedirs(dir_from, dir_to):
pagelist = listdir(dir_from)
for pagename in pagelist:
pagedir_from = opj(dir_from, pagename)
pagedir_to = opj(dir_to, pagename)
# first we copy all, even the stuff we convert later:
copy_dir(pagedir_from, pagedir_to)
rev_from = opj(pagedir_from, 'revisions')
rev_to = opj(pagedir_to, 'revisions')
if os.path.exists(rev_from):
revlist = listdir(rev_from)
for rfile in revlist:
rev = int(rfile)
r_from = opj(rev_from, rfile)
r_to = opj(rev_to, rfile)
tocrlf(r_from, r_to)
def process_pagedirs(dir_from, dir_to):
pagelist = listdir(dir_from)
for pagename in pagelist:
pagedir_from = opj(dir_from, pagename)
pagedir_to = opj(dir_to, pagename)
# first we copy all, even the stuff we convert later:
copy_dir(pagedir_from, pagedir_to)
rev_from = opj(pagedir_from, 'revisions')
rev_to = opj(pagedir_to, 'revisions')
if os.path.exists(rev_from):
revlist = listdir(rev_from)
for rfile in revlist:
rev = int(rfile)
r_from = opj(rev_from, rfile)
r_to = opj(rev_to, rfile)
tocrlf(r_from, r_to)
def convert_textdir(dir_from, dir_to, is_backupdir=0):
for fname_from in listdir(dir_from):
if is_backupdir:
fname, timestamp = fname_from.split('.')
else:
fname = fname_from
try:
os.mkdir(opj(dir_to, 'pages', fname))
except:
pass
try:
os.mkdir(opj(dir_to, 'pages', fname, 'backup'))
except:
pass
try:
os.mkdir(opj(dir_to, 'pages', fname, 'cache'))
except:
pass
if is_backupdir:
fname_to = opj('pages', fname, 'backup', timestamp)
else:
fname_to = opj('pages', fname, 'text')
copy_file(opj(dir_from, fname_from), opj(dir_to, fname_to))
def convert_userdir(dir_from, dir_to, enc_from, enc_to):
os.mkdir(dir_to)
for fname in listdir(dir_from):
convert_file(opj(dir_from, fname), opj(dir_to, fname),
enc_from, enc_to)
revlist = listdir(rev_from)
for rfile in revlist:
rev = int(rfile)
r_from = opj(rev_from, rfile)
r_to = opj(rev_to, rfile)
tocrlf(r_from, r_to)
origdir = 'data.pre-mig7'
try:
os.rename('data', origdir)
except OSError:
print "You need to be in the directory where your copy of the 'data' directory is located."
sys.exit(1)
os.makedirs(opj('data','pages'))
process_pagedirs(opj(origdir, 'pages'), opj('data', 'pages'))
copy_dir(opj(origdir, 'plugin'), opj('data', 'plugin'))
copy_dir(opj(origdir, 'user'), opj('data', 'user'))
copy_file(opj(origdir, 'edit-log'), opj('data', 'edit-log'))
copy_file(opj(origdir, 'event-log'), opj('data', 'event-log'))
copy_file(opj(origdir, 'intermap.txt'), opj('data', 'intermap.txt'))
fields[0] = qf_convert_string(fields[0], enc_from, enc_to)
fields[5] = convert_string(fields[5], enc_from, enc_to)
line = '\t'.join(fields)
file_to.write(line)
origdir = 'data.pre-mig2'
# Backup original dir and create new empty dir
try:
os.rename('data', origdir)
os.mkdir('data')
except OSError:
print "You need to be in the directory where your copy of the 'data' directory is located."
sys.exit(1)
convert_textdir(opj(origdir, 'text'), opj('data', 'text'), from_encoding, to_encoding)
convert_textdir(opj(origdir, 'backup'), opj('data', 'backup'), from_encoding, to_encoding, 1)
convert_pagedir(opj(origdir, 'pages'), opj('data', 'pages'), from_encoding, to_encoding)
convert_userdir(opj(origdir, 'user'), opj('data', 'user'), from_encoding, to_encoding)
convert_editlog(opj(origdir, 'editlog'), opj('data', 'editlog'), from_encoding, to_encoding)
copy_file(opj(origdir, 'event.log'), opj('data', 'event.log'))
copy_dir(opj(origdir, 'plugin'), opj('data', 'plugin'))
copy_file(opj(origdir, 'intermap.txt'), opj('data', 'intermap.txt'))
def gather_pagedirs(dir_from, is_backupdir=0):
""" this gathers information from the pagedirs, i.e. text and backup
files (and also the local editlog) and tries to merge/synchronize
with the informations gathered from editlog
"""
global pagelist
pagelist = listdir(dir_from)
for pagename in pagelist:
editlog_from = opj(dir_from, pagename, 'edit-log')
gather_editlog(dir_from, editlog_from)
entry = info.get(pagename, {})
loglist = [] # editlog timestamps of page revisions
for ts, data in entry.items():
if data[1][2] in [
'SAVE',
'SAVENEW',
'SAVE/REVERT',
]:
loglist.append(ts)
loglist.sort()
lleftover = loglist[:]
# remember the latest log entry
if lleftover:
llatest = lleftover[-1]
else:
llatest = None
backupdir_from = opj(dir_from, pagename, 'backup')
if os.path.exists(backupdir_from):
backuplist = listdir(backupdir_from)
bleftover = backuplist[:]
for bfile in backuplist:
backup_from = opj(backupdir_from, bfile)
ts = long(bfile)
if ts in loglist: # we have an editlog entry, exact match
entry[ts][0] = backup_from
lleftover.remove(ts)
bleftover.remove(bfile)
text_from = opj(dir_from, pagename, 'text')
found_text = False
if os.path.exists(
text_from
): # we have a text file, it should match latest log entry
exists[pagename] = True
mtime = os.path.getmtime(text_from)
if llatest and llatest in lleftover:
ts = llatest
if abs(wikiutil.timestamp2version(mtime) -
ts) < 2000000: # less than a second diff
entry[ts][0] = text_from
lleftover.remove(ts)
found_text = True
else: # we have no log entries left 8(
ts = wikiutil.timestamp2version(mtime)
data = [
ts, '', 'SAVE', pagename, '', '', '', '',
'missing editlog entry for this page version'
]
entry[ts] = [text_from, data]
else:
# this page was maybe deleted, so we remember for later:
exists[pagename] = False
if llatest in lleftover: # if a page is deleted, the last log entry has no file
entry[llatest][0] = None
lleftover.remove(llatest)
if os.path.exists(backupdir_from):
backuplist = listdir(backupdir_from)
for bfile in backuplist:
if not bfile in bleftover: continue
backup_from = opj(backupdir_from, bfile)
bts = long(bfile) # must be long for py 2.2.x
for ts in lleftover:
tdiff = abs(bts - ts)
if tdiff < 2000000: # editlog, inexact match
entry[ts][0] = backup_from
lleftover.remove(ts)
bleftover.remove(bfile)
elif 3599000000 <= tdiff <= 3601000000: # editlog, win32 daylight saving bug
entry[ts][0] = backup_from
lleftover.remove(ts)
bleftover.remove(bfile)
print "Warning: Win32 daylight saving bug encountered & fixed!"
if len(bleftover) == 1 and len(
lleftover) == 1: # only 1 left, must be this
backup_from = opj(backupdir_from, bleftover[0])
entry[lleftover[0]][0] = backup_from
lleftover = []
bleftover = []
# fake some log entries
for bfile in bleftover:
backup_from = opj(backupdir_from, bfile)
bts = long(bfile) # must be long py 2.2.x
data = [
ts, '', 'SAVE', pagename, '', '', '', '',
'missing editlog entry for this page version'
]
entry[bts] = [backup_from, data]
# check if we still haven't matched the "text" file
if not found_text and os.path.exists(text_from):
if llatest in lleftover: # latest log entry still free
entry[llatest][
0] = text_from # take it. do not care about mtime of file.
lleftover.remove(llatest)
else: # log for "text" file is missing or latest was taken by other rev 8(
mtime = os.path.getmtime(text_from)
ts = wikiutil.timestamp2version(
mtime) # take mtime, we have nothing better
data = [
ts, '', 'SAVE', pagename, '', '', '', '',
'missing editlog entry for this page version'
]
entry[ts] = [text_from, data]
# delete unmatching log entries
for ts in lleftover:
#print "XXX Deleting leftover log entry: %r" % entry[ts]
del entry[ts]
info[pagename] = entry
data = editlog[ts]
f.write('\t'.join(data) + '\n')
f.close()
origdir = 'data.pre-mig5'
# Backup original dir and create new empty dir
try:
os.rename('data', origdir)
os.mkdir('data')
except OSError:
print "You need to be in the directory where your copy of the 'data' directory is located."
sys.exit(1)
gather_editlog(origdir, opj(origdir, 'edit-log'))
gather_pagedirs(opj(origdir, 'pages'))
remove_trash(opj(origdir, 'pages'))
generate_pages(origdir, 'data')
generate_editlog(origdir, 'data')
copy_dir(opj(origdir, 'plugin'), opj('data', 'plugin'))
copy_dir(opj(origdir, 'user'), opj('data', 'user'))
copy_file(opj(origdir, 'event-log'), opj('data', 'event-log'))
copy_file(opj(origdir, 'intermap.txt'), opj('data', 'intermap.txt'))
bm = open(opj(dir_from, fname)).read().strip()
bm = str(wikiutil.timestamp2version(float(bm)))
f = open(opj(dir_to, fname), 'w')
f.write(bm)
f.close()
else:
copy_file(opj(dir_from, fname), opj(dir_to, fname))
origdir = 'data.pre-mig4'
# Backup original dir and create new empty dir
try:
os.rename('data', origdir)
os.mkdir('data')
except OSError:
print "You need to be in the directory where your copy of the 'data' directory is located."
sys.exit(1)
convert_pagedir(opj(origdir, 'pages'), opj('data', 'pages'))
convert_editlog(opj(origdir, 'edit-log'), opj('data', 'edit-log'))
convert_eventlog(opj(origdir, 'event.log'), opj('data', 'event-log'))
convert_userdir(opj(origdir, 'user'), opj('data', 'user'))
copy_dir(opj(origdir, 'plugin'), opj('data', 'plugin'))
copy_file(opj(origdir, 'intermap.txt'), opj('data', 'intermap.txt'))
def convert_pagedir(dir_from, dir_to, is_backupdir=0):
os.mkdir(dir_to)
for pagedir in listdir(dir_from):
text_from = opj(dir_from, pagedir, 'text')
text_to = opj(dir_to, pagedir, 'text')
os.mkdir(opj(dir_to, pagedir))
copy_file(text_from, text_to)
backupdir_from = opj(dir_from, pagedir, 'backup')
backupdir_to = opj(dir_to, pagedir, 'backup')
if os.path.exists(backupdir_from):
os.mkdir(backupdir_to)
for ts in listdir(backupdir_from):
ts_usec = str(convert_ts(float(ts)))
backup_from = opj(backupdir_from, ts)
backup_to = opj(backupdir_to, ts_usec)
copy_file(backup_from, backup_to)
editlog_from = opj(dir_from, pagedir, 'edit-log')
editlog_to = opj(dir_to, pagedir, 'edit-log')
convert_editlog(editlog_from, editlog_to)
#cachedir_from = opj(dir_from, pagedir, 'cache')
#cachedir_to = opj(dir_to, pagedir, 'cache')
#if os.path.exists(cachedir_from):
# os.mkdir(cachedir_to)
# try:
# copy_file(
# opj(cachedir_from, 'hitcounts'),
# opj(cachedir_to, 'hitcounts'))
# except: pass
attachdir_from = opj(dir_from, pagedir, 'attachments')
attachdir_to = opj(dir_to, pagedir, 'attachments')
if os.path.exists(attachdir_from):
try:
copy_dir(attachdir_from, attachdir_to)
except:
pass
