Beispiel #1
0
def convertUsers(srcdir, dstdir):
    """ Convert users files

    @param srcdir: old users dir
    @param dstdir: new users dir
    """
    charset = 'utf-8'

    # Create dstdir
    if not os.path.exists(dstdir):
        try:
            os.mkdir(dstdir)
        except OSError:
            migutil.fatalError("can't create user directory at '%s'" % dstdir)

    if not os.path.isdir(srcdir):
        migutil.fatalError("can't find user directory at '%s'" % srcdir)

    for name in migutil.listdir(srcdir):
        if name == 'README' or name.endswith('.trail'):
            # Copy as is
            migutil.copy_file(join(srcdir, name), join(dstdir, name))
        else:
            srcfile = join(srcdir, name)
            f = codecs.open(srcfile, 'rb', charset)
            text = f.read()
            f.close()
            text = convertUserData(text)
            dstfile = join(dstdir, name)
            f = codecs.open(dstfile, 'wb', charset)
            f.write(text)
            f.close()
            print "Converted '%s' to '%s'" % (srcfile, dstfile)
Beispiel #2
0
def convertUsers(srcdir, dstdir):
    """ Convert users files

    @param srcdir: old users dir
    @param dstdir: new users dir
    """
    charset = 'utf-8'
    
    # Create dstdir
    if not os.path.exists(dstdir):
        try:
            os.mkdir(dstdir)
        except OSError:
            migutil.fatalError("can't create user directory at '%s'" % dstdir)

    if not os.path.isdir(srcdir):
        migutil.fatalError("can't find user directory at '%s'" % srcdir)

    for name in migutil.listdir(srcdir):
        if name == 'README' or name.endswith('.trail'):
            # Copy as is
            migutil.copy_file(join(srcdir, name), join(dstdir, name))
        else:
            srcfile = join(srcdir, name)
            f = codecs.open(srcfile, 'rb', charset)
            text = f.read()
            f.close()
            text = convertUserData(text)
            dstfile = join(dstdir, name)
            f = codecs.open(dstfile, 'wb', charset)
            f.write(text)
            f.close()
            print "Converted '%s' to '%s'" % (srcfile, dstfile)
Beispiel #3
0
 def init_dest(self):
     try:
         os.makedirs(self.ddata)
     except:
         pass
     os.makedirs(opj(self.ddata, 'pages'))
     os.makedirs(opj(self.ddata, 'user'))
     copy_dir(opj(self.sdata, 'plugin'), opj(self.ddata, 'plugin'))
     copy_file(opj(self.sdata, 'intermap.txt'), opj(self.ddata, 'intermap.txt'))
Beispiel #4
0
 def init_dest(self):
     try:
         os.makedirs(self.ddata)
     except:
         pass
     os.makedirs(opj(self.ddata, "pages"))
     os.makedirs(opj(self.ddata, "user"))
     copy_dir(opj(self.sdata, "plugin"), opj(self.ddata, "plugin"))
     copy_file(opj(self.sdata, "intermap.txt"), opj(self.ddata, "intermap.txt"))
Beispiel #5
0
def convert_userdir(dir_from, dir_to):
    os.mkdir(dir_to)
    for fname in listdir(dir_from):
        if fname.endswith('.bookmark'):
            bm = open(opj(dir_from, fname)).read().strip()
            bm = str(wikiutil.timestamp2version(float(bm)))
            f = open(opj(dir_to, fname), 'w')
            f.write(bm)
            f.close()
        else:
            copy_file(opj(dir_from, fname), opj(dir_to, fname))
Beispiel #6
0
def convert_userdir(dir_from, dir_to):
    os.mkdir(dir_to)
    for fname in listdir(dir_from):
        if fname.endswith('.bookmark'):
            bm = open(opj(dir_from, fname)).read().strip()
            bm = str(wikiutil.timestamp2version(float(bm)))
            f = open(opj(dir_to, fname), 'w')
            f.write(bm)
            f.close()
        else:
            copy_file(opj(dir_from, fname), opj(dir_to, fname))
def generate_pages(dir_from, dir_to):
    for pagename in info2:
        entry = info2.get(pagename, {})
        tslist = entry.keys()
        if tslist:
            pagedir = opj(dir_to, 'pages', pagename)
            os.makedirs(opj(pagedir, 'revisions'))
            editlog_file = opj(pagedir, 'edit-log')
            f = open(editlog_file, 'w')
            rev = 0
            tslist.sort()
            for ts in tslist:
                rev += 1
                revstr = '%08d' % rev
                file_from, data = entry[ts]
                data[0] = str(ts)
                data[1] = revstr
                if data[2].endswith('/REVERT'):
                    # replace the timestamp with the revision number
                    revertts = long(data[7])  # must be long for py 2.2.x
                    try:
                        revertrev = int(entry[revertts][1][1])
                    except KeyError:
                        # never should trigger...
                        print "********* KeyError %s entry[%d][1][1] **********" % (
                            pagename, revertts)
                        revertrev = 0
                    data[7] = '%08d' % revertrev
                f.write('\t'.join(data) + '\n')
                if file_from is not None:
                    file_to = opj(pagedir, 'revisions', revstr)
                    copy_file(file_from, file_to)
            f.close()

            curr_file = opj(pagedir, 'current')
            f = open(curr_file, 'w')
            f.write(revstr)
            f.close()

        att_from = opj(dir_from, 'pages', pagename, 'attachments')
        if os.path.exists(att_from):
            att_to = opj(pagedir, 'attachments')
            copy_dir(att_from, att_to)
Beispiel #8
0
def generate_pages(dir_from, dir_to):
    for pagename in info2:
        entry = info2.get(pagename, {})
        tslist = entry.keys()
        if tslist:
            pagedir = opj(dir_to, 'pages', pagename)
            os.makedirs(opj(pagedir, 'revisions'))
            editlog_file = opj(pagedir, 'edit-log')
            f = open(editlog_file, 'w')
            rev = 0
            tslist.sort()
            for ts in tslist:
                rev += 1
                revstr = '%08d' % rev
                file_from, data = entry[ts]
                data[0] = str(ts)
                data[1] = revstr
                if data[2].endswith('/REVERT'):
                    # replace the timestamp with the revision number
                    revertts = long(data[7]) # must be long for py 2.2.x
                    try:
                        revertrev = int(entry[revertts][1][1])
                    except KeyError:
                        # never should trigger...
                        print "********* KeyError %s entry[%d][1][1] **********" % (pagename, revertts)
                        revertrev = 0
                    data[7] = '%08d' % revertrev
                f.write('\t'.join(data)+'\n')
                if file_from is not None:
                    file_to = opj(pagedir, 'revisions', revstr)
                    copy_file(file_from, file_to)
            f.close()
                
            curr_file = opj(pagedir, 'current')
            f = open(curr_file, 'w')
            f.write(revstr)
            f.close()

        att_from = opj(dir_from, 'pages', pagename, 'attachments')
        if os.path.exists(att_from):
            att_to = opj(pagedir, 'attachments')
            copy_dir(att_from, att_to)
Beispiel #9
0
def convert_textdir(dir_from, dir_to, is_backupdir=0):
    for fname_from in listdir(dir_from):
        if is_backupdir:
            fname, timestamp = fname_from.split('.')
        else:
            fname = fname_from
        try:
            os.mkdir(opj(dir_to, 'pages', fname))
        except: pass
        try:
            os.mkdir(opj(dir_to, 'pages', fname, 'backup'))
        except: pass
        try:
            os.mkdir(opj(dir_to, 'pages', fname, 'cache'))
        except: pass
        if is_backupdir:
            fname_to = opj('pages', fname, 'backup', timestamp)
        else:
            fname_to = opj('pages', fname, 'text')
        copy_file(opj(dir_from, fname_from), opj(dir_to, fname_to))
Beispiel #10
0
def convert_pagedir(dir_from, dir_to, is_backupdir=0):
    os.mkdir(dir_to)
    for pagedir in listdir(dir_from):
        text_from = opj(dir_from, pagedir, 'text')
        text_to = opj(dir_to, pagedir, 'text')
        os.mkdir(opj(dir_to, pagedir))
        copy_file(text_from, text_to)

        backupdir_from = opj(dir_from, pagedir, 'backup')
        backupdir_to = opj(dir_to, pagedir, 'backup')
        if os.path.exists(backupdir_from):
            os.mkdir(backupdir_to)
            for ts in listdir(backupdir_from):
                ts_usec = str(convert_ts(float(ts)))
                backup_from = opj(backupdir_from, ts)
                backup_to = opj(backupdir_to, ts_usec)
                copy_file(backup_from, backup_to)

        editlog_from = opj(dir_from, pagedir, 'edit-log')
        editlog_to = opj(dir_to, pagedir, 'edit-log')
        convert_editlog(editlog_from, editlog_to)

        #cachedir_from = opj(dir_from, pagedir, 'cache')
        #cachedir_to = opj(dir_to, pagedir, 'cache')
        #if os.path.exists(cachedir_from):
        #    os.mkdir(cachedir_to)
        #    try:
        #        copy_file(
        #            opj(cachedir_from, 'hitcounts'),
        #            opj(cachedir_to, 'hitcounts'))
        #    except: pass

        attachdir_from = opj(dir_from, pagedir, 'attachments')
        attachdir_to = opj(dir_to, pagedir, 'attachments')
        if os.path.exists(attachdir_from):
            try:
                copy_dir(attachdir_from, attachdir_to)
            except:
                pass
Beispiel #11
0
def convert_pagedir(dir_from, dir_to, is_backupdir=0):
    os.mkdir(dir_to)
    for pagedir in listdir(dir_from):
        text_from = opj(dir_from, pagedir, 'text')
        text_to = opj(dir_to, pagedir, 'text')
        os.mkdir(opj(dir_to, pagedir))
        copy_file(text_from, text_to)

        backupdir_from = opj(dir_from, pagedir, 'backup')
        backupdir_to = opj(dir_to, pagedir, 'backup')
        if os.path.exists(backupdir_from):
            os.mkdir(backupdir_to)
            for ts in listdir(backupdir_from):
                ts_usec = str(convert_ts(float(ts)))
                backup_from = opj(backupdir_from, ts)
                backup_to = opj(backupdir_to, ts_usec)
                copy_file(backup_from, backup_to)

        editlog_from = opj(dir_from, pagedir, 'edit-log')
        editlog_to = opj(dir_to, pagedir, 'edit-log')
        convert_editlog(editlog_from, editlog_to)

        #cachedir_from = opj(dir_from, pagedir, 'cache')
        #cachedir_to = opj(dir_to, pagedir, 'cache')
        #if os.path.exists(cachedir_from):
        #    os.mkdir(cachedir_to)
        #    try:
        #        copy_file(
        #            opj(cachedir_from, 'hitcounts'),
        #            opj(cachedir_to, 'hitcounts'))
        #    except: pass

        attachdir_from = opj(dir_from, pagedir, 'attachments')
        attachdir_to = opj(dir_to, pagedir, 'attachments')
        if os.path.exists(attachdir_from):
            try:
                copy_dir(attachdir_from, attachdir_to)
            except: pass
def convert_textdir(dir_from, dir_to, is_backupdir=0):
    for fname_from in listdir(dir_from):
        if is_backupdir:
            fname, timestamp = fname_from.split('.')
        else:
            fname = fname_from
        try:
            os.mkdir(opj(dir_to, 'pages', fname))
        except:
            pass
        try:
            os.mkdir(opj(dir_to, 'pages', fname, 'backup'))
        except:
            pass
        try:
            os.mkdir(opj(dir_to, 'pages', fname, 'cache'))
        except:
            pass
        if is_backupdir:
            fname_to = opj('pages', fname, 'backup', timestamp)
        else:
            fname_to = opj('pages', fname, 'text')
        copy_file(opj(dir_from, fname_from), opj(dir_to, fname_to))
Beispiel #13
0
 def copy(self, attach_dir):
     """ copy attachment file from orig path to new destination """
     attfile = self.name.encode('utf-8')
     dest = opj(attach_dir, attfile)
     copy_file(self.path, dest)
        data = editlog[ts]
        f.write('\t'.join(data) + '\n')
    f.close()


origdir = 'data.pre-mig5'

# Backup original dir and create new empty dir
try:
    os.rename('data', origdir)
    os.mkdir('data')
except OSError:
    print "You need to be in the directory where your copy of the 'data' directory is located."
    sys.exit(1)

gather_editlog(origdir, opj(origdir, 'edit-log'))
gather_pagedirs(opj(origdir, 'pages'))

remove_trash(opj(origdir, 'pages'))

generate_pages(origdir, 'data')
generate_editlog(origdir, 'data')

copy_dir(opj(origdir, 'plugin'), opj('data', 'plugin'))

copy_dir(opj(origdir, 'user'), opj('data', 'user'))

copy_file(opj(origdir, 'event-log'), opj('data', 'event-log'))

copy_file(opj(origdir, 'intermap.txt'), opj('data', 'intermap.txt'))
        f.write("\t".join(data) + "\n")
    f.close()


origdir = "data.pre-mig5"

# Backup original dir and create new empty dir
try:
    os.rename("data", origdir)
    os.mkdir("data")
except OSError:
    print "You need to be in the directory where your copy of the 'data' directory is located."
    sys.exit(1)

gather_editlog(origdir, opj(origdir, "edit-log"))
gather_pagedirs(opj(origdir, "pages"))

remove_trash(opj(origdir, "pages"))

generate_pages(origdir, "data")
generate_editlog(origdir, "data")


copy_dir(opj(origdir, "plugin"), opj("data", "plugin"))

copy_dir(opj(origdir, "user"), opj("data", "user"))

copy_file(opj(origdir, "event-log"), opj("data", "event-log"))

copy_file(opj(origdir, "intermap.txt"), opj("data", "intermap.txt"))
def generate_pages(dir_from, dir_to):
    revactions = ['SAVE', 'SAVENEW', 'SAVE/REVERT', ] # these actions create revisions
    for pn in info:
        entry = info.get(pn, {})
        tslist = entry.keys()
        if tslist:
            pagedir = opj(dir_to, 'pages', pn)
            revdir = opj(pagedir, 'revisions')
            os.makedirs(revdir)
            editlog_file = opj(pagedir, 'edit-log')
            f = open(editlog_file, 'w')
            revnew = 0
            tslist.sort()
            for ts in tslist:
                data = entry[ts][1]
                datanew = data[:]
                (timestamp, rev, action, pagename, ip, host, id, extra, comment) = data
                revstr = '%08d' % rev
                if action in revactions:
                    revnew += 1
                    revnewstr = '%08d' % revnew
                    entry[ts][0] = revnew # remember what new revno we chose
                else: # ATTNEW,ATTDRW,ATTDEL
                    revnewstr = '99999999'
                if action.endswith('/REVERT'):
                    # replace the old revno with the correct new revno
                    revertrevold = int(extra)
                    revertrevnew = 0
                    for ts2 in tslist:
                        data2 = entry[ts2][1]
                        (timestamp2, rev2, action2, pagename2, ip2, host2, id2, extra2, comment2) = data2
                        if rev2 == revertrevold:
                            revertrevnew = entry[ts2][0]
                    datanew[7] = '%08d' % revertrevnew

                datanew[1] = revnewstr
                f.write('\t'.join(datanew)+'\n') # does make a CRLF on win32 in the file

                if action in revactions: # we DO have a page rev for this one
                    file_from = opj(dir_from, 'pages', pn, 'revisions', revstr)
                    file_to = opj(revdir, revnewstr)
                    copy_file(file_from, file_to)
            f.close()

            # check if page exists or is deleted in orig dir
            pagedir_from = opj(dir_from, 'pages', pn)
            revdir_from = opj(pagedir_from, 'revisions')
            try:
                curr_file_from = opj(pagedir_from, 'current')
                currentfrom = open(curr_file_from).read().strip() # try to access it
                page_exists = 1
            except:
                page_exists = 0

            # re-make correct DELETED status!
            if page_exists:
                curr_file = opj(pagedir, 'current')
                f = open(curr_file, 'w')
                f.write("%08d\n" % revnew) # we add a \n, so it is easier to hack in there manually
                f.close()

        att_from = opj(dir_from, 'pages', pn, 'attachments')
        if os.path.exists(att_from):
            att_to = opj(pagedir, 'attachments')
            copy_dir(att_from, att_to)
            fields[5] = convert_string(fields[5], enc_from, enc_to)
            line = '\t'.join(fields)
            file_to.write(line)

origdir = 'data.pre-mig2'

# Backup original dir and create new empty dir
try:
    os.rename('data', origdir)
    os.mkdir('data')
except OSError:
    print "You need to be in the directory where your copy of the 'data' directory is located."
    sys.exit(1)

convert_textdir(opj(origdir, 'text'), opj('data', 'text'), from_encoding, to_encoding)

convert_textdir(opj(origdir, 'backup'), opj('data', 'backup'), from_encoding, to_encoding, 1)

convert_pagedir(opj(origdir, 'pages'), opj('data', 'pages'), from_encoding, to_encoding)

convert_userdir(opj(origdir, 'user'), opj('data', 'user'), from_encoding, to_encoding)

convert_editlog(opj(origdir, 'editlog'), opj('data', 'editlog'), from_encoding, to_encoding)

copy_file(opj(origdir, 'event.log'), opj('data', 'event.log'))

copy_dir(opj(origdir, 'plugin'), opj('data', 'plugin'))

copy_file(opj(origdir, 'intermap.txt'), opj('data', 'intermap.txt'))

Beispiel #18
0
 def copy(self, attach_dir):
     """ copy attachment file from orig path to new destination """
     attfile = self.name.encode('utf-8')
     dest = opj(attach_dir, attfile)
     copy_file(self.path, dest)