def convertUsers(srcdir, dstdir): """ Convert users files @param srcdir: old users dir @param dstdir: new users dir """ charset = 'utf-8' # Create dstdir if not os.path.exists(dstdir): try: os.mkdir(dstdir) except OSError: migutil.fatalError("can't create user directory at '%s'" % dstdir) if not os.path.isdir(srcdir): migutil.fatalError("can't find user directory at '%s'" % srcdir) for name in migutil.listdir(srcdir): if name == 'README' or name.endswith('.trail'): # Copy as is migutil.copy_file(join(srcdir, name), join(dstdir, name)) else: srcfile = join(srcdir, name) f = codecs.open(srcfile, 'rb', charset) text = f.read() f.close() text = convertUserData(text) dstfile = join(dstdir, name) f = codecs.open(dstfile, 'wb', charset) f.write(text) f.close() print "Converted '%s' to '%s'" % (srcfile, dstfile)
def init_dest(self): try: os.makedirs(self.ddata) except: pass os.makedirs(opj(self.ddata, 'pages')) os.makedirs(opj(self.ddata, 'user')) copy_dir(opj(self.sdata, 'plugin'), opj(self.ddata, 'plugin')) copy_file(opj(self.sdata, 'intermap.txt'), opj(self.ddata, 'intermap.txt'))
def init_dest(self): try: os.makedirs(self.ddata) except: pass os.makedirs(opj(self.ddata, "pages")) os.makedirs(opj(self.ddata, "user")) copy_dir(opj(self.sdata, "plugin"), opj(self.ddata, "plugin")) copy_file(opj(self.sdata, "intermap.txt"), opj(self.ddata, "intermap.txt"))
def convert_userdir(dir_from, dir_to): os.mkdir(dir_to) for fname in listdir(dir_from): if fname.endswith('.bookmark'): bm = open(opj(dir_from, fname)).read().strip() bm = str(wikiutil.timestamp2version(float(bm))) f = open(opj(dir_to, fname), 'w') f.write(bm) f.close() else: copy_file(opj(dir_from, fname), opj(dir_to, fname))
def generate_pages(dir_from, dir_to): for pagename in info2: entry = info2.get(pagename, {}) tslist = entry.keys() if tslist: pagedir = opj(dir_to, 'pages', pagename) os.makedirs(opj(pagedir, 'revisions')) editlog_file = opj(pagedir, 'edit-log') f = open(editlog_file, 'w') rev = 0 tslist.sort() for ts in tslist: rev += 1 revstr = '%08d' % rev file_from, data = entry[ts] data[0] = str(ts) data[1] = revstr if data[2].endswith('/REVERT'): # replace the timestamp with the revision number revertts = long(data[7]) # must be long for py 2.2.x try: revertrev = int(entry[revertts][1][1]) except KeyError: # never should trigger... print "********* KeyError %s entry[%d][1][1] **********" % ( pagename, revertts) revertrev = 0 data[7] = '%08d' % revertrev f.write('\t'.join(data) + '\n') if file_from is not None: file_to = opj(pagedir, 'revisions', revstr) copy_file(file_from, file_to) f.close() curr_file = opj(pagedir, 'current') f = open(curr_file, 'w') f.write(revstr) f.close() att_from = opj(dir_from, 'pages', pagename, 'attachments') if os.path.exists(att_from): att_to = opj(pagedir, 'attachments') copy_dir(att_from, att_to)
def generate_pages(dir_from, dir_to): for pagename in info2: entry = info2.get(pagename, {}) tslist = entry.keys() if tslist: pagedir = opj(dir_to, 'pages', pagename) os.makedirs(opj(pagedir, 'revisions')) editlog_file = opj(pagedir, 'edit-log') f = open(editlog_file, 'w') rev = 0 tslist.sort() for ts in tslist: rev += 1 revstr = '%08d' % rev file_from, data = entry[ts] data[0] = str(ts) data[1] = revstr if data[2].endswith('/REVERT'): # replace the timestamp with the revision number revertts = long(data[7]) # must be long for py 2.2.x try: revertrev = int(entry[revertts][1][1]) except KeyError: # never should trigger... print "********* KeyError %s entry[%d][1][1] **********" % (pagename, revertts) revertrev = 0 data[7] = '%08d' % revertrev f.write('\t'.join(data)+'\n') if file_from is not None: file_to = opj(pagedir, 'revisions', revstr) copy_file(file_from, file_to) f.close() curr_file = opj(pagedir, 'current') f = open(curr_file, 'w') f.write(revstr) f.close() att_from = opj(dir_from, 'pages', pagename, 'attachments') if os.path.exists(att_from): att_to = opj(pagedir, 'attachments') copy_dir(att_from, att_to)
def convert_textdir(dir_from, dir_to, is_backupdir=0): for fname_from in listdir(dir_from): if is_backupdir: fname, timestamp = fname_from.split('.') else: fname = fname_from try: os.mkdir(opj(dir_to, 'pages', fname)) except: pass try: os.mkdir(opj(dir_to, 'pages', fname, 'backup')) except: pass try: os.mkdir(opj(dir_to, 'pages', fname, 'cache')) except: pass if is_backupdir: fname_to = opj('pages', fname, 'backup', timestamp) else: fname_to = opj('pages', fname, 'text') copy_file(opj(dir_from, fname_from), opj(dir_to, fname_to))
def convert_pagedir(dir_from, dir_to, is_backupdir=0): os.mkdir(dir_to) for pagedir in listdir(dir_from): text_from = opj(dir_from, pagedir, 'text') text_to = opj(dir_to, pagedir, 'text') os.mkdir(opj(dir_to, pagedir)) copy_file(text_from, text_to) backupdir_from = opj(dir_from, pagedir, 'backup') backupdir_to = opj(dir_to, pagedir, 'backup') if os.path.exists(backupdir_from): os.mkdir(backupdir_to) for ts in listdir(backupdir_from): ts_usec = str(convert_ts(float(ts))) backup_from = opj(backupdir_from, ts) backup_to = opj(backupdir_to, ts_usec) copy_file(backup_from, backup_to) editlog_from = opj(dir_from, pagedir, 'edit-log') editlog_to = opj(dir_to, pagedir, 'edit-log') convert_editlog(editlog_from, editlog_to) #cachedir_from = opj(dir_from, pagedir, 'cache') #cachedir_to = opj(dir_to, pagedir, 'cache') #if os.path.exists(cachedir_from): # os.mkdir(cachedir_to) # try: # copy_file( # opj(cachedir_from, 'hitcounts'), # opj(cachedir_to, 'hitcounts')) # except: pass attachdir_from = opj(dir_from, pagedir, 'attachments') attachdir_to = opj(dir_to, pagedir, 'attachments') if os.path.exists(attachdir_from): try: copy_dir(attachdir_from, attachdir_to) except: pass
def copy(self, attach_dir): """ copy attachment file from orig path to new destination """ attfile = self.name.encode('utf-8') dest = opj(attach_dir, attfile) copy_file(self.path, dest)
data = editlog[ts] f.write('\t'.join(data) + '\n') f.close() origdir = 'data.pre-mig5' # Backup original dir and create new empty dir try: os.rename('data', origdir) os.mkdir('data') except OSError: print "You need to be in the directory where your copy of the 'data' directory is located." sys.exit(1) gather_editlog(origdir, opj(origdir, 'edit-log')) gather_pagedirs(opj(origdir, 'pages')) remove_trash(opj(origdir, 'pages')) generate_pages(origdir, 'data') generate_editlog(origdir, 'data') copy_dir(opj(origdir, 'plugin'), opj('data', 'plugin')) copy_dir(opj(origdir, 'user'), opj('data', 'user')) copy_file(opj(origdir, 'event-log'), opj('data', 'event-log')) copy_file(opj(origdir, 'intermap.txt'), opj('data', 'intermap.txt'))
f.write("\t".join(data) + "\n") f.close() origdir = "data.pre-mig5" # Backup original dir and create new empty dir try: os.rename("data", origdir) os.mkdir("data") except OSError: print "You need to be in the directory where your copy of the 'data' directory is located." sys.exit(1) gather_editlog(origdir, opj(origdir, "edit-log")) gather_pagedirs(opj(origdir, "pages")) remove_trash(opj(origdir, "pages")) generate_pages(origdir, "data") generate_editlog(origdir, "data") copy_dir(opj(origdir, "plugin"), opj("data", "plugin")) copy_dir(opj(origdir, "user"), opj("data", "user")) copy_file(opj(origdir, "event-log"), opj("data", "event-log")) copy_file(opj(origdir, "intermap.txt"), opj("data", "intermap.txt"))
def generate_pages(dir_from, dir_to): revactions = ['SAVE', 'SAVENEW', 'SAVE/REVERT', ] # these actions create revisions for pn in info: entry = info.get(pn, {}) tslist = entry.keys() if tslist: pagedir = opj(dir_to, 'pages', pn) revdir = opj(pagedir, 'revisions') os.makedirs(revdir) editlog_file = opj(pagedir, 'edit-log') f = open(editlog_file, 'w') revnew = 0 tslist.sort() for ts in tslist: data = entry[ts][1] datanew = data[:] (timestamp, rev, action, pagename, ip, host, id, extra, comment) = data revstr = '%08d' % rev if action in revactions: revnew += 1 revnewstr = '%08d' % revnew entry[ts][0] = revnew # remember what new revno we chose else: # ATTNEW,ATTDRW,ATTDEL revnewstr = '99999999' if action.endswith('/REVERT'): # replace the old revno with the correct new revno revertrevold = int(extra) revertrevnew = 0 for ts2 in tslist: data2 = entry[ts2][1] (timestamp2, rev2, action2, pagename2, ip2, host2, id2, extra2, comment2) = data2 if rev2 == revertrevold: revertrevnew = entry[ts2][0] datanew[7] = '%08d' % revertrevnew datanew[1] = revnewstr f.write('\t'.join(datanew)+'\n') # does make a CRLF on win32 in the file if action in revactions: # we DO have a page rev for this one file_from = opj(dir_from, 'pages', pn, 'revisions', revstr) file_to = opj(revdir, revnewstr) copy_file(file_from, file_to) f.close() # check if page exists or is deleted in orig dir pagedir_from = opj(dir_from, 'pages', pn) revdir_from = opj(pagedir_from, 'revisions') try: curr_file_from = opj(pagedir_from, 'current') currentfrom = open(curr_file_from).read().strip() # try to access it page_exists = 1 except: page_exists = 0 # re-make correct DELETED status! if page_exists: curr_file = opj(pagedir, 'current') f = open(curr_file, 'w') f.write("%08d\n" % revnew) # we add a \n, so it is easier to hack in there manually f.close() att_from = opj(dir_from, 'pages', pn, 'attachments') if os.path.exists(att_from): att_to = opj(pagedir, 'attachments') copy_dir(att_from, att_to)
fields[5] = convert_string(fields[5], enc_from, enc_to) line = '\t'.join(fields) file_to.write(line) origdir = 'data.pre-mig2' # Backup original dir and create new empty dir try: os.rename('data', origdir) os.mkdir('data') except OSError: print "You need to be in the directory where your copy of the 'data' directory is located." sys.exit(1) convert_textdir(opj(origdir, 'text'), opj('data', 'text'), from_encoding, to_encoding) convert_textdir(opj(origdir, 'backup'), opj('data', 'backup'), from_encoding, to_encoding, 1) convert_pagedir(opj(origdir, 'pages'), opj('data', 'pages'), from_encoding, to_encoding) convert_userdir(opj(origdir, 'user'), opj('data', 'user'), from_encoding, to_encoding) convert_editlog(opj(origdir, 'editlog'), opj('data', 'editlog'), from_encoding, to_encoding) copy_file(opj(origdir, 'event.log'), opj('data', 'event.log')) copy_dir(opj(origdir, 'plugin'), opj('data', 'plugin')) copy_file(opj(origdir, 'intermap.txt'), opj('data', 'intermap.txt'))