def _load_dict(self): request = self.request dict_name = self.name page = Page(request, dict_name) if page.exists(): arena = "pagedicts" key = wikiutil.quoteWikinameFS(dict_name) cache = caching.CacheEntry(request, arena, key, scope="wiki", use_pickle=True) try: cache_mtime = cache.mtime() page_mtime = wikiutil.version2timestamp(page.mtime_usecs()) # TODO: fix up-to-date check mtime granularity problems. # # cache_mtime is float while page_mtime is integer # The comparision needs to be done on the lowest type of both if int(cache_mtime) > int(page_mtime): # cache is uptodate return cache.content() else: raise caching.CacheError except caching.CacheError: # either cache does not exist, is erroneous or not uptodate: recreate it d = super(WikiDict, self)._load_dict() cache.update(d) return d else: raise DictDoesNotExistError(dict_name)
def _load_group(self): request = self.request group_name = self.name page = Page(request, group_name) if page.exists(): arena = 'pagegroups' key = wikiutil.quoteWikinameFS(group_name) cache = caching.CacheEntry(request, arena, key, scope='wiki', use_pickle=True) try: cache_mtime = cache.mtime() page_mtime = wikiutil.version2timestamp(page.mtime_usecs()) # TODO: fix up-to-date check mtime granularity problems. # # cache_mtime is float while page_mtime is integer # The comparision needs to be done on the lowest type of both if int(cache_mtime) > int(page_mtime): # cache is uptodate return cache.content() else: raise caching.CacheError except caching.CacheError: # either cache does not exist, is erroneous or not uptodate: recreate it members, member_groups = super(WikiGroup, self)._load_group() cache.update((members, member_groups)) return members, member_groups else: raise GroupDoesNotExistError(group_name)
def _load_group(self): request = self.request group_name = self.name page = Page(request, group_name) if page.exists(): arena = 'pagegroups' key = wikiutil.quoteWikinameFS(group_name) cache = caching.CacheEntry(request, arena, key, scope='wiki', use_pickle=True) try: cache_mtime = cache.mtime() page_mtime = wikiutil.version2timestamp(page.mtime_usecs()) # TODO: fix up-to-date check mtime granularity problems. # # cache_mtime is float while page_mtime is integer # The comparision needs to be done on the lowest type of both if int(cache_mtime) > int(page_mtime): # cache is uptodate return cache.content() else: raise caching.CacheError except caching.CacheError: # either cache does not exist, is erroneous or not uptodate: recreate it members, member_groups = super(WikiGroup, self)._load_group() cache.update((members, member_groups)) return members, member_groups else: raise GroupDoesNotExistError(group_name)
def packagePages(self, pagelist, filename, function): """ Puts pages from pagelist into filename and calls function on them on installation. """ request = self.request try: os.remove(filename) except OSError: pass zf = zipfile.ZipFile(filename, "w", COMPRESSION_LEVEL) cnt = 0 script = [packLine(['MoinMoinPackage', '1']), ] for pagename in pagelist: pagename = pagename.strip() page = Page(request, pagename) if page.exists(): cnt += 1 script.append(packLine([function, str(cnt), pagename])) timestamp = wikiutil.version2timestamp(page.mtime_usecs()) zi = zipfile.ZipInfo(filename=str(cnt), date_time=datetime.fromtimestamp(timestamp).timetuple()[:6]) zi.compress_type = COMPRESSION_LEVEL zf.writestr(zi, page.get_raw_body().encode("utf-8")) else: #import sys #print >>sys.stderr, "Could not find the page %s." % pagename.encode("utf-8") pass script += [packLine(['Print', 'Installed MoinMaster page bundle %s.' % os.path.basename(filename)])] zf.writestr(MOIN_PACKAGE_FILE, u"\n".join(script).encode("utf-8")) zf.close()
def collectpackage(self, pagelist, fileobject, pkgname="", include_attachments=False): """ Expects a list of pages as an argument, and fileobject to be an open file object, which a zipfile will get written to. @param pagelist: pages to package @param fileobject: open file object to write to @param pkgname: optional file name, to prevent self packaging @rtype: string or None @return: error message, if one happened @rtype: boolean @param include_attachments: True if you want attachments collected """ _ = self.request.getText COMPRESSION_LEVEL = zipfile.ZIP_DEFLATED pages = [] for pagename in pagelist: pagename = wikiutil.normalize_pagename(pagename, self.request.cfg) if pagename: page = Page(self.request, pagename) if page.exists() and self.request.user.may.read(pagename): pages.append(page) if not pages: return (_('No pages like "%s"!') % wikiutil.escape(pagelist)) # Set zipfile output zf = zipfile.ZipFile(fileobject, "w", COMPRESSION_LEVEL) cnt = 0 userid = user.getUserIdentification(self.request) script = [packLine(['MoinMoinPackage', '1']), ] for page in pages: cnt += 1 files = _get_files(self.request, page.page_name) script.append(packLine(["AddRevision", str(cnt), page.page_name, userid, "Created by the PackagePages action."])) timestamp = wikiutil.version2timestamp(page.mtime_usecs()) # avoid getting strange exceptions from zipfile in case of pre-1980 timestamps nineteeneighty = (10 * 365 + 3) * 24 * 3600 # 1970 + 10y + 3d timestamp = max(nineteeneighty, timestamp) # zip can not store timestamps before 1980 zi = zipfile.ZipInfo(filename=str(cnt), date_time=datetime.fromtimestamp(timestamp).timetuple()[:6]) zi.compress_type = COMPRESSION_LEVEL zf.writestr(zi, page.get_raw_body().encode("utf-8")) if include_attachments: for attname in files: if attname != pkgname: cnt += 1 zipname = "%d_attachment" % cnt script.append(packLine(["AddAttachment", zipname, attname, page.page_name, userid, "Created by the PackagePages action."])) filename = AttachFile.getFilename(self.request, page.page_name, attname) zf.write(filename, zipname) script += [packLine(['Print', 'Thank you for using PackagePages!'])] zf.writestr(MOIN_PACKAGE_FILE, u"\n".join(script).encode("utf-8")) zf.close()
def packagePages(self, pagelist, filename, function): """ Puts pages from pagelist into filename and calls function on them on installation. """ request = self.request try: os.remove(filename) except OSError: pass # page LanguageSetup needs no packing! existing_pages = [ pagename for pagename in pagelist if Page(request, pagename).exists() and pagename != 'LanguageSetup' ] if not existing_pages: return zf = zipfile.ZipFile(filename, "w", COMPRESSION_LEVEL) script = [ packLine(['MoinMoinPackage', '1']), ] cnt = 0 for pagename in existing_pages: pagename = pagename.strip() page = Page(request, pagename) files = _get_files(request, pagename) for attname in files: cnt += 1 zipname = "%d" % cnt script.append( packLine([ "ReplaceUnderlayAttachment", zipname, attname, pagename ])) attpath = AttachFile.getFilename(request, pagename, attname) zf.write(attpath, zipname) cnt += 1 zipname = "%d" % cnt script.append(packLine([function, zipname, pagename])) timestamp = wikiutil.version2timestamp(page.mtime_usecs()) zi = zipfile.ZipInfo( filename=zipname, date_time=datetime.fromtimestamp(timestamp).timetuple()[:6]) zi.compress_type = COMPRESSION_LEVEL zf.writestr(zi, page.get_raw_body().encode("utf-8")) script += [ packLine([ 'Print', 'Installed MoinMaster page bundle %s.' % os.path.basename(filename) ]) ] zf.writestr(MOIN_PACKAGE_FILE, u"\n".join(script).encode("utf-8")) zf.close()
def _index_page_rev(self, request, connection, pagename, revno, mode='update'): """ Index a page revision. @param request: request suitable for indexing @param connection: the Indexer connection object @param pagename: the page name @param revno: page revision number (int) @param mode: 'add' = just add, no checks 'update' = check if already in index and update if needed (mtime) """ page = Page(request, pagename, rev=revno) request.page = page # XXX for what is this needed? wikiname = request.cfg.interwikiname or u"Self" revision = str(page.get_real_rev()) itemid = "%s:%s:%s" % (wikiname, pagename, revision) mtime = page.mtime_usecs() doc = self._get_document(connection, itemid, mtime, mode) logging.debug("%s %s %r" % (pagename, revision, doc)) if doc: mimetype = 'text/%s' % page.pi['format'] # XXX improve this fields = {} fields['wikiname'] = wikiname fields['pagename'] = pagename fields['attachment'] = '' # this is a real page, not an attachment fields['mtime'] = str(mtime) fields['revision'] = revision fields['title'] = pagename fields['content'] = page.get_raw_body() fields['lang'], fields['stem_lang'] = self._get_languages(page) fields['author'] = page.edit_info().get('editor', '?') multivalued_fields = {} multivalued_fields['mimetype'] = [ mt for mt in [mimetype] + mimetype.split('/') ] multivalued_fields['domain'] = self._get_domains(page) multivalued_fields['linkto'] = page.getPageLinks(request) multivalued_fields['category'] = self._get_categories(page) self._add_fields_to_document(request, doc, fields, multivalued_fields) try: connection.replace(doc) except xappy.IndexerError, err: logging.warning("IndexerError at %r %r %r (%s)" % (wikiname, pagename, revision, str(err)))
def convert_editlog(page, output = None, overwrite = False): pagedir = page.getPagePath() pagename = wikiname(pagedir) if not output: output = pagename edit_log = editlog.EditLog(request, page.getPagePath('edit-log')) changes = {} for log in edit_log: # not supported. perhaps add anyway? if log.action in ('ATTNEW', 'ATTDEL', 'ATTDRW'): continue # 1201095949 192.168.2.23 E start [email protected] author = log.hostname if log.userid: userdata = user.User(request, log.userid) if userdata.name: author = userdata.name try: action = { 'SAVE' : 'E', 'SAVENEW' : 'C', 'SAVE/REVERT' : 'R', }[log.action] except KeyError: action = log.action mtime = str(log.ed_time_usecs / USEC) changes[mtime] = u"\t".join([mtime, log.addr, action, dw.cleanID(log.pagename), author, log.comment]) # see if we have missing entries, try to recover page = Page(request, pagename) if len(page.getRevList()) != len(changes): print "RECOVERING edit-log, missing %d entries" % (len(page.getRevList()) - len(changes)) for rev in page.getRevList(): page = Page(request, pagename, rev = rev) mtime = page.mtime_usecs() / USEC if not mtime: pagefile, realrev, exists = page.get_rev(rev = rev); if os.path.exists(pagefile): mtime = int(os.path.getmtime(pagefile)) print "Recovered %s: %s" % (rev, mtime) mtime = str(mtime) if not changes.has_key(mtime): changes[mtime] = u"\t".join([mtime, '127.0.0.1', '?', dw.cleanID(pagename), 'root', 'recovered entry']) print "ADDING %s" % mtime changes = sorted(changes.values()) out_file = os.path.join(output_dir, 'meta', dw.metaFN(output, '.changes')) writefile(out_file, changes, overwrite = overwrite)
def packagePages(self, pagelist, filename, function): """ Puts pages from pagelist into filename and calls function on them on installation. """ request = self.request try: os.remove(filename) except OSError: pass # page LanguageSetup needs no packing! existing_pages = [ pagename for pagename in pagelist if Page(request, pagename).exists() and pagename != "LanguageSetup" ] if not existing_pages: return zf = zipfile.ZipFile(filename, "w", COMPRESSION_LEVEL) script = [packLine(["MoinMoinPackage", "1"])] fallback_timestamp = int(time.time()) cnt = 0 for pagename in existing_pages: pagename = pagename.strip() page = Page(request, pagename) files = _get_files(request, pagename) for attname in files: cnt += 1 zipname = "%d" % cnt script.append(packLine(["ReplaceUnderlayAttachment", zipname, attname, pagename])) attpath = AttachFile.getFilename(request, pagename, attname) zf.write(attpath, zipname) cnt += 1 zipname = "%d" % cnt script.append(packLine([function, zipname, pagename])) timestamp = wikiutil.version2timestamp(page.mtime_usecs()) if not timestamp: # page.mtime_usecs() returns 0 for underlay pages timestamp = fallback_timestamp dt = datetime.fromtimestamp(timestamp) zi = zipfile.ZipInfo(filename=zipname, date_time=dt.timetuple()[:6]) zi.compress_type = COMPRESSION_LEVEL zf.writestr(zi, page.get_raw_body().encode("utf-8")) script += [packLine(["Print", "Installed MoinMaster page bundle %s." % os.path.basename(filename)])] zf.writestr(MOIN_PACKAGE_FILE, u"\n".join(script).encode("utf-8")) zf.close()
def _index_page_rev(self, request, connection, pagename, revno, mode='update'): """ Index a page revision. @param request: request suitable for indexing @param connection: the Indexer connection object @param pagename: the page name @param revno: page revision number (int) @param mode: 'add' = just add, no checks 'update' = check if already in index and update if needed (mtime) """ page = Page(request, pagename, rev=revno) request.page = page # XXX for what is this needed? wikiname = request.cfg.interwikiname or u"Self" revision = str(page.get_real_rev()) itemid = "%s:%s:%s" % (wikiname, pagename, revision) mtime = page.mtime_usecs() doc = self._get_document(connection, itemid, mtime, mode) logging.debug("%s %s %r" % (pagename, revision, doc)) if doc: mimetype = 'text/%s' % page.pi['format'] # XXX improve this fields = {} fields['wikiname'] = wikiname fields['pagename'] = pagename fields['attachment'] = '' # this is a real page, not an attachment fields['mtime'] = str(mtime) fields['revision'] = revision fields['title'] = pagename fields['content'] = page.get_raw_body() fields['lang'], fields['stem_lang'] = self._get_languages(page) fields['author'] = page.edit_info().get('editor', '?') multivalued_fields = {} multivalued_fields['mimetype'] = [mt for mt in [mimetype] + mimetype.split('/')] multivalued_fields['domain'] = self._get_domains(page) multivalued_fields['linkto'] = page.getPageLinks(request) multivalued_fields['category'] = self._get_categories(page) self._add_fields_to_document(request, doc, fields, multivalued_fields) try: connection.replace(doc) except xappy.IndexerError, err: logging.warning("IndexerError at %r %r %r (%s)" % ( wikiname, pagename, revision, str(err)))
def execute(pagename, request): pagename_header = '%s-%s.zip' % (pagename, datetime.now().isoformat()[:10]) pagename_header = pagename_header.encode('ascii', 'ignore') request.content_type = 'application/zip' request.headers['Content-Disposition'] = \ 'attachment; filename="%s"' % pagename_header args = values_to_form(request.values) try: args = args['args'][0] except (KeyError, IndexError): args = u'' pagelist, metakeys, _ = metatable_parseargs(request, args, get_all_keys=True) renameDict = dict() for page in pagelist: metas = get_metas(request, page, ["gwikirename"], abs_attach=False, checkAccess=False) renameList = metas["gwikirename"] if renameList: renameDict[page] = renameList output = StringIO() zip = zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) userid = user.getUserIdentification(request) script = [ packLine(['MoinMoinPackage', '1']), ] counter = 0 for pagename in pagelist: counter += 1 page = Page(request, pagename) timestamp = wikiutil.version2timestamp(page.mtime_usecs()) # Underlay pages are in epoch 0, zipfile in python 2.7 does # not support this. if not timestamp: pagefile, rev, exists = page.get_rev() if rev == 99999999: # We should never get here log.error("Page %s neither in pages or underlay, skipping." % (pagename)) continue timestamp = os.path.getctime(pagefile) pagetext = page.get_raw_body().encode("utf-8") filename = str(counter) zinfo = zipfile.ZipInfo( filename=filename, date_time=datetime.fromtimestamp(timestamp).timetuple()[:6]) zinfo.compress_type = zipfile.ZIP_DEFLATED zip.writestr(zinfo, pagetext) targetNameList = renameDict.get(pagename, [pagename]) for targetName in targetNameList: script.append( packLine(["AddRevision", filename, targetName, userid, ""])) for attachment in _get_files(request, pagename): counter += 1 sourcefile = AttachFile.getFilename(request, pagename, attachment) filename = str(counter) + "-attachment" zip.write(sourcefile, filename) script.append( packLine([ "AddAttachment", filename, attachment, pagename, userid, "" ])) zip.writestr(MOIN_PACKAGE_FILE, u"\n".join(script).encode("utf-8")) zip.close() request.write(output.getvalue())
def convertfile(page, output=None, overwrite=False): pagedir = page.getPagePath() pagename = wikiname(pagedir) if not output: output = pagename print "Converting %s" % pagename if page.isUnderlayPage(): print "underlay: %s" % page.request.cfg.data_underlay_dir print "underlay: %s" % request.cfg.data_underlay_dir print "SKIP UNDERLAY: %s" % pagename return False current_exists = page.exists() current_rev = page.current_rev() if convert_attic: revs = page.getRevList() else: revs = [current_rev] # Generate random ID Number for collision avoidance when attachments in Namespace have the same name randomID = random.randint(101, 999) for rev in revs: page = Page(request, pagename, rev=rev) pagefile, realrev, exists = page.get_rev(rev=rev) mtime = page.mtime_usecs() / USEC if not mtime: if os.path.exists(pagefile) != exists: raise Exception, "IT SHOULD NOT HAPPEN" if os.path.exists(pagefile): mtime = int(os.path.getmtime(pagefile)) print "recovered %s: %s" % (rev, mtime) if not mtime: print "NO REVISION: for %s" % pagefile continue if rev == current_rev: out_file = os.path.join(output_dir, 'pages', dw.wikiFN(output)) if not convert_attic and not exists: # if not converting attic, allow current version may not exist anymore continue else: out_file = os.path.join(output_dir, 'attic', dw.wikiFN(output, str(mtime))) content = moin2doku(pagename, page.get_raw_body(), randomID) if len(content) == 0: # raise Exception, "No content" print "NO CONTENT: exists: %s,%s" % (exists, os.path.exists(pagefile)) writefile(out_file, content, overwrite=overwrite) copystat(pagefile, out_file) ID = dw.cleanID(output) copy_attachments(page, dw.getNS(ID), randomID) # convert edit-log, it's always present even if current page is not convert_editlog(page, output=output, overwrite=overwrite) # add to redirect.conf if filenames differ # and page must exist (no redirect for deleted pages) if redirect_conf and current_exists: # redirect dokuwiki plugin is quite picky # - it doesn't understand if entries are not lowercase # - it doesn't understand if paths are separated by forward slash old_page = pagename.lower().replace('/', ':').replace(' ', '_') if old_page != ID: redirect_map[old_page] = ID print "Converted %s as %s" % (pagename, dw.wikiFN(output)) return True
def convert_editlog(page, output=None, overwrite=False): pagedir = page.getPagePath() pagename = wikiname(pagedir) if not output: output = pagename edit_log = editlog.EditLog(request, page.getPagePath('edit-log')) changes = {} for log in edit_log: # not supported. perhaps add anyway? if log.action in ('ATTNEW', 'ATTDEL', 'ATTDRW'): continue # 1201095949 192.168.2.23 E start [email protected] author = log.hostname if log.userid: userdata = user.User(request, log.userid) if userdata.name: author = userdata.name try: action = { 'SAVE': 'E', 'SAVENEW': 'C', 'SAVE/REVERT': 'R', }[log.action] except KeyError: action = log.action mtime = str(log.ed_time_usecs / USEC) changes[mtime] = u"\t".join([ mtime, log.addr, action, dw.cleanID(log.pagename), author, log.comment ]) # see if we have missing entries, try to recover page = Page(request, pagename) if len(page.getRevList()) != len(changes): print "RECOVERING edit-log, missing %d entries" % ( len(page.getRevList()) - len(changes)) for rev in page.getRevList(): page = Page(request, pagename, rev=rev) mtime = page.mtime_usecs() / USEC if not mtime: pagefile, realrev, exists = page.get_rev(rev=rev) if os.path.exists(pagefile): mtime = int(os.path.getmtime(pagefile)) print "Recovered %s: %s" % (rev, mtime) mtime = str(mtime) if not changes.has_key(mtime): changes[mtime] = u"\t".join([ mtime, '127.0.0.1', '?', dw.cleanID(pagename), 'root', 'recovered entry' ]) print "ADDING %s" % mtime changes = sorted(changes.values()) out_file = os.path.join(output_dir, 'meta', dw.metaFN(output, '.changes')) writefile(out_file, changes, overwrite=overwrite)
def execute(pagename, request): pagename_header = "%s-%s.zip" % (pagename, datetime.now().isoformat()[:10]) pagename_header = pagename_header.encode("ascii", "ignore") request.content_type = "application/zip" request.headers["Content-Disposition"] = 'attachment; filename="%s"' % pagename_header args = values_to_form(request.values) try: args = args["args"][0] except (KeyError, IndexError): args = u"" pagelist, metakeys, _ = metatable_parseargs(request, args, get_all_keys=True) renameDict = dict() for page in pagelist: metas = get_metas(request, page, ["gwikirename"], abs_attach=False, checkAccess=False) renameList = metas["gwikirename"] if renameList: renameDict[page] = renameList output = StringIO() zip = zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) userid = user.getUserIdentification(request) script = [packLine(["MoinMoinPackage", "1"])] counter = 0 for pagename in pagelist: counter += 1 page = Page(request, pagename) timestamp = wikiutil.version2timestamp(page.mtime_usecs()) # Underlay pages are in epoch 0, zipfile in python 2.7 does # not support this. if not timestamp: pagefile, rev, exists = page.get_rev() if rev == 99999999: # We should never get here log.error("Page %s neither in pages or underlay, skipping." % (pagename)) continue timestamp = os.path.getctime(pagefile) pagetext = page.get_raw_body().encode("utf-8") filename = str(counter) zinfo = zipfile.ZipInfo(filename=filename, date_time=datetime.fromtimestamp(timestamp).timetuple()[:6]) zinfo.compress_type = zipfile.ZIP_DEFLATED zip.writestr(zinfo, pagetext) targetNameList = renameDict.get(pagename, [pagename]) for targetName in targetNameList: script.append(packLine(["AddRevision", filename, targetName, userid, ""])) for attachment in _get_files(request, pagename): counter += 1 sourcefile = AttachFile.getFilename(request, pagename, attachment) filename = str(counter) + "-attachment" zip.write(sourcefile, filename) script.append(packLine(["AddAttachment", filename, attachment, pagename, userid, ""])) zip.writestr(MOIN_PACKAGE_FILE, u"\n".join(script).encode("utf-8")) zip.close() request.write(output.getvalue())
def convertfile(page, output = None, overwrite = False): pagedir = page.getPagePath() pagename = wikiname(pagedir) if not output: output = pagename if page.isUnderlayPage(): print "underlay: %s" % page.request.cfg.data_underlay_dir print "underlay: %s" % request.cfg.data_underlay_dir print "SKIP UNDERLAY: %s" % pagename return False current_exists = page.exists() current_rev = page.current_rev() if convert_attic: revs = page.getRevList() else: revs = [current_rev] # Generate random ID Number for collision avoidance when attachments in Namespace have the same name randomID = random.randint(101,999) for rev in revs: page = Page(request, pagename, rev = rev) pagefile, realrev, exists = page.get_rev(rev = rev); mtime = page.mtime_usecs() / USEC if not mtime: if os.path.exists(pagefile) != exists: raise Exception, "IT SHOULD NOT HAPPEN" if os.path.exists(pagefile): mtime = int(os.path.getmtime(pagefile)) print "recovered %s: %s" % (rev, mtime) if not mtime: print "NO REVISION: for %s" % pagefile continue if rev == current_rev: out_file = os.path.join(output_dir, 'pages', dw.wikiFN(output)) if not convert_attic and not exists: # if not converting attic, allow current version may not exist anymore continue else: out_file = os.path.join(output_dir, 'attic', dw.wikiFN(output, str(mtime))) content = moin2doku(pagename, page.get_raw_body(),randomID) if len(content) == 0: # raise Exception, "No content" print "NO CONTENT: exists: %s,%s" % (exists, os.path.exists(pagefile)) writefile(out_file, content, overwrite = overwrite) copystat(pagefile, out_file) ID = dw.cleanID(output) copy_attachments(page, dw.getNS(ID),randomID) # convert edit-log, it's always present even if current page is not convert_editlog(page, output = output, overwrite = overwrite) # add to redirect.conf if filenames differ # and page must exist (no redirect for deleted pages) if redirect_conf and current_exists: # redirect dokuwiki plugin is quite picky # - it doesn't understand if entries are not lowercase # - it doesn't understand if paths are separated by forward slash old_page = pagename.lower().replace('/', ':').replace(' ', '_') if old_page != ID: redirect_map[old_page] = ID print "Converted %s as %s" % (pagename, dw.wikiFN(output)) return True