def run(self, query):
before_wiki()
if query:
q = And([
Term(WIKINAME, app.cfg.interwikiname),
Regex(NAME_EXACT, query)
])
else:
q = Every()
for current_rev in app.storage.search(q, limit=None):
current_name = current_rev.meta[NAME]
current_revid = current_rev.meta[REVID]
print "Destroying historical revisions of {0!r}:".format(
current_name)
has_historical_revision = False
for rev in current_rev.item.iter_revs():
revid = rev.meta[REVID]
if revid == current_revid:
# fixup metadata and overwrite existing revision; modified time will be updated if changed
changed = False
meta = dict(rev.meta)
if REV_NUMBER in meta and meta[
REV_NUMBER] > 1 or REV_NUMBER not in meta:
changed = True
meta[REV_NUMBER] = 1
if PARENTID in meta:
changed = True
del meta[PARENTID]
if changed:
current_rev.item.store_revision(meta,
current_rev.data,
overwrite=True)
print " (current rev meta data updated)"
continue
has_historical_revision = True
name = rev.meta[NAME]
if name == current_name:
print " Destroying revision {0}".format(revid)
else:
print " Destroying revision {0} (named {1!r})".format(
revid, name)
current_rev.item.destroy_revision(revid)
if not has_historical_revision:
print " (no historical revisions)"
print "Finished reducing backend."
def run(self,
directory='HTML',
theme='topside_cms',
exclude_ns='userprofiles',
user=None,
query=None):
if theme:
app.cfg.user_defaults[THEME_NAME] = theme
exclude_ns = exclude_ns.split(',') if exclude_ns else []
before_wiki()
norm = os.path.normpath
join = os.path.join
if '/' in directory:
# user has specified complete path to root
html_root = directory
else:
html_root = norm(join(app.cfg.wikiconfig_dir, directory))
repo_root = norm(join(app.cfg.wikiconfig_dir))
moinmoin = norm(join(app.cfg.wikiconfig_dir, 'src', 'moin'))
# override ACLs with permission to read all items
for namespace, acls in app.cfg.acl_mapping:
acls['before'] = 'All:read'
# create an empty output directory after deleting any existing directory
print(
'Creating output directory {0}, starting to copy supporting files'.
format(html_root))
if os.path.exists(html_root):
shutil.rmtree(html_root, ignore_errors=False)
else:
os.makedirs(html_root)
# create subdirectories and copy static css, icons, images into "static" subdirectory
shutil.copytree(norm(join(moinmoin, 'static')),
norm(join(html_root, 'static')))
shutil.copytree(norm(join(repo_root, 'wiki_local')),
norm(join(html_root, '+serve/wiki_local')))
# copy files from xstatic packaging into "+serve" subdirectory
pkg = app.cfg.pkg
xstatic_dirs = [
'font_awesome', 'jquery', 'jquery_tablesorter', 'autosize'
]
if theme in [
'basic',
]:
xstatic_dirs.append('bootstrap')
for dirs in xstatic_dirs:
xs = XStatic(getattr(pkg, dirs),
root_url='/static',
provider='local',
protocol='http')
shutil.copytree(xs.base_dir, norm(join(html_root, '+serve', dirs)))
# copy directories for theme's static files
theme = app.cfg.user_defaults[THEME_NAME]
if theme == 'topside_cms':
# topside_cms uses topside CSS files
from_dir = norm(join(moinmoin, 'themes/topside/static'))
else:
from_dir = norm(join(moinmoin, 'themes', theme, 'static'))
to_dir = norm(join(html_root, '_themes', theme))
shutil.copytree(from_dir, to_dir)
# convert: <img alt="svg" src="/+get/+7cb364b8ca5d4b7e960a4927c99a2912/svg" />
# to: <img alt="svg" src="+get/svg" />
invalid_src = re.compile(r' src="/\+get/\+[0-9a-f]{32}/')
valid_src = ' src="+get/'
# get ready to render and copy individual items
names = []
home_page = None
get_dir = norm(join(
html_root, '+get')) # images and other raw data from wiki content
os.makedirs(get_dir)
if query:
q = And([
Term(WIKINAME, app.cfg.interwikiname),
Regex(NAME_EXACT, query)
])
else:
q = Every()
print('Starting to dump items')
for current_rev in app.storage.search(q, limit=None, sortedby="name"):
if current_rev.namespace in exclude_ns:
# we usually do not copy userprofiles, no one can login to a static wiki
continue
if not current_rev.name:
# TODO: we skip nameless tickets, but named tickets and comments are processed with ugly names
continue
try:
item_name = current_rev.fqname.fullname
rendered = show_item(
item_name, CURRENT) # @@@ userid is needed for acls here
# convert / characters in sub-items and namespaces and save names for index
file_name = item_name.replace('/', SLASH)
filename = norm(join(html_root, file_name))
names.append(file_name)
except Forbidden:
print('Failed to dump {0}: Forbidden'.format(current_rev.name))
continue
except KeyError:
print('Failed to dump {0}: KeyError'.format(current_rev.name))
continue
if not isinstance(rendered, str):
print('Rendering failed for {0} with response {1}'.format(
file_name, rendered))
continue
# make hrefs relative to current folder
rendered = rendered.replace('href="/', 'href="')
rendered = rendered.replace('src="/static/', 'src="static/')
rendered = rendered.replace('src="/+serve/', 'src="+serve/')
rendered = rendered.replace(
'href="+index/"',
'href="+index"') # trailing slash changes relative position
rendered = rendered.replace('<a href="">', '<a href="{0}">'.format(
app.cfg.default_root)) # TODO: fix basic theme
# remove item ID from: src="/+get/+7cb364b8ca5d4b7e960a4927c99a2912/svg"
rendered = re.sub(invalid_src, valid_src, rendered)
rendered = self.subitems(rendered)
# copy raw data for all items to output /+get directory; images are required, text items are of marginal/no benefit
item = app.storage[current_rev.fqname.fullname]
rev = item[CURRENT]
with open(get_dir + '/' + file_name, 'wb') as f:
shutil.copyfileobj(rev.data, f)
# save rendered items or raw data to dump directory root
contenttype = item.meta['contenttype'].split(';')[0]
if contenttype in CONTENTTYPE_MEDIA and filename.endswith(
CONTENTTYPE_MEDIA_SUFFIX):
# do not put a rendered html-formatted file with a name like video.mp4 into root; browsers want raw data
with open(filename, 'wb') as f:
rev.data.seek(0)
shutil.copyfileobj(rev.data, f)
print('Saved file named {0} as raw data'.format(filename))
else:
with open(filename, 'wb') as f:
f.write(rendered.encode('utf8'))
print('Saved file named {0}'.format(filename))
if current_rev.name == app.cfg.default_root:
# make duplicates of home page that are easy to find in directory list and open with a click
for target in [(current_rev.name + '.html'),
('_' + current_rev.name + '.html')]:
with open(norm(join(html_root, target)), 'wb') as f:
f.write(rendered.encode('utf8'))
home_page = rendered # save a copy for creation of index page
if home_page:
# create an index page by replacing the content of the home page with a list of items
# work around differences in basic and modernized theme layout
# TODO: this is likely to break as new themes are added
if theme == 'basic':
start = '<div class="moin-content" role="main">' # basic
end = '<footer class="navbar moin-footer">'
div_end = '</div>'
else:
start = '<div id="moin-content">' # modernized , topside, topside cms
end = '<footer id="moin-footer">'
div_end = '</div></div>'
# build a page named "+index" containing links to all wiki items
ul = '<h1>Index</h1><ul>{0}</ul>'
li = '<li><a href="{0}">{1}</a></li>'
links = []
names.sort()
for name in names:
links.append(li.format(name, name.replace(SLASH, '/')))
name_links = ul.format('\n'.join(links))
try:
part1 = home_page.split(start)[0]
part2 = home_page.split(end)[1]
page = part1 + start + name_links + div_end + end + part2
except IndexError:
page = home_page
print('Error: failed to find {0} in item named {1}'.format(
end, app.cfg.default_root))
for target in ['+index', '_+index.html']:
with open(norm(join(html_root, target)), 'wb') as f:
f.write(page.encode('utf8'))
else:
print(
'Error: no item matching name in app.cfg.default_root was found'
)
def recurse(self, elem, page_href):
# on first call, elem.tag.name=='page'.
# Descendants (body, div, p, include, page, etc.) are processed by recursing through DOM
# stack is used to detect transclusion loops
page_href_new = elem.get(moin_page.page_href)
if page_href_new:
page_href_new = Iri(page_href_new)
if page_href_new != page_href:
page_href = page_href_new
self.stack.append(page_href)
else:
self.stack.append(None)
else:
self.stack.append(None)
try:
if elem.tag == xinclude.include:
# we have already recursed several levels and found a transclusion: "{{SomePage}}" or <<Include(...)>>
# process the transclusion and add it to the DOM. Subsequent recursions will traverse through
# the transclusion's elements.
href = elem.get(xinclude.href)
xpointer = elem.get(xinclude.xpointer)
xp_include_pages = None
xp_include_sort = None
xp_include_items = None
xp_include_skipitems = None
xp_include_heading = None
xp_include_level = None
if xpointer:
# we are working on an <<Include(abc)>> macro, not a {{transclusion}}
xp = XPointer(xpointer)
xp_include = None
xp_namespaces = {}
for entry in xp:
uri = None
name = entry.name.split(':', 1)
if len(name) > 1:
prefix, name = name
uri = xp_namespaces.get(prefix, False)
else:
name = name[0]
if uri is None and name == 'xmlns':
d_prefix, d_uri = entry.data.split('=', 1)
xp_namespaces[d_prefix] = d_uri
elif uri == moin_page.namespace and name == 'include':
xp_include = XPointer(entry.data)
if xp_include:
for entry in xp_include:
name, data = entry.name, entry.data_unescape
# TODO: These do not include all parameters in moin 1.9 Include macro docs:
# <<Include(pagename, heading, level, from="regex", to="regex", sort=ascending|descending, items=n, skipitems=n, titlesonly, editlink)>>
# these are currently unsupported in moin 2.0: from, to, titlesonly, editlink
if name == 'pages': # pages == pagename in moin 1.9
xp_include_pages = data
elif name == 'sort':
xp_include_sort = data
elif name == 'items':
xp_include_items = int(data)
elif name == 'skipitems':
xp_include_skipitems = int(data)
elif name == 'heading':
xp_include_heading = data
elif name == 'level':
xp_include_level = data
included_elements = []
if href:
# We have a single page to transclude or include
href = Iri(href)
link = Iri(scheme='wiki', authority='')
if href.scheme == 'wiki':
if href.authority:
raise ValueError(
"can't handle xinclude for non-local authority"
)
else:
path = href.path[1:]
elif href.scheme == 'wiki.local':
page = page_href
path = href.path
if path[0] == '':
# /subitem
tmp = page.path[1:]
tmp.extend(path[1:])
path = tmp
elif path[0] == '..':
# ../sisteritem
path = page.path[1:] + path[1:]
else:
raise ValueError(
"can't handle xinclude for schemes other than wiki or wiki.local"
)
link.path = path
if flaskg.user.may.read(unicode(path)):
page = Item.create(unicode(path))
pages = ((page, link), )
else:
# ACLs prevent user from viewing a transclusion - show message
message = moin_page.p(children=(_(
'Access Denied, transcluded content suppressed.')))
attrib = {html.class_: 'warning'}
div = ET.Element(moin_page.div,
attrib,
children=(message, ))
container = ET.Element(moin_page.body,
children=(div, ))
return [
container, 0
] # replace transclusion with container's child
elif xp_include_pages:
# we have regex of pages to include: <<Include(^qqq)>>
query = And([
Term(WIKINAME, app.cfg.interwikiname),
Regex(NAME_EXACT, xp_include_pages)
])
reverse = xp_include_sort == 'descending'
results = flaskg.storage.search(query,
sortedby=NAME_EXACT,
reverse=reverse,
limit=None)
pagelist = [result.name for result in results]
if xp_include_skipitems is not None:
pagelist = pagelist[xp_include_skipitems:]
if xp_include_items is not None:
pagelist = pagelist[xp_include_items + 1:]
pages = ((Item.create(p),
Iri(scheme='wiki', authority='', path='/' + p))
for p in pagelist)
if not pagelist:
msg = _(
'Error: no items found matching "<<Include({0})>>"'
).format(xp_include_pages)
attrib = {html.class_: 'moin-error'}
strong = ET.Element(moin_page.strong, attrib, (msg, ))
included_elements.append(strong)
for page, p_href in pages:
if p_href.path[0] != '/':
p_href.path = IriPath('/' + '/'.join(p_href.path))
if p_href in self.stack:
# we have a transclusion loop, create an error message showing list of pages forming loop
loop = self.stack[self.stack.index(p_href):]
loop = [
u'{0}'.format(ref.path[1:])
for ref in loop if ref is not None
] + [page.name]
msg = u'Error: Transclusion loop via: ' + u', '.join(
loop)
attrib = {html.class_: 'moin-error'}
strong = ET.Element(moin_page.strong, attrib, (msg, ))
included_elements.append(strong)
continue
if xp_include_heading is not None:
attrib = {xlink.href: p_href}
children = (xp_include_heading or page.name, )
elem_a = ET.Element(moin_page.a,
attrib,
children=children)
attrib = {
moin_page.outline_level: xp_include_level or '1'
}
elem_h = ET.Element(moin_page.h,
attrib,
children=(elem_a, ))
included_elements.append(elem_h)
page_doc = page.content.internal_representation(
attributes=Arguments(keyword=elem.attrib))
if isinstance(page.rev.data, file):
page.rev.data.close()
self.recurse(page_doc, page_href)
# The href needs to be an absolute URI, without the prefix "wiki://"
page_doc = mark_item_as_transclusion(page_doc, p_href.path)
included_elements.append(page_doc)
if len(included_elements) > 1:
# use a div as container
result = ET.Element(moin_page.div)
result.extend(included_elements)
elif included_elements:
result = included_elements[0]
else:
result = None
# end of processing for transclusion; the "result" will get inserted into the DOM below
return result
# Traverse the DOM by calling self.recurse with each child of the current elem.
# Starting elem.tag.name=='page'.
container = []
i = 0
while i < len(elem):
child = elem[i]
if isinstance(child, ET.Node):
ret = self.recurse(child, page_href)
if ret:
# Either child or a descendant of child is a transclusion.
# See top of this script for notes on why these DOM adjustments are required.
if isinstance(ret, ET.Node
) and elem.tag.name in NO_BLOCK_CHILDREN:
body = ret[0]
if len(body) == 0:
# the transcluded item is empty, insert an empty span into DOM
attrib = Attributes(ret).convert()
elem[i] = ET.Element(moin_page.span,
attrib=attrib)
elif (isinstance(body[0], ET.Node)
and (len(body) > 1 or body[0].tag.name
not in ('p', 'object', 'a'))):
# Complex case: "some text {{BlockItem}} more text" or "\n{{BlockItem}}\n" where
# the BlockItem body contains multiple p's, a table, preformatted text, etc.
# These block elements cannot be made a child of the current elem, so we create
# a container to replace elem.
# Create nodes to hold any siblings before and after current child (elem[i])
before = copy.deepcopy(elem)
after = copy.deepcopy(elem)
before[:] = elem[0:i]
after[:] = elem[i + 1:]
if len(before):
# there are siblings before transclude, save them in container
container.append(before)
new_trans_ptr = len(container)
# get attributes from page node;
# we expect {class: "moin-transclusion"; data-href: "http://some.org/somepage"}
attrib = Attributes(ret).convert()
# current elem will likely be replaced by container so we need to copy data-lineno attr
if html.data_lineno in elem.attrib:
attrib[html.data_lineno] = elem.attrib[
html.data_lineno]
# make new div node to hold transclusion, copy children, and save in container
div = ET.Element(moin_page.div,
attrib=attrib,
children=body[:])
container.append(
div) # new_trans_ptr is index to this
if len(after):
container.append(after)
if elem.tag.name == 'a':
# invalid input [[MyPage|{{BlockItem}}]],
# best option is to retain A-tag and fail html validation
# TODO: error may not be obvious to user - add error message
elem[i] = div
else:
# move up 1 level in recursion where elem becomes the child and
# is usually replaced by container
return [container, new_trans_ptr]
else:
# default action for inline transclusions or odd things like circular transclusion error messages
classes = child.attrib.get(html.class_,
'').split()
classes += ret.attrib.get(html.class_,
'').split()
ret.attrib[html.class_] = ' '.join(classes)
elem[i] = ret
elif isinstance(ret, types.ListType):
# a container has been returned.
# Note: there are multiple places where a container may be constructed
ret_container, trans_ptr = ret
# trans_ptr points to the transclusion within ret_container.
# Here the transclusion will always contain a block level element
if elem.tag.name in NO_BLOCK_CHILDREN:
# Complex case, transclusion effects grand-parent, great-grand-parent, e.g.:
# "/* comment {{BlockItem}} */" or "text ''italic {{BlockItem}} italic'' text"
# elem is an inline element, build a bigger container to replace elem's parent,
before = copy.deepcopy(elem)
after = copy.deepcopy(elem)
before[:] = elem[0:i] + ret_container[
0:trans_ptr]
after[:] = ret_container[trans_ptr +
1:] + elem[i + 1:]
if len(before):
container.append(before)
new_trans_ptr = len(container)
# child may have classes like "comment" that must be added to transcluded element
classes = child.attrib.get(
moin_page.class_, '').split()
# must use moin_page.class_ above, but use html.class below per html_out.py code
classes += ret_container[trans_ptr].attrib.get(
html.class_, '').split()
ret_container[trans_ptr].attrib[
html.class_] = ' '.join(classes)
container.append(ret_container[trans_ptr]
) # the transclusion
if len(after):
container.append(after)
return [container, new_trans_ptr]
else:
# elem is a block element
for grandchild in child:
if isinstance(
grandchild, ET.Node
) and grandchild.tag.name == u'include':
# the include may have classes that must be added to transcluded element
classes = grandchild.attrib.get(
html.class_, '').split()
classes += ret_container[
trans_ptr].attrib.get(
html.class_, '').split()
ret_container[trans_ptr].attrib[
html.class_] = ' '.join(classes)
# replace child element with the container generated in lower recursion
elem[i:i +
1] = ret_container # elem[i] is the child
else:
# default action for any ret not fitting special cases above,
# e.g. tranclusion is within a table cell
elem[i] = ret
# we are finished with this child, advance to next sibling
i += 1
finally:
self.stack.pop()
def search(self, key):
#results = self.searcher.find("content", key)
results = self.searcher.search(Regex("content", key))
#print len(results)
return (len(results), results)