def load_talk(self, page_dir, page_title):
"""Load talk for page.
page_dir - path to directory with page dump.
page_title - page title in Allura Wiki
"""
talk_data = self._talk(page_dir)
if not talk_data:
return
text = mediawiki2markdown(talk_data['text'])
page = WM.Page.query.get(app_config_id=self.wiki.config._id,
title=page_title)
if not page:
return
thread = M.Thread.query.get(ref_id=page.index_id())
if not thread:
return
timestamp = datetime.datetime.strptime(talk_data['timestamp'],
self.TIMESTAMP_FMT)
c.user = (M.User.query.get(username=talk_data['username'].lower())
or M.User.anonymous())
thread.add_post(text=text,
discussion_id=thread.discussion_id,
thread_id=thread._id,
timestamp=timestamp,
ignore_security=True)
allura_base.log.info('Loaded talk for page %s' % page_title)
def load_pages(self):
"""Load pages with edit history from json to Allura wiki tool"""
allura_base.log.info('Loading pages into allura...')
for page_dir in self._pages():
for page in self._history(page_dir):
p = WM.Page.upsert(page['title'])
p.viewable_by = ['all']
p.text = mediawiki_internal_links2markdown(
mediawiki2markdown(page['text']), page['title'])
timestamp = datetime.datetime.strptime(page['timestamp'],
self.TIMESTAMP_FMT)
p.mod_date = timestamp
c.user = (M.User.query.get(username=page['username'].lower())
or M.User.anonymous())
ss = p.commit()
ss.mod_date = ss.timestamp = timestamp
# set home to main page
if page['title'] == 'Main_Page':
gl = WM.Globals.query.get(app_config_id=self.wiki.config._id)
if gl is not None:
gl.root = page['title']
allura_base.log.info('Loaded history of page %s (%s)' %
(page['page_id'], page['title']))
self.load_talk(page_dir, page['title'])
self.load_attachments(page_dir, page['title'])
def load_talk(self, page_dir, page_title):
"""Load talk for page.
page_dir - path to directory with page dump.
page_title - page title in Allura Wiki
"""
talk_data = self._talk(page_dir)
if not talk_data:
return
text = mediawiki2markdown(talk_data['text'])
page = WM.Page.query.get(app_config_id=self.wiki.config._id,
title=page_title)
if not page:
return
thread = M.Thread.query.get(ref_id=page.index_id())
if not thread:
return
timestamp = datetime.datetime.strptime(talk_data['timestamp'],
self.TIMESTAMP_FMT)
c.user = (M.User.query.get(username=talk_data['username'].lower())
or M.User.anonymous())
thread.add_post(
text=text,
discussion_id=thread.discussion_id,
thread_id=thread._id,
timestamp=timestamp,
ignore_security=True)
log.info('Loaded talk for page %s', page_title)
def load_pages(self):
"""Load pages with edit history from json to Allura wiki tool"""
log.info('Loading pages into allura...')
for page_dir in self._pages():
for page in self._history(page_dir):
p = WM.Page.upsert(page['title'])
p.viewable_by = ['all']
p.text = mediawiki_internal_links2markdown(
mediawiki2markdown(page['text']),
page['title'])
timestamp = datetime.datetime.strptime(page['timestamp'],
self.TIMESTAMP_FMT)
p.mod_date = timestamp
c.user = (M.User.query.get(username=page['username'].lower())
or M.User.anonymous())
ss = p.commit()
ss.mod_date = ss.timestamp = timestamp
# set home to main page
if page['title'] == 'Main_Page':
gl = WM.Globals.query.get(app_config_id=self.wiki.config._id)
if gl is not None:
gl.root = page['title']
log.info('Loaded history of page %s (%s)', page['page_id'], page['title'])
self.load_talk(page_dir, page['title'])
self.load_attachments(page_dir, page['title'])
def test_mediawiki2markdown():
mediawiki_text = """
'''bold''' ''italics''
== Getting started ==
* [http://www.mediawiki.org/wiki/Manual:Configuration_settings Configuration]
* [http://www.mediawiki.org/wiki/Manual:FAQ MediaWiki FAQ]
<plugin>.plugin()
"""
mediawiki_output = converters.mediawiki2markdown(mediawiki_text)
assert "**bold** _italics_" in mediawiki_output
assert "## Getting started" in mediawiki_output
assert "* [MediaWiki FAQ](http://www.mediawiki.org/wiki/Manual:FAQ)" in mediawiki_output
assert "<plugin>.plugin()" in mediawiki_output
def test_mediawiki2markdown():
mediawiki_text = """
'''bold''' ''italics''
== Getting started ==
* [http://www.mediawiki.org/wiki/Manual:Configuration_settings Configuration]
* [http://www.mediawiki.org/wiki/Manual:FAQ MediaWiki FAQ]
<plugin>.plugin()
"""
mediawiki_output = converters.mediawiki2markdown(mediawiki_text)
assert "**bold** _italics_" in mediawiki_output
assert "## Getting started" in mediawiki_output
assert ("* [MediaWiki FAQ](http://www.mediawiki.org/wiki/Manual:FAQ)"
in mediawiki_output)
assert '<plugin>.plugin()' in mediawiki_output
def convert_markup(self, text, filename):
"""Convert any supported github markup into Allura-markdown.
Conversion happens in 4 phases:
1. Convert source text to a html using h.render_any_markup.
2. Rewrite links that match the wiki URL prefix with new location.
3. Convert resulting html to a markdown using html2text, if available.
4. Convert gollum tags
If html2text module isn't available then only phases 1 and 2 will be executed.
Files in mediawiki format are converted using mediawiki2markdown
if html2text is available.
"""
name, ext = os.path.splitext(filename)
if ext in self.markdown_exts:
return text
try:
import html2text
html2text.BODY_WIDTH = 0
except ImportError:
html2text = None
if ext and ext in self.mediawiki_exts:
if html2text:
text = mediawiki2markdown(text)
text = self.convert_gollum_tags(text)
# Don't have html here, so we can't call self._rewrite_links.
# Falling back to simpler rewriter.
prefix = self.github_wiki_url
new_prefix = self.app.url
if not prefix.endswith('/'):
prefix += '/'
if not new_prefix.endswith('/'):
new_prefix += '/'
_re = re.compile(r'%s(\S*)' % prefix)
def repl(m):
return new_prefix + self._convert_page_name(m.group(1))
text = _re.sub(repl, text)
else:
text = h.render_any_markup(filename, text)
text = self.rewrite_links(text, self.github_wiki_url, self.app.url)
return text
elif ext and ext in self.textile_exts:
text = self._prepare_textile_text(text)
text = h.render_any_markup(filename, text)
text = self.rewrite_links(text, self.github_wiki_url, self.app.url)
if html2text:
text = html2text.html2text(text)
text = self.convert_gollum_tags(text)
text = text.replace('<notextile>', '').replace('</notextile>', '')
text = text.replace('<notextile>', '').replace('</notextile>', '')
text = text.replace('<notextile>', '').replace('</notextile>', '')
return text
else:
text = h.render_any_markup(filename, text)
text = self.rewrite_links(text, self.github_wiki_url, self.app.url)
if html2text:
text = html2text.html2text(text)
text = self.convert_gollum_tags(text)
return text
def convert_markup(self, text, filename):
"""Convert any supported github markup into Allura-markdown.
Conversion happens in 4 phases:
1. Convert source text to a html using h.render_any_markup.
2. Rewrite links that match the wiki URL prefix with new location.
3. Convert resulting html to a markdown using html2text, if available.
4. Convert gollum tags
If html2text module isn't available then only phases 1 and 2 will be executed.
Files in mediawiki format are converted using mediawiki2markdown
if html2text is available.
"""
name, ext = os.path.splitext(filename)
if ext in self.markdown_exts:
text = self.github_markdown_converter.convert(text)
return self.convert_gollum_tags(text)
try:
import html2text
html2text.BODY_WIDTH = 0
except ImportError:
html2text = None
if ext and ext in self.mediawiki_exts:
if html2text:
text = mediawiki2markdown(text)
text = self.convert_gollum_tags(text)
# Don't have html here, so we can't call self._rewrite_links.
# Falling back to simpler rewriter.
prefix = self.github_wiki_url
new_prefix = self.app.url
if not prefix.endswith('/'):
prefix += '/'
if not new_prefix.endswith('/'):
new_prefix += '/'
_re = re.compile(r'%s(\S*)' % prefix)
def repl(m):
return new_prefix + self._convert_page_name(m.group(1))
text = _re.sub(repl, text)
else:
text = h.render_any_markup(filename, text)
text = self.rewrite_links(text, self.github_wiki_url,
self.app.url)
return text
elif ext and ext in self.textile_exts:
text = self._prepare_textile_text(text)
text = six.text_type(h.render_any_markup(filename, text))
text = self.rewrite_links(text, self.github_wiki_url, self.app.url)
if html2text:
text = html2text.html2text(text)
text = self.convert_gollum_tags(text)
text = text.replace('<notextile>',
'').replace('< notextile>',
'').replace('</notextile>', '')
text = text.replace('<notextile>',
'').replace('</notextile>', '')
text = text.replace('<notextile>',
'').replace('</notextile>', '')
return text
else:
text = h.render_any_markup(filename, text)
text = self.rewrite_links(text, self.github_wiki_url, self.app.url)
if html2text:
text = html2text.html2text(text)
text = self.convert_gollum_tags(text)
return text
