def command(self):
self.basic_setup()
self.handle_options()
if self.options.extract:
self.extractor.extract()
if self.options.load:
self.loader = MediawikiLoader(self.options)
self.loader.load()
class Wiki2MarkDownCommand(WikiCommand):
"""Import MediaWiki to Allura Wiki tool"""
min_args = 1
max_args = None
summary = 'Import wiki from mediawiki-dump to allura wiki'
parser = WikiCommand.standard_parser(verbose=True)
parser.add_option('-e', '--extract-only', action='store_true',
dest='extract',
help='Store data from the mediawiki-dump '
'on the local filesystem; not load into Allura')
parser.add_option('-l', '--load-only', action='store_true', dest='load',
help='Load into Allura previously-extracted data')
parser.add_option('-d', '--dump-dir', dest='dump_dir', default='',
help='Directory for dump files')
parser.add_option('-n', '--neighborhood', dest='nbhd', default='',
help='Neighborhood name to load data')
parser.add_option('-p', '--project', dest='project', default='',
help='Project shortname to load data into')
parser.add_option('-s', '--source', dest='source', default='',
help='Database type to extract from (only mysql for now)')
parser.add_option('--db_name', dest='db_name', default='mediawiki',
help='Database name')
parser.add_option('--host', dest='host', default='localhost',
help='Database host')
parser.add_option('--port', dest='port', type='int', default=0,
help='Database port')
parser.add_option('--user', dest='user', default='',
help='User for database connection')
parser.add_option('--password', dest='password', default='',
help='Password for database connection')
parser.add_option('-a', '--attachments-dir', dest='attachments_dir',
help='Path to directory with mediawiki attachments dump',
default='')
def command(self):
self.basic_setup()
self.handle_options()
if self.options.extract:
self.extractor.extract()
if self.options.load:
self.loader = MediawikiLoader(self.options)
self.loader.load()
def handle_options(self):
if not self.options.dump_dir:
allura_base.log.error('You must specify directory for dump files')
exit(2)
if not self.options.extract and not self.options.load:
# if action doesn't specified - do both
self.options.extract = True
self.options.load = True
if self.options.load and (not self.options.project
or not self.options.nbhd):
allura_base.log.error('You must specify neighborhood and project '
'to load data')
exit(2)
if self.options.extract:
if self.options.source == 'mysql':
self.extractor = MySQLExtractor(self.options)
elif self.options.source in ('sqlite', 'postgres', 'sql-dump'):
allura_base.log.error('This source not implemented yet.'
'Only mysql for now')
exit(2)
else:
allura_base.log.error('You must specify valid data source')
exit(2)
if not self.options.attachments_dir:
allura_base.log.error('You must specify path to directory '
'with mediawiki attachmets dump.')
exit(2)
def setUp(self):
setup_basic_test()
self.options = mock.Mock()
# need test project with installed wiki app
self.options.nbhd = 'Adobe'
self.options.project = '--init--'
nbhd = M.Neighborhood.query.get(name=self.options.nbhd)
h.set_context(self.options.project, 'wiki', neighborhood=nbhd)
# monkey-patch MediawikiLoader for test
def pages(self):
yield 1
yield 2
def history(self, page_dir):
data = {
1: [
{
'title': 'Test title',
'text': "'''bold''' ''italics''",
'page_id': 1,
'timestamp': '20120808000001',
'username': '******'
},
{
'title': 'Test title',
'text': "'''bold'''",
'page_id': 1,
'timestamp': '20120809000001',
'username': '******'
},
],
2: [
{
'title': 'Main',
'text': "Main text rev 1",
'page_id': 2,
'timestamp': '20120808000001',
'username': '******'
},
{
'title': 'Main',
'text': "Main text rev 2",
'page_id': 2,
'timestamp': '20120809000001',
'username': '******'
},
],
}
for page in data[page_dir]:
yield page
def talk(self, page_dir):
data = {
1: {
'text': "''Talk page'' for page 1.",
'username': '******',
'timestamp': '20120809000001'
},
2: {
'text': "''Talk page'' for page 2.",
'username': '******',
'timestamp': '20120809000001'
},
}
return data[page_dir]
def attachments(self, *args, **kwargs):
# make 'empty' iterator
if False:
yield
MediawikiLoader._pages = pages
MediawikiLoader._history = history
MediawikiLoader._talk = talk
MediawikiLoader._attachments = attachments
self.loader = MediawikiLoader(self.options)
class TestMediawikiLoader(object):
def setUp(self):
setup_basic_test()
self.options = mock.Mock()
# need test project with installed wiki app
self.options.nbhd = 'Adobe'
self.options.project = '--init--'
nbhd = M.Neighborhood.query.get(name=self.options.nbhd)
h.set_context(self.options.project, 'wiki', neighborhood=nbhd)
# monkey-patch MediawikiLoader for test
def pages(self):
yield 1
yield 2
def history(self, page_dir):
data = {
1: [
{
'title': 'Test title',
'text': "'''bold''' ''italics''",
'page_id': 1,
'timestamp': '20120808000001',
'username': '******'
},
{
'title': 'Test title',
'text': "'''bold'''",
'page_id': 1,
'timestamp': '20120809000001',
'username': '******'
},
],
2: [
{
'title': 'Main',
'text': "Main text rev 1",
'page_id': 2,
'timestamp': '20120808000001',
'username': '******'
},
{
'title': 'Main',
'text': "Main text rev 2",
'page_id': 2,
'timestamp': '20120809000001',
'username': '******'
},
],
}
for page in data[page_dir]:
yield page
def talk(self, page_dir):
data = {
1: {
'text': "''Talk page'' for page 1.",
'username': '******',
'timestamp': '20120809000001'
},
2: {
'text': "''Talk page'' for page 2.",
'username': '******',
'timestamp': '20120809000001'
},
}
return data[page_dir]
def attachments(self, *args, **kwargs):
# make 'empty' iterator
if False:
yield
MediawikiLoader._pages = pages
MediawikiLoader._history = history
MediawikiLoader._talk = talk
MediawikiLoader._attachments = attachments
self.loader = MediawikiLoader(self.options)
def get_page(self, title):
return WM.Page.query.get(app_config_id=context.app.config._id,
title=title)
def get_post(self, title):
page = self.get_page(title)
thread = M.Thread.query.get(ref_id=page.index_id())
return M.Post.query.get(discussion_id=thread.discussion_id,
thread_id=thread._id)
@skipif(module_not_available('mediawiki'))
@mock.patch('allura.model.discuss.g.director')
def test_load_pages(self, director):
"""Test that pages, edit history and talk loaded properly"""
self.loader.load_pages()
page = self.get_page('Test title')
assert page.mod_date == datetime.strptime('20120809000001',
self.loader.TIMESTAMP_FMT)
assert page.authors()[0].username == 'test-user'
assert '**bold**' in page.text
# _italics should be only in the first revision of page
assert '_italics_' not in page
page = page.get_version(1)
assert '**bold** _italics_' in page.text
assert page.mod_date == datetime.strptime('20120808000001',
self.loader.TIMESTAMP_FMT)
assert page.authors()[0].username == 'test-user'
page = self.get_page('Main')
assert page.mod_date == datetime.strptime('20120809000001',
self.loader.TIMESTAMP_FMT)
assert page.authors()[0].username == '*anonymous'
assert 'Main text rev 2' in page.text
page = page.get_version(1)
assert page.mod_date == datetime.strptime('20120808000001',
self.loader.TIMESTAMP_FMT)
assert page.authors()[0].username == '*anonymous'
assert 'Main text rev 1' in page.text
# Check that talk pages loaded
post = self.get_post('Test title')
assert post.timestamp == datetime.strptime('20120809000001',
self.loader.TIMESTAMP_FMT)
assert post.author().username == 'test-user'
assert '_Talk page_ for page 1.' in post.text
post = self.get_post('Main')
assert post.timestamp == datetime.strptime('20120809000001',
self.loader.TIMESTAMP_FMT)
assert post.author().username == '*anonymous'
assert '_Talk page_ for page 2.' in post.text
