def handle_noargs(self, **options):
pname = smart_decode(options.get('pname'))
dname = smart_decode(options.get('dname'))
release = smart_decode(options.get('release'))
url = smart_decode(options.get('url'))
syncer = smart_decode(options.get('syncer'))
create_doc_local(pname, dname, release, syncer, url)
def test_maven_parse_joda162_doc_code(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'joda162doc',
'index.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
create_doc_db('project1', 'manual', '3.0', '',
'doc.syncer.generic_syncer.SingleURLSyncer',
'doc.parser.common_parsers.MavenParser')
print('Syncing doc')
sync_doc(pname, dname, release)
print('Synced doc')
document = parse_doc(pname, dname, release, True)
page = Page.objects.filter(document=document).filter(
title='User Guide').all()[0]
section = Section.objects.filter(page=page).filter(
title='Direct access').all()[0]
self.assertEqual(0, section.title_references.count())
# Only 2, because small references are not added! Yay!
self.assertEqual(2, section.code_references.count())
self.assertEqual(1, section.code_snippets.count())
section = Section.objects.filter(page=page).filter(
title='Using a MutableDateTime').all()[0]
self.assertEqual(1, section.title_references.count())
# Include the ref in the title too!
self.assertEqual(2, section.code_references.count())
self.assertEqual(1, section.code_snippets.count())
def test_docbook_parse_ht4_doc(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'httpclient402doc',
'index.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
create_doc_db('project1', 'manual', '3.0', '',
'doc.syncer.generic_syncer.SingleURLSyncer',
'doc.parser.common_parsers.NewDocBookParser')
sync_doc(pname, dname, release)
document = parse_doc(pname, dname, release, False)
page = Page.objects.filter(document=document).filter(
title='Chapter 2. Connection management').all()[0]
self.assertEqual('/html/body', page.xpath)
self.assertEqual(23, page.sections.count())
section = Section.objects.filter(page=page).filter(
number='2.3.1.').all()[0]
self.assertEqual('2.3.1. Route computation', section.title)
self.assertEqual(182, section.word_count)
self.assertEqual('2.3.', section.parent.number)
# With code snippets:
section = Section.objects.filter(page=page).filter(
number='2.5.').all()[0]
self.assertEqual('2.5. Socket factories', section.title)
self.assertEqual(110, section.word_count)
self.assertEqual('2.', section.parent.number)
self.assertTrue(section.parent.parent is None)
def test_docbook_parse_ht4_doc(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'httpclient402doc',
'index.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
create_doc_db('project1', 'manual', '3.0', '',
'doc.syncer.generic_syncer.SingleURLSyncer',
'doc.parser.common_parsers.NewDocBookParser')
sync_doc(pname, dname, release)
document = parse_doc(pname, dname, release, False)
page = Page.objects.filter(document=document).filter(
title='Chapter 2. Connection management').all()[0]
self.assertEqual('/html/body', page.xpath)
self.assertEqual(23, page.sections.count())
section = Section.objects.filter(page=page).filter(
number='2.3.1.').all()[0]
self.assertEqual('2.3.1. Route computation', section.title)
self.assertEqual(182, section.word_count)
self.assertEqual('2.3.', section.parent.number)
# With code snippets:
section = Section.objects.filter(page=page).filter(
number='2.5.').all()[0]
self.assertEqual('2.5. Socket factories', section.title)
self.assertEqual(110, section.word_count)
self.assertEqual('2.', section.parent.number)
self.assertTrue(section.parent.parent is None)
def test_docbook_parse_sp25_doc(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'spring25doc', 'index.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
create_doc_db('project1', 'manual', '3.0', '',
'doc.syncer.generic_syncer.SingleURLSyncer',
'doc.parser.common_parsers.NewDocBookParser')
print('Syncing Doc')
sync_doc(pname, dname, release)
print('Synced Doc')
document = parse_doc(pname, dname, release, False)
page = Page.objects.filter(document=document).filter(
title='Chapter 22. Email').all()[0]
self.assertEqual('/html/body', page.xpath)
self.assertEqual(11, page.sections.count())
section = Section.objects.filter(page=page).filter(
number='22.3.1.1.').all()[0]
self.assertEqual('22.3.1.1. Attachments', section.title)
self.assertEqual(78, section.word_count)
self.assertEqual('22.3.1.', section.parent.number)
section = Section.objects.filter(page=page).filter(
number='22.2.').all()[0]
self.assertEqual('22.2. Usage', section.title)
self.assertEqual(51, section.word_count)
self.assertEqual('22.', section.parent.number)
self.assertTrue(section.parent.parent is None)
def test_maven_parse_joda162_doc_code(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'joda162doc', 'index.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
create_doc_db('project1', 'manual', '3.0', '',
'doc.syncer.generic_syncer.SingleURLSyncer',
'doc.parser.common_parsers.MavenParser')
print('Syncing doc')
sync_doc(pname, dname, release)
print('Synced doc')
document = parse_doc(pname, dname, release, True)
page = Page.objects.filter(document=document).filter(
title='User Guide').all()[0]
section = Section.objects.filter(page=page).filter(
title='Direct access').all()[0]
self.assertEqual(0, section.title_references.count())
# Only 2, because small references are not added! Yay!
self.assertEqual(2, section.code_references.count())
self.assertEqual(1, section.code_snippets.count())
section = Section.objects.filter(page=page).filter(
title='Using a MutableDateTime').all()[0]
self.assertEqual(1, section.title_references.count())
# Include the ref in the title too!
self.assertEqual(2, section.code_references.count())
self.assertEqual(1, section.code_snippets.count())
def test_docbook_parse_hib3_doc_code(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'hib35doc', 'index.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
create_doc_db('project1', 'manual', '3.0', '',
'doc.syncer.generic_syncer.SingleURLSyncer',
'doc.parser.common_parsers.NewDocBookParser')
print('Syncing doc')
sync_doc(pname, dname, release)
print('Synced doc')
document = parse_doc(pname, dname, release, True)
page = Page.objects.filter(document=document).filter(
title='Chapter 12. Transactions and Concurrency').all()[0]
section = Section.objects.filter(page=page).filter(
number='12.3.4.').all()[0]
self.assertEqual(0, section.title_references.count())
self.assertEqual(13, section.code_references.count())
self.assertEqual(0, section.code_snippets.count())
section = Section.objects.filter(page=page).filter(
number='12.3.3.').all()[0]
self.assertEqual(0, section.title_references.count())
self.assertEqual(12, section.code_references.count())
self.assertEqual(1, section.code_snippets.count())
def test_docbook_parse_hib3_doc_code(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'hib35doc',
'index.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
create_doc_db('project1', 'manual', '3.0', '',
'doc.syncer.generic_syncer.SingleURLSyncer',
'doc.parser.common_parsers.NewDocBookParser')
print('Syncing doc')
sync_doc(pname, dname, release)
print('Synced doc')
document = parse_doc(pname, dname, release, True)
page = Page.objects.filter(document=document).filter(
title='Chapter 12. Transactions and Concurrency').all()[0]
section = Section.objects.filter(page=page).filter(
number='12.3.4.').all()[0]
self.assertEqual(0, section.title_references.count())
self.assertEqual(13, section.code_references.count())
self.assertEqual(0, section.code_snippets.count())
section = Section.objects.filter(page=page).filter(
number='12.3.3.').all()[0]
self.assertEqual(0, section.title_references.count())
self.assertEqual(12, section.code_references.count())
self.assertEqual(1, section.code_snippets.count())
def handle_noargs(self, **options):
pname = smart_decode(options.get('pname'))
dname = smart_decode(options.get('dname'))
release = smart_decode(options.get('release'))
url = smart_decode(options.get('url'))
syncer = smart_decode(options.get('syncer'))
create_doc_local(pname, dname, release, syncer, url)
def handle_noargs(self, **options):
pname = smart_decode(options.get('pname'))
dname = smart_decode(options.get('dname'))
release = smart_decode(options.get('release'))
url = smart_decode(options.get('url'))
syncer = smart_decode(options.get('syncer'))
parser = smart_decode(options.get('parser'))
create_doc_db(pname, dname, release, url, syncer, parser)
if (options.get('local', False)):
create_doc_local(pname, dname, release, syncer, url)
def test_sync_doc_remote(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'http://hc.apache.org/httpcomponents-client-ga/tutorial/html/index.html')
sync_doc(pname, dname, release)
doc_key = dname + release
model = load_model(pname, DOC_PATH, doc_key)
self.assertEqual(9, len(model.pages))
for page_key in model.pages:
path = urlparse(page_key).path
self.assertTrue(os.path.exists(path))
def test_sync_doc_remote(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
create_doc_local(
pname, dname, release, 'doc.syncer.generic_syncer.SingleURLSyncer',
'http://hc.apache.org/httpcomponents-client-ga/tutorial/html/index.html'
)
sync_doc(pname, dname, release)
doc_key = dname + release
model = load_model(pname, DOC_PATH, doc_key)
self.assertEqual(9, len(model.pages))
for page_key in model.pages:
path = urlparse(page_key).path
self.assertTrue(os.path.exists(path))
def test_sync_doc_local(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'httpclient402doc',
'index.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
sync_doc(pname, dname, release)
doc_key = dname + release
model = load_model(pname, DOC_PATH, doc_key)
self.assertEqual(8, len(model.pages))
for page_key in model.pages:
path = urlparse(page_key).path
self.assertTrue(os.path.exists(path))
def test_sync_doc_local(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'httpclient402doc',
'index.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
sync_doc(pname, dname, release)
doc_key = dname + release
model = load_model(pname, DOC_PATH, doc_key)
self.assertEqual(8, len(model.pages))
for page_key in model.pages:
path = urlparse(page_key).path
self.assertTrue(os.path.exists(path))
def test_maven_parse_htclient_doc(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'httpclient31doc',
'userguide.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
create_doc_db('project1', 'manual', '3.0', '',
'doc.syncer.generic_syncer.SingleURLSyncer',
'doc.parser.common_parsers.MavenParser')
print('Syncing doc')
sync_doc(pname, dname, release)
print('Synced doc')
document = parse_doc(pname, dname, release, False)
page = Page.objects.filter(document=document).filter(
title='HttpClient exception handling guide').all()[0]
self.assertEqual('/html/body', page.xpath)
self.assertEqual(22, page.sections.count())
section = Section.objects.filter(page=page).filter(
title='Idempotent methods').all()[0]
self.assertEqual('Idempotent methods', section.title)
self.assertEqual(113, section.word_count)
self.assertEqual('', section.number)
self.assertEqual('HTTP transport safety', section.parent.title)
section = Section.objects.filter(page=page).filter(
title='Custom exception handler').all()[0]
self.assertEqual('Custom exception handler', section.title)
self.assertEqual('', section.number)
self.assertEqual(132, section.word_count)
self.assertTrue(section.parent is None)
def test_maven_parse_joda162_doc(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'joda162doc',
'index.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
create_doc_db('project1', 'manual', '3.0', '',
'doc.syncer.generic_syncer.SingleURLSyncer',
'doc.parser.common_parsers.MavenParser')
print('Syncing doc')
sync_doc(pname, dname, release)
print('Synced doc')
document = parse_doc(pname, dname, release, False)
page = Page.objects.filter(document=document).filter(
title='User Guide').all()[0]
self.assertEqual('/html/body', page.xpath)
self.assertEqual(35, page.sections.count())
section = Section.objects.filter(page=page).filter(
title='Direct access').all()[0]
self.assertEqual('Direct access', section.title)
self.assertEqual(110, section.word_count)
self.assertEqual('', section.number)
self.assertEqual('Input and Output', section.parent.title)
self.assertEqual(162, section.parent.word_count)
section = Section.objects.filter(page=page).filter(
title='Advanced features').all()[0]
self.assertEqual('Advanced features', section.title)
self.assertEqual(2, section.word_count)
self.assertTrue(section.parent is None)
def test_maven_parse_htclient_doc(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'httpclient31doc',
'userguide.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
create_doc_db('project1', 'manual', '3.0', '',
'doc.syncer.generic_syncer.SingleURLSyncer',
'doc.parser.common_parsers.MavenParser')
print('Syncing doc')
sync_doc(pname, dname, release)
print('Synced doc')
document = parse_doc(pname, dname, release, False)
page = Page.objects.filter(document=document).filter(
title='HttpClient exception handling guide').all()[0]
self.assertEqual('/html/body', page.xpath)
self.assertEqual(22, page.sections.count())
section = Section.objects.filter(page=page).filter(
title='Idempotent methods').all()[0]
self.assertEqual('Idempotent methods', section.title)
self.assertEqual(113, section.word_count)
self.assertEqual('', section.number)
self.assertEqual('HTTP transport safety', section.parent.title)
section = Section.objects.filter(page=page).filter(
title='Custom exception handler').all()[0]
self.assertEqual('Custom exception handler', section.title)
self.assertEqual('', section.number)
self.assertEqual(132, section.word_count)
self.assertTrue(section.parent is None)
def test_docbook_parse_sp25_doc(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'spring25doc',
'index.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
create_doc_db('project1', 'manual', '3.0', '',
'doc.syncer.generic_syncer.SingleURLSyncer',
'doc.parser.common_parsers.NewDocBookParser')
print('Syncing Doc')
sync_doc(pname, dname, release)
print('Synced Doc')
document = parse_doc(pname, dname, release, False)
page = Page.objects.filter(document=document).filter(
title='Chapter 22. Email').all()[0]
self.assertEqual('/html/body', page.xpath)
self.assertEqual(11, page.sections.count())
section = Section.objects.filter(page=page).filter(
number='22.3.1.1.').all()[0]
self.assertEqual('22.3.1.1. Attachments', section.title)
self.assertEqual(78, section.word_count)
self.assertEqual('22.3.1.', section.parent.number)
section = Section.objects.filter(page=page).filter(
number='22.2.').all()[0]
self.assertEqual('22.2. Usage', section.title)
self.assertEqual(51, section.word_count)
self.assertEqual('22.', section.parent.number)
self.assertTrue(section.parent.parent is None)
def test_maven_parse_joda162_doc(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'joda162doc', 'index.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
create_doc_db('project1', 'manual', '3.0', '',
'doc.syncer.generic_syncer.SingleURLSyncer',
'doc.parser.common_parsers.MavenParser')
print('Syncing doc')
sync_doc(pname, dname, release)
print('Synced doc')
document = parse_doc(pname, dname, release, False)
page = Page.objects.filter(document=document).filter(
title='User Guide').all()[0]
self.assertEqual('/html/body', page.xpath)
self.assertEqual(35, page.sections.count())
section = Section.objects.filter(page=page).filter(
title='Direct access').all()[0]
self.assertEqual('Direct access', section.title)
self.assertEqual(110, section.word_count)
self.assertEqual('', section.number)
self.assertEqual('Input and Output', section.parent.title)
self.assertEqual(162, section.parent.word_count)
section = Section.objects.filter(page=page).filter(
title='Advanced features').all()[0]
self.assertEqual('Advanced features', section.title)
self.assertEqual(2, section.word_count)
self.assertTrue(section.parent is None)
def test_docbook_parse_hib3_doc(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'hib35doc', 'index.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
create_doc_db('project1', 'manual', '3.0', '',
'doc.syncer.generic_syncer.SingleURLSyncer',
'doc.parser.common_parsers.NewDocBookParser')
print('Syncing doc')
sync_doc(pname, dname, release)
print('Synced doc')
document = parse_doc(pname, dname, release, False)
page = Page.objects.filter(document=document).filter(
title='Chapter 12. Transactions and Concurrency').all()[0]
self.assertEqual('/html/body', page.xpath)
self.assertEqual(18, page.sections.count())
section = Section.objects.filter(page=page).filter(
number='12.3.4.').all()[0]
self.assertEqual('12.3.4. Customizing automatic versioning',
section.title)
self.assertTrue(276, section.word_count)
self.assertEqual('12.3.', section.parent.number)
section = Section.objects.filter(page=page).filter(
number='12.5.').all()[0]
self.assertEqual('12.5. Connection release modes', section.title)
self.assertEqual(371, section.word_count)
self.assertEqual('12.', section.parent.number)
self.assertTrue(section.parent.parent is None)
def test_docbook_parse_hib3_doc(self):
pname = 'project1'
release = '3.0'
dname = 'manual'
test_doc = os.path.join(settings.TESTDATA, 'hib35doc',
'index.html')
test_doc = os.path.normpath(test_doc)
create_doc_local(pname, dname, release,
'doc.syncer.generic_syncer.SingleURLSyncer',
'file://' + test_doc)
create_doc_db('project1', 'manual', '3.0', '',
'doc.syncer.generic_syncer.SingleURLSyncer',
'doc.parser.common_parsers.NewDocBookParser')
print('Syncing doc')
sync_doc(pname, dname, release)
print('Synced doc')
document = parse_doc(pname, dname, release, False)
page = Page.objects.filter(document=document).filter(
title='Chapter 12. Transactions and Concurrency').all()[0]
self.assertEqual('/html/body', page.xpath)
self.assertEqual(18, page.sections.count())
section = Section.objects.filter(page=page).filter(
number='12.3.4.').all()[0]
self.assertEqual('12.3.4. Customizing automatic versioning', section.title)
self.assertTrue(276, section.word_count)
self.assertEqual('12.3.', section.parent.number)
section = Section.objects.filter(page=page).filter(
number='12.5.').all()[0]
self.assertEqual('12.5. Connection release modes', section.title)
self.assertEqual(371, section.word_count)
self.assertEqual('12.', section.parent.number)
self.assertTrue(section.parent.parent is None)
def create_documentation(self):
create_doc_local(self.pname, 'manual', self.release, 'foo', 'url')
doc = create_doc_db(self.pname, 'manual', self.release, 'url', 'foo',
'foo')
page1 = Page(document=doc, title='HTTP Server')
page1.save()
page2 = Page(document=doc, title='HTTP Client')
page2.save()
section1 = Section(page=page1, number='1.',
title='Implementing foo bar')
section1.save()
section11 = Section(page=page1, number='1.1',
title='Reference', parent=section1)
section11.save()
section2 = Section(page=page2, number='2.',
title='Implementing the Client')
section2.save()
section21 = Section(page=page2, number='2.1',
title='Implementing the Client Again')
section21.save()
coderef1 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='@p1.Annotation1',
source='d',
kind_hint=self.ann_kind,
local_context=section1,
mid_context=section1,
global_context=page1
)
coderef1.save()
self.code_refs.append(coderef1)
coderef2 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='@Annotation2',
source='d',
kind_hint=self.ann_kind,
local_context=section1,
mid_context=section1,
global_context=page1
)
coderef2.save()
self.code_refs.append(coderef2)
snippet_content = r'''
@Annotation1(foobarbaz=2)
public class FooBar {
public void main(String arg) {
System.out.println();
RecodocClient rc = new RecodocClient();
}
}
'''
snippet1 = CodeSnippet(
project=self.project,
project_release=self.releasedb,
language='j',
source='d',
snippet_text=snippet_content,
local_context=section2,
mid_context=section2,
global_context=page2
)
snippet1.save()
self.code_snippets.append(snippet1)
def test_create_doc_local(self):
create_doc_local('project1', 'manual', '3.0', 'foo.syncer')
create_doc_local('project1', 'manual', '3.1', 'foo.syncer')
path = get_doc_path('project1', root=True)
self.assertEqual(2, len(os.listdir(path)))
def test_list_doc_local(self):
create_doc_local('project1', 'manual', '3.0', 'foo.syncer')
create_doc_local('project1', 'manual', '3.1', 'foo.syncer')
self.assertEqual(2, len(list_doc_local('project1')))
def test_create_doc_local(self):
create_doc_local('project1', 'manual', '3.0', 'foo.syncer')
create_doc_local('project1', 'manual', '3.1', 'foo.syncer')
path = get_doc_path('project1', root=True)
self.assertEqual(2, len(os.listdir(path)))
def test_list_doc_local(self):
create_doc_local('project1', 'manual', '3.0', 'foo.syncer')
create_doc_local('project1', 'manual', '3.1', 'foo.syncer')
self.assertEqual(2, len(list_doc_local('project1')))