def test_apache_parser(self):
create_channel_db('project1', 'cf', 'coreforum',
'channel.syncer.common_syncers.ApacheMailSyncer',
'channel.parser.common_parsers.ApacheMailParser',
'http://mail-archives.apache.org/mod_mbox/hc-httpclient-users/'
)
create_channel_local('project1', 'coreforum',
'channel.syncer.common_syncers.ApacheMailSyncer',
'http://mail-archives.apache.org/mod_mbox/hc-httpclient-users/'
)
pname = 'project1'
cname = 'coreforum'
toc_refresh(pname, cname)
toc_download_section(pname, cname, start=0, end=1)
toc_download_entries(pname, cname, 9, 99)
parse_channel(pname, cname, True)
self.assertEqual(8, Message.objects.all().count())
messages = list(Message.objects.all())
for message in messages:
print('{0} by {1} on {2} (wc: {3})'.format(
message.title, message.author, message.msg_date,
message.word_count))
print(' {0} snippets and {1} references'.format(
message.code_snippets.count(),
message.code_references.count()))
print(' Snippets:')
for code_snippet in message.code_snippets.all():
print(' {0}'.format(code_snippet.language))
for ref in message.code_references.all():
print(' {0}: {1}'.format(ref.kind_hint.kind, ref.content))
# Test Snippets
third_to_last = messages[-3]
self.assertEqual('l', third_to_last.code_snippets.all()[0].language)
self.assertEqual('jx', third_to_last.code_snippets.all()[1].language)
# Test Refs
fourth_to_last = messages[-4]
refs = [ref.content.strip() for ref in
fourth_to_last.code_references.all()]
self.assertEqual(4, len(refs))
self.assertTrue('EasySSLProtocolSocketFactory' in refs)
self.assertTrue('SSL' in refs)
# Test Post-Processing!
channel = post_process_channel(pname, cname)
self.assertEqual(4, channel.threads.count())
second_thread = channel.threads.all()[1]
self.assertEqual(3, second_thread.messages.count())
indexes = [msg.index for msg in second_thread.messages.all()]
self.assertEqual([0, 1, 2], indexes)
self.assertFalse(second_thread.messages.all()[0].title.lower()
.startswith('re'))
self.assertTrue(second_thread.messages.all()[1].title.lower()
.startswith('re'))
def handle_noargs(self, **options):
pname = smart_decode(options.get('pname'))
cname = smart_decode(options.get('cname'))
cfull_name = smart_decode(options.get('cfull_name'))
url = smart_decode(options.get('url'))
syncer = smart_decode(options.get('syncer'))
parser = smart_decode(options.get('parser'))
create_channel_db(pname, cfull_name, cname, syncer, parser, url)
if (options.get('local', False)):
create_channel_local(pname, cname, syncer, url)
def handle_noargs(self, **options):
pname = smart_decode(options.get('pname'))
cname = smart_decode(options.get('cname'))
cfull_name = smart_decode(options.get('cfull_name'))
url = smart_decode(options.get('url'))
syncer = smart_decode(options.get('syncer'))
parser = smart_decode(options.get('parser'))
create_channel_db(pname, cfull_name, cname, syncer, parser, url)
if (options.get('local', False)):
create_channel_local(pname, cname, syncer, url)
def test_phpbb_parser(self):
create_channel_db('project1', 'cf', 'coreforum',
'channel.syncer.common_syncers.PHPBBForumSyncer',
'channel.parser.common_parsers.PHPBBForumParser',
'https://forum.hibernate.org/viewforum.php?f=1'
)
create_channel_local('project1', 'coreforum',
'channel.syncer.common_syncers.PHPBBForumSyncer',
'https://forum.hibernate.org/viewforum.php?f=1'
)
pname = 'project1'
cname = 'coreforum'
toc_refresh(pname, cname)
toc_download_section(pname, cname, start=0, end=2)
toc_download_entries(pname, cname, 1023, 1025)
parse_channel(pname, cname, True)
self.assertEqual(23, Message.objects.all().count())
messages = list(Message.objects.all())
for message in messages:
print(message.url)
print('{0} by {1} on {2} (wc: {3})'.format(
message.title, message.author, message.msg_date,
message.word_count))
print(' {0} snippets and {1} references'.format(
message.code_snippets.count(),
message.code_references.count()))
print(' Snippets:')
for code_snippet in message.code_snippets.all():
print(' {0}'.format(code_snippet.language))
for ref in message.code_references.all():
print(' {0}: {1}'.format(ref.kind_hint.kind, ref.content))
# Test Snippets
first_message = messages[0]
self.assertEqual('x', first_message.code_snippets.all()[0].language)
# Test Author
self.assertEqual(first_message.author.nickname, 'mhellkamp')
# Test Date
self.assertEqual(first_message.msg_date, datetime(2003, 8, 29, 10, 16))
# Test Refs
second_message = messages[1]
refs = [ref.content.strip() for ref in
second_message.code_references.all()]
# Because of the title 4 + 1 = 5
self.assertEqual(5, len(refs))
self.assertTrue('DBCP' in refs)
self.assertTrue('C3P0' in refs)
def test_fudeclipse_parser(self):
create_channel_db('project1', 'cf', 'coreforum',
'channel.syncer.common_syncers.FUDEclipseForumSyncer',
'channel.parser.common_parsers.FUDEclipseForumParser',
'http://www.eclipse.org/forums/index.php/sf/thread/59/'
)
create_channel_local('project1', 'coreforum',
'channel.syncer.common_syncers.FUDEclipseForumSyncer',
'http://www.eclipse.org/forums/index.php/sf/thread/59/'
)
pname = 'project1'
cname = 'coreforum'
toc_refresh(pname, cname)
toc_download_section(pname, cname, start=0, end=2)
toc_download_entries(pname, cname, 0, 6)
parse_channel(pname, cname, True)
self.assertEqual(18, Message.objects.all().count())
messages = list(Message.objects.all())
for message in messages:
print('{0} by {1} on {2} (wc: {3})'.format(
message.title, message.author, message.msg_date,
message.word_count))
print(' {0} snippets and {1} references'.format(
message.code_snippets.count(),
message.code_references.count()))
print(' Snippets:')
for code_snippet in message.code_snippets.all():
print(' {0}'.format(code_snippet.language))
for ref in message.code_references.all():
print(' {0}: {1}'.format(ref.kind_hint.kind, ref.content))
first_message = messages[0]
# Test Title
self.assertEqual(first_message.title,
'looping back to a previous step')
# Test Author
self.assertEqual(first_message.author.nickname, 'No real name')
# Test Date
self.assertEqual(first_message.msg_date, datetime(2010, 8, 26, 7, 50))
# Test Refs
refs = [ref.content.strip() for ref in
first_message.code_references.all()]
self.assertEqual(3, len(refs))
self.assertTrue('ActivityElements' in refs)
def test_fudeclipse_syncer(self):
create_channel_db('project1', 'cf', 'coreforum',
'channel.syncer.common_syncers.FUDEclipseForumSyncer',
'foo.parser',
'http://www.eclipse.org/forums/index.php/sf/thread/13/'
)
create_channel_local('project1', 'coreforum',
'channel.syncer.common_syncers.FUDEclipseForumSyncer',
'http://www.eclipse.org/forums/index.php/sf/thread/13/'
)
pname = 'project1'
cname = 'coreforum'
toc_refresh(pname, cname)
model = load_model(pname, STHREAD_PATH, cname)
self.assertEqual(
'http://www.eclipse.org/forums/index.php/sf/thread/13/1/0/',
model.toc_sections[0].url)
self.assertFalse(model.toc_sections[0].downloaded)
self.assertTrue(len(model.toc_sections) >= 247)
for i in xrange(0, 247):
self.assertEqual(i, model.toc_sections[i].index)
toc_download_section(pname, cname, start=0, end=4)
model = load_model(pname, STHREAD_PATH, cname)
self.assertTrue(model.toc_sections[0].downloaded)
self.assertTrue(model.toc_sections[1].downloaded)
self.assertTrue(model.toc_sections[2].downloaded)
self.assertTrue(model.toc_sections[3].downloaded)
self.assertFalse(model.toc_sections[4].downloaded)
self.assertEqual(160, len(model.entries))
self.assertEqual(0, model.entries[0].index)
self.assertFalse(model.entries[0].downloaded)
self.assertEqual(1000, model.entries[40].index)
self.assertEqual(1001, model.entries[41].index)
#self.assertTrue(model.entries[26].url.find('t=59') > -1)
toc_download_entries(pname, cname, 1039, 1040)
model = load_model(pname, STHREAD_PATH, cname)
self.assertTrue(model.entries[79].downloaded)
self.assertFalse(model.entries[80].downloaded)
path = os.path.join(settings.PROJECT_FS_ROOT,
model.entries[79].local_paths[0])
self.assertTrue(os.path.exists(path))
def test_phpbb_syncer(self):
create_channel_db('project1', 'cf', 'coreforum',
'channel.syncer.common_syncers.PHPBBForumSyncer', 'foo.parser',
'https://forum.hibernate.org/viewforum.php?f=1'
)
create_channel_local('project1', 'coreforum',
'channel.syncer.common_syncers.PHPBBForumSyncer',
'https://forum.hibernate.org/viewforum.php?f=1'
)
pname = 'project1'
cname = 'coreforum'
toc_refresh(pname, cname)
model = load_model(pname, STHREAD_PATH, cname)
self.assertEqual(
'https://forum.hibernate.org/viewforum.php?f=1&sd=a&start=0',
model.toc_sections[0].url)
self.assertFalse(model.toc_sections[0].downloaded)
self.assertTrue(len(model.toc_sections) > 2349)
for i in xrange(0, 2349):
self.assertEqual(i, model.toc_sections[i].index)
toc_download_section(pname, cname, start=0, end=4)
model = load_model(pname, STHREAD_PATH, cname)
self.assertTrue(model.toc_sections[0].downloaded)
self.assertTrue(model.toc_sections[1].downloaded)
self.assertTrue(model.toc_sections[2].downloaded)
self.assertTrue(model.toc_sections[3].downloaded)
self.assertFalse(model.toc_sections[4].downloaded)
self.assertEqual(100, len(model.entries))
self.assertEqual(0, model.entries[0].index)
self.assertFalse(model.entries[0].downloaded)
self.assertEqual(1000, model.entries[25].index)
self.assertEqual(1001, model.entries[26].index)
self.assertTrue(model.entries[26].url.find('t=59') > -1)
toc_download_entries(pname, cname, 1024, 1025)
model = load_model(pname, STHREAD_PATH, cname)
self.assertTrue(model.entries[49].downloaded)
self.assertFalse(model.entries[50].downloaded)
path = os.path.join(settings.PROJECT_FS_ROOT,
model.entries[49].local_paths[0])
self.assertTrue(os.path.exists(path))
path = os.path.join(settings.PROJECT_FS_ROOT,
model.entries[49].local_paths[1])
self.assertTrue(os.path.exists(path))
def test_apache_syncer(self):
create_channel_db('project1', 'cf', 'coreforum',
'channel.syncer.common_syncers.ApacheMailSyncer', 'foo.parser',
'http://mail-archives.apache.org/mod_mbox/hc-httpclient-users/'
)
create_channel_local('project1', 'coreforum',
'channel.syncer.common_syncers.ApacheMailSyncer',
'http://mail-archives.apache.org/mod_mbox/hc-httpclient-users/'
)
pname = 'project1'
cname = 'coreforum'
toc_refresh(pname, cname)
model = load_model(pname, STHREAD_PATH, cname)
self.assertEqual(
'http://mail-archives.apache.org/mod_mbox/hc-httpclient-users/200410.mbox/date',
model.toc_sections[0].url)
self.assertFalse(model.toc_sections[0].downloaded)
self.assertTrue(len(model.toc_sections) >= 79)
for i in xrange(0, 79):
self.assertEqual(i, model.toc_sections[i].index)
toc_download_section(pname, cname, start=0, end=4)
model = load_model(pname, STHREAD_PATH, cname)
self.assertTrue(model.toc_sections[0].downloaded)
self.assertTrue(model.toc_sections[1].downloaded)
self.assertTrue(model.toc_sections[2].downloaded)
self.assertTrue(model.toc_sections[3].downloaded)
self.assertFalse(model.toc_sections[4].downloaded)
self.assertEqual(316, len(model.entries))
self.assertEqual(0, model.entries[0].index)
self.assertFalse(model.entries[0].downloaded)
self.assertEqual(1000, model.entries[17].index)
self.assertEqual(1001, model.entries[18].index)
self.assertTrue(model.entries[18].url.find('xbox.localdomain') > -1)
toc_download_entries(pname, cname, 0, 1)
model = load_model(pname, STHREAD_PATH, cname)
self.assertTrue(model.entries[0].downloaded)
self.assertFalse(model.entries[1].downloaded)
path = os.path.join(settings.PROJECT_FS_ROOT,
model.entries[0].local_paths[0])
self.assertTrue(os.path.exists(path))
def create_channel(self):
create_channel_local(self.pname, 'usermail', 'foo', 'url')
channel = create_channel_db(self.pname, 'user mail', 'usermail',
'foo', 'foo', 'url')
thread1 = SupportThread(
title='HTTP Server Question',
channel=channel)
thread1.save()
thread2 = SupportThread(
title='Http client question',
channel=channel)
thread2.save()
thread3 = SupportThread(
title='Random Question',
channel=channel)
thread3.save()
message1 = Message(
title='HTTP Server Question',
index=0,
sthread=thread1
)
message1.save()
message2 = Message(
title='RE: HTTP Server Question',
index=1,
sthread=thread1
)
message2.save()
message3 = Message(
title='Http client question',
index=0,
sthread=thread2
)
message3.save()
message4 = Message(
title='RE: Http client question',
index=1,
sthread=thread2
)
message4.save()
message5 = Message(
title='Random Question',
index=0,
sthread=thread3
)
message5.save()
message6 = Message(
title='RE: RE: HTTP Server Question',
index=0,
sthread=thread1
)
message6.save()
coderef1 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='clazz1',
source='s',
kind_hint=self.class_kind,
local_context=message1,
global_context=thread1
)
coderef1.save()
self.code_refs.append(coderef1)
coderef2 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='Enum1',
source='s',
kind_hint=self.class_kind,
local_context=message2,
global_context=thread1,
)
coderef2.save()
self.code_refs.append(coderef2)
coderef3 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='Disco',
source='s',
kind_hint=self.class_kind,
local_context=message2,
global_context=thread1,
)
coderef3.save()
self.code_refs.append(coderef3)
coderef4 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='RecodocClient',
source='s',
kind_hint=self.class_kind,
local_context=message2,
global_context=thread1,
)
coderef4.save()
self.code_refs.append(coderef4)
coderef5 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='foo.RecodocClient',
source='s',
kind_hint=self.class_kind,
local_context=message2,
global_context=thread1,
)
coderef5.save()
self.code_refs.append(coderef5)
coderef6 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='callMethod1("hello")',
source='s',
kind_hint=self.method_kind,
local_context=message2,
global_context=thread1,
)
coderef6.save()
self.code_refs.append(coderef6)
coderef7 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='callMethod1("hello", 2)',
source='s',
kind_hint=self.method_kind,
local_context=message2,
global_context=thread1,
)
coderef7.save()
self.code_refs.append(coderef7)
coderef8 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='callMethod1(p3.Foo, p3.Bar)',
source='s',
kind_hint=self.method_kind,
local_context=message2,
global_context=thread1,
)
coderef8.save()
self.code_refs.append(coderef8)
coderef9 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='equals()',
source='s',
kind_hint=self.method_kind,
local_context=message2,
global_context=thread1,
)
coderef9.save()
self.code_refs.append(coderef9)
coderef10 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='list.add(foo)',
source='s',
kind_hint=self.class_kind,
local_context=message2,
global_context=thread1,
)
coderef10.save()
self.code_refs.append(coderef10)
coderef11 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='list.add(foo)',
source='s',
kind_hint=self.method_kind,
local_context=message2,
global_context=thread1,
parent_reference=coderef10,
)
coderef11.save()
self.code_refs.append(coderef11)
coderef12 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='nonexistentmethod',
source='s',
kind_hint=self.method_kind,
local_context=message2,
global_context=thread1,
parent_reference=coderef10,
)
coderef12.save()
self.code_refs.append(coderef12)
# Index = 14
coderef13 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='recodoc.callMethod10(foo)',
source='s',
kind_hint=self.class_kind,
local_context=message5,
global_context=thread3,
)
coderef13.save()
self.code_refs.append(coderef13)
# Index = 15
coderef14 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='recodoc.callMethod10(foo)',
source='s',
kind_hint=self.method_kind,
local_context=message5,
global_context=thread3,
parent_reference=coderef13,
)
coderef14.save()
self.code_refs.append(coderef14)
coderef15 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='getClient2()',
source='s',
kind_hint=self.method_kind,
local_context=message2,
global_context=thread1,
)
coderef15.save()
self.code_refs.append(coderef15)
# Index = 17
coderef16 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='method100()',
source='s',
kind_hint=self.method_kind,
local_context=message2,
global_context=thread1,
)
coderef16.save()
self.code_refs.append(coderef16)
coderef17 = SingleCodeReference(
project=self.project,
project_release=self.releasedb,
content='method200()',
source='s',
kind_hint=self.method_kind,
local_context=message2,
global_context=thread1,
)
coderef17.save()
self.code_refs.append(coderef17)
snippet_content = r'''
@Annotation1
public class FooBar extends FooBarSuper {
public void main(String arg) {
System.out.println();
RecodocClient obj = new RecodocClient();
List list = null;
aFoo.callMethod10();
}
'''
snippet1 = CodeSnippet(
project=self.project,
project_release=self.releasedb,
language='j',
source='s',
snippet_text=snippet_content,
local_context=message3,
global_context=thread2
)
snippet1.save()
self.code_snippets.append(snippet1)
def handle_noargs(self, **options):
pname = smart_decode(options.get('pname'))
cname = smart_decode(options.get('cname'))
url = smart_decode(options.get('url'))
syncer = smart_decode(options.get('syncer'))
create_channel_local(pname, cname, syncer, url)
def handle_noargs(self, **options):
pname = smart_decode(options.get('pname'))
cname = smart_decode(options.get('cname'))
url = smart_decode(options.get('url'))
syncer = smart_decode(options.get('syncer'))
create_channel_local(pname, cname, syncer, url)
def test_list_doc_local(self):
create_channel_local('project1', 'coreforum', 'foo.syncer',
'http://foo')
create_channel_local('project1', 'webforum', 'foo.syncer',
'http://foo')
self.assertEqual(2, len(list_channels_local('project1')))
def test_create_doc_local(self):
create_channel_local('project1', 'coreforum', 'foo.syncer', 'foo.com')
create_channel_local('project1', 'webforum', 'foo.syncer', 'foo.com')
path = get_channel_path('project1', root=True)
self.assertEqual(2, len(os.listdir(path)))