def test_apache_parser(self): create_channel_db('project1', 'cf', 'coreforum', 'channel.syncer.common_syncers.ApacheMailSyncer', 'channel.parser.common_parsers.ApacheMailParser', 'http://mail-archives.apache.org/mod_mbox/hc-httpclient-users/' ) create_channel_local('project1', 'coreforum', 'channel.syncer.common_syncers.ApacheMailSyncer', 'http://mail-archives.apache.org/mod_mbox/hc-httpclient-users/' ) pname = 'project1' cname = 'coreforum' toc_refresh(pname, cname) toc_download_section(pname, cname, start=0, end=1) toc_download_entries(pname, cname, 9, 99) parse_channel(pname, cname, True) self.assertEqual(8, Message.objects.all().count()) messages = list(Message.objects.all()) for message in messages: print('{0} by {1} on {2} (wc: {3})'.format( message.title, message.author, message.msg_date, message.word_count)) print(' {0} snippets and {1} references'.format( message.code_snippets.count(), message.code_references.count())) print(' Snippets:') for code_snippet in message.code_snippets.all(): print(' {0}'.format(code_snippet.language)) for ref in message.code_references.all(): print(' {0}: {1}'.format(ref.kind_hint.kind, ref.content)) # Test Snippets third_to_last = messages[-3] self.assertEqual('l', third_to_last.code_snippets.all()[0].language) self.assertEqual('jx', third_to_last.code_snippets.all()[1].language) # Test Refs fourth_to_last = messages[-4] refs = [ref.content.strip() for ref in fourth_to_last.code_references.all()] self.assertEqual(4, len(refs)) self.assertTrue('EasySSLProtocolSocketFactory' in refs) self.assertTrue('SSL' in refs) # Test Post-Processing! channel = post_process_channel(pname, cname) self.assertEqual(4, channel.threads.count()) second_thread = channel.threads.all()[1] self.assertEqual(3, second_thread.messages.count()) indexes = [msg.index for msg in second_thread.messages.all()] self.assertEqual([0, 1, 2], indexes) self.assertFalse(second_thread.messages.all()[0].title.lower() .startswith('re')) self.assertTrue(second_thread.messages.all()[1].title.lower() .startswith('re'))
def handle_noargs(self, **options): pname = smart_decode(options.get('pname')) cname = smart_decode(options.get('cname')) cfull_name = smart_decode(options.get('cfull_name')) url = smart_decode(options.get('url')) syncer = smart_decode(options.get('syncer')) parser = smart_decode(options.get('parser')) create_channel_db(pname, cfull_name, cname, syncer, parser, url) if (options.get('local', False)): create_channel_local(pname, cname, syncer, url)
def test_phpbb_parser(self): create_channel_db('project1', 'cf', 'coreforum', 'channel.syncer.common_syncers.PHPBBForumSyncer', 'channel.parser.common_parsers.PHPBBForumParser', 'https://forum.hibernate.org/viewforum.php?f=1' ) create_channel_local('project1', 'coreforum', 'channel.syncer.common_syncers.PHPBBForumSyncer', 'https://forum.hibernate.org/viewforum.php?f=1' ) pname = 'project1' cname = 'coreforum' toc_refresh(pname, cname) toc_download_section(pname, cname, start=0, end=2) toc_download_entries(pname, cname, 1023, 1025) parse_channel(pname, cname, True) self.assertEqual(23, Message.objects.all().count()) messages = list(Message.objects.all()) for message in messages: print(message.url) print('{0} by {1} on {2} (wc: {3})'.format( message.title, message.author, message.msg_date, message.word_count)) print(' {0} snippets and {1} references'.format( message.code_snippets.count(), message.code_references.count())) print(' Snippets:') for code_snippet in message.code_snippets.all(): print(' {0}'.format(code_snippet.language)) for ref in message.code_references.all(): print(' {0}: {1}'.format(ref.kind_hint.kind, ref.content)) # Test Snippets first_message = messages[0] self.assertEqual('x', first_message.code_snippets.all()[0].language) # Test Author self.assertEqual(first_message.author.nickname, 'mhellkamp') # Test Date self.assertEqual(first_message.msg_date, datetime(2003, 8, 29, 10, 16)) # Test Refs second_message = messages[1] refs = [ref.content.strip() for ref in second_message.code_references.all()] # Because of the title 4 + 1 = 5 self.assertEqual(5, len(refs)) self.assertTrue('DBCP' in refs) self.assertTrue('C3P0' in refs)
def test_fudeclipse_parser(self): create_channel_db('project1', 'cf', 'coreforum', 'channel.syncer.common_syncers.FUDEclipseForumSyncer', 'channel.parser.common_parsers.FUDEclipseForumParser', 'http://www.eclipse.org/forums/index.php/sf/thread/59/' ) create_channel_local('project1', 'coreforum', 'channel.syncer.common_syncers.FUDEclipseForumSyncer', 'http://www.eclipse.org/forums/index.php/sf/thread/59/' ) pname = 'project1' cname = 'coreforum' toc_refresh(pname, cname) toc_download_section(pname, cname, start=0, end=2) toc_download_entries(pname, cname, 0, 6) parse_channel(pname, cname, True) self.assertEqual(18, Message.objects.all().count()) messages = list(Message.objects.all()) for message in messages: print('{0} by {1} on {2} (wc: {3})'.format( message.title, message.author, message.msg_date, message.word_count)) print(' {0} snippets and {1} references'.format( message.code_snippets.count(), message.code_references.count())) print(' Snippets:') for code_snippet in message.code_snippets.all(): print(' {0}'.format(code_snippet.language)) for ref in message.code_references.all(): print(' {0}: {1}'.format(ref.kind_hint.kind, ref.content)) first_message = messages[0] # Test Title self.assertEqual(first_message.title, 'looping back to a previous step') # Test Author self.assertEqual(first_message.author.nickname, 'No real name') # Test Date self.assertEqual(first_message.msg_date, datetime(2010, 8, 26, 7, 50)) # Test Refs refs = [ref.content.strip() for ref in first_message.code_references.all()] self.assertEqual(3, len(refs)) self.assertTrue('ActivityElements' in refs)
def test_fudeclipse_syncer(self): create_channel_db('project1', 'cf', 'coreforum', 'channel.syncer.common_syncers.FUDEclipseForumSyncer', 'foo.parser', 'http://www.eclipse.org/forums/index.php/sf/thread/13/' ) create_channel_local('project1', 'coreforum', 'channel.syncer.common_syncers.FUDEclipseForumSyncer', 'http://www.eclipse.org/forums/index.php/sf/thread/13/' ) pname = 'project1' cname = 'coreforum' toc_refresh(pname, cname) model = load_model(pname, STHREAD_PATH, cname) self.assertEqual( 'http://www.eclipse.org/forums/index.php/sf/thread/13/1/0/', model.toc_sections[0].url) self.assertFalse(model.toc_sections[0].downloaded) self.assertTrue(len(model.toc_sections) >= 247) for i in xrange(0, 247): self.assertEqual(i, model.toc_sections[i].index) toc_download_section(pname, cname, start=0, end=4) model = load_model(pname, STHREAD_PATH, cname) self.assertTrue(model.toc_sections[0].downloaded) self.assertTrue(model.toc_sections[1].downloaded) self.assertTrue(model.toc_sections[2].downloaded) self.assertTrue(model.toc_sections[3].downloaded) self.assertFalse(model.toc_sections[4].downloaded) self.assertEqual(160, len(model.entries)) self.assertEqual(0, model.entries[0].index) self.assertFalse(model.entries[0].downloaded) self.assertEqual(1000, model.entries[40].index) self.assertEqual(1001, model.entries[41].index) #self.assertTrue(model.entries[26].url.find('t=59') > -1) toc_download_entries(pname, cname, 1039, 1040) model = load_model(pname, STHREAD_PATH, cname) self.assertTrue(model.entries[79].downloaded) self.assertFalse(model.entries[80].downloaded) path = os.path.join(settings.PROJECT_FS_ROOT, model.entries[79].local_paths[0]) self.assertTrue(os.path.exists(path))
def test_phpbb_syncer(self): create_channel_db('project1', 'cf', 'coreforum', 'channel.syncer.common_syncers.PHPBBForumSyncer', 'foo.parser', 'https://forum.hibernate.org/viewforum.php?f=1' ) create_channel_local('project1', 'coreforum', 'channel.syncer.common_syncers.PHPBBForumSyncer', 'https://forum.hibernate.org/viewforum.php?f=1' ) pname = 'project1' cname = 'coreforum' toc_refresh(pname, cname) model = load_model(pname, STHREAD_PATH, cname) self.assertEqual( 'https://forum.hibernate.org/viewforum.php?f=1&sd=a&start=0', model.toc_sections[0].url) self.assertFalse(model.toc_sections[0].downloaded) self.assertTrue(len(model.toc_sections) > 2349) for i in xrange(0, 2349): self.assertEqual(i, model.toc_sections[i].index) toc_download_section(pname, cname, start=0, end=4) model = load_model(pname, STHREAD_PATH, cname) self.assertTrue(model.toc_sections[0].downloaded) self.assertTrue(model.toc_sections[1].downloaded) self.assertTrue(model.toc_sections[2].downloaded) self.assertTrue(model.toc_sections[3].downloaded) self.assertFalse(model.toc_sections[4].downloaded) self.assertEqual(100, len(model.entries)) self.assertEqual(0, model.entries[0].index) self.assertFalse(model.entries[0].downloaded) self.assertEqual(1000, model.entries[25].index) self.assertEqual(1001, model.entries[26].index) self.assertTrue(model.entries[26].url.find('t=59') > -1) toc_download_entries(pname, cname, 1024, 1025) model = load_model(pname, STHREAD_PATH, cname) self.assertTrue(model.entries[49].downloaded) self.assertFalse(model.entries[50].downloaded) path = os.path.join(settings.PROJECT_FS_ROOT, model.entries[49].local_paths[0]) self.assertTrue(os.path.exists(path)) path = os.path.join(settings.PROJECT_FS_ROOT, model.entries[49].local_paths[1]) self.assertTrue(os.path.exists(path))
def test_apache_syncer(self): create_channel_db('project1', 'cf', 'coreforum', 'channel.syncer.common_syncers.ApacheMailSyncer', 'foo.parser', 'http://mail-archives.apache.org/mod_mbox/hc-httpclient-users/' ) create_channel_local('project1', 'coreforum', 'channel.syncer.common_syncers.ApacheMailSyncer', 'http://mail-archives.apache.org/mod_mbox/hc-httpclient-users/' ) pname = 'project1' cname = 'coreforum' toc_refresh(pname, cname) model = load_model(pname, STHREAD_PATH, cname) self.assertEqual( 'http://mail-archives.apache.org/mod_mbox/hc-httpclient-users/200410.mbox/date', model.toc_sections[0].url) self.assertFalse(model.toc_sections[0].downloaded) self.assertTrue(len(model.toc_sections) >= 79) for i in xrange(0, 79): self.assertEqual(i, model.toc_sections[i].index) toc_download_section(pname, cname, start=0, end=4) model = load_model(pname, STHREAD_PATH, cname) self.assertTrue(model.toc_sections[0].downloaded) self.assertTrue(model.toc_sections[1].downloaded) self.assertTrue(model.toc_sections[2].downloaded) self.assertTrue(model.toc_sections[3].downloaded) self.assertFalse(model.toc_sections[4].downloaded) self.assertEqual(316, len(model.entries)) self.assertEqual(0, model.entries[0].index) self.assertFalse(model.entries[0].downloaded) self.assertEqual(1000, model.entries[17].index) self.assertEqual(1001, model.entries[18].index) self.assertTrue(model.entries[18].url.find('xbox.localdomain') > -1) toc_download_entries(pname, cname, 0, 1) model = load_model(pname, STHREAD_PATH, cname) self.assertTrue(model.entries[0].downloaded) self.assertFalse(model.entries[1].downloaded) path = os.path.join(settings.PROJECT_FS_ROOT, model.entries[0].local_paths[0]) self.assertTrue(os.path.exists(path))
def create_channel(self): create_channel_local(self.pname, 'usermail', 'foo', 'url') channel = create_channel_db(self.pname, 'user mail', 'usermail', 'foo', 'foo', 'url') thread1 = SupportThread( title='HTTP Server Question', channel=channel) thread1.save() thread2 = SupportThread( title='Http client question', channel=channel) thread2.save() thread3 = SupportThread( title='Random Question', channel=channel) thread3.save() message1 = Message( title='HTTP Server Question', index=0, sthread=thread1 ) message1.save() message2 = Message( title='RE: HTTP Server Question', index=1, sthread=thread1 ) message2.save() message3 = Message( title='Http client question', index=0, sthread=thread2 ) message3.save() message4 = Message( title='RE: Http client question', index=1, sthread=thread2 ) message4.save() message5 = Message( title='Random Question', index=0, sthread=thread3 ) message5.save() message6 = Message( title='RE: RE: HTTP Server Question', index=0, sthread=thread1 ) message6.save() coderef1 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='clazz1', source='s', kind_hint=self.class_kind, local_context=message1, global_context=thread1 ) coderef1.save() self.code_refs.append(coderef1) coderef2 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='Enum1', source='s', kind_hint=self.class_kind, local_context=message2, global_context=thread1, ) coderef2.save() self.code_refs.append(coderef2) coderef3 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='Disco', source='s', kind_hint=self.class_kind, local_context=message2, global_context=thread1, ) coderef3.save() self.code_refs.append(coderef3) coderef4 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='RecodocClient', source='s', kind_hint=self.class_kind, local_context=message2, global_context=thread1, ) coderef4.save() self.code_refs.append(coderef4) coderef5 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='foo.RecodocClient', source='s', kind_hint=self.class_kind, local_context=message2, global_context=thread1, ) coderef5.save() self.code_refs.append(coderef5) coderef6 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='callMethod1("hello")', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, ) coderef6.save() self.code_refs.append(coderef6) coderef7 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='callMethod1("hello", 2)', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, ) coderef7.save() self.code_refs.append(coderef7) coderef8 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='callMethod1(p3.Foo, p3.Bar)', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, ) coderef8.save() self.code_refs.append(coderef8) coderef9 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='equals()', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, ) coderef9.save() self.code_refs.append(coderef9) coderef10 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='list.add(foo)', source='s', kind_hint=self.class_kind, local_context=message2, global_context=thread1, ) coderef10.save() self.code_refs.append(coderef10) coderef11 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='list.add(foo)', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, parent_reference=coderef10, ) coderef11.save() self.code_refs.append(coderef11) coderef12 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='nonexistentmethod', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, parent_reference=coderef10, ) coderef12.save() self.code_refs.append(coderef12) # Index = 14 coderef13 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='recodoc.callMethod10(foo)', source='s', kind_hint=self.class_kind, local_context=message5, global_context=thread3, ) coderef13.save() self.code_refs.append(coderef13) # Index = 15 coderef14 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='recodoc.callMethod10(foo)', source='s', kind_hint=self.method_kind, local_context=message5, global_context=thread3, parent_reference=coderef13, ) coderef14.save() self.code_refs.append(coderef14) coderef15 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='getClient2()', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, ) coderef15.save() self.code_refs.append(coderef15) # Index = 17 coderef16 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='method100()', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, ) coderef16.save() self.code_refs.append(coderef16) coderef17 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='method200()', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, ) coderef17.save() self.code_refs.append(coderef17) snippet_content = r''' @Annotation1 public class FooBar extends FooBarSuper { public void main(String arg) { System.out.println(); RecodocClient obj = new RecodocClient(); List list = null; aFoo.callMethod10(); } ''' snippet1 = CodeSnippet( project=self.project, project_release=self.releasedb, language='j', source='s', snippet_text=snippet_content, local_context=message3, global_context=thread2 ) snippet1.save() self.code_snippets.append(snippet1)
def handle_noargs(self, **options): pname = smart_decode(options.get('pname')) cname = smart_decode(options.get('cname')) url = smart_decode(options.get('url')) syncer = smart_decode(options.get('syncer')) create_channel_local(pname, cname, syncer, url)
def test_list_doc_local(self): create_channel_local('project1', 'coreforum', 'foo.syncer', 'http://foo') create_channel_local('project1', 'webforum', 'foo.syncer', 'http://foo') self.assertEqual(2, len(list_channels_local('project1')))
def test_create_doc_local(self): create_channel_local('project1', 'coreforum', 'foo.syncer', 'foo.com') create_channel_local('project1', 'webforum', 'foo.syncer', 'foo.com') path = get_channel_path('project1', root=True) self.assertEqual(2, len(os.listdir(path)))