def parse_single_code_references(text, kind_hint, kind_strategies, kinds, kinds_hierarchies=ALL_KINDS_HIERARCHIES, save_index=False, strict=False, find_context=False, code_words=None): single_refs = [] matches = [] kind_text = kind_hint.kind if kind_text not in kind_strategies: kind_text = 'unknown' for strategy in kind_strategies[kind_text]: matches.extend(strategy.match(text)) if code_words is not None: matches.extend(parse_text_code_words(text, code_words)) # Sort to get correct indices matches.sort(key=lambda match: match[0][0]) avoided = process_matches(text, matches, single_refs, kinds, kinds_hierarchies, save_index, find_context) if len(single_refs) == 0 and not avoided and not strict: code = SingleCodeReference(content=text, kind_hint=kind_hint) code.save() single_refs.append(code) return single_refs
def parse_single_code_references(text, kind_hint, kind_strategies, kinds, kinds_hierarchies=ALL_KINDS_HIERARCHIES, save_index=False, strict=False, find_context=False, code_words=None): single_refs = [] matches = [] kind_text = kind_hint.kind if kind_text not in kind_strategies: kind_text = 'unknown' for strategy in kind_strategies[kind_text]: matches.extend(strategy.match(text)) if code_words is not None: matches.extend(parse_text_code_words(text, code_words)) # Sort to get correct indices matches.sort(key=lambda match: match[0][0]) avoided = process_matches(text, matches, single_refs, kinds, kinds_hierarchies, save_index, find_context) if len(single_refs) == 0 and not avoided and not strict: code = SingleCodeReference(content=text, kind_hint=kind_hint) code.save() single_refs.append(code) return single_refs
def process_matches(text, matches, single_refs, kinds, kinds_hierarchies, save_index, find_context): filtered = set() index = 0 avoided = False for match in matches: if is_valid_match(match, matches, filtered): (parent, children) = match content = text[parent[0]:parent[1]] if parent[2] == IGNORE_KIND: avoided = True continue main_reference = SingleCodeReference(content=content, kind_hint=kinds[parent[2]]) if save_index: main_reference.index = index if find_context: main_reference.sentence = find_sentence( text, parent[0], parent[1]) main_reference.paragraph = find_paragraph( text, parent[0], parent[1]) main_reference.save() single_refs.append(main_reference) # Process children process_children_matches(text, matches, children, index, single_refs, kinds, kinds_hierarchies, save_index, find_context) index += 1 else: filtered.add(match) return avoided
def create_documentation2(self): page1 = Page.objects.get(title='HTTP Server') section1 = Section.objects.get(title='Implementing foo bar') section11 = Section.objects.get(number='1.1') #section2 = Section.objects.get(number='2.') # Index = 19 coderef1 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='ChildClazz', source='d', kind_hint=self.class_kind, local_context=section11, mid_context=section1, global_context=page1 ) coderef1.save() self.code_refs.append(coderef1)
def process_matches(text, matches, single_refs, kinds, kinds_hierarchies, save_index, find_context): filtered = set() index = 0 avoided = False for match in matches: if is_valid_match(match, matches, filtered): (parent, children) = match content = text[parent[0]:parent[1]] if parent[2] == IGNORE_KIND: avoided = True continue main_reference = SingleCodeReference( content=content, kind_hint=kinds[parent[2]]) if save_index: main_reference.index = index if find_context: main_reference.sentence = find_sentence(text, parent[0], parent[1]) main_reference.paragraph = find_paragraph(text, parent[0], parent[1]) main_reference.save() single_refs.append(main_reference) # Process children process_children_matches(text, matches, children, index, single_refs, kinds, kinds_hierarchies, save_index, find_context) index += 1 else: filtered.add(match) return avoided
def _process_name(self, bindings, declarations, nodes, mcode): for i, binding in enumerate(bindings): declaration = declarations[i] parts = binding.split(self.HANDLE_SEPARATOR) kind = parts[0][2] kind_hint = None content = binding if kind == self.TYPE_KIND: kind_hint = self.class_kind elif kind == self.METHOD_KIND: kind_hint = self.method_kind elif kind == self.FIELD_KIND: kind_hint = self.field_kind elif kind == self.ANNOTATION_KIND: kind_hint = self.annotation_kind elif kind == self.ENUM_KIND: kind_hint = self.enumeration_kind else: logger.warning('Not a high level code reference {0}'.\ format(kind)) if kind_hint is not None: code = SingleCodeReference(content=content, xpath=mcode.xpath, file_path=mcode.file_path, kind_hint=kind_hint, declaration=declaration, snippet=mcode, source=mcode.source, index=(mcode.index * -1000) - i) code.project = self.project if mcode.project_release is not None: code.project_release = mcode.project_release code.local_context = mcode.local_context code.mid_context = mcode.mid_context code.global_context = mcode.global_context code.save()
def _process_name(self, bindings, declarations, nodes, mcode): for i, binding in enumerate(bindings): declaration = declarations[i] parts = binding.split(self.HANDLE_SEPARATOR) kind = parts[0][2] kind_hint = None content = binding if kind == self.TYPE_KIND: kind_hint = self.class_kind elif kind == self.METHOD_KIND: kind_hint = self.method_kind elif kind == self.FIELD_KIND: kind_hint = self.field_kind elif kind == self.ANNOTATION_KIND: kind_hint = self.annotation_kind elif kind == self.ENUM_KIND: kind_hint = self.enumeration_kind else: logger.warning('Not a high level code reference {0}'.\ format(kind)) if kind_hint is not None: code = SingleCodeReference( content=content, xpath=mcode.xpath, file_path=mcode.file_path, kind_hint=kind_hint, declaration=declaration, snippet=mcode, source=mcode.source, index=(mcode.index * -1000) - i) code.project = self.project if mcode.project_release is not None: code.project_release = mcode.project_release code.local_context = mcode.local_context code.mid_context = mcode.mid_context code.global_context = mcode.global_context code.save()
def process_children_matches(text, matches, children, index, single_refs, kinds, kinds_hierarchies, save_index, find_context): for i, child in enumerate(children): content = text[child[0]:child[1]] parent_reference = find_parent_reference(child[2], single_refs, kinds_hierarchies) child_reference = SingleCodeReference( content=content, kind_hint=kinds[child[2]], child_index=i, parent_reference=parent_reference) if save_index: child_reference.index = index if find_context: child_reference.sentence = find_sentence(text, child[0], child[1]) child_reference.paragraph = find_paragraph(text, child[0], child[1]) child_reference.save() single_refs.append(child_reference)
def process_children_matches(text, matches, children, index, single_refs, kinds, kinds_hierarchies, save_index, find_context): for i, child in enumerate(children): content = text[child[0]:child[1]] parent_reference = find_parent_reference(child[2], single_refs, kinds_hierarchies) child_reference = SingleCodeReference( content=content, kind_hint=kinds[child[2]], child_index=i, parent_reference=parent_reference) if save_index: child_reference.index = index if find_context: child_reference.sentence = find_sentence(text, child[0], child[1]) child_reference.paragraph = find_paragraph(text, child[0], child[1]) child_reference.save() single_refs.append(child_reference)
def create_documentation(self): create_doc_local(self.pname, 'manual', self.release, 'foo', 'url') doc = create_doc_db(self.pname, 'manual', self.release, 'url', 'foo', 'foo') page1 = Page(document=doc, title='HTTP Server') page1.save() page2 = Page(document=doc, title='HTTP Client') page2.save() section1 = Section(page=page1, number='1.', title='Implementing foo bar') section1.save() section11 = Section(page=page1, number='1.1', title='Reference', parent=section1) section11.save() section2 = Section(page=page2, number='2.', title='Implementing the Client') section2.save() section21 = Section(page=page2, number='2.1', title='Implementing the Client Again') section21.save() coderef1 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='@p1.Annotation1', source='d', kind_hint=self.ann_kind, local_context=section1, mid_context=section1, global_context=page1 ) coderef1.save() self.code_refs.append(coderef1) coderef2 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='@Annotation2', source='d', kind_hint=self.ann_kind, local_context=section1, mid_context=section1, global_context=page1 ) coderef2.save() self.code_refs.append(coderef2) snippet_content = r''' @Annotation1(foobarbaz=2) public class FooBar { public void main(String arg) { System.out.println(); RecodocClient rc = new RecodocClient(); } } ''' snippet1 = CodeSnippet( project=self.project, project_release=self.releasedb, language='j', source='d', snippet_text=snippet_content, local_context=section2, mid_context=section2, global_context=page2 ) snippet1.save() self.code_snippets.append(snippet1)
def create_channel2(self): thread1 = SupportThread.objects.get(title='HTTP Server Question') message2 = Message.objects.get(title='RE: HTTP Server Question') thread2 = SupportThread.objects.get(title='Http client question') message4 = Message.objects.get(title='RE: Http client question') message5 = Message.objects.get(title='RE: RE: HTTP Server Question') # Index = 20 coderef1 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='GeneralClient.getClient2()', source='s', kind_hint=self.class_kind, local_context=message2, global_context=thread1, ) coderef1.save() self.code_refs.append(coderef1) coderef2 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='GeneralClient.getClient2()', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, parent_reference=coderef1, ) coderef2.save() self.code_refs.append(coderef2) # Index = 22 coderef3 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='RecodocClient.getClient2()', source='s', kind_hint=self.class_kind, local_context=message2, global_context=thread1, ) coderef3.save() self.code_refs.append(coderef3) coderef4 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='RecodocClient.getClient2()', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, parent_reference=coderef3, ) coderef4.save() self.code_refs.append(coderef4) # Index = 24 coderef5 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='callMethod10()', source='s', kind_hint=self.method_kind, local_context=message4, global_context=thread2, ) coderef5.save() self.code_refs.append(coderef5) # Index = 25 coderef6 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='callMethod10()', source='s', kind_hint=self.method_kind, local_context=message5, global_context=thread1, ) coderef6.save() self.code_refs.append(coderef6) # Index = 26 coderef7 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='callMethod1000()', source='s', kind_hint=self.method_kind, local_context=message5, global_context=thread1, ) coderef7.save() self.code_refs.append(coderef7) # Index = 27 coderef8 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='AM', source='s', kind_hint=self.field_kind, local_context=message5, global_context=thread1, ) coderef8.save() self.code_refs.append(coderef8) # Index = 28 coderef9 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='CLIENT_NAME', source='s', kind_hint=self.field_kind, local_context=message5, global_context=thread1, ) coderef9.save() self.code_refs.append(coderef9) # Index = 29 coderef10 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='callMethod10', source='s', kind_hint=self.field_kind, local_context=message4, global_context=thread2, ) coderef10.save() self.code_refs.append(coderef10)
def create_channel(self): create_channel_local(self.pname, 'usermail', 'foo', 'url') channel = create_channel_db(self.pname, 'user mail', 'usermail', 'foo', 'foo', 'url') thread1 = SupportThread( title='HTTP Server Question', channel=channel) thread1.save() thread2 = SupportThread( title='Http client question', channel=channel) thread2.save() thread3 = SupportThread( title='Random Question', channel=channel) thread3.save() message1 = Message( title='HTTP Server Question', index=0, sthread=thread1 ) message1.save() message2 = Message( title='RE: HTTP Server Question', index=1, sthread=thread1 ) message2.save() message3 = Message( title='Http client question', index=0, sthread=thread2 ) message3.save() message4 = Message( title='RE: Http client question', index=1, sthread=thread2 ) message4.save() message5 = Message( title='Random Question', index=0, sthread=thread3 ) message5.save() message6 = Message( title='RE: RE: HTTP Server Question', index=0, sthread=thread1 ) message6.save() coderef1 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='clazz1', source='s', kind_hint=self.class_kind, local_context=message1, global_context=thread1 ) coderef1.save() self.code_refs.append(coderef1) coderef2 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='Enum1', source='s', kind_hint=self.class_kind, local_context=message2, global_context=thread1, ) coderef2.save() self.code_refs.append(coderef2) coderef3 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='Disco', source='s', kind_hint=self.class_kind, local_context=message2, global_context=thread1, ) coderef3.save() self.code_refs.append(coderef3) coderef4 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='RecodocClient', source='s', kind_hint=self.class_kind, local_context=message2, global_context=thread1, ) coderef4.save() self.code_refs.append(coderef4) coderef5 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='foo.RecodocClient', source='s', kind_hint=self.class_kind, local_context=message2, global_context=thread1, ) coderef5.save() self.code_refs.append(coderef5) coderef6 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='callMethod1("hello")', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, ) coderef6.save() self.code_refs.append(coderef6) coderef7 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='callMethod1("hello", 2)', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, ) coderef7.save() self.code_refs.append(coderef7) coderef8 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='callMethod1(p3.Foo, p3.Bar)', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, ) coderef8.save() self.code_refs.append(coderef8) coderef9 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='equals()', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, ) coderef9.save() self.code_refs.append(coderef9) coderef10 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='list.add(foo)', source='s', kind_hint=self.class_kind, local_context=message2, global_context=thread1, ) coderef10.save() self.code_refs.append(coderef10) coderef11 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='list.add(foo)', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, parent_reference=coderef10, ) coderef11.save() self.code_refs.append(coderef11) coderef12 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='nonexistentmethod', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, parent_reference=coderef10, ) coderef12.save() self.code_refs.append(coderef12) # Index = 14 coderef13 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='recodoc.callMethod10(foo)', source='s', kind_hint=self.class_kind, local_context=message5, global_context=thread3, ) coderef13.save() self.code_refs.append(coderef13) # Index = 15 coderef14 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='recodoc.callMethod10(foo)', source='s', kind_hint=self.method_kind, local_context=message5, global_context=thread3, parent_reference=coderef13, ) coderef14.save() self.code_refs.append(coderef14) coderef15 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='getClient2()', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, ) coderef15.save() self.code_refs.append(coderef15) # Index = 17 coderef16 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='method100()', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, ) coderef16.save() self.code_refs.append(coderef16) coderef17 = SingleCodeReference( project=self.project, project_release=self.releasedb, content='method200()', source='s', kind_hint=self.method_kind, local_context=message2, global_context=thread1, ) coderef17.save() self.code_refs.append(coderef17) snippet_content = r''' @Annotation1 public class FooBar extends FooBarSuper { public void main(String arg) { System.out.println(); RecodocClient obj = new RecodocClient(); List list = null; aFoo.callMethod10(); } ''' snippet1 = CodeSnippet( project=self.project, project_release=self.releasedb, language='j', source='s', snippet_text=snippet_content, local_context=message3, global_context=thread2 ) snippet1.save() self.code_snippets.append(snippet1)