def _parse_paragraphs(self, message, load, text_paragraphs): for para_index, paragraph in enumerate(text_paragraphs): text = merge_lines(paragraph, False) kind_hint = self.kinds['unknown'] for i, code in enumerate( parse_single_code_references( text, kind_hint, self.kind_strategies, self.kinds, find_context=True, strict=True, code_words=load.code_words)): code.file_path = message.file_path code.url = message.url code.index = i + (para_index * 1000) code.project = self.channel.project code.local_context = message if load.entry is not None: code.global_context = load.entry code.save()
def _save_snippets(self, message, load, snippets): for index, (snippet, language) in enumerate(snippets): if language == REPLY_LANGUAGE: continue text = merge_lines(snippet) code = CodeSnippet(file_path=message.file_path, url=message.url, language=language, source=CHANNEL_SOURCE, snippet_text=text, index=index) code.local_context = message if load.entry is not None: code.global_context = load.entry code.project = self.channel.project code.save()
def _save_snippets(self, message, load, snippets): for index, (snippet, language) in enumerate(snippets): if language == REPLY_LANGUAGE: continue text = merge_lines(snippet) code = CodeSnippet( file_path=message.file_path, url=message.url, language=language, source=CHANNEL_SOURCE, snippet_text=text, index=index) code.local_context = message if load.entry is not None: code.global_context = load.entry code.project = self.channel.project code.save()
def _parse_paragraphs(self, message, load, text_paragraphs): for para_index, paragraph in enumerate(text_paragraphs): text = merge_lines(paragraph, False) kind_hint = self.kinds['unknown'] for i, code in enumerate( parse_single_code_references(text, kind_hint, self.kind_strategies, self.kinds, find_context=True, strict=True, code_words=load.code_words)): code.file_path = message.file_path code.url = message.url code.index = i + (para_index * 1000) code.project = self.channel.project code.local_context = message if load.entry is not None: code.global_context = load.entry code.save()
def is_exception_trace_lines(lines): exception_lines = 0 for line in lines: for pattern in EXCEPTION_PATTERNS: if pattern.search(line): exception_lines += 1 break lines_confidence = float(exception_lines) / len(lines) one_line_confidence = 0.0 one_line = su.merge_lines(lines, False) if EXCEPTION_LINE_PATTERN.search(one_line): one_line_confidence = 1.0 confidence = max(lines_confidence, one_line_confidence) if - 0.10 <= (confidence - THRESHOLD_EXCEPTION) <= 0: logger.info('Almost reached EXCEPTION threshold') logger.info(lines) return (confidence >= THRESHOLD_EXCEPTION, confidence)
def is_reply_header(lines): text = su.merge_lines(lines, False).strip() return (WROTE_RE.match(text) is not None, 1.0)