def _find_parent( search_start_offset: int, file: AnnotationManager, closest_left_node_id: int, ) -> Optional[bblfsh.Node]: """ Compute the UAST parent of the `TokenAnnotation` as the LCA of the closest left and right \ Babelfish nodes. :param search_start_offset: Offset of the current node. :param file: Source code annotated with `UASTAnnotation` and `TokenAnnotation`. :param closest_left_node_id: bblfsh node of the closest parent already gone through. :return: The `bblfsh.Node` of the found parent or None if no parent was found. """ left_ancestors = set() current_left_ancestor_id = closest_left_node_id parents = file.get(UASTAnnotation).parents while current_left_ancestor_id in parents: left_ancestors.add(id(parents[current_left_ancestor_id])) current_left_ancestor_id = id(parents[current_left_ancestor_id]) for future_vnode in file.iter_by_type( TokenAnnotation, start_offset=search_start_offset): if future_vnode.has_node: break else: return None current_right_ancestor_id = id(future_vnode.node) while current_right_ancestor_id in parents: if id(parents[current_right_ancestor_id]) in left_ancestors: return parents[current_right_ancestor_id] current_right_ancestor_id = id(parents[current_right_ancestor_id]) return None
def file_to_old_parse_file_format( file: AnnotationManager ) -> Tuple[List["VirtualNode"], Dict[int, bblfsh.Node]]: """ Convert `AnnotationManager` instance to the deprecated output format of \ `FeatureExtractor._parse_file()`. The function exists for backward compatibility and should be removed after the refactoring is \ finished. :param file: file annotated with `UASTAnnotation`, `PathAnnotation` and `RawTokenAnnotation`. \ It is expected to be the output of `FeatureExtractor._parse_file()`. :return: The old `FeatureExtractor._parse_file()` output format, that is \ Tuple with `VirtualNode`-s and `bbfsh.Node` id to parent mapping. """ vnodes = [] path = file.get(PathAnnotation).path raw_lines_data = file.sequence.splitlines(keepends=True) line_lens = [0] + [len(d) for d in raw_lines_data] line_lens[-1] += 1 _line_start_offsets = numpy.array(line_lens).cumsum() for annotation in file.iter_by_type(RawTokenAnnotation): vnode = VirtualNode( file[annotation.span], _to_position(raw_lines_data, _line_start_offsets, annotation.start), _to_position(raw_lines_data, _line_start_offsets, annotation.stop), is_accumulated_indentation=False, path=path, node=annotation.node, y=None, ) vnodes.append(vnode) return vnodes, file.get(UASTAnnotation).parents
def test_add(self): code = "0123456789" annotated_data = AnnotationManager(code) annotations = [Annotation(0, 3), Annotation(3, 3), Annotation(3, 4)] another_annotation = AnotherAnnotation(0, len(code) - 1) annotated_data.add(*annotations) annotated_data.add(another_annotation) overlapping_annotations = annotations + [ Annotation(0, 1), Annotation(1, 4), Annotation(3, 3), Annotation(2, 5) ] for annotation in overlapping_annotations: with self.assertRaises(ValueError): annotated_data.add(annotation) ok_annotations = [ Annotation(0, 0), Annotation(4, 4), Annotation(9, 11), Annotation(4, 9) ] for annotation in ok_annotations: annotated_data.add(annotation)
def setUp(self): config = FormatAnalyzer._load_config(get_config()) self.annotated_file = AnnotationManager.from_file(self.file) self.final_config = config["train"]["javascript"] self.extractor = FeatureExtractor( language="javascript", **self.final_config["feature_extractor"]) self.annotated_file = AnnotationManager.from_file(self.file)
def test_get(self): code = "0123456789" annotated_data = AnnotationManager(code) with self.assertRaises(Exception): annotated_data.get(Annotation) annotated_data.add(Annotation(4, 7)) annotated_data.get(Annotation, (4, 7)) with self.assertRaises(Exception): annotated_data.get(Annotation)
def test_getitem(self): am = AnnotationManager("") self.assertEqual(am[:1], "") with self.assertRaises(IndexError): am[0] seq = "0123456789" am = AnnotationManager(seq) self.assertEqual(am[0], seq[0]) self.assertEqual(am[0:5], seq[0:5]) self.assertEqual(am[(5, 7)], seq[5:7]) self.assertEqual(am[7:5], seq[7:5]) with self.assertRaises(IndexError): am[len(seq) + 1]
def _annotate_files( self, files: Iterable[UnicodeFile], lines: Optional[List[List[int]]] = None ) -> List[AnnotationManager]: parsed_files = [] for i, file in enumerate(files): path = file.path file = AnnotationManager.from_file(file) if lines is not None and lines[i] is not None: file.add( LinesToCheckAnnotation(0, len(file), frozenset(lines[i]))) try: self._parse_file(file) except AssertionError as e: self._log.warning("could not parse %s: error '%s', skipping", path, e) if self.debug_parsing: import traceback traceback.print_exc() input("Press Enter to continue…") continue self._classify_vnodes(file) self._merge_classes_to_composite_labels(file) self._add_noops(file) parsed_files.append(file) self._fill_vnode_parents(file) vnodes_parsed_number = sum( file.count(TokenAnnotation) for file in parsed_files) self._log.debug("Parsed %d vnodes", vnodes_parsed_number) return parsed_files
def test_classify_vnodes_with_trailing_space(self): contents = self.contents + " " file = BytesToUnicodeConverter.convert_file( File(content=contents.encode(), uast=self.uast, language="javascript", path="test")) annotated_file = AnnotationManager.from_file(file) self.extractor._parse_file(annotated_file) self.extractor._classify_vnodes(annotated_file) text = "".join( annotated_file[token.span] for token in annotated_file.iter_by_type(AtomicTokenAnnotation)) self.assertEqual(text, contents) cls_counts = Counter() old_stop = 0 for annotations in annotated_file.iter_by_type_nested( AtomicTokenAnnotation, ClassAnnotation): self.assertEqual(old_stop, annotations.start) if ClassAnnotation in annotations: cls_counts.update( map(CLASSES.__getitem__, annotations[ClassAnnotation].cls)) old_stop = annotations.stop self.assertEqual(len(contents), old_stop) self.assertEqual(cls_counts[CLS_SPACE_INC], cls_counts[CLS_SPACE_DEC] + 1) self.assertGreater(cls_counts[CLS_SPACE_INC], 0) self.assertGreater(cls_counts[CLS_SPACE], 0) self.assertGreater(cls_counts[CLS_NEWLINE], 0) self.assertGreater(cls_counts[CLS_SINGLE_QUOTE], 0) self.assertTrue(cls_counts[CLS_SINGLE_QUOTE] % 2 == 0)
def test_positions(self): test_js_code_filepath = Path( __file__).parent / "browser-policy-content.js" with open(str(test_js_code_filepath), mode="rb") as f: code = f.read() uast = bblfsh.BblfshClient("0.0.0.0:9432").parse(filename="", language="javascript", contents=code).uast converter = BytesToUnicodeConverter(code) code_uni = converter.convert_content() uast_uni = converter.convert_uast(uast) file = UnicodeFile(content=code_uni, uast=uast_uni, language="javascript", path="test.js") annotated_data = AnnotationManager.from_file(file) self.extractor._parse_file(annotated_data) nodes, _ = file_to_old_parse_file_format(annotated_data) for index, (node1, node2) in enumerate(zip(nodes, nodes[1:])): self.assertLessEqual( node1.start.line, node2.start.line, "Start line position decrease for %d, %d nodes" % (index, index + 1)) self.assertLessEqual( node1.start.offset, node2.start.offset, "Start offset position decrease for %d, %d nodes" % (index, index + 1))
def _merge_classes_to_composite_labels(self, file: AnnotationManager) -> None: """ Build "composite" `TokenAnnotation` and `LabelAnnotation` from predictable atomic tokens. :param file: Source code annotated with `AtomicTokenAnnotation`, `ClassAnnotation`, \ `AccumulatedIndentationAnnotation`. """ def _class_seq_to_annotations(start, stop, current_class_seq): if NEWLINE_INDEX not in current_class_seq or \ current_class_seq[0] == NEWLINE_INDEX: # if there are no trailing whitespaces or tabs yield TokenAnnotation(start, stop) yield LabelAnnotation(start, stop, tuple(current_class_seq)) else: index = current_class_seq.index(NEWLINE_INDEX) middle = start + index yield TokenAnnotation(start, middle) yield TokenAnnotation(middle, stop) yield LabelAnnotation(start, middle, tuple(current_class_seq[:index])) yield LabelAnnotation(middle, stop, tuple(current_class_seq[index:])) start, stop, current_class_seq = None, None, [] for annotations in file.iter_by_type_nested( AtomicTokenAnnotation, ClassAnnotation, AccumulatedIndentationAnnotation): has_target = ClassAnnotation in annotations acc_indent = AccumulatedIndentationAnnotation in annotations if (not has_target and not acc_indent or (has_target and annotations[ClassAnnotation].cls[0] in QUOTES_INDEX)): if current_class_seq: file.add(*_class_seq_to_annotations( start, stop, current_class_seq)) start, stop, current_class_seq = None, None, [] file.add( annotations[AtomicTokenAnnotation].to_token_annotation()) if ClassAnnotation in annotations: file.add( annotations[ClassAnnotation].to_target_annotation()) else: if not current_class_seq: start = annotations.start stop = annotations.stop if not acc_indent: current_class_seq.extend(annotations[ClassAnnotation].cls) if current_class_seq: file.add( *_class_seq_to_annotations(start, stop, current_class_seq))
def _file_to_vnodes_and_parents( file: AnnotationManager ) -> Tuple[List["VirtualNode"], Dict[int, bblfsh.Node]]: """ Convert one `AnnotationManager` instance to the deprecated format of \ `FeatureExtractor._annotate_files()` (`_parse_vnodes()` before refactoring). The old format is a sequence of vnodes and vnodes parents mapping. Used by `files_to_old_parse_file_format` to generate the old `_parse_vnodes`-like output format for a sequence of `AnnotationManager`-s. This function is different from `file_to_old_parse_file_format()` because it is created for `_parse_vnodes()` backward compatibility and `file_to_old_parse_file_format()` for `_parse_file()` backward compatibility. The function exists for backward compatibility and should be removed after the refactoring is \ finished. :param file: file annotated with `Path`-, `Token`-, `Label`-, `TokenParent`- `Annotation`. :return: Tuple with `VirtualNode`-s and node id to parents mapping. """ vnodes = [] path = file.get(PathAnnotation).path raw_lines_data = file.sequence.splitlines(keepends=True) line_lens = [0] + [len(d) for d in raw_lines_data] line_lens[-1] += 1 _line_start_offsets = numpy.array(line_lens).cumsum() vnode_parents = {} for annotations in file.iter_by_type_nested(TokenAnnotation, LabelAnnotation, TokenParentAnnotation): vnode = VirtualNode( file[annotations.span], _to_position(raw_lines_data, _line_start_offsets, annotations.start), _to_position(raw_lines_data, _line_start_offsets, annotations.stop), is_accumulated_indentation=False, path=path, node=annotations[TokenAnnotation].node, y=annotations[LabelAnnotation].label if LabelAnnotation in annotations else None, ) vnodes.append(vnode) vnode_parents[id(vnode)] = annotations[TokenParentAnnotation].parent \ if TokenParentAnnotation in annotations else None return vnodes, vnode_parents
def test_find_overlapping_annotation(self): code = "0123456789" annotated_data = AnnotationManager(code) annotations = [Annotation(0, 3), Annotation(3, 3), Annotation(3, 4)] annotated_data.add(*annotations[::-1]) for annotation in annotations: self.assertEqual( annotated_data.find_overlapping_annotation( Annotation, *annotation.span), annotation) self.assertEqual( annotated_data.find_overlapping_annotation(Annotation, 1, 2), annotations[0]) self.assertEqual( annotated_data.find_overlapping_annotation(Annotation, 3, 5), annotations[2]) self.assertEqual( annotated_data.find_overlapping_annotation(Annotation, 2, 4), annotations[0]) for span in [(4, 4), (4, 5), (5, 5)]: with self.assertRaises(NoAnnotation): annotated_data.find_overlapping_annotation(Annotation, *span)
def test_vnode_positions(self): test_js_code_filepath = Path(__file__).parent / "jquery.layout.js" with open(str(test_js_code_filepath), mode="rb") as f: code = f.read() uast = bblfsh.BblfshClient("0.0.0.0:9432").parse( filename="", language="javascript", contents=code).uast file = BytesToUnicodeConverter.convert_file( File(content=code, uast=uast, language="javascript", path="test.js")) annotated_data = AnnotationManager.from_file(file) self.extractor._parse_file(annotated_data) # Just should not fail self.extractor._classify_vnodes(annotated_data)
def test_iter_annotations(self): code = "0123456789" annotated_data = AnnotationManager(code) annotations = [Annotation(0, 3), Annotation(3, 3), Annotation(3, 4)] another_annotation = AnotherAnnotation(0, len(code) - 1) annotated_data.add(*annotations[::-1]) annotated_data.add(another_annotation) res = [ AnnotationsSpan(0, 3, { Annotation: annotations[0], AnotherAnnotation: another_annotation }), AnnotationsSpan(3, 3, { Annotation: annotations[1], AnotherAnnotation: another_annotation }), AnnotationsSpan(3, 4, { Annotation: annotations[2], AnotherAnnotation: another_annotation }) ] self.assertEqual( list( annotated_data.iter_by_type_nested(Annotation, AnotherAnnotation)), res) annotations = list( annotated_data.iter_by_type_nested(AnotherAnnotation, Annotation)) res = [ AnnotationsSpan(0, len(code) - 1, {AnotherAnnotation: another_annotation}) ] self.assertEqual(annotations, res)
def test_iter_annotation(self): code = "0123456789" annotated_data = AnnotationManager(code) with self.assertRaises(KeyError): list(annotated_data.iter_by_type(Annotation)) annotations = [Annotation(0, 3), Annotation(3, 3), Annotation(3, 4)] annotated_data.add(*annotations[::-1]) self.assertEqual(list(annotated_data.iter_by_type(Annotation)), annotations) more_annotations = [ Annotation(0, 0), Annotation(4, 4), Annotation(4, 7) ] annotated_data.add(*more_annotations) self.assertEqual( list(annotated_data.iter_by_type(Annotation)), sorted(annotations + more_annotations, key=lambda x: x.span))
def test_parse_file_comment_after_regexp(self): code = b"x = // comment\n/<regexp>/;" uast = bblfsh.BblfshClient("0.0.0.0:9432").parse(filename="", language="javascript", contents=code).uast file = BytesToUnicodeConverter.convert_file( File(uast=uast, content=code, language="javascript", path="")) annotated_file = AnnotationManager.from_file(file) self.extractor._parse_file(annotated_file) self.assertEqual( "".join( annotated_file[token.span] for token in annotated_file.iter_by_type(RawTokenAnnotation)), code.decode())
def test_parse_file_with_trailing_space(self): contents = self.contents + " " file = BytesToUnicodeConverter.convert_file( File(content=contents.encode(), uast=self.uast, language="javascript", path="test")) annotated_data = AnnotationManager.from_file(file) self.extractor._parse_file(annotated_data) nodes, _ = file_to_old_parse_file_format(annotated_data) offset, line, col = nodes[-1].end self.assertEqual(len(contents), offset) # Space token always ends on the same line self.assertEqual(len(contents.splitlines()), line) self.assertEqual("".join(n.value for n in nodes), contents)
def test_parse_file_exact_match(self): test_js_code_filepath = str( Path(__file__).parent / "for_parse_test.js.xz") with lzma.open(test_js_code_filepath, mode="rb") as f: code = f.read() uast = bblfsh.BblfshClient("0.0.0.0:9432").parse(filename="", language="javascript", contents=code).uast file = BytesToUnicodeConverter.convert_file( File(uast=uast, content=code, language="javascript", path="")) annotated_file = AnnotationManager.from_file(file) self.extractor._parse_file(annotated_file) nodes, _ = file_to_old_parse_file_format(annotated_file) self.assertEqual("".join(n.value for n in nodes), code.decode()) self.assertEqual( "".join( annotated_file[token.span] for token in annotated_file.iter_by_type(RawTokenAnnotation)), code.decode())
def _fill_vnode_parents(self, file: AnnotationManager): closest_left_node_id = None uast_annotation = file.get(UASTAnnotation) for annotation in file.iter_by_type(TokenAnnotation): if annotation.has_node: closest_left_node_id = id(annotation.node) file.add( TokenParentAnnotation( *annotation.span, uast_annotation.parents[closest_left_node_id])) else: parent = (self._find_parent(annotation.stop, file, closest_left_node_id) or uast_annotation.uast) file.add(TokenParentAnnotation(*annotation.span, parent))
def test_check_interval_crossing(self): data = [ ((9, 19), (19, 20), False), ((19, 20), (9, 19), False), ((1, 3), (2, 4), True), ((2, 4), (1, 3), True), ((-2, 4), (1, 3), True), ((-2, 3), (1, 3), True), ((1, 3), (1, 3), True), ((1, 3), (6, 7), False), ((10, 30), (6, 7), False), ((10, 10), (10, 10), True), ((10, 30), (10, 10), False), ((10, 10), (10, 30), False), ((10, 10), (5, 30), True), ((5, 30), (10, 10), True), ] for i, (interval1, interval2, res) in enumerate(data): self.assertEqual( AnnotationManager._check_spans_overlap(*interval1, *interval2), res, "Case # %d" % i)
def test_count(self): seq = "0123456789" am = AnnotationManager(seq) self.assertEqual(am.count(Annotation), 0) am.add(Annotation(0, 1)) self.assertEqual(am.count(Annotation), 1) am.add(Annotation(4, 5), Annotation(3, 4), Annotation(4, 4)) self.assertEqual(am.count(Annotation), 4) self.assertEqual(am.count(AnotherAnnotation), 0) am.add(AnotherAnnotation(4, 8)) self.assertEqual(am.count(Annotation), 4) self.assertEqual(am.count(AnotherAnnotation), 1) am.add(AnotherAnnotation(0, 3), AnotherAnnotation(3, 4), AnotherAnnotation(8, 8)) self.assertEqual(am.count(Annotation), 4) self.assertEqual(am.count(AnotherAnnotation), 4)
def _parse_file(self, file: AnnotationManager) -> None: """ Annotate source code with `RawTokenAnnotation`-s. Given the source text and the corresponding UAST this function covers all code with a `RawTokenAnnotation`-s. :param file: Source code annotated with `UASTAnnotation`. """ # TODO(zurk): rename this function when the refactoring is finished. contents = file.sequence # build the line mapping lines = contents.splitlines(keepends=True) # Check if there is a newline in the end of file. Yes, you can just check # lines[-1][-1] == "\n" but if someone decide to use weird '\u2028' unicode character for # new line this condition gives wrong result. eof_new_line = lines[-1].splitlines()[0] != lines[-1] if eof_new_line: # We add last line as empty one because it actually exists, but .splitlines() does not # return it. lines.append("") line_offsets = numpy.zeros(len(lines) + 1, dtype=numpy.int32) pos = 0 for i, line in enumerate(lines): line_offsets[i] = pos pos += len(line) line_offsets[-1] = pos + 1 # walk the tree: collect nodes with assigned tokens node_tokens = [] queue = [file.get(UASTAnnotation).uast] while queue: node = queue.pop() if node.internal_type in self.node_fixtures: self.node_fixtures[node.internal_type](node) queue.extend(node.children) if (node.token or node.start_position and node.end_position and node.start_position != node.end_position and not node.children): node_tokens.append(node) node_tokens.sort(key=lambda n: n.start_position.offset) sentinel = bblfsh.Node() sentinel.start_position.offset = len(contents) sentinel.start_position.line = len(lines) node_tokens.append(sentinel) # scan `node_tokens` and fill the gaps with imaginary nodes pos = 0 parser = self.tokens.PARSER for node in node_tokens: if node.start_position.offset < pos: continue if node.start_position.offset > pos: sumlen = 0 diff = contents[pos:node.start_position.offset] for match in parser.finditer(diff): offsets = [] for suboff in (match.start(), match.end()): offsets.append(pos + suboff) token = match.group() sumlen += len(token) file.add(RawTokenAnnotation(*offsets)) assert sumlen == node.start_position.offset - pos, \ "missed some imaginary tokens: \"%s\"" % diff if node is sentinel: break uast_node_annot = list( VirtualNode.from_node(node, contents, self.token_unwrappers)) file.add(*uast_node_annot) pos = node.end_position.offset
def _add_noops(self, file: AnnotationManager) -> None: """ Add `TokenAnnotation` with zero length in between `TokenAnnotation` without labeled nodes. Such zero length annotations means that some formatting sequence can be inserted to the annotation position. :param file: Source code annotated with `TokenAnnotation` and `LabelAnnotation`. """ noop_target = (CLASS_INDEX[CLS_NOOP], ) if not len(file): return prev_annotations = None for i, annotations in enumerate( file.iter_by_type_nested(TokenAnnotation, LabelAnnotation)): if i == 0: if LabelAnnotation not in annotations: file.add(TokenAnnotation(0, 0)) file.add(LabelAnnotation(0, 0, noop_target)) else: if LabelAnnotation not in prev_annotations and \ LabelAnnotation not in annotations: file.add( TokenAnnotation(annotations.start, annotations.start)) file.add( LabelAnnotation(annotations.start, annotations.start, noop_target)) prev_annotations = annotations if LabelAnnotation not in annotations: file.add(TokenAnnotation(annotations.stop, annotations.stop)) file.add( LabelAnnotation(annotations.stop, annotations.stop, noop_target))
def _classify_vnodes(self, file: AnnotationManager) -> None: """ Annotate source code with `AtomicTokenAnnotation`, `ClassAnnotation` and \ `AccumulatedIndentationAnnotation`. `ClassAnnotation` contains the index of the corresponding class to predict. We detect indentation changes, so several whitespace nodes are merged together. :param file: Source code annotated with `RawTokenAnnotation`. """ indentation = [] for token in file.iter_by_type(RawTokenAnnotation): token_value = file[token.span] if token.has_node: file.add(token.to_atomic_token_annotation()) continue if not token_value.isspace(): if token_value == "'": file.add( ClassAnnotation(*token.span, (CLASS_INDEX[CLS_SINGLE_QUOTE], ))) elif token_value == '"': file.add( ClassAnnotation(*token.span, (CLASS_INDEX[CLS_DOUBLE_QUOTE], ))) file.add(token.to_atomic_token_annotation()) continue lines = token_value.splitlines(keepends=True) if lines[-1].splitlines()[0] != lines[-1]: # We add last line as empty one to mimic .split("\n") behaviour lines.append("") if len(lines) == 1: # only tabs and spaces are possible for i, char in enumerate(token_value): if char == "\t": cls = (CLASS_INDEX[CLS_TAB], ) else: cls = (CLASS_INDEX[CLS_SPACE], ) offset = token.start file.add(ClassAnnotation(offset + i, offset + i + 1, cls)) file.add(AtomicTokenAnnotation(offset + i, offset + i + 1)) continue line_offset = 0 traling_chars = lines[0].splitlines()[0] if traling_chars: # node contains trailing whitespaces from the previous line assert set(traling_chars) <= {" ", "\t"} file.add( ClassAnnotation( token.start, token.start + len(traling_chars), tuple(CLASS_INDEX[CLS_SPACE if yi == " " else CLS_TAB] for yi in traling_chars))) file.add( AtomicTokenAnnotation(token.start, token.start + len(traling_chars))) lines[0] = lines[0][len(traling_chars):] line_offset += len(traling_chars) for line in lines[:-1]: # `line` ends with \r\n, we prepend \r to the newline node start_offset = token.start + line_offset file.add( ClassAnnotation(start_offset, start_offset + len(line), (NEWLINE_INDEX, ))) file.add( AtomicTokenAnnotation(start_offset, start_offset + len(line))) line_offset += len(line) line = lines[-1].splitlines()[0] if lines[-1] else "" my_indent = list(line) offset = token.stop offset -= len(line) try: for ws in indentation: my_indent.remove(ws) except ValueError: if my_indent: # mixed tabs and spaces, do not classify file.add(AtomicTokenAnnotation(offset, token.stop)) continue # indentation decreases if indentation[:len(line)]: file.add(AtomicTokenAnnotation(offset, token.stop)) file.add( AccumulatedIndentationAnnotation(offset, token.stop)) dec_class = [] for char in indentation[len(line):]: if char == "\t": cls = CLASS_INDEX[CLS_TAB_DEC] else: cls = CLASS_INDEX[CLS_SPACE_DEC] dec_class.append(cls) file.add(AtomicTokenAnnotation(token.stop, token.stop)) # It is not possible to have multiple zero-length intervals so we can only add it # with joined class file.add( ClassAnnotation(token.stop, token.stop, tuple(dec_class))) indentation = indentation[:len(line)] else: # indentation is stable or increases if indentation: file.add( AtomicTokenAnnotation(offset, offset + len(indentation))) file.add( AccumulatedIndentationAnnotation( offset, offset + len(indentation))) offset += len(indentation) for char in my_indent: indentation.append(char) for i, char in enumerate(my_indent): if char == "\t": cls = (CLASS_INDEX[CLS_TAB_INC], ) else: cls = (CLASS_INDEX[CLS_SPACE_INC], ) file.add(AtomicTokenAnnotation(offset + i, offset + i + 1)) file.add(ClassAnnotation(offset + i, offset + i + 1, cls)) offset += len(my_indent)
def test_len(self): self.assertEqual(0, len(AnnotationManager(""))) self.assertEqual(100, len(AnnotationManager(100 * "1")))