def generate_test_segments_func(elems): """Roughly generate test segments. This function isn't totally robust, but good enough for testing. Use with caution. """ buff = [] raw_file = "".join(elems) templated_file = TemplatedFile.from_string(raw_file) idx = 0 for elem in elems: if elem == "<indent>": buff.append( Indent(pos_marker=PositionMarker.from_point( idx, idx, templated_file))) continue elif elem == "<dedent>": buff.append( Dedent(pos_marker=PositionMarker.from_point( idx, idx, templated_file))) continue seg_kwargs = {} if set(elem) <= {" ", "\t"}: SegClass = WhitespaceSegment elif set(elem) <= {"\n"}: SegClass = NewlineSegment elif elem == "(": SegClass = SymbolSegment seg_kwargs = {"name": "bracket_open"} elif elem == ")": SegClass = SymbolSegment seg_kwargs = {"name": "bracket_close"} elif elem.startswith("--"): SegClass = CommentSegment seg_kwargs = {"name": "inline_comment"} elif elem.startswith('"'): SegClass = CodeSegment seg_kwargs = {"name": "double_quote"} elif elem.startswith("'"): SegClass = CodeSegment seg_kwargs = {"name": "single_quote"} else: SegClass = CodeSegment # Set a none position marker which we'll realign at the end. buff.append( SegClass(raw=elem, pos_marker=PositionMarker( slice(idx, idx + len(elem)), slice(idx, idx + len(elem)), templated_file, ), **seg_kwargs)) idx += len(elem) return tuple(buff)
def test__parser__base_segments_stubs(): """Test stub methods that have no implementation in base class.""" template = TemplatedFile.from_string("foobar") rs1 = RawSegment("foobar", PositionMarker(slice(0, 6), slice(0, 6), template)) base_segment = BaseSegment(segments=[rs1]) with pytest.raises(NotImplementedError): base_segment.edit("foo")
def test__parser__base_segments_raw_compare(): """Test comparison of raw segments.""" template = TemplatedFile.from_string("foobar") rs1 = RawSegment("foobar", PositionMarker(slice(0, 6), slice(0, 6), template)) rs2 = RawSegment("foobar", PositionMarker(slice(0, 6), slice(0, 6), template)) assert rs1 == rs2
def test_markers__setting_position_raw(): """Test that we can correctly infer positions from strings & locations.""" templ = TemplatedFile.from_string("foobar") # Check inference in the template assert templ.get_line_pos_of_char_pos(2, source=True) == (1, 3) assert templ.get_line_pos_of_char_pos(2, source=False) == (1, 3) # Now check it passes through pos = PositionMarker(slice(2, 5), slice(2, 5), templ) # Can we infer positions correctly? assert pos.working_loc == (1, 3)
def test__parser__base_segments_base_compare(): """Test comparison of base segments.""" template = TemplatedFile.from_string("foobar") rs1 = RawSegment("foobar", PositionMarker(slice(0, 6), slice(0, 6), template)) rs2 = RawSegment("foobar", PositionMarker(slice(0, 6), slice(0, 6), template)) ds1 = DummySegment([rs1]) ds2 = DummySegment([rs2]) dsa2 = DummyAuxSegment([rs2]) # Check for equality assert ds1 == ds2 # Check a different match on the same details are not the same assert ds1 != dsa2
def lex( self, raw: Union[str, TemplatedFile] ) -> Tuple[Tuple[BaseSegment, ...], List[SQLLexError]]: """Take a string or TemplatedFile and return segments. If we fail to match the *whole* string, then we must have found something that we cannot lex. If that happens we should package it up as unlexable and keep track of the exceptions. """ # Make sure we've got a string buffer and a template # regardless of what was passed in. if isinstance(raw, str): template = TemplatedFile.from_string(raw) str_buff = raw else: template = raw str_buff = str(template) # Lex the string to get a tuple of LexedElement element_buffer: List[LexedElement] = [] while True: res = self.lex_match(str_buff, self.lexer_matchers) element_buffer += res.elements if res.forward_string: resort_res = self.last_resort_lexer.match(res.forward_string) if not resort_res: # If we STILL can't match, then just panic out. raise SQLLexError( f"Fatal. Unable to lex characters: {0!r}".format( res.forward_string[:10] + "..." if len(res.forward_string) > 9 else res. forward_string)) str_buff = resort_res.forward_string element_buffer += resort_res.elements else: break # Map tuple LexedElement to list of TemplateElement. # This adds the template_slice to the object. templated_buffer = self.map_template_slices(element_buffer, template) # Turn lexed elements into segments. segments: Tuple[RawSegment, ...] = self.elements_to_segments( templated_buffer, template) # Generate any violations violations: List[SQLLexError] = self.violations_from_segments(segments) return segments, violations
def test_linted_file_ignore_masked_violations( noqa: dict, violations: List[SQLBaseError], expected ): """Test that _ignore_masked_violations() correctly filters violations.""" ignore_mask = [Linter.parse_noqa(**c) for c in noqa] lf = linter.LintedFile( path="", violations=violations, time_dict={}, tree=None, ignore_mask=ignore_mask, templated_file=TemplatedFile.from_string(""), ) result = lf._ignore_masked_violations(violations) expected_violations = [v for i, v in enumerate(violations) if i in expected] assert expected_violations == result
def test_markers__setting_position_working(): """Test that we can correctly set positions manually.""" templ = TemplatedFile.from_string("foobar") pos = PositionMarker(slice(2, 5), slice(2, 5), templ, 4, 4) # Can we NOT infer when we're told. assert pos.working_loc == (4, 4)
def generate_test_segments_func(elems): """Roughly generate test segments. This function isn't totally robust, but good enough for testing. Use with caution. """ buff = [] raw_file = "".join(elems) templated_file = TemplatedFile.from_string(raw_file) idx = 0 for elem in elems: if elem == "<indent>": buff.append( Indent(pos_marker=PositionMarker.from_point( idx, idx, templated_file))) continue elif elem == "<dedent>": buff.append( Dedent(pos_marker=PositionMarker.from_point( idx, idx, templated_file))) continue if set(elem) <= {" ", "\t"}: cls = RawSegment.make(" ", name="whitespace", type="whitespace", _is_code=False) elif set(elem) <= {"\n"}: cls = RawSegment.make("\n", name="newline", type="newline", _is_code=False) elif elem == "(": cls = RawSegment.make("(", name="bracket_open") elif elem == ")": cls = RawSegment.make(")", name="bracket_close") elif elem.startswith("--"): cls = RawSegment.make("--", name="inline_comment", _is_code=False) elif elem.startswith('"'): cls = RawSegment.make('"', name="double_quote") elif elem.startswith("'"): cls = RawSegment.make("'", name="single_quote") else: cls = RawSegment.make("") # Set a none position marker which we'll realign at the end. buff.append( cls( elem, pos_marker=PositionMarker( slice(idx, idx + len(elem)), slice(idx, idx + len(elem)), templated_file, ), )) idx += len(elem) return tuple(buff)
def test__linted_file__slice_source_file_using_patches(source_patches, source_only_slices, raw_source_string, expected_result, caplog): """Test _slice_source_file_using_patches. This is part of fix_string(). """ with caplog.at_level(logging.DEBUG, logger="sqlfluff.linter"): result = LintedFile._slice_source_file_using_patches( source_patches, source_only_slices, raw_source_string) assert result == expected_result templated_file_1 = TemplatedFile.from_string("abc") templated_file_2 = TemplatedFile( "{# blah #}{{ foo }}bc", "<testing>", "abc", [ TemplatedFileSlice("comment", slice(0, 10), slice(0, 0)), TemplatedFileSlice("templated", slice(10, 19), slice(0, 1)), TemplatedFileSlice("literal", slice(19, 21), slice(1, 3)), ], [ RawFileSlice("{# blah #}", "comment", 0), RawFileSlice("{{ foo }}", "templated", 10), RawFileSlice("bc", "literal", 19), ], )