def test__parser__base_segments_raw_compare(): """Test comparison of raw segments.""" fp1 = FilePositionMarker.from_fresh() fp2 = FilePositionMarker.from_fresh() rs1 = RawSegment("foobar", fp1) rs2 = RawSegment("foobar", fp2) assert rs1 == rs2
def raw_seg_list(): """A generic list of raw segments to test against.""" return [ RawSegment("bar", FilePositionMarker()), RawSegment("foo", FilePositionMarker().advance_by("bar")), RawSegment("bar", FilePositionMarker().advance_by("barfoo")), ]
def test__parser__base_segments_base_compare(): """Test comparison of base segments.""" rs1 = RawSegment("foobar", FilePositionMarker()) rs2 = RawSegment("foobar", FilePositionMarker()) ds1 = DummySegment([rs1]) ds2 = DummySegment([rs2]) dsa2 = DummyAuxSegment([rs2]) # Check for equality assert ds1 == ds2 # Check a different match on the same details are not the same assert ds1 != dsa2
def test__parser__lexer_multimatcher(caplog): """Test the RepeatedMultiMatcher.""" matcher = RepeatedMultiMatcher( SingletonMatcher("dot", ".", RawSegment.make(".", name="dot", is_code=True)), RegexMatcher("test", r"#[^#]*#", RawSegment.make("test", name="test")), ) start_pos = FilePositionMarker() with caplog.at_level(logging.DEBUG): res = matcher.match("..#..#..#", start_pos) assert res.new_string == "#" # Should match right up to the final element assert res.new_pos == start_pos.advance_by("..#..#..") assert len(res.segments) == 5 assert res.segments[2].raw == "#..#"
def assert_matches(instring, matcher, matchstring): """Assert that a matcher does or doesn't work on a string. The optional `matchstring` argument, which can optionally be None, allows to either test positive matching of a particular string or negative matching (that it explicitly) doesn't match. """ start_pos = FilePositionMarker() res = matcher.match(instring, start_pos) # Check we've got the right type assert isinstance(res, LexMatch) if matchstring is None: assert res.new_string == instring assert res.new_pos == start_pos assert res.segments == () # tuple else: new_pos = start_pos.advance_by(matchstring) assert res.new_string == instring[len(matchstring):] assert res.new_pos == new_pos assert len(res.segments) == 1 assert res.segments[0].raw == matchstring
def _crawl_tree(cls, tree, variable_names, raw): """Crawl the tree looking for occurrences of the undeclared values.""" # First iterate through children for elem in tree.iter_child_nodes(): yield from cls._crawl_tree(elem, variable_names, raw) # Then assess self if isinstance(tree, jinja2.nodes.Name) and tree.name in variable_names: line_no = tree.lineno line = raw.split("\n")[line_no - 1] pos = line.index(tree.name) + 1 # Generate the charpos. +1 is for the newline characters themselves charpos = (sum( len(raw_line) + 1 for raw_line in raw.split("\n")[:line_no - 1]) + pos) # NB: The positions returned here will be *inconsistent* with those # from the linter at the moment, because these are references to the # structure of the file *before* templating. yield SQLTemplaterError( "Undefined jinja template variable: {0!r}".format(tree.name), pos=FilePositionMarker(None, line_no, pos, charpos), )
def test__parser__base_segments_raw_compare(): """Test comparison of raw segments.""" rs1 = RawSegment("foobar", FilePositionMarker()) rs2 = RawSegment("foobar", FilePositionMarker()) assert rs1 == rs2
def test__parser__base_segments_raw_init(): """Test initialisation. Other tests just use the fixture.""" RawSegment("foobar", FilePositionMarker())
def raw_seg(): """Construct a raw segment as a fixture.""" fp = FilePositionMarker().advance_by("abc") return RawSegment("foobar", fp)
def test__parser__base_segments_raw_init(): """Test initialisation. Other tests just use the fixture.""" fp = FilePositionMarker.from_fresh() RawSegment("foobar", fp)
result = Linter.parse_noqa(input, 0) if not isinstance(expected, type): assert result == expected else: # With exceptions, just check the type, not the contents. assert isinstance(result, expected) @pytest.mark.parametrize( "noqa,violations,expected", [ [ [], [ DummyLintError( FilePositionMarker(statement_index=None, line_no=1)) ], [ 0, ], ], [ [dict(comment="noqa: L001", line_no=1)], [ DummyLintError( FilePositionMarker(statement_index=None, line_no=1)) ], [], ], [ [dict(comment="noqa: L001", line_no=2)],
def process( self, *, in_str: str, fname: Optional[str] = None, config=None ) -> Tuple[Optional[TemplatedFile], list]: """Process a string and return the new string. Note that the arguments are enforced as keywords because Templaters can have differences in their `process` method signature. A Templater that only supports reading from a file would need the following signature: process(*, fname, in_str=None, config=None) (arguments are swapped) Args: in_str (:obj:`str`): The input string. fname (:obj:`str`, optional): The filename of this string. This is mostly for loading config files at runtime. config (:obj:`FluffConfig`): A specific config to use for this templating operation. Only necessary for some templaters. """ if not config: raise ValueError( "For the jinja templater, the `process()` method requires a config object." ) # Load the context live_context = self.get_context(fname=fname, config=config) # Apply dbt builtin functions if we're allowed. apply_dbt_builtins = config.get_section( (self.templater_selector, self.name, "apply_dbt_builtins") ) if apply_dbt_builtins: # This feels a bit wrong defining these here, they should probably # be configurable somewhere sensible. But for now they're not. # TODO: Come up with a better solution. dbt_builtins = self._generate_dbt_builtins() for name in dbt_builtins: # Only apply if it hasn't already been set at this stage. if name not in live_context: live_context[name] = dbt_builtins[name] env = self._get_jinja_env() # Load macros from path (if applicable) macros_path = config.get_section( (self.templater_selector, self.name, "load_macros_from_path") ) if macros_path: live_context.update( self._extract_macros_from_path(macros_path, env=env, ctx=live_context) ) # Load config macros, these will take precedence over macros from the path live_context.update( self._extract_macros_from_config(config=config, env=env, ctx=live_context) ) live_context.update(self._extract_libraries_from_config(config=config)) # Load the template, passing the global context. try: template = env.from_string(in_str, globals=live_context) except TemplateSyntaxError as err: # Something in the template didn't parse, return the original # and a violation around what happened. (len(line) for line in in_str.split("\n")[: err.lineno]) return ( TemplatedFile(source_str=in_str, fname=fname), [ SQLTemplaterError( "Failure to parse jinja template: {0}.".format(err), pos=FilePositionMarker( None, err.lineno, None, # Calculate the charpos for sorting. sum( len(line) for line in in_str.split("\n")[: err.lineno - 1] ), ), ) ], ) violations = [] # Attempt to identify any undeclared variables. The majority # will be found during the _crawl_tree step rather than this # first Exception which serves only to catch catastrophic errors. try: syntax_tree = env.parse(in_str) undefined_variables = meta.find_undeclared_variables(syntax_tree) except Exception as err: # TODO: Add a url here so people can get more help. raise SQLTemplaterError( "Failure in identifying Jinja variables: {0}.".format(err) ) # Get rid of any that *are* actually defined. for val in live_context: if val in undefined_variables: undefined_variables.remove(val) if undefined_variables: # Lets go through and find out where they are: for val in self._crawl_tree(syntax_tree, undefined_variables, in_str): violations.append(val) try: # NB: Passing no context. Everything is loaded when the template is loaded. out_str = template.render() # Slice the file once rendered. raw_sliced, sliced_file, out_str = self.slice_file( in_str, out_str, config=config ) return ( TemplatedFile( source_str=in_str, templated_str=out_str, fname=fname, sliced_file=sliced_file, raw_sliced=raw_sliced, ), violations, ) except (TemplateError, TypeError) as err: templater_logger.info("Unrecoverable Jinja Error: %s", err) violations.append( SQLTemplaterError( ( "Unrecoverable failure in Jinja templating: {0}. Have you configured " "your variables? https://docs.sqlfluff.com/en/latest/configuration.html" ).format(err) ) ) return None, violations