def test__config__load_user_appdir_config(mock_listdir, mock_path_exists,
mock_xdg_home):
"""Test loading config from user appdir."""
xdg_config_path = os.environ.get("XDG_CONFIG_HOME") + "/sqlfluff"
def path_exists(x):
if x == os.path.expanduser("~/.config/sqlfluff"):
return False
if x == xdg_config_path:
return False
else:
return True
mock_path_exists.side_effect = path_exists
c = ConfigLoader()
with patch.object(appdirs, attribute="system", new="darwin"):
resolved_path = c._get_user_config_dir_path()
c.load_user_appdir_config()
assert resolved_path == os.path.expanduser(
"~/Library/Application Support/sqlfluff")
mock_path_exists.assert_has_calls([
call(xdg_config_path),
call(os.path.expanduser("~/Library/Application Support/sqlfluff")),
])
def test__config__load_file_f():
"""Test loading config from a file path."""
c = ConfigLoader()
cfg = c.load_config_at_path(
os.path.join("test", "fixtures", "config", "inheritance_a",
"testing.sql"))
assert cfg == config_a
def test__config__load_toml():
"""Test loading config from a pyproject.toml file."""
c = ConfigLoader()
cfg = c.load_default_config_file(
os.path.join("test", "fixtures", "config", "toml"),
"pyproject.toml",
)
assert cfg == {
"core": {
"testing_int": 5,
"testing_bar": 7.698,
"testing_bool": False,
"testing_arr": ["a", "b", "c"],
"testing_inline_table": {
"x": 1
},
},
"bar": {
"foo": "foobar"
},
"fnarr": {
"fnarr": {
"foo": "foobar"
}
},
}
def test__config__iter_config_paths_right_order():
"""Test that config paths are fetched ordered by priority."""
c = ConfigLoader()
cfg_paths = c.iter_config_locations_up_to_path(
os.path.join("test", "fixtures", "config", "inheritance_a", "nested",
"blah.sql"),
working_path="test/fixtures",
)
assert list(cfg_paths) == [
str(Path(p).resolve()) for p in [
"test/fixtures",
"test/fixtures/config",
"test/fixtures/config/inheritance_a",
"test/fixtures/config/inheritance_a/nested",
]
]
def test__config__find_sqlfluffignore_in_same_directory():
"""Test find ignore file in the same directory as sql file."""
ignore_files = ConfigLoader.find_ignore_config_files(
path="test/fixtures/linter/sqlfluffignore/path_b/query_b.sql",
working_path="test/fixtures/linter/sqlfluffignore/",
)
assert ignore_files == {
os.path.abspath(
"test/fixtures/linter/sqlfluffignore/path_b/.sqlfluffignore"),
os.path.abspath("test/fixtures/linter/sqlfluffignore/.sqlfluffignore"),
}
def test__config__load_nested():
"""Test nested overwrite and order of precedence of config files in the same directory."""
c = ConfigLoader()
cfg = c.load_config_up_to_path(
os.path.join("test", "fixtures", "config", "inheritance_a", "nested",
"blah.sql"))
assert cfg == {
"core": {
"testing_val": "foobar",
"testing_int": 1,
"testing_bar": 7.698
},
"bar": {
"foo": "foobar"
},
"fnarr": {
"fnarr": {
"foo": "foobar"
}
},
}
def load_default_config() -> dict:
"""Loads the default configuration for the plugin."""
return ConfigLoader.get_global().load_default_config_file(
file_dir=os.path.dirname(__file__),
file_name="plugin_default_config.cfg",
)
def paths_from_path(
self,
path: str,
ignore_file_name: str = ".sqlfluffignore",
ignore_non_existent_files: bool = False,
ignore_files: bool = True,
working_path: str = os.getcwd(),
) -> List[str]:
"""Return a set of sql file paths from a potentially more ambiguous path string.
Here we also deal with the .sqlfluffignore file if present.
When a path to a file to be linted is explicitly passed
we look for ignore files in all directories that are parents of the file,
up to the current directory.
If the current directory is not a parent of the file we only
look for an ignore file in the direct parent of the file.
"""
if not os.path.exists(path):
if ignore_non_existent_files:
return []
else:
raise OSError("Specified path does not exist")
# Files referred to exactly are also ignored if
# matched, but we warn the users when that happens
is_exact_file = os.path.isfile(path)
path_walk: WalkableType
if is_exact_file:
# When the exact file to lint is passed, we
# fill path_walk with an input that follows
# the structure of `os.walk`:
# (root, directories, files)
dirpath = os.path.dirname(path)
files = [os.path.basename(path)]
path_walk = [(dirpath, None, files)]
else:
path_walk = list(os.walk(path))
ignore_file_paths = ConfigLoader.find_ignore_config_files(
path=path,
working_path=working_path,
ignore_file_name=ignore_file_name)
# Add paths that could contain "ignore files"
# to the path_walk list
path_walk_ignore_file = [
(
os.path.dirname(ignore_file_path),
None,
# Only one possible file, since we only
# have one "ignore file name"
[os.path.basename(ignore_file_path)],
) for ignore_file_path in ignore_file_paths
]
path_walk += path_walk_ignore_file
# If it's a directory then expand the path!
buffer = []
ignores = {}
for dirpath, _, filenames in path_walk:
for fname in filenames:
fpath = os.path.join(dirpath, fname)
# Handle potential .sqlfluffignore files
if ignore_files and fname == ignore_file_name:
with open(fpath) as fh:
spec = pathspec.PathSpec.from_lines("gitwildmatch", fh)
ignores[dirpath] = spec
# We don't need to process the ignore file any futher
continue
# We won't purge files *here* because there's an edge case
# that the ignore file is processed after the sql file.
# Scan for remaining files
for ext in (self.config.get(
"sql_file_exts", default=".sql").lower().split(",")):
# is it a sql file?
if fname.lower().endswith(ext):
buffer.append(fpath)
if not ignore_files:
return sorted(buffer)
# Check the buffer for ignore items and normalise the rest.
# It's a set, so we can do natural deduplication.
filtered_buffer = set()
for fpath in buffer:
abs_fpath = os.path.abspath(fpath)
for ignore_base, ignore_spec in ignores.items():
abs_ignore_base = os.path.abspath(ignore_base)
if abs_fpath.startswith(abs_ignore_base +
os.sep) and ignore_spec.match_file(
os.path.relpath(
abs_fpath, abs_ignore_base)):
# This file is ignored, skip it.
if is_exact_file:
linter_logger.warning(
"Exact file path %s was given but "
"it was ignored by a %s pattern in %s, "
"re-run with `--disregard-sqlfluffignores` to "
"skip %s" % (
path,
ignore_file_name,
ignore_base,
ignore_file_name,
))
break
else:
npath = os.path.normpath(fpath)
# For debugging, log if we already have the file.
if npath in filtered_buffer:
linter_logger.debug( # pragma: no cover
"Developer Warning: Path crawler attempted to "
"requeue the same file twice. %s is already in "
"filtered buffer.",
npath,
)
filtered_buffer.add(npath)
# Return a sorted list
return sorted(filtered_buffer)
def paths_from_path(
self,
path: str,
ignore_file_name: str = ".sqlfluffignore",
ignore_non_existent_files: bool = False,
ignore_files: bool = True,
working_path: str = os.getcwd(),
) -> List[str]:
"""Return a set of sql file paths from a potentially more ambiguous path string.
Here we also deal with the .sqlfluffignore file if present.
When a path to a file to be linted is explicitly passed
we look for ignore files in all directories that are parents of the file,
up to the current directory.
If the current directory is not a parent of the file we only
look for an ignore file in the direct parent of the file.
"""
if not os.path.exists(path):
if ignore_non_existent_files:
return []
else:
raise IOError("Specified path does not exist")
# Files referred to exactly are also ignored if
# matched, but we warn the users when that happens
is_exact_file = os.path.isfile(path)
if is_exact_file:
# When the exact file to lint is passed, we
# fill path_walk with an input that follows
# the structure of `os.walk`:
# (root, directories, files)
dirpath = os.path.dirname(path)
files = [os.path.basename(path)]
ignore_file_paths = ConfigLoader.find_ignore_config_files(
path=path,
working_path=working_path,
ignore_file_name=ignore_file_name)
# Add paths that could contain "ignore files"
# to the path_walk list
path_walk_ignore_file = [
(
os.path.dirname(ignore_file_path),
None,
# Only one possible file, since we only
# have one "ignore file name"
[os.path.basename(ignore_file_path)],
) for ignore_file_path in ignore_file_paths
]
path_walk: WalkableType = [(dirpath, None, files)
] + path_walk_ignore_file
else:
path_walk = os.walk(path)
# If it's a directory then expand the path!
buffer = []
ignore_set = set()
for dirpath, _, filenames in path_walk:
for fname in filenames:
fpath = os.path.join(dirpath, fname)
# Handle potential .sqlfluffignore files
if ignore_files and fname == ignore_file_name:
with open(fpath, "r") as fh:
spec = pathspec.PathSpec.from_lines("gitwildmatch", fh)
matches = spec.match_tree(dirpath)
for m in matches:
ignore_path = os.path.join(dirpath, m)
ignore_set.add(os.path.abspath(ignore_path))
# We don't need to process the ignore file any futher
continue
# We won't purge files *here* because there's an edge case
# that the ignore file is processed after the sql file.
# Scan for remaining files
for ext in self.config.get("sql_file_exts",
default=".sql").split(","):
# is it a sql file?
if fname.endswith(ext):
buffer.append(fpath)
if not ignore_files:
return sorted(buffer)
# Check the buffer for ignore items and normalise the rest.
filtered_buffer = []
for fpath in buffer:
if os.path.abspath(fpath) not in ignore_set:
filtered_buffer.append(os.path.normpath(fpath))
elif is_exact_file:
linter_logger.warning(
"Exact file path %s was given but "
"it was ignored by a %s pattern, "
"re-run with `--disregard-sqlfluffignores` to "
"skip %s" % (
path,
ignore_file_name,
ignore_file_name,
))
# Return
return sorted(filtered_buffer)