def run_autofix_test(
tmpdir: py.path.local,
method: typing.Callable[[typing.List[str]], int],
not_pretty_formatted_path: str,
formatted_path: str,
) -> None:
tmpdir.mkdir("src")
not_pretty_formatted_tmp_path = tmpdir.join("src").join(
basename(not_pretty_formatted_path))
# It is a relative paths as KTLint==0.41.0 dropped support for absolute paths
not_pretty_formatted_tmp_strpath = str(
tmpdir.bestrelpath(not_pretty_formatted_tmp_path))
copyfile(not_pretty_formatted_path, not_pretty_formatted_tmp_path)
with change_dir_context(tmpdir.strpath):
parameters = ["--autofix", not_pretty_formatted_tmp_strpath]
status_code = method(parameters)
if status_code != 1:
raise UnexpectedStatusCode(parameters=parameters,
expected_status_code=1,
actual_status_code=status_code)
# file was formatted (shouldn't trigger linter again)
with change_dir_context(tmpdir.strpath):
parameters = ["--autofix", not_pretty_formatted_tmp_strpath]
status_code = method(parameters)
if status_code != 0:
raise UnexpectedStatusCode(parameters=parameters,
expected_status_code=0,
actual_status_code=status_code)
assert not_pretty_formatted_tmp_path.read_text("utf-8") == py.path.local(
formatted_path).read_text("utf-8")
def model_name(tmpdir: py.path.local):
model = "unittest_model"
temp_model_dir = tmpdir.mkdir("models")
shutil.copytree("tests/data/models/sample_model",
temp_model_dir.join(model))
with mock.patch("src.model.MODELS_DIR", temp_model_dir):
yield model
def config(tmpdir: py.path.local) -> SDConfig:
'''Clone the module so we can modify it per test.'''
cnf = SDConfig()
data = tmpdir.mkdir('data')
keys = data.mkdir('keys')
os.chmod(str(keys), 0o700)
store = data.mkdir('store')
tmp = data.mkdir('tmp')
sqlite = data.join('db.sqlite')
# GPG 2.1+ requires gpg-agent, see #4013
gpg_agent_config = str(keys.join('gpg-agent.conf'))
with open(gpg_agent_config, 'w+') as f:
f.write('allow-loopback-pinentry')
gpg = gnupg.GPG('gpg2', homedir=str(keys))
for ext in ['sec', 'pub']:
file_path = path.join(path.dirname(__file__), 'files',
'test_journalist_key.{}'.format(ext))
with open(file_path) as f:
gpg.import_keys(f.read())
cnf.SECUREDROP_DATA_ROOT = str(data)
cnf.GPG_KEY_DIR = str(keys)
cnf.STORE_DIR = str(store)
cnf.TEMP_DIR = str(tmp)
cnf.DATABASE_FILE = str(sqlite)
# create the db file
subprocess.check_call(['sqlite3', cnf.DATABASE_FILE, '.databases'])
return cnf
def tmp_home_factory(tmpdir: py.path.local):
fake_home_dir = str(tmpdir.mkdir('home'))
def tmp_home():
return Path(fake_home_dir)
return tmp_home
def test_update_dependencies_with_valid_path(
tmpdir: py.path.local,
package: GreatExpectationsContribPackageManifest):
requirements_file = tmpdir.mkdir("tmp").join("requirements.txt")
contents = """
altair>=4.0.0,<5 # package
Click>=7.1.2 # package
mistune>=0.8.4,<2.0.0 # package
numpy>=1.14.1 # package
ruamel.yaml>=0.16,<0.17.18 # package
"""
requirements_file.write(contents)
package._update_dependencies(str(requirements_file))
assert package.dependencies == [
Dependency(text="altair",
link="https://pypi.org/project/altair",
version="<5, >=4.0.0"),
Dependency(text="Click",
link="https://pypi.org/project/Click",
version=">=7.1.2"),
Dependency(
text="mistune",
link="https://pypi.org/project/mistune",
version="<2.0.0, >=0.8.4",
),
Dependency(text="numpy",
link="https://pypi.org/project/numpy",
version=">=1.14.1"),
Dependency(
text="ruamel.yaml",
link="https://pypi.org/project/ruamel.yaml",
version="<0.17.18, >=0.16",
),
]
def fill_mock_conf(conf: T.Dict[T.Text, T.Any],
tmpdir: py.path.local) -> T.Text:
base_dir = tmpdir.mkdir("conf")
conf_path = base_dir.join("kofi.yml")
with open(conf_path, "w") as f:
f.write(yaml.dump(conf))
return conf_path
def dataset_masked(tmpdir: py.path.local):
dataset = "unittest_dataset"
temp_dataset_dir = tmpdir.mkdir("datasets")
shutil.copytree("tests/data/datasets/sample_masked",
temp_dataset_dir.join(dataset))
with mock.patch("src.dataset.common.DATASETS_DIR", temp_dataset_dir):
yield dataset
def test_update_from_package_info_with_valid_path(
tmpdir: py.path.local,
package: GreatExpectationsContribPackageManifest):
package_info_file = tmpdir.mkdir("tmp").join("package_info.yml")
contents = """
"""
package_info_file.write(contents)
package._update_from_package_info(str(package_info_file))
def run_autofix_test(
tmpdir: py.path.local,
method: typing.Callable[[typing.List[str]], int],
not_pretty_formatted_path: str,
formatted_path: str,
) -> None:
tmpdir.mkdir("src")
not_pretty_formatted_tmp_path = tmpdir.join("src").join(
basename(not_pretty_formatted_path)).strpath
copyfile(not_pretty_formatted_path, not_pretty_formatted_tmp_path)
with change_dir_context(tmpdir.strpath):
assert method(["--autofix", not_pretty_formatted_tmp_path]) == 1
# file was formatted (shouldn't trigger linter again)
with change_dir_context(tmpdir.strpath):
assert method(["--autofix", not_pretty_formatted_tmp_path]) == 0
assert __read_file(not_pretty_formatted_tmp_path) == __read_file(
formatted_path)
def test_update_from_package_info_with_valid_path_with_missing_keys(
tmpdir: py.path.local,
package: GreatExpectationsContribPackageManifest):
code_owners = [GitHubUser("John Doe")]
domain_experts = [DomainExpert("Charles Dickens")]
package.code_owners = code_owners
package.domain_experts = domain_experts
package_info_file = tmpdir.mkdir("tmp").join("package_info.yml")
contents = """
"""
package_info_file.write(contents)
package._update_from_package_info(str(package_info_file))
# Unchanged attrs since file state is invalid
assert package.code_owners == code_owners
assert package.domain_experts == domain_experts
def test_secrets_manager_file(tmpdir: py.path.local):
tmp = tmpdir.mkdir("file_test").dirname
os.environ["FLYTE_SECRETS_DEFAULT_DIR"] = tmp
sec = SecretsManager()
f = os.path.join(tmp, "test")
with open(f, "w+") as w:
w.write("my-password")
with pytest.raises(ValueError):
sec.get("test", "")
with pytest.raises(ValueError):
sec.get("", "x")
# Group dir not exists
with pytest.raises(ValueError):
sec.get("group", "test")
g = os.path.join(tmp, "group")
os.makedirs(g)
f = os.path.join(g, "test")
with open(f, "w+") as w:
w.write("my-password")
assert sec.get("group", "test") == "my-password"
del os.environ["FLYTE_SECRETS_DEFAULT_DIR"]
def make_image_tempdir(tmpdir: py.path.local) -> pl.Path:
return pl.Path(tmpdir.mkdir('photos'))
def test_gitignore(capsys, tmpdir: py.path.local):
import_content = """
import b
import a
"""
def main_check(args):
try:
main.main(args)
except SystemExit:
pass
return capsys.readouterr()
subprocess.run(["git", "init", str(tmpdir)])
python_file = tmpdir.join("has_imports.py")
python_file.write(import_content)
tmpdir.join("no_imports.py").write("...")
out, error = main_check([str(python_file), "--skip-gitignore", "--filter-files", "--check"])
assert "has_imports.py" in error and "no_imports.py" not in error
tmpdir.join(".gitignore").write("has_imports.py")
out, error = main_check([str(python_file), "--check"])
assert "has_imports.py" in error and "no_imports.py" not in error
out, error = main_check([str(python_file), "--skip-gitignore", "--filter-files", "--check"])
assert "Skipped" in out
# Should work with nested directories
tmpdir.mkdir("nested_dir")
tmpdir.join(".gitignore").write("nested_dir/has_imports.py")
subfolder_file = tmpdir.join("nested_dir/has_imports.py")
subfolder_file.write(import_content)
out, error = main_check([str(tmpdir), "--skip-gitignore", "--filter-files", "--check"])
assert "has_imports.py" in error and "nested_dir/has_imports.py" not in error
# Should work with relative path
currentdir = os.getcwd()
os.chdir(tmpdir)
out, error = main_check([".", "--skip-gitignore", "--filter-files", "--check"])
assert "has_imports.py" in error and "nested_dir/has_imports.py" not in error
tmpdir.join(".gitignore").write(
"""
nested_dir/has_imports.py
has_imports.py
"""
)
out, error = main_check([".", "--skip-gitignore", "--filter-files", "--check"])
assert "Skipped" in out
os.chdir(currentdir)
# Should work with multiple git projects
tmpdir.join(".git").remove()
tmpdir.join(".gitignore").remove()
# git_project0
# | has_imports_ignored.py ignored
# | has_imports.py should check
git_project0 = tmpdir.mkdir("git_project0")
subprocess.run(["git", "init", str(git_project0)])
git_project0.join(".gitignore").write("has_imports_ignored.py")
git_project0.join("has_imports_ignored.py").write(import_content)
git_project0.join("has_imports.py").write(import_content)
# git_project1
# | has_imports.py should check
# | nested_dir
# | has_imports_ignored.py ignored
# | has_imports.py should check
# | nested_dir_ignored ignored
# | has_imports.py ignored from folder
git_project1 = tmpdir.mkdir("git_project1")
subprocess.run(["git", "init", str(git_project1)])
git_project1.join(".gitignore").write(
"""
nested_dir/has_imports_ignored.py
nested_dir_ignored
"""
)
git_project1.join("has_imports.py").write(import_content)
nested_dir = git_project1.mkdir("nested_dir")
nested_dir.join("has_imports.py").write(import_content)
nested_dir.join("has_imports_ignored.py").write(import_content)
git_project1.mkdir("nested_dir_ignored").join("has_imports.py").write(import_content)
should_check = [
"/has_imports.py",
"/nested_dir/has_imports.py",
"/git_project0/has_imports.py",
"/git_project1/has_imports.py",
"/git_project1/nested_dir/has_imports.py",
]
out, error = main_check([str(tmpdir), "--skip-gitignore", "--filter-files", "--check"])
if os.name == "nt":
should_check = [sc.replace("/", "\\") for sc in should_check]
assert all(f"{str(tmpdir)}{file}" in error for file in should_check)
out, error = main_check([str(tmpdir), "--skip-gitignore", "--filter-files"])
assert all(f"{str(tmpdir)}{file}" in out for file in should_check)
# Should work when git project contains symlinks
if os.name != "nt":
git_project0.join("has_imports_ignored.py").write(import_content)
git_project0.join("has_imports.py").write(import_content)
tmpdir.join("has_imports.py").write(import_content)
tmpdir.join("nested_dir").join("has_imports.py").write(import_content)
git_project0.join("ignore_link.py").mksymlinkto(tmpdir.join("has_imports.py"))
git_project0.join("ignore_link").mksymlinkto(tmpdir.join("nested_dir"))
git_project0.join(".gitignore").write("ignore_link.py\nignore_link", mode="a")
out, error = main_check(
[str(git_project0), "--skip-gitignore", "--filter-files", "--check"]
)
should_check = ["/git_project0/has_imports.py"]
assert all(f"{str(tmpdir)}{file}" in error for file in should_check)
out, error = main_check([str(git_project0), "--skip-gitignore", "--filter-files"])
assert all(f"{str(tmpdir)}{file}" in out for file in should_check)
def recipes_folder(tmpdir: py.path.local):
"""Prepares a temp dir with '/recipes' folder as configured"""
orig_cwd = tmpdir.chdir()
yield tmpdir.mkdir(TEST_RECIPES_FOLDER)
orig_cwd.chdir()
