def test_end_to_end_full_path_spectrum(s3, tmpdir_factory, monkeypatch,
land_path, fail_path, pass_path,
log_path):
monkeypatch.setattr(fs, "S3FileSystem", mock_get_file)
from data_linter.validation import run_validation
test_folder = "tests/data/end_to_end1/"
land_folder = "tests/data/end_to_end1/land/"
config_path = os.path.join(test_folder, "config.yaml")
with open(config_path) as yml:
config = yaml.safe_load(yml)
if not fail_path.startswith("s3://"):
fail_path = tmpdir_factory.mktemp(fail_path)
fail_path = fail_path.strpath
if not pass_path.startswith("s3://"):
pass_path = tmpdir_factory.mktemp(pass_path)
pass_path = pass_path.strpath
if not log_path.startswith("s3://"):
log_path = tmpdir_factory.mktemp(log_path)
log_path = log_path.strpath
config["land-base-path"] = land_path
config["fail-base-path"] = fail_path
config["pass-base-path"] = pass_path
config["log-base-path"] = log_path
set_up_s3(s3, land_folder, config)
run_validation(config)
def test_parquet_linting(s3):
from data_linter.validation import run_validation
config = {
"land-base-path": "s3://land/",
"fail-base-path": "s3://fail/",
"pass-base-path": "s3://pass/",
"log-base-path": "s3://log/",
"compress-data": True,
"remove-tables-on-pass": False,
"all-must-pass": True,
"tables": {
"table1": {
"required": True,
"metadata": "tests/data/end_to_end2/metadata/table1.json",
"expect-header": True,
}
},
}
land_folder = "tests/data/end_to_end2/land/"
set_up_s3(s3, land_folder, config)
config["land-base-path"] = land_folder
run_validation(config)
def test_end_to_end_no_creds_error():
from data_linter.validation import run_validation
test_folder = "tests/data/end_to_end1/"
config_path = os.path.join(test_folder, "config.yaml")
with pytest.raises(Exception):
run_validation(config_path)
def test_end_to_end_no_creds_error():
from data_linter.validation import run_validation
from botocore.exceptions import NoCredentialsError
test_folder = "tests/data/end_to_end1/"
config_path = os.path.join(test_folder, "config.yaml")
with pytest.raises(NoCredentialsError):
run_validation(config_path)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"-v",
"--version",
action="version",
version="%(prog)s {version}".format(version=dl.__version__),
)
parser.add_argument("-c", "--config_path", help="Path to a config.yaml")
args = parser.parse_args()
run_validation(args.config_path)
def test_end_to_end_all_validators(s3, monkeypatch, validator):
monkeypatch.setattr(fs, "S3FileSystem", mock_get_file)
from data_linter.validation import run_validation
test_folder = "tests/data/end_to_end1/"
land_folder = "tests/data/end_to_end1/land/"
config_path = os.path.join(test_folder, "config.yaml")
with open(config_path) as f:
config = yaml.safe_load(f)
config["validator-engine"] = validator
set_up_s3(s3, land_folder, config)
run_validation(config)
def test_end_to_end(s3, monkeypatch):
monkeypatch.setattr(fs, "S3FileSystem", mock_get_file)
from data_linter.validation import run_validation
test_folder = "tests/data/end_to_end1/"
land_folder = "tests/data/end_to_end1/land/"
config_path = os.path.join(test_folder, "config.yaml")
with open(config_path) as f:
config = yaml.safe_load(f)
# if config.get("validator-engine", "pandas") == "pandas":
# config_path = os.path.join(test_folder, "config2.yaml")
set_up_s3(s3, land_folder, config)
run_validation(config_path)
os.system(
f"python data_linter/command_line.py --config-path {config_path}")