def add_new_column():
test_schema["properties"]["third"] = {"type": "string"}
with test_schema_path.open("w") as fp:
json.dump(test_schema, fp)
run(f"git add {test_schema_path}")
run(["git", "commit", "-m", "Add a new column"])
return checkout_transpile_schemas(tmp_git / "schemas", "HEAD",
"HEAD~1", tmp_path / "integration")
def add_test_schema() -> Tuple[Path, Path]:
test_schema_path.parent.mkdir(parents=True, exist_ok=False)
with test_schema_path.open("w") as fp:
json.dump(test_schema, fp)
run(f"git add {test_schema_path}")
run(["git", "commit", "-m", "Add a test schema"])
return checkout_transpile_schemas(tmp_git / "schemas", "HEAD",
"HEAD~1", tmp_path / "integration")
def test_git_untracked_files(tmp_git: Path):
assert not git_untracked_files()
# schemas folder is checked by default
run("touch schemas/new_file")
# but not the tests directory
run("touch tests/new_file")
assert git_untracked_files() == ["schemas/new_file"]
assert git_untracked_files(directories=["schemas", "tests"]) == [
"schemas/new_file",
"tests/new_file",
]
def test_managed_git_state_stash_with_conflict(tmp_git: Path):
"""Conflicts made during visits are NOT handled, but the stash maintains history."""
filename = tmp_git / "README.md"
filename.open("w+").write("test")
diff = run("git diff")
assert len(diff) > 0, run("git status")
assert git_stash_size() == 0
with pytest.raises(subprocess.CalledProcessError) as excinfo:
with managed_git_state():
assert git_stash_size() == 1
run("git checkout HEAD~1")
filename.open("w+").write("test1")
assert "apply" in str(excinfo.value)
assert git_stash_size() == 1
def test_managed_git_state(tmp_git: Path):
original = run("git rev-parse HEAD")
with managed_git_state():
run("git checkout HEAD~1")
assert run("git rev-parse HEAD") != original
assert run("git rev-parse master"), "cannot see reference to master"
assert run("git rev-parse HEAD") == original
def test_managed_git_state_stash(tmp_git: Path):
"""Assert that top level stash is maintained when no changes are made during visits of revisions."""
filename = tmp_git / "README.md"
original = run("git rev-parse HEAD")
filename.open("w+").write("test")
diff = run("git diff")
assert len(diff) > 0, run("git status")
assert git_stash_size() == 0
with managed_git_state():
assert git_stash_size() == 1
run("git checkout HEAD~1")
with managed_git_state():
assert git_stash_size() == 1
run("git checkout HEAD~1")
assert git_stash_size() == 0
assert run("git rev-parse HEAD") == original
assert run("git diff") == diff
def diff(base_ref, head_ref, input_directory, output_directory):
# check that the correct tools are installed
run("diff --version")
run("git --version")
run("jsonschema-transpiler --version")
schemas_path = Path(input_directory)
integration_path = Path(output_directory)
head_rev_path, base_rev_path = checkout_transpile_schemas(
schemas_path, head_ref, base_ref, integration_path)
write_schema_diff(head_rev_path, base_rev_path, integration_path)
def tmp_git(tmp_path: Path) -> Path:
"""Copy the entire repository with the current revision.
To check the state of a failed test, change directories to the temporary
directory surfaced by pytest.
"""
curdir = os.getcwd()
origin = ROOT
workdir = tmp_path / "mps"
resolved_head_ref = resolve_ref("HEAD")
run(f"git clone {origin} {workdir}")
os.chdir(workdir)
# make branches available by checking them out, but ensure state ends up on HEAD
run(f"git checkout main")
run(f"git checkout {resolved_head_ref}")
yield workdir
os.chdir(curdir)
def diff(base_ref, head_ref, input_directory, output_directory):
# check that the correct tools are installed
run("diff --version")
run("git --version")
run("jsonschema-transpiler --version")
schemas_path = Path(input_directory)
integration_path = Path(output_directory)
head_rev_path, base_rev_path = checkout_transpile_schemas(
schemas_path, head_ref, base_ref, integration_path
)
# also compute the columns for each of these
def write_compact(path: Path):
for p in path.glob("*.bq"):
out = p.parent / p.name.replace(".bq", ".txt")
bq_schema = json.loads(p.read_text())
out.write_text("\n".join(compute_compact_columns(bq_schema)))
write_compact(head_rev_path)
write_compact(base_rev_path)
write_schema_diff(
head_rev_path,
base_rev_path,
integration_path,
prefix="bq_schema",
options="--new-file --exclude *.txt",
)
write_schema_diff(
head_rev_path,
base_rev_path,
integration_path,
prefix="compact_schema",
options="--new-file --exclude *.bq",
)
def test_dummy_git_env(tmp_git: Path):
assert Path(run("git remote get-url origin")) == ROOT
assert tmp_git != ROOT
from mozilla_pipeline_schemas.utils import run
from mozilla_pipeline_schemas.sink import transform_sink
from pathlib import Path
import tempfile
# Only run these relatively expensive checks once per test run
ENABLE_MPS_BIGQUERY_TESTS = False
BIGQUERY_MISCONFIGURED_REASON = "failed to configure mps bigquery"
try:
run("diff --version")
run("git --version")
run("jsonschema-transpiler --version")
ENABLE_MPS_BIGQUERY_TESTS = True
except Exception as ex:
BIGQUERY_MISCONFIGURED_REASON = f"{BIGQUERY_MISCONFIGURED_REASON}: {str(ex)}"
# the target directory is relative to this module
ENABLE_MPS_SINK_TESTS = False
SINK_MISCONFIGURED_REASON = "failed to configure mps bigquery sink"
root = Path(__file__).parent.parent
try:
transform_sink(root / "validation/telemetry/main.4.min.pass.json",
root / "target")
ENABLE_MPS_SINK_TESTS = True
except Exception as ex:
SINK_MISCONFIGURED_REASON = f"{SINK_MISCONFIGURED_REASON}: {str(ex)}"
def runif_cli_configured(func):
"""Decorator that checks if java dependencies are installed."""
import pytest
def test_run():
assert run("echo hello world") == "hello world"
with pytest.raises(subprocess.CalledProcessError):
run("false")
run("false", check=False)
