def add_new_column(): test_schema["properties"]["third"] = {"type": "string"} with test_schema_path.open("w") as fp: json.dump(test_schema, fp) run(f"git add {test_schema_path}") run(["git", "commit", "-m", "Add a new column"]) return checkout_transpile_schemas(tmp_git / "schemas", "HEAD", "HEAD~1", tmp_path / "integration")
def add_test_schema() -> Tuple[Path, Path]: test_schema_path.parent.mkdir(parents=True, exist_ok=False) with test_schema_path.open("w") as fp: json.dump(test_schema, fp) run(f"git add {test_schema_path}") run(["git", "commit", "-m", "Add a test schema"]) return checkout_transpile_schemas(tmp_git / "schemas", "HEAD", "HEAD~1", tmp_path / "integration")
def test_git_untracked_files(tmp_git: Path): assert not git_untracked_files() # schemas folder is checked by default run("touch schemas/new_file") # but not the tests directory run("touch tests/new_file") assert git_untracked_files() == ["schemas/new_file"] assert git_untracked_files(directories=["schemas", "tests"]) == [ "schemas/new_file", "tests/new_file", ]
def test_managed_git_state_stash_with_conflict(tmp_git: Path): """Conflicts made during visits are NOT handled, but the stash maintains history.""" filename = tmp_git / "README.md" filename.open("w+").write("test") diff = run("git diff") assert len(diff) > 0, run("git status") assert git_stash_size() == 0 with pytest.raises(subprocess.CalledProcessError) as excinfo: with managed_git_state(): assert git_stash_size() == 1 run("git checkout HEAD~1") filename.open("w+").write("test1") assert "apply" in str(excinfo.value) assert git_stash_size() == 1
def test_managed_git_state(tmp_git: Path): original = run("git rev-parse HEAD") with managed_git_state(): run("git checkout HEAD~1") assert run("git rev-parse HEAD") != original assert run("git rev-parse master"), "cannot see reference to master" assert run("git rev-parse HEAD") == original
def test_managed_git_state_stash(tmp_git: Path): """Assert that top level stash is maintained when no changes are made during visits of revisions.""" filename = tmp_git / "README.md" original = run("git rev-parse HEAD") filename.open("w+").write("test") diff = run("git diff") assert len(diff) > 0, run("git status") assert git_stash_size() == 0 with managed_git_state(): assert git_stash_size() == 1 run("git checkout HEAD~1") with managed_git_state(): assert git_stash_size() == 1 run("git checkout HEAD~1") assert git_stash_size() == 0 assert run("git rev-parse HEAD") == original assert run("git diff") == diff
def diff(base_ref, head_ref, input_directory, output_directory): # check that the correct tools are installed run("diff --version") run("git --version") run("jsonschema-transpiler --version") schemas_path = Path(input_directory) integration_path = Path(output_directory) head_rev_path, base_rev_path = checkout_transpile_schemas( schemas_path, head_ref, base_ref, integration_path) write_schema_diff(head_rev_path, base_rev_path, integration_path)
def tmp_git(tmp_path: Path) -> Path: """Copy the entire repository with the current revision. To check the state of a failed test, change directories to the temporary directory surfaced by pytest. """ curdir = os.getcwd() origin = ROOT workdir = tmp_path / "mps" resolved_head_ref = resolve_ref("HEAD") run(f"git clone {origin} {workdir}") os.chdir(workdir) # make branches available by checking them out, but ensure state ends up on HEAD run(f"git checkout main") run(f"git checkout {resolved_head_ref}") yield workdir os.chdir(curdir)
def diff(base_ref, head_ref, input_directory, output_directory): # check that the correct tools are installed run("diff --version") run("git --version") run("jsonschema-transpiler --version") schemas_path = Path(input_directory) integration_path = Path(output_directory) head_rev_path, base_rev_path = checkout_transpile_schemas( schemas_path, head_ref, base_ref, integration_path ) # also compute the columns for each of these def write_compact(path: Path): for p in path.glob("*.bq"): out = p.parent / p.name.replace(".bq", ".txt") bq_schema = json.loads(p.read_text()) out.write_text("\n".join(compute_compact_columns(bq_schema))) write_compact(head_rev_path) write_compact(base_rev_path) write_schema_diff( head_rev_path, base_rev_path, integration_path, prefix="bq_schema", options="--new-file --exclude *.txt", ) write_schema_diff( head_rev_path, base_rev_path, integration_path, prefix="compact_schema", options="--new-file --exclude *.bq", )
def test_dummy_git_env(tmp_git: Path): assert Path(run("git remote get-url origin")) == ROOT assert tmp_git != ROOT
from mozilla_pipeline_schemas.utils import run from mozilla_pipeline_schemas.sink import transform_sink from pathlib import Path import tempfile # Only run these relatively expensive checks once per test run ENABLE_MPS_BIGQUERY_TESTS = False BIGQUERY_MISCONFIGURED_REASON = "failed to configure mps bigquery" try: run("diff --version") run("git --version") run("jsonschema-transpiler --version") ENABLE_MPS_BIGQUERY_TESTS = True except Exception as ex: BIGQUERY_MISCONFIGURED_REASON = f"{BIGQUERY_MISCONFIGURED_REASON}: {str(ex)}" # the target directory is relative to this module ENABLE_MPS_SINK_TESTS = False SINK_MISCONFIGURED_REASON = "failed to configure mps bigquery sink" root = Path(__file__).parent.parent try: transform_sink(root / "validation/telemetry/main.4.min.pass.json", root / "target") ENABLE_MPS_SINK_TESTS = True except Exception as ex: SINK_MISCONFIGURED_REASON = f"{SINK_MISCONFIGURED_REASON}: {str(ex)}" def runif_cli_configured(func): """Decorator that checks if java dependencies are installed.""" import pytest
def test_run(): assert run("echo hello world") == "hello world" with pytest.raises(subprocess.CalledProcessError): run("false") run("false", check=False)