def test_scrub():
"""Verifies that sensitive data in user config file is scrubbed when
scrubbing is enabled."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.make_sample_config()
result = helpers.execute_command(
["-v", "snapshot", "--name", "test", "--module", "test"],
command_input="y\n")
run_assertions(result)
with open(snapshot_config_file()) as f:
assert "*" * 20 in f.read()
cleanup()
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_install_overwrite():
"""Verifies that the Minipresto library can be installed over an existing
library."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(
["-v", "lib_install", "--version", "0.0.0"], command_input="y\n"
)
assert result.exit_code == 0
assert os.path.isdir(os.path.join(helpers.MINIPRESTO_USER_DIR, "lib"))
assert "Removing existing library directory" in result.output
cleanup()
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_running():
"""Ensures the `module` command can output metadata for running modules."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["-v", "provision", "--module", "test"])
result = helpers.execute_command(["-v", "modules", "--json", "--running"])
assert result.exit_code == 0
assert all((
'"type": "catalog"' in result.output,
'"type": "catalog"' in result.output,
'"containers":' in result.output,
))
helpers.execute_command(["-v", "down", "--sig-kill"])
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_snapshot_no_directory():
"""Verifies that a snapshot can be created when there is no existing
snapshots directory in the Minipresto user home directory."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
subprocess.call(f"rm -rf {helpers.MINIPRESTO_USER_SNAPSHOTS_DIR}",
shell=True)
result = helpers.execute_command(
["-v", "snapshot", "--name", "test", "--module", "test"],
command_input="y\n",
)
run_assertions(result)
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_valid_user_config():
"""Ensures that valid, user-defined Presto/JVM config can be successfully
appended to Presto config files."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command([
"-v",
"--env",
"CONFIG=query.max-stage-count=85\nquery.max-execution-time=1h",
"--env",
"JVM_CONFIG=-Xmx2G\n-Xms1G",
"provision",
"--module",
"test",
])
assert result.exit_code == 0
assert ("Appending user-defined Presto config to Presto container config"
in result.output)
jvm_config = subprocess.Popen(
f"docker exec -i presto cat /usr/lib/presto/etc/jvm.config",
shell=True,
stdout=subprocess.PIPE,
universal_newlines=True,
)
presto_config = subprocess.Popen(
f"docker exec -i presto cat /usr/lib/presto/etc/config.properties",
shell=True,
stdout=subprocess.PIPE,
universal_newlines=True,
)
jvm_config, _ = jvm_config.communicate()
presto_config, _ = presto_config.communicate()
assert all(("-Xmx2G" in jvm_config, "-Xms1G" in jvm_config))
assert all((
"query.max-stage-count=85" in presto_config,
"query.max-execution-time=1h" in presto_config,
))
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_valid_name():
"""Tests that all valid characters can be present and succeed for a given
snapshot name."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
result = helpers.execute_command(
["-v", "snapshot", "--name", "my-test_123", "--module", "test"],
command_input="y\n",
)
run_assertions(result, snapshot_name="my-test_123")
assert "Creating snapshot of specified modules" in result.output
cleanup(snapshot_name="my-test_123")
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def main():
helpers.log_status(__file__)
test_daemon_off_all(
["-v", "down"],
["-v", "provision"],
["-v", "remove"],
[
"-v",
"snapshot",
"--name",
"test",
], # Applicable only when snapshotting active environment
["-v", "modules", "--running"],
)
test_env()
test_multiple_env()
test_invalid_env()
test_invalid_lib()
def test_no_containers():
"""Verifies that the down command functions appropriately when no containers
are running."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(["-v", "down"])
assert result.exit_code == 0
assert "No containers to bring down" in result.output
docker_client = docker.from_env()
containers = docker_client.containers.list(
filters={"label": RESOURCE_LABEL})
assert len(containers) == 0, "There should be no running containers"
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_invalid_env():
"""Verifies that an invalid environment variable will cause the CLI to exit
with a non-zero status code."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(
["-v", "--env", "COMPOSE_PROJECT_NAMEtest", "version"])
assert result.exit_code == 2
assert "Invalid key-value pair" in result.output
result = helpers.execute_command(["-v", "--env", "=", "version"])
assert result.exit_code == 2
assert "Invalid key-value pair" in result.output
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_standalone():
"""Verifies that a standalone Presto container is provisioned when no
options are passed in."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(["-v", "provision"])
assert result.exit_code == 0
assert "Provisioning standalone" in result.output
containers = get_containers()
assert len(containers) == 1
for container in containers:
assert container.name == "presto"
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_invalid_module():
"""Verifies that a non-zero status code is returned when attempting to
provision an invalid module."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command([
"-v", "provision", "--module", "hive-s3", "--module",
"not-a-real-module"
])
assert result.exit_code == 2
assert "Invalid module" in result.output
containers = get_containers()
assert len(containers) == 0
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_no_scrub():
"""Verifies that the user config file is retained in full when scrubbing is
disabled."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.make_sample_config()
result = helpers.execute_command(
["-v", "snapshot", "--name", "test", "--module", "test", "--no-scrub"],
True,
command_input="y\n",
)
run_assertions(result)
with open(snapshot_config_file()) as f:
assert "*" * 20 not in f.read()
cleanup()
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_docker_native():
"""Ensures that native Docker Compose command options can be appended to the
provisioning command.
This function also calls the bootstrap script test functions."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(
["-v", "provision", "--module", "test", "--docker-native", "--build"])
containers = get_containers()
assert "Received native Docker Compose options" in result.output
assert len(containers) == 2
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
test_bootstrap_script(result)
test_bootstrap_re_execute()
cleanup()
def test_invalid_lib():
"""Verifies that Minipresto exists with a user error if pointing to an
invalid library."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
# Real directory, but ain't a real library
result = helpers.execute_command(
["-v", "--env", "LIB_PATH=/tmp/", "modules"])
assert result.exit_code == 2
assert "You must provide a path to a compatible Minipresto library" in result.output
# Fake directory
result = helpers.execute_command(
["-v", "--env", "LIB_PATH=/gucci-is-overrated/", "modules"])
assert result.exit_code == 2
assert "You must provide a path to a compatible Minipresto library" in result.output
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_volumes():
"""Verifies that volumes with the standard Minipresto label applied to them
are removed."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["provision", "--module", "test"])
helpers.execute_command(["down", "--sig-kill"])
result = helpers.execute_command(["-v", "remove", "--volumes"])
assert result.exit_code == 0
assert "Volume removed:" in result.output
assert_docker_resource_count({
"resource_type": docker_client.volumes,
"label": RESOURCE_LABEL,
"expected_count": 0,
})
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_keep():
"""Verifies that the `--keep` flag works as expected."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["-v", "provision", "--module", "test"])
result = helpers.execute_command(["-v", "down", "--keep"])
assert "Stopped container" in result.output
assert "Removed container" not in result.output
docker_client = docker.from_env()
containers = docker_client.containers.list(
filters={"label": RESOURCE_LABEL}, all=True)
for container in containers:
assert container.name.lower() == "presto" or container.name.lower(
) == "test"
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_daemon_off_all(*args):
"""Verifies that each Minipresto command properly exits properly if the
Docker daemon is off or unresponsive."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
def run_daemon_assertions(result):
"""Runs standard assertions."""
assert result.exit_code == 2, f"Invalid exit code: {result.exit_code}"
assert (
"Error when pinging the Docker server. Is the Docker daemon running?"
in result.output), f"Unexpected output: {result.output}"
helpers.stop_docker_daemon()
for arg in args:
result = helpers.execute_command(arg)
run_daemon_assertions(result)
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_remove_dependent_resources_force():
"""Verifies that a dependent resources can be forcibly removed. Note that
even forcing a resource removal will not work if it is tied to a running
container.
Images can be forcibly removed if tied to a stop container. Volumes cannot
be removed if tied to any container, whether it is active or stopped. This
is a Docker-level restriction."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["provision", "--module", "test"])
subprocess.call("docker stop test", shell=True)
result = helpers.execute_command([
"-v",
"remove",
"--images",
"--label",
"com.starburst.tests.module.test=catalog-test",
"--force",
], )
assert result.exit_code == 0
assert "Image removed:" in result.output
assert_docker_resource_count(
{
"resource_type": docker_client.images,
"label": "com.starburst.tests.module.test=catalog-test",
"expected_count": 0,
},
{
"resource_type": docker_client.volumes,
"label": "com.starburst.tests.module.test=catalog-test",
"expected_count": 1,
},
)
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_specific_directory_invalid():
"""Tests that the snapshot file cannot be saved in an invalid directory."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
result = helpers.execute_command(
[
"-v",
"snapshot",
"--name",
"test",
"--module",
"test",
"--directory",
"/tmppp/",
],
command_input="y\n",
)
assert "Cannot save snapshot in nonexistent directory:" in result.output
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_invalid_label():
"""Verifies that images with the Minipresto label applied to them are
removed."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["provision", "--module", "test"])
helpers.execute_command(["down", "--sig-kill"])
result = helpers.execute_command(
["-v", "remove", "--images", "--label", "not-real-label=not-real"])
assert result.exit_code == 0
assert "Image removed:" not in result.output
assert_docker_resource_count({
"resource_type": docker_client.images,
"label": "com.starburst.tests.module.test=catalog-test",
"expected_count": 1,
})
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_remove_dependent_resources_stopped():
"""Verifies that a dependent resources (tied to stopped containers) cannot
be removed."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["provision", "--module", "test"])
subprocess.call("docker stop test", shell=True)
result = helpers.execute_command([
"-v",
"remove",
"--images",
"--volumes",
"--label",
"com.starburst.tests.module.test=catalog-test",
], )
assert result.exit_code == 0
assert all((
"Cannot remove volume:" in result.output,
"Cannot remove image:" in result.output,
))
assert_docker_resource_count(
{
"resource_type": docker_client.images,
"label": "com.starburst.tests.module.test=catalog-test",
"expected_count": 1,
},
{
"resource_type": docker_client.volumes,
"label": "com.starburst.tests.module.test=catalog-test",
"expected_count": 1,
},
)
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_command_snapshot_file():
"""Verifies that an environment can be provisioned from a snapshot command
file (these are written when a snapshot is created so that other users can
easily reproduce the environment)."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
command_snapshot_file = os.path.join(helpers.MINIPRESTO_USER_SNAPSHOTS_DIR,
"test", "provision-snapshot.sh")
process = subprocess.Popen(
command_snapshot_file,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
stdout, _ = process.communicate()
assert process.returncode == 0
assert "Environment provisioning complete" in stdout
cleanup()
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_running_containers():
"""Verifies that the down command works when multiple containers are
running. This also verifies the --sig-kill option works."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["-v", "provision", "--module", "test"])
result = helpers.execute_command(["-v", "down", "--sig-kill"])
assert result.exit_code == 0
assert all((
"Stopped container" in result.output,
"Removed container" in result.output,
"test" in result.output,
"presto" in result.output,
))
docker_client = docker.from_env()
containers = docker_client.containers.list(
filters={"label": RESOURCE_LABEL})
assert len(containers) == 0, "There should be no running containers"
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def main():
helpers.log_status(__file__)
# test_no_directory()
test_reset_with_directory()
def main():
helpers.log_status(__file__)
test_version()
def main():
helpers.log_status(__file__)
test_install()
test_install_overwrite()
test_invalid_ver()
