def cleanup():
"""Brings down containers and removes resources."""
helpers.execute_command([
"remove",
"--images",
"--volumes",
"--label",
"com.starburst.tests.module.test=catalog-test",
])
def cleanup(snapshot_name="test"):
"""Removes test snapshot tarball and turns off running resources."""
if not snapshot_name == "test":
subprocess.call(
f"rm -rf {os.path.join(helpers.MINIPRESTO_USER_SNAPSHOTS_DIR, snapshot_name)}.tar.gz",
shell=True,
)
else:
subprocess.call(f"rm -rf {helpers.SNAPSHOT_FILE}", shell=True)
helpers.execute_command(["down", "--sig-kill"])
def test_specific_directory():
"""Tests that the snapshot file can be saved in a user-specified
directory."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
result = helpers.execute_command(
[
"-v",
"snapshot",
"--name",
"test",
"--module",
"test",
"--directory",
"/tmp/",
],
command_input="y\n",
)
run_assertions(result, True, check_path=os.path.join(os.sep, "tmp"))
assert "Creating snapshot of specified modules" in result.output
subprocess.call("rm -rf /tmp/test.tar.gz", shell=True)
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_multiple_env():
"""Verifies that multiple environment variables can be successfully passed
in."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command([
"-v",
"--env",
"COMPOSE_PROJECT_NAME=test",
"--env",
"STARBURST_VER=338-e.1",
"--env",
"PRESTO=is=awesome",
"version",
])
assert result.exit_code == 0
assert all((
'"COMPOSE_PROJECT_NAME": "test"' in result.output,
'"STARBURST_VER": "338-e.1"' in result.output,
'"PRESTO": "is=awesome"' in result.output,
))
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_provision_append():
"""Verifies that modules can be appended to already-running environments."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["-v", "provision", "--module", "test"])
result = helpers.execute_command(
["-v", "provision", "--module", "postgres"])
containers = get_containers()
assert result.exit_code == 0
assert "Identified the following running modules" in result.output
assert len(containers) == 3 # presto, test, and postgres
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_duplicate_config_props():
"""Ensures that duplicate configuration properties in Presto are logged as a
warning to the user."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["-v", "provision"])
cmd_chunk = (
f"$'query.max-stage-count=85\nquery.max-stage-count=100"
f"\nquery.max-execution-time=1h\nquery.max-execution-time=2h'")
subprocess.Popen(
f'docker exec -i presto sh -c "echo {cmd_chunk} >> /usr/lib/presto/etc/config.properties"',
shell=True,
stdout=subprocess.PIPE,
universal_newlines=True,
)
cmd_chunk = "$'-Xms1G\n-Xms1G'"
subprocess.Popen(
f'docker exec -i presto sh -c "echo {cmd_chunk} >> /usr/lib/presto/etc/jvm.config"',
shell=True,
stdout=subprocess.PIPE,
universal_newlines=True,
)
# Hard stop to allow commands to process
time.sleep(2)
helpers.execute_command(["-v", "down", "--sig-kill", "--keep"])
result = helpers.execute_command(["-v", "provision"])
assert all((
"Duplicate Presto configuration properties detected in config.properties"
in result.output,
"query.max-stage-count=85" in result.output,
"query.max-stage-count=100" in result.output,
"query.max-execution-time=1h" in result.output,
"query.max-execution-time=2h" in result.output,
"Duplicate Presto configuration properties detected in jvm.config"
in result.output,
"-Xms1G" in result.output,
"-Xms1G" in result.output,
))
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_running():
"""Ensures the `module` command can output metadata for running modules."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["-v", "provision", "--module", "test"])
result = helpers.execute_command(["-v", "modules", "--json", "--running"])
assert result.exit_code == 0
assert all((
'"type": "catalog"' in result.output,
'"type": "catalog"' in result.output,
'"containers":' in result.output,
))
helpers.execute_command(["-v", "down", "--sig-kill"])
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_snapshot_active_env():
"""Verifies that a snapshot can be successfully created from an active
environment."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
helpers.execute_command(["-v", "provision", "--module", "test"])
result = helpers.execute_command(
["-v", "snapshot", "--name", "test"],
command_input="y\n",
)
run_assertions(result, False)
assert "Creating snapshot of active environment" in result.output
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_invalid_env():
"""Verifies that an invalid environment variable will cause the CLI to exit
with a non-zero status code."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(
["-v", "--env", "COMPOSE_PROJECT_NAMEtest", "version"])
assert result.exit_code == 2
assert "Invalid key-value pair" in result.output
result = helpers.execute_command(["-v", "--env", "=", "version"])
assert result.exit_code == 2
assert "Invalid key-value pair" in result.output
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_version():
"""Tests for correct version output."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(["version"])
assert pkg_resources.require("Minipresto")[0].version in result.output
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_valid_module():
"""Ensures the `module` command works when providing a valid module name."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(["-v", "modules", "--module", "test"])
assert result.exit_code == 0
assert all(("Module: test" in result.output, "Test module"
in result.output))
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_keep():
"""Verifies that the `--keep` flag works as expected."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["-v", "provision", "--module", "test"])
result = helpers.execute_command(["-v", "down", "--keep"])
assert "Stopped container" in result.output
assert "Removed container" not in result.output
docker_client = docker.from_env()
containers = docker_client.containers.list(
filters={"label": RESOURCE_LABEL}, all=True)
for container in containers:
assert container.name.lower() == "presto" or container.name.lower(
) == "test"
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_volumes():
"""Verifies that volumes with the standard Minipresto label applied to them
are removed."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["provision", "--module", "test"])
helpers.execute_command(["down", "--sig-kill"])
result = helpers.execute_command(["-v", "remove", "--volumes"])
assert result.exit_code == 0
assert "Volume removed:" in result.output
assert_docker_resource_count({
"resource_type": docker_client.volumes,
"label": RESOURCE_LABEL,
"expected_count": 0,
})
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_invalid_lib():
"""Verifies that Minipresto exists with a user error if pointing to an
invalid library."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
# Real directory, but ain't a real library
result = helpers.execute_command(
["-v", "--env", "LIB_PATH=/tmp/", "modules"])
assert result.exit_code == 2
assert "You must provide a path to a compatible Minipresto library" in result.output
# Fake directory
result = helpers.execute_command(
["-v", "--env", "LIB_PATH=/gucci-is-overrated/", "modules"])
assert result.exit_code == 2
assert "You must provide a path to a compatible Minipresto library" in result.output
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_env():
"""Verifies that an environment variable can be successfully passed in."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(
["-v", "--env", "COMPOSE_PROJECT_NAME=test", "version"])
assert result.exit_code == 0
assert "COMPOSE_PROJECT_NAME" and "test" in result.output
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_remove_dependent_resources_force():
"""Verifies that a dependent resources can be forcibly removed. Note that
even forcing a resource removal will not work if it is tied to a running
container.
Images can be forcibly removed if tied to a stop container. Volumes cannot
be removed if tied to any container, whether it is active or stopped. This
is a Docker-level restriction."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["provision", "--module", "test"])
subprocess.call("docker stop test", shell=True)
result = helpers.execute_command([
"-v",
"remove",
"--images",
"--label",
"com.starburst.tests.module.test=catalog-test",
"--force",
], )
assert result.exit_code == 0
assert "Image removed:" in result.output
assert_docker_resource_count(
{
"resource_type": docker_client.images,
"label": "com.starburst.tests.module.test=catalog-test",
"expected_count": 0,
},
{
"resource_type": docker_client.volumes,
"label": "com.starburst.tests.module.test=catalog-test",
"expected_count": 1,
},
)
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_invalid_label():
"""Verifies that images with the Minipresto label applied to them are
removed."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["provision", "--module", "test"])
helpers.execute_command(["down", "--sig-kill"])
result = helpers.execute_command(
["-v", "remove", "--images", "--label", "not-real-label=not-real"])
assert result.exit_code == 0
assert "Image removed:" not in result.output
assert_docker_resource_count({
"resource_type": docker_client.images,
"label": "com.starburst.tests.module.test=catalog-test",
"expected_count": 1,
})
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_install():
"""Verifies that the Minipresto library can be installed."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
# Install 0.0.0 since it's always around as a test release
result = helpers.execute_command(["-v", "lib_install", "--version", "0.0.0"])
assert result.exit_code == 0
assert os.path.isdir(os.path.join(helpers.MINIPRESTO_USER_DIR, "lib"))
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_invalid_module():
"""Ensures Minipresto exists with a user error if an invalid module name is
provided."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(
["-v", "modules", "--module", "not-a-real-module"])
assert result.exit_code == 2
assert "Invalid module" in result.output
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_remove_dependent_resources_stopped():
"""Verifies that a dependent resources (tied to stopped containers) cannot
be removed."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["provision", "--module", "test"])
subprocess.call("docker stop test", shell=True)
result = helpers.execute_command([
"-v",
"remove",
"--images",
"--volumes",
"--label",
"com.starburst.tests.module.test=catalog-test",
], )
assert result.exit_code == 0
assert all((
"Cannot remove volume:" in result.output,
"Cannot remove image:" in result.output,
))
assert_docker_resource_count(
{
"resource_type": docker_client.images,
"label": "com.starburst.tests.module.test=catalog-test",
"expected_count": 1,
},
{
"resource_type": docker_client.volumes,
"label": "com.starburst.tests.module.test=catalog-test",
"expected_count": 1,
},
)
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_json():
"""Ensures the `module` command can output module metadata in JSON
format."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(
["-v", "modules", "--module", "test", "--json"])
assert result.exit_code == 0
assert all(('"type": "catalog"' in result.output, '"test":'
in result.output))
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_invalid_ver():
"""Verifies that an error is raised if an incorrect version is passed to the
command."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(["-v", "lib_install", "--version", "YEE-PRESTO"])
assert result.exit_code == 1
assert not os.path.isdir(os.path.join(helpers.MINIPRESTO_USER_DIR, "lib"))
cleanup()
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_running_containers():
"""Verifies that the down command works when multiple containers are
running. This also verifies the --sig-kill option works."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
helpers.execute_command(["-v", "provision", "--module", "test"])
result = helpers.execute_command(["-v", "down", "--sig-kill"])
assert result.exit_code == 0
assert all((
"Stopped container" in result.output,
"Removed container" in result.output,
"test" in result.output,
"presto" in result.output,
))
docker_client = docker.from_env()
containers = docker_client.containers.list(
filters={"label": RESOURCE_LABEL})
assert len(containers) == 0, "There should be no running containers"
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_bootstrap_re_execute():
"""Ensures that bootstrap scripts do not execute if they have already
executed."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(["-v", "provision", "--module", "test"])
assert result.exit_code == 0
assert all((
"Bootstrap already executed in container 'presto'. Skipping.",
"Bootstrap already executed in container 'test'. Skipping.",
))
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_snapshot_standalone():
"""Verifies that a the standlone Presto module can be snapshotted."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
result = helpers.execute_command(
["-v", "snapshot", "--name", "test"],
command_input="y\n",
)
run_assertions(result, False)
assert "Snapshotting Presto resources only" in result.output
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_invalid_name():
"""Tests that all valid characters can be present and succeed for a given
snapshot name."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
result = helpers.execute_command(
["-v", "snapshot", "--name", "##.my-test?", "--module", "test"],
command_input="y\n",
)
assert result.exit_code == 2
assert "Illegal character found in provided filename" in result.output
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_force():
"""Verifies that the user can override the check to see if the resulting
tarball exists."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(
["-v", "snapshot", "--name", "test", "--module", "test", "--force"],
command_input="y\n",
)
run_assertions(result)
assert "Creating snapshot of specified modules" in result.output
cleanup()
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_all_modules():
"""Ensures that all module metadata is printed to the console if a module
name is not passed to the command."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(["-v", "modules"])
assert result.exit_code == 0
assert all((
"Module: test" in result.output,
"Description:" in result.output,
"Incompatible Modules:" in result.output,
))
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_incompatible_modules():
"""Verifies that chosen modules are not mutually-exclusive."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
result = helpers.execute_command(
["-v", "provision", "--module", "ldap", "--module", "test"])
assert result.exit_code == 2
assert all((
"Incompatible modules detected" in result.output,
"incompatible with module" in result.output,
"ldap" in result.output,
))
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
def test_snapshot_no_directory():
"""Verifies that a snapshot can be created when there is no existing
snapshots directory in the Minipresto user home directory."""
helpers.log_status(cast(FrameType, currentframe()).f_code.co_name)
cleanup()
subprocess.call(f"rm -rf {helpers.MINIPRESTO_USER_SNAPSHOTS_DIR}",
shell=True)
result = helpers.execute_command(
["-v", "snapshot", "--name", "test", "--module", "test"],
command_input="y\n",
)
run_assertions(result)
helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
