def test_scrub(): """Verifies that sensitive data in user config file is scrubbed when scrubbing is enabled.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.make_sample_config() result = helpers.execute_command( ["-v", "snapshot", "--name", "test", "--module", "test"], command_input="y\n") run_assertions(result) with open(snapshot_config_file()) as f: assert "*" * 20 in f.read() cleanup() helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_install_overwrite(): """Verifies that the Minipresto library can be installed over an existing library.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command( ["-v", "lib_install", "--version", "0.0.0"], command_input="y\n" ) assert result.exit_code == 0 assert os.path.isdir(os.path.join(helpers.MINIPRESTO_USER_DIR, "lib")) assert "Removing existing library directory" in result.output cleanup() helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_running(): """Ensures the `module` command can output metadata for running modules.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.execute_command(["-v", "provision", "--module", "test"]) result = helpers.execute_command(["-v", "modules", "--json", "--running"]) assert result.exit_code == 0 assert all(( '"type": "catalog"' in result.output, '"type": "catalog"' in result.output, '"containers":' in result.output, )) helpers.execute_command(["-v", "down", "--sig-kill"]) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_snapshot_no_directory(): """Verifies that a snapshot can be created when there is no existing snapshots directory in the Minipresto user home directory.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) cleanup() subprocess.call(f"rm -rf {helpers.MINIPRESTO_USER_SNAPSHOTS_DIR}", shell=True) result = helpers.execute_command( ["-v", "snapshot", "--name", "test", "--module", "test"], command_input="y\n", ) run_assertions(result) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_valid_user_config(): """Ensures that valid, user-defined Presto/JVM config can be successfully appended to Presto config files.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command([ "-v", "--env", "CONFIG=query.max-stage-count=85\nquery.max-execution-time=1h", "--env", "JVM_CONFIG=-Xmx2G\n-Xms1G", "provision", "--module", "test", ]) assert result.exit_code == 0 assert ("Appending user-defined Presto config to Presto container config" in result.output) jvm_config = subprocess.Popen( f"docker exec -i presto cat /usr/lib/presto/etc/jvm.config", shell=True, stdout=subprocess.PIPE, universal_newlines=True, ) presto_config = subprocess.Popen( f"docker exec -i presto cat /usr/lib/presto/etc/config.properties", shell=True, stdout=subprocess.PIPE, universal_newlines=True, ) jvm_config, _ = jvm_config.communicate() presto_config, _ = presto_config.communicate() assert all(("-Xmx2G" in jvm_config, "-Xms1G" in jvm_config)) assert all(( "query.max-stage-count=85" in presto_config, "query.max-execution-time=1h" in presto_config, )) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_valid_name(): """Tests that all valid characters can be present and succeed for a given snapshot name.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) cleanup() result = helpers.execute_command( ["-v", "snapshot", "--name", "my-test_123", "--module", "test"], command_input="y\n", ) run_assertions(result, snapshot_name="my-test_123") assert "Creating snapshot of specified modules" in result.output cleanup(snapshot_name="my-test_123") helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def main(): helpers.log_status(__file__) test_daemon_off_all( ["-v", "down"], ["-v", "provision"], ["-v", "remove"], [ "-v", "snapshot", "--name", "test", ], # Applicable only when snapshotting active environment ["-v", "modules", "--running"], ) test_env() test_multiple_env() test_invalid_env() test_invalid_lib()
def test_no_containers(): """Verifies that the down command functions appropriately when no containers are running.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command(["-v", "down"]) assert result.exit_code == 0 assert "No containers to bring down" in result.output docker_client = docker.from_env() containers = docker_client.containers.list( filters={"label": RESOURCE_LABEL}) assert len(containers) == 0, "There should be no running containers" helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_invalid_env(): """Verifies that an invalid environment variable will cause the CLI to exit with a non-zero status code.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command( ["-v", "--env", "COMPOSE_PROJECT_NAMEtest", "version"]) assert result.exit_code == 2 assert "Invalid key-value pair" in result.output result = helpers.execute_command(["-v", "--env", "=", "version"]) assert result.exit_code == 2 assert "Invalid key-value pair" in result.output helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_standalone(): """Verifies that a standalone Presto container is provisioned when no options are passed in.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command(["-v", "provision"]) assert result.exit_code == 0 assert "Provisioning standalone" in result.output containers = get_containers() assert len(containers) == 1 for container in containers: assert container.name == "presto" helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_invalid_module(): """Verifies that a non-zero status code is returned when attempting to provision an invalid module.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command([ "-v", "provision", "--module", "hive-s3", "--module", "not-a-real-module" ]) assert result.exit_code == 2 assert "Invalid module" in result.output containers = get_containers() assert len(containers) == 0 helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_no_scrub(): """Verifies that the user config file is retained in full when scrubbing is disabled.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.make_sample_config() result = helpers.execute_command( ["-v", "snapshot", "--name", "test", "--module", "test", "--no-scrub"], True, command_input="y\n", ) run_assertions(result) with open(snapshot_config_file()) as f: assert "*" * 20 not in f.read() cleanup() helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_docker_native(): """Ensures that native Docker Compose command options can be appended to the provisioning command. This function also calls the bootstrap script test functions.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command( ["-v", "provision", "--module", "test", "--docker-native", "--build"]) containers = get_containers() assert "Received native Docker Compose options" in result.output assert len(containers) == 2 helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) test_bootstrap_script(result) test_bootstrap_re_execute() cleanup()
def test_invalid_lib(): """Verifies that Minipresto exists with a user error if pointing to an invalid library.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) # Real directory, but ain't a real library result = helpers.execute_command( ["-v", "--env", "LIB_PATH=/tmp/", "modules"]) assert result.exit_code == 2 assert "You must provide a path to a compatible Minipresto library" in result.output # Fake directory result = helpers.execute_command( ["-v", "--env", "LIB_PATH=/gucci-is-overrated/", "modules"]) assert result.exit_code == 2 assert "You must provide a path to a compatible Minipresto library" in result.output helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_volumes(): """Verifies that volumes with the standard Minipresto label applied to them are removed.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.execute_command(["provision", "--module", "test"]) helpers.execute_command(["down", "--sig-kill"]) result = helpers.execute_command(["-v", "remove", "--volumes"]) assert result.exit_code == 0 assert "Volume removed:" in result.output assert_docker_resource_count({ "resource_type": docker_client.volumes, "label": RESOURCE_LABEL, "expected_count": 0, }) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_keep(): """Verifies that the `--keep` flag works as expected.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.execute_command(["-v", "provision", "--module", "test"]) result = helpers.execute_command(["-v", "down", "--keep"]) assert "Stopped container" in result.output assert "Removed container" not in result.output docker_client = docker.from_env() containers = docker_client.containers.list( filters={"label": RESOURCE_LABEL}, all=True) for container in containers: assert container.name.lower() == "presto" or container.name.lower( ) == "test" helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_daemon_off_all(*args): """Verifies that each Minipresto command properly exits properly if the Docker daemon is off or unresponsive.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) def run_daemon_assertions(result): """Runs standard assertions.""" assert result.exit_code == 2, f"Invalid exit code: {result.exit_code}" assert ( "Error when pinging the Docker server. Is the Docker daemon running?" in result.output), f"Unexpected output: {result.output}" helpers.stop_docker_daemon() for arg in args: result = helpers.execute_command(arg) run_daemon_assertions(result) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_remove_dependent_resources_force(): """Verifies that a dependent resources can be forcibly removed. Note that even forcing a resource removal will not work if it is tied to a running container. Images can be forcibly removed if tied to a stop container. Volumes cannot be removed if tied to any container, whether it is active or stopped. This is a Docker-level restriction.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.execute_command(["provision", "--module", "test"]) subprocess.call("docker stop test", shell=True) result = helpers.execute_command([ "-v", "remove", "--images", "--label", "com.starburst.tests.module.test=catalog-test", "--force", ], ) assert result.exit_code == 0 assert "Image removed:" in result.output assert_docker_resource_count( { "resource_type": docker_client.images, "label": "com.starburst.tests.module.test=catalog-test", "expected_count": 0, }, { "resource_type": docker_client.volumes, "label": "com.starburst.tests.module.test=catalog-test", "expected_count": 1, }, ) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_specific_directory_invalid(): """Tests that the snapshot file cannot be saved in an invalid directory.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) cleanup() result = helpers.execute_command( [ "-v", "snapshot", "--name", "test", "--module", "test", "--directory", "/tmppp/", ], command_input="y\n", ) assert "Cannot save snapshot in nonexistent directory:" in result.output helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_invalid_label(): """Verifies that images with the Minipresto label applied to them are removed.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.execute_command(["provision", "--module", "test"]) helpers.execute_command(["down", "--sig-kill"]) result = helpers.execute_command( ["-v", "remove", "--images", "--label", "not-real-label=not-real"]) assert result.exit_code == 0 assert "Image removed:" not in result.output assert_docker_resource_count({ "resource_type": docker_client.images, "label": "com.starburst.tests.module.test=catalog-test", "expected_count": 1, }) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_remove_dependent_resources_stopped(): """Verifies that a dependent resources (tied to stopped containers) cannot be removed.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.execute_command(["provision", "--module", "test"]) subprocess.call("docker stop test", shell=True) result = helpers.execute_command([ "-v", "remove", "--images", "--volumes", "--label", "com.starburst.tests.module.test=catalog-test", ], ) assert result.exit_code == 0 assert all(( "Cannot remove volume:" in result.output, "Cannot remove image:" in result.output, )) assert_docker_resource_count( { "resource_type": docker_client.images, "label": "com.starburst.tests.module.test=catalog-test", "expected_count": 1, }, { "resource_type": docker_client.volumes, "label": "com.starburst.tests.module.test=catalog-test", "expected_count": 1, }, ) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_command_snapshot_file(): """Verifies that an environment can be provisioned from a snapshot command file (these are written when a snapshot is created so that other users can easily reproduce the environment).""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) command_snapshot_file = os.path.join(helpers.MINIPRESTO_USER_SNAPSHOTS_DIR, "test", "provision-snapshot.sh") process = subprocess.Popen( command_snapshot_file, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, ) stdout, _ = process.communicate() assert process.returncode == 0 assert "Environment provisioning complete" in stdout cleanup() helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_running_containers(): """Verifies that the down command works when multiple containers are running. This also verifies the --sig-kill option works.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.execute_command(["-v", "provision", "--module", "test"]) result = helpers.execute_command(["-v", "down", "--sig-kill"]) assert result.exit_code == 0 assert all(( "Stopped container" in result.output, "Removed container" in result.output, "test" in result.output, "presto" in result.output, )) docker_client = docker.from_env() containers = docker_client.containers.list( filters={"label": RESOURCE_LABEL}) assert len(containers) == 0, "There should be no running containers" helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def main(): helpers.log_status(__file__) # test_no_directory() test_reset_with_directory()
def main(): helpers.log_status(__file__) test_version()
def main(): helpers.log_status(__file__) test_install() test_install_overwrite() test_invalid_ver()