def test_remove_dependent_resources_running(): """Verifies that a dependent resources (tied to active containers) cannot be removed.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.execute_command(["provision", "--module", "test"]) result = helpers.execute_command(["-v", "remove", "--images", "--volumes"], command_input="y\n") assert result.exit_code == 0 assert all(( "Cannot remove volume:" in result.output, "Cannot remove image:" in result.output, )) assert_docker_resource_count( { "resource_type": docker_client.images, "label": "com.starburst.tests.module.test=catalog-test", "expected_count": 1, }, { "resource_type": docker_client.volumes, "label": "com.starburst.tests.module.test=catalog-test", "expected_count": 1, }, ) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_all(): """Verifies that all Minipresto resources are removed.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.execute_command(["provision", "--module", "test"]) helpers.execute_command(["down", "--sig-kill"]) result = helpers.execute_command(["-v", "remove", "--images", "--volumes"], command_input="y\n") assert result.exit_code == 0 assert all(("Volume removed:" in result.output, "Image removed:" in result.output)) assert_docker_resource_count( { "resource_type": docker_client.images, "label": RESOURCE_LABEL, "expected_count": 0, }, { "resource_type": docker_client.volumes, "label": RESOURCE_LABEL, "expected_count": 0, }, ) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_label(): """Verifies that only images with the given label are removed.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.execute_command(["provision", "--module", "test"]) helpers.execute_command(["down", "--sig-kill"]) result = helpers.execute_command([ "-v", "remove", "--images", "--label", "com.starburst.tests.module.test=catalog-test", ], ) assert result.exit_code == 0 assert "Image removed:" in result.output assert_docker_resource_count( { "resource_type": docker_client.images, "label": "com.starburst.tests.module.test=catalog-test", "expected_count": 0, }, { "resource_type": docker_client.images, "label": "com.starburst.tests.module=presto", "expected_count": 1, }, ) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_multiple_env(): """Verifies that multiple environment variables can be successfully passed in.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command([ "-v", "--env", "COMPOSE_PROJECT_NAME=test", "--env", "STARBURST_VER=338-e.1", "--env", "PRESTO=is=awesome", "version", ]) assert result.exit_code == 0 assert all(( '"COMPOSE_PROJECT_NAME": "test"' in result.output, '"STARBURST_VER": "338-e.1"' in result.output, '"PRESTO": "is=awesome"' in result.output, )) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_bootstrap_script(result): """Ensures that bootstrap scripts properly execute in containers.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) assert all(( "Successfully executed bootstrap script in container: 'presto'", "Successfully executed bootstrap script in container: 'test'", )) presto_bootstrap_check = subprocess.Popen( f"docker exec -i presto ls /usr/lib/presto/etc/", shell=True, stdout=subprocess.PIPE, universal_newlines=True, ) test_bootstrap_check = subprocess.Popen( f"docker exec -i test cat /root/test_bootstrap.txt", shell=True, stdout=subprocess.PIPE, universal_newlines=True, ) presto_bootstrap_check, _ = presto_bootstrap_check.communicate() test_bootstrap_check, _ = test_bootstrap_check.communicate() assert "test_bootstrap.txt" in presto_bootstrap_check assert "hello world" in test_bootstrap_check helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_specific_directory(): """Tests that the snapshot file can be saved in a user-specified directory.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) cleanup() result = helpers.execute_command( [ "-v", "snapshot", "--name", "test", "--module", "test", "--directory", "/tmp/", ], command_input="y\n", ) run_assertions(result, True, check_path=os.path.join(os.sep, "tmp")) assert "Creating snapshot of specified modules" in result.output subprocess.call("rm -rf /tmp/test.tar.gz", shell=True) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_version(): """Tests for correct version output.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command(["version"]) assert pkg_resources.require("Minipresto")[0].version in result.output helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_reset_with_directory(): """Verifies that the configuration directory is only removed and restored with the user's approval. This is a valid test case for both 'yes' and 'no' responses.""" import time helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) subprocess.call(f"mkdir {helpers.MINIPRESTO_USER_DIR}", shell=True, stdout=subprocess.DEVNULL) start_time = time.time() end_time = 2.0 output = "" while time.time() - start_time <= end_time: process = subprocess.Popen( "minipresto -v config --reset", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, ) while True: output_line = process.stdout.readline() if output_line == "": break output, _ = process.communicate() # Get full output (stdout + stderr) if time.time() >= end_time: process.terminate() break process = subprocess.Popen( "minipresto -v config --reset", stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, universal_newlines=True, ) output = process.communicate(input="y\n", timeout=1)[0] process.terminate() assert process.returncode == 0 assert all(( "Configuration directory exists" in output, "Created Minipresto configuration directory" in output, "Opening existing config file at path" in output, )) assert os.path.isdir(helpers.MINIPRESTO_USER_DIR) assert os.path.isfile(helpers.CONFIG_FILE) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_edit_valid_config(): """Verifies that the user can edit an existing configuration file.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) subprocess.call(f"rm -rf {helpers.MINIPRESTO_USER_DIR}", shell=True) subprocess.call(f"mkdir {helpers.MINIPRESTO_USER_DIR}", shell=True) helpers.make_sample_config() return_code = subprocess.call(f"minipresto config", shell=True) assert return_code == 0 helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_valid_module(): """Ensures the `module` command works when providing a valid module name.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command(["-v", "modules", "--module", "test"]) assert result.exit_code == 0 assert all(("Module: test" in result.output, "Test module" in result.output)) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_env(): """Verifies that an environment variable can be successfully passed in.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command( ["-v", "--env", "COMPOSE_PROJECT_NAME=test", "version"]) assert result.exit_code == 0 assert "COMPOSE_PROJECT_NAME" and "test" in result.output helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_install(): """Verifies that the Minipresto library can be installed.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) cleanup() # Install 0.0.0 since it's always around as a test release result = helpers.execute_command(["-v", "lib_install", "--version", "0.0.0"]) assert result.exit_code == 0 assert os.path.isdir(os.path.join(helpers.MINIPRESTO_USER_DIR, "lib")) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_invalid_module(): """Ensures Minipresto exists with a user error if an invalid module name is provided.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command( ["-v", "modules", "--module", "not-a-real-module"]) assert result.exit_code == 2 assert "Invalid module" in result.output helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_invalid_ver(): """Verifies that an error is raised if an incorrect version is passed to the command.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command(["-v", "lib_install", "--version", "YEE-PRESTO"]) assert result.exit_code == 1 assert not os.path.isdir(os.path.join(helpers.MINIPRESTO_USER_DIR, "lib")) cleanup() helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_json(): """Ensures the `module` command can output module metadata in JSON format.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command( ["-v", "modules", "--module", "test", "--json"]) assert result.exit_code == 0 assert all(('"type": "catalog"' in result.output, '"test":' in result.output)) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_no_directory(): """Verifies that a configuration directory and config file are created with config --reset.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) subprocess.call(f"rm -rf {helpers.MINIPRESTO_USER_DIR}", shell=True) return_code = subprocess.call("minipresto config", shell=True) assert return_code == 0 assert os.path.isdir(helpers.MINIPRESTO_USER_DIR) assert os.path.isfile(helpers.CONFIG_FILE) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_edit_invalid_config(): """Verifies that an error is not thrown if the config is 'invalid' (such as a missing section or value required to perform an action). This is because there is default behavior built in, and all major functions should still work without a valid configuration file.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) subprocess.call(f"rm -rf {helpers.MINIPRESTO_USER_DIR}", shell=True) subprocess.call(f"mkdir {helpers.MINIPRESTO_USER_DIR}", shell=True) subprocess.call(f"touch {helpers.CONFIG_FILE}", shell=True) return_code = subprocess.call(f"minipresto config", shell=True) assert return_code == 0 helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_snapshot_standalone(): """Verifies that a the standlone Presto module can be snapshotted.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) cleanup() result = helpers.execute_command( ["-v", "snapshot", "--name", "test"], command_input="y\n", ) run_assertions(result, False) assert "Snapshotting Presto resources only" in result.output helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_bootstrap_re_execute(): """Ensures that bootstrap scripts do not execute if they have already executed.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command(["-v", "provision", "--module", "test"]) assert result.exit_code == 0 assert all(( "Bootstrap already executed in container 'presto'. Skipping.", "Bootstrap already executed in container 'test'. Skipping.", )) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_invalid_name(): """Tests that all valid characters can be present and succeed for a given snapshot name.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) cleanup() result = helpers.execute_command( ["-v", "snapshot", "--name", "##.my-test?", "--module", "test"], command_input="y\n", ) assert result.exit_code == 2 assert "Illegal character found in provided filename" in result.output helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_provision_append(): """Verifies that modules can be appended to already-running environments.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.execute_command(["-v", "provision", "--module", "test"]) result = helpers.execute_command( ["-v", "provision", "--module", "postgres"]) containers = get_containers() assert result.exit_code == 0 assert "Identified the following running modules" in result.output assert len(containers) == 3 # presto, test, and postgres helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_all_modules(): """Ensures that all module metadata is printed to the console if a module name is not passed to the command.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command(["-v", "modules"]) assert result.exit_code == 0 assert all(( "Module: test" in result.output, "Description:" in result.output, "Incompatible Modules:" in result.output, )) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_snapshot_inactive_env(): """Verifies that a snapshot can be successfully created from an inactive environment.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) cleanup() result = helpers.execute_command( ["-v", "snapshot", "--name", "test", "--module", "test"], command_input="y\n", ) run_assertions(result) assert "Creating snapshot of specified modules" in result.output helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_force(): """Verifies that the user can override the check to see if the resulting tarball exists.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command( ["-v", "snapshot", "--name", "test", "--module", "test", "--force"], command_input="y\n", ) run_assertions(result) assert "Creating snapshot of specified modules" in result.output cleanup() helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_duplicate_config_props(): """Ensures that duplicate configuration properties in Presto are logged as a warning to the user.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.execute_command(["-v", "provision"]) cmd_chunk = ( f"$'query.max-stage-count=85\nquery.max-stage-count=100" f"\nquery.max-execution-time=1h\nquery.max-execution-time=2h'") subprocess.Popen( f'docker exec -i presto sh -c "echo {cmd_chunk} >> /usr/lib/presto/etc/config.properties"', shell=True, stdout=subprocess.PIPE, universal_newlines=True, ) cmd_chunk = "$'-Xms1G\n-Xms1G'" subprocess.Popen( f'docker exec -i presto sh -c "echo {cmd_chunk} >> /usr/lib/presto/etc/jvm.config"', shell=True, stdout=subprocess.PIPE, universal_newlines=True, ) # Hard stop to allow commands to process time.sleep(2) helpers.execute_command(["-v", "down", "--sig-kill", "--keep"]) result = helpers.execute_command(["-v", "provision"]) assert all(( "Duplicate Presto configuration properties detected in config.properties" in result.output, "query.max-stage-count=85" in result.output, "query.max-stage-count=100" in result.output, "query.max-execution-time=1h" in result.output, "query.max-execution-time=2h" in result.output, "Duplicate Presto configuration properties detected in jvm.config" in result.output, "-Xms1G" in result.output, "-Xms1G" in result.output, )) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_install_overwrite(): """Verifies that the Minipresto library can be installed over an existing library.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command( ["-v", "lib_install", "--version", "0.0.0"], command_input="y\n" ) assert result.exit_code == 0 assert os.path.isdir(os.path.join(helpers.MINIPRESTO_USER_DIR, "lib")) assert "Removing existing library directory" in result.output cleanup() helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_incompatible_modules(): """Verifies that chosen modules are not mutually-exclusive.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command( ["-v", "provision", "--module", "ldap", "--module", "test"]) assert result.exit_code == 2 assert all(( "Incompatible modules detected" in result.output, "incompatible with module" in result.output, "ldap" in result.output, )) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup()
def test_running(): """Ensures the `module` command can output metadata for running modules.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.execute_command(["-v", "provision", "--module", "test"]) result = helpers.execute_command(["-v", "modules", "--json", "--running"]) assert result.exit_code == 0 assert all(( '"type": "catalog"' in result.output, '"type": "catalog"' in result.output, '"containers":' in result.output, )) helpers.execute_command(["-v", "down", "--sig-kill"]) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_scrub(): """Verifies that sensitive data in user config file is scrubbed when scrubbing is enabled.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) helpers.make_sample_config() result = helpers.execute_command( ["-v", "snapshot", "--name", "test", "--module", "test"], command_input="y\n") run_assertions(result) with open(snapshot_config_file()) as f: assert "*" * 20 in f.read() cleanup() helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)
def test_snapshot_no_directory(): """Verifies that a snapshot can be created when there is no existing snapshots directory in the Minipresto user home directory.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) cleanup() subprocess.call(f"rm -rf {helpers.MINIPRESTO_USER_SNAPSHOTS_DIR}", shell=True) result = helpers.execute_command( ["-v", "snapshot", "--name", "test", "--module", "test"], command_input="y\n", ) run_assertions(result) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name)