def create_remote_keytab(self, keytab_id: str, principals: list=[]) -> str: """ Create a remote keytab for the specified list of principals """ name = "{}.{}.keytab".format(keytab_id, str(uuid.uuid4())) log.info("Creating keytab: %s", name) if not principals: log.info("Using predefined principals") principals = self.principals if not principals: log.error("No principals specified not creating keytab") return None log.info("Deleting any previous keytab just in case (kadmin will append to it)") sdk_cmd.marathon_task_exec(self.task_id, "rm {}".format(name)) kadmin_options = ["-l"] kadmin_cmd = "ext" kadmin_args = ["-k", name] kadmin_args.extend(principals) self.__run_kadmin(kadmin_options, kadmin_cmd, kadmin_args) keytab_absolute_path = os.path.join("/var/lib/mesos/slave/slaves", self.kdc_host_id, "frameworks", self.framework_id, "executors", self.task_id, "runs/latest", name) return keytab_absolute_path
def _create_keystore_truststore(cn: str, marathon_task: str): pub_path = "{}_pub.crt".format(cn) priv_path = "{}_priv.key".format(cn) keystore_path = "{}_keystore.jks".format(cn) truststore_path = "{}_truststore.jks".format(cn) log.info( "Generating keystore and truststore, task:{}".format(marathon_task)) dcos_ca_bundle = fetch_dcos_ca_bundle(marathon_task) # Convert to a PKCS12 key output = sdk_cmd.marathon_task_exec( marathon_task, 'bash -c "export RANDFILE=/mnt/mesos/sandbox/.rnd && ' 'openssl pkcs12 -export -in {} -inkey {} ' '-out keypair.p12 -name keypair -passout pass:export ' '-CAfile {} -caname root"'.format(pub_path, priv_path, dcos_ca_bundle)) assert output[0] is 0 log.info("Generating certificate: importing into keystore and truststore") # Import into the keystore and truststore output = sdk_cmd.marathon_task_exec( marathon_task, "keytool -importkeystore " "-deststorepass changeit -destkeypass changeit -destkeystore {} " "-srckeystore keypair.p12 -srcstoretype PKCS12 -srcstorepass export " "-alias keypair".format(keystore_path)) assert output[0] is 0 output = sdk_cmd.marathon_task_exec( marathon_task, "keytool -import -trustcacerts -noprompt " "-file {} -storepass changeit " "-keystore {}".format(dcos_ca_bundle, truststore_path)) assert output[0] is 0
def create_tls_artifacts(cn: str, marathon_task: str) -> str: pub_path = "{}_pub.crt".format(cn) priv_path = "{}_priv.key".format(cn) log.info("Generating certificate. cn={}, task={}".format(cn, marathon_task)) output = sdk_cmd.marathon_task_exec( marathon_task, "openssl req -nodes -newkey rsa:2048 -keyout {} -out request.csr " '-subj "/C=US/ST=CA/L=SF/O=Mesosphere/OU=Mesosphere/CN={}"'.format(priv_path, cn), ) assert output[0] == 0 rc, raw_csr, _ = sdk_cmd.marathon_task_exec(marathon_task, "cat request.csr") assert rc == 0 request = {"certificate_request": raw_csr} output = sdk_cmd.marathon_task_exec( marathon_task, "curl --insecure -L -X POST " "-H 'Authorization: token={}' " "leader.mesos/ca/api/v2/sign " "-d '{}'".format(sdk_utils.dcos_token(), json.dumps(request)), ) assert output[0] == 0 # Write the public cert to the client certificate = json.loads(output[1])["result"]["certificate"] output = sdk_cmd.marathon_task_exec( marathon_task, "bash -c \"echo '{}' > {}\"".format(certificate, pub_path) ) assert output[0] == 0 _create_keystore_truststore(cn, marathon_task) return "CN={},OU=Mesosphere,O=Mesosphere,L=SF,ST=CA,C=US".format(cn)
def create_remote_keytab(self, keytab_id: str, principals: list = []) -> str: """ Create a remote keytab for the specified list of principals """ name = "{}.{}.keytab".format(keytab_id, str(uuid.uuid4())) log.info("Creating keytab: %s", name) if not principals: log.info("Using predefined principals") principals = self.principals if not principals: log.error("No principals specified not creating keytab") return None log.info( "Deleting any previous keytab just in case (kadmin will append to it)" ) sdk_cmd.marathon_task_exec(self.task_id, "rm {}".format(name)) kadmin_options = ["-l"] kadmin_cmd = "ext" kadmin_args = ["-k", name] kadmin_args.extend(principals) self.__run_kadmin(kadmin_options, kadmin_cmd, kadmin_args) keytab_absolute_path = os.path.join("/var/lib/mesos/slave/slaves", self.kdc_host_id, "frameworks", self.framework_id, "executors", self.task_id, "runs/latest", name) return keytab_absolute_path
def _create_keystore_truststore(cn: str, marathon_task: str): pub_path = "{}_pub.crt".format(cn) priv_path = "{}_priv.key".format(cn) keystore_path = "{}_keystore.jks".format(cn) truststore_path = "{}_truststore.jks".format(cn) log.info("Generating keystore and truststore, task:{}".format(marathon_task)) dcos_ca_bundle = fetch_dcos_ca_bundle(marathon_task) # Convert to a PKCS12 key output = sdk_cmd.marathon_task_exec( marathon_task, 'bash -c "export RANDFILE=/mnt/mesos/sandbox/.rnd && ' 'openssl pkcs12 -export -in {} -inkey {} ' '-out keypair.p12 -name keypair -passout pass:export ' '-CAfile {} -caname root"'.format(pub_path, priv_path, dcos_ca_bundle)) assert output[0] is 0 log.info("Generating certificate: importing into keystore and truststore") # Import into the keystore and truststore output = sdk_cmd.marathon_task_exec( marathon_task, "keytool -importkeystore " "-deststorepass changeit -destkeypass changeit -destkeystore {} " "-srckeystore keypair.p12 -srcstoretype PKCS12 -srcstorepass export " "-alias keypair".format(keystore_path)) assert output[0] is 0 output = sdk_cmd.marathon_task_exec( marathon_task, "keytool -import -trustcacerts -noprompt " "-file {} -storepass changeit " "-keystore {}".format(dcos_ca_bundle, truststore_path)) assert output[0] is 0
def fetch_dcos_ca_bundle(marathon_task: str) -> str: """Fetch the DC/OS CA bundle from the leading Mesos master""" local_bundle_file = "dcos-ca.crt" cmd = ["curl", "-L", "--insecure", "-v", "leader.mesos/ca/dcos-ca.crt", "-o", local_bundle_file] sdk_cmd.marathon_task_exec(marathon_task, " ".join(cmd)) return local_bundle_file
def test_user_can_auth_and_write_and_read(hdfs_client, kerberos): sdk_auth.kinit(hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("hdfs")) test_filename = "test_auth_write_read-{}".format(str(uuid.uuid4())) write_cmd = "/bin/bash -c \"{}\"".format(config.hdfs_write_command(config.TEST_CONTENT_SMALL, test_filename)) sdk_cmd.marathon_task_exec(hdfs_client["id"], write_cmd) read_cmd = "/bin/bash -c \"{}\"".format(config.hdfs_read_command(test_filename)) _, stdout, _ = sdk_cmd.marathon_task_exec(hdfs_client["id"], read_cmd) assert stdout == config.TEST_CONTENT_SMALL
def test_user_can_auth_and_write_and_read(hdfs_client, kerberos): sdk_auth.kinit(hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("hdfs")) test_filename = "test_auth_write_read" # must be unique among tests in this suite write_cmd = "/bin/bash -c '{}'".format(config.hdfs_write_command(config.TEST_CONTENT_SMALL, test_filename)) sdk_cmd.marathon_task_exec(hdfs_client["id"], write_cmd) read_cmd = "/bin/bash -c '{}'".format(config.hdfs_read_command(test_filename)) _, stdout, _ = sdk_cmd.marathon_task_exec(hdfs_client["id"], read_cmd) assert stdout == config.TEST_CONTENT_SMALL
def test_tls_ciphers(kafka_service): task_name = "kafka-0-broker" task_id = sdk_tasks.get_task_ids(config.SERVICE_NAME, task_name)[0] endpoint = sdk_cmd.svc_cli( config.PACKAGE_NAME, config.SERVICE_NAME, "endpoints {}".format(BROKER_TLS_ENDPOINT), parse_json=True, )[1]["dns"][0] ciphers_config_path = [ "service", "security", "transport_encryption", "ciphers" ] expected_ciphers = set( test_utils.get_in( ciphers_config_path, sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "describe", parse_json=True)[1], "", ).rstrip().split(",")) print("The expected cyphers are\n", expected_ciphers) openssl_ciphers = sdk_security.openssl_ciphers() missing_openssl_ciphers = cipher_suites.missing_openssl_ciphers( openssl_ciphers) possible_openssl_ciphers = openssl_ciphers - missing_openssl_ciphers enabled_ciphers = set() assert openssl_ciphers, "OpenSSL ciphers should be non-empty" assert expected_ciphers, "Expected ciphers should be non-empty" assert possible_openssl_ciphers, "Possible OpenSSL ciphers should be non-empty" sdk_cmd.marathon_task_exec(task_id, "openssl version") # Output OpenSSL version. print("\n{} OpenSSL ciphers missing from the cipher_suites module:".format( len(missing_openssl_ciphers))) print("\n".join(test_utils.sort(list(missing_openssl_ciphers)))) print("\n{} expected ciphers:".format(len(expected_ciphers))) print("\n".join(test_utils.sort(list(expected_ciphers)))) print("\n{} ciphers will be checked:".format( len(possible_openssl_ciphers))) for openssl_cipher in test_utils.sort(list(possible_openssl_ciphers)): print("{} ({})".format(cipher_suites.rfc_name(openssl_cipher), openssl_cipher)) for openssl_cipher in possible_openssl_ciphers: if sdk_security.is_cipher_enabled(config.SERVICE_NAME, task_name, openssl_cipher, endpoint): enabled_ciphers.add(cipher_suites.rfc_name(openssl_cipher)) print("{} ciphers enabled out of {}:".format( len(enabled_ciphers), len(possible_openssl_ciphers))) print("\n".join(test_utils.sort(list(enabled_ciphers)))) assert expected_ciphers == enabled_ciphers, "Enabled ciphers should match expected ciphers"
def fetch_dcos_ca_bundle(marathon_task: str) -> str: """Fetch the DC/OS CA bundle from the leading Mesos master""" local_bundle_file = "dcos-ca.crt" cmd = [ "curl", "-L", "--insecure", "-v", "leader.mesos/ca/dcos-ca.crt", "-o", local_bundle_file ] sdk_cmd.marathon_task_exec(marathon_task, " ".join(cmd)) return local_bundle_file
def test_user_can_auth_and_write_and_read(hdfs_client, kerberos): sdk_auth.kinit(hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("hdfs")) test_filename = "test_auth_write_read" # must be unique among tests in this suite write_cmd = "/bin/bash -c '{}'".format( config.hdfs_write_command(config.TEST_CONTENT_SMALL, test_filename)) sdk_cmd.marathon_task_exec(hdfs_client["id"], write_cmd) read_cmd = "/bin/bash -c '{}'".format( config.hdfs_read_command(test_filename)) _, stdout, _ = sdk_cmd.marathon_task_exec(hdfs_client["id"], read_cmd) assert stdout == config.TEST_CONTENT_SMALL
def test_user_can_auth_and_write_and_read(hdfs_client, kerberos): sdk_auth.kinit(hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("hdfs")) test_filename = "test_auth_write_read-{}".format(str(uuid.uuid4())) write_cmd = "/bin/bash -c \"{}\"".format( config.hdfs_write_command(config.TEST_CONTENT_SMALL, test_filename)) sdk_cmd.marathon_task_exec(hdfs_client["id"], write_cmd) read_cmd = "/bin/bash -c \"{}\"".format( config.hdfs_read_command(test_filename)) _, stdout, _ = sdk_cmd.marathon_task_exec(hdfs_client["id"], read_cmd) assert stdout == config.TEST_CONTENT_SMALL
def _modify_role_acls( action: str, roles: list, user: str, marathon_task: str, topic: str, zookeeper_endpoint: str, env_str: str = None, ) -> tuple: if not action.startswith("--"): action = "--{}".format(action) cmd_list = [ "kafka-acls", "--topic", topic, "--authorizer-properties", "zookeeper.connect={}".format(zookeeper_endpoint), action, "--force", "--allow-principal", "User:{}".format(user), ] cmd_list.extend(roles) cmd = auth.get_bash_command(" ".join(cmd_list), env_str) LOG.info("Running: %s", cmd) output = sdk_cmd.marathon_task_exec(marathon_task, cmd) LOG.info(output) return output
def test_verify_https_ports(hdfs_client, node_type, port): """ Verify that HTTPS port is open name, journal and data node types. """ task_id = "{}-0-node".format(node_type) host = sdk_hosts.autoip_host(config.SERVICE_NAME, task_id, port) cmd = [ "curl", "-v", "--cacert", hdfs_client["dcos_ca_bundle"], "https://{host}".format(host=host), ] rc, stdout, stderr = sdk_cmd.marathon_task_exec(hdfs_client["id"], " ".join(cmd)) assert not rc assert "SSL connection using TLS1.2 / ECDHE_RSA_AES_128_GCM_SHA256" in stderr assert "server certificate verification OK" in stderr assert "common name: {}.{} (matched)".format(task_id, config.SERVICE_NAME) in stderr # In the Kerberos case we expect a 401 error assert "401 Authentication required" in stdout
def test_config_with_custom_yml(configure_package) -> None: sdk_install.install( config.PACKAGE_NAME, service_name=config.SERVICE_NAME, expected_running_tasks=config.DEFAULT_TASK_COUNT, ) decoded_base_64_yml = "logging.json: true" base_64_yml = base64.b64encode(decoded_base_64_yml.encode("utf-8")).decode("utf-8") elasticsearch_url = "http://" + sdk_hosts.vip_host(config.SERVICE_NAME, "coordinator", 9200) sdk_install.install( config.KIBANA_PACKAGE_NAME, config.KIBANA_SERVICE_NAME, 0, {"kibana": {"elasticsearch_url": elasticsearch_url, "custom_kibana_yml": base_64_yml}}, wait_for_deployment=False, insert_strict_options=False, ) cmd = "bash -c 'grep \"{}\" kibana-*/config/kibana.yml'".format(decoded_base_64_yml) rc, stdout, stderr = sdk_cmd.marathon_task_exec(config.KIBANA_SERVICE_NAME, cmd) assert rc == 0 and decoded_base_64_yml in stdout config.check_kibana_adminrouter_integration("service/{}/".format(config.KIBANA_SERVICE_NAME))
def kdestroy(marathon_task_id: str): """ Performs a kdestroy command to erase an auth session for a principal. :param task_id: The marathon task in whose environment the kdestroy will run. """ log.info("Erasing auth session:") rc, stdout, stderr = sdk_cmd.marathon_task_exec(marathon_task_id, "kdestroy") if rc != 0: raise RuntimeError("Failed ({}) to erase auth session\nstdout: {}\nstderr: {}".format(rc, stdout, stderr))
def task_exec(self, task_id, cmd): full_cmd = " ".join([ "export JAVA_HOME=$(ls -d ${MESOS_SANDBOX}/jdk*/jre/) &&", "export TASK_IP=$(${MESOS_SANDBOX}/bootstrap --get-task-ip) &&", "CASSANDRA_DIRECTORY=$(ls -d ${MESOS_SANDBOX}/apache-cassandra-*/) &&", cmd, ]) return sdk_cmd.marathon_task_exec(task_id, "bash -c '{}'".format(full_cmd))
def task_exec(self, task_id, cmd): full_cmd = " ".join( [ "export JAVA_HOME=$(ls -d ${MESOS_SANDBOX}/jdk*/jre/) &&", "export TASK_IP=$(${MESOS_SANDBOX}/bootstrap --get-task-ip) &&", "CASSANDRA_DIRECTORY=$(ls -d ${MESOS_SANDBOX}/apache-cassandra-*/) &&", cmd, ] ) return sdk_cmd.marathon_task_exec(task_id, "bash -c '{}'".format(full_cmd))
def task_exec(self, task_id, cmd): full_cmd = " ".join( [ "export JAVA_HOME=$(ls -d ${MESOS_SANDBOX}/jdk*/jre/) &&", "export TASK_IP=$(${MESOS_SANDBOX}/bootstrap --get-task-ip) &&", "ELASTICSEARCH_DIRECTORY=$(ls -d ${MESOS_SANDBOX}/elasticsearch-*/) &&", cmd, ] ) return sdk_cmd.marathon_task_exec(task_id, "bash -c '{}'".format(full_cmd))
def check_kibana_plugin_installed(plugin_name, service_name=SERVICE_NAME): task_sandbox = sdk_cmd.get_task_sandbox_path(service_name) # Environment variables aren't available on DC/OS 1.9 so we manually inject # MESOS_SANDBOX (and can't use ELASTIC_VERSION). # # TODO(mpereira): improve this by making task environment variables # available in task_exec commands on 1.9. # Ticket: https://jira.mesosphere.com/browse/INFINITY-3360 cmd = "bash -c 'KIBANA_DIRECTORY=$(ls -d {}/kibana-*-linux-x86_64); $KIBANA_DIRECTORY/bin/kibana-plugin list'".format(task_sandbox) _, stdout, _ = sdk_cmd.marathon_task_exec(service_name, cmd) return plugin_name in stdout
def check_kibana_plugin_installed(plugin_name, service_name=SERVICE_NAME): task_sandbox = sdk_cmd.get_task_sandbox_path(service_name) # Environment variables aren't available on DC/OS 1.9 so we manually inject # MESOS_SANDBOX (and can't use ELASTIC_VERSION). # # TODO(mpereira): improve this by making task environment variables # available in task_exec commands on 1.9. # Ticket: https://jira.mesosphere.com/browse/INFINITY-3360 cmd = "bash -c 'KIBANA_DIRECTORY=$(ls -d {}/kibana-*-linux-x86_64); $KIBANA_DIRECTORY/bin/kibana-plugin list'".format( task_sandbox) _, stdout, _ = sdk_cmd.marathon_task_exec(service_name, cmd) return plugin_name in stdout
def create_offers_file(self): scheduler_vip = sdk_hosts.scheduler_vip_host(self.service_name, "api") scheduler = self.scheduler_tasks[0] rc, stdout, stderr = sdk_cmd.marathon_task_exec( scheduler["id"], "curl -s {}/v1/debug/offers".format(scheduler_vip), print_output=False ) if rc != 0 or stderr: log.error( "Could not get scheduler offers\nstdout: '%s'\nstderr: '%s'", stdout[:100], stderr ) else: self.write_file("service_v1_debug_offers.html", stdout)
def kinit(marathon_task_id: str, keytab: str, principal: str): """ Performs a kinit command to authenticate the specified principal. :param marathon_task_id: The marathon task in whose environment the kinit will run. :param keytab: The keytab used by kinit to authenticate. :param principal: The name of the principal the user wants to authenticate as. """ kinit_cmd = "kinit -kt {keytab} {principal}".format(keytab=keytab, principal=principal) log.info("Authenticating principal=%s with keytab=%s: %s", principal, keytab, kinit_cmd) rc, stdout, stderr = sdk_cmd.marathon_task_exec(marathon_task_id, kinit_cmd) if rc != 0: raise RuntimeError("Failed ({}) to authenticate with keytab={} principal={}\n" \ "stdout: {}\n" \ "stderr: {}".format(rc, keytab, principal, stdout, stderr))
def create_tls_artifacts(cn: str, marathon_task: str) -> str: pub_path = "{}_pub.crt".format(cn) priv_path = "{}_priv.key".format(cn) log.info("Generating certificate. cn={}, task={}".format( cn, marathon_task)) output = sdk_cmd.marathon_task_exec( marathon_task, "openssl req -nodes -newkey rsa:2048 -keyout {} -out request.csr " '-subj "/C=US/ST=CA/L=SF/O=Mesosphere/OU=Mesosphere/CN={}"'.format( priv_path, cn), ) assert output[0] == 0 rc, raw_csr, _ = sdk_cmd.marathon_task_exec(marathon_task, "cat request.csr") assert rc == 0 request = {"certificate_request": raw_csr} output = sdk_cmd.marathon_task_exec( marathon_task, "curl --insecure -L -X POST " "-H 'Authorization: token={}' " "leader.mesos/ca/api/v2/sign " "-d '{}'".format(sdk_utils.dcos_token(), json.dumps(request)), ) assert output[0] == 0 # Write the public cert to the client certificate = json.loads(output[1])["result"]["certificate"] output = sdk_cmd.marathon_task_exec( marathon_task, "bash -c \"echo '{}' > {}\"".format(certificate, pub_path)) assert output[0] == 0 _create_keystore_truststore(cn, marathon_task) return "CN={},OU=Mesosphere,O=Mesosphere,L=SF,ST=CA,C=US".format(cn)
def wait_for_brokers(client: str, brokers: list): """ Run bootstrap on the specified client to resolve the list of brokers """ LOG.info("Running bootstrap to wait for DNS resolution") bootstrap_cmd = [ "/opt/bootstrap", "-print-env=false", "-template=false", "-install-certs=false", "--self-resolve=false", "-resolve-hosts", ",".join(brokers), ] bootstrap_output = sdk_cmd.marathon_task_exec(client, " ".join(bootstrap_cmd)) LOG.info(bootstrap_output) assert "SDK Bootstrap successful" in " ".join(str(bo) for bo in bootstrap_output) return True
def configuration_ids(self) -> List[str]: scheduler_vip = sdk_hosts.scheduler_vip_host(self.service_name, "api") scheduler = self.scheduler_tasks[0] rc, stdout, stderr = sdk_cmd.marathon_task_exec( scheduler["id"], "curl -s {}/v1/configurations".format(scheduler_vip), print_output=False, ) if rc != 0 or stderr: raise Exception( "Could not get scheduler configuration IDs\nstdout: '%s'\nstderr: '%s'", stdout, stderr, ) else: return json.loads(stdout)
def __run_kadmin(self, options: list, cmd: str, args: list): """ Invokes Kerberos' kadmin binary inside the container to run some command. :param options (list): A list of options given to kadmin. :param cmd (str): The name of the sub command to run. :param args (list): A list of arguments passed to the sub command. This should also include any flags needed to be set for the sub command. :raises a generic Exception if the invocation fails. """ kadmin_cmd = "/usr/sbin/kadmin {options} {cmd} {args}".format( options=' '.join(options), cmd=cmd, args=' '.join(args) ) log.info("Running kadmin: {}".format(kadmin_cmd)) rc, stdout, stderr = sdk_cmd.marathon_task_exec(self.task_id, kadmin_cmd) if rc != 0: raise RuntimeError("Failed ({}) to invoke kadmin: {}\nstdout: {}\nstderr: {}".format(rc, kadmin_cmd, stdout, stderr))
def __run_kadmin(self, options: list, cmd: str, args: list): """ Invokes Kerberos' kadmin binary inside the container to run some command. :param options (list): A list of options given to kadmin. :param cmd (str): The name of the sub command to run. :param args (list): A list of arguments passed to the sub command. This should also include any flags needed to be set for the sub command. :raises a generic Exception if the invocation fails. """ kadmin_cmd = "/usr/sbin/kadmin {options} {cmd} {args}".format( options=' '.join(options), cmd=cmd, args=' '.join(args)) log.info("Running kadmin: {}".format(kadmin_cmd)) rc, stdout, stderr = sdk_cmd.marathon_task_exec( self.task_id, kadmin_cmd) if rc != 0: raise RuntimeError( "Failed ({}) to invoke kadmin: {}\nstdout: {}\nstderr: {}". format(rc, kadmin_cmd, stdout, stderr))
def test_verify_https_ports(hdfs_client, node_type, port): """ Verify that HTTPS port is open name, journal and data node types. """ task_id = "{}-0-node".format(node_type) host = sdk_hosts.autoip_host( config.SERVICE_NAME, task_id, port) cmd = ["curl", "-v", "--cacert", hdfs_client["dcos_ca_bundle"], "https://{host}".format(host=host), ] rc, stdout, stderr = sdk_cmd.marathon_task_exec(hdfs_client["id"], " ".join(cmd)) assert not rc assert "SSL connection using TLS1.2 / ECDHE_RSA_AES_128_GCM_SHA256" in stderr assert "server certificate verification OK" in stderr assert "common name: {}.{} (matched)".format(task_id, config.SERVICE_NAME) in stderr # In the Kerberos case we expect a 401 error assert "401 Authentication required" in stdout
def read_wrapper(): return sdk_cmd.marathon_task_exec(marathon_task, read_cmd)
def write_wrapper(): return sdk_cmd.marathon_task_exec(marathon_task, write_cmd)
def _run_client_command(): rc, stdout, stderr = sdk_cmd.marathon_task_exec(CLIENT_APP_NAME, "/bin/bash -c '{}'".format(hdfs_command)) return (success_check(rc, stdout, stderr), stdout, stderr)
def read_wrapper(): LOG.info("Running: %s", read_cmd) rc, stdout, stderr = sdk_cmd.marathon_task_exec(task, read_cmd) LOG.info("rc=%s\nstdout=%s\nstderr=%s\n", rc, stdout, stderr) return rc, stdout, stderr
def test_users_have_appropriate_permissions(hdfs_client, kerberos): # "hdfs" is a superuser sdk_auth.kinit(hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("hdfs")) log.info("Creating directory for alice") make_user_directory_cmd = config.hdfs_command("mkdir -p /users/alice") sdk_cmd.marathon_task_exec(hdfs_client["id"], make_user_directory_cmd) change_ownership_cmd = config.hdfs_command("chown alice:users /users/alice") sdk_cmd.marathon_task_exec(hdfs_client["id"], change_ownership_cmd) change_permissions_cmd = config.hdfs_command("chmod 700 /users/alice") sdk_cmd.marathon_task_exec(hdfs_client["id"], change_permissions_cmd) test_filename = "test_user_permissions-{}".format(str(uuid.uuid4())) # alice has read/write access to her directory sdk_auth.kdestroy(hdfs_client["id"]) sdk_auth.kinit(hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("alice")) write_access_cmd = "/bin/bash -c \"{}\"".format(config.hdfs_write_command( config.TEST_CONTENT_SMALL, "/users/alice/{}".format(test_filename))) log.info("Alice can write: %s", write_access_cmd) rc, stdout, _ = sdk_cmd.marathon_task_exec(hdfs_client["id"], write_access_cmd) assert stdout == '' and rc == 0 read_access_cmd = config.hdfs_read_command("/users/alice/{}".format(test_filename)) log.info("Alice can read: %s", read_access_cmd) _, stdout, _ = sdk_cmd.marathon_task_exec(hdfs_client["id"], read_access_cmd) assert stdout == config.TEST_CONTENT_SMALL ls_cmd = config.hdfs_command("ls /users/alice") _, stdout, _ = sdk_cmd.marathon_task_exec(hdfs_client["id"], ls_cmd) assert "/users/alice/{}".format(test_filename) in stdout # bob doesn't have read/write access to alice's directory sdk_auth.kdestroy(hdfs_client["id"]) sdk_auth.kinit(hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("bob")) log.info("Bob tries to wrtie to alice's directory: %s", write_access_cmd) _, _, stderr = sdk_cmd.marathon_task_exec(hdfs_client["id"], write_access_cmd) log.info("Bob can't write to alice's directory: %s", write_access_cmd) assert "put: Permission denied: user=bob" in stderr log.info("Bob tries to read from alice's directory: %s", read_access_cmd) _, _, stderr = sdk_cmd.marathon_task_exec(hdfs_client["id"], read_access_cmd) log.info("Bob can't read from alice's directory: %s", read_access_cmd) assert "cat: Permission denied: user=bob" in stderr
def test_users_have_appropriate_permissions(hdfs_client, kerberos): # "hdfs" is a superuser sdk_auth.kinit(hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("hdfs")) log.info("Creating directory for alice") make_user_directory_cmd = config.hdfs_command("mkdir -p /users/alice") sdk_cmd.marathon_task_exec(hdfs_client["id"], make_user_directory_cmd) change_ownership_cmd = config.hdfs_command( "chown alice:users /users/alice") sdk_cmd.marathon_task_exec(hdfs_client["id"], change_ownership_cmd) change_permissions_cmd = config.hdfs_command("chmod 700 /users/alice") sdk_cmd.marathon_task_exec(hdfs_client["id"], change_permissions_cmd) test_filename = "test_user_permissions-{}".format(str(uuid.uuid4())) # alice has read/write access to her directory sdk_auth.kdestroy(hdfs_client["id"]) sdk_auth.kinit(hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("alice")) write_access_cmd = "/bin/bash -c \"{}\"".format( config.hdfs_write_command(config.TEST_CONTENT_SMALL, "/users/alice/{}".format(test_filename))) log.info("Alice can write: %s", write_access_cmd) rc, stdout, _ = sdk_cmd.marathon_task_exec(hdfs_client["id"], write_access_cmd) assert stdout == '' and rc == 0 read_access_cmd = config.hdfs_read_command( "/users/alice/{}".format(test_filename)) log.info("Alice can read: %s", read_access_cmd) _, stdout, _ = sdk_cmd.marathon_task_exec(hdfs_client["id"], read_access_cmd) assert stdout == config.TEST_CONTENT_SMALL ls_cmd = config.hdfs_command("ls /users/alice") _, stdout, _ = sdk_cmd.marathon_task_exec(hdfs_client["id"], ls_cmd) assert "/users/alice/{}".format(test_filename) in stdout # bob doesn't have read/write access to alice's directory sdk_auth.kdestroy(hdfs_client["id"]) sdk_auth.kinit(hdfs_client["id"], keytab=config.KEYTAB, principal=kerberos.get_principal("bob")) log.info("Bob tries to wrtie to alice's directory: %s", write_access_cmd) _, _, stderr = sdk_cmd.marathon_task_exec(hdfs_client["id"], write_access_cmd) log.info("Bob can't write to alice's directory: %s", write_access_cmd) assert "put: Permission denied: user=bob" in stderr log.info("Bob tries to read from alice's directory: %s", read_access_cmd) _, _, stderr = sdk_cmd.marathon_task_exec(hdfs_client["id"], read_access_cmd) log.info("Bob can't read from alice's directory: %s", read_access_cmd) assert "cat: Permission denied: user=bob" in stderr
def hdfs_cmd(cmd): LOGGER.info('Running command %s', cmd) rc, _, _ = sdk_cmd.marathon_task_exec(HDFS_CLIENT_ID, "bin/hdfs dfs -{}".format(cmd)) if rc != 0: raise Exception("HDFS command failed with code {}: {}".format(rc, cmd))