def test_s3_env():
creds = s3.get_credentials()
args = [
"--conf spark.mesos.driverEnv.AWS_ACCESS_KEY_ID={}".format(
creds.access_key),
"--conf spark.mesos.driverEnv.AWS_SECRET_ACCESS_KEY={}".format(
creds.secret_key)
]
args.append("--class S3Job")
linecount_path = os.path.join(THIS_DIR, 'resources', 'linecount.txt')
s3.upload_file(linecount_path)
# download/read linecount.txt only
utils.run_tests(app_url=utils.dcos_test_jar_url(),
app_args="--readUrl {} --countOnly".format(
s3.s3n_url('linecount.txt')),
expected_output="Read 3 lines",
args=args)
# download/read linecount.txt, reupload as linecount-env.txt
utils.run_tests(app_url=utils.dcos_test_jar_url(),
app_args="--readUrl {} --writeUrl {}".format(
s3.s3n_url('linecount.txt'),
s3.s3n_url('linecount-env.txt')),
expected_output="Read 3 lines",
args=args)
assert len(list(s3.list("linecount-env.txt"))) > 0
def test_s3_secrets():
linecount_path = os.path.join(THIS_DIR, 'resources', 'linecount.txt')
s3.upload_file(linecount_path)
creds = s3.get_credentials()
def make_credential_secret(path, val):
sdk_security.delete_secret(path)
rc, stdout, stderr = sdk_cmd.run_raw_cli(
"security secrets create /{} -v {}".format(path, val))
assert rc == 0, "Failed to create secret {}, stderr: {}, stdout: {}".format(
path, stderr, stdout)
aws_access_key_path = "aws_access_key_id"
make_credential_secret(aws_access_key_path, creds.access_key)
aws_secret_key_path = "aws_secret_access_key"
make_credential_secret(aws_secret_key_path, creds.secret_key)
args = [
"--conf spark.mesos.containerizer=mesos",
"--conf spark.mesos.driver.secret.names=/{key},/{secret}".format(
key=aws_access_key_path, secret=aws_secret_key_path),
"--conf spark.mesos.driver.secret.envkeys=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY",
"--class S3Job"
]
try:
# download/read linecount.txt only
utils.run_tests(app_url=utils.dcos_test_jar_url(),
app_args="--readUrl {} --countOnly".format(
s3.s3n_url('linecount.txt')),
expected_output="Read 3 lines",
args=args)
# download/read linecount.txt, reupload as linecount-secret.txt:
utils.run_tests(app_url=utils.dcos_test_jar_url(),
app_args="--readUrl {} --writeUrl {}".format(
s3.s3n_url('linecount.txt'),
s3.s3n_url('linecount-secret.txt')),
expected_output="Read 3 lines",
args=args)
assert len(list(s3.list("linecount-secret.txt"))) > 0
finally:
sdk_security.delete_secret(aws_access_key_path)
sdk_security.delete_secret(aws_secret_key_path)
# "labels":{
# "labels":[
# {"key":"key1","value":"val1"},
# {"key":"key2","value":"val2"}]}}]
network_info = task['container']['network_infos'][0]
assert network_info['name'] == "dcos"
labels = network_info['labels']['labels']
assert len(labels) == 2
assert labels[0]['key'] == "key1"
assert labels[0]['value'] == "val1"
assert labels[1]['key'] == "key2"
assert labels[1]['value'] == "val2"
# Your session credentials are tied to your IP. They work locally, but will not work from the spark job.
@pytest.mark.skipif(s3.get_credentials().token is not None,
reason="Session credentials won't work")
@sdk_utils.dcos_ee_only
@pytest.mark.sanity
@pytest.mark.smoke
def test_s3_secrets():
linecount_path = os.path.join(THIS_DIR, 'resources', 'linecount.txt')
s3.upload_file(linecount_path)
creds = s3.get_credentials()
def make_credential_secret(path, val):
sdk_security.delete_secret(path)
rc, stdout, stderr = sdk_cmd.run_raw_cli(
"security secrets create /{} -v {}".format(path, val))
assert rc == 0, "Failed to create secret {}, stderr: {}, stdout: {}".format(
expected_output="Pi is roughly 3",
args=["--py-files {}".format(py_file_url)])
@pytest.mark.sanity
@pytest.mark.smoke
def test_r():
r_script_path = os.path.join(THIS_DIR, 'jobs', 'R', 'dataframe.R')
r_script_url = utils.upload_file(r_script_path)
utils.run_tests(app_url=r_script_url,
app_args='',
expected_output="Justin")
# Your session credentials are tied to your IP. They work locally, but will not work from the spark job.
@pytest.mark.skipif(s3.get_credentials().token is not None, reason="Session credentials won't work")
@sdk_utils.dcos_ee_only
@pytest.mark.sanity
@pytest.mark.smoke
def test_s3_secrets():
linecount_path = os.path.join(THIS_DIR, 'resources', 'linecount.txt')
s3.upload_file(linecount_path)
creds = s3.get_credentials()
def make_credential_secret(path, val):
sdk_security.delete_secret(path)
rc, stdout, stderr = sdk_cmd.run_raw_cli("security secrets create /{} -v {}".format(path, val))
assert rc == 0, "Failed to create secret {}, stderr: {}, stdout: {}".format(path, stderr, stdout)
aws_access_key_path = "aws_access_key_id"
make_credential_secret(aws_access_key_path, creds.access_key)