def setup_spark(configure_security_spark, configure_universe):
try:
spark_utils.require_spark(
user="******", # Run as root on centos
use_bootstrap_ip=True) # Needed on GPU nodes
spark_utils.upload_file(os.environ["SCALA_TEST_JAR_PATH"])
yield
finally:
spark_utils.teardown_spark()
def test_python():
python_script_path = os.path.join(THIS_DIR, 'jobs', 'python', 'pi_with_include.py')
python_script_url = utils.upload_file(python_script_path)
py_file_path = os.path.join(THIS_DIR, 'jobs', 'python', 'PySparkTestInclude.py')
py_file_url = utils.upload_file(py_file_path)
utils.run_tests(app_url=python_script_url,
app_args="30",
expected_output="Pi is roughly 3",
args=["--py-files {}".format(py_file_url)])
def setup_spark(configure_security, configure_universe):
try:
utils.require_spark()
utils.upload_file(os.environ["SCALA_TEST_JAR_PATH"])
shakedown.run_dcos_command(
'package install --cli dcos-enterprise-cli --yes')
yield
finally:
utils.teardown_spark()
def setup_spark(kerberized_kafka, configure_security_spark,
configure_universe):
try:
# need to do this here also in case this test is run first
# and the jar hasn't been updated
utils.upload_file(os.environ["SCALA_TEST_JAR_PATH"])
utils.require_spark()
yield
finally:
utils.teardown_spark()
def _test_spark_docker_image(docker_image):
utils.upload_dcos_test_jar()
utils.require_spark(
additional_options={'service': {
'docker-image': docker_image
}})
expected_groups_count = 12000
num_mappers = 4
value_size_bytes = 100
num_reducers = 4
sleep = 500
python_script_path = os.path.join(THIS_DIR, 'jobs', 'python',
'shuffle_app.py')
python_script_url = utils.upload_file(python_script_path)
utils.run_tests(
app_url=python_script_url,
app_args="{} {} {} {} {}".format(num_mappers, expected_groups_count,
value_size_bytes, num_reducers,
sleep),
expected_output="Groups count: {}".format(expected_groups_count),
args=[
"--conf spark.executor.cores=1", "--conf spark.cores.max=4",
"--conf spark.scheduler.minRegisteredResourcesRatio=1",
"--conf spark.scheduler.maxRegisteredResourcesWaitingTime=3m"
])
utils.teardown_spark()
def test_disconnect_from_master():
python_script_path = os.path.join(THIS_DIR, 'jobs', 'python',
'long_running.py')
python_script_url = utils.upload_file(python_script_path)
task_id = utils.submit_job(
python_script_url,
"{} {}".format(LONG_RUNNING_FW_NUM_TASKS, LONG_RUNNING_RUN_TIME_SEC), [
"--conf", "spark.mesos.driver.failoverTimeout=1800", "--conf",
"spark.cores.max=1"
])
# Wait until executor is running
utils.wait_for_executors_running(LONG_RUNNING_FW_NAME,
LONG_RUNNING_FW_NUM_TASKS)
# Block the driver's connection to Mesos master
framework_info = shakedown.get_service(LONG_RUNNING_FW_NAME)
(driver_host, port) = _parse_fw_pid_host_port(framework_info["pid"])
_block_master_connection(driver_host, port)
# The connection will timeout after 15 minutes of inactivity.
# Add 5 minutes to make sure the master has detected the disconnection.
# The framework will be considered disconnected => failover_timeout kicks in.
LOGGER.info(
"Waiting {} seconds for connection with master to timeout...".format(
MASTER_CONNECTION_TIMEOUT_SEC))
time.sleep(MASTER_CONNECTION_TIMEOUT_SEC + 5 * 60)
# Restore the connection. The driver should reconnect.
_unblock_master_connection(driver_host)
# The executor and driver should finish.
utils.check_job_output(task_id, "Job completed successfully")
def test_driver_executor_tls():
'''
Put keystore and truststore as secrets in DC/OS secret store.
Run SparkPi job with TLS enabled, referencing those secrets.
Make sure other secrets still show up.
'''
python_script_path = os.path.join(THIS_DIR, 'jobs', 'python',
'pi_with_secret.py')
python_script_url = utils.upload_file(python_script_path)
resources_folder = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'resources')
keystore_file = 'server.jks'
truststore_file = 'trust.jks'
keystore_path = os.path.join(resources_folder,
'{}.base64'.format(keystore_file))
truststore_path = os.path.join(resources_folder,
'{}.base64'.format(truststore_file))
keystore_secret = '__dcos_base64__keystore'
truststore_secret = '__dcos_base64__truststore'
my_secret = 'mysecret'
my_secret_content = 'secretcontent'
shakedown.run_dcos_command(
'security secrets create /{} --value-file {}'.format(
keystore_secret, keystore_path))
shakedown.run_dcos_command(
'security secrets create /{} --value-file {}'.format(
truststore_secret, truststore_path))
shakedown.run_dcos_command('security secrets create /{} --value {}'.format(
my_secret, my_secret_content))
password = '******'
try:
utils.run_tests(
app_url=python_script_url,
app_args="30 {} {}".format(my_secret, my_secret_content),
expected_output="Pi is roughly 3",
args=[
"--keystore-secret-path",
keystore_secret,
"--truststore-secret-path",
truststore_secret,
"--private-key-password",
format(password),
"--keystore-password",
format(password),
"--truststore-password",
format(password),
"--conf",
"spark.mesos.driver.secret.names={}".format(my_secret),
"--conf",
"spark.mesos.driver.secret.filenames={}".format(my_secret),
"--conf",
"spark.mesos.driver.secret.envkeys={}".format(my_secret),
])
finally:
shakedown.run_dcos_command(
'security secrets delete /{}'.format(keystore_secret))
shakedown.run_dcos_command(
'security secrets delete /{}'.format(truststore_secret))
shakedown.run_dcos_command(
'security secrets delete /{}'.format(my_secret))
def test_jar(app_name=utils.SPARK_APP_NAME):
master_url = ("https"
if utils.is_strict() else "http") + "://leader.mesos:5050"
spark_job_runner_args = '{} dcos \\"*\\" spark:only 2 --auth-token={}'.format(
master_url, shakedown.dcos_acs_token())
jar_url = utils.upload_file(os.getenv('TEST_JAR_PATH'))
utils.run_tests(
app_url=jar_url,
app_args=spark_job_runner_args,
expected_output="All tests passed",
app_name=app_name,
args=[
"--class",
'com.typesafe.spark.test.mesos.framework.runners.SparkJobRunner'
])
def test_r():
r_script_path = os.path.join(THIS_DIR, 'jobs', 'R', 'dataframe.R')
r_script_url = utils.upload_file(r_script_path)
utils.run_tests(app_url=r_script_url,
app_args='',
expected_output="Justin")
