def setup_spark(configure_security_spark, configure_universe): try: spark_utils.require_spark( user="******", # Run as root on centos use_bootstrap_ip=True) # Needed on GPU nodes spark_utils.upload_file(os.environ["SCALA_TEST_JAR_PATH"]) yield finally: spark_utils.teardown_spark()
def test_python(): python_script_path = os.path.join(THIS_DIR, 'jobs', 'python', 'pi_with_include.py') python_script_url = utils.upload_file(python_script_path) py_file_path = os.path.join(THIS_DIR, 'jobs', 'python', 'PySparkTestInclude.py') py_file_url = utils.upload_file(py_file_path) utils.run_tests(app_url=python_script_url, app_args="30", expected_output="Pi is roughly 3", args=["--py-files {}".format(py_file_url)])
def setup_spark(configure_security, configure_universe): try: utils.require_spark() utils.upload_file(os.environ["SCALA_TEST_JAR_PATH"]) shakedown.run_dcos_command( 'package install --cli dcos-enterprise-cli --yes') yield finally: utils.teardown_spark()
def setup_spark(kerberized_kafka, configure_security_spark, configure_universe): try: # need to do this here also in case this test is run first # and the jar hasn't been updated utils.upload_file(os.environ["SCALA_TEST_JAR_PATH"]) utils.require_spark() yield finally: utils.teardown_spark()
def _test_spark_docker_image(docker_image): utils.upload_dcos_test_jar() utils.require_spark( additional_options={'service': { 'docker-image': docker_image }}) expected_groups_count = 12000 num_mappers = 4 value_size_bytes = 100 num_reducers = 4 sleep = 500 python_script_path = os.path.join(THIS_DIR, 'jobs', 'python', 'shuffle_app.py') python_script_url = utils.upload_file(python_script_path) utils.run_tests( app_url=python_script_url, app_args="{} {} {} {} {}".format(num_mappers, expected_groups_count, value_size_bytes, num_reducers, sleep), expected_output="Groups count: {}".format(expected_groups_count), args=[ "--conf spark.executor.cores=1", "--conf spark.cores.max=4", "--conf spark.scheduler.minRegisteredResourcesRatio=1", "--conf spark.scheduler.maxRegisteredResourcesWaitingTime=3m" ]) utils.teardown_spark()
def test_disconnect_from_master(): python_script_path = os.path.join(THIS_DIR, 'jobs', 'python', 'long_running.py') python_script_url = utils.upload_file(python_script_path) task_id = utils.submit_job( python_script_url, "{} {}".format(LONG_RUNNING_FW_NUM_TASKS, LONG_RUNNING_RUN_TIME_SEC), [ "--conf", "spark.mesos.driver.failoverTimeout=1800", "--conf", "spark.cores.max=1" ]) # Wait until executor is running utils.wait_for_executors_running(LONG_RUNNING_FW_NAME, LONG_RUNNING_FW_NUM_TASKS) # Block the driver's connection to Mesos master framework_info = shakedown.get_service(LONG_RUNNING_FW_NAME) (driver_host, port) = _parse_fw_pid_host_port(framework_info["pid"]) _block_master_connection(driver_host, port) # The connection will timeout after 15 minutes of inactivity. # Add 5 minutes to make sure the master has detected the disconnection. # The framework will be considered disconnected => failover_timeout kicks in. LOGGER.info( "Waiting {} seconds for connection with master to timeout...".format( MASTER_CONNECTION_TIMEOUT_SEC)) time.sleep(MASTER_CONNECTION_TIMEOUT_SEC + 5 * 60) # Restore the connection. The driver should reconnect. _unblock_master_connection(driver_host) # The executor and driver should finish. utils.check_job_output(task_id, "Job completed successfully")
def test_driver_executor_tls(): ''' Put keystore and truststore as secrets in DC/OS secret store. Run SparkPi job with TLS enabled, referencing those secrets. Make sure other secrets still show up. ''' python_script_path = os.path.join(THIS_DIR, 'jobs', 'python', 'pi_with_secret.py') python_script_url = utils.upload_file(python_script_path) resources_folder = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'resources') keystore_file = 'server.jks' truststore_file = 'trust.jks' keystore_path = os.path.join(resources_folder, '{}.base64'.format(keystore_file)) truststore_path = os.path.join(resources_folder, '{}.base64'.format(truststore_file)) keystore_secret = '__dcos_base64__keystore' truststore_secret = '__dcos_base64__truststore' my_secret = 'mysecret' my_secret_content = 'secretcontent' shakedown.run_dcos_command( 'security secrets create /{} --value-file {}'.format( keystore_secret, keystore_path)) shakedown.run_dcos_command( 'security secrets create /{} --value-file {}'.format( truststore_secret, truststore_path)) shakedown.run_dcos_command('security secrets create /{} --value {}'.format( my_secret, my_secret_content)) password = '******' try: utils.run_tests( app_url=python_script_url, app_args="30 {} {}".format(my_secret, my_secret_content), expected_output="Pi is roughly 3", args=[ "--keystore-secret-path", keystore_secret, "--truststore-secret-path", truststore_secret, "--private-key-password", format(password), "--keystore-password", format(password), "--truststore-password", format(password), "--conf", "spark.mesos.driver.secret.names={}".format(my_secret), "--conf", "spark.mesos.driver.secret.filenames={}".format(my_secret), "--conf", "spark.mesos.driver.secret.envkeys={}".format(my_secret), ]) finally: shakedown.run_dcos_command( 'security secrets delete /{}'.format(keystore_secret)) shakedown.run_dcos_command( 'security secrets delete /{}'.format(truststore_secret)) shakedown.run_dcos_command( 'security secrets delete /{}'.format(my_secret))
def test_jar(app_name=utils.SPARK_APP_NAME): master_url = ("https" if utils.is_strict() else "http") + "://leader.mesos:5050" spark_job_runner_args = '{} dcos \\"*\\" spark:only 2 --auth-token={}'.format( master_url, shakedown.dcos_acs_token()) jar_url = utils.upload_file(os.getenv('TEST_JAR_PATH')) utils.run_tests( app_url=jar_url, app_args=spark_job_runner_args, expected_output="All tests passed", app_name=app_name, args=[ "--class", 'com.typesafe.spark.test.mesos.framework.runners.SparkJobRunner' ])
def test_r(): r_script_path = os.path.join(THIS_DIR, 'jobs', 'R', 'dataframe.R') r_script_url = utils.upload_file(r_script_path) utils.run_tests(app_url=r_script_url, app_args='', expected_output="Justin")