Beispiel #1
0
def test_teragen():
    if utils.hdfs_enabled():
        jar_url = 'https://downloads.mesosphere.io/spark/examples/spark-terasort-1.0-jar-with-dependencies_2.11.jar'
        utils.run_tests(app_url=jar_url,
                        app_args="1g hdfs:///terasort_in",
                        expected_output="Number of records written",
                        app_name="/spark",
                        args=["--class", "com.github.ehiggs.spark.terasort.TeraGen"])
Beispiel #2
0
def teardown_module(module):
    shakedown.uninstall_package_and_wait(SPARK_PACKAGE_NAME)
    if utils.hdfs_enabled():
        shakedown.uninstall_package_and_wait(HDFS_PACKAGE_NAME, HDFS_SERVICE_NAME)
        _run_janitor(HDFS_SERVICE_NAME)
Beispiel #3
0
def test_terasort():
    if utils.hdfs_enabled():
        _delete_hdfs_terasort_files()
        _run_teragen()
        _run_terasort()
        _run_teravalidate()
Beispiel #4
0
def setup_module(module):
    if utils.hdfs_enabled():
        utils.require_hdfs()
    utils.require_spark()
    utils.upload_file(os.environ["SCALA_TEST_JAR_PATH"])
Beispiel #5
0
def setup_module(module):
    if utils.hdfs_enabled():
        utils.require_hdfs()
    utils.require_spark()