def test_teragen():
if utils.hdfs_enabled():
jar_url = 'https://downloads.mesosphere.io/spark/examples/spark-terasort-1.0-jar-with-dependencies_2.11.jar'
utils.run_tests(app_url=jar_url,
app_args="1g hdfs:///terasort_in",
expected_output="Number of records written",
app_name="/spark",
args=["--class", "com.github.ehiggs.spark.terasort.TeraGen"])
def teardown_module(module):
shakedown.uninstall_package_and_wait(SPARK_PACKAGE_NAME)
if utils.hdfs_enabled():
shakedown.uninstall_package_and_wait(HDFS_PACKAGE_NAME, HDFS_SERVICE_NAME)
_run_janitor(HDFS_SERVICE_NAME)
def test_terasort():
if utils.hdfs_enabled():
_delete_hdfs_terasort_files()
_run_teragen()
_run_terasort()
_run_teravalidate()
def setup_module(module):
if utils.hdfs_enabled():
utils.require_hdfs()
utils.require_spark()
utils.upload_file(os.environ["SCALA_TEST_JAR_PATH"])
def setup_module(module):
if utils.hdfs_enabled():
utils.require_hdfs()
utils.require_spark()