class TestSparkAppSubmit(TestCase): masters = [ SparkMaster.local(), SparkMaster.local(1), SparkMaster.yarn_client(), SparkMaster.yarn_cluster() ] input_path = os.path.join(os.path.dirname(__file__), "resources", "spark", "input.txt") def _spark_application_template_(self, master): return SparkApplication().application( application_jar=os.path.join(os.path.dirname(__file__), "resources", "spark", "SparkExample.jar"), main_class="example.spark.WordCounter").master(master) def spark_app_config_template(self, master, name=str(uuid.uuid4())): _config = Configuration.create() _config.set(section=name, key=TaskOptions.SPARK_APP_CONFIG_MASTER, value=master) _config.set(section=name, key=TaskOptions.SPARK_APP_CONFIG_APPLICATION_JAR, value=os.path.join(os.path.dirname(__file__), "resources", "spark", "SparkExample.jar")) _config.set(section=name, key=TaskOptions.SPARK_APP_CONFIG_MAIN_CLASS, value="example.spark.WordCounter") return _config @skipUnless(has_command('spark-submit'), "Cannot find spark-submit command-line utility") def test_spark_app_submit(self): # self.run_test(application=self._spark_application_template_(SparkMaster.local())) self._run_(application=self._spark_application_template_( SparkMaster.local(1))) # self.run_test(application=self._spark_application_template_(SparkMaster.yarn_cluster())) # self.run_test(application=self._spark_application_template_(SparkMaster.yarn_client())) @skipUnless(has_command('spark-submit'), "Cannot find spark-submit command-line utility") def test_preconfigured_spark_app_submit(self): section = str(uuid.uuid4()) _app_config = self.spark_app_config_template( master=SparkMaster.local(1), name=section) self._run_( application=SparkApplication(config=_app_config, name=section)) def _run_(self, application, test_id=str(uuid.uuid4())): basedir = LocalFS(os.path.join("/tmp", "test_spark", test_id)) try: basedir.create_directory() _app_input = self.input_path _app_output_dir = os.path.join(basedir.path, "output") status = application.run('file:' + _app_input, 'file:' + _app_output_dir) self.assertTrue(status.is_ok(), status.stderr()) self.assertTrue(os.path.exists(_app_output_dir), status.stderr()) finally: basedir.delete_directory()