Python WordCountPySparkTask Examples

Example #1

File: test_emr_spark_task.py Project: databand-ai/dbnd

 def test_word_count_pyspark(self):
     logging.info("Running %s", WordCountPySparkTask)
     actual = WordCountPySparkTask(
         text=TEXT_FILE, task_version=str(random.random()), override=conf_override
     )
     actual.dbnd_run()
     print(target(actual.counters.path, "part-00000").read())

Example #2

File: test_livy_spark_task.py Project: databand-ai/dbnd

 def test_word_count_pyspark(self):
     with dbnd_config(disable_tracker_api()):
         actual = WordCountPySparkTask(
             text=TEXT_FILE,
             task_version=str(random.random()),
             override=conf_override,
         )
         actual.dbnd_run()
         print(target(actual.counters.path, "part-00000").read())

Example #3

File: test_databricks_local.py Project: turbaszek/dbnd

    def test_simple_python(self, d_hook):
        run_states = [
            RunState("RUNNING", "gla", "bla"),
            RunState("TERMINATED", "SUCCESS", "yey"),
        ]
        d_hook_instance = d_hook.return_value
        d_hook_instance.submit_run = my_submit_run
        d_hook_instance.get_run_state.side_effect = run_states
        d_hook_instance.get_run_page_url.return_value = (
            "https://databand.ai-test-databricks")

        WordCountPySparkTask(
            text=__file__,
            spark_engine="databricks",
            task_version="now",
            override=test_config,
        ).dbnd_run()

Example #4

File: test_spark_task.py Project: turbaszek/dbnd

 def test_word_count_pyspark(self):
     actual = WordCountPySparkTask(text=__file__)
     actual.dbnd_run()
     print(target(actual.counters.path, "part-00000").read())