Esempi in Python per SparkSubmitHook._resolve_should_track_driver_status

Esempio n. 1

File: test_spark_submit_hook.py Progetto: rubeshdcube/airflow-1

    def test_resolve_should_track_driver_status(self):
        # Given
        hook_default = SparkSubmitHook(conn_id='')
        hook_spark_yarn_cluster = SparkSubmitHook(conn_id='spark_yarn_cluster')
        hook_spark_k8s_cluster = SparkSubmitHook(conn_id='spark_k8s_cluster')
        hook_spark_default_mesos = SparkSubmitHook(
            conn_id='spark_default_mesos')
        hook_spark_home_set = SparkSubmitHook(conn_id='spark_home_set')
        hook_spark_home_not_set = SparkSubmitHook(conn_id='spark_home_not_set')
        hook_spark_binary_set = SparkSubmitHook(conn_id='spark_binary_set')
        hook_spark_binary_and_home_set = SparkSubmitHook(
            conn_id='spark_binary_and_home_set')
        hook_spark_standalone_cluster = SparkSubmitHook(
            conn_id='spark_standalone_cluster')

        # When
        should_track_driver_status_default = hook_default \
            ._resolve_should_track_driver_status()
        should_track_driver_status_spark_yarn_cluster = hook_spark_yarn_cluster \
            ._resolve_should_track_driver_status()
        should_track_driver_status_spark_k8s_cluster = hook_spark_k8s_cluster \
            ._resolve_should_track_driver_status()
        should_track_driver_status_spark_default_mesos = hook_spark_default_mesos \
            ._resolve_should_track_driver_status()
        should_track_driver_status_spark_home_set = hook_spark_home_set \
            ._resolve_should_track_driver_status()
        should_track_driver_status_spark_home_not_set = hook_spark_home_not_set \
            ._resolve_should_track_driver_status()
        should_track_driver_status_spark_binary_set = hook_spark_binary_set \
            ._resolve_should_track_driver_status()
        should_track_driver_status_spark_binary_and_home_set = \
            hook_spark_binary_and_home_set._resolve_should_track_driver_status()
        should_track_driver_status_spark_standalone_cluster = \
            hook_spark_standalone_cluster._resolve_should_track_driver_status()

        # Then
        self.assertEqual(should_track_driver_status_default, False)
        self.assertEqual(should_track_driver_status_spark_yarn_cluster, False)
        self.assertEqual(should_track_driver_status_spark_k8s_cluster, False)
        self.assertEqual(should_track_driver_status_spark_default_mesos, False)
        self.assertEqual(should_track_driver_status_spark_home_set, False)
        self.assertEqual(should_track_driver_status_spark_home_not_set, False)
        self.assertEqual(should_track_driver_status_spark_binary_set, False)
        self.assertEqual(should_track_driver_status_spark_binary_and_home_set,
                         False)
        self.assertEqual(should_track_driver_status_spark_standalone_cluster,
                         True)

Esempio n. 2

File: test_spark_submit_hook.py Progetto: AdamUnger/incubator-airflow

    def test_resolve_should_track_driver_status(self):
        # Given
        hook_default = SparkSubmitHook(conn_id='')
        hook_spark_yarn_cluster = SparkSubmitHook(conn_id='spark_yarn_cluster')
        hook_spark_k8s_cluster = SparkSubmitHook(conn_id='spark_k8s_cluster')
        hook_spark_default_mesos = SparkSubmitHook(conn_id='spark_default_mesos')
        hook_spark_home_set = SparkSubmitHook(conn_id='spark_home_set')
        hook_spark_home_not_set = SparkSubmitHook(conn_id='spark_home_not_set')
        hook_spark_binary_set = SparkSubmitHook(conn_id='spark_binary_set')
        hook_spark_binary_and_home_set = SparkSubmitHook(
            conn_id='spark_binary_and_home_set')
        hook_spark_standalone_cluster = SparkSubmitHook(
            conn_id='spark_standalone_cluster')

        # When
        should_track_driver_status_default = hook_default \
            ._resolve_should_track_driver_status()
        should_track_driver_status_spark_yarn_cluster = hook_spark_yarn_cluster \
            ._resolve_should_track_driver_status()
        should_track_driver_status_spark_k8s_cluster = hook_spark_k8s_cluster \
            ._resolve_should_track_driver_status()
        should_track_driver_status_spark_default_mesos = hook_spark_default_mesos \
            ._resolve_should_track_driver_status()
        should_track_driver_status_spark_home_set = hook_spark_home_set \
            ._resolve_should_track_driver_status()
        should_track_driver_status_spark_home_not_set = hook_spark_home_not_set \
            ._resolve_should_track_driver_status()
        should_track_driver_status_spark_binary_set = hook_spark_binary_set \
            ._resolve_should_track_driver_status()
        should_track_driver_status_spark_binary_and_home_set = \
            hook_spark_binary_and_home_set._resolve_should_track_driver_status()
        should_track_driver_status_spark_standalone_cluster = \
            hook_spark_standalone_cluster._resolve_should_track_driver_status()

        # Then
        self.assertEqual(should_track_driver_status_default, False)
        self.assertEqual(should_track_driver_status_spark_yarn_cluster, False)
        self.assertEqual(should_track_driver_status_spark_k8s_cluster, False)
        self.assertEqual(should_track_driver_status_spark_default_mesos, False)
        self.assertEqual(should_track_driver_status_spark_home_set, False)
        self.assertEqual(should_track_driver_status_spark_home_not_set, False)
        self.assertEqual(should_track_driver_status_spark_binary_set, False)
        self.assertEqual(should_track_driver_status_spark_binary_and_home_set, False)
        self.assertEqual(should_track_driver_status_spark_standalone_cluster, True)