def test_execute(self):
# Given / When
operator = SparkSqlOperator(task_id='spark_sql_job',
dag=self.dag,
**self._config)
self.assertEqual(self._config['sql'], operator._sql)
self.assertEqual(self._config['conn_id'], operator._conn_id)
self.assertEqual(self._config['total_executor_cores'],
operator._total_executor_cores)
self.assertEqual(self._config['executor_cores'],
operator._executor_cores)
self.assertEqual(self._config['executor_memory'],
operator._executor_memory)
self.assertEqual(self._config['keytab'], operator._keytab)
self.assertEqual(self._config['principal'], operator._principal)
self.assertEqual(self._config['executor_memory'],
operator._executor_memory)
self.assertEqual(self._config['keytab'], operator._keytab)
self.assertEqual(self._config['principal'], operator._principal)
self.assertEqual(self._config['master'], operator._master)
self.assertEqual(self._config['name'], operator._name)
self.assertEqual(self._config['num_executors'],
operator._num_executors)
self.assertEqual(self._config['verbose'], operator._verbose)
self.assertEqual(self._config['yarn_queue'], operator._yarn_queue)
def test_execute(self):
# Given / When
operator = SparkSqlOperator(task_id='spark_sql_job',
dag=self.dag,
**self._config)
assert self._config['sql'] == operator._sql
assert self._config['conn_id'] == operator._conn_id
assert self._config[
'total_executor_cores'] == operator._total_executor_cores
assert self._config['executor_cores'] == operator._executor_cores
assert self._config['executor_memory'] == operator._executor_memory
assert self._config['keytab'] == operator._keytab
assert self._config['principal'] == operator._principal
assert self._config['executor_memory'] == operator._executor_memory
assert self._config['keytab'] == operator._keytab
assert self._config['principal'] == operator._principal
assert self._config['master'] == operator._master
assert self._config['name'] == operator._name
assert self._config['num_executors'] == operator._num_executors
assert self._config['verbose'] == operator._verbose
assert self._config['yarn_queue'] == operator._yarn_queue
# [END howto_operator_spark_submit]
# [START howto_operator_spark_jdbc]
jdbc_to_spark_job = SparkJDBCOperator(
cmd_type='jdbc_to_spark',
jdbc_table="foo",
spark_conf={},
spark_jars="${SPARK_HOME}/jars/postgresql-42.2.12.jar",
jdbc_driver="org.postgresql.Driver",
metastore_table="bar",
save_mode="overwrite",
save_format="JSON",
task_id="jdbc_to_spark_job")
spark_to_jdbc_job = SparkJDBCOperator(
cmd_type='spark_to_jdbc',
jdbc_table="foo",
spark_conf={},
spark_jars="${SPARK_HOME}/jars/postgresql-42.2.12.jar",
jdbc_driver="org.postgresql.Driver",
metastore_table="bar",
save_mode="append",
task_id="spark_to_jdbc_job")
# [END howto_operator_spark_jdbc]
# [START howto_operator_spark_sql]
sql_job = SparkSqlOperator(sql="SELECT * FROM bar",
master="local",
task_id="sql_job")
# [END howto_operator_spark_sql]