def test_hook_correct_region():
with patch(HOOK) as mock_hook:
dataproc_task = DataProcSparkOperator(task_id=TASK_ID,
region=GCP_REGION)
dataproc_task.execute(None)
mock_hook.return_value.submit.assert_called_once_with(
mock.ANY, mock.ANY, GCP_REGION, mock.ANY)
def test_hook_correct_region(self):
with patch('airflow.contrib.operators.dataproc_operator.DataProcHook'
) as mock_hook:
dataproc_task = DataProcSparkOperator(task_id=TASK_ID,
region=REGION)
dataproc_task.execute(None)
mock_hook.return_value.submit.assert_called_once_with(
mock.ANY, mock.ANY, REGION)
def test_hook_correct_region(self):
with patch('airflow.contrib.operators.dataproc_operator.DataProcHook') as mock_hook:
dataproc_task = DataProcSparkOperator(
task_id=TASK_ID,
region=REGION
)
dataproc_task.execute(None)
mock_hook.return_value.submit.assert_called_once_with(mock.ANY, mock.ANY, REGION)
def test_hook_correct_region():
with patch(HOOK) as mock_hook:
dataproc_task = DataProcSparkOperator(
task_id=TASK_ID,
region=GCP_REGION
)
dataproc_task.execute(None)
mock_hook.return_value.submit.assert_called_once_with(mock.ANY, mock.ANY,
GCP_REGION, mock.ANY)
def test_correct_job_definition(self, mock_hook, mock_uuid):
# Expected job
job_definition = deepcopy(DATAPROC_JOB_TO_SUBMIT)
job_definition['job']['sparkJob'] = {'mainClass': 'main_class'}
job_definition['job']['reference']['projectId'] = None
job_definition['job']['reference']['jobId'] = DATAPROC_JOB_ID + "_test"
# Prepare job using operator
task = DataProcSparkOperator(task_id=TASK_ID,
region=GCP_REGION,
cluster_name=CLUSTER_NAME,
job_name=DATAPROC_JOB_ID,
labels=LABELS,
main_class="main_class")
task.execute(context=None)
self.assertDictEqual(job_definition, task.job_template.job)
