def test_job_create_gcs(self, mock_hook):
mock_hook.return_value.create_transfer_job.return_value = VALID_TRANSFER_JOB_GCS_RAW
body = deepcopy(VALID_TRANSFER_JOB_GCS)
del (body['name'])
op = GcpTransferServiceJobCreateOperator(body=body, task_id=TASK_ID)
result = op.execute(None)
mock_hook.assert_called_once_with(api_version='v1', gcp_conn_id='google_cloud_default')
mock_hook.return_value.create_transfer_job.assert_called_once_with(body=VALID_TRANSFER_JOB_GCS_RAW)
self.assertEqual(result, VALID_TRANSFER_JOB_GCS_RAW)
def test_job_create_gcs(self, mock_hook):
mock_hook.return_value.create_transfer_job.return_value = VALID_TRANSFER_JOB_GCS_RAW
body = deepcopy(VALID_TRANSFER_JOB_GCS)
del body['name']
op = GcpTransferServiceJobCreateOperator(body=body, task_id=TASK_ID)
result = op.execute(None)
mock_hook.assert_called_once_with(api_version='v1', gcp_conn_id='google_cloud_default')
mock_hook.return_value.create_transfer_job.assert_called_once_with(body=VALID_TRANSFER_JOB_GCS_RAW)
self.assertEqual(result, VALID_TRANSFER_JOB_GCS_RAW)
def test_job_create_multiple(self, aws_hook, gcp_hook):
aws_hook.return_value.get_credentials.return_value = Credentials(
TEST_AWS_ACCESS_KEY_ID, TEST_AWS_ACCESS_SECRET, None
)
gcp_hook.return_value.create_transfer_job.return_value = VALID_TRANSFER_JOB_AWS_RAW
body = deepcopy(VALID_TRANSFER_JOB_AWS)
op = GcpTransferServiceJobCreateOperator(body=body, task_id=TASK_ID)
result = op.execute(None)
self.assertEqual(result, VALID_TRANSFER_JOB_AWS_RAW)
op = GcpTransferServiceJobCreateOperator(body=body, task_id=TASK_ID)
result = op.execute(None)
self.assertEqual(result, VALID_TRANSFER_JOB_AWS_RAW)
def test_job_create_aws(self, aws_hook, mock_hook):
mock_hook.return_value.create_transfer_job.return_value = VALID_TRANSFER_JOB_AWS_RAW
aws_hook.return_value.get_credentials.return_value = Credentials(
TEST_AWS_ACCESS_KEY_ID, TEST_AWS_ACCESS_SECRET, None
)
body = deepcopy(VALID_TRANSFER_JOB_AWS)
del body['name']
op = GcpTransferServiceJobCreateOperator(body=body, task_id=TASK_ID)
result = op.execute(None)
mock_hook.assert_called_once_with(api_version='v1', gcp_conn_id='google_cloud_default')
mock_hook.return_value.create_transfer_job.assert_called_once_with(body=VALID_TRANSFER_JOB_AWS_RAW)
self.assertEqual(result, VALID_TRANSFER_JOB_AWS_RAW)
def test_job_create_aws(self, aws_hook, mock_hook):
mock_hook.return_value.create_transfer_job.return_value = VALID_TRANSFER_JOB_AWS_RAW
aws_hook.return_value.get_credentials.return_value = Credentials(
TEST_AWS_ACCESS_KEY_ID, TEST_AWS_ACCESS_SECRET, None
)
body = deepcopy(VALID_TRANSFER_JOB_AWS)
del (body['name'])
op = GcpTransferServiceJobCreateOperator(body=body, task_id=TASK_ID)
result = op.execute(None)
mock_hook.assert_called_once_with(api_version='v1', gcp_conn_id='google_cloud_default')
mock_hook.return_value.create_transfer_job.assert_called_once_with(body=VALID_TRANSFER_JOB_AWS_RAW)
self.assertEqual(result, VALID_TRANSFER_JOB_AWS_RAW)
def test_templates(self, _):
dag_id = 'test_dag_id'
self.dag = DAG(dag_id, default_args={'start_date': DEFAULT_DATE})
op = GcpTransferServiceJobCreateOperator(
body={"description": "{{ dag.dag_id }}"},
gcp_conn_id='{{ dag.dag_id }}',
aws_conn_id='{{ dag.dag_id }}',
task_id='task-id',
dag=self.dag,
)
ti = TaskInstance(op, DEFAULT_DATE)
ti.render_templates()
self.assertEqual(dag_id, getattr(op, 'body')[DESCRIPTION])
self.assertEqual(dag_id, getattr(op, 'gcp_conn_id'))
self.assertEqual(dag_id, getattr(op, 'aws_conn_id'))
def test_job_create_multiple(self, aws_hook, gcp_hook):
aws_hook.return_value.get_credentials.return_value = Credentials(
TEST_AWS_ACCESS_KEY_ID, TEST_AWS_ACCESS_SECRET, None)
gcp_hook.return_value.create_transfer_job.return_value = VALID_TRANSFER_JOB_AWS_RAW
body = deepcopy(VALID_TRANSFER_JOB_AWS)
op = GcpTransferServiceJobCreateOperator(body=body, task_id=TASK_ID)
result = op.execute(None)
self.assertEqual(result, VALID_TRANSFER_JOB_AWS_RAW)
op = GcpTransferServiceJobCreateOperator(body=body, task_id=TASK_ID)
result = op.execute(None)
self.assertEqual(result, VALID_TRANSFER_JOB_AWS_RAW)
# [END howto_operator_gcp_transfer_update_job_body]
list_filter_dict = {FILTER_PROJECT_ID: GCP_PROJECT_ID, FILTER_JOB_NAMES: []}
# [START howto_operator_gcp_transfer_default_args]
default_args = {'start_date': days_ago(1)}
# [END howto_operator_gcp_transfer_default_args]
with models.DAG(
'example_gcp_transfer',
default_args=default_args,
schedule_interval=None # Override to match your needs
) as dag:
# [START howto_operator_gcp_transfer_create_job]
create_transfer_job_from_aws = GcpTransferServiceJobCreateOperator(
task_id="create_transfer_job_from_aws", body=aws_to_gcs_transfer_body)
# [END howto_operator_gcp_transfer_create_job]
wait_for_operation_to_start = GCPTransferServiceWaitForJobStatusSensor(
task_id="wait_for_operation_to_start",
job_name=
"{{task_instance.xcom_pull('create_transfer_job_from_aws')['name']}}",
project_id=GCP_PROJECT_ID,
expected_statuses={GcpTransferOperationStatus.IN_PROGRESS},
poke_interval=WAIT_FOR_OPERATION_POKE_INTERVAL,
)
# [START howto_operator_gcp_transfer_pause_operation]
pause_operation = GcpTransferServiceOperationPauseOperator(
task_id="pause_operation",
operation_name=
