def execute(self, context):
transfer_hook = GCPTransferServiceHook(gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to)
s3_creds = S3Hook(aws_conn_id=self.aws_conn_id).get_credentials()
job = transfer_hook.create_transfer_job(
project_id=self.project_id,
description=self.description,
schedule=self.schedule,
transfer_spec={
'awsS3DataSource': {
'bucketName': self.s3_bucket,
'awsAccessKey': {
'accessKeyId': s3_creds.access_key,
'secretAccessKey': s3_creds.secret_key,
}
},
'gcsDataSink': {
'bucketName': self.gcs_bucket,
},
'objectConditions': self.object_conditions,
'transferOptions': self.transfer_options,
})
if self.wait:
transfer_hook.wait_for_transfer_job(job)
def execute(self, context):
hook = GCPTransferServiceHook(api_version=self.api_version,
gcp_conn_id=self.gcp_conn_id)
operations_list = hook.list_transfer_operations(
request_filter=self.filter)
self.log.info(operations_list)
return operations_list
def setUp(self):
with mock.patch.object(GCPTransferServiceHook,
'__init__',
return_value=None):
self.conn = mock.Mock()
self.transfer_hook = GCPTransferServiceHook()
self.transfer_hook._conn = self.conn
def execute(self, context):
hook = GCPTransferServiceHook(gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to)
body = self._create_body()
TransferJobPreprocessor(body=body, aws_conn_id=self.aws_conn_id).process_body()
job = hook.create_transfer_job(body=body)
if self.wait:
hook.wait_for_transfer_job(job, timeout=self.timeout)
def poke(self, context):
hook = GCPTransferServiceHook(gcp_conn_id=self.gcp_cloud_conn_id)
operations = hook.list_transfer_operations(
filter={'project_id': self.project_id, 'job_names': [self.job_name]}
)
check = GCPTransferServiceHook.operations_contain_expected_statuses(
operations=operations, expected_statuses=self.expected_statuses
)
if check:
self.xcom_push(key="sensed_operations", value=operations, context=context)
return check
def execute(self, context):
transfer_hook = GCPTransferServiceHook(gcp_conn_id=self.gcp_conn_id,
delegate_to=self.delegate_to)
job = transfer_hook.create_transfer_job(
project_id=self.project_id,
description=self.description,
schedule=self.schedule,
transfer_spec={
'gcsDataSource': {
'bucketName': self.source_bucket,
},
'gcsDataSink': {
'bucketName': self.destination_bucket,
},
'objectConditions': self.object_conditions,
'transferOptions': self.transfer_options,
})
if self.wait:
transfer_hook.wait_for_transfer_job(job)
def execute(self, context):
hook = GCPTransferServiceHook(api_version=self.api_version,
gcp_conn_id=self.gcp_conn_id)
hook.cancel_transfer_operation(operation_name=self.operation_name)
def execute(self, context):
self._validate_inputs()
hook = GCPTransferServiceHook(api_version=self.api_version,
gcp_conn_id=self.gcp_conn_id)
hook.delete_transfer_job(job_name=self.job_name,
project_id=self.project_id)
def execute(self, context):
TransferJobPreprocessor(body=self.body,
aws_conn_id=self.aws_conn_id).process_body()
hook = GCPTransferServiceHook(api_version=self.api_version,
gcp_conn_id=self.gcp_conn_id)
return hook.update_transfer_job(job_name=self.job_name, body=self.body)
def setUp(self):
with mock.patch(
'airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__',
new=mock_base_gcp_hook_no_default_project_id,
):
self.gct_hook = GCPTransferServiceHook(gcp_conn_id='test')