def test_object_create_acl(self, mock_hook):
operator = GoogleCloudStorageObjectCreateAclEntryOperator(
bucket="test-bucket",
object_name="test-object",
entity="test-entity",
role="test-role",
user_project="test-user-project",
task_id="id")
operator.execute(None)
mock_hook.return_value.insert_object_acl.assert_called_once_with(
bucket_name="test-bucket",
object_name="test-object",
entity="test-entity",
role="test-role",
user_project="test-user-project")
def test_object_create_acl(self, mock_hook):
operator = GoogleCloudStorageObjectCreateAclEntryOperator(
bucket="test-bucket",
object_name="test-object",
entity="test-entity",
role="test-role",
generation="test-generation",
user_project="test-user-project",
task_id="id"
)
operator.execute(None)
mock_hook.return_value.insert_object_acl.assert_called_once_with(
bucket_name="test-bucket",
object_name="test-object",
entity="test-entity",
role="test-role",
generation="test-generation",
user_project="test-user-project"
)
)
# [START howto_operator_gcs_bucket_create_acl_entry_task]
gcs_bucket_create_acl_entry_task = GoogleCloudStorageBucketCreateAclEntryOperator(
bucket=BUCKET_1,
entity=GCS_ACL_ENTITY,
role=GCS_ACL_BUCKET_ROLE,
task_id="gcs_bucket_create_acl_entry_task",
)
# [END howto_operator_gcs_bucket_create_acl_entry_task]
# [START howto_operator_gcs_object_create_acl_entry_task]
gcs_object_create_acl_entry_task = GoogleCloudStorageObjectCreateAclEntryOperator(
bucket=BUCKET_1,
object_name=BUCKET_FILE_LOCATION,
entity=GCS_ACL_ENTITY,
role=GCS_ACL_OBJECT_ROLE,
task_id="gcs_object_create_acl_entry_task",
)
# [END howto_operator_gcs_object_create_acl_entry_task]
download_file = GoogleCloudStorageDownloadOperator(
task_id="download_file",
object_name=BUCKET_FILE_LOCATION,
bucket=BUCKET_1,
filename=PATH_TO_SAVED_FILE,
)
copy_file = GoogleCloudStorageToGoogleCloudStorageOperator(
task_id="copy_file",
source_bucket=BUCKET_1,
html_content=EmailTemplate,
dag=dag)
rpt_cnt = 0
for itr_rpt in report_config[rpt_folder_cnt]['reports']:
rpt_name = report_config[rpt_folder_cnt]['reports'][rpt_cnt][
'report_name']
task_gcs_sensor = GoogleCloudStorageObjectSensor(
task_id='gcs_sensor_' + rpt_name,
bucket=rpt_bucket_name,
object='%s/%s_%s.txt' % (rpt_folder_name, "{{ ds }}", rpt_name),
google_cloud_conn_id='gcp_project_deng',
dag=dag)
task_gcs_give_acl = GoogleCloudStorageObjectCreateAclEntryOperator(
task_id='giveaclaccess_gcs_' + rpt_name,
object_name='%s/%s_%s.txt' %
(rpt_folder_name, "{{ ds }}", rpt_name),
entity='allUsers',
role='READER',
bucket='rpt_bucket_name',
google_cloud_storage_conn_id='gcp_project_deng',
dag=dag)
task_start >> task_start_report >> task_gcs_sensor >> task_gcs_give_acl
rpt_cnt += 1
task_gcs_give_acl >> task_trigger_report_email >> task_finish
rpt_folder_cnt += 1
prev_task = next_dep(sql_export_task, prev_task)
prev_task = next_dep(sql_export_task2, prev_task)
# ############################################## #
# ### IMPORTING SQL TO INSTANCE 2 ############## #
# ############################################## #
import_url_split = urlsplit(IMPORT_URI)
# For import to work we need to add the Cloud SQL instance's Service Account
# read access to the target GCS object.
# [START howto_operator_cloudsql_import_gcs_permissions]
sql_gcp_add_object_permission_task = GoogleCloudStorageObjectCreateAclEntryOperator(
entity="user-{{ task_instance.xcom_pull("
"'sql_instance_create_task2', key='service_account_email')"
" }}",
role="READER",
bucket=import_url_split[1], # netloc (bucket)
object_name=import_url_split[2][1:], # path (strip first '/')
task_id='sql_gcp_add_object_permission_task',
)
prev_task = next_dep(sql_gcp_add_object_permission_task, prev_task)
# For import to work we also need to add the Cloud SQL instance's Service Account
# write access to the whole bucket!.
sql_gcp_add_bucket_permission_2_task = GoogleCloudStorageBucketCreateAclEntryOperator(
entity="user-{{ task_instance.xcom_pull("
"'sql_instance_create_task2', key='service_account_email') "
"}}",
role="WRITER",
bucket=import_url_split[1], # netloc
task_id='sql_gcp_add_bucket_permission_2_task',
task_queryoracle_to_gcs = OracleToGoogleCloudStorageOperator(
task_id='oraclespoolreport_to_gcs_' + rpt_query_name,
oracle_conn_id='oracle_oprtpdb',
google_cloud_storage_conn_id='gcp_project_deng',
provide_context=True,
sql=rpt_query,
# params={
# 'table_name': tbl,
# 'control': ctrl_tbl
# },
bucket='la_reporting_external',
filename='%s/%s_%s.csv' %
(rpt_name, rpt_query_name, "{{ next_ds_nodash }}"),
export_format='csv',
field_delimiter=',',
dag=dag)
task_gcs_give_acl = GoogleCloudStorageObjectCreateAclEntryOperator(
task_id='giveaclaccess_gcs_' + rpt_query_name,
object_name='%s/%s_%s.csv' %
(rpt_name, rpt_query_name, "{{ next_ds_nodash }}"),
entity='allUsers',
role='READER',
bucket='la_reporting_external',
google_cloud_storage_conn_id='gcp_project_deng',
dag=dag)
task_start >> task_start_report >> task_queryoracle_to_gcs >> task_gcs_give_acl
rpt_qry_cnt += 1
task_gcs_give_acl >> task_send_report_email >> task_finish
instance=INSTANCE_NAME,
task_id='sql_export_task')
# [END howto_operator_cloudsql_export]
prev_task = next_dep(sql_export_task, prev_task)
# ############################################## #
# ### IMPORTING SQL TO INSTANCE 2 ############## #
# ############################################## #
# For import to work we need to add the Cloud SQL instance's Service Account
# read access to the target GCS object.
# [START howto_operator_cloudsql_import_gcs_permissions]
sql_gcp_add_object_permission = GoogleCloudStorageObjectCreateAclEntryOperator(
entity=
"user-{{ task_instance.xcom_pull('sql_instance_create_2', key='service_account_email') }}",
role="READER",
bucket=re.match(r'gs:\/\/(\S*)\/', IMPORT_URI).group(1),
object_name=re.match(r'gs:\/\/[^\/]*\/(\S*)', IMPORT_URI).group(1),
task_id='sql_gcp_add_object_permission',
)
# [END howto_operator_cloudsql_import_gcs_permissions]
prev_task = next_dep(sql_gcp_add_object_permission, prev_task)
# [START howto_operator_cloudsql_import]
sql_import_task = CloudSqlInstanceImportOperator(project_id=PROJECT_ID,
body=import_body,
instance=INSTANCE_NAME2,
task_id='sql_import_task')
# [END howto_operator_cloudsql_import]
prev_task = next_dep(sql_import_task, prev_task)
# ############################################## #
