dag_id='s3_bucket_tagging_dag',
schedule_interval=None,
start_date=days_ago(2),
max_active_runs=1,
tags=['example'],
) as dag:
create_bucket = S3CreateBucketOperator(
task_id='s3_bucket_tagging_dag_create',
bucket_name=BUCKET_NAME,
region_name='us-east-1',
)
delete_bucket = S3DeleteBucketOperator(
task_id='s3_bucket_tagging_dag_delete',
bucket_name=BUCKET_NAME,
force_delete=True,
)
# [START howto_operator_s3_bucket_tagging]
get_tagging = S3GetBucketTaggingOperator(
task_id='s3_bucket_tagging_dag_get_tagging', bucket_name=BUCKET_NAME)
put_tagging = S3PutBucketTaggingOperator(
task_id='s3_bucket_tagging_dag_put_tagging',
bucket_name=BUCKET_NAME,
key=TAG_KEY,
value=TAG_VALUE)
delete_tagging = S3DeleteBucketTaggingOperator(
task_id='s3_bucket_tagging_dag_delete_tagging',
task_id="create_s3_bucket", bucket_name=S3BUCKET_NAME, region_name='us-east-1'
)
upload_to_s3 = PythonOperator(task_id='upload_file_to_s3', python_callable=upload_file)
create_gcs_bucket = GCSCreateBucketOperator(
task_id="create_bucket",
bucket_name=GCS_BUCKET,
project_id=GCP_PROJECT_ID,
)
# [START howto_transfer_s3togcs_operator]
transfer_to_gcs = S3ToGCSOperator(
task_id='s3_to_gcs_task', bucket=S3BUCKET_NAME, prefix=PREFIX, dest_gcs="gs://" + GCS_BUCKET
)
# [END howto_transfer_s3togcs_operator]
delete_s3_bucket = S3DeleteBucketOperator(
task_id='delete_s3_bucket', bucket_name=S3BUCKET_NAME, force_delete=True
)
delete_gcs_bucket = GCSDeleteBucketOperator(task_id='delete_gcs_bucket', bucket_name=GCS_BUCKET)
(
create_s3_bucket
>> upload_to_s3
>> create_gcs_bucket
>> transfer_to_gcs
>> delete_s3_bucket
>> delete_gcs_bucket
)
BUCKET_NAME = os.environ.get('BUCKET_NAME', 'test-s3-bucket-tagging')
TAG_KEY = os.environ.get('TAG_KEY', 'test-s3-bucket-tagging-key')
TAG_VALUE = os.environ.get('TAG_VALUE', 'test-s3-bucket-tagging-value')
# [START howto_operator_s3_bucket_tagging]
with DAG(
dag_id='s3_bucket_tagging_dag',
schedule_interval=None,
start_date=datetime(2021, 1, 1),
catchup=False,
default_args={"bucket_name": BUCKET_NAME},
max_active_runs=1,
tags=['example'],
) as dag:
create_bucket = S3CreateBucketOperator(task_id='s3_bucket_tagging_dag_create', region_name='us-east-1')
delete_bucket = S3DeleteBucketOperator(task_id='s3_bucket_tagging_dag_delete', force_delete=True)
get_tagging = S3GetBucketTaggingOperator(task_id='s3_bucket_tagging_dag_get_tagging')
put_tagging = S3PutBucketTaggingOperator(
task_id='s3_bucket_tagging_dag_put_tagging', key=TAG_KEY, value=TAG_VALUE
)
delete_tagging = S3DeleteBucketTaggingOperator(task_id='s3_bucket_tagging_dag_delete_tagging')
create_bucket >> put_tagging >> get_tagging >> delete_tagging >> delete_bucket
# [END howto_operator_s3_bucket_tagging]
def setUp(self):
self.delete_bucket_operator = S3DeleteBucketOperator(task_id=TASK_ID, bucket_name=BUCKET_NAME,)
