def test_execute(self, mock_hook): operator = GCSCreateBucketOperator( task_id=TASK_ID, bucket_name=TEST_BUCKET, resource={ "lifecycle": { "rule": [{"action": {"type": "Delete"}, "condition": {"age": 7}}] } }, storage_class="MULTI_REGIONAL", location="EU", labels={"env": "prod"}, project_id=TEST_PROJECT, ) operator.execute(None) mock_hook.return_value.create_bucket.assert_called_once_with( bucket_name=TEST_BUCKET, storage_class="MULTI_REGIONAL", location="EU", labels={"env": "prod"}, project_id=TEST_PROJECT, resource={ "lifecycle": { "rule": [{"action": {"type": "Delete"}, "condition": {"age": 7}}] } }, )
'GCP_GCS_PATH_TO_TRANSFORM_SCRIPT', 'test.py' ) PATH_TO_UPLOAD_FILE = os.environ.get( "GCP_GCS_PATH_TO_UPLOAD_FILE", "test-gcs-example.txt" ) PATH_TO_SAVED_FILE = os.environ.get( "GCP_GCS_PATH_TO_SAVED_FILE", "test-gcs-example-download.txt" ) BUCKET_FILE_LOCATION = PATH_TO_UPLOAD_FILE.rpartition("/")[-1] with models.DAG( "example_gcs", default_args=default_args, schedule_interval=None, tags=['example'], ) as dag: create_bucket1 = GCSCreateBucketOperator( task_id="create_bucket1", bucket_name=BUCKET_1, project_id=PROJECT_ID ) create_bucket2 = GCSCreateBucketOperator( task_id="create_bucket2", bucket_name=BUCKET_2, project_id=PROJECT_ID ) list_buckets = GCSListObjectsOperator( task_id="list_buckets", bucket=BUCKET_1 ) list_buckets_result = BashOperator( task_id="list_buckets_result", bash_command="echo \"{{ task_instance.xcom_pull('list_buckets') }}\"", )
AdsInsights.Field.clicks, AdsInsights.Field.impressions, ] PARAMETERS = {'level': 'ad', 'date_preset': 'yesterday'} # [END howto_FB_ADS_variables] with models.DAG( "example_facebook_ads_to_gcs", schedule_interval='@once', # Override to match your needs start_date=datetime(2021, 1, 1), catchup=False, ) as dag: create_bucket = GCSCreateBucketOperator( task_id="create_bucket", bucket_name=GCS_BUCKET, project_id=GCP_PROJECT_ID, ) create_dataset = BigQueryCreateEmptyDatasetOperator( task_id="create_dataset", dataset_id=DATASET_NAME, ) create_table = BigQueryCreateEmptyTableOperator( task_id="create_table", dataset_id=DATASET_NAME, table_id=TABLE_NAME, schema_fields=[ {'name': 'campaign_name', 'type': 'STRING', 'mode': 'NULLABLE'}, {'name': 'campaign_id', 'type': 'STRING', 'mode': 'NULLABLE'},
BUCKET_FILE_LOCATION = PATH_TO_UPLOAD_FILE.rpartition("/")[-1] with models.DAG( "example_gcs", start_date=START_DATE, catchup=False, schedule_interval='@once', tags=['example'], ) as dag: create_bucket1 = GCSCreateBucketOperator( task_id="create_bucket1", bucket_name=BUCKET_1, project_id=PROJECT_ID, resource={ "iamConfiguration": { "uniformBucketLevelAccess": { "enabled": False, }, }, }, ) create_bucket2 = GCSCreateBucketOperator(task_id="create_bucket2", bucket_name=BUCKET_2, project_id=PROJECT_ID) list_buckets = GCSListObjectsOperator(task_id="list_buckets", bucket=BUCKET_1) list_buckets_result = BashOperator( task_id="list_buckets_result",
from airflow.providers.google.cloud.operators.gcs import GCSListObjectsOperator from airflow.providers.google.cloud.operators.gcs import GCSDeleteBucketOperator from airflow import models import uuid import os from airflow.utils.dates import days_ago from airflow.utils.state import State UUID = uuid.uuid4() PROJECT_ID = os.environ.get("GOOGLE_CLOUD_PROJECT", "leah-playground") BUCKET_NAME = f"leah-{UUID}" with models.DAG( "example_gcs", start_date=days_ago(1), schedule_interval=None, ) as dag: create_bucket = GCSCreateBucketOperator(task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID) list_objects = GCSListObjectsOperator(task_id="list_objects", bucket=BUCKET_NAME) list_buckets_result = BashOperator( task_id="list_buckets_result", bash_command="echo \"{{ task_instance.xcom_pull('list_objects') }}\"", ) delete_bucket = GCSDeleteBucketOperator(task_id="delete_bucket", bucket_name=BUCKET_NAME) create_bucket >> list_objects >> delete_bucket if __name__ == "__main__": dag.clear(dag_run_state=State.NONE) dag.run()
from airflow.utils.dates import days_ago from airflow.utils.state import State # Assumes existence of Airflow Variable set to name of GCP Project PROJECT_ID = models.Variable.get("gcp_project") with models.DAG( "example_gcs", start_date=days_ago(1), schedule_interval=None, ) as dag: generate_uuid = PythonOperator(task_id='generate_uuid', python_callable=lambda: str(uuid.uuid4())) create_bucket = GCSCreateBucketOperator( task_id="create_bucket", bucket_name="{{ task_instance.xcom_pull('generate_uuid') }}", project_id=PROJECT_ID) list_objects = GCSListObjectsOperator( task_id="list_objects", bucket="{{ task_instance.xcom_pull('generate_uuid') }}") list_buckets_result = BashOperator( task_id="list_buckets_result", bash_command="echo \"{{ task_instance.xcom_pull('list_objects') }}\"", ) delete_bucket = GCSDeleteBucketOperator( task_id="delete_bucket", bucket_name="{{ task_instance.xcom_pull('generate_uuid') }}") generate_uuid >> create_bucket >> list_objects >> list_buckets_result >> delete_bucket if __name__ == "__main__":