Python GCSCreateBucketOperator示例

示例#1

文件： test_gcs.py 项目： stijndehaes/airflow

    def test_execute(self, mock_hook):
        operator = GCSCreateBucketOperator(
            task_id=TASK_ID,
            bucket_name=TEST_BUCKET,
            resource={
                "lifecycle": {
                    "rule": [{"action": {"type": "Delete"}, "condition": {"age": 7}}]
                }
            },
            storage_class="MULTI_REGIONAL",
            location="EU",
            labels={"env": "prod"},
            project_id=TEST_PROJECT,
        )

        operator.execute(None)
        mock_hook.return_value.create_bucket.assert_called_once_with(
            bucket_name=TEST_BUCKET,
            storage_class="MULTI_REGIONAL",
            location="EU",
            labels={"env": "prod"},
            project_id=TEST_PROJECT,
            resource={
                "lifecycle": {
                    "rule": [{"action": {"type": "Delete"}, "condition": {"age": 7}}]
                }
            },
        )

示例#2

文件： example_gcs.py 项目： yyz940922/airflow_date_issue_corrected

    'GCP_GCS_PATH_TO_TRANSFORM_SCRIPT', 'test.py'
)
PATH_TO_UPLOAD_FILE = os.environ.get(
    "GCP_GCS_PATH_TO_UPLOAD_FILE", "test-gcs-example.txt"
)
PATH_TO_SAVED_FILE = os.environ.get(
    "GCP_GCS_PATH_TO_SAVED_FILE", "test-gcs-example-download.txt"
)

BUCKET_FILE_LOCATION = PATH_TO_UPLOAD_FILE.rpartition("/")[-1]

with models.DAG(
    "example_gcs", default_args=default_args, schedule_interval=None, tags=['example'],
) as dag:
    create_bucket1 = GCSCreateBucketOperator(
        task_id="create_bucket1", bucket_name=BUCKET_1, project_id=PROJECT_ID
    )

    create_bucket2 = GCSCreateBucketOperator(
        task_id="create_bucket2", bucket_name=BUCKET_2, project_id=PROJECT_ID
    )

    list_buckets = GCSListObjectsOperator(
        task_id="list_buckets", bucket=BUCKET_1
    )

    list_buckets_result = BashOperator(
        task_id="list_buckets_result",
        bash_command="echo \"{{ task_instance.xcom_pull('list_buckets') }}\"",
    )

示例#3

文件： example_facebook_ads_to_gcs.py 项目： youngyjd/incubator-airflow

    AdsInsights.Field.clicks,
    AdsInsights.Field.impressions,
]
PARAMETERS = {'level': 'ad', 'date_preset': 'yesterday'}
# [END howto_FB_ADS_variables]

with models.DAG(
    "example_facebook_ads_to_gcs",
    schedule_interval='@once',  # Override to match your needs
    start_date=datetime(2021, 1, 1),
    catchup=False,
) as dag:

    create_bucket = GCSCreateBucketOperator(
        task_id="create_bucket",
        bucket_name=GCS_BUCKET,
        project_id=GCP_PROJECT_ID,
    )

    create_dataset = BigQueryCreateEmptyDatasetOperator(
        task_id="create_dataset",
        dataset_id=DATASET_NAME,
    )

    create_table = BigQueryCreateEmptyTableOperator(
        task_id="create_table",
        dataset_id=DATASET_NAME,
        table_id=TABLE_NAME,
        schema_fields=[
            {'name': 'campaign_name', 'type': 'STRING', 'mode': 'NULLABLE'},
            {'name': 'campaign_id', 'type': 'STRING', 'mode': 'NULLABLE'},

示例#4

BUCKET_FILE_LOCATION = PATH_TO_UPLOAD_FILE.rpartition("/")[-1]

with models.DAG(
        "example_gcs",
        start_date=START_DATE,
        catchup=False,
        schedule_interval='@once',
        tags=['example'],
) as dag:
    create_bucket1 = GCSCreateBucketOperator(
        task_id="create_bucket1",
        bucket_name=BUCKET_1,
        project_id=PROJECT_ID,
        resource={
            "iamConfiguration": {
                "uniformBucketLevelAccess": {
                    "enabled": False,
                },
            },
        },
    )

    create_bucket2 = GCSCreateBucketOperator(task_id="create_bucket2",
                                             bucket_name=BUCKET_2,
                                             project_id=PROJECT_ID)

    list_buckets = GCSListObjectsOperator(task_id="list_buckets",
                                          bucket=BUCKET_1)

    list_buckets_result = BashOperator(
        task_id="list_buckets_result",

示例#5

文件： example_dag.py 项目： dandhlee/testrepo-githubapp

from airflow.providers.google.cloud.operators.gcs import GCSListObjectsOperator
from airflow.providers.google.cloud.operators.gcs import GCSDeleteBucketOperator
from airflow import models
import uuid
import os
from airflow.utils.dates import days_ago
from airflow.utils.state import State
UUID = uuid.uuid4()
PROJECT_ID = os.environ.get("GOOGLE_CLOUD_PROJECT", "leah-playground")
BUCKET_NAME = f"leah-{UUID}"

with models.DAG(
    "example_gcs",
    start_date=days_ago(1),
    schedule_interval=None,
) as dag:
    create_bucket = GCSCreateBucketOperator(task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID)

    list_objects = GCSListObjectsOperator(task_id="list_objects", bucket=BUCKET_NAME)
    list_buckets_result = BashOperator(
        task_id="list_buckets_result",
        bash_command="echo \"{{ task_instance.xcom_pull('list_objects') }}\"",
    )
    delete_bucket = GCSDeleteBucketOperator(task_id="delete_bucket", bucket_name=BUCKET_NAME)

    create_bucket >> list_objects >> delete_bucket


if __name__ == "__main__":
    dag.clear(dag_run_state=State.NONE)
    dag.run()

示例#6

from airflow.utils.dates import days_ago
from airflow.utils.state import State

# Assumes existence of Airflow Variable set to name of GCP Project
PROJECT_ID = models.Variable.get("gcp_project")

with models.DAG(
        "example_gcs",
        start_date=days_ago(1),
        schedule_interval=None,
) as dag:
    generate_uuid = PythonOperator(task_id='generate_uuid',
                                   python_callable=lambda: str(uuid.uuid4()))
    create_bucket = GCSCreateBucketOperator(
        task_id="create_bucket",
        bucket_name="{{ task_instance.xcom_pull('generate_uuid') }}",
        project_id=PROJECT_ID)
    list_objects = GCSListObjectsOperator(
        task_id="list_objects",
        bucket="{{ task_instance.xcom_pull('generate_uuid') }}")
    list_buckets_result = BashOperator(
        task_id="list_buckets_result",
        bash_command="echo \"{{ task_instance.xcom_pull('list_objects') }}\"",
    )
    delete_bucket = GCSDeleteBucketOperator(
        task_id="delete_bucket",
        bucket_name="{{ task_instance.xcom_pull('generate_uuid') }}")

    generate_uuid >> create_bucket >> list_objects >> list_buckets_result >> delete_bucket

if __name__ == "__main__":