def test_load_templated_yaml(self): dag = DAG(dag_id='example_cloudbuild_operator', start_date=TEST_DEFAULT_DATE) with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w+t') as build: build.writelines(""" steps: - name: 'ubuntu' args: ['echo', 'Hello {{ params.name }}!'] """) build.seek(0) body_path = build.name operator = CloudBuildCreateBuildOperator( body=body_path, task_id="task-id", dag=dag, params={'name': 'airflow'} ) operator.prepare_template() ti = TaskInstance(operator, TEST_DEFAULT_DATE) ti.render_templates() expected_body = {'steps': [ {'name': 'ubuntu', 'args': ['echo', 'Hello airflow!'] } ] } self.assertEqual(expected_body, operator.body)
def test_minimal_green_path(self, mock_hook): mock_hook.return_value.create_build.return_value = TEST_CREATE_BODY operator = CloudBuildCreateBuildOperator( body=TEST_CREATE_BODY, project_id=TEST_PROJECT_ID, task_id="task-id" ) result = operator.execute({}) self.assertIs(result, TEST_CREATE_BODY)
def test_storage_source_replace(self, hook_mock): hook_mock.return_value.create_build.return_value = TEST_CREATE_BODY current_body = { # [START howto_operator_gcp_cloud_build_source_gcs_url] "source": {"storageSource": "gs://bucket-name/object-name.tar.gz"}, # [END howto_operator_gcp_cloud_build_source_gcs_url] "steps": [ { "name": "gcr.io/cloud-builders/docker", "args": ["build", "-t", "gcr.io/$PROJECT_ID/docker-image", "."], } ], "images": ["gcr.io/$PROJECT_ID/docker-image"], } operator = CloudBuildCreateBuildOperator( body=current_body, project_id=TEST_PROJECT_ID, task_id="task-id" ) operator.execute({}) expected_result = { # [START howto_operator_gcp_cloud_build_source_gcs_dict] "source": {"storageSource": {"bucket": "bucket-name", "object": "object-name.tar.gz"}}, # [END howto_operator_gcp_cloud_build_source_gcs_dict] "steps": [ { "name": "gcr.io/cloud-builders/docker", "args": ["build", "-t", "gcr.io/$PROJECT_ID/docker-image", "."], } ], "images": ["gcr.io/$PROJECT_ID/docker-image"], } hook_mock.create_build(body=expected_result, project_id=TEST_PROJECT_ID)
def test_repo_source_replace(self, hook_mock): hook_mock.return_value.create_build.return_value = TEST_CREATE_BODY current_body = { # [START howto_operator_gcp_cloud_build_source_repo_url] "source": { "repoSource": "https://source.developers.google.com/p/airflow-project/r/airflow-repo" }, # [END howto_operator_gcp_cloud_build_source_repo_url] "steps": [{ "name": "gcr.io/cloud-builders/docker", "args": ["build", "-t", "gcr.io/$PROJECT_ID/docker-image", "."], }], "images": ["gcr.io/$PROJECT_ID/docker-image"], } operator = CloudBuildCreateBuildOperator(body=current_body, project_id=TEST_PROJECT_ID, task_id="task-id") return_value = operator.execute({}) expected_body = { # [START howto_operator_gcp_cloud_build_source_repo_dict] "source": { "repoSource": { "projectId": "airflow-project", "repoName": "airflow-repo", "branchName": "master", } }, # [END howto_operator_gcp_cloud_build_source_repo_dict] "steps": [{ "name": "gcr.io/cloud-builders/docker", "args": ["build", "-t", "gcr.io/$PROJECT_ID/docker-image", "."], }], "images": ["gcr.io/$PROJECT_ID/docker-image"], } hook_mock.return_value.create_build.assert_called_once_with( body=expected_body, project_id=TEST_PROJECT_ID) self.assertEqual(return_value, TEST_CREATE_BODY)
def test_missing_input(self, body): with self.assertRaisesRegex(AirflowException, "The required parameter 'body' is missing"): CloudBuildCreateBuildOperator(body=body, project_id=TEST_PROJECT_ID, task_id="task-id")
"images": ["gcr.io/$PROJECT_ID/$REPO_NAME"], } # [END howto_operator_create_build_from_repo_body] with models.DAG( "example_gcp_cloud_build", schedule_interval='@once', start_date=START_DATE, catchup=False, tags=["example"], ) as build_dag: # [START howto_operator_create_build_from_storage] create_build_from_storage = CloudBuildCreateBuildOperator( task_id="create_build_from_storage", project_id=GCP_PROJECT_ID, build=create_build_from_storage_body ) # [END howto_operator_create_build_from_storage] # [START howto_operator_create_build_from_storage_result] create_build_from_storage_result = BashOperator( bash_command=f"echo { create_build_from_storage.output['results'] }", task_id="create_build_from_storage_result", ) # [END howto_operator_create_build_from_storage_result] # [START howto_operator_create_build_from_repo] create_build_from_repo = CloudBuildCreateBuildOperator( task_id="create_build_from_repo", project_id=GCP_PROJECT_ID, build=create_build_from_repo_body ) # [END howto_operator_create_build_from_repo]
}], "images": ["gcr.io/$PROJECT_ID/$REPO_NAME"], } # [END howto_operator_create_build_from_repo_body] with models.DAG( "example_gcp_cloud_build", schedule_interval='@once', start_date=START_DATE, catchup=False, tags=["example"], ) as build_dag: # [START howto_operator_create_build_from_storage] create_build_from_storage = CloudBuildCreateBuildOperator( task_id="create_build_from_storage", project_id=GCP_PROJECT_ID, build=create_build_from_storage_body) # [END howto_operator_create_build_from_storage] # [START howto_operator_create_build_from_storage_result] create_build_from_storage_result = BashOperator( bash_command=f"echo { create_build_from_storage.output['results'] }", task_id="create_build_from_storage_result", ) # [END howto_operator_create_build_from_storage_result] # [START howto_operator_create_build_from_repo] create_build_from_repo = CloudBuildCreateBuildOperator( task_id="create_build_from_repo", project_id=GCP_PROJECT_ID, build=create_build_from_repo_body)
bash_command=""" cd {0} python ml_sklearn.py {1} {2} {3} {4} {5} """.format(pathScript, pathEncoder, "IrisClassificacao", "ModeloIris", 2, 0)) [t3] with TaskGroup("validate", tooltip="validate") as validate: t4 = BashOperator(dag=dag, task_id='validate', bash_command=""" cd {0} python validateModel.py {1} {2} """.format(pathScript, "IrisClassificacao", pathModel)) [t4] with TaskGroup("buildImage", tooltip="buildImage") as buildImage: t5 = CloudBuildCreateBuildOperator( dag=dag, task_id='buildImage', project_id='robotic-tide-284315', body='/opt/airflow/dags/deployApi/apiIris/cloudbuild.yaml', gcp_conn_id='my_gcp_connection', api_version='v1', params={'name': 'Airflow'}) [t5] end = DummyOperator(task_id='end') start >> etl >> preProcessing >> model >> validate >> buildImage >> end