def test_load_templated_yaml(self):
dag = DAG(dag_id='example_cloudbuild_operator', start_date=TEST_DEFAULT_DATE)
with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w+t') as build:
build.writelines("""
steps:
- name: 'ubuntu'
args: ['echo', 'Hello {{ params.name }}!']
""")
build.seek(0)
body_path = build.name
operator = CloudBuildCreateBuildOperator(
body=body_path,
task_id="task-id", dag=dag,
params={'name': 'airflow'}
)
operator.prepare_template()
ti = TaskInstance(operator, TEST_DEFAULT_DATE)
ti.render_templates()
expected_body = {'steps': [
{'name': 'ubuntu',
'args': ['echo', 'Hello airflow!']
}
]
}
self.assertEqual(expected_body, operator.body)
def test_minimal_green_path(self, mock_hook):
mock_hook.return_value.create_build.return_value = TEST_CREATE_BODY
operator = CloudBuildCreateBuildOperator(
body=TEST_CREATE_BODY, project_id=TEST_PROJECT_ID, task_id="task-id"
)
result = operator.execute({})
self.assertIs(result, TEST_CREATE_BODY)
def test_storage_source_replace(self, hook_mock):
hook_mock.return_value.create_build.return_value = TEST_CREATE_BODY
current_body = {
# [START howto_operator_gcp_cloud_build_source_gcs_url]
"source": {"storageSource": "gs://bucket-name/object-name.tar.gz"},
# [END howto_operator_gcp_cloud_build_source_gcs_url]
"steps": [
{
"name": "gcr.io/cloud-builders/docker",
"args": ["build", "-t", "gcr.io/$PROJECT_ID/docker-image", "."],
}
],
"images": ["gcr.io/$PROJECT_ID/docker-image"],
}
operator = CloudBuildCreateBuildOperator(
body=current_body, project_id=TEST_PROJECT_ID, task_id="task-id"
)
operator.execute({})
expected_result = {
# [START howto_operator_gcp_cloud_build_source_gcs_dict]
"source": {"storageSource": {"bucket": "bucket-name", "object": "object-name.tar.gz"}},
# [END howto_operator_gcp_cloud_build_source_gcs_dict]
"steps": [
{
"name": "gcr.io/cloud-builders/docker",
"args": ["build", "-t", "gcr.io/$PROJECT_ID/docker-image", "."],
}
],
"images": ["gcr.io/$PROJECT_ID/docker-image"],
}
hook_mock.create_build(body=expected_result, project_id=TEST_PROJECT_ID)
def test_repo_source_replace(self, hook_mock):
hook_mock.return_value.create_build.return_value = TEST_CREATE_BODY
current_body = {
# [START howto_operator_gcp_cloud_build_source_repo_url]
"source": {
"repoSource":
"https://source.developers.google.com/p/airflow-project/r/airflow-repo"
},
# [END howto_operator_gcp_cloud_build_source_repo_url]
"steps": [{
"name":
"gcr.io/cloud-builders/docker",
"args":
["build", "-t", "gcr.io/$PROJECT_ID/docker-image", "."],
}],
"images": ["gcr.io/$PROJECT_ID/docker-image"],
}
operator = CloudBuildCreateBuildOperator(body=current_body,
project_id=TEST_PROJECT_ID,
task_id="task-id")
return_value = operator.execute({})
expected_body = {
# [START howto_operator_gcp_cloud_build_source_repo_dict]
"source": {
"repoSource": {
"projectId": "airflow-project",
"repoName": "airflow-repo",
"branchName": "master",
}
},
# [END howto_operator_gcp_cloud_build_source_repo_dict]
"steps": [{
"name":
"gcr.io/cloud-builders/docker",
"args":
["build", "-t", "gcr.io/$PROJECT_ID/docker-image", "."],
}],
"images": ["gcr.io/$PROJECT_ID/docker-image"],
}
hook_mock.return_value.create_build.assert_called_once_with(
body=expected_body, project_id=TEST_PROJECT_ID)
self.assertEqual(return_value, TEST_CREATE_BODY)
def test_missing_input(self, body):
with self.assertRaisesRegex(AirflowException, "The required parameter 'body' is missing"):
CloudBuildCreateBuildOperator(body=body, project_id=TEST_PROJECT_ID, task_id="task-id")
"images": ["gcr.io/$PROJECT_ID/$REPO_NAME"],
}
# [END howto_operator_create_build_from_repo_body]
with models.DAG(
"example_gcp_cloud_build",
schedule_interval='@once',
start_date=START_DATE,
catchup=False,
tags=["example"],
) as build_dag:
# [START howto_operator_create_build_from_storage]
create_build_from_storage = CloudBuildCreateBuildOperator(
task_id="create_build_from_storage", project_id=GCP_PROJECT_ID, build=create_build_from_storage_body
)
# [END howto_operator_create_build_from_storage]
# [START howto_operator_create_build_from_storage_result]
create_build_from_storage_result = BashOperator(
bash_command=f"echo { create_build_from_storage.output['results'] }",
task_id="create_build_from_storage_result",
)
# [END howto_operator_create_build_from_storage_result]
# [START howto_operator_create_build_from_repo]
create_build_from_repo = CloudBuildCreateBuildOperator(
task_id="create_build_from_repo", project_id=GCP_PROJECT_ID, build=create_build_from_repo_body
)
# [END howto_operator_create_build_from_repo]
}],
"images": ["gcr.io/$PROJECT_ID/$REPO_NAME"],
}
# [END howto_operator_create_build_from_repo_body]
with models.DAG(
"example_gcp_cloud_build",
schedule_interval='@once',
start_date=START_DATE,
catchup=False,
tags=["example"],
) as build_dag:
# [START howto_operator_create_build_from_storage]
create_build_from_storage = CloudBuildCreateBuildOperator(
task_id="create_build_from_storage",
project_id=GCP_PROJECT_ID,
build=create_build_from_storage_body)
# [END howto_operator_create_build_from_storage]
# [START howto_operator_create_build_from_storage_result]
create_build_from_storage_result = BashOperator(
bash_command=f"echo { create_build_from_storage.output['results'] }",
task_id="create_build_from_storage_result",
)
# [END howto_operator_create_build_from_storage_result]
# [START howto_operator_create_build_from_repo]
create_build_from_repo = CloudBuildCreateBuildOperator(
task_id="create_build_from_repo",
project_id=GCP_PROJECT_ID,
build=create_build_from_repo_body)
bash_command="""
cd {0}
python ml_sklearn.py {1} {2} {3} {4} {5}
""".format(pathScript, pathEncoder, "IrisClassificacao",
"ModeloIris", 2, 0))
[t3]
with TaskGroup("validate", tooltip="validate") as validate:
t4 = BashOperator(dag=dag,
task_id='validate',
bash_command="""
cd {0}
python validateModel.py {1} {2}
""".format(pathScript, "IrisClassificacao", pathModel))
[t4]
with TaskGroup("buildImage", tooltip="buildImage") as buildImage:
t5 = CloudBuildCreateBuildOperator(
dag=dag,
task_id='buildImage',
project_id='robotic-tide-284315',
body='/opt/airflow/dags/deployApi/apiIris/cloudbuild.yaml',
gcp_conn_id='my_gcp_connection',
api_version='v1',
params={'name': 'Airflow'})
[t5]
end = DummyOperator(task_id='end')
start >> etl >> preProcessing >> model >> validate >> buildImage >> end
