def test_deploy_execute(self, mock_hook):
mock_hook.return_value.get_function.side_effect = mock.Mock(
side_effect=HttpError(resp=MOCK_RESP_404, content=b'not found'))
mock_hook.return_value.create_new_function.return_value = True
op = GcfFunctionDeployOperator(
project_id=GCP_PROJECT_ID,
location=GCP_LOCATION,
body=deepcopy(VALID_BODY),
task_id="id"
)
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.return_value.get_function.assert_called_once_with(
'projects/test_project_id/locations/test_region/functions/helloWorld'
)
expected_body = deepcopy(VALID_BODY)
expected_body['labels'] = {
'airflow-version': 'v' + version.replace('.', '-').replace('+', '-')
}
mock_hook.return_value.create_new_function.assert_called_once_with(
project_id='test_project_id',
location='test_region',
body=expected_body
)
def test_valid_source_code_union_field(self, source_code, mock_hook):
mock_hook.return_value.upload_function_zip.return_value = 'https://uploadUrl'
mock_hook.return_value.get_function.side_effect = mock.Mock(
side_effect=HttpError(resp=MOCK_RESP_404, content=b'not found'))
mock_hook.return_value.create_new_function.return_value = True
body = deepcopy(VALID_BODY)
body.pop('sourceUploadUrl', None)
body.pop('sourceArchiveUrl', None)
body.pop('sourceRepository', None)
body.pop('sourceRepositoryUrl', None)
zip_path = source_code.pop('zip_path', None)
body.update(source_code)
op = GcfFunctionDeployOperator(project_id="test_project_id",
location="test_region",
body=body,
task_id="id",
zip_path=zip_path)
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
if zip_path:
mock_hook.return_value.upload_function_zip.assert_called_once_with(
parent='projects/test_project_id/locations/test_region',
zip_path='/path/to/file')
mock_hook.return_value.get_function.assert_called_once_with(
'projects/test_project_id/locations/test_region/functions/helloWorld'
)
mock_hook.return_value.create_new_function.assert_called_once_with(
'projects/test_project_id/locations/test_region', body)
mock_hook.reset_mock()
def test_valid_source_code_union_field(self, source_code, mock_hook):
mock_hook.return_value.upload_function_zip.return_value = 'https://uploadUrl'
mock_hook.return_value.get_function.side_effect = mock.Mock(
side_effect=HttpError(resp=MOCK_RESP_404, content=b'not found'))
mock_hook.return_value.create_new_function.return_value = True
body = deepcopy(VALID_BODY)
body.pop('sourceUploadUrl', None)
body.pop('sourceArchiveUrl', None)
body.pop('sourceRepository', None)
body.pop('sourceRepositoryUrl', None)
zip_path = source_code.pop('zip_path', None)
body.update(source_code)
op = GcfFunctionDeployOperator(
project_id="test_project_id",
location="test_region",
body=body,
task_id="id",
zip_path=zip_path
)
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
if zip_path:
mock_hook.return_value.upload_function_zip.assert_called_once_with(
parent='projects/test_project_id/locations/test_region',
zip_path='/path/to/file'
)
mock_hook.return_value.get_function.assert_called_once_with(
'projects/test_project_id/locations/test_region/functions/helloWorld'
)
mock_hook.return_value.create_new_function.assert_called_once_with(
'projects/test_project_id/locations/test_region',
body
)
mock_hook.reset_mock()
def test_valid_trigger_union_field(self, trigger, mock_hook):
mock_hook.return_value.upload_function_zip.return_value = 'https://uploadUrl'
mock_hook.return_value.get_function.side_effect = mock.Mock(
side_effect=HttpError(resp=MOCK_RESP_404, content=b'not found'))
mock_hook.return_value.create_new_function.return_value = True
body = deepcopy(VALID_BODY)
body.pop('httpsTrigger', None)
body.pop('eventTrigger', None)
body.update(trigger)
op = GcfFunctionDeployOperator(
project_id="test_project_id",
location="test_region",
body=body,
task_id="id",
)
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.return_value.get_function.assert_called_once_with(
'projects/test_project_id/locations/test_region/functions/helloWorld'
)
mock_hook.return_value.create_new_function.assert_called_once_with(
project_id='test_project_id',
location='test_region',
body=body
)
mock_hook.reset_mock()
def test_extra_parameter(self, mock_hook):
mock_hook.return_value.create_new_function.return_value = True
body = deepcopy(VALID_BODY)
body['extra_parameter'] = 'extra'
op = GcfFunctionDeployOperator(project_id="test_project_id",
location="test_region",
body=body,
task_id="id")
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.reset_mock()
def test_valid_labels_field(self, labels, mock_hook):
mock_hook.return_value.list_functions.return_value = []
mock_hook.return_value.create_new_function.return_value = True
body = deepcopy(VALID_BODY)
body['labels'] = labels
op = GcfFunctionDeployOperator(project_id="test_project_id",
location="test_region",
body=body,
task_id="id")
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.reset_mock()
def test_body_empty_or_missing_fields(self, body, message, mock_hook):
mock_hook.return_value.upload_function_zip.return_value = 'https://uploadUrl'
with self.assertRaises(AirflowException) as cm:
op = GcfFunctionDeployOperator(project_id="test_project_id",
location="test_region",
body=body,
task_id="id")
op.execute(None)
err = cm.exception
self.assertIn(message, str(err))
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.reset_mock()
def test_extra_parameter(self, mock_hook):
mock_hook.return_value.create_new_function.return_value = True
body = deepcopy(VALID_BODY)
body['extra_parameter'] = 'extra'
op = GcfFunctionDeployOperator(
project_id="test_project_id",
location="test_region",
body=body,
task_id="id"
)
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.reset_mock()
def test_validation_disabled(self, mock_hook):
mock_hook.return_value.create_new_function.return_value = True
body = {
"name": "function_name",
"some_invalid_body_field": "some_invalid_body_field_value"
}
op = GcfFunctionDeployOperator(project_id="test_project_id",
location="test_region",
body=body,
validate_body=False,
task_id="id")
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.reset_mock()
def test_invalid_field_values(self, key, value, message, mock_hook):
mock_hook.return_value.create_new_function.return_value = True
body = deepcopy(VALID_BODY)
body[key] = value
with self.assertRaises(AirflowException) as cm:
op = GcfFunctionDeployOperator(project_id="test_project_id",
location="test_region",
body=body,
task_id="id")
op.execute(None)
err = cm.exception
self.assertIn(message, str(err))
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.reset_mock()
def test_body_empty_or_missing_fields(self, body, message, mock_hook):
mock_hook.return_value.upload_function_zip.return_value = 'https://uploadUrl'
with self.assertRaises(AirflowException) as cm:
op = GcfFunctionDeployOperator(
project_id="test_project_id",
location="test_region",
body=body,
task_id="id"
)
op.execute(None)
err = cm.exception
self.assertIn(message, str(err))
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.reset_mock()
def test_valid_labels_field(self, labels, mock_hook):
mock_hook.return_value.list_functions.return_value = []
mock_hook.return_value.create_new_function.return_value = True
body = deepcopy(VALID_BODY)
body['labels'] = labels
op = GcfFunctionDeployOperator(
project_id="test_project_id",
location="test_region",
body=body,
task_id="id"
)
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.reset_mock()
def test_empty_project_id_is_ok(self, mock_hook):
operator = GcfFunctionDeployOperator(location="test_region",
body=deepcopy(VALID_BODY),
task_id="id")
operator._hook.get_function.side_effect = \
HttpError(resp=MOCK_RESP_404, content=b'not found')
operator.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
new_body = deepcopy(VALID_BODY)
new_body['labels'] = {
'airflow-version':
'v' + version.replace('.', '-').replace('+', '-')
}
mock_hook.return_value.create_new_function.assert_called_once_with(
project_id=None, location="test_region", body=new_body)
def test_body_validation_simple(self, mock_hook):
mock_hook.return_value.create_new_function.return_value = True
body = deepcopy(VALID_BODY)
body['name'] = ''
with self.assertRaises(AirflowException) as cm:
op = GcfFunctionDeployOperator(project_id="test_project_id",
location="test_region",
body=body,
task_id="id")
op.execute(None)
err = cm.exception
self.assertIn("The body field 'name' of value '' does not match",
str(err))
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.reset_mock()
def test_invalid_field_values(self, key, value, message, mock_hook):
mock_hook.return_value.create_new_function.return_value = True
body = deepcopy(VALID_BODY)
body[key] = value
with self.assertRaises(AirflowException) as cm:
op = GcfFunctionDeployOperator(
project_id="test_project_id",
location="test_region",
body=body,
task_id="id"
)
op.execute(None)
err = cm.exception
self.assertIn(message, str(err))
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.reset_mock()
def test_validation_disabled(self, mock_hook):
mock_hook.return_value.create_new_function.return_value = True
body = {
"name": "function_name",
"some_invalid_body_field": "some_invalid_body_field_value"
}
op = GcfFunctionDeployOperator(
project_id="test_project_id",
location="test_region",
body=body,
validate_body=False,
task_id="id"
)
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.reset_mock()
def test_body_validation_simple(self, mock_hook):
mock_hook.return_value.create_new_function.return_value = True
body = deepcopy(VALID_BODY)
body['name'] = ''
with self.assertRaises(AirflowException) as cm:
op = GcfFunctionDeployOperator(
project_id="test_project_id",
location="test_region",
body=body,
task_id="id"
)
op.execute(None)
err = cm.exception
self.assertIn("The body field 'name' of value '' does not match",
str(err))
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.reset_mock()
def test_empty_project_id_is_ok(self, mock_hook):
operator = GcfFunctionDeployOperator(
location="test_region",
body=deepcopy(VALID_BODY),
task_id="id"
)
operator._hook.get_function.side_effect = \
HttpError(resp=MOCK_RESP_404, content=b'not found')
operator.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
new_body = deepcopy(VALID_BODY)
new_body['labels'] = {
'airflow-version': 'v' + version.replace('.', '-').replace('+', '-')}
mock_hook.return_value.create_new_function.assert_called_once_with(
project_id=None,
location="test_region",
body=new_body)
def test_invalid_trigger_union_field(self, trigger, message, mock_hook):
mock_hook.return_value.upload_function_zip.return_value = 'https://uploadUrl'
body = deepcopy(VALID_BODY)
body.pop('httpsTrigger', None)
body.pop('eventTrigger', None)
body.update(trigger)
with self.assertRaises(AirflowException) as cm:
op = GcfFunctionDeployOperator(
project_id="test_project_id",
location="test_region",
body=body,
task_id="id",
)
op.execute(None)
err = cm.exception
self.assertIn(message, str(err))
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.reset_mock()
def test_invalid_trigger_union_field(self, trigger, message, mock_hook):
mock_hook.return_value.upload_function_zip.return_value = 'https://uploadUrl'
body = deepcopy(VALID_BODY)
body.pop('httpsTrigger', None)
body.pop('eventTrigger', None)
body.update(trigger)
with self.assertRaises(AirflowException) as cm:
op = GcfFunctionDeployOperator(
project_id="test_project_id",
location="test_region",
body=body,
task_id="id",
)
op.execute(None)
err = cm.exception
self.assertIn(message, str(err))
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.reset_mock()
def test_empty_body(self, mock_hook):
with self.assertRaises(AirflowException) as cm:
GcfFunctionDeployOperator(project_id="test_project_id",
location="test_region",
body=None,
task_id="id")
err = cm.exception
self.assertIn("The required parameter 'body' is missing", str(err))
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
def test_deploy_execute(self, mock_hook):
mock_hook.return_value.get_function.side_effect = mock.Mock(
side_effect=HttpError(resp=MOCK_RESP_404, content=b'not found'))
mock_hook.return_value.create_new_function.return_value = True
op = GcfFunctionDeployOperator(project_id=GCP_PROJECT_ID,
location=GCP_LOCATION,
body=deepcopy(VALID_BODY),
task_id="id")
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.return_value.get_function.assert_called_once_with(
'projects/test_project_id/locations/test_region/functions/helloWorld'
)
expected_body = deepcopy(VALID_BODY)
expected_body['labels'] = {
'airflow-version':
'v' + version.replace('.', '-').replace('+', '-')
}
mock_hook.return_value.create_new_function.assert_called_once_with(
'projects/test_project_id/locations/test_region', expected_body)
def test_update_function_if_exists(self, mock_hook):
mock_hook.return_value.get_function.return_value = True
mock_hook.return_value.update_function.return_value = True
op = GcfFunctionDeployOperator(project_id=GCP_PROJECT_ID,
location=GCP_LOCATION,
body=deepcopy(VALID_BODY),
task_id="id")
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.return_value.get_function.assert_called_once_with(
'projects/test_project_id/locations/test_region/functions/helloWorld'
)
expected_body = deepcopy(VALID_BODY)
expected_body['labels'] = {
'airflow-version':
'v' + version.replace('.', '-').replace('+', '-')
}
mock_hook.return_value.update_function.assert_called_once_with(
'projects/test_project_id/locations/test_region/functions/helloWorld',
expected_body, expected_body.keys())
mock_hook.return_value.create_new_function.assert_not_called()
def test_update_function_if_exists(self, mock_hook):
mock_hook.return_value.get_function.return_value = True
mock_hook.return_value.update_function.return_value = True
op = GcfFunctionDeployOperator(
project_id=GCP_PROJECT_ID,
location=GCP_LOCATION,
body=deepcopy(VALID_BODY),
task_id="id"
)
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.return_value.get_function.assert_called_once_with(
'projects/test_project_id/locations/test_region/functions/helloWorld'
)
expected_body = deepcopy(VALID_BODY)
expected_body['labels'] = {
'airflow-version': 'v' + version.replace('.', '-').replace('+', '-')
}
mock_hook.return_value.update_function.assert_called_once_with(
'projects/test_project_id/locations/test_region/functions/helloWorld',
expected_body, expected_body.keys())
mock_hook.return_value.create_new_function.assert_not_called()
def test_valid_trigger_union_field(self, trigger, mock_hook):
mock_hook.return_value.upload_function_zip.return_value = 'https://uploadUrl'
mock_hook.return_value.get_function.side_effect = mock.Mock(
side_effect=HttpError(resp=MOCK_RESP_404, content=b'not found'))
mock_hook.return_value.create_new_function.return_value = True
body = deepcopy(VALID_BODY)
body.pop('httpsTrigger', None)
body.pop('eventTrigger', None)
body.update(trigger)
op = GcfFunctionDeployOperator(
project_id="test_project_id",
location="test_region",
body=body,
task_id="id",
)
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.return_value.get_function.assert_called_once_with(
'projects/test_project_id/locations/test_region/functions/helloWorld'
)
mock_hook.return_value.create_new_function.assert_called_once_with(
project_id='test_project_id', location='test_region', body=body)
mock_hook.reset_mock()
'location': LOCATION,
'body': body,
'validate_body': VALIDATE_BODY
}
# [END howto_operator_gcf_deploy_args]
# [START howto_operator_gcf_deploy_variants]
if SOURCE_ARCHIVE_URL:
body['sourceArchiveUrl'] = SOURCE_ARCHIVE_URL
elif SOURCE_REPOSITORY:
body['sourceRepository'] = {'url': SOURCE_REPOSITORY}
elif ZIP_PATH:
body['sourceUploadUrl'] = ''
default_args['zip_path'] = ZIP_PATH
elif SOURCE_UPLOAD_URL:
body['sourceUploadUrl'] = SOURCE_UPLOAD_URL
else:
raise Exception("Please provide one of the source_code parameters")
# [END howto_operator_gcf_deploy_variants]
with models.DAG('example_gcp_function_deploy_delete',
default_args=default_args,
schedule_interval=datetime.timedelta(days=1)) as dag:
# [START howto_operator_gcf_deploy]
deploy_task = GcfFunctionDeployOperator(task_id="gcf_deploy_task",
name=FUNCTION_NAME)
# [END howto_operator_gcf_deploy]
delete_task = GcfFunctionDeleteOperator(task_id="gcf_delete_task",
name=FUNCTION_NAME)
deploy_task >> delete_task
set_defaults_version = MLEngineSetDefaultVersionOperator(
task_id="set-default-version",
project_id=PROJECT,
model_name=MODEL_NAME,
version_name=VERSION_NAME,
)
function_body = {
"name":
"projects/topic-sentiment-269614/locations/us-east1/functions/analyze-ui",
"entryPoint": "analyze",
"runtime": "python37",
"httpsTrigger": {},
"sourceRepository": {
"url":
"https://source.developers.google.com/projects/topic-sentiment-269614/repos/github_metacreek_springboard/fixed-aliases/production-api/paths/api"
},
"environmentVariables": {
"DOMAIN_LOOKUP_PATH": DOMAIN_LOOKUP_PATH
}
}
deploy_cloud_function = GcfFunctionDeployOperator(task_id='create_function',
project_id=PROJECT,
location=REGION,
body=function_body)
# Dag definition
begin >> create_version >> set_defaults_version >> deploy_cloud_function >> end
# [END howto_operator_gcf_default_args]
# [START howto_operator_gcf_deploy_variants]
if SOURCE_ARCHIVE_URL:
body['sourceArchiveUrl'] = SOURCE_ARCHIVE_URL
elif SOURCE_REPOSITORY:
body['sourceRepository'] = {'url': SOURCE_REPOSITORY}
elif ZIP_PATH:
body['sourceUploadUrl'] = ''
default_args['zip_path'] = ZIP_PATH
elif SOURCE_UPLOAD_URL:
body['sourceUploadUrl'] = SOURCE_UPLOAD_URL
else:
raise Exception("Please provide one of the source_code parameters")
# [END howto_operator_gcf_deploy_variants]
with models.DAG('example_gcp_function_deploy_delete',
default_args=default_args,
schedule_interval=datetime.timedelta(days=1)) as dag:
# [START howto_operator_gcf_deploy]
deploy_task = GcfFunctionDeployOperator(task_id="gcf_deploy_task",
name=FUNCTION_NAME,
project_id=PROJECT_ID,
location=LOCATION,
body=body,
validate_body=VALIDATE_BODY)
# [END howto_operator_gcf_deploy]
delete_task = GcfFunctionDeleteOperator(task_id="gcf_delete_task",
name=FUNCTION_NAME)
deploy_task >> delete_task
if GCF_SOURCE_ARCHIVE_URL:
body['sourceArchiveUrl'] = GCF_SOURCE_ARCHIVE_URL
elif GCF_SOURCE_REPOSITORY:
body['sourceRepository'] = {'url': GCF_SOURCE_REPOSITORY}
elif GCF_ZIP_PATH:
body['sourceUploadUrl'] = ''
default_args['zip_path'] = GCF_ZIP_PATH
elif GCF_SOURCE_UPLOAD_URL:
body['sourceUploadUrl'] = GCF_SOURCE_UPLOAD_URL
else:
raise Exception("Please provide one of the source_code parameters")
# [END howto_operator_gcf_deploy_variants]
with models.DAG(
'example_gcp_function',
default_args=default_args,
schedule_interval=None # Override to match your needs
) as dag:
# [START howto_operator_gcf_deploy]
deploy_task = GcfFunctionDeployOperator(task_id="gcf_deploy_task",
project_id=GCP_PROJECT_ID,
location=GCP_LOCATION,
body=body,
validate_body=GCP_VALIDATE_BODY)
# [END howto_operator_gcf_deploy]
# [START howto_operator_gcf_delete]
delete_task = GcfFunctionDeleteOperator(task_id="gcf_delete_task",
name=FUNCTION_NAME)
# [END howto_operator_gcf_delete]
deploy_task >> delete_task
