def _validate_credentials(self) -> None:
"""Validate credentials.
Raises:
DataOutConnectorValueError: If credential combination does not meet
criteria.
"""
if not self.api_secret:
raise errors.DataOutConnectorValueError(
'Missing api secret. Please check you have api_secret set as a Cloud '
'Composer variable as specified in the TCRM documentation.')
valid_payload_types = (PayloadTypes.FIREBASE.value,
PayloadTypes.GTAG.value)
if self.payload_type not in valid_payload_types:
raise errors.DataOutConnectorValueError(
f'Unsupport payload_type: {self.payload_type}. Supported '
'payload_type is gtag or firebase.')
if (self.payload_type == PayloadTypes.FIREBASE.value
and not self.firebase_app_id):
raise errors.DataOutConnectorValueError(
'Wrong payload_type or missing firebase_app_id. Please make sure '
'firebase_app_id is set when payload_type is firebase.')
if (self.payload_type == PayloadTypes.GTAG.value
and not self.measurement_id):
raise errors.DataOutConnectorValueError(
'Wrong payload_type or missing measurement_id. Please make sure '
'measurement_id is set when payload_type is gtag.')
def _ensure_payload_is_dict(self, event: Dict[str, Any]) -> None:
"""Ensures the event contains key of payload and the value of payload is dict.
When the event is loaded from BigQuery, the value of payload is string,
whereas the value of payload is dict when event is loaded from Google
Storage because the raw data is stored as JSON files.
Args:
event: Event to ensure.
Returns:
The same event as input that is ensured.
"""
if 'payload' not in event:
raise errors.DataOutConnectorValueError(
error_num=errors.ErrorNameIDMap.
GA4_HOOK_ERROR_MISSING_PAYLOAD_IN_EVENT)
payload = event['payload']
# Payload is from Google Cloud Storage.
if isinstance(payload, dict):
return
try:
# Payload is coming from BQ and must be converted to a dictionary
event['payload'] = json.loads(payload)
except json.decoder.JSONDecodeError:
raise errors.DataOutConnectorValueError(
error_num=errors.ErrorNameIDMap.
GA4_HOOK_ERROR_INVALID_JSON_STRUCTURE)
def _parse_validate_result(self, event: Dict[str, Any],
response: requests.Response):
"""Parses the response returned from the debug API.
The response body contains a JSON to indicate the validated result.
For example:
{
"validationMessages": [
{
"fieldPath": "timestamp_micros"
"description": "Measurement timestamp_micros has timestamp....",
"validationCode": "VALUE_INVALID"
}]
}
The fieldPath indicates which part of your payload JSON contains invalid
value, when fieldPath doesn't exist, the fieldPath can be found in
description as well.
Args:
event: The event that contains the index and the payload.
response: The HTTP response from the debug API.
"""
if response.status_code >= 500:
raise errors.DataOutConnectorValueError(
error_num=errors.ErrorNameIDMap.
RETRIABLE_GA4_HOOK_ERROR_HTTP_ERROR)
elif response.status_code != 200:
raise errors.DataOutConnectorValueError(
error_num=errors.ErrorNameIDMap.
NON_RETRIABLE_ERROR_EVENT_NOT_SENT)
try:
validation_result = response.json()
except json.JSONDecodeError:
raise errors.DataOutConnectorValueError(
error_num=errors.ErrorNameIDMap.
RETRIABLE_GA4_HOOK_ERROR_HTTP_ERROR.value)
# Payload is valid: validation messages are only returned if there is a
# problem with the payload.
if not validation_result['validationMessages']:
return
# The validation API only ever returns one message.
message = validation_result['validationMessages'][0]
field_path = message['fieldPath']
description = message['description']
for property_name in _ERROR_TYPES:
if field_path == property_name or property_name in description:
raise errors.DataOutConnectorValueError(
error_num=_ERROR_TYPES[property_name])
# Prevent from losing error message if it is undefined due to API change.
logging.error('id: %s, fieldPath: %s, description: %s', event['id'],
message['fieldPath'], message['description'])
raise errors.DataOutConnectorValueError(
error_num=errors.ErrorNameIDMap.GA4_HOOK_ERROR_INVALID_VALUES)
def _validate_init_params(
self, user_list_name: str,
membership_lifespan: int,
upload_key_type: str,
create_list: bool,
app_id: str
) -> None:
"""Validates user_list_name and membership_lifespan parameters.
Args:
user_list_name: The name of the user list to add members to.
membership_lifespan: Number of days a user's cookie stays.
upload_key_type: The upload key type.
create_list: A flag to enable a new list creation if a list called
user_list_name doesn't exist.
app_id: The mobile app id for creating new user list when the
upload_key_type is MOBILE_ADVERTISING_ID.
Raises:
DataOutConnectorValueError if any init argument is invalid.
"""
if not user_list_name:
raise errors.DataOutConnectorValueError(
'User list name is empty.',
errors.ErrorNameIDMap.ADS_CM_HOOK_ERROR_EMPTY_USER_LIST_NAME)
if membership_lifespan < 0 or (membership_lifespan > 540 and
membership_lifespan != 10000):
raise errors.DataOutConnectorValueError(
('Membership lifespan in days should be between 0 and 540 inclusive '
'or 10000 for no expiration.'),
errors.ErrorNameIDMap.ADS_CM_HOOK_ERROR_INVALID_MEMBERSHIP_LIFESPAN)
if upload_key_type not in (
ads_hook_v2.CUSTOMER_MATCH_UPLOAD_KEY_CONTACT_INFO,
ads_hook_v2.CUSTOMER_MATCH_UPLOAD_KEY_MOBILE_ADVERTISING_ID,
ads_hook_v2.CUSTOMER_MATCH_UPLOAD_KEY_CRM_ID):
raise errors.DataOutConnectorValueError(
('Invalid upload key type. Valid upload key type: CONTACT_INFO, '
'MOBILE_ADVERTISING_ID or CRM_ID'),
errors.ErrorNameIDMap.ADS_CM_HOOK_ERROR_INVALID_UPLOAD_KEY_TYPE)
if (upload_key_type
== ads_hook_v2.CUSTOMER_MATCH_UPLOAD_KEY_MOBILE_ADVERTISING_ID and
create_list and not app_id):
raise errors.DataOutConnectorValueError(
'app_id needs to be specified for '
'MOBILE_ADVERTISING_ID when create_list is True.',
errors.ErrorNameIDMap.ADS_CM_HOOK_ERROR_MISSING_APPID)
def _send_validate_request(self, payload: Dict[str,
Any]) -> requests.Response:
"""Sends the GA4 payload to the debug API for data validating.
By adding the key-value pair
(validationBehavior: ENFORCE_RECOMMENDATIONS), the API will check the
payload thoroughly, this is recommended because the Measurement Protocol
API won't check the data and it fails silently, you might not know what
happened to your data.
Args:
payload: the JSON payload of the GA4 event.
Returns:
The response from the debug API.
"""
validating_payload = dict(payload)
validating_payload['validationBehavior'] = 'ENFORCE_RECOMMENDATIONS'
try:
response = requests.post(self.validate_url,
json=validating_payload)
except requests.ConnectionError:
raise errors.DataOutConnectorValueError(
error_num=errors.ErrorNameIDMap.
RETRIABLE_GA4_HOOK_ERROR_HTTP_ERROR)
return response
def _get_developer_token(self) -> str:
"""Gets developer token from connection configuration.
Returns:
dev_token: Developer token of Google Ads API.
Raises:
DataOutConnectorValueError: If connection is not available or if password
is missing in the connection.
"""
conn = self.get_connection(self.http_conn_id)
if not conn:
raise errors.DataOutConnectorValueError(
'Cannot get connection {id}.'.format(id=self.http_conn_id),
errors.ErrorNameIDMap.
RETRIABLE_ADS_UAC_HOOK_ERROR_FAIL_TO_GET_AIRFLOW_CONNECTION)
if not conn.password:
raise errors.DataOutConnectorValueError(
'Missing dev token. Please check connection {id} and its password.'
.format(id=self.http_conn_id), errors.ErrorNameIDMap.
RETRIABLE_ADS_UAC_HOOK_ERROR_MISSING_DEV_TOKEN)
return conn.password
def _validate_tracking_id(self, tracking_id: str) -> None:
"""Validates tracking matches the common pattern.
The tracking id must comply the specified pattern 'UA-XXXXX-Y' to proceed
the send_hit function.
Args:
tracking_id: GA's property or tracking ID for GA to identify hits.
Raises:
DataOutConnectorValueError: If the tracking id format is invalid.
"""
if not re.match(_GA_TRACKING_ID_REGEX, tracking_id):
raise errors.DataOutConnectorValueError(
'Invalid Tracking ID Format. The expected format is `UA-XXXXX-Y`.',
errors.ErrorNameIDMap.GA_HOOK_ERROR_INVALID_TRACKING_ID_FORMAT)
def _validate_app_conversion_payload(self, payload: Dict[str,
Any]) -> None:
"""Validates payload sent to UAC.
Args:
payload: The payload to be validated before sending to Google Ads UAC.
Raises:
DataOutConnectorValueError: If some value is missing or in wrong format.
"""
for key in _REQUIRED_FIELDS:
if payload.get(key) is None:
raise errors.DataOutConnectorValueError(
"""Missing {key} in payload.""".format(key=key), errors.
ErrorNameIDMap.ADS_UAC_HOOK_ERROR_MISSING_MANDATORY_FIELDS)
if payload.get('app_event_type') not in [
item.value for item in AppEventType
]:
raise errors.DataOutConnectorValueError(
"""Unsupported app event type in
payload. Example: 'first_open', 'session_start', 'in_app_purchase',
'view_item_list', 'view_item', 'view_search_results',
'add_to_cart', 'ecommerce_purchase', 'custom'.""", errors.
ErrorNameIDMap.ADS_UAC_HOOK_ERROR_UNSUPPORTED_APP_EVENT_TYPE)
if (payload.get('app_event_name')
and payload.get('app_event_type') != 'custom'):
raise errors.DataOutConnectorValueError(
"""App event type must be 'custom' when app event name exists.""",
errors.ErrorNameIDMap.ADS_UAC_HOOK_ERROR_WRONG_APP_EVENT_TYPE)
match = _RDID_REGEX.match(payload.get('rdid'))
if not match:
raise errors.DataOutConnectorValueError(
"""Wrong raw device id format in
payload. Should be compatible with RFC4122.""", errors.
ErrorNameIDMap.ADS_UAC_HOOK_ERROR_WRONG_RAW_DEVICE_ID_FORMAT)
if payload.get('id_type') not in [item.value for item in IdType]:
raise errors.DataOutConnectorValueError(
"""Wrong raw device id type in
payload. Example: 'advertisingid', 'idfa'.""", errors.ErrorNameIDMap.
ADS_UAC_HOOK_ERROR_WRONG_RAW_DEVICE_ID_TYPE)
if payload.get('lat') != 0 and payload.get('lat') != 1:
raise errors.DataOutConnectorValueError(
"""Wrong limit-ad-tracking status in payload. Example: 0, 1.""",
errors.ErrorNameIDMap.ADS_UAC_HOOK_ERROR_WRONG_LAT_STATUS)
def _validate_uid_or_cid(self, cid: Optional[str],
uid: Optional[str]) -> None:
"""Validates uid or cid.
Each payload must include cid (client id) or uid (user id) in it; this
function verifies either uid or cid are set.
Args:
cid: Client id to check.
uid: User id to check.
Raises:
DataOutConnectorValueError: If input parameter didn't cover either cid or
uid.
"""
if not cid and not uid:
raise errors.DataOutConnectorValueError(
'Hit must have cid or uid.',
error_num=errors.ErrorNameIDMap.
GA_HOOK_ERROR_MISSING_CID_OR_UID)
def upload_customer_match_user_list(
self,
customer_id: str,
user_list_name: str,
create_new_list: bool,
payloads: List[type_alias.Payload],
upload_key_type: str,
app_id: Optional[str] = None,
membership_life_span_days: int = DEFAULT_MEMBERSHIP_LIFESPAN_DAYS
) -> None:
"""Uploads customer match user list.
Args:
customer_id: The customer id.
user_list_name: The name of the user list.
create_new_list: The flag that indicates if new list should be created if
there is no existing list with given user_list_name.
payloads: The list of data for the customer match user list.
upload_key_type: The type of customer match user list.
app_id: The mobile app id for creating new user list when the
upload_key_type is MOBILE_ADVERTISING_ID.
membership_life_span_days: Number of days a user's cookie stays.
Raises:
DataOutConnectorError raised when errors occurred.
"""
if not user_list_name:
raise errors.DataOutConnectorValueError(
msg='user_list_name is empty.')
# Raises error because create new list requested, but no app id
# provided.
if (create_new_list and
upload_key_type == CUSTOMER_MATCH_UPLOAD_KEY_MOBILE_ADVERTISING_ID and
not app_id):
raise errors.DataOutConnectorValueError(
msg=('When upload_key_type is MOBILE_ADVERTISING_ID, new list '
'cannot be created without an app_id.'))
try:
user_list_resource_name = self._get_user_list_resource_name(
user_list_name, customer_id)
# Raises error because no list exists and create new list not requested.
if not user_list_resource_name and not create_new_list:
raise errors.DataOutConnectorValueError(
msg=(f'Customer match user list: {user_list_name} does not exist.'
' and create_new_list = False'))
if not user_list_resource_name:
user_list_resource_name = self._create_customer_match_user_list(
customer_id, user_list_name,
upload_key_type, app_id, membership_life_span_days)
offline_user_data_job_operations = self._build_user_data_job_operations(
payloads, upload_key_type)
offline_user_data_job_resource_name = self._create_offline_user_data_job(
user_list_resource_name, customer_id)
self.log.info(f'offline_user_data_job_resource_name='
f'{offline_user_data_job_resource_name}')
self._add_users_to_customer_match_user_list(
offline_user_data_job_resource_name, offline_user_data_job_operations)
self._run_offline_user_data_job(offline_user_data_job_resource_name)
self._get_offline_user_data_job_status(
offline_user_data_job_resource_name, customer_id)
except json.JSONDecodeError as json_decode_error:
raise errors.DataOutConnectorError(
msg=str(json_decode_error), error=json_decode_error
) from json_decode_error
except requests.RequestException as request_exception:
self._handle_request_exception(request_exception)