def events_blobs_generator(self) -> Generator[blob.Blob, None, None]:
"""Generates blobs of retriable failed events stored in monitoring.
Retrieves all retriable failed events (of same dag_name and location) from
the monitoring table back until the last retry or the beginning of the table
if no previous retries found.
A retry entity will be logged in monitoring table if is_retry is True.
Yields:
A blob object containing events from a page with length of
_DEFAULT_PAGE_SIZE from the monitoring table.
"""
sql = (
'SELECT `info` '
f'FROM `{self.dataset_id}`.`{self.table_id}` '
'WHERE `dag_name`=%(dag_name)s '
' AND `location`=%(location)s AND `type_id`>9 '
' AND `type_id`<50 '
' AND `timestamp`>('
' SELECT IFNULL(`max_timestamp`, CAST("2020-1-1" AS TIMESTAMP)) '
' FROM (SELECT max(`timestamp`) AS `max_timestamp`'
f' FROM `{self.dataset_id}`.`{self.table_id}` '
' WHERE `type_id`=%(type_id)s '
' AND `dag_name`=%(dag_name)s '
' AND `location`=%(location)s '
' HAVING MAX(`timestamp`)=MAX(`timestamp`) '
' UNION ALL '
' SELECT NULL '
' ORDER BY `max_timestamp` DESC '
' LIMIT 1)'
')')
bq_cursor = self.get_conn().cursor()
bq_cursor.execute(
sql, {
'dag_name': self.dag_name,
'location': self.input_location,
'type_id': MonitoringEntityMap.BLOB.value
})
if self.enable_monitoring:
self.store_retry(dag_name=self.dag_name,
location=self.input_location)
i = 0
events = []
row = bq_cursor.fetchone()
while row is not None:
events.append(json.loads(row[0]))
i += 1
if i == _DEFAULT_PAGE_SIZE:
yield blob.Blob(events, self.url)
i = 0
events = []
row = bq_cursor.fetchone()
if events:
yield blob.Blob(events, self.url)
def test_execute_appends_empty_reports_when_no_events_to_send(self):
blb = blob.Blob(events=[], location='blob')
(self.dc_operator.input_hook.events_blobs_generator.return_value
) = fake_events_generator([blb] * 2)
self.dc_operator.output_hook.send_events.return_value = blob.Blob(
events=[], location='')
reports = self.dc_operator.execute({})
self.assertListEqual(reports, [[], []])
def test_ads_cm_hook_send_events_crm_id_missing_user_id(self):
"""Test hook send_events fail due to missing crm id."""
hook = self.create_ads_cm_hook(upload_key_type='CRM_ID')
blb = blob.Blob(events=[{}], location='')
hook.send_events(blb)
hook.get_user_list_id.assert_not_called()
def payload_validation_case(self,
ga4_error_value,
field_path=None,
description=''):
validation_messages = {
'validationMessages': [{
'fieldPath': field_path,
'description': description
}]
}
if not field_path:
validation_messages['validationMessages'][0][
'fieldPath'] = field_path
blb = blob.Blob(events=[self.test_event], location='')
with mock.patch('requests.post') as mock_resp:
mock_resp.return_value = mock.MagicMock()
mock_resp.return_value.status_code = 200
mock_resp.return_value.json.return_value = validation_messages
blb = self.test_gtag_hook.send_events(blb)
self.assertEqual(len(blb.events), 1)
self.assertEqual(len(blb.failed_events), 1)
for failed_event in blb.failed_events:
self.assertEqual(failed_event[2], ga4_error_value)
def test_ads_cm_hook_send_events_contact_empty_event(self):
"""Test hook send_events fail due to payload being empty."""
hook = self.create_ads_cm_hook()
blb = blob.Blob(events=[{}], location='')
hook.send_events(blb)
hook.get_user_list_id.assert_not_called()
def test_init(self):
blb = blob.Blob([{'': ''}], 'Location', 0)
self.assertTupleEqual((blb.events, blb.location, blb.position,
blb.num_rows, blb.failed_events, blb.reports),
([{
'': ''
}], 'Location', 0, 1, [], []))
def test_ads_cm_hook_send_events_contact_info_bad_phone_number(self):
"""Test hook send_events fail due to incorrect phone number format."""
hook = self.create_ads_cm_hook()
self.contact_info_event_email['hashedPhoneNumber'] = 'bad phone number'
blb = blob.Blob(events=[self.contact_info_event_email], location='')
hook.send_events(blb)
hook.get_user_list_id.assert_not_called()
def _execute_and_assert_num_of_failed_event(self, events,
num_failed_events):
"""Wraps calling send_event and assertion to avoid duplication."""
blb = blob.Blob(events=events, location='')
blb = self.cm_hook.send_events(blb)
self.assertEqual(len(blb.failed_events), num_failed_events)
def _execute_and_assert_one_failed_event(self, events):
"""Wraps calling send_event and assertion to avoid duplication."""
response = _create_api_response([_SUCCESS])
self.mock_service.mutate.return_value = response
blb = blob.Blob(events=events, location='')
blb = self.test_hook.send_events(blb)
self.assertEqual(len(blb.failed_events), 1)
def test_execute_appends_reports_after_sending_events(self):
(self.dc_operator.input_hook.events_blobs_generator.return_value
) = fake_events_generator([self.blob] * 2)
(self.dc_operator.output_hook.send_events.return_value) = blob.Blob(
events=[], location='', reports=([0], [1]))
reports = self.dc_operator.execute({})
self.assertListEqual(reports, [([0], [1]), ([0], [1])])
def test_ga_hook_send_events_utf8_event_batching(self):
with mock.patch.object(self.test_hook, 'send_hit') as patched_send_hook:
events = list(self.utf8_event for x in range(20))
blb = blob.Blob(events=events, location='')
self.test_hook.send_events(blb)
# at 4K each, we can fit 4 in each batch
self.assertEqual(patched_send_hook.call_count, 5)
def test_ga4_hook_send_event_with_illegal_json(self):
test_event = {'id': 1, 'payload': '{'}
blb = blob.Blob(events=[test_event], location='')
blb = self.test_gtag_hook.send_events(blb)
self.assertEqual(len(blb.failed_events), 1)
self.assertEqual(
blb.failed_events[0][2],
errors.ErrorNameIDMap.GA4_HOOK_ERROR_INVALID_JSON_STRUCTURE.value)
def test_ads_cm_hook_send_events_mobile_advertising_no_id(self):
"""Test hook send_events fail due to missing mobile advertising id."""
hook = self.create_ads_cm_hook(upload_key_type='MOBILE_ADVERTISING_ID')
mobile_id_event = {}
blb = blob.Blob(events=[mobile_id_event], location='')
hook.send_events(blb)
hook.get_user_list_id.assert_not_called()
def test_ads_cm_hook_send_events_crm_id(self):
"""Test hook send_events success with crm id."""
hook = self.create_ads_cm_hook(upload_key_type='CRM_ID')
blb = blob.Blob(events=[self.crm_id_event], location='')
blb = hook.send_events(blb)
self.assertListEqual([], blb.failed_events)
hook.add_members_to_user_list.assert_called()
def test_ads_cm_hook_send_events_mobile_advertising_good_id(self):
"""Test hook send_events success with mobile advertising id."""
hook = self.create_ads_cm_hook(upload_key_type='MOBILE_ADVERTISING_ID')
blb = blob.Blob(events=[self.mobile_id_event], location='')
blb = hook.send_events(blb)
self.assertListEqual([], blb.failed_events)
hook.add_members_to_user_list.assert_called()
def test_ga4_hook_send_event_with_retriable_http_error(self):
blb = blob.Blob(events=[self.test_event], location='')
with mock.patch('requests.post') as mock_resp:
mock_resp.side_effect = requests.ConnectionError
blb = self.test_gtag_hook.send_events(blb)
self.assertEqual(len(blb.failed_events), 1)
self.assertEqual(
blb.failed_events[0][2], errors.ErrorNameIDMap.
RETRIABLE_GA4_HOOK_ERROR_HTTP_ERROR.value)
def test_ads_cm_hook_send_events_contact_info_email_only(self):
"""Test hook send_events success with email only payload."""
hook = self.create_ads_cm_hook()
blb = blob.Blob(events=[self.contact_info_event_email], location='')
blb = hook.send_events(blb)
self.assertListEqual([], blb.failed_events)
hook.add_members_to_user_list.assert_called()
def test_execute_returns_none_if_return_report_is_false(self):
(self.dc_operator_no_report.input_hook.events_blobs_generator.
return_value) = fake_events_generator([self.blob])
(self.dc_operator_no_report.output_hook.send_events.return_value
) = blob.Blob(events=[], location='', reports=([0, 1], []))
result = self.dc_operator_no_report.execute({})
self.assertIsNone(result)
def test_execute_when_monitoring_is_disabled(self):
(self.dc_operator.input_hook.events_blobs_generator.return_value
) = fake_events_generator([self.blob] * 2)
(self.dc_operator.output_hook.send_events.return_value) = blob.Blob(
events=[], location='', reports=([0], [1]))
self.dc_operator_disable_monitoring.execute({'test': 100})
self.mock_monitoring_hook.return_value.store_blob.assert_not_called()
self.mock_monitoring_hook.return_value.store_events.assert_not_called()
def test_upload_failed_due_to_authentication_errors(self):
"""Test Authenticating related error occurs."""
events = [copy.deepcopy(_event_test_conversion)]
self.mock_service.mutate.side_effect = RefreshError(
'invalid_client: Unauthorized')
blb = blob.Blob(events=events, location='')
blb = self.test_hook.send_events(blb)
self.assertEqual(len(blb.failed_events), 1)
def test_ads_cm_hook_send_events_create_new_list_is_false(self):
"""Test hook send_events fail due to create_list incorrect value."""
hook = self.create_ads_cm_hook(create_list=True)
hook.get_user_list_id.side_effect = (
errors.DataOutConnectorValueError())
blb = blob.Blob(events=[self.contact_info_event_email], location='')
hook = self.create_ads_cm_hook(create_list=False)
hook.send_events(blb)
hook.create_user_list.assert_not_called()
def test_non_retriable_error_occurs(self):
"""Test non retriable error occurs and retry is not triggered."""
events = [copy.deepcopy(_event_test_conversion)] * 2
response = _create_api_response([_NON_RETRIABLE_ERR, _SUCCESS])
self.mock_service.mutate.return_value = response
blb = blob.Blob(events=events, location='')
blb = self.test_hook.send_events(blb)
self.assertEqual(self.mock_service.mutate.call_count, 1)
self.assertEqual(len(blb.failed_events), 1)
def test_ads_cm_hook_send_events_contact_info_add_members_to_list(self):
"""Test hook send_events success with contact info payload."""
hook = self.create_ads_cm_hook(create_list=True)
hook.add_members_to_user_list.side_effect = (
errors.DataOutConnectorSendUnsuccessfulError())
blb = blob.Blob(events=[self.contact_info_event_email], location='')
hook = self.create_ads_cm_hook()
blb = hook.send_events(blb)
self.assertListEqual([], blb.failed_events)
hook.add_members_to_user_list.assert_called()
def test_ga4_hook_parse_validate_result_failed_with_4xx(self):
blb = blob.Blob(events=[self.test_event], location='')
with mock.patch('requests.post') as mock_resp:
mock_resp.return_value = mock.MagicMock()
mock_resp.return_value.status_code = 404
blb = self.test_gtag_hook.send_events(blb)
self.assertEqual(len(blb.failed_events), 1)
self.assertEqual(
blb.failed_events[0][2],
errors.ErrorNameIDMap.NON_RETRIABLE_ERROR_EVENT_NOT_SENT.value)
def test_ads_cm_hook_send_events_create_new_list(self):
"""Test hook send_events successful."""
hook = self.create_ads_cm_hook(create_list=True)
hook.get_user_list_id.side_effect = (
errors.DataOutConnectorValueError())
blb = blob.Blob(events=[self.contact_info_event_email], location='')
blb = hook.send_events(blb)
self.assertListEqual([], blb.failed_events)
hook.get_user_list_id.assert_called_once()
hook.add_members_to_user_list.assert_called_once()
def test_ads_cm_hook_send_events_contact_info_with_address_info(self):
"""Test hook send_events success with address info payload."""
hook = self.create_ads_cm_hook()
contact_info = {}
contact_info.update(self.contact_info_event_email)
contact_info.update(self.address_info)
blb = blob.Blob(events=[contact_info], location='')
blb = hook.send_events(blb)
self.assertListEqual([], blb.failed_events)
hook.add_members_to_user_list.assert_called()
def test_send_events_with_expected_output(self):
sample_data_list = [self.sample_data] * 5
sample_blb = blob.Blob(sample_data_list, 'location')
self.adapter.register_uri('POST',
self.sample_url,
complete_qs=True,
json={'attributed': True})
self.test_hook.dry_run = False
result_blb = self.test_hook.send_events(sample_blb)
self.assertListEqual([], result_blb.failed_events)
self.assertEqual(5, len(result_blb.reports))
def test_mixed_retriable_and_non_retriable_error_occurs(self):
events = [copy.deepcopy(_event_test_conversion)] * 3
resp1 = _create_api_response(
[_RETRIABLE_ERR, _NON_RETRIABLE_ERR, _SUCCESS])
resp2 = _create_api_response([_SUCCESS])
self.mock_service.mutate.side_effect = [resp1, resp2]
blb = blob.Blob(events=events, location='')
blb = self.test_hook.send_events(blb)
self.assertEqual(self.mock_service.mutate.call_count, 2)
self.assertEqual(len(blb.failed_events), 1)
def test_ga4_hook_send_event(self):
blb = blob.Blob(events=[self.test_event], location='')
with mock.patch('requests.post') as mock_resp:
mock_resp.return_value = mock.MagicMock()
mock_resp.return_value.status_code = 200
mock_resp.return_value.json.return_value = {
'validationMessages': []
}
blb = self.test_gtag_hook.send_events(blb)
self.assertEqual(len(blb.events), 1)
self.assertEqual(len(blb.failed_events), 0)
def test_ga_hook_send_events_small_event_batch_contents(self):
with mock.patch.object(self.test_hook, 'send_hit') as patched_send_hook:
events = list(self.small_event for x in range(20))
blb = blob.Blob(events=events, location='')
self.test_hook.send_events(blb)
expected_str = ('tid=UA-12323-4&v=1&t=event&z=1558517072202080&'
'ec=ClientID&ea=test_event_action&el=20190423&ev=1&'
'cid=12345.67890')
expected_payload = '\n'.join([expected_str] * 20)
patched_send_hook.assert_called_once_with(
expected_payload, send_type=ga_hook.SendTypes.BATCH)
