def events_blobs_generator(self) -> Generator[blob.Blob, None, None]:
"""Generates blobs of retriable failed events stored in monitoring.
Retrieves all retriable failed events (of same dag_name and location) from
the monitoring table back until the last retry or the beginning of the table
if no previous retries found.
A retry entity will be logged in monitoring table if is_retry is True.
Yields:
A blob object containing events from a page with length of
_DEFAULT_PAGE_SIZE from the monitoring table.
"""
sql = (
'SELECT `info` '
f'FROM `{self.dataset_id}`.`{self.table_id}` '
'WHERE `dag_name`=%(dag_name)s '
' AND `location`=%(location)s AND `type_id`>9 '
' AND `type_id`<50 '
' AND `timestamp`>('
' SELECT IFNULL(`max_timestamp`, CAST("2020-1-1" AS TIMESTAMP)) '
' FROM (SELECT max(`timestamp`) AS `max_timestamp`'
f' FROM `{self.dataset_id}`.`{self.table_id}` '
' WHERE `type_id`=%(type_id)s '
' AND `dag_name`=%(dag_name)s '
' AND `location`=%(location)s '
' HAVING MAX(`timestamp`)=MAX(`timestamp`) '
' UNION ALL '
' SELECT NULL '
' ORDER BY `max_timestamp` DESC '
' LIMIT 1)'
')')
bq_cursor = self.get_conn().cursor()
bq_cursor.execute(
sql, {
'dag_name': self.dag_name,
'location': self.input_location,
'type_id': MonitoringEntityMap.BLOB.value
})
if self.enable_monitoring:
self.store_retry(dag_name=self.dag_name,
location=self.input_location)
i = 0
events = []
row = bq_cursor.fetchone()
while row is not None:
events.append(json.loads(row[0]))
i += 1
if i == _DEFAULT_PAGE_SIZE:
yield blob.Blob(events, self.url)
i = 0
events = []
row = bq_cursor.fetchone()
if events:
yield blob.Blob(events, self.url)
def test_execute_appends_empty_reports_when_no_events_to_send(self):
blb = blob.Blob(events=[], location='blob')
(self.dc_operator.input_hook.events_blobs_generator.
return_value) = fake_events_generator([blb] * 2)
self.dc_operator.output_hook.send_events.return_value = blob.Blob(
events=[], location='')
reports = self.dc_operator.execute({})
self.assertListEqual(reports, [[], []])
def test_ads_cm_hook_send_events_contact_empty_event(self):
"""Test hook send_events fail due to payload being empty."""
hook = self.create_ads_cm_hook()
blb = blob.Blob(events=[{}], location='')
hook.send_events(blb)
hook.get_user_list_id.assert_not_called()
def test_ads_cm_hook_send_events_crm_id_missing_user_id(self):
"""Test hook send_events fail due to missing crm id."""
hook = self.create_ads_cm_hook(upload_key_type='CRM_ID')
blb = blob.Blob(events=[{}], location='')
hook.send_events(blb)
hook.get_user_list_id.assert_not_called()
def test_ads_cm_hook_send_events_contact_info_bad_phone_number(self):
"""Test hook send_events fail due to incorrect phone number format."""
hook = self.create_ads_cm_hook()
self.contact_info_event_email['hashedPhoneNumber'] = 'bad phone number'
blb = blob.Blob(events=[self.contact_info_event_email], location='')
hook.send_events(blb)
hook.get_user_list_id.assert_not_called()
def _execute_and_assert_num_of_failed_event(self, events,
num_failed_events):
"""Wraps calling send_event and assertion to avoid duplication."""
blb = blob.Blob(events=events, location='')
blb = self.cm_hook.send_events(blb)
self.assertEqual(len(blb.failed_events), num_failed_events)
def test_ads_cm_hook_send_events_contact_info_email_only(self):
"""Test hook send_events success with email only payload."""
hook = self.create_ads_cm_hook()
blb = blob.Blob(events=[self.contact_info_event_email], location='')
blb = hook.send_events(blb)
self.assertListEqual([], blb.failed_events)
hook.add_members_to_user_list.assert_called()
def test_execute_returns_none_if_return_report_is_false(self):
(self.dc_operator_no_report.input_hook.events_blobs_generator.
return_value) = fake_events_generator([self.blob])
(self.dc_operator_no_report.output_hook.send_events.
return_value) = blob.Blob(events=[], location='', reports=([0, 1], []))
result = self.dc_operator_no_report.execute({})
self.assertIsNone(result)
def _execute_and_assert_one_failed_event(self, events):
"""Wraps calling send_event and assertion to avoid duplication."""
response = _create_api_response([_SUCCESS])
self.mock_service.mutate.return_value = response
blb = blob.Blob(events=events, location='')
blb = self.test_hook.send_events(blb)
self.assertEqual(len(blb.failed_events), 1)
def test_ads_cm_hook_send_events_mobile_advertising_good_id(self):
"""Test hook send_events success with mobile advertising id."""
hook = self.create_ads_cm_hook(upload_key_type='MOBILE_ADVERTISING_ID')
blb = blob.Blob(events=[self.mobile_id_event], location='')
blb = hook.send_events(blb)
self.assertListEqual([], blb.failed_events)
hook.add_members_to_user_list.assert_called()
def test_execute_appends_reports_after_sending_events(self):
(self.dc_operator.input_hook.events_blobs_generator.
return_value) = fake_events_generator([self.blob] * 2)
(self.dc_operator.output_hook.send_events.
return_value) = blob.Blob(events=[], location='', reports=([0], [1]))
reports = self.dc_operator.execute({})
self.assertListEqual(reports, [([0], [1]), ([0], [1])])
def test_ads_cm_hook_send_events_mobile_advertising_no_id(self):
"""Test hook send_events fail due to missing mobile advertising id."""
hook = self.create_ads_cm_hook(upload_key_type='MOBILE_ADVERTISING_ID')
mobile_id_event = {}
blb = blob.Blob(events=[mobile_id_event], location='')
hook.send_events(blb)
hook.get_user_list_id.assert_not_called()
def test_ads_cm_hook_send_events_crm_id(self):
"""Test hook send_events success with crm id."""
hook = self.create_ads_cm_hook(upload_key_type='CRM_ID')
blb = blob.Blob(events=[self.crm_id_event], location='')
blb = hook.send_events(blb)
self.assertListEqual([], blb.failed_events)
hook.add_members_to_user_list.assert_called()
def test_upload_failed_due_to_authentication_errors(self):
"""Test Authenticating related error occurs."""
events = [copy.deepcopy(_event_test_conversion)]
self.mock_service.mutate.side_effect = RefreshError(
'invalid_client: Unauthorized')
blb = blob.Blob(events=events, location='')
blb = self.test_hook.send_events(blb)
self.assertEqual(len(blb.failed_events), 1)
def test_init(self):
blb = blob.Blob([{'': ''}], 'Location', 0)
self.assertTupleEqual((blb.events,
blb.location,
blb.position,
blb.num_rows,
blb.failed_events,
blb.reports),
([{'': ''}], 'Location', 0, 1, [], []))
def test_ga_hook_send_events_utf8_event_batching(self):
with mock.patch.object(self.test_hook,
'send_hit') as patched_send_hook:
events = list(self.utf8_event for x in range(20))
blb = blob.Blob(events=events, location='')
self.test_hook.send_events(blb)
# at 4K each, we can fit 4 in each batch
self.assertEqual(patched_send_hook.call_count, 5)
def test_execute_when_monitoring_is_disabled(self):
(self.dc_operator.input_hook.events_blobs_generator.
return_value) = fake_events_generator([self.blob] * 2)
(self.dc_operator.output_hook.send_events.
return_value) = blob.Blob(events=[], location='', reports=([0], [1]))
self.dc_operator_disable_monitoring.execute({'test': 100})
self.mock_monitoring_hook.return_value.store_blob.assert_not_called()
self.mock_monitoring_hook.return_value.store_events.assert_not_called()
def test_ads_cm_hook_send_events_create_new_list_is_false(self):
"""Test hook send_events fail due to create_list incorrect value."""
hook = self.create_ads_cm_hook(create_list=True)
hook.get_user_list_id.side_effect = (
errors.DataOutConnectorValueError())
blb = blob.Blob(events=[self.contact_info_event_email], location='')
hook = self.create_ads_cm_hook(create_list=False)
hook.send_events(blb)
hook.create_user_list.assert_not_called()
def test_ads_cm_hook_send_events_create_new_list(self):
"""Test hook send_events successful."""
hook = self.create_ads_cm_hook(create_list=True)
hook.get_user_list_id.side_effect = (
errors.DataOutConnectorValueError())
blb = blob.Blob(events=[self.contact_info_event_email], location='')
blb = hook.send_events(blb)
self.assertListEqual([], blb.failed_events)
hook.get_user_list_id.assert_called_once()
hook.add_members_to_user_list.assert_called_once()
def test_ads_cm_hook_send_events_contact_info_with_address_info(self):
"""Test hook send_events success with address info payload."""
hook = self.create_ads_cm_hook()
contact_info = {}
contact_info.update(self.contact_info_event_email)
contact_info.update(self.address_info)
blb = blob.Blob(events=[contact_info], location='')
blb = hook.send_events(blb)
self.assertListEqual([], blb.failed_events)
hook.add_members_to_user_list.assert_called()
def test_ads_cm_hook_send_events_contact_info_add_members_to_list(self):
"""Test hook send_events success with contact info payload."""
hook = self.create_ads_cm_hook(create_list=True)
hook.add_members_to_user_list.side_effect = (
errors.DataOutConnectorSendUnsuccessfulError())
blb = blob.Blob(events=[self.contact_info_event_email], location='')
hook = self.create_ads_cm_hook()
blb = hook.send_events(blb)
self.assertListEqual([], blb.failed_events)
hook.add_members_to_user_list.assert_called()
def test_non_retriable_error_occurs(self):
"""Test non retriable error occurs and retry is not triggered."""
events = [copy.deepcopy(_event_test_conversion)] * 2
response = _create_api_response([_NON_RETRIABLE_ERR, _SUCCESS])
self.mock_service.mutate.return_value = response
blb = blob.Blob(events=events, location='')
blb = self.test_hook.send_events(blb)
self.assertEqual(self.mock_service.mutate.call_count, 1)
self.assertEqual(len(blb.failed_events), 1)
def test_send_events_with_expected_output(self):
sample_data_list = [self.sample_data] * 5
sample_blb = blob.Blob(sample_data_list, 'location')
self.adapter.register_uri('POST',
self.sample_url,
complete_qs=True,
json={'attributed': True})
self.test_hook.dry_run = False
result_blb = self.test_hook.send_events(sample_blb)
self.assertListEqual([], result_blb.failed_events)
self.assertEqual(5, len(result_blb.reports))
def test_mixed_retriable_and_non_retriable_error_occurs(self):
events = [copy.deepcopy(_event_test_conversion)] * 3
resp1 = _create_api_response(
[_RETRIABLE_ERR, _NON_RETRIABLE_ERR, _SUCCESS])
resp2 = _create_api_response([_SUCCESS])
self.mock_service.mutate.side_effect = [resp1, resp2]
blb = blob.Blob(events=events, location='')
blb = self.test_hook.send_events(blb)
self.assertEqual(self.mock_service.mutate.call_count, 2)
self.assertEqual(len(blb.failed_events), 1)
def test_ads_cm_hook_send_events_contact_info_with_bad_address_info(self):
"""Test hook send_events fail due to incorrect address info format."""
hook = self.create_ads_cm_hook()
contact_info = {}
contact_info.update(self.contact_info_event_email)
del self.address_info['zipCode']
contact_info.update(self.address_info)
blb = blob.Blob(events=[contact_info], location='')
blb = hook.send_events(blb)
self.assertListEqual([], blb.failed_events)
hook.add_members_to_user_list.assert_called_with(
1, [self.contact_info_event_email])
def test_retriable_error_occurs(self):
"""Test retriable error occurs and retry is triggered."""
events = [copy.deepcopy(_event_test_conversion)] * 2
first_response = _create_api_response([_RETRIABLE_ERR, _SUCCESS])
second_response = _create_api_response([_SUCCESS])
self.mock_service.mutate.side_effect = [
first_response, second_response
]
blb = blob.Blob(events=events, location='')
blb = self.test_hook.send_events(blb)
self.assertEqual(self.mock_service.mutate.call_count, 2)
self.assertEqual(len(blb.failed_events), 0)
def test_ga_hook_send_events_small_event_batch_contents(self):
with mock.patch.object(self.test_hook,
'send_hit') as patched_send_hook:
events = list(self.small_event for x in range(20))
blb = blob.Blob(events=events, location='')
self.test_hook.send_events(blb)
expected_str = (
'tid=UA-12323-4&v=1&t=event&z=1558517072202080&'
'ec=ClientID&ea=test_event_action&el=20190423&ev=1&'
'cid=12345.67890')
expected_payload = '\n'.join([expected_str] * 20)
patched_send_hook.assert_called_once_with(
expected_payload, send_type=ga_hook.SendTypes.BATCH)
def test_send_events_with_failed_output(self):
sample_data_list = [self.sample_data] * 5
sample_blb = blob.Blob(sample_data_list, 'location')
self.adapter.register_uri('POST',
self.sample_url,
complete_qs=True,
status_code=400,
reason='Bad Request')
self.test_hook.dry_run = False
result_blb = self.test_hook.send_events(sample_blb)
failed_index = [event[0] for event in result_blb.failed_events]
self.assertCountEqual([0, 1, 2, 3, 4], failed_index)
self.assertEqual(5, len(result_blb.reports))
def _query_results_to_blob(self, query_results: Dict[str, Any],
start_index: int, num_rows: int) -> blob.Blob:
"""Converts query results of BigQuery to event blob.
Args:
query_results: Raw query results.
start_index: Start index of BigQuery table rows.
num_rows: Number of rows processed.
Returns:
blob: Event blob containing event list and status.
"""
if query_results is None:
return None
events = self._query_results_to_maps_list(query_results)
return blob.Blob(events=events,
location=self.url,
position=start_index,
num_rows=num_rows)
def test_ga_hook_send_events_returns_expected(self):
"""Test GoogleAnalyticsHook returns successful and not event indixes."""
events = list(self.small_event for x in range(4))
bad_event = {
'cid': '12345.67890',
'ec': 'ClientID',
'ea': 'test_event_action',
'el': '20190423',
'ev': 1,
'z': '1558517072202080' * 10000
}
events.extend([bad_event] * 2)
expected = [(4, bad_event, 81), (5, bad_event, 81)]
blb = blob.Blob(events=events, location='')
with mock.patch.object(self.test_hook, 'send_hit'):
blb = self.test_hook.send_events(blb)
# 4 successful events and 2 unsuccessful event
self.assertListEqual(blb.failed_events, expected)
