def events_blobs_generator(self) -> Generator[blob.Blob, None, None]: """Generates blobs of retriable failed events stored in monitoring. Retrieves all retriable failed events (of same dag_name and location) from the monitoring table back until the last retry or the beginning of the table if no previous retries found. A retry entity will be logged in monitoring table if is_retry is True. Yields: A blob object containing events from a page with length of _DEFAULT_PAGE_SIZE from the monitoring table. """ sql = ( 'SELECT `info` ' f'FROM `{self.dataset_id}`.`{self.table_id}` ' 'WHERE `dag_name`=%(dag_name)s ' ' AND `location`=%(location)s AND `type_id`>9 ' ' AND `type_id`<50 ' ' AND `timestamp`>(' ' SELECT IFNULL(`max_timestamp`, CAST("2020-1-1" AS TIMESTAMP)) ' ' FROM (SELECT max(`timestamp`) AS `max_timestamp`' f' FROM `{self.dataset_id}`.`{self.table_id}` ' ' WHERE `type_id`=%(type_id)s ' ' AND `dag_name`=%(dag_name)s ' ' AND `location`=%(location)s ' ' HAVING MAX(`timestamp`)=MAX(`timestamp`) ' ' UNION ALL ' ' SELECT NULL ' ' ORDER BY `max_timestamp` DESC ' ' LIMIT 1)' ')') bq_cursor = self.get_conn().cursor() bq_cursor.execute( sql, { 'dag_name': self.dag_name, 'location': self.input_location, 'type_id': MonitoringEntityMap.BLOB.value }) if self.enable_monitoring: self.store_retry(dag_name=self.dag_name, location=self.input_location) i = 0 events = [] row = bq_cursor.fetchone() while row is not None: events.append(json.loads(row[0])) i += 1 if i == _DEFAULT_PAGE_SIZE: yield blob.Blob(events, self.url) i = 0 events = [] row = bq_cursor.fetchone() if events: yield blob.Blob(events, self.url)
def test_execute_appends_empty_reports_when_no_events_to_send(self): blb = blob.Blob(events=[], location='blob') (self.dc_operator.input_hook.events_blobs_generator.return_value ) = fake_events_generator([blb] * 2) self.dc_operator.output_hook.send_events.return_value = blob.Blob( events=[], location='') reports = self.dc_operator.execute({}) self.assertListEqual(reports, [[], []])
def test_ads_cm_hook_send_events_crm_id_missing_user_id(self): """Test hook send_events fail due to missing crm id.""" hook = self.create_ads_cm_hook(upload_key_type='CRM_ID') blb = blob.Blob(events=[{}], location='') hook.send_events(blb) hook.get_user_list_id.assert_not_called()
def payload_validation_case(self, ga4_error_value, field_path=None, description=''): validation_messages = { 'validationMessages': [{ 'fieldPath': field_path, 'description': description }] } if not field_path: validation_messages['validationMessages'][0][ 'fieldPath'] = field_path blb = blob.Blob(events=[self.test_event], location='') with mock.patch('requests.post') as mock_resp: mock_resp.return_value = mock.MagicMock() mock_resp.return_value.status_code = 200 mock_resp.return_value.json.return_value = validation_messages blb = self.test_gtag_hook.send_events(blb) self.assertEqual(len(blb.events), 1) self.assertEqual(len(blb.failed_events), 1) for failed_event in blb.failed_events: self.assertEqual(failed_event[2], ga4_error_value)
def test_ads_cm_hook_send_events_contact_empty_event(self): """Test hook send_events fail due to payload being empty.""" hook = self.create_ads_cm_hook() blb = blob.Blob(events=[{}], location='') hook.send_events(blb) hook.get_user_list_id.assert_not_called()
def test_init(self): blb = blob.Blob([{'': ''}], 'Location', 0) self.assertTupleEqual((blb.events, blb.location, blb.position, blb.num_rows, blb.failed_events, blb.reports), ([{ '': '' }], 'Location', 0, 1, [], []))
def test_ads_cm_hook_send_events_contact_info_bad_phone_number(self): """Test hook send_events fail due to incorrect phone number format.""" hook = self.create_ads_cm_hook() self.contact_info_event_email['hashedPhoneNumber'] = 'bad phone number' blb = blob.Blob(events=[self.contact_info_event_email], location='') hook.send_events(blb) hook.get_user_list_id.assert_not_called()
def _execute_and_assert_num_of_failed_event(self, events, num_failed_events): """Wraps calling send_event and assertion to avoid duplication.""" blb = blob.Blob(events=events, location='') blb = self.cm_hook.send_events(blb) self.assertEqual(len(blb.failed_events), num_failed_events)
def _execute_and_assert_one_failed_event(self, events): """Wraps calling send_event and assertion to avoid duplication.""" response = _create_api_response([_SUCCESS]) self.mock_service.mutate.return_value = response blb = blob.Blob(events=events, location='') blb = self.test_hook.send_events(blb) self.assertEqual(len(blb.failed_events), 1)
def test_execute_appends_reports_after_sending_events(self): (self.dc_operator.input_hook.events_blobs_generator.return_value ) = fake_events_generator([self.blob] * 2) (self.dc_operator.output_hook.send_events.return_value) = blob.Blob( events=[], location='', reports=([0], [1])) reports = self.dc_operator.execute({}) self.assertListEqual(reports, [([0], [1]), ([0], [1])])
def test_ga_hook_send_events_utf8_event_batching(self): with mock.patch.object(self.test_hook, 'send_hit') as patched_send_hook: events = list(self.utf8_event for x in range(20)) blb = blob.Blob(events=events, location='') self.test_hook.send_events(blb) # at 4K each, we can fit 4 in each batch self.assertEqual(patched_send_hook.call_count, 5)
def test_ga4_hook_send_event_with_illegal_json(self): test_event = {'id': 1, 'payload': '{'} blb = blob.Blob(events=[test_event], location='') blb = self.test_gtag_hook.send_events(blb) self.assertEqual(len(blb.failed_events), 1) self.assertEqual( blb.failed_events[0][2], errors.ErrorNameIDMap.GA4_HOOK_ERROR_INVALID_JSON_STRUCTURE.value)
def test_ads_cm_hook_send_events_mobile_advertising_no_id(self): """Test hook send_events fail due to missing mobile advertising id.""" hook = self.create_ads_cm_hook(upload_key_type='MOBILE_ADVERTISING_ID') mobile_id_event = {} blb = blob.Blob(events=[mobile_id_event], location='') hook.send_events(blb) hook.get_user_list_id.assert_not_called()
def test_ads_cm_hook_send_events_crm_id(self): """Test hook send_events success with crm id.""" hook = self.create_ads_cm_hook(upload_key_type='CRM_ID') blb = blob.Blob(events=[self.crm_id_event], location='') blb = hook.send_events(blb) self.assertListEqual([], blb.failed_events) hook.add_members_to_user_list.assert_called()
def test_ads_cm_hook_send_events_mobile_advertising_good_id(self): """Test hook send_events success with mobile advertising id.""" hook = self.create_ads_cm_hook(upload_key_type='MOBILE_ADVERTISING_ID') blb = blob.Blob(events=[self.mobile_id_event], location='') blb = hook.send_events(blb) self.assertListEqual([], blb.failed_events) hook.add_members_to_user_list.assert_called()
def test_ga4_hook_send_event_with_retriable_http_error(self): blb = blob.Blob(events=[self.test_event], location='') with mock.patch('requests.post') as mock_resp: mock_resp.side_effect = requests.ConnectionError blb = self.test_gtag_hook.send_events(blb) self.assertEqual(len(blb.failed_events), 1) self.assertEqual( blb.failed_events[0][2], errors.ErrorNameIDMap. RETRIABLE_GA4_HOOK_ERROR_HTTP_ERROR.value)
def test_ads_cm_hook_send_events_contact_info_email_only(self): """Test hook send_events success with email only payload.""" hook = self.create_ads_cm_hook() blb = blob.Blob(events=[self.contact_info_event_email], location='') blb = hook.send_events(blb) self.assertListEqual([], blb.failed_events) hook.add_members_to_user_list.assert_called()
def test_execute_returns_none_if_return_report_is_false(self): (self.dc_operator_no_report.input_hook.events_blobs_generator. return_value) = fake_events_generator([self.blob]) (self.dc_operator_no_report.output_hook.send_events.return_value ) = blob.Blob(events=[], location='', reports=([0, 1], [])) result = self.dc_operator_no_report.execute({}) self.assertIsNone(result)
def test_execute_when_monitoring_is_disabled(self): (self.dc_operator.input_hook.events_blobs_generator.return_value ) = fake_events_generator([self.blob] * 2) (self.dc_operator.output_hook.send_events.return_value) = blob.Blob( events=[], location='', reports=([0], [1])) self.dc_operator_disable_monitoring.execute({'test': 100}) self.mock_monitoring_hook.return_value.store_blob.assert_not_called() self.mock_monitoring_hook.return_value.store_events.assert_not_called()
def test_upload_failed_due_to_authentication_errors(self): """Test Authenticating related error occurs.""" events = [copy.deepcopy(_event_test_conversion)] self.mock_service.mutate.side_effect = RefreshError( 'invalid_client: Unauthorized') blb = blob.Blob(events=events, location='') blb = self.test_hook.send_events(blb) self.assertEqual(len(blb.failed_events), 1)
def test_ads_cm_hook_send_events_create_new_list_is_false(self): """Test hook send_events fail due to create_list incorrect value.""" hook = self.create_ads_cm_hook(create_list=True) hook.get_user_list_id.side_effect = ( errors.DataOutConnectorValueError()) blb = blob.Blob(events=[self.contact_info_event_email], location='') hook = self.create_ads_cm_hook(create_list=False) hook.send_events(blb) hook.create_user_list.assert_not_called()
def test_non_retriable_error_occurs(self): """Test non retriable error occurs and retry is not triggered.""" events = [copy.deepcopy(_event_test_conversion)] * 2 response = _create_api_response([_NON_RETRIABLE_ERR, _SUCCESS]) self.mock_service.mutate.return_value = response blb = blob.Blob(events=events, location='') blb = self.test_hook.send_events(blb) self.assertEqual(self.mock_service.mutate.call_count, 1) self.assertEqual(len(blb.failed_events), 1)
def test_ads_cm_hook_send_events_contact_info_add_members_to_list(self): """Test hook send_events success with contact info payload.""" hook = self.create_ads_cm_hook(create_list=True) hook.add_members_to_user_list.side_effect = ( errors.DataOutConnectorSendUnsuccessfulError()) blb = blob.Blob(events=[self.contact_info_event_email], location='') hook = self.create_ads_cm_hook() blb = hook.send_events(blb) self.assertListEqual([], blb.failed_events) hook.add_members_to_user_list.assert_called()
def test_ga4_hook_parse_validate_result_failed_with_4xx(self): blb = blob.Blob(events=[self.test_event], location='') with mock.patch('requests.post') as mock_resp: mock_resp.return_value = mock.MagicMock() mock_resp.return_value.status_code = 404 blb = self.test_gtag_hook.send_events(blb) self.assertEqual(len(blb.failed_events), 1) self.assertEqual( blb.failed_events[0][2], errors.ErrorNameIDMap.NON_RETRIABLE_ERROR_EVENT_NOT_SENT.value)
def test_ads_cm_hook_send_events_create_new_list(self): """Test hook send_events successful.""" hook = self.create_ads_cm_hook(create_list=True) hook.get_user_list_id.side_effect = ( errors.DataOutConnectorValueError()) blb = blob.Blob(events=[self.contact_info_event_email], location='') blb = hook.send_events(blb) self.assertListEqual([], blb.failed_events) hook.get_user_list_id.assert_called_once() hook.add_members_to_user_list.assert_called_once()
def test_ads_cm_hook_send_events_contact_info_with_address_info(self): """Test hook send_events success with address info payload.""" hook = self.create_ads_cm_hook() contact_info = {} contact_info.update(self.contact_info_event_email) contact_info.update(self.address_info) blb = blob.Blob(events=[contact_info], location='') blb = hook.send_events(blb) self.assertListEqual([], blb.failed_events) hook.add_members_to_user_list.assert_called()
def test_send_events_with_expected_output(self): sample_data_list = [self.sample_data] * 5 sample_blb = blob.Blob(sample_data_list, 'location') self.adapter.register_uri('POST', self.sample_url, complete_qs=True, json={'attributed': True}) self.test_hook.dry_run = False result_blb = self.test_hook.send_events(sample_blb) self.assertListEqual([], result_blb.failed_events) self.assertEqual(5, len(result_blb.reports))
def test_mixed_retriable_and_non_retriable_error_occurs(self): events = [copy.deepcopy(_event_test_conversion)] * 3 resp1 = _create_api_response( [_RETRIABLE_ERR, _NON_RETRIABLE_ERR, _SUCCESS]) resp2 = _create_api_response([_SUCCESS]) self.mock_service.mutate.side_effect = [resp1, resp2] blb = blob.Blob(events=events, location='') blb = self.test_hook.send_events(blb) self.assertEqual(self.mock_service.mutate.call_count, 2) self.assertEqual(len(blb.failed_events), 1)
def test_ga4_hook_send_event(self): blb = blob.Blob(events=[self.test_event], location='') with mock.patch('requests.post') as mock_resp: mock_resp.return_value = mock.MagicMock() mock_resp.return_value.status_code = 200 mock_resp.return_value.json.return_value = { 'validationMessages': [] } blb = self.test_gtag_hook.send_events(blb) self.assertEqual(len(blb.events), 1) self.assertEqual(len(blb.failed_events), 0)
def test_ga_hook_send_events_small_event_batch_contents(self): with mock.patch.object(self.test_hook, 'send_hit') as patched_send_hook: events = list(self.small_event for x in range(20)) blb = blob.Blob(events=events, location='') self.test_hook.send_events(blb) expected_str = ('tid=UA-12323-4&v=1&t=event&z=1558517072202080&' 'ec=ClientID&ea=test_event_action&el=20190423&ev=1&' 'cid=12345.67890') expected_payload = '\n'.join([expected_str] * 20) patched_send_hook.assert_called_once_with( expected_payload, send_type=ga_hook.SendTypes.BATCH)