def test_trigger_when_paused(self): # not paused with patch.object(RepeatRecord, 'fire') as mock_fire: with patch.object(RepeatRecord, 'postpone_by') as mock_postpone_fire: # calls process_repeat_record(): self.repeat_record = self.repeater.register( CaseAccessors(self.domain_obj).get_case(CASE_ID)) self.assertEqual(mock_fire.call_count, 1) self.assertEqual(mock_postpone_fire.call_count, 0) # paused self.repeater.pause() # re fetch repeat record repeat_record_id = self.repeat_record.get_id self.repeat_record = RepeatRecord.get(repeat_record_id) process_repeat_record(self.repeat_record) self.assertEqual(mock_fire.call_count, 1) self.assertEqual(mock_postpone_fire.call_count, 1) # resumed self.repeater.resume() # re fetch repeat record repeat_record_id = self.repeat_record.get_id self.repeat_record = RepeatRecord.get(repeat_record_id) process_repeat_record(self.repeat_record) self.assertEqual(mock_fire.call_count, 2) self.assertEqual(mock_postpone_fire.call_count, 1)
def test_trigger_when_paused(self): # not paused with patch.object(RepeatRecord, 'fire') as mock_fire: with patch.object(RepeatRecord, 'postpone_by') as mock_postpone_fire: # calls process_repeat_record(): self.repeat_record = self.repeater.register(CaseAccessors(self.domain).get_case(CASE_ID)) self.assertEqual(mock_fire.call_count, 1) self.assertEqual(mock_postpone_fire.call_count, 0) # paused self.repeater.pause() # re fetch repeat record repeat_record_id = self.repeat_record.get_id self.repeat_record = RepeatRecord.get(repeat_record_id) process_repeat_record(self.repeat_record) self.assertEqual(mock_fire.call_count, 1) self.assertEqual(mock_postpone_fire.call_count, 1) # resumed self.repeater.resume() # re fetch repeat record repeat_record_id = self.repeat_record.get_id self.repeat_record = RepeatRecord.get(repeat_record_id) process_repeat_record(self.repeat_record) self.assertEqual(mock_fire.call_count, 2) self.assertEqual(mock_postpone_fire.call_count, 1)
def test_trigger_when_paused(self): # not paused with patch.object(RepeatRecord, 'fire') as mock_fire: with patch.object(RepeatRecord, 'postpone_by') as mock_postpone_fire: process_repeat_record(self.repeat_record) self.assertEqual(mock_fire.call_count, 1) self.assertEqual(mock_postpone_fire.call_count, 0) # paused self.repeater.pause() # re fetch repeat record repeat_record_id = self.repeat_record.get_id self.repeat_record = RepeatRecord.get(repeat_record_id) process_repeat_record(self.repeat_record) self.assertEqual(mock_fire.call_count, 1) self.assertEqual(mock_postpone_fire.call_count, 1) # resumed self.repeater.resume() # re fetch repeat record repeat_record_id = self.repeat_record.get_id self.repeat_record = RepeatRecord.get(repeat_record_id) process_repeat_record(self.repeat_record) self.assertEqual(mock_fire.call_count, 2) self.assertEqual(mock_postpone_fire.call_count, 1)
def handle(self, *args, **options): domain = options.get('domain') repeater_id = options.get('repeater_id') state = options.get('state') records_file_path = options.get('records_file_path') if records_file_path: self._load_record_ids_from_file(records_file_path) records = self.record_ids record_count = len(records) elif domain and repeater_id: records = iter_repeat_records_by_domain(domain, repeater_id=repeater_id, state=state) record_count = get_repeat_record_count(domain, repeater_id=repeater_id, state=state) else: raise CommandError("Insufficient Arguments") for record in with_progress_bar(records, length=record_count): if isinstance(record, str): record_id = record try: record = RepeatRecord.get(record_id) except ResourceNotFound: self.ws.append([record_id, '', 'Not Found']) continue self._add_row(record) file_name = self._save_file(repeater_id, state) print("Report saved in file:{filename}".format(filename=file_name))
def allowed_to_forward(self, user): # if this user is already in the repeater queue don't add another one if queued_payload(user.domain, user.user_id): return False # If this user has already been forwarded without any changes, then # don't send it to BETS again successful_records = user.user_data.get('BETS_user_repeat_record_ids') if successful_records: latest_record_id = successful_records.split(" ")[-1] latest_record = RepeatRecord.get(latest_record_id) previous_payload = latest_record.attempts[-1].info if latest_record.attempts else "{}" try: previous_payload_json = json.loads(previous_payload) current_payload = get_bets_user_json(user.domain, user) del previous_payload_json['user_data']['id_device_number'] del previous_payload_json['user_data']['id_device_body'] del current_payload['user_data']['id_device_number'] del current_payload['user_data']['id_device_body'] if current_payload == previous_payload_json: return False except (TypeError, ValueError): pass return (user.user_data.get('user_level', None) == 'real' and any(self._is_relevant_location(loc) for loc in user.get_sql_locations(self.domain)))
def _validate_record(r, domain): try: payload = RepeatRecord.get(r) except ResourceNotFound: return None if payload.domain != domain: return None return payload
def openmrs_test_fire(request, domain, repeater_id, record_id): repeater = OpenmrsRepeater.get(repeater_id) record = RepeatRecord.get(record_id) assert repeater.domain == domain assert record.domain == domain assert record.repeater_id == repeater.get_id attempt = repeater.fire_for_record(record) return JsonResponse(attempt.to_json())
def _validate_record(r, domain): from corehq.motech.repeaters.models import RepeatRecord try: payload = RepeatRecord.get(r) except ResourceNotFound: return None if payload.domain != domain: return None return payload
def _get_couch_repeat_record(domain, record_id): from corehq.motech.repeaters.models import RepeatRecord try: couch_record = RepeatRecord.get(record_id) except ResourceNotFound: return None if couch_record.domain != domain: return None return couch_record
def get_record_or_404(request, domain, record_id): try: record = RepeatRecord.get(record_id) except ResourceNotFound: raise Http404() if record.domain != domain: raise Http404() return record
def requeue_repeat_record(request, domain): try: record = RepeatRecord.get(request.POST.get('record_id')) except ResourceNotFound: return HttpResponse(status=404) record.requeue() record.save() if record.cancelled: return HttpResponse(status=400) return HttpResponse('OK')
def delete_all_repeat_records(): from .models import RepeatRecord results = RepeatRecord.get_db().view('repeaters/repeat_records', reduce=False).all() for result in results: try: repeat_record = RepeatRecord.get(result['id']) except Exception: pass else: repeat_record.delete()
def test_success(self): repeat_record = self.repeater.register( CaseAccessors(self.domain).get_case(CASE_ID)) repeat_record = RepeatRecord.get(repeat_record.record_id) # Should be marked as successful after a successful run with patch('corehq.motech.repeaters.models.simple_post' ) as mock_simple_post: mock_simple_post.return_value.status_code = 200 repeat_record.fire() self.assertTrue(repeat_record.succeeded)
class RepeaterFailureTest(BaseRepeaterTest): domain = 'repeat-fail' def setUp(self): super().setUp() # Create the case before creating the repeater, so that the # repeater doesn't fire for the case creation. Each test # registers this case, and the repeater will fire then. self.post_xml(self.xform_xml, self.domain) self.connx = ConnectionSettings.objects.create( domain=self.domain, url='case-repeater-url', ) self.repeater = CaseRepeater( domain=self.domain, connection_settings_id=self.connx.id, format='case_json', ) self.repeater.save() def tearDown(self): self.repeater.delete() self.connx.delete() FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain) delete_all_repeat_records() super().tearDown() def test_get_payload_exception(self): repeat_record = self.repeater.register( CommCareCase.objects.get_case(CASE_ID, self.domain)) with self.assertRaises(Exception): with patch.object(CaseRepeater, 'get_payload', side_effect=Exception('Boom!')): repeat_record.fire() self.assertEqual(repeat_record.failure_reason, 'Boom!') self.assertFalse(repeat_record.succeeded) def test_failure(self): case = CommCareCase.objects.get_case(CASE_ID, self.domain) with patch('corehq.motech.repeaters.models.simple_request', side_effect=RequestException('Boom!')): rr = self.repeater.register(case) # calls repeat_record.fire() # Fetch the repeat_record revision that was updated repeat_record = RepeatRecord.get(rr.record_id) self.assertEqual(repeat_record.failure_reason, 'Boom!') self.assertFalse(repeat_record.succeeded)
def allowed_to_forward(self, location): # if this location is already in the repeater queue, don't forward again if queued_payload(location.domain, location.location_id): return False # If this location has already been forwarded without any changes, then # don't send it to BETS again successful_records = location.metadata.get('BETS_location_repeat_record_ids') if successful_records: latest_record_id = successful_records.split(" ")[-1] latest_record = RepeatRecord.get(latest_record_id) previous_payload = latest_record.attempts[-1].info if latest_record.attempts else "{}" try: previous_payload_json = json.loads(previous_payload) current_payload = get_bets_location_json(location) if current_payload == previous_payload_json: return False except (TypeError, ValueError): pass return (location.metadata.get('is_test') != "yes" and location.location_type.code in self.location_types_to_forward)
class RepeaterTest(BaseRepeaterTest): domain = "repeater-test" def setUp(self): super(RepeaterTest, self).setUp() self.case_connx = ConnectionSettings.objects.create( domain=self.domain, url='case-repeater-url', ) self.case_repeater = CaseRepeater( domain=self.domain, connection_settings_id=self.case_connx.id, format='case_json', ) self.case_repeater.save() self.form_connx = ConnectionSettings.objects.create( domain=self.domain, url='form-repeater-url', ) self.form_repeater = FormRepeater( domain=self.domain, connection_settings_id=self.form_connx.id, format='form_json') self.form_repeater.save() self.log = [] with patch('corehq.motech.repeaters.models.simple_request', return_value=MockResponse(status_code=500, reason="Borked")) as mock_fire: self.post_xml(self.xform_xml, self.domain) self.initial_fire_call_count = mock_fire.call_count def tearDown(self): self.case_repeater.delete() self.case_connx.delete() self.form_repeater.delete() self.form_connx.delete() FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain) delete_all_repeat_records() super(RepeaterTest, self).tearDown() def repeat_records(self): return super(RepeaterTest, self).repeat_records(self.domain) # whatever value specified will be doubled since both case and form repeater are active def _create_additional_repeat_records(self, count): for _ in range(count): instance_id = uuid.uuid4().hex xform_xml = XFORM_XML_TEMPLATE.format( "https://www.commcarehq.org/test/repeater/", USER_ID, instance_id, self.case_block) with patch('corehq.motech.repeaters.models.simple_request', return_value=MockResponse(status_code=500, reason="Borked")): self.post_xml(xform_xml, self.domain) def test_skip_device_logs(self): devicelog_xml = XFORM_XML_TEMPLATE.format(DEVICE_LOG_XMLNS, USER_ID, '1234', '') self.post_xml(devicelog_xml, self.domain) for repeat_record in self.repeat_records(): self.assertNotEqual(repeat_record.payload_id, '1234') def test_skip_duplicates(self): """ Ensure that submitting a duplicate form does not create extra RepeatRecords """ self.assertEqual(len(self.repeat_records()), 2) # this form is already submitted during setUp so a second submission should be a duplicate form = self.post_xml(self.xform_xml, self.domain).xform self.assertTrue(form.is_duplicate) self.assertEqual(len(self.repeat_records()), 2) def test_repeater_failed_sends(self): """ This tests records that fail are requeued later """ def now(): return datetime.utcnow() repeat_records = self.repeat_records() self.assertEqual(len(repeat_records), 2) for repeat_record in repeat_records: with patch('corehq.motech.repeaters.models.simple_request', return_value=MockResponse( status_code=404, reason='Not Found')) as mock_request: repeat_record.fire() self.assertEqual(mock_request.call_count, 1) # Enqueued repeat records have next_check incremented by 48 hours next_check_time = now() + timedelta(minutes=60) + timedelta(hours=48) repeat_records = RepeatRecord.all( domain=self.domain, due_before=now() + timedelta(minutes=15), ) self.assertEqual(len(repeat_records), 0) repeat_records = RepeatRecord.all( domain=self.domain, due_before=next_check_time, ) self.assertEqual(len(repeat_records), 2) def test_update_failure_next_check(self): now = datetime.utcnow() record = RepeatRecord( domain=self.domain, repeater_id=self.case_repeater.get_id, next_check=now, ) self.assertIsNone(record.last_checked) attempt = record.make_set_next_try_attempt(None) record.add_attempt(attempt) self.assertTrue(record.last_checked > now) self.assertEqual(record.next_check, record.last_checked + MIN_RETRY_WAIT) def test_repeater_successful_send(self): repeat_records = self.repeat_records() for repeat_record in repeat_records: with patch('corehq.motech.repeaters.models.simple_request') as mock_request, \ patch.object(ConnectionSettings, 'get_auth_manager') as mock_manager: mock_request.return_value.status_code = 200 mock_manager.return_value = 'MockAuthManager' repeat_record.fire() self.assertEqual(mock_request.call_count, 1) mock_request.assert_called_with( self.domain, repeat_record.repeater.get_url(repeat_record), repeat_record.get_payload(), headers=repeat_record.repeater.get_headers(repeat_record), auth_manager='MockAuthManager', verify=repeat_record.repeater.verify, notify_addresses=[], payload_id=repeat_record.payload_id, method="POST", ) # The following is pretty fickle and depends on which of # - corehq.motech.repeaters.signals # - casexml.apps.case.signals # gets loaded first. # This is deterministic but easily affected by minor code changes repeat_records = self.repeat_records() for repeat_record in repeat_records: self.assertEqual(repeat_record.succeeded, True) self.assertEqual(repeat_record.next_check, None) self.assertEqual(len(self.repeat_records()), 0) self.post_xml(self.update_xform_xml, self.domain) self.assertEqual(len(self.repeat_records()), 2) def test_check_repeat_records(self): self.assertEqual(len(self.repeat_records()), 2) self.assertEqual(self.initial_fire_call_count, 2) with patch( 'corehq.motech.repeaters.models.simple_request') as mock_fire: check_repeaters() self.assertEqual(mock_fire.call_count, 0) def test_repeat_record_status_check(self): self.assertEqual(len(self.repeat_records()), 2) # Do not trigger cancelled records for repeat_record in self.repeat_records(): repeat_record.cancelled = True repeat_record.save() with patch( 'corehq.motech.repeaters.models.simple_request') as mock_fire: check_repeaters() self.assertEqual(mock_fire.call_count, 0) # trigger force send records if not cancelled and tries not exhausted for repeat_record in self.repeat_records(): with patch('corehq.motech.repeaters.models.simple_request', return_value=MockResponse(status_code=200, reason='')) as mock_fire: repeat_record.fire(force_send=True) self.assertEqual(mock_fire.call_count, 1) # all records should be in SUCCESS state after force try for repeat_record in self.repeat_records(): self.assertEqual(repeat_record.state, RECORD_SUCCESS_STATE) self.assertEqual(repeat_record.overall_tries, 1) # not trigger records succeeded triggered after cancellation with patch( 'corehq.motech.repeaters.models.simple_request') as mock_fire: check_repeaters() self.assertEqual(mock_fire.call_count, 0) for repeat_record in self.repeat_records(): self.assertEqual(repeat_record.state, RECORD_SUCCESS_STATE) def test_retry_process_repeat_record_locking(self): self.assertEqual(len(self.repeat_records()), 2) with patch('corehq.motech.repeaters.tasks.retry_process_repeat_record' ) as mock_process: check_repeaters() self.assertEqual(mock_process.delay.call_count, 0) for record in self.repeat_records(): # Resetting next_check should allow them to be requeued record.next_check = datetime.utcnow() record.save() with patch('corehq.motech.repeaters.tasks.retry_process_repeat_record' ) as mock_process: check_repeaters() self.assertEqual(mock_process.delay.call_count, 2) def test_automatic_cancel_repeat_record(self): case = CommCareCase.objects.get_case(CASE_ID, self.domain) rr = self.case_repeater.register(case) # Fetch the revision that was updated: repeat_record = RepeatRecord.get(rr.record_id) self.assertEqual(1, repeat_record.overall_tries) with patch('corehq.motech.repeaters.models.simple_request', side_effect=Exception('Boom!')): for __ in range(repeat_record.max_possible_tries - repeat_record.overall_tries): repeat_record.fire() self.assertEqual(True, repeat_record.cancelled) repeat_record.requeue() self.assertEqual(0, repeat_record.overall_tries) self.assertNotEqual(None, repeat_record.next_check)
class TestRepeaterFormat(BaseRepeaterTest): domain = 'test-fmt' @classmethod def setUpClass(cls): super(TestRepeaterFormat, cls).setUpClass() cls.payload = 'some random case' class NewCaseGenerator(BasePayloadGenerator): format_name = 'new_format' format_label = 'XML' deprecated_format_names = ('new_format_alias', ) def get_payload(self, repeat_record, payload_doc): return cls.payload RegisterGenerator.get_collection(CaseRepeater).add_new_format( NewCaseGenerator) cls.new_generator = NewCaseGenerator def setUp(self): super().setUp() self.post_xml(self.xform_xml, self.domain) self.connx = ConnectionSettings.objects.create( domain=self.domain, url='case-repeater-url', ) self.repeater = CaseRepeater( domain=self.domain, connection_settings_id=self.connx.id, format='new_format', ) # SQL Repeater Model restricts format and would raise error on unexpected values self.repeater.save(sync_to_sql=False) def tearDown(self): self.repeater.delete(sync_to_sql=False) self.connx.delete() FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain) delete_all_repeat_records() super().tearDown() def test_new_format_same_name(self): class NewCaseGenerator(BasePayloadGenerator): format_name = 'case_xml' format_label = 'XML' def get_payload(self, repeat_record, payload_doc): return self.payload with self.assertRaises(DuplicateFormatException): RegisterGenerator.get_collection(CaseRepeater).add_new_format( NewCaseGenerator) def test_new_format_second_default(self): class NewCaseGenerator(BasePayloadGenerator): format_name = 'rubbish' format_label = 'XML' def get_payload(self, repeat_record, payload_doc): return self.payload with self.assertRaises(DuplicateFormatException): RegisterGenerator.get_collection(CaseRepeater).add_new_format( NewCaseGenerator, is_default=True) def test_new_format_payload(self): case = CommCareCase.objects.get_case(CASE_ID, self.domain) with patch('corehq.motech.repeaters.models.simple_request') as mock_request, \ patch.object(ConnectionSettings, 'get_auth_manager') as mock_manager: mock_request.return_value.status_code = 200 mock_manager.return_value = 'MockAuthManager' rr = self.repeater.register(case) repeat_record = RepeatRecord.get(rr.record_id) headers = self.repeater.get_headers(repeat_record) mock_request.assert_called_with( self.domain, self.connx.url, self.payload, auth_manager='MockAuthManager', headers=headers, notify_addresses=[], payload_id='ABC123CASEID', verify=self.repeater.verify, method="POST", )
with patch('corehq.motech.repeaters.models.simple_request', side_effect=RequestException('Boom!')): rr = self.repeater.register(case) # calls repeat_record.fire() # Fetch the repeat_record revision that was updated repeat_record = RepeatRecord.get(rr.record_id) self.assertEqual(repeat_record.failure_reason, 'Boom!') self.assertFalse(repeat_record.succeeded) def test_unexpected_failure(self): case = CommCareCase.objects.get_case(CASE_ID, self.domain) with patch('corehq.motech.repeaters.models.simple_request', side_effect=Exception('Boom!')): rr = self.repeater.register(case) repeat_record = RepeatRecord.get(rr.record_id) self.assertEqual(repeat_record.failure_reason, 'Internal Server Error') self.assertFalse(repeat_record.succeeded) def test_success(self): case = CommCareCase.objects.get_case(CASE_ID, self.domain) # Should be marked as successful after a successful run with patch('corehq.motech.repeaters.models.simple_request' ) as mock_simple_post: mock_simple_post.return_value.status_code = 200 rr = self.repeater.register(case) repeat_record = RepeatRecord.get(rr.record_id) self.assertTrue(repeat_record.succeeded)