def test_trigger_when_paused(self):
# not paused
with patch.object(RepeatRecord, 'fire') as mock_fire:
with patch.object(RepeatRecord,
'postpone_by') as mock_postpone_fire:
# calls process_repeat_record():
self.repeat_record = self.repeater.register(
CaseAccessors(self.domain_obj).get_case(CASE_ID))
self.assertEqual(mock_fire.call_count, 1)
self.assertEqual(mock_postpone_fire.call_count, 0)
# paused
self.repeater.pause()
# re fetch repeat record
repeat_record_id = self.repeat_record.get_id
self.repeat_record = RepeatRecord.get(repeat_record_id)
process_repeat_record(self.repeat_record)
self.assertEqual(mock_fire.call_count, 1)
self.assertEqual(mock_postpone_fire.call_count, 1)
# resumed
self.repeater.resume()
# re fetch repeat record
repeat_record_id = self.repeat_record.get_id
self.repeat_record = RepeatRecord.get(repeat_record_id)
process_repeat_record(self.repeat_record)
self.assertEqual(mock_fire.call_count, 2)
self.assertEqual(mock_postpone_fire.call_count, 1)
def test_trigger_when_paused(self):
# not paused
with patch.object(RepeatRecord, 'fire') as mock_fire:
with patch.object(RepeatRecord, 'postpone_by') as mock_postpone_fire:
# calls process_repeat_record():
self.repeat_record = self.repeater.register(CaseAccessors(self.domain).get_case(CASE_ID))
self.assertEqual(mock_fire.call_count, 1)
self.assertEqual(mock_postpone_fire.call_count, 0)
# paused
self.repeater.pause()
# re fetch repeat record
repeat_record_id = self.repeat_record.get_id
self.repeat_record = RepeatRecord.get(repeat_record_id)
process_repeat_record(self.repeat_record)
self.assertEqual(mock_fire.call_count, 1)
self.assertEqual(mock_postpone_fire.call_count, 1)
# resumed
self.repeater.resume()
# re fetch repeat record
repeat_record_id = self.repeat_record.get_id
self.repeat_record = RepeatRecord.get(repeat_record_id)
process_repeat_record(self.repeat_record)
self.assertEqual(mock_fire.call_count, 2)
self.assertEqual(mock_postpone_fire.call_count, 1)
def test_trigger_when_paused(self):
# not paused
with patch.object(RepeatRecord, 'fire') as mock_fire:
with patch.object(RepeatRecord,
'postpone_by') as mock_postpone_fire:
process_repeat_record(self.repeat_record)
self.assertEqual(mock_fire.call_count, 1)
self.assertEqual(mock_postpone_fire.call_count, 0)
# paused
self.repeater.pause()
# re fetch repeat record
repeat_record_id = self.repeat_record.get_id
self.repeat_record = RepeatRecord.get(repeat_record_id)
process_repeat_record(self.repeat_record)
self.assertEqual(mock_fire.call_count, 1)
self.assertEqual(mock_postpone_fire.call_count, 1)
# resumed
self.repeater.resume()
# re fetch repeat record
repeat_record_id = self.repeat_record.get_id
self.repeat_record = RepeatRecord.get(repeat_record_id)
process_repeat_record(self.repeat_record)
self.assertEqual(mock_fire.call_count, 2)
self.assertEqual(mock_postpone_fire.call_count, 1)
def test_trigger_when_paused(self):
# not paused
with patch.object(RepeatRecord, 'fire') as mock_fire:
with patch.object(RepeatRecord, 'postpone_by') as mock_postpone_fire:
process_repeat_record(self.repeat_record)
self.assertEqual(mock_fire.call_count, 1)
self.assertEqual(mock_postpone_fire.call_count, 0)
# paused
self.repeater.pause()
# re fetch repeat record
repeat_record_id = self.repeat_record.get_id
self.repeat_record = RepeatRecord.get(repeat_record_id)
process_repeat_record(self.repeat_record)
self.assertEqual(mock_fire.call_count, 1)
self.assertEqual(mock_postpone_fire.call_count, 1)
# resumed
self.repeater.resume()
# re fetch repeat record
repeat_record_id = self.repeat_record.get_id
self.repeat_record = RepeatRecord.get(repeat_record_id)
process_repeat_record(self.repeat_record)
self.assertEqual(mock_fire.call_count, 2)
self.assertEqual(mock_postpone_fire.call_count, 1)
def handle(self, *args, **options):
domain = options.get('domain')
repeater_id = options.get('repeater_id')
state = options.get('state')
records_file_path = options.get('records_file_path')
if records_file_path:
self._load_record_ids_from_file(records_file_path)
records = self.record_ids
record_count = len(records)
elif domain and repeater_id:
records = iter_repeat_records_by_domain(domain, repeater_id=repeater_id, state=state)
record_count = get_repeat_record_count(domain, repeater_id=repeater_id, state=state)
else:
raise CommandError("Insufficient Arguments")
for record in with_progress_bar(records, length=record_count):
if isinstance(record, str):
record_id = record
try:
record = RepeatRecord.get(record_id)
except ResourceNotFound:
self.ws.append([record_id, '', 'Not Found'])
continue
self._add_row(record)
file_name = self._save_file(repeater_id, state)
print("Report saved in file:{filename}".format(filename=file_name))
def handle(self, *args, **options):
domain = options.get('domain')
repeater_id = options.get('repeater_id')
state = options.get('state')
records_file_path = options.get('records_file_path')
if records_file_path:
self._load_record_ids_from_file(records_file_path)
records = self.record_ids
record_count = len(records)
elif domain and repeater_id:
records = iter_repeat_records_by_domain(domain,
repeater_id=repeater_id,
state=state)
record_count = get_repeat_record_count(domain,
repeater_id=repeater_id,
state=state)
else:
raise CommandError("Insufficient Arguments")
for record in with_progress_bar(records, length=record_count):
if isinstance(record, str):
record_id = record
try:
record = RepeatRecord.get(record_id)
except ResourceNotFound:
self.ws.append([record_id, '', 'Not Found'])
continue
self._add_row(record)
file_name = self._save_file(repeater_id, state)
print("Report saved in file:{filename}".format(filename=file_name))
def allowed_to_forward(self, user):
# if this user is already in the repeater queue don't add another one
if queued_payload(user.domain, user.user_id):
return False
# If this user has already been forwarded without any changes, then
# don't send it to BETS again
successful_records = user.user_data.get('BETS_user_repeat_record_ids')
if successful_records:
latest_record_id = successful_records.split(" ")[-1]
latest_record = RepeatRecord.get(latest_record_id)
previous_payload = latest_record.attempts[-1].info if latest_record.attempts else "{}"
try:
previous_payload_json = json.loads(previous_payload)
current_payload = get_bets_user_json(user.domain, user)
del previous_payload_json['user_data']['id_device_number']
del previous_payload_json['user_data']['id_device_body']
del current_payload['user_data']['id_device_number']
del current_payload['user_data']['id_device_body']
if current_payload == previous_payload_json:
return False
except (TypeError, ValueError):
pass
return (user.user_data.get('user_level', None) == 'real'
and any(self._is_relevant_location(loc)
for loc in user.get_sql_locations(self.domain)))
def _validate_record(r, domain):
try:
payload = RepeatRecord.get(r)
except ResourceNotFound:
return None
if payload.domain != domain:
return None
return payload
def openmrs_test_fire(request, domain, repeater_id, record_id):
repeater = OpenmrsRepeater.get(repeater_id)
record = RepeatRecord.get(record_id)
assert repeater.domain == domain
assert record.domain == domain
assert record.repeater_id == repeater.get_id
attempt = repeater.fire_for_record(record)
return JsonResponse(attempt.to_json())
def _validate_record(r, domain):
from corehq.motech.repeaters.models import RepeatRecord
try:
payload = RepeatRecord.get(r)
except ResourceNotFound:
return None
if payload.domain != domain:
return None
return payload
def openmrs_test_fire(request, domain, repeater_id, record_id):
repeater = OpenmrsRepeater.get(repeater_id)
record = RepeatRecord.get(record_id)
assert repeater.domain == domain
assert record.domain == domain
assert record.repeater_id == repeater.get_id
attempt = repeater.fire_for_record(record)
return JsonResponse(attempt.to_json())
def _get_couch_repeat_record(domain, record_id):
from corehq.motech.repeaters.models import RepeatRecord
try:
couch_record = RepeatRecord.get(record_id)
except ResourceNotFound:
return None
if couch_record.domain != domain:
return None
return couch_record
def get_record_or_404(request, domain, record_id):
try:
record = RepeatRecord.get(record_id)
except ResourceNotFound:
raise Http404()
if record.domain != domain:
raise Http404()
return record
def get_record_or_404(request, domain, record_id):
try:
record = RepeatRecord.get(record_id)
except ResourceNotFound:
raise Http404()
if record.domain != domain:
raise Http404()
return record
def requeue_repeat_record(request, domain):
try:
record = RepeatRecord.get(request.POST.get('record_id'))
except ResourceNotFound:
return HttpResponse(status=404)
record.requeue()
record.save()
if record.cancelled:
return HttpResponse(status=400)
return HttpResponse('OK')
def delete_all_repeat_records():
from .models import RepeatRecord
results = RepeatRecord.get_db().view('repeaters/repeat_records', reduce=False).all()
for result in results:
try:
repeat_record = RepeatRecord.get(result['id'])
except Exception:
pass
else:
repeat_record.delete()
def requeue_repeat_record(request, domain):
try:
record = RepeatRecord.get(request.POST.get('record_id'))
except ResourceNotFound:
return HttpResponse(status=404)
record.requeue()
record.save()
if record.cancelled:
return HttpResponse(status=400)
return HttpResponse('OK')
def test_success(self):
repeat_record = self.repeater.register(
CaseAccessors(self.domain).get_case(CASE_ID))
repeat_record = RepeatRecord.get(repeat_record.record_id)
# Should be marked as successful after a successful run
with patch('corehq.motech.repeaters.models.simple_post'
) as mock_simple_post:
mock_simple_post.return_value.status_code = 200
repeat_record.fire()
self.assertTrue(repeat_record.succeeded)
class RepeaterFailureTest(BaseRepeaterTest):
domain = 'repeat-fail'
def setUp(self):
super().setUp()
# Create the case before creating the repeater, so that the
# repeater doesn't fire for the case creation. Each test
# registers this case, and the repeater will fire then.
self.post_xml(self.xform_xml, self.domain)
self.connx = ConnectionSettings.objects.create(
domain=self.domain,
url='case-repeater-url',
)
self.repeater = CaseRepeater(
domain=self.domain,
connection_settings_id=self.connx.id,
format='case_json',
)
self.repeater.save()
def tearDown(self):
self.repeater.delete()
self.connx.delete()
FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain)
delete_all_repeat_records()
super().tearDown()
def test_get_payload_exception(self):
repeat_record = self.repeater.register(
CommCareCase.objects.get_case(CASE_ID, self.domain))
with self.assertRaises(Exception):
with patch.object(CaseRepeater,
'get_payload',
side_effect=Exception('Boom!')):
repeat_record.fire()
self.assertEqual(repeat_record.failure_reason, 'Boom!')
self.assertFalse(repeat_record.succeeded)
def test_failure(self):
case = CommCareCase.objects.get_case(CASE_ID, self.domain)
with patch('corehq.motech.repeaters.models.simple_request',
side_effect=RequestException('Boom!')):
rr = self.repeater.register(case) # calls repeat_record.fire()
# Fetch the repeat_record revision that was updated
repeat_record = RepeatRecord.get(rr.record_id)
self.assertEqual(repeat_record.failure_reason, 'Boom!')
self.assertFalse(repeat_record.succeeded)
def allowed_to_forward(self, location):
# if this location is already in the repeater queue, don't forward again
if queued_payload(location.domain, location.location_id):
return False
# If this location has already been forwarded without any changes, then
# don't send it to BETS again
successful_records = location.metadata.get('BETS_location_repeat_record_ids')
if successful_records:
latest_record_id = successful_records.split(" ")[-1]
latest_record = RepeatRecord.get(latest_record_id)
previous_payload = latest_record.attempts[-1].info if latest_record.attempts else "{}"
try:
previous_payload_json = json.loads(previous_payload)
current_payload = get_bets_location_json(location)
if current_payload == previous_payload_json:
return False
except (TypeError, ValueError):
pass
return (location.metadata.get('is_test') != "yes"
and location.location_type.code in self.location_types_to_forward)
class RepeaterTest(BaseRepeaterTest):
domain = "repeater-test"
def setUp(self):
super(RepeaterTest, self).setUp()
self.case_connx = ConnectionSettings.objects.create(
domain=self.domain,
url='case-repeater-url',
)
self.case_repeater = CaseRepeater(
domain=self.domain,
connection_settings_id=self.case_connx.id,
format='case_json',
)
self.case_repeater.save()
self.form_connx = ConnectionSettings.objects.create(
domain=self.domain,
url='form-repeater-url',
)
self.form_repeater = FormRepeater(
domain=self.domain,
connection_settings_id=self.form_connx.id,
format='form_json')
self.form_repeater.save()
self.log = []
with patch('corehq.motech.repeaters.models.simple_request',
return_value=MockResponse(status_code=500,
reason="Borked")) as mock_fire:
self.post_xml(self.xform_xml, self.domain)
self.initial_fire_call_count = mock_fire.call_count
def tearDown(self):
self.case_repeater.delete()
self.case_connx.delete()
self.form_repeater.delete()
self.form_connx.delete()
FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain)
delete_all_repeat_records()
super(RepeaterTest, self).tearDown()
def repeat_records(self):
return super(RepeaterTest, self).repeat_records(self.domain)
# whatever value specified will be doubled since both case and form repeater are active
def _create_additional_repeat_records(self, count):
for _ in range(count):
instance_id = uuid.uuid4().hex
xform_xml = XFORM_XML_TEMPLATE.format(
"https://www.commcarehq.org/test/repeater/", USER_ID,
instance_id, self.case_block)
with patch('corehq.motech.repeaters.models.simple_request',
return_value=MockResponse(status_code=500,
reason="Borked")):
self.post_xml(xform_xml, self.domain)
def test_skip_device_logs(self):
devicelog_xml = XFORM_XML_TEMPLATE.format(DEVICE_LOG_XMLNS, USER_ID,
'1234', '')
self.post_xml(devicelog_xml, self.domain)
for repeat_record in self.repeat_records():
self.assertNotEqual(repeat_record.payload_id, '1234')
def test_skip_duplicates(self):
"""
Ensure that submitting a duplicate form does not create extra RepeatRecords
"""
self.assertEqual(len(self.repeat_records()), 2)
# this form is already submitted during setUp so a second submission should be a duplicate
form = self.post_xml(self.xform_xml, self.domain).xform
self.assertTrue(form.is_duplicate)
self.assertEqual(len(self.repeat_records()), 2)
def test_repeater_failed_sends(self):
"""
This tests records that fail are requeued later
"""
def now():
return datetime.utcnow()
repeat_records = self.repeat_records()
self.assertEqual(len(repeat_records), 2)
for repeat_record in repeat_records:
with patch('corehq.motech.repeaters.models.simple_request',
return_value=MockResponse(
status_code=404,
reason='Not Found')) as mock_request:
repeat_record.fire()
self.assertEqual(mock_request.call_count, 1)
# Enqueued repeat records have next_check incremented by 48 hours
next_check_time = now() + timedelta(minutes=60) + timedelta(hours=48)
repeat_records = RepeatRecord.all(
domain=self.domain,
due_before=now() + timedelta(minutes=15),
)
self.assertEqual(len(repeat_records), 0)
repeat_records = RepeatRecord.all(
domain=self.domain,
due_before=next_check_time,
)
self.assertEqual(len(repeat_records), 2)
def test_update_failure_next_check(self):
now = datetime.utcnow()
record = RepeatRecord(
domain=self.domain,
repeater_id=self.case_repeater.get_id,
next_check=now,
)
self.assertIsNone(record.last_checked)
attempt = record.make_set_next_try_attempt(None)
record.add_attempt(attempt)
self.assertTrue(record.last_checked > now)
self.assertEqual(record.next_check,
record.last_checked + MIN_RETRY_WAIT)
def test_repeater_successful_send(self):
repeat_records = self.repeat_records()
for repeat_record in repeat_records:
with patch('corehq.motech.repeaters.models.simple_request') as mock_request, \
patch.object(ConnectionSettings, 'get_auth_manager') as mock_manager:
mock_request.return_value.status_code = 200
mock_manager.return_value = 'MockAuthManager'
repeat_record.fire()
self.assertEqual(mock_request.call_count, 1)
mock_request.assert_called_with(
self.domain,
repeat_record.repeater.get_url(repeat_record),
repeat_record.get_payload(),
headers=repeat_record.repeater.get_headers(repeat_record),
auth_manager='MockAuthManager',
verify=repeat_record.repeater.verify,
notify_addresses=[],
payload_id=repeat_record.payload_id,
method="POST",
)
# The following is pretty fickle and depends on which of
# - corehq.motech.repeaters.signals
# - casexml.apps.case.signals
# gets loaded first.
# This is deterministic but easily affected by minor code changes
repeat_records = self.repeat_records()
for repeat_record in repeat_records:
self.assertEqual(repeat_record.succeeded, True)
self.assertEqual(repeat_record.next_check, None)
self.assertEqual(len(self.repeat_records()), 0)
self.post_xml(self.update_xform_xml, self.domain)
self.assertEqual(len(self.repeat_records()), 2)
def test_check_repeat_records(self):
self.assertEqual(len(self.repeat_records()), 2)
self.assertEqual(self.initial_fire_call_count, 2)
with patch(
'corehq.motech.repeaters.models.simple_request') as mock_fire:
check_repeaters()
self.assertEqual(mock_fire.call_count, 0)
def test_repeat_record_status_check(self):
self.assertEqual(len(self.repeat_records()), 2)
# Do not trigger cancelled records
for repeat_record in self.repeat_records():
repeat_record.cancelled = True
repeat_record.save()
with patch(
'corehq.motech.repeaters.models.simple_request') as mock_fire:
check_repeaters()
self.assertEqual(mock_fire.call_count, 0)
# trigger force send records if not cancelled and tries not exhausted
for repeat_record in self.repeat_records():
with patch('corehq.motech.repeaters.models.simple_request',
return_value=MockResponse(status_code=200,
reason='')) as mock_fire:
repeat_record.fire(force_send=True)
self.assertEqual(mock_fire.call_count, 1)
# all records should be in SUCCESS state after force try
for repeat_record in self.repeat_records():
self.assertEqual(repeat_record.state, RECORD_SUCCESS_STATE)
self.assertEqual(repeat_record.overall_tries, 1)
# not trigger records succeeded triggered after cancellation
with patch(
'corehq.motech.repeaters.models.simple_request') as mock_fire:
check_repeaters()
self.assertEqual(mock_fire.call_count, 0)
for repeat_record in self.repeat_records():
self.assertEqual(repeat_record.state, RECORD_SUCCESS_STATE)
def test_retry_process_repeat_record_locking(self):
self.assertEqual(len(self.repeat_records()), 2)
with patch('corehq.motech.repeaters.tasks.retry_process_repeat_record'
) as mock_process:
check_repeaters()
self.assertEqual(mock_process.delay.call_count, 0)
for record in self.repeat_records():
# Resetting next_check should allow them to be requeued
record.next_check = datetime.utcnow()
record.save()
with patch('corehq.motech.repeaters.tasks.retry_process_repeat_record'
) as mock_process:
check_repeaters()
self.assertEqual(mock_process.delay.call_count, 2)
def test_automatic_cancel_repeat_record(self):
case = CommCareCase.objects.get_case(CASE_ID, self.domain)
rr = self.case_repeater.register(case)
# Fetch the revision that was updated:
repeat_record = RepeatRecord.get(rr.record_id)
self.assertEqual(1, repeat_record.overall_tries)
with patch('corehq.motech.repeaters.models.simple_request',
side_effect=Exception('Boom!')):
for __ in range(repeat_record.max_possible_tries -
repeat_record.overall_tries):
repeat_record.fire()
self.assertEqual(True, repeat_record.cancelled)
repeat_record.requeue()
self.assertEqual(0, repeat_record.overall_tries)
self.assertNotEqual(None, repeat_record.next_check)
class TestRepeaterFormat(BaseRepeaterTest):
domain = 'test-fmt'
@classmethod
def setUpClass(cls):
super(TestRepeaterFormat, cls).setUpClass()
cls.payload = 'some random case'
class NewCaseGenerator(BasePayloadGenerator):
format_name = 'new_format'
format_label = 'XML'
deprecated_format_names = ('new_format_alias', )
def get_payload(self, repeat_record, payload_doc):
return cls.payload
RegisterGenerator.get_collection(CaseRepeater).add_new_format(
NewCaseGenerator)
cls.new_generator = NewCaseGenerator
def setUp(self):
super().setUp()
self.post_xml(self.xform_xml, self.domain)
self.connx = ConnectionSettings.objects.create(
domain=self.domain,
url='case-repeater-url',
)
self.repeater = CaseRepeater(
domain=self.domain,
connection_settings_id=self.connx.id,
format='new_format',
)
# SQL Repeater Model restricts format and would raise error on unexpected values
self.repeater.save(sync_to_sql=False)
def tearDown(self):
self.repeater.delete(sync_to_sql=False)
self.connx.delete()
FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain)
delete_all_repeat_records()
super().tearDown()
def test_new_format_same_name(self):
class NewCaseGenerator(BasePayloadGenerator):
format_name = 'case_xml'
format_label = 'XML'
def get_payload(self, repeat_record, payload_doc):
return self.payload
with self.assertRaises(DuplicateFormatException):
RegisterGenerator.get_collection(CaseRepeater).add_new_format(
NewCaseGenerator)
def test_new_format_second_default(self):
class NewCaseGenerator(BasePayloadGenerator):
format_name = 'rubbish'
format_label = 'XML'
def get_payload(self, repeat_record, payload_doc):
return self.payload
with self.assertRaises(DuplicateFormatException):
RegisterGenerator.get_collection(CaseRepeater).add_new_format(
NewCaseGenerator, is_default=True)
def test_new_format_payload(self):
case = CommCareCase.objects.get_case(CASE_ID, self.domain)
with patch('corehq.motech.repeaters.models.simple_request') as mock_request, \
patch.object(ConnectionSettings, 'get_auth_manager') as mock_manager:
mock_request.return_value.status_code = 200
mock_manager.return_value = 'MockAuthManager'
rr = self.repeater.register(case)
repeat_record = RepeatRecord.get(rr.record_id)
headers = self.repeater.get_headers(repeat_record)
mock_request.assert_called_with(
self.domain,
self.connx.url,
self.payload,
auth_manager='MockAuthManager',
headers=headers,
notify_addresses=[],
payload_id='ABC123CASEID',
verify=self.repeater.verify,
method="POST",
)
with patch('corehq.motech.repeaters.models.simple_request',
side_effect=RequestException('Boom!')):
rr = self.repeater.register(case) # calls repeat_record.fire()
# Fetch the repeat_record revision that was updated
repeat_record = RepeatRecord.get(rr.record_id)
self.assertEqual(repeat_record.failure_reason, 'Boom!')
self.assertFalse(repeat_record.succeeded)
def test_unexpected_failure(self):
case = CommCareCase.objects.get_case(CASE_ID, self.domain)
with patch('corehq.motech.repeaters.models.simple_request',
side_effect=Exception('Boom!')):
rr = self.repeater.register(case)
repeat_record = RepeatRecord.get(rr.record_id)
self.assertEqual(repeat_record.failure_reason, 'Internal Server Error')
self.assertFalse(repeat_record.succeeded)
def test_success(self):
case = CommCareCase.objects.get_case(CASE_ID, self.domain)
# Should be marked as successful after a successful run
with patch('corehq.motech.repeaters.models.simple_request'
) as mock_simple_post:
mock_simple_post.return_value.status_code = 200
rr = self.repeater.register(case)
repeat_record = RepeatRecord.get(rr.record_id)
self.assertTrue(repeat_record.succeeded)