def test_repeat_record_created(self): ''' Tests that whenever an application with a repeater is saved that a repeat record is created. ''' application = Application(domain=self.domain) application.save() repeat_records = RepeatRecord.all(domain=self.domain, due_before=datetime.utcnow()) self.assertEqual(len(repeat_records), 0) app_structure_repeater = AppStructureRepeater(domain=self.domain, url=self.forwarding_url) app_structure_repeater.save() application.save() repeat_records = RepeatRecord.all(domain=self.domain, due_before=datetime.utcnow()) self.assertEqual(len(repeat_records), 1) for repeat_record in repeat_records: self.assertEqual(repeat_record.url, self.forwarding_url) self.assertEqual(repeat_record.get_payload(), application.get_id) repeat_record.delete() application.delete() app_structure_repeater.delete()
def check_repeaters(): start = datetime.utcnow() LIMIT = 100 while True: number_locked = 0 # take LIMIT records off the top # the assumption is that they all get 'popped' in the for loop # the only exception I can see is if there's a problem with the # locking, a large number of locked tasks could pile up at the top, # so make a provision for that worst case repeat_records = RepeatRecord.all(due_before=start, limit=LIMIT + number_locked) if repeat_records.count() <= number_locked: # don't keep spinning if there's nothing left to fetch return for repeat_record in repeat_records: now = datetime.utcnow() # abort if taking too long, so the next task can take over if now - start > CHECK_REPEATERS_INTERVAL: return if repeat_record.acquire_lock(start): repeat_record.fire() repeat_record.save() repeat_record.release_lock() else: number_locked += 1
def tearDown(self): self.case_repeater.delete() self.form_repeater.delete() XFormInstance.get(instance_id).delete() repeat_records = RepeatRecord.all() for repeat_record in repeat_records: repeat_record.delete()
def check_repeaters(request, domain): if request.method == 'GET': return HttpResponse(PAGE % {'status': '', 'domain': domain}) elif request.method == 'POST': start = datetime.utcnow() repeat_records = RepeatRecord.all(domain, due_before=start, limit=100) process_repeater_list(repeat_records) return HttpResponse(PAGE % {'status': 'Done', 'domain': domain})
def test_repeater(self): CommCareCase.get(case_id) now = datetime.utcnow() repeat_records = RepeatRecord.all(domain=self.domain, due_before=now) self.assertEqual(len(repeat_records), 2) self.clear_log() for repeat_record in repeat_records: repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404])) repeat_record.save() for (url, status, data) in self.log: self.assertEqual(status, 404) self.clear_log() in30min = now + timedelta(minutes=30) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now + timedelta(minutes=15)) self.assertEqual(len(repeat_records), 0) repeat_records = RepeatRecord.all(domain=self.domain, due_before=in30min + timedelta(seconds=1)) self.assertEqual(len(repeat_records), 2) for repeat_record in repeat_records: self.assertLess(abs(in30min - repeat_record.next_check), timedelta(seconds=1)) repeat_record.fire(post_fn=self.make_post_fn([404, 200])) repeat_record.save() self.assertEqual(len(self.log), 4) self.assertEqual(self.log[1], (self.form_repeater.url, 200, xform_xml)) self.assertEqual(self.log[3][:2], (self.case_repeater.url, 200)) check_xml_line_by_line(self, self.log[3][2], case_block) repeat_records = RepeatRecord.all(domain=self.domain, due_before=in30min) for repeat_record in repeat_records: self.assertEqual(repeat_record.succeeded, True) self.assertEqual(repeat_record.next_check, None)
def check_repeaters(): now = datetime.utcnow() repeat_records = RepeatRecord.all(due_before=now) for repeat_record in repeat_records: repeat_record.fire() try: repeat_record.save() except ResourceConflict: logging.error("ResourceConflict with repeat_record %s: %s" % (repeat_record.get_id, repeat_record.to_json())) raise
def handle(self, *args, **options): if len(args) == 1: domain = args[0] else: raise CommandError("Usage: %s\n%s" % (self.args, self.help)) next_year = datetime.datetime.now() + datetime.timedelta(days=365) records = RepeatRecord.all(domain=domain, due_before=next_year) for record in records: record.fire(post_fn=simple_post) record.save() print "{} {}".format(record._id, "successful" if record.succeeded else "failed")
def loop(): # take LIMIT records off the top # the assumption is that they all get 'popped' in the for loop # the only exception I can see is if there's a problem with the # locking, a large number of locked tasks could pile up at the top, # so make a provision for that worst case number_locked = progress_report['number_locked'] repeat_records = RepeatRecord.all(due_before=start, limit=LIMIT + number_locked) return process_repeater_list(repeat_records, start=start, cutoff=start + CHECK_REPEATERS_INTERVAL, progress_report=progress_report)
def handle(self, *args, **options): if len(args) == 1: domain = args[0] else: raise CommandError('Usage: %s\n%s' % (self.args, self.help)) next_year = datetime.datetime.utcnow() + datetime.timedelta(days=365) records = RepeatRecord.all(domain=domain, due_before=next_year) for record in records: record.fire(post_fn=simple_post) record.save() print '{} {}'.format( record._id, 'successful' if record.succeeded else 'failed')
def test_repeater(self): CommCareCase.get(case_id) def now(): return datetime.utcnow() repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 2) self.clear_log() for repeat_record in repeat_records: repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404])) repeat_record.save() for (url, status, data) in self.log: self.assertEqual(status, 404) self.clear_log() next_check_time = now() + timedelta(minutes=60) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now() + timedelta(minutes=15)) self.assertEqual(len(repeat_records), 0) repeat_records = RepeatRecord.all(domain=self.domain, due_before=next_check_time + timedelta(seconds=2)) self.assertEqual(len(repeat_records), 2) for repeat_record in repeat_records: self.assertLess(abs(next_check_time - repeat_record.next_check), timedelta(seconds=2)) repeat_record.fire(post_fn=self.make_post_fn([404, 200])) repeat_record.save() self.assertEqual(len(self.log), 4) self.assertEqual(self.log[1], (self.form_repeater.url, 200, xform_xml)) self.assertEqual(self.log[3][:2], (self.case_repeater.url, 200)) check_xml_line_by_line(self, self.log[3][2], case_block) repeat_records = RepeatRecord.all(domain=self.domain, due_before=next_check_time) for repeat_record in repeat_records: self.assertEqual(repeat_record.succeeded, True) self.assertEqual(repeat_record.next_check, None) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 0) self.post_xml(update_xform_xml) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 2)
def test_ignore_document(self): """ When get_payload raises IgnoreDocument, fire should call update_success """ @RegisterGenerator(FormRepeater, 'new_format', 'XML') class NewFormGenerator(BasePayloadGenerator): def get_payload(self, repeat_record, payload_doc): raise IgnoreDocument repeat_records = RepeatRecord.all(domain=self.domain, ) for repeat_record_ in repeat_records: repeat_record_.fire() self.assertIsNone(repeat_record_.next_check) self.assertTrue(repeat_record_.succeeded)
def test_repeater(self): CommCareCase.get(case_id) def now(): return datetime.utcnow() repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 2) self.clear_log() for repeat_record in repeat_records: repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404])) repeat_record.save() for (url, status, data, headers) in self.log: self.assertEqual(status, 404) self.clear_log() next_check_time = now() + timedelta(minutes=60) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now() + timedelta(minutes=15)) self.assertEqual(len(repeat_records), 0) repeat_records = RepeatRecord.all(domain=self.domain, due_before=next_check_time + timedelta(seconds=2)) self.assertEqual(len(repeat_records), 2) for repeat_record in repeat_records: self.assertLess(abs(next_check_time - repeat_record.next_check), timedelta(seconds=2)) repeat_record.fire(post_fn=self.make_post_fn([404, 200])) repeat_record.save() self.assertEqual(len(self.log), 4) self.assertEqual(self.log[1][:3], (self.form_repeater.url, 200, xform_xml)) self.assertIn('received-on', self.log[1][3]) self.assertEqual(self.log[3][:2], (self.case_repeater.url, 200)) self.assertIn('server-modified-on', self.log[3][3]) check_xml_line_by_line(self, self.log[3][2], case_block) repeat_records = RepeatRecord.all(domain=self.domain, due_before=next_check_time) for repeat_record in repeat_records: self.assertEqual(repeat_record.succeeded, True) self.assertEqual(repeat_record.next_check, None) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 0) self.post_xml(update_xform_xml) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 2)
def loop(): # take LIMIT records off the top # the assumption is that they all get 'popped' in the for loop # the only exception I can see is if there's a problem with the # locking, a large number of locked tasks could pile up at the top, # so make a provision for that worst case number_locked = progress_report['number_locked'] repeat_records = RepeatRecord.all( due_before=start, limit=LIMIT + number_locked ) return process_repeater_list( repeat_records, start=start, cutoff=start + CHECK_REPEATERS_INTERVAL, progress_report=progress_report )
def test_ignore_document(self): """ When get_payload raises IgnoreDocument, fire should call update_success """ @RegisterGenerator(FormRepeater, 'new_format', 'XML') class NewFormGenerator(BasePayloadGenerator): def get_payload(self, repeat_record, payload_doc): raise IgnoreDocument repeat_records = RepeatRecord.all( domain=self.domain, ) for repeat_record_ in repeat_records: repeat_record_.fire() self.assertIsNone(repeat_record_.next_check) self.assertTrue(repeat_record_.succeeded)
def loop(): number_locked = 0 # take LIMIT records off the top # the assumption is that they all get 'popped' in the for loop # the only exception I can see is if there's a problem with the # locking, a large number of locked tasks could pile up at the top, # so make a provision for that worst case repeat_records = RepeatRecord.all( due_before=start, limit=LIMIT + number_locked ) if repeat_records.count() <= number_locked: # don't keep spinning if there's nothing left to fetch return False for repeat_record in repeat_records: now = datetime.utcnow() # abort if taking too long, so the next task can take over if now - start > CHECK_REPEATERS_INTERVAL: return False if repeat_record.acquire_lock(start): if repeat_record.repeater.doc_type.endswith(DELETED): if not repeat_record.doc_type.endswith(DELETED): repeat_record.doc_type += DELETED progress_report['deleted'].append(repeat_record.get_id) else: repeat_record.fire() if repeat_record.succeeded: progress_report['success'].append(repeat_record.get_id) else: progress_report['fail'].append(repeat_record.get_id) repeat_record.save() repeat_record.release_lock() else: progress_report['locked'].append(repeat_record.get_id) number_locked += 1 return True
def repeat_records(cls, domain_name): return RepeatRecord.all(domain=domain_name, due_before=datetime.utcnow())
def tearDown(self): self.repeater.delete() repeat_records = RepeatRecord.all() for repeat_record in repeat_records: repeat_record.delete()
def test_repeater(self): # this test should probably be divided into more units CommCareCase.get(case_id) def now(): return datetime.utcnow() repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 2) self.clear_log() records_by_repeater_id = {} for repeat_record in repeat_records: repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404])) repeat_record.save() records_by_repeater_id[repeat_record.repeater_id] = repeat_record for (url, status, data, headers) in self.log: self.assertEqual(status, 404) self.clear_log() next_check_time = now() + timedelta(minutes=60) repeat_records = RepeatRecord.all( domain=self.domain, due_before=now() + timedelta(minutes=15), ) self.assertEqual(len(repeat_records), 0) repeat_records = RepeatRecord.all( domain=self.domain, due_before=next_check_time + timedelta(seconds=2), ) self.assertEqual(len(repeat_records), 2) for repeat_record in repeat_records: self.assertLess(abs(next_check_time - repeat_record.next_check), timedelta(seconds=3)) repeat_record.fire(post_fn=self.make_post_fn([404, 200])) repeat_record.save() self.assertEqual(len(self.log), 4) # The following is pretty fickle and depends on which of # - corehq.apps.receiverwrapper.signals # - casexml.apps.case.signals # gets loaded first. # This is deterministic but easily affected by minor code changes # check case stuff rec = records_by_repeater_id[self.case_repeater.get_id] self.assertEqual(self.log[1][:2], (self.case_repeater.get_url(rec), 200)) self.assertIn('server-modified-on', self.log[1][3]) check_xml_line_by_line(self, self.log[1][2], case_block) # check form stuff rec = records_by_repeater_id[self.form_repeater.get_id] self.assertEqual(self.log[3][:3], (self.form_repeater.get_url(rec), 200, xform_xml)) self.assertIn('received-on', self.log[3][3]) repeat_records = RepeatRecord.all( domain=self.domain, due_before=next_check_time, ) for repeat_record in repeat_records: self.assertEqual(repeat_record.succeeded, True) self.assertEqual(repeat_record.next_check, None) self.assertEqual(len(self.repeat_records(self.domain)), 0) self.post_xml(update_xform_xml, self.domain) self.assertEqual(len(self.repeat_records(self.domain)), 2)
def test_repeater(self): CommCareCase.get(case_id) def now(): return datetime.utcnow() repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 2) self.clear_log() for repeat_record in repeat_records: repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404])) repeat_record.save() for (url, status, data, headers) in self.log: self.assertEqual(status, 404) self.clear_log() next_check_time = now() + timedelta(minutes=60) repeat_records = RepeatRecord.all( domain=self.domain, due_before=now() + timedelta(minutes=15), ) self.assertEqual(len(repeat_records), 0) repeat_records = RepeatRecord.all( domain=self.domain, due_before=next_check_time + timedelta(seconds=2), ) self.assertEqual(len(repeat_records), 2) for repeat_record in repeat_records: self.assertLess(abs(next_check_time - repeat_record.next_check), timedelta(seconds=2)) repeat_record.fire(post_fn=self.make_post_fn([404, 200])) repeat_record.save() self.assertEqual(len(self.log), 4) # The following is pretty fickle and depends on which of # - corehq.apps.receiverwrapper.signals # - casexml.apps.case.signals # gets loaded first. # This is deterministic but easily affected by minor code changes # check case stuff self.assertEqual(self.log[1][:2], (self.case_repeater.url, 200)) self.assertIn('server-modified-on', self.log[1][3]) check_xml_line_by_line(self, self.log[1][2], case_block) # check form stuff self.assertEqual(self.log[3][:3], (self.form_repeater.url, 200, xform_xml)) self.assertIn('received-on', self.log[3][3]) repeat_records = RepeatRecord.all( domain=self.domain, due_before=next_check_time, ) for repeat_record in repeat_records: self.assertEqual(repeat_record.succeeded, True) self.assertEqual(repeat_record.next_check, None) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 0) self.post_xml(update_xform_xml) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 2)
def repeat_records(self): return RepeatRecord.all(domain=self.domain, due_before=datetime.utcnow())