def test_repeat_record_created(self): ''' Tests that whenever an application with a repeater is saved that a repeat record is created. ''' application = Application(domain=self.domain) application.save() repeat_records = RepeatRecord.all(domain=self.domain, due_before=datetime.utcnow()) self.assertEqual(len(repeat_records), 0) app_structure_repeater = AppStructureRepeater(domain=self.domain, url=self.forwarding_url) app_structure_repeater.save() application.save() repeat_records = RepeatRecord.all(domain=self.domain, due_before=datetime.utcnow()) self.assertEqual(len(repeat_records), 1) for repeat_record in repeat_records: self.assertEqual(repeat_record.url, self.forwarding_url) self.assertEqual(repeat_record.get_payload(), application.get_id) repeat_record.delete() application.delete() app_structure_repeater.delete()
def testLocks(self): r = RepeatRecord(domain='test') r.save() r2 = RepeatRecord.get(r._id) self.assertTrue(r.acquire_lock(datetime.utcnow())) r3 = RepeatRecord.get(r._id) self.assertFalse(r2.acquire_lock(datetime.utcnow())) self.assertFalse(r3.acquire_lock(datetime.utcnow())) r.release_lock() r4 = RepeatRecord.get(r._id) self.assertTrue(r4.acquire_lock(datetime.utcnow()))
def check_repeaters(): start = datetime.utcnow() LIMIT = 100 while True: number_locked = 0 # take LIMIT records off the top # the assumption is that they all get 'popped' in the for loop # the only exception I can see is if there's a problem with the # locking, a large number of locked tasks could pile up at the top, # so make a provision for that worst case repeat_records = RepeatRecord.all(due_before=start, limit=LIMIT + number_locked) if repeat_records.count() <= number_locked: # don't keep spinning if there's nothing left to fetch return for repeat_record in repeat_records: now = datetime.utcnow() # abort if taking too long, so the next task can take over if now - start > CHECK_REPEATERS_INTERVAL: return if repeat_record.acquire_lock(start): repeat_record.fire() repeat_record.save() repeat_record.release_lock() else: number_locked += 1
def tearDown(self): self.case_repeater.delete() self.form_repeater.delete() XFormInstance.get(instance_id).delete() repeat_records = RepeatRecord.all() for repeat_record in repeat_records: repeat_record.delete()
def handle(self, *args, **options): domain = args[0] db = Domain.get_db() doc_ids = [ r['id'] for r in db.view( 'domain/docs', startkey=[domain, 'RepeatRecord'], endkey=[domain, 'RepeatRecord', {}], reduce=False, ) ] count = len(doc_ids) print 'found %s doc ids' % count latest = datetime.min latest_doc = None for i, doc in enumerate(iter_docs(db, doc_ids)): wrapped = RepeatRecord.wrap(doc) if i % 100 == 0: print 'checked %s / %s' % (i, count) if wrapped.last_checked and wrapped.last_checked > latest: latest = wrapped.last_checked latest_doc = wrapped print 'new latest: %s' % latest if latest_doc: print 'latest repeater date is %s' % latest print 'latest repeater is %s' % latest_doc._id else: print 'no relevant repeaters found'
def check_repeaters(request, domain): if request.method == 'GET': return HttpResponse(PAGE % {'status': '', 'domain': domain}) elif request.method == 'POST': start = datetime.utcnow() repeat_records = RepeatRecord.all(domain, due_before=start, limit=100) process_repeater_list(repeat_records) return HttpResponse(PAGE % {'status': 'Done', 'domain': domain})
def test_repeater(self): CommCareCase.get(case_id) now = datetime.utcnow() repeat_records = RepeatRecord.all(domain=self.domain, due_before=now) self.assertEqual(len(repeat_records), 2) self.clear_log() for repeat_record in repeat_records: repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404])) repeat_record.save() for (url, status, data) in self.log: self.assertEqual(status, 404) self.clear_log() in30min = now + timedelta(minutes=30) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now + timedelta(minutes=15)) self.assertEqual(len(repeat_records), 0) repeat_records = RepeatRecord.all(domain=self.domain, due_before=in30min + timedelta(seconds=1)) self.assertEqual(len(repeat_records), 2) for repeat_record in repeat_records: self.assertLess(abs(in30min - repeat_record.next_check), timedelta(seconds=1)) repeat_record.fire(post_fn=self.make_post_fn([404, 200])) repeat_record.save() self.assertEqual(len(self.log), 4) self.assertEqual(self.log[1], (self.form_repeater.url, 200, xform_xml)) self.assertEqual(self.log[3][:2], (self.case_repeater.url, 200)) check_xml_line_by_line(self, self.log[3][2], case_block) repeat_records = RepeatRecord.all(domain=self.domain, due_before=in30min) for repeat_record in repeat_records: self.assertEqual(repeat_record.succeeded, True) self.assertEqual(repeat_record.next_check, None)
def check_repeaters(): now = datetime.utcnow() repeat_records = RepeatRecord.all(due_before=now) for repeat_record in repeat_records: repeat_record.fire() try: repeat_record.save() except ResourceConflict: logging.error("ResourceConflict with repeat_record %s: %s" % (repeat_record.get_id, repeat_record.to_json())) raise
def handle(self, *args, **options): if len(args) == 1: domain = args[0] else: raise CommandError("Usage: %s\n%s" % (self.args, self.help)) next_year = datetime.datetime.now() + datetime.timedelta(days=365) records = RepeatRecord.all(domain=domain, due_before=next_year) for record in records: record.fire(post_fn=simple_post) record.save() print "{} {}".format(record._id, "successful" if record.succeeded else "failed")
def loop(): # take LIMIT records off the top # the assumption is that they all get 'popped' in the for loop # the only exception I can see is if there's a problem with the # locking, a large number of locked tasks could pile up at the top, # so make a provision for that worst case number_locked = progress_report['number_locked'] repeat_records = RepeatRecord.all(due_before=start, limit=LIMIT + number_locked) return process_repeater_list(repeat_records, start=start, cutoff=start + CHECK_REPEATERS_INTERVAL, progress_report=progress_report)
def handle(self, *args, **options): if len(args) == 1: domain = args[0] else: raise CommandError('Usage: %s\n%s' % (self.args, self.help)) next_year = datetime.datetime.utcnow() + datetime.timedelta(days=365) records = RepeatRecord.all(domain=domain, due_before=next_year) for record in records: record.fire(post_fn=simple_post) record.save() print '{} {}'.format( record._id, 'successful' if record.succeeded else 'failed')
def test_repeater(self): CommCareCase.get(case_id) def now(): return datetime.utcnow() repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 2) self.clear_log() for repeat_record in repeat_records: repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404])) repeat_record.save() for (url, status, data) in self.log: self.assertEqual(status, 404) self.clear_log() next_check_time = now() + timedelta(minutes=60) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now() + timedelta(minutes=15)) self.assertEqual(len(repeat_records), 0) repeat_records = RepeatRecord.all(domain=self.domain, due_before=next_check_time + timedelta(seconds=2)) self.assertEqual(len(repeat_records), 2) for repeat_record in repeat_records: self.assertLess(abs(next_check_time - repeat_record.next_check), timedelta(seconds=2)) repeat_record.fire(post_fn=self.make_post_fn([404, 200])) repeat_record.save() self.assertEqual(len(self.log), 4) self.assertEqual(self.log[1], (self.form_repeater.url, 200, xform_xml)) self.assertEqual(self.log[3][:2], (self.case_repeater.url, 200)) check_xml_line_by_line(self, self.log[3][2], case_block) repeat_records = RepeatRecord.all(domain=self.domain, due_before=next_check_time) for repeat_record in repeat_records: self.assertEqual(repeat_record.succeeded, True) self.assertEqual(repeat_record.next_check, None) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 0) self.post_xml(update_xform_xml) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 2)
def test_ignore_document(self): """ When get_payload raises IgnoreDocument, fire should call update_success """ @RegisterGenerator(FormRepeater, 'new_format', 'XML') class NewFormGenerator(BasePayloadGenerator): def get_payload(self, repeat_record, payload_doc): raise IgnoreDocument repeat_records = RepeatRecord.all(domain=self.domain, ) for repeat_record_ in repeat_records: repeat_record_.fire() self.assertIsNone(repeat_record_.next_check) self.assertTrue(repeat_record_.succeeded)
def test_repeater(self): CommCareCase.get(case_id) def now(): return datetime.utcnow() repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 2) self.clear_log() for repeat_record in repeat_records: repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404])) repeat_record.save() for (url, status, data, headers) in self.log: self.assertEqual(status, 404) self.clear_log() next_check_time = now() + timedelta(minutes=60) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now() + timedelta(minutes=15)) self.assertEqual(len(repeat_records), 0) repeat_records = RepeatRecord.all(domain=self.domain, due_before=next_check_time + timedelta(seconds=2)) self.assertEqual(len(repeat_records), 2) for repeat_record in repeat_records: self.assertLess(abs(next_check_time - repeat_record.next_check), timedelta(seconds=2)) repeat_record.fire(post_fn=self.make_post_fn([404, 200])) repeat_record.save() self.assertEqual(len(self.log), 4) self.assertEqual(self.log[1][:3], (self.form_repeater.url, 200, xform_xml)) self.assertIn('received-on', self.log[1][3]) self.assertEqual(self.log[3][:2], (self.case_repeater.url, 200)) self.assertIn('server-modified-on', self.log[3][3]) check_xml_line_by_line(self, self.log[3][2], case_block) repeat_records = RepeatRecord.all(domain=self.domain, due_before=next_check_time) for repeat_record in repeat_records: self.assertEqual(repeat_record.succeeded, True) self.assertEqual(repeat_record.next_check, None) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 0) self.post_xml(update_xform_xml) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 2)
def handle(self, *args, **options): domain = args[0] old_db = Domain.get_db() new_db = RepeatRecord.get_db() assert old_db.dbname != new_db.dbname doc_ids = get_doc_ids(domain, 'RepeatRecord', old_db) count = len(doc_ids) chunksize = 250 for i, docs in enumerate(chunked(iter_docs(old_db, doc_ids, chunksize), chunksize)): for doc in docs: if '_rev' in doc: del doc['_rev'] new_db.bulk_save(docs, new_edits=False) print 'checked %s / %s' % (i * chunksize, count)
def loop(): # take LIMIT records off the top # the assumption is that they all get 'popped' in the for loop # the only exception I can see is if there's a problem with the # locking, a large number of locked tasks could pile up at the top, # so make a provision for that worst case number_locked = progress_report['number_locked'] repeat_records = RepeatRecord.all( due_before=start, limit=LIMIT + number_locked ) return process_repeater_list( repeat_records, start=start, cutoff=start + CHECK_REPEATERS_INTERVAL, progress_report=progress_report )
def test_ignore_document(self): """ When get_payload raises IgnoreDocument, fire should call update_success """ @RegisterGenerator(FormRepeater, 'new_format', 'XML') class NewFormGenerator(BasePayloadGenerator): def get_payload(self, repeat_record, payload_doc): raise IgnoreDocument repeat_records = RepeatRecord.all( domain=self.domain, ) for repeat_record_ in repeat_records: repeat_record_.fire() self.assertIsNone(repeat_record_.next_check) self.assertTrue(repeat_record_.succeeded)
def handle(self, *args, **options): domain = args[0] old_db = Domain.get_db() new_db = RepeatRecord.get_db() assert old_db.dbname != new_db.dbname doc_ids = [ r["id"] for r in old_db.view( "domain/docs", startkey=[domain, "RepeatRecord"], endkey=[domain, "RepeatRecord", {}], reduce=False ) ] count = len(doc_ids) chunksize = 250 for i, docs in enumerate(chunked(iter_docs(old_db, doc_ids, chunksize), chunksize)): for doc in docs: if "_rev" in doc: del doc["_rev"] new_db.bulk_save(docs, new_edits=False) print "checked %s / %s" % (i * chunksize, count)
def handle(self, *args, **options): domain = args[0] db = Domain.get_db() doc_ids = get_doc_ids(domain, "RepeatRecord", db) count = len(doc_ids) print "found %s doc ids" % count latest = datetime.min latest_doc = None for i, doc in enumerate(iter_docs(db, doc_ids)): wrapped = RepeatRecord.wrap(doc) if i % 100 == 0: print "checked %s / %s" % (i, count) if wrapped.last_checked and wrapped.last_checked > latest: latest = wrapped.last_checked latest_doc = wrapped print "new latest: %s" % latest if latest_doc: print "latest repeater date is %s" % latest print "latest repeater is %s" % latest_doc._id else: print "no relevant repeaters found"
def loop(): number_locked = 0 # take LIMIT records off the top # the assumption is that they all get 'popped' in the for loop # the only exception I can see is if there's a problem with the # locking, a large number of locked tasks could pile up at the top, # so make a provision for that worst case repeat_records = RepeatRecord.all( due_before=start, limit=LIMIT + number_locked ) if repeat_records.count() <= number_locked: # don't keep spinning if there's nothing left to fetch return False for repeat_record in repeat_records: now = datetime.utcnow() # abort if taking too long, so the next task can take over if now - start > CHECK_REPEATERS_INTERVAL: return False if repeat_record.acquire_lock(start): if repeat_record.repeater.doc_type.endswith(DELETED): if not repeat_record.doc_type.endswith(DELETED): repeat_record.doc_type += DELETED progress_report['deleted'].append(repeat_record.get_id) else: repeat_record.fire() if repeat_record.succeeded: progress_report['success'].append(repeat_record.get_id) else: progress_report['fail'].append(repeat_record.get_id) repeat_record.save() repeat_record.release_lock() else: progress_report['locked'].append(repeat_record.get_id) number_locked += 1 return True
def check_inline_form_repeaters(post_fn=None): """old-style FormRepeater grandfathered in""" now = datetime.utcnow() forms = XFormInstance.view('receiverwrapper/forms_with_pending_repeats', startkey="", endkey=json_format_datetime(now), include_docs=True, ) for form in forms: if hasattr(form, 'repeats'): for repeat_record in form.repeats: record = dict(repeat_record) url = record.pop('url') record = RepeatRecord.wrap(record) record.payload_id = form.get_id record._repeater = FormRepeater(url=url) record.fire(post_fn=post_fn) record = record.to_json() record['url'] = url repeat_record.update(record) form.save()
def handle(self, *args, **options): domain = args[0] old_db = Domain.get_db() new_db = RepeatRecord.get_db() assert old_db.dbname != new_db.dbname doc_ids = [ r['id'] for r in old_db.view( 'domain/docs', startkey=[domain, 'RepeatRecord'], endkey=[domain, 'RepeatRecord', {}], reduce=False, ) ] count = len(doc_ids) chunksize = 250 for i, docs in enumerate( chunked(iter_docs(old_db, doc_ids, chunksize), chunksize)): for doc in docs: if '_rev' in doc: del doc['_rev'] new_db.bulk_save(docs, new_edits=False) print 'checked %s / %s' % (i * chunksize, count)
def repeat_records(self): return RepeatRecord.all(domain=self.domain, due_before=datetime.utcnow())
def repeat_records(cls, domain_name): return RepeatRecord.all(domain=domain_name, due_before=datetime.utcnow())
def test_repeater(self): CommCareCase.get(case_id) def now(): return datetime.utcnow() repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 2) self.clear_log() for repeat_record in repeat_records: repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404])) repeat_record.save() for (url, status, data, headers) in self.log: self.assertEqual(status, 404) self.clear_log() next_check_time = now() + timedelta(minutes=60) repeat_records = RepeatRecord.all( domain=self.domain, due_before=now() + timedelta(minutes=15), ) self.assertEqual(len(repeat_records), 0) repeat_records = RepeatRecord.all( domain=self.domain, due_before=next_check_time + timedelta(seconds=2), ) self.assertEqual(len(repeat_records), 2) for repeat_record in repeat_records: self.assertLess(abs(next_check_time - repeat_record.next_check), timedelta(seconds=2)) repeat_record.fire(post_fn=self.make_post_fn([404, 200])) repeat_record.save() self.assertEqual(len(self.log), 4) # The following is pretty fickle and depends on which of # - corehq.apps.receiverwrapper.signals # - casexml.apps.case.signals # gets loaded first. # This is deterministic but easily affected by minor code changes # check case stuff self.assertEqual(self.log[1][:2], (self.case_repeater.url, 200)) self.assertIn('server-modified-on', self.log[1][3]) check_xml_line_by_line(self, self.log[1][2], case_block) # check form stuff self.assertEqual(self.log[3][:3], (self.form_repeater.url, 200, xform_xml)) self.assertIn('received-on', self.log[3][3]) repeat_records = RepeatRecord.all( domain=self.domain, due_before=next_check_time, ) for repeat_record in repeat_records: self.assertEqual(repeat_record.succeeded, True) self.assertEqual(repeat_record.next_check, None) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 0) self.post_xml(update_xform_xml) repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 2)
def tearDown(self): self.repeater.delete() repeat_records = RepeatRecord.all() for repeat_record in repeat_records: repeat_record.delete()
def test_repeater(self): # this test should probably be divided into more units CommCareCase.get(case_id) def now(): return datetime.utcnow() repeat_records = RepeatRecord.all(domain=self.domain, due_before=now()) self.assertEqual(len(repeat_records), 2) self.clear_log() records_by_repeater_id = {} for repeat_record in repeat_records: repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404])) repeat_record.save() records_by_repeater_id[repeat_record.repeater_id] = repeat_record for (url, status, data, headers) in self.log: self.assertEqual(status, 404) self.clear_log() next_check_time = now() + timedelta(minutes=60) repeat_records = RepeatRecord.all( domain=self.domain, due_before=now() + timedelta(minutes=15), ) self.assertEqual(len(repeat_records), 0) repeat_records = RepeatRecord.all( domain=self.domain, due_before=next_check_time + timedelta(seconds=2), ) self.assertEqual(len(repeat_records), 2) for repeat_record in repeat_records: self.assertLess(abs(next_check_time - repeat_record.next_check), timedelta(seconds=3)) repeat_record.fire(post_fn=self.make_post_fn([404, 200])) repeat_record.save() self.assertEqual(len(self.log), 4) # The following is pretty fickle and depends on which of # - corehq.apps.receiverwrapper.signals # - casexml.apps.case.signals # gets loaded first. # This is deterministic but easily affected by minor code changes # check case stuff rec = records_by_repeater_id[self.case_repeater.get_id] self.assertEqual(self.log[1][:2], (self.case_repeater.get_url(rec), 200)) self.assertIn('server-modified-on', self.log[1][3]) check_xml_line_by_line(self, self.log[1][2], case_block) # check form stuff rec = records_by_repeater_id[self.form_repeater.get_id] self.assertEqual(self.log[3][:3], (self.form_repeater.get_url(rec), 200, xform_xml)) self.assertIn('received-on', self.log[3][3]) repeat_records = RepeatRecord.all( domain=self.domain, due_before=next_check_time, ) for repeat_record in repeat_records: self.assertEqual(repeat_record.succeeded, True) self.assertEqual(repeat_record.next_check, None) self.assertEqual(len(self.repeat_records(self.domain)), 0) self.post_xml(update_xform_xml, self.domain) self.assertEqual(len(self.repeat_records(self.domain)), 2)