def test_repeat_record_created(self):
'''
Tests that whenever an application with a repeater is saved that a repeat record is created.
'''
application = Application(domain=self.domain)
application.save()
repeat_records = RepeatRecord.all(domain=self.domain,
due_before=datetime.utcnow())
self.assertEqual(len(repeat_records), 0)
app_structure_repeater = AppStructureRepeater(domain=self.domain,
url=self.forwarding_url)
app_structure_repeater.save()
application.save()
repeat_records = RepeatRecord.all(domain=self.domain,
due_before=datetime.utcnow())
self.assertEqual(len(repeat_records), 1)
for repeat_record in repeat_records:
self.assertEqual(repeat_record.url, self.forwarding_url)
self.assertEqual(repeat_record.get_payload(), application.get_id)
repeat_record.delete()
application.delete()
app_structure_repeater.delete()
def testLocks(self):
r = RepeatRecord(domain='test')
r.save()
r2 = RepeatRecord.get(r._id)
self.assertTrue(r.acquire_lock(datetime.utcnow()))
r3 = RepeatRecord.get(r._id)
self.assertFalse(r2.acquire_lock(datetime.utcnow()))
self.assertFalse(r3.acquire_lock(datetime.utcnow()))
r.release_lock()
r4 = RepeatRecord.get(r._id)
self.assertTrue(r4.acquire_lock(datetime.utcnow()))
def testLocks(self):
r = RepeatRecord(domain='test')
r.save()
r2 = RepeatRecord.get(r._id)
self.assertTrue(r.acquire_lock(datetime.utcnow()))
r3 = RepeatRecord.get(r._id)
self.assertFalse(r2.acquire_lock(datetime.utcnow()))
self.assertFalse(r3.acquire_lock(datetime.utcnow()))
r.release_lock()
r4 = RepeatRecord.get(r._id)
self.assertTrue(r4.acquire_lock(datetime.utcnow()))
def check_repeaters():
start = datetime.utcnow()
LIMIT = 100
while True:
number_locked = 0
# take LIMIT records off the top
# the assumption is that they all get 'popped' in the for loop
# the only exception I can see is if there's a problem with the
# locking, a large number of locked tasks could pile up at the top,
# so make a provision for that worst case
repeat_records = RepeatRecord.all(due_before=start,
limit=LIMIT + number_locked)
if repeat_records.count() <= number_locked:
# don't keep spinning if there's nothing left to fetch
return
for repeat_record in repeat_records:
now = datetime.utcnow()
# abort if taking too long, so the next task can take over
if now - start > CHECK_REPEATERS_INTERVAL:
return
if repeat_record.acquire_lock(start):
repeat_record.fire()
repeat_record.save()
repeat_record.release_lock()
else:
number_locked += 1
def tearDown(self):
self.case_repeater.delete()
self.form_repeater.delete()
XFormInstance.get(instance_id).delete()
repeat_records = RepeatRecord.all()
for repeat_record in repeat_records:
repeat_record.delete()
def tearDown(self):
self.case_repeater.delete()
self.form_repeater.delete()
XFormInstance.get(instance_id).delete()
repeat_records = RepeatRecord.all()
for repeat_record in repeat_records:
repeat_record.delete()
def handle(self, *args, **options):
domain = args[0]
db = Domain.get_db()
doc_ids = [
r['id'] for r in db.view(
'domain/docs',
startkey=[domain, 'RepeatRecord'],
endkey=[domain, 'RepeatRecord', {}],
reduce=False,
)
]
count = len(doc_ids)
print 'found %s doc ids' % count
latest = datetime.min
latest_doc = None
for i, doc in enumerate(iter_docs(db, doc_ids)):
wrapped = RepeatRecord.wrap(doc)
if i % 100 == 0:
print 'checked %s / %s' % (i, count)
if wrapped.last_checked and wrapped.last_checked > latest:
latest = wrapped.last_checked
latest_doc = wrapped
print 'new latest: %s' % latest
if latest_doc:
print 'latest repeater date is %s' % latest
print 'latest repeater is %s' % latest_doc._id
else:
print 'no relevant repeaters found'
def check_repeaters():
start = datetime.utcnow()
LIMIT = 100
while True:
number_locked = 0
# take LIMIT records off the top
# the assumption is that they all get 'popped' in the for loop
# the only exception I can see is if there's a problem with the
# locking, a large number of locked tasks could pile up at the top,
# so make a provision for that worst case
repeat_records = RepeatRecord.all(due_before=start, limit=LIMIT + number_locked)
if repeat_records.count() <= number_locked:
# don't keep spinning if there's nothing left to fetch
return
for repeat_record in repeat_records:
now = datetime.utcnow()
# abort if taking too long, so the next task can take over
if now - start > CHECK_REPEATERS_INTERVAL:
return
if repeat_record.acquire_lock(start):
repeat_record.fire()
repeat_record.save()
repeat_record.release_lock()
else:
number_locked += 1
def check_repeaters(request, domain):
if request.method == 'GET':
return HttpResponse(PAGE % {'status': '', 'domain': domain})
elif request.method == 'POST':
start = datetime.utcnow()
repeat_records = RepeatRecord.all(domain, due_before=start, limit=100)
process_repeater_list(repeat_records)
return HttpResponse(PAGE % {'status': 'Done', 'domain': domain})
def check_repeaters(request, domain):
if request.method == 'GET':
return HttpResponse(PAGE % {'status': '', 'domain': domain})
elif request.method == 'POST':
start = datetime.utcnow()
repeat_records = RepeatRecord.all(domain, due_before=start, limit=100)
process_repeater_list(repeat_records)
return HttpResponse(PAGE % {'status': 'Done', 'domain': domain})
def test_repeater(self):
CommCareCase.get(case_id)
now = datetime.utcnow()
repeat_records = RepeatRecord.all(domain=self.domain, due_before=now)
self.assertEqual(len(repeat_records), 2)
self.clear_log()
for repeat_record in repeat_records:
repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404]))
repeat_record.save()
for (url, status, data) in self.log:
self.assertEqual(status, 404)
self.clear_log()
in30min = now + timedelta(minutes=30)
repeat_records = RepeatRecord.all(domain=self.domain, due_before=now + timedelta(minutes=15))
self.assertEqual(len(repeat_records), 0)
repeat_records = RepeatRecord.all(domain=self.domain, due_before=in30min + timedelta(seconds=1))
self.assertEqual(len(repeat_records), 2)
for repeat_record in repeat_records:
self.assertLess(abs(in30min - repeat_record.next_check), timedelta(seconds=1))
repeat_record.fire(post_fn=self.make_post_fn([404, 200]))
repeat_record.save()
self.assertEqual(len(self.log), 4)
self.assertEqual(self.log[1], (self.form_repeater.url, 200, xform_xml))
self.assertEqual(self.log[3][:2], (self.case_repeater.url, 200))
check_xml_line_by_line(self, self.log[3][2], case_block)
repeat_records = RepeatRecord.all(domain=self.domain, due_before=in30min)
for repeat_record in repeat_records:
self.assertEqual(repeat_record.succeeded, True)
self.assertEqual(repeat_record.next_check, None)
def check_repeaters():
now = datetime.utcnow()
repeat_records = RepeatRecord.all(due_before=now)
for repeat_record in repeat_records:
repeat_record.fire()
try:
repeat_record.save()
except ResourceConflict:
logging.error("ResourceConflict with repeat_record %s: %s" % (repeat_record.get_id, repeat_record.to_json()))
raise
def handle(self, *args, **options):
if len(args) == 1:
domain = args[0]
else:
raise CommandError("Usage: %s\n%s" % (self.args, self.help))
next_year = datetime.datetime.now() + datetime.timedelta(days=365)
records = RepeatRecord.all(domain=domain, due_before=next_year)
for record in records:
record.fire(post_fn=simple_post)
record.save()
print "{} {}".format(record._id, "successful" if record.succeeded else "failed")
def loop():
# take LIMIT records off the top
# the assumption is that they all get 'popped' in the for loop
# the only exception I can see is if there's a problem with the
# locking, a large number of locked tasks could pile up at the top,
# so make a provision for that worst case
number_locked = progress_report['number_locked']
repeat_records = RepeatRecord.all(due_before=start,
limit=LIMIT + number_locked)
return process_repeater_list(repeat_records,
start=start,
cutoff=start + CHECK_REPEATERS_INTERVAL,
progress_report=progress_report)
def handle(self, *args, **options):
if len(args) == 1:
domain = args[0]
else:
raise CommandError('Usage: %s\n%s' % (self.args, self.help))
next_year = datetime.datetime.utcnow() + datetime.timedelta(days=365)
records = RepeatRecord.all(domain=domain, due_before=next_year)
for record in records:
record.fire(post_fn=simple_post)
record.save()
print '{} {}'.format(
record._id, 'successful' if record.succeeded else 'failed')
def test_repeater(self):
CommCareCase.get(case_id)
def now():
return datetime.utcnow()
repeat_records = RepeatRecord.all(domain=self.domain, due_before=now())
self.assertEqual(len(repeat_records), 2)
self.clear_log()
for repeat_record in repeat_records:
repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404]))
repeat_record.save()
for (url, status, data) in self.log:
self.assertEqual(status, 404)
self.clear_log()
next_check_time = now() + timedelta(minutes=60)
repeat_records = RepeatRecord.all(domain=self.domain,
due_before=now() +
timedelta(minutes=15))
self.assertEqual(len(repeat_records), 0)
repeat_records = RepeatRecord.all(domain=self.domain,
due_before=next_check_time +
timedelta(seconds=2))
self.assertEqual(len(repeat_records), 2)
for repeat_record in repeat_records:
self.assertLess(abs(next_check_time - repeat_record.next_check),
timedelta(seconds=2))
repeat_record.fire(post_fn=self.make_post_fn([404, 200]))
repeat_record.save()
self.assertEqual(len(self.log), 4)
self.assertEqual(self.log[1], (self.form_repeater.url, 200, xform_xml))
self.assertEqual(self.log[3][:2], (self.case_repeater.url, 200))
check_xml_line_by_line(self, self.log[3][2], case_block)
repeat_records = RepeatRecord.all(domain=self.domain,
due_before=next_check_time)
for repeat_record in repeat_records:
self.assertEqual(repeat_record.succeeded, True)
self.assertEqual(repeat_record.next_check, None)
repeat_records = RepeatRecord.all(domain=self.domain, due_before=now())
self.assertEqual(len(repeat_records), 0)
self.post_xml(update_xform_xml)
repeat_records = RepeatRecord.all(domain=self.domain, due_before=now())
self.assertEqual(len(repeat_records), 2)
def test_ignore_document(self):
"""
When get_payload raises IgnoreDocument, fire should call update_success
"""
@RegisterGenerator(FormRepeater, 'new_format', 'XML')
class NewFormGenerator(BasePayloadGenerator):
def get_payload(self, repeat_record, payload_doc):
raise IgnoreDocument
repeat_records = RepeatRecord.all(domain=self.domain, )
for repeat_record_ in repeat_records:
repeat_record_.fire()
self.assertIsNone(repeat_record_.next_check)
self.assertTrue(repeat_record_.succeeded)
def test_repeater(self):
CommCareCase.get(case_id)
def now():
return datetime.utcnow()
repeat_records = RepeatRecord.all(domain=self.domain, due_before=now())
self.assertEqual(len(repeat_records), 2)
self.clear_log()
for repeat_record in repeat_records:
repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404]))
repeat_record.save()
for (url, status, data, headers) in self.log:
self.assertEqual(status, 404)
self.clear_log()
next_check_time = now() + timedelta(minutes=60)
repeat_records = RepeatRecord.all(domain=self.domain, due_before=now() + timedelta(minutes=15))
self.assertEqual(len(repeat_records), 0)
repeat_records = RepeatRecord.all(domain=self.domain, due_before=next_check_time + timedelta(seconds=2))
self.assertEqual(len(repeat_records), 2)
for repeat_record in repeat_records:
self.assertLess(abs(next_check_time - repeat_record.next_check), timedelta(seconds=2))
repeat_record.fire(post_fn=self.make_post_fn([404, 200]))
repeat_record.save()
self.assertEqual(len(self.log), 4)
self.assertEqual(self.log[1][:3], (self.form_repeater.url, 200, xform_xml))
self.assertIn('received-on', self.log[1][3])
self.assertEqual(self.log[3][:2], (self.case_repeater.url, 200))
self.assertIn('server-modified-on', self.log[3][3])
check_xml_line_by_line(self, self.log[3][2], case_block)
repeat_records = RepeatRecord.all(domain=self.domain, due_before=next_check_time)
for repeat_record in repeat_records:
self.assertEqual(repeat_record.succeeded, True)
self.assertEqual(repeat_record.next_check, None)
repeat_records = RepeatRecord.all(domain=self.domain, due_before=now())
self.assertEqual(len(repeat_records), 0)
self.post_xml(update_xform_xml)
repeat_records = RepeatRecord.all(domain=self.domain, due_before=now())
self.assertEqual(len(repeat_records), 2)
def handle(self, *args, **options):
domain = args[0]
old_db = Domain.get_db()
new_db = RepeatRecord.get_db()
assert old_db.dbname != new_db.dbname
doc_ids = get_doc_ids(domain, 'RepeatRecord', old_db)
count = len(doc_ids)
chunksize = 250
for i, docs in enumerate(chunked(iter_docs(old_db, doc_ids, chunksize), chunksize)):
for doc in docs:
if '_rev' in doc:
del doc['_rev']
new_db.bulk_save(docs, new_edits=False)
print 'checked %s / %s' % (i * chunksize, count)
def test_repeat_record_created(self):
'''
Tests that whenever an application with a repeater is saved that a repeat record is created.
'''
application = Application(domain=self.domain)
application.save()
repeat_records = RepeatRecord.all(domain=self.domain, due_before=datetime.utcnow())
self.assertEqual(len(repeat_records), 0)
app_structure_repeater = AppStructureRepeater(domain=self.domain, url=self.forwarding_url)
app_structure_repeater.save()
application.save()
repeat_records = RepeatRecord.all(domain=self.domain, due_before=datetime.utcnow())
self.assertEqual(len(repeat_records), 1)
for repeat_record in repeat_records:
self.assertEqual(repeat_record.url, self.forwarding_url)
self.assertEqual(repeat_record.get_payload(), application.get_id)
repeat_record.delete()
application.delete()
app_structure_repeater.delete()
def loop():
# take LIMIT records off the top
# the assumption is that they all get 'popped' in the for loop
# the only exception I can see is if there's a problem with the
# locking, a large number of locked tasks could pile up at the top,
# so make a provision for that worst case
number_locked = progress_report['number_locked']
repeat_records = RepeatRecord.all(
due_before=start,
limit=LIMIT + number_locked
)
return process_repeater_list(
repeat_records,
start=start,
cutoff=start + CHECK_REPEATERS_INTERVAL,
progress_report=progress_report
)
def test_ignore_document(self):
"""
When get_payload raises IgnoreDocument, fire should call update_success
"""
@RegisterGenerator(FormRepeater, 'new_format', 'XML')
class NewFormGenerator(BasePayloadGenerator):
def get_payload(self, repeat_record, payload_doc):
raise IgnoreDocument
repeat_records = RepeatRecord.all(
domain=self.domain,
)
for repeat_record_ in repeat_records:
repeat_record_.fire()
self.assertIsNone(repeat_record_.next_check)
self.assertTrue(repeat_record_.succeeded)
def handle(self, *args, **options):
domain = args[0]
old_db = Domain.get_db()
new_db = RepeatRecord.get_db()
assert old_db.dbname != new_db.dbname
doc_ids = [
r["id"]
for r in old_db.view(
"domain/docs", startkey=[domain, "RepeatRecord"], endkey=[domain, "RepeatRecord", {}], reduce=False
)
]
count = len(doc_ids)
chunksize = 250
for i, docs in enumerate(chunked(iter_docs(old_db, doc_ids, chunksize), chunksize)):
for doc in docs:
if "_rev" in doc:
del doc["_rev"]
new_db.bulk_save(docs, new_edits=False)
print "checked %s / %s" % (i * chunksize, count)
def handle(self, *args, **options):
domain = args[0]
db = Domain.get_db()
doc_ids = get_doc_ids(domain, "RepeatRecord", db)
count = len(doc_ids)
print "found %s doc ids" % count
latest = datetime.min
latest_doc = None
for i, doc in enumerate(iter_docs(db, doc_ids)):
wrapped = RepeatRecord.wrap(doc)
if i % 100 == 0:
print "checked %s / %s" % (i, count)
if wrapped.last_checked and wrapped.last_checked > latest:
latest = wrapped.last_checked
latest_doc = wrapped
print "new latest: %s" % latest
if latest_doc:
print "latest repeater date is %s" % latest
print "latest repeater is %s" % latest_doc._id
else:
print "no relevant repeaters found"
def loop():
number_locked = 0
# take LIMIT records off the top
# the assumption is that they all get 'popped' in the for loop
# the only exception I can see is if there's a problem with the
# locking, a large number of locked tasks could pile up at the top,
# so make a provision for that worst case
repeat_records = RepeatRecord.all(
due_before=start,
limit=LIMIT + number_locked
)
if repeat_records.count() <= number_locked:
# don't keep spinning if there's nothing left to fetch
return False
for repeat_record in repeat_records:
now = datetime.utcnow()
# abort if taking too long, so the next task can take over
if now - start > CHECK_REPEATERS_INTERVAL:
return False
if repeat_record.acquire_lock(start):
if repeat_record.repeater.doc_type.endswith(DELETED):
if not repeat_record.doc_type.endswith(DELETED):
repeat_record.doc_type += DELETED
progress_report['deleted'].append(repeat_record.get_id)
else:
repeat_record.fire()
if repeat_record.succeeded:
progress_report['success'].append(repeat_record.get_id)
else:
progress_report['fail'].append(repeat_record.get_id)
repeat_record.save()
repeat_record.release_lock()
else:
progress_report['locked'].append(repeat_record.get_id)
number_locked += 1
return True
def check_inline_form_repeaters(post_fn=None):
"""old-style FormRepeater grandfathered in"""
now = datetime.utcnow()
forms = XFormInstance.view('receiverwrapper/forms_with_pending_repeats',
startkey="",
endkey=json_format_datetime(now),
include_docs=True,
)
for form in forms:
if hasattr(form, 'repeats'):
for repeat_record in form.repeats:
record = dict(repeat_record)
url = record.pop('url')
record = RepeatRecord.wrap(record)
record.payload_id = form.get_id
record._repeater = FormRepeater(url=url)
record.fire(post_fn=post_fn)
record = record.to_json()
record['url'] = url
repeat_record.update(record)
form.save()
def handle(self, *args, **options):
domain = args[0]
old_db = Domain.get_db()
new_db = RepeatRecord.get_db()
assert old_db.dbname != new_db.dbname
doc_ids = [
r['id'] for r in old_db.view(
'domain/docs',
startkey=[domain, 'RepeatRecord'],
endkey=[domain, 'RepeatRecord', {}],
reduce=False,
)
]
count = len(doc_ids)
chunksize = 250
for i, docs in enumerate(
chunked(iter_docs(old_db, doc_ids, chunksize), chunksize)):
for doc in docs:
if '_rev' in doc:
del doc['_rev']
new_db.bulk_save(docs, new_edits=False)
print 'checked %s / %s' % (i * chunksize, count)
def repeat_records(self):
return RepeatRecord.all(domain=self.domain, due_before=datetime.utcnow())
def repeat_records(cls, domain_name):
return RepeatRecord.all(domain=domain_name, due_before=datetime.utcnow())
def test_repeater(self):
CommCareCase.get(case_id)
def now():
return datetime.utcnow()
repeat_records = RepeatRecord.all(domain=self.domain, due_before=now())
self.assertEqual(len(repeat_records), 2)
self.clear_log()
for repeat_record in repeat_records:
repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404]))
repeat_record.save()
for (url, status, data, headers) in self.log:
self.assertEqual(status, 404)
self.clear_log()
next_check_time = now() + timedelta(minutes=60)
repeat_records = RepeatRecord.all(
domain=self.domain,
due_before=now() + timedelta(minutes=15),
)
self.assertEqual(len(repeat_records), 0)
repeat_records = RepeatRecord.all(
domain=self.domain,
due_before=next_check_time + timedelta(seconds=2),
)
self.assertEqual(len(repeat_records), 2)
for repeat_record in repeat_records:
self.assertLess(abs(next_check_time - repeat_record.next_check),
timedelta(seconds=2))
repeat_record.fire(post_fn=self.make_post_fn([404, 200]))
repeat_record.save()
self.assertEqual(len(self.log), 4)
# The following is pretty fickle and depends on which of
# - corehq.apps.receiverwrapper.signals
# - casexml.apps.case.signals
# gets loaded first.
# This is deterministic but easily affected by minor code changes
# check case stuff
self.assertEqual(self.log[1][:2], (self.case_repeater.url, 200))
self.assertIn('server-modified-on', self.log[1][3])
check_xml_line_by_line(self, self.log[1][2], case_block)
# check form stuff
self.assertEqual(self.log[3][:3],
(self.form_repeater.url, 200, xform_xml))
self.assertIn('received-on', self.log[3][3])
repeat_records = RepeatRecord.all(
domain=self.domain,
due_before=next_check_time,
)
for repeat_record in repeat_records:
self.assertEqual(repeat_record.succeeded, True)
self.assertEqual(repeat_record.next_check, None)
repeat_records = RepeatRecord.all(domain=self.domain, due_before=now())
self.assertEqual(len(repeat_records), 0)
self.post_xml(update_xform_xml)
repeat_records = RepeatRecord.all(domain=self.domain, due_before=now())
self.assertEqual(len(repeat_records), 2)
def tearDown(self):
self.repeater.delete()
repeat_records = RepeatRecord.all()
for repeat_record in repeat_records:
repeat_record.delete()
def test_repeater(self):
# this test should probably be divided into more units
CommCareCase.get(case_id)
def now():
return datetime.utcnow()
repeat_records = RepeatRecord.all(domain=self.domain, due_before=now())
self.assertEqual(len(repeat_records), 2)
self.clear_log()
records_by_repeater_id = {}
for repeat_record in repeat_records:
repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404]))
repeat_record.save()
records_by_repeater_id[repeat_record.repeater_id] = repeat_record
for (url, status, data, headers) in self.log:
self.assertEqual(status, 404)
self.clear_log()
next_check_time = now() + timedelta(minutes=60)
repeat_records = RepeatRecord.all(
domain=self.domain,
due_before=now() + timedelta(minutes=15),
)
self.assertEqual(len(repeat_records), 0)
repeat_records = RepeatRecord.all(
domain=self.domain,
due_before=next_check_time + timedelta(seconds=2),
)
self.assertEqual(len(repeat_records), 2)
for repeat_record in repeat_records:
self.assertLess(abs(next_check_time - repeat_record.next_check),
timedelta(seconds=3))
repeat_record.fire(post_fn=self.make_post_fn([404, 200]))
repeat_record.save()
self.assertEqual(len(self.log), 4)
# The following is pretty fickle and depends on which of
# - corehq.apps.receiverwrapper.signals
# - casexml.apps.case.signals
# gets loaded first.
# This is deterministic but easily affected by minor code changes
# check case stuff
rec = records_by_repeater_id[self.case_repeater.get_id]
self.assertEqual(self.log[1][:2], (self.case_repeater.get_url(rec), 200))
self.assertIn('server-modified-on', self.log[1][3])
check_xml_line_by_line(self, self.log[1][2], case_block)
# check form stuff
rec = records_by_repeater_id[self.form_repeater.get_id]
self.assertEqual(self.log[3][:3],
(self.form_repeater.get_url(rec), 200, xform_xml))
self.assertIn('received-on', self.log[3][3])
repeat_records = RepeatRecord.all(
domain=self.domain,
due_before=next_check_time,
)
for repeat_record in repeat_records:
self.assertEqual(repeat_record.succeeded, True)
self.assertEqual(repeat_record.next_check, None)
self.assertEqual(len(self.repeat_records(self.domain)), 0)
self.post_xml(update_xform_xml, self.domain)
self.assertEqual(len(self.repeat_records(self.domain)), 2)
def tearDown(self):
self.repeater.delete()
repeat_records = RepeatRecord.all()
for repeat_record in repeat_records:
repeat_record.delete()
def repeat_records(cls, domain_name):
return RepeatRecord.all(domain=domain_name,
due_before=datetime.utcnow())
