def handle(self, *args, **options):
count = 1
limit, batch, quiet = options['limit'], options['batch'], options['quiet']
while True:
if not quiet:
print 'Fetching new batch'
total, records = get_records_to_process('CASE XFORM MISMATCH', batch)
if not quiet:
print 'Records in batch:', total
if not total:
return
for record in records:
if 'case_id' not in record:
record['exception'].archive('system')
continue
case_id = record['case_id']
try:
rebuild_case(case_id)
record['exception'].archive('system')
if not quiet:
print 'rebuilt %s/%s cases' % (count, total)
except Exception, e:
logging.exception("couldn't rebuild case {id}. {msg}".format(id=case_id, msg=str(e)))
finally:
count += 1
def handle(self, *args, **options):
if len(args) == 1:
case_id = args[0]
else:
raise CommandError('Usage: %s\n%s' % (self.args, self.help))
rebuild_case(case_id)
def get_with_rebuild(cls, id):
try:
return cls.get(id)
except ResourceNotFound:
from casexml.apps.case.cleanup import rebuild_case
case = rebuild_case(id)
if case is None:
raise
return case
def testReconcileActions(self):
now = datetime.utcnow()
# make sure we timestamp everything so they have the right order
case_id = post_util(create=True, form_extras={'received_on': now})
post_util(case_id=case_id,
p1='p1-1',
p2='p2-1',
form_extras={'received_on': now + timedelta(seconds=1)})
post_util(case_id=case_id,
p2='p2-2',
p3='p3-2',
form_extras={'received_on': now + timedelta(seconds=2)})
case = CommCareCase.get(case_id)
original_actions = [deepcopy(a) for a in case.actions]
original_form_ids = [id for id in case.xform_ids]
self.assertEqual(3, len(original_actions))
self.assertEqual(3, len(original_form_ids))
self._assertListEqual(original_actions, case.actions)
# test reordering
case.actions = [case.actions[2], case.actions[1], case.actions[0]]
self._assertListNotEqual(original_actions, case.actions)
case.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication
case.actions = case.actions * 3
self.assertEqual(9, len(case.actions))
self._assertListNotEqual(original_actions, case.actions)
case.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication, even when dates are off
case.actions = original_actions + [deepcopy(case.actions[2])]
case.actions[-1].server_date = case.actions[
-1].server_date + timedelta(seconds=1)
self._assertListNotEqual(original_actions, case.actions)
case.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication with different properties is actually
# treated differently
case.actions = original_actions + [deepcopy(case.actions[2])]
case.actions[-1].updated_unknown_properties['new'] = 'mismatch'
self.assertEqual(4, len(case.actions))
self._assertListNotEqual(original_actions, case.actions)
case.reconcile_actions()
self._assertListNotEqual(original_actions, case.actions)
# test clean slate rebuild
case = rebuild_case(case_id)
self._assertListEqual(original_actions, primary_actions(case))
self._assertListEqual(original_form_ids, case.xform_ids)
def tag_forms_as_deleted_rebuild_associated_cases(formlist, deletion_id, deleted_cases=None):
from casexml.apps.case.cleanup import rebuild_case
if deleted_cases is None:
deleted_cases = set()
cases_to_rebuild = set()
for form in formlist:
form['doc_type'] += DELETED_SUFFIX
form['-deletion_id'] = deletion_id
cases_to_rebuild.update(get_case_ids_from_form(form))
XFormInstance.get_db().bulk_save(formlist)
for case in cases_to_rebuild - deleted_cases:
rebuild_case(case)
def handle(self, *args, **options):
if len(args) == 1:
domain = args[0]
else:
raise CommandError('Usage: %s\n%s' % (self.args, self.help))
ids = get_case_ids_in_domain(domain)
for count, case_id in enumerate(ids):
try:
rebuild_case(case_id)
if count % 100 == 0:
print 'rebuilt %s/%s cases' % (count, len(ids))
except Exception, e:
logging.exception("couldn't rebuild case {id}. {msg}".format(id=case_id, msg=str(e)))
def handle(self, *args, **options):
if len(args) == 1:
domain = args[0]
else:
raise CommandError('Usage: %s\n%s' % (self.args, self.help))
ids = get_case_ids_in_domain(domain)
for count, case_id in enumerate(ids):
try:
rebuild_case(case_id)
if count % 100 == 0:
print 'rebuilt %s/%s cases' % (count, len(ids))
except Exception, e:
logging.exception("couldn't rebuild case {id}. {msg}".format(
id=case_id, msg=str(e)))
def testRebuildEmpty(self):
self.assertEqual(None, rebuild_case('notarealid'))
try:
CommCareCase.get_with_rebuild('notarealid')
self.fail('get with rebuild should still fail for unknown cases')
except ResourceNotFound:
pass
def _rebuild_case_with_retries(self, case_id):
"""
Rebuild a case with retries
- retry in 5 min if failure occurs after (default_retry_delay)
- retry a total of 3 times
"""
from casexml.apps.case.cleanup import rebuild_case
try:
rebuild_case(case_id)
except (UnexpectedDeletedXForm, ResourceConflict) as exc:
try:
self.retry(exc=exc)
except MaxRetriesExceededError:
notify_exception(
"Maximum Retries Exceeded while rebuilding case {} during deletion.".format(case_id)
)
def testRebuildEmpty(self):
self.assertEqual(None, rebuild_case('notarealid'))
try:
CommCareCase.get_with_rebuild('notarealid')
self.fail('get with rebuild should still fail for unknown cases')
except ResourceNotFound:
pass
def handle_problematic_data(datalist_tup, csv_writer, verbose=False, rebuild=False):
case_data = CommCareCase.get_db().view("_all_docs", keys=[d[1] for d in datalist_tup])
cases = set([c["id"] for c in case_data if "id" in c])
for domain, case_id, form_id, received_on in datalist_tup:
error = "action_missing" if case_id in cases else "nonexistent_case"
csv_writer.writerow([domain, case_id, form_id, received_on, error])
if verbose and error == "nonexistent_case":
logger.info("Case (%s) from form (%s) does not exist" % (case_id, form_id))
elif verbose and error == "action_missing":
logger.info("Case (%s) missing action for form (%s)" % (case_id, form_id))
if rebuild:
if verbose:
logger.info("rebuilding case (%s) from scratch" % case_id)
try:
rebuild_case(case_id)
except Exception as e:
logger.info("Case Rebuild Failure: %s" % e)
def handle_problematic_data(datalist_tup, csv_writer, verbose=False, rebuild=False):
case_data = CommCareCase.get_db().view('_all_docs', keys=[d[1] for d in datalist_tup])
cases = set([c["id"] for c in case_data if 'id' in c])
for domain, case_id, form_id, received_on in datalist_tup:
error = "action_missing" if case_id in cases else "nonexistent_case"
csv_writer.writerow([domain, case_id, form_id, received_on, error])
if verbose and error == "nonexistent_case":
logger.info("Case (%s) from form (%s) does not exist" % (case_id, form_id))
elif verbose and error == "action_missing":
logger.info("Case (%s) missing action for form (%s)" % (case_id, form_id))
if rebuild:
if verbose:
logger.info("rebuilding case (%s) from scratch" % case_id)
try:
rebuild_case(case_id)
except Exception as e:
logger.info("Case Rebuild Failure: %s" % e)
def get_with_rebuild(cls, id):
try:
return cls.get(id)
except ResourceNotFound:
from casexml.apps.case.cleanup import rebuild_case
case = rebuild_case(id)
if case is None:
raise
return case
def testRebuildCreateCase(self):
case_id = post_util(create=True)
post_util(case_id=case_id, p1='p1', p2='p2')
# delete initial case
case = CommCareCase.get(case_id)
case.delete()
case = rebuild_case(case_id)
self.assertEqual(case.p1, 'p1')
self.assertEqual(case.p2, 'p2')
self.assertEqual(2, len(primary_actions(case))) # create + update
case.delete()
try:
CommCareCase.get(case_id)
self.fail('get should fail on deleted cases')
except ResourceNotFound:
pass
case = CommCareCase.get_with_rebuild(case_id)
self.assertEqual(case.p1, 'p1')
self.assertEqual(case.p2, 'p2')
def testReconcileActions(self):
now = datetime.utcnow()
# make sure we timestamp everything so they have the right order
case_id = post_util(create=True, form_extras={'received_on': now})
post_util(case_id=case_id, p1='p1-1', p2='p2-1', form_extras={'received_on': now + timedelta(seconds=1)})
post_util(case_id=case_id, p2='p2-2', p3='p3-2', form_extras={'received_on': now + timedelta(seconds=2)})
case = CommCareCase.get(case_id)
original_actions = [deepcopy(a) for a in case.actions]
original_form_ids = [id for id in case.xform_ids]
self.assertEqual(3, len(original_actions))
self.assertEqual(3, len(original_form_ids))
self._assertListEqual(original_actions, case.actions)
# test reordering
case.actions = [case.actions[2], case.actions[1], case.actions[0]]
self._assertListNotEqual(original_actions, case.actions)
case.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication
case.actions = case.actions * 3
self.assertEqual(9, len(case.actions))
self._assertListNotEqual(original_actions, case.actions)
case.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication, even when dates are off
case.actions = original_actions + [deepcopy(case.actions[2])]
case.actions[-1].server_date = case.actions[-1].server_date + timedelta(seconds=1)
self._assertListNotEqual(original_actions, case.actions)
case.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication with different properties is actually
# treated differently
case.actions = original_actions + [deepcopy(case.actions[2])]
case.actions[-1].updated_unknown_properties['new'] = 'mismatch'
self.assertEqual(4, len(case.actions))
self._assertListNotEqual(original_actions, case.actions)
case.reconcile_actions()
self._assertListNotEqual(original_actions, case.actions)
# test clean slate rebuild
case = rebuild_case(case_id)
self._assertListEqual(original_actions, primary_actions(case))
self._assertListEqual(original_form_ids, case.xform_ids)
def testRebuildCreateCase(self):
case_id = post_util(create=True)
post_util(case_id=case_id, p1='p1', p2='p2')
# delete initial case
case = CommCareCase.get(case_id)
case.delete()
case = rebuild_case(case_id)
self.assertEqual(case.p1, 'p1')
self.assertEqual(case.p2, 'p2')
self.assertEqual(2, len(primary_actions(case))) # create + update
case.delete()
try:
CommCareCase.get(case_id)
self.fail('get should fail on deleted cases')
except ResourceNotFound:
pass
case = CommCareCase.get_with_rebuild(case_id)
self.assertEqual(case.p1, 'p1')
self.assertEqual(case.p2, 'p2')
def rebuild_case_view(request, domain, case_id):
case = get_document_or_404(CommCareCase, domain, case_id)
rebuild_case(case_id)
messages.success(request, _(u'Case %s was rebuilt from its forms.' % case.name))
return HttpResponseRedirect(reverse('case_details', args=[domain, case_id]))
@require_case_view_permission
@login_and_domain_required
@require_GET
def case_xml(request, domain, case_id):
case = _get_case_or_404(domain, case_id)
version = request.GET.get('version', V2)
return HttpResponse(case.to_xml(version), content_type='text/xml')
@require_case_view_permission
@require_permission(Permissions.edit_data)
@require_POST
def rebuild_case_view(request, domain, case_id):
case = _get_case_or_404(domain, case_id)
rebuild_case(case_id)
messages.success(request,
_(u'Case %s was rebuilt from its forms.' % case.name))
return HttpResponseRedirect(reverse('case_details', args=[domain,
case_id]))
@require_case_view_permission
@require_permission(Permissions.edit_data)
@require_POST
def close_case_view(request, domain, case_id):
case = _get_case_or_404(domain, case_id)
if case.closed:
messages.info(request, u'Case {} is already closed.'.format(case.name))
else:
form_id = close_case(case_id, domain, request.couch_user)
def rebuild_form_cases(sender, xform, *args, **kwargs):
from casexml.apps.case.xform import get_case_ids_from_form
from casexml.apps.case.cleanup import rebuild_case
for case_id in get_case_ids_from_form(xform):
rebuild_case(case_id)
def rebuild_case_view(request, domain, case_id):
case = _get_case_or_404(domain, case_id)
rebuild_case(case_id)
messages.success(request, _("Case %s was rebuilt from its forms." % case.name))
return HttpResponseRedirect(reverse("case_details", args=[domain, case_id]))
