def test_case_pillow_indicators(self):
since = get_current_seq(XFormInstance.get_db())
self._save_doc_to_db('indicator_form.json', XFormInstance)
case_id = self._save_doc_to_db('indicator_case.json', CommCareCase)
case_instance = CommCareCase.get(case_id)
# FormDataInCaseIndicatorDef (For those forgotten properties)
forgotten_property = FormDataInCaseIndicatorDefinition.increment_or_create_unique(
INDICATOR_TEST_NAMESPACE,
INDICATOR_TEST_DOMAIN,
slug='club_name',
question_id='location.club',
case_type='song_tag',
xmlns='http://openrosa.org/formdesigner/indicator-create-xmlns',
)
forgotten_property.save()
self.case_pillow.process_changes(since=since, forever=False)
indicator_case = IndicatorCase.get(case_id)
self.assertEqual(indicator_case.get_id, case_instance.get_id)
self.assertNotEqual(indicator_case.get_db().dbname,
case_instance.get_db().dbname)
self.assertNotEqual(indicator_case.computed_, {})
def handle_one(self, domain, case_type, chunk_size):
self.log('Copying {case_type} cases in {domain}'
.format(case_type=case_type, domain=domain))
old_db = CommCareCase.get_db()
new_db = IndicatorCase.get_db()
assert old_db.uri != new_db.uri
# this dbaccessor pulls from old_db
case_ids = get_case_ids_in_domain(domain, case_type)
self.delete_bad_doc_types(case_ids, chunk_size)
case_dict_chunks = chunked(iter_docs(old_db, case_ids, chunk_size),
chunk_size)
for case_dicts in case_dict_chunks:
for case_dict in case_dicts:
del case_dict['_rev']
case_dict.pop('_attachments', None)
case_dict['doc_type'] = "IndicatorCase"
try:
results = new_db.bulk_save(case_dicts)
except BulkSaveError as error:
results = error.results
for result in results:
if result.get('error') == 'conflict':
self.log('- OK: [{id}] is already in the indicator db'
.format(id=result.get('id')))
elif 'error' in result:
self.log('- ERROR: [{id}] ({result})'.format(
id=result.get('id'),
result=json.dumps(result)
))
else:
self.log('- ADDED: [{id}] saved to indicator db'.format(
id=result.get('id')
))
def test_case_pillow_indicators(self):
self._save_doc_to_db('indicator_form.json', XFormInstance)
case_id = self._save_doc_to_db('indicator_case.json', CommCareCase)
case_instance = CommCareCase.get(case_id)
# FormDataInCaseIndicatorDef (For those forgotten properties)
forgotten_property = FormDataInCaseIndicatorDefinition.increment_or_create_unique(
INDICATOR_TEST_NAMESPACE,
INDICATOR_TEST_DOMAIN,
slug='club_name',
question_id='location.club',
case_type='song_tag',
xmlns='http://openrosa.org/formdesigner/indicator-create-xmlns',
)
forgotten_property.save()
self.case_pillow.process_changes(since=None, forever=False)
indicator_case = IndicatorCase.get(case_id)
self.assertEqual(indicator_case.get_id, case_instance.get_id)
self.assertNotEqual(
indicator_case.get_db().dbname, case_instance.get_db().dbname
)
self.assertNotEqual(indicator_case.computed_, {})
def delete_bad_doc_types(self, case_ids, chunk_size):
"""
No view in this db includes CommCareCases, so check manually
"""
db = IndicatorCase.get_db()
case_dict_chunks = chunked(iter_docs(db, case_ids, chunk_size),
chunk_size)
to_delete = []
for case_dicts in case_dict_chunks:
for case_dict in case_dicts:
if case_dict['doc_type'] == 'CommCareCase':
to_delete.append(case_dict['_id'])
elif case_dict['doc_type'] != 'IndicatorCase':
raise Exception("Unexpected case type {} found"
.format(case_dict['doc_type']))
assert db.uri != CommCareCase.get_db().uri
print "Deleting {} docs from db '{}'".format(len(to_delete), db.dbname)
iter_bulk_delete(db, to_delete, chunk_size)
def delete_bad_doc_types(self, case_ids, chunk_size):
"""
No view in this db includes CommCareCases, so check manually
"""
db = IndicatorCase.get_db()
case_dict_chunks = chunked(iter_docs(db, case_ids, chunk_size),
chunk_size)
to_delete = []
for case_dicts in case_dict_chunks:
for case_dict in case_dicts:
if case_dict['doc_type'] == 'CommCareCase':
to_delete.append(case_dict['_id'])
elif case_dict['doc_type'] != 'IndicatorCase':
raise Exception("Unexpected case type {} found"
.format(case_dict['doc_type']))
assert db.uri != CommCareCase.get_db().uri
print "Deleting {} docs from db '{}'".format(len(to_delete), db.dbname)
iter_bulk_delete(db, to_delete, chunk_size)
def handle_one(self, domain, case_type, chunk_size):
self.log('Copying {case_type} cases in {domain}'
.format(case_type=case_type, domain=domain))
old_db = CommCareCase.get_db()
new_db = IndicatorCase.get_db()
assert old_db.uri != new_db.uri
case_ids = old_db.view(
'case/all_cases',
startkey=["all type", domain, case_type],
endkey=["all type", domain, case_type, {}],
reduce=False,
wrapper=lambda x: x['id']
).all()
self.delete_bad_doc_types(case_ids, chunk_size)
case_dict_chunks = chunked(iter_docs(old_db, case_ids, chunk_size),
chunk_size)
for case_dicts in case_dict_chunks:
for case_dict in case_dicts:
del case_dict['_rev']
case_dict.pop('_attachments', None)
case_dict['doc_type'] = "IndicatorCase"
try:
results = new_db.bulk_save(case_dicts)
except BulkSaveError as error:
results = error.results
for result in results:
if result.get('error') == 'conflict':
self.log('- OK: [{id}] is already in the indicator db'
.format(id=result.get('id')))
elif 'error' in result:
self.log('- ERROR: [{id}] ({result})'.format(
id=result.get('id'),
result=json.dumps(result)
))
else:
self.log('- ADDED: [{id}] saved to indicator db'.format(
id=result.get('id')
))
def test_case_pillow_indicators(self):
self._save_doc_to_db("indicator_form.json", XFormInstance)
case_id = self._save_doc_to_db("indicator_case.json", CommCareCase)
case_instance = CommCareCase.get(case_id)
# FormDataInCaseIndicatorDef (For those forgotten properties)
forgotten_property = FormDataInCaseIndicatorDefinition.increment_or_create_unique(
INDICATOR_TEST_NAMESPACE,
INDICATOR_TEST_DOMAIN,
slug="club_name",
question_id="location.club",
case_type="song_tag",
xmlns="http://openrosa.org/formdesigner/indicator-create-xmlns",
)
forgotten_property.save()
self.case_pillow.run_burst()
indicator_case = IndicatorCase.get(case_id)
self.assertEqual(indicator_case.get_id, case_instance.get_id)
self.assertNotEqual(indicator_case.get_db().dbname, case_instance.get_db().dbname)
self.assertNotEqual(indicator_case.computed_, {})
def process_indicators_for_case(namespaces, domain, doc_dict):
case_type = doc_dict.get('type')
if not case_type:
return
case_indicator_defs = []
for namespace in namespaces:
case_indicator_defs.extend(CaseIndicatorDefinition.get_all(
namespace,
domain,
case_type=case_type
))
try:
indicator_case = IndicatorCase.wrap_for_indicator_db(doc_dict)
indicator_case.update_indicators_in_bulk(
case_indicator_defs, logger=pillow_logging,
save_on_update=False
)
indicator_case.save()
except Exception as e:
pillow_logging.error(
"Error creating for MVP Indicator for form %(form_id)s: "
"%(error)s" % {
'form_id': doc_dict['_id'],
'error': e,
},
)
raise
# Now Update Data From Case to All Related Xforms (ewwww)
xform_ids = doc_dict.get('xform_ids', [])
if not xform_ids:
return
for xform_id in xform_ids:
related_xform_indicators = []
try:
# first try to get the doc from the indicator DB
xform_doc = IndicatorXForm.get(xform_id)
except ResourceNotFound:
# if that fails fall back to the main DB
try:
xform_dict = XFormInstance.get_db().get(xform_id)
xform_doc = IndicatorXForm.wrap_for_indicator_db(xform_dict)
related_xform_indicators = get_form_indicators(namespaces, domain, xform_doc.xmlns)
except ResourceNotFound:
pillow_logging.error(
"Could not find an XFormInstance with id %(xform_id)s "
"related to Case %(case_id)s" % {
'xform_id': xform_id,
'case_id': doc_dict['_id'],
}
)
continue
if not xform_doc.xmlns:
continue
for namespace in namespaces:
related_xform_indicators.extend(
CaseDataInFormIndicatorDefinition.get_all(
namespace,
domain,
xmlns=xform_doc.xmlns
)
)
xform_doc.update_indicators_in_bulk(
related_xform_indicators,
logger=pillow_logging,
save_on_update=False
)
xform_doc.save()
def process_indicators_for_case(namespaces, domain, doc_dict):
case_type = doc_dict.get('type')
if not case_type:
return
case_indicator_defs = []
for namespace in namespaces:
case_indicator_defs.extend(
CaseIndicatorDefinition.get_all(namespace,
domain,
case_type=case_type))
try:
indicator_case = IndicatorCase.wrap_for_indicator_db(doc_dict)
indicator_case.update_indicators_in_bulk(case_indicator_defs,
logger=pillow_logging,
save_on_update=False)
indicator_case.save()
except Exception as e:
pillow_logging.error(
"Error creating for MVP Indicator for form %(form_id)s: "
"%(error)s" % {
'form_id': doc_dict['_id'],
'error': e,
}, )
raise
# Now Update Data From Case to All Related Xforms (ewwww)
xform_ids = doc_dict.get('xform_ids', [])
if not xform_ids:
return
for xform_id in xform_ids:
related_xform_indicators = []
try:
# first try to get the doc from the indicator DB
xform_doc = IndicatorXForm.get(xform_id)
except ResourceNotFound:
# if that fails fall back to the main DB
try:
xform_dict = XFormInstance.get_db().get(xform_id)
xform_doc = IndicatorXForm.wrap_for_indicator_db(xform_dict)
related_xform_indicators = get_form_indicators(
namespaces, domain, xform_doc.xmlns)
except ResourceNotFound:
pillow_logging.error(
"Could not find an XFormInstance with id %(xform_id)s "
"related to Case %(case_id)s" % {
'xform_id': xform_id,
'case_id': doc_dict['_id'],
})
continue
if not xform_doc.xmlns:
continue
for namespace in namespaces:
related_xform_indicators.extend(
CaseDataInFormIndicatorDefinition.get_all(
namespace, domain, xmlns=xform_doc.xmlns))
xform_doc.update_indicators_in_bulk(related_xform_indicators,
logger=pillow_logging,
save_on_update=False)
xform_doc.save()
def process_indicators(self, namespaces, domain, doc_dict):
case_type = doc_dict.get('type')
if not case_type:
return
case_indicator_defs = []
for namespace in namespaces:
case_indicator_defs.extend(CaseIndicatorDefinition.get_all(
namespace,
domain,
case_type=case_type
))
if not case_indicator_defs:
return
try:
indicator_case = IndicatorCase.get_or_create_from_dict(doc_dict)[0]
indicator_case.update_indicators_in_bulk(
case_indicator_defs, logger=pillow_eval_logging
)
except Exception as e:
pillow_eval_logging.error(
"Error creating for MVP Indicator for form %(form_id)s: "
"%(error)s" % {
'form_id': doc_dict['_id'],
'error': e,
},
)
# Now Update Data From Case to All Related Xforms (ewwww)
xform_ids = doc_dict.get('xform_ids', [])
if not xform_ids:
return
for xform_id in xform_ids:
try:
xform_dict = XFormInstance.get_db().get(xform_id)
xform_doc = IndicatorXForm.get_or_create_from_dict(xform_dict)[0]
except ResourceNotFound:
pillow_eval_logging.error(
"Could not find an XFormInstance with id %(xform_id)s "
"related to Case %(case_id)s" % {
'xform_id': xform_id,
'case_id': doc_dict['_id'],
}
)
continue
if not xform_doc.xmlns:
continue
related_xform_defs = []
for namespace in namespaces:
related_xform_defs.extend(
CaseDataInFormIndicatorDefinition.get_all(
namespace,
domain,
xmlns=xform_doc.xmlns
)
)
xform_doc.update_indicators_in_bulk(
related_xform_defs,
logger=pillow_eval_logging,
)
