def testReportXFormTransform(self):
form = XFORM_SINGLE_CASE
report_pillow = ReportXFormPillow(online=False)
form['domain'] = settings.ES_XFORM_FULL_INDEX_DOMAINS[0]
processed_form = report_pillow.change_transform(form)
mapping = report_pillow.default_mapping
#root level
for k, v in processed_form['form'].items():
if k in mapping['properties']['form']['properties']:
if isinstance(v, dict):
self.assertFalse(
'#value' in v,
msg=
"Error, processed dict contains a #value dict for key [%s] when it shouldn't"
% k)
else:
#it's not a dict, so that means it's ok
if k in form:
self.assertEqual(form[k], v)
else:
self.assertTrue(isinstance(v, dict))
if isinstance(form['form'][k], dict):
#if the original is a dict, then, make sure the keys are the same
self.assertFalse('#value' in v)
self.assertEqual(sorted(form['form'][k].keys()),
sorted(v.keys()))
else:
self.assertTrue('#value' in v)
self.assertEqual(form['form'][k], v['#value'])
def testBlanktoNulls(self):
orig = {
'_id': 'blank_strings',
'form': {
'case': {
"@xmlns": "http://commcarehq.org/case/transaction/v2",
"@date_modified": "2013-10-14T10:59:44Z",
"@user_id": "someuser",
"@case_id": "mycase",
"index": "",
"attachment": "",
"create": "",
"update": "",
}
}
}
dict_props = ['index', 'attachment', 'create', 'update']
pillow = ReportXFormPillow(online=False)
all_blank = copy.deepcopy(orig)
all_blank['domain'] = settings.ES_XFORM_FULL_INDEX_DOMAINS[0]
for_indexing = pillow.change_transform(all_blank)
for prop in dict_props:
self.assertIsNone(for_indexing['form']['case'][prop])
all_dicts = copy.deepcopy(orig)
all_dicts['domain'] = settings.ES_XFORM_FULL_INDEX_DOMAINS[0]
for prop in dict_props:
all_dicts['form']['case'][prop] = {}
for_index2 = pillow.change_transform(all_dicts)
for prop in dict_props:
self.assertIsNotNone(for_index2['form']['case'][prop])
def testReportXFormTransform(self):
form = XFORM_SINGLE_CASE
report_pillow = ReportXFormPillow(online=False)
form['domain'] = settings.ES_XFORM_FULL_INDEX_DOMAINS[0]
processed_form = report_pillow.change_transform(form)
mapping = report_pillow.default_mapping
#root level
for k, v in processed_form['form'].items():
if k in mapping['properties']['form']['properties']:
if isinstance(v, dict):
self.assertFalse('#value' in v, msg="Error, processed dict contains a #value dict for key [%s] when it shouldn't" % k)
else:
#it's not a dict, so that means it's ok
if k in form:
self.assertEqual(form[k], v)
else:
self.assertTrue(isinstance(v, dict))
if isinstance(form['form'][k], dict):
#if the original is a dict, then, make sure the keys are the same
self.assertFalse('#value' in v)
self.assertEqual(sorted(form['form'][k].keys()), sorted(v.keys()))
else:
self.assertTrue('#value' in v)
self.assertEqual(form['form'][k], v['#value'])
def finish_handle(self):
filepath = os.path.join(settings.FILEPATH, 'corehq','pillows','mappings','reportxform_mapping.py')
xform_pillow = ReportXFormPillow(create_index=False)
#current index
#check current index
aliased_indices = xform_pillow.check_alias()
current_index = xform_pillow.es_index
sys.stderr.write("current index:\n")
sys.stderr.write('REPORT_XFORM_INDEX="%s"\n' % current_index)
#regenerate the mapping dict
mapping = reportxform_mapping.REPORT_XFORM_MAPPING
xform_pillow.default_mapping = mapping
delattr(xform_pillow, '_calc_meta_cache')
calc_index = "%s_%s" % (xform_pillow.es_index_prefix, xform_pillow.calc_meta())
if calc_index not in aliased_indices and calc_index != current_index:
sys.stderr.write("\n\tWarning, current index %s is not aliased at the moment\n" % current_index)
sys.stderr.write("\tCurrent live aliased index: %s\n\n" % (','.join(aliased_indices)))
if calc_index != current_index:
sys.stderr.write("REPORT_XFORM_INDEX hash has changed, please update \n\t%s\n\tREPORT_XFORM_INDEX property with the line below:\n" % filepath)
sys.stdout.write('REPORT_XFORM_INDEX="%s"\n' % calc_index)
else:
sys.stderr.write("REPORT_XFORM_INDEX unchanged\n")
def testConvertAndRestoreReportXFormDicts(self):
pillow = ReportXFormPillow(online=False)
orig = CONCEPT_XFORM
orig['domain'] = settings.ES_XFORM_FULL_INDEX_DOMAINS[0]
for_indexing = pillow.change_transform(orig)
restored = restore_property_dict(for_indexing)
#appVersion might be munged in meta, so swapping it out
orig_appversion = orig['form']['meta']['appVersion']
restored_appversion = restored['form']['meta']['appVersion']
if isinstance(orig_appversion, dict):
self.assertEqual(restored_appversion, orig_appversion['#text'])
else:
self.assertEqual(restored_appversion, orig_appversion)
del(orig['form']['meta']['appVersion'])
del(restored['form']['meta']['appVersion'])
# user_type gets added in change_transform
del(restored['user_type'])
del(for_indexing['user_type'])
# inserted during the transform
del(restored['inserted_at'])
del(for_indexing['inserted_at'])
del(restored['form']['meta']['app_build_version'])
del(restored['form']['meta']['commcare_version'])
del(restored['form']['meta']['geo_point'])
self.assertNotEqual(for_indexing, orig)
self.assertNotEqual(for_indexing, restored)
self.assertEqual(orig, restored)
def testBlanktoNulls(self):
orig = {
'_id': 'blank_strings',
'form': {
'case': {
"@xmlns": "http://commcarehq.org/case/transaction/v2",
"@date_modified": "2013-10-14T10:59:44Z",
"@user_id": "someuser",
"@case_id": "mycase",
"index": "",
"attachment": "",
"create": "",
"update": "",
}
}
}
dict_props = ['index', 'attachment', 'create', 'update']
pillow = ReportXFormPillow(online=False)
all_blank = copy.deepcopy(orig)
all_blank['domain'] = settings.ES_XFORM_FULL_INDEX_DOMAINS[0]
for_indexing = pillow.change_transform(all_blank)
for prop in dict_props:
self.assertIsNone(for_indexing['form']['case'][prop])
all_dicts = copy.deepcopy(orig)
all_dicts['domain'] = settings.ES_XFORM_FULL_INDEX_DOMAINS[0]
for prop in dict_props:
all_dicts['form']['case'][prop] = {}
for_index2 = pillow.change_transform(all_dicts)
for prop in dict_props:
self.assertIsNotNone(for_index2['form']['case'][prop])
def testComputedConversion(self):
"""
Since we set dyanmic=True on reportxforms, need to do conversions on the computed_ properties
so call conversion on computed_ dict as well, this test ensures that it's converted on change_transform
:return:
"""
orig = {
'_id': 'blank_strings',
'form': {
'case': {
"@xmlns": "http://commcarehq.org/case/transaction/v2",
"@date_modified": "2013-10-14T10:59:44Z",
"@user_id": "someuser",
"@case_id": "mycase",
"index": "",
"attachment": "",
"create": "",
"update": "",
}
},
'computed_': {
"mvp_indicators": {
"last_muac": {
"updated": "2013-02-04T21:54:28Z",
"version": 1,
"type": "FormDataAliasIndicatorDefinition",
"multi_value": False,
"value": None
},
"muac": {
"updated": "2013-02-04T21:54:28Z",
"version": 1,
"type": "FormDataAliasIndicatorDefinition",
"multi_value": False,
"value": {
"#text": "",
"@concept_id": "1343"
}
},
"vaccination_status": {
"updated": "2013-02-04T21:54:28Z",
"version": 1,
"type": "FormDataAliasIndicatorDefinition",
"multi_value": False,
"value": "yes"
},
}
}
}
pillow = ReportXFormPillow(online=False)
orig['domain'] = settings.ES_XFORM_FULL_INDEX_DOMAINS[0]
for_indexing = pillow.change_transform(orig)
restored = restore_property_dict(for_indexing)
self.assertNotEqual(orig['computed_'], for_indexing['computed_'])
self.assertEqual(orig['computed_'], restored['computed_'])
def testComputedConversion(self):
"""
Since we set dyanmic=True on reportxforms, need to do conversions on the computed_ properties
so call conversion on computed_ dict as well, this test ensures that it's converted on change_transform
:return:
"""
orig = {
'_id': 'blank_strings',
'form': {
'case': {
"@xmlns": "http://commcarehq.org/case/transaction/v2",
"@date_modified": "2013-10-14T10:59:44Z",
"@user_id": "someuser",
"@case_id": "mycase",
"index": "",
"attachment": "",
"create": "",
"update": "",
}
},
'computed_': {
"mvp_indicators": {
"last_muac": {
"updated": "2013-02-04T21:54:28Z",
"version": 1,
"type": "FormDataAliasIndicatorDefinition",
"multi_value": False,
"value": None
},
"muac": {
"updated": "2013-02-04T21:54:28Z",
"version": 1,
"type": "FormDataAliasIndicatorDefinition",
"multi_value": False,
"value": {
"#text": "",
"@concept_id": "1343"
}
},
"vaccination_status": {
"updated": "2013-02-04T21:54:28Z",
"version": 1,
"type": "FormDataAliasIndicatorDefinition",
"multi_value": False,
"value": "yes"
},
}
}
}
pillow = ReportXFormPillow(online=False)
orig['domain'] = settings.ES_XFORM_FULL_INDEX_DOMAINS[0]
for_indexing = pillow.change_transform(orig)
restored = restore_property_dict(for_indexing)
self.assertNotEqual(orig['computed_'], for_indexing['computed_'])
self.assertEqual(orig['computed_'], restored['computed_'])
def test_xformPillowTransform(self):
"""
Test to make sure report xform and reportxform pillows strip the appVersion dict to match the
mappings
"""
pillows = [ReportXFormPillow(online=False),XFormPillow(online=False)]
bad_appVersion = {
"_id": "foo",
"domain": settings.ES_XFORM_FULL_INDEX_DOMAINS[0],
"form": {
"meta": {
"@xmlns": "http://openrosa.org/jr/xforms",
"username": "******",
"instanceID": "foo",
"userID": "some_user_id",
"timeEnd": "2013-09-20T01:33:12Z",
"appVersion": {
"@xmlns": "http://commcarehq.org/xforms",
"#text": "CCODK:\"2.5.1\"(11126). v236 CC2.5b[11126] on April-15-2013"
},
"timeStart": "2013-09-19T01:13:20Z",
"deviceID": "somedevice"
}
}
}
for pillow in pillows:
cleaned = pillow.change_transform(bad_appVersion)
self.assertFalse(isinstance(cleaned['form']['meta']['appVersion'], dict))
self.assertTrue(isinstance(cleaned['form']['meta']['appVersion'], str))
self.assertTrue(cleaned['form']['meta']['appVersion'], "CCODK:\"2.5.1\"(11126). v236 CC2.5b[11126] on April-15-2013")
def testConceptReportConversion(self):
pillow = ReportXFormPillow(online=False)
orig = CONCEPT_XFORM
orig['domain'] = settings.ES_XFORM_FULL_INDEX_DOMAINS[0]
for_indexing = pillow.change_transform(orig)
self.assertTrue(isinstance(for_indexing['form']['last_visit'], dict))
self.assertTrue('#value' in for_indexing['form']['last_visit'])
self.assertTrue(isinstance(for_indexing['form']['member_available'], dict))
self.assertTrue(isinstance(for_indexing['form']['member_available']['#text'], dict))
self.assertTrue(isinstance(for_indexing['form']['member_available']['@concept_id'], dict))
self.assertEqual(for_indexing['form']['member_available'],
{
"#text": {
"#value": "yes"
},
"@concept_id": {
"#value": "1890"
}
}
)
self.assertEqual(for_indexing['form']['modern_fp'],
[
{
"fp_type": {
"#text": {
"#value": "iud"
},
"@concept_id": {
"#value": "374"
}
}
},
{
"fp_type": {
"#text": {
"#value": "ij"
},
"@concept_id": {
"#value": "374"
}
}
}
]
)
def testConceptReportConversion(self):
pillow = ReportXFormPillow(online=False)
orig = CONCEPT_XFORM
orig['domain'] = settings.ES_XFORM_FULL_INDEX_DOMAINS[0]
for_indexing = pillow.change_transform(orig)
self.assertTrue(isinstance(for_indexing['form']['last_visit'], dict))
self.assertTrue('#value' in for_indexing['form']['last_visit'])
self.assertTrue(isinstance(for_indexing['form']['member_available'], dict))
self.assertTrue(isinstance(for_indexing['form']['member_available']['#text'], dict))
self.assertTrue(isinstance(for_indexing['form']['member_available']['@concept_id'], dict))
self.assertEqual(for_indexing['form']['member_available'],
{
"#text": {
"#value": "yes"
},
"@concept_id": {
"#value": "1890"
}
}
)
self.assertEqual(for_indexing['form']['modern_fp'],
[
{
"fp_type": {
"#text": {
"#value": "iud"
},
"@concept_id": {
"#value": "374"
}
}
},
{
"fp_type": {
"#text": {
"#value": "ij"
},
"@concept_id": {
"#value": "374"
}
}
}
]
)
def testConvertAndRestoreReportXFormDicts(self):
pillow = ReportXFormPillow(online=False)
orig = CONCEPT_XFORM
orig['domain'] = settings.ES_XFORM_FULL_INDEX_DOMAINS[0]
for_indexing = pillow.change_transform(orig)
restored = restore_property_dict(for_indexing)
#appVersion might be munged in meta, so swapping it out
orig_appversion = orig['form']['meta']['appVersion']
restored_appversion = restored['form']['meta']['appVersion']
if isinstance(orig_appversion, dict):
self.assertEqual(restored_appversion, orig_appversion['#text'])
else:
self.assertEqual(restored_appversion, orig_appversion)
del (orig['form']['meta']['appVersion'])
del (restored['form']['meta']['appVersion'])
self.assertNotEqual(for_indexing, orig)
self.assertNotEqual(for_indexing, restored)
self.assertEqual(orig, restored)
def testSubCaseForm(self):
"""
Ensure that the dict format converter never touches any sub property that has a key of 'case'
this is our way of handling case blocks. The properties in the case block ought not to be touched
this current form only captures
"""
pillow = ReportXFormPillow(online=False)
orig = {
'_id': 'nested_case_blocks',
'form': {
'case': {
"@xmlns": "http://commcarehq.org/case/transaction/v2",
"@date_modified": "2013-10-14T10:59:44Z",
"@user_id": "someuser",
"@case_id": "mycase",
},
'subcase_0': {
'case': {
"@xmlns": "http://commcarehq.org/case/transaction/v2",
"index": {
"parent": {
"@case_type": "household",
"#text": "some_parent"
}
},
"@date_modified": "2013-10-12T11:59:41Z",
"create": {
"case_type": "child",
"owner_id": "some_owner",
"case_name": "hello there"
},
"@user_id": "someuser",
"update": {
"first_name": "asdlfjkasdf",
"surname": "askljvlajskdlrwe",
"dob": "2011-03-21",
"sex": "male",
"weight_date": "never",
"household_head_health_id": "",
"dob_known": "yes",
"health_id": "",
"length_date": "never",
"dob_calc": "2011-03-21"
},
"@case_id": "subcaseid"
}
},
'really': {
'nested': {
'case': {
"@xmlns": "http://commcarehq.org/case/transaction/v2",
"index": {
"parent": {
"@case_type": "household",
"#text": "some_parent"
}
},
"@date_modified": "2013-10-12T11:59:41Z",
"create": {
"case_type": "child",
"owner_id": "some_owner",
"case_name": "hello there"
},
"@user_id": "someuser",
"update": {
"first_name": "asdlfjkasdf",
"surname": "askljvlajskdlrwe",
"dob": "2011-03-21",
"sex": "male",
"weight_date": "never",
"household_head_health_id": "",
"dob_known": "yes",
"health_id": "",
"length_date": "never",
"dob_calc": "2011-03-21"
},
"@case_id": "subcaseid2"
}
}
},
'array_cases': [
{'case': {'foo': 'bar'}},
{'case': {'boo': 'bar'}},
{'case': {'poo': 'bar'}},
]
}
}
orig['domain'] = settings.ES_XFORM_FULL_INDEX_DOMAINS[0]
for_indexing = pillow.change_transform(orig)
self.assertEqual(orig['form']['case'], for_indexing['form']['case'])
self.assertEqual(orig['form']['subcase_0']['case'], for_indexing['form']['subcase_0']['case'])
self.assertEqual(orig['form']['really']['nested']['case'], for_indexing['form']['really']['nested']['case'])
def testSubCaseForm(self):
"""
Ensure that the dict format converter never touches any sub property that has a key of 'case'
this is our way of handling case blocks. The properties in the case block ought not to be touched
this current form only captures
"""
pillow = ReportXFormPillow(online=False)
orig = {
'_id': 'nested_case_blocks',
'form': {
'case': {
"@xmlns": "http://commcarehq.org/case/transaction/v2",
"@date_modified": "2013-10-14T10:59:44Z",
"@user_id": "someuser",
"@case_id": "mycase",
},
'subcase_0': {
'case': {
"@xmlns": "http://commcarehq.org/case/transaction/v2",
"index": {
"parent": {
"@case_type": "household",
"#text": "some_parent"
}
},
"@date_modified": "2013-10-12T11:59:41Z",
"create": {
"case_type": "child",
"owner_id": "some_owner",
"case_name": "hello there"
},
"@user_id": "someuser",
"update": {
"first_name": "asdlfjkasdf",
"surname": "askljvlajskdlrwe",
"dob": "2011-03-21",
"sex": "male",
"weight_date": "never",
"household_head_health_id": "",
"dob_known": "yes",
"health_id": "",
"length_date": "never",
"dob_calc": "2011-03-21"
},
"@case_id": "subcaseid"
}
},
'really': {
'nested': {
'case': {
"@xmlns":
"http://commcarehq.org/case/transaction/v2",
"index": {
"parent": {
"@case_type": "household",
"#text": "some_parent"
}
},
"@date_modified": "2013-10-12T11:59:41Z",
"create": {
"case_type": "child",
"owner_id": "some_owner",
"case_name": "hello there"
},
"@user_id": "someuser",
"update": {
"first_name": "asdlfjkasdf",
"surname": "askljvlajskdlrwe",
"dob": "2011-03-21",
"sex": "male",
"weight_date": "never",
"household_head_health_id": "",
"dob_known": "yes",
"health_id": "",
"length_date": "never",
"dob_calc": "2011-03-21"
},
"@case_id": "subcaseid2"
}
}
},
'array_cases': [
{
'case': {
'foo': 'bar'
}
},
{
'case': {
'boo': 'bar'
}
},
{
'case': {
'poo': 'bar'
}
},
]
}
}
orig['domain'] = settings.ES_XFORM_FULL_INDEX_DOMAINS[0]
for_indexing = pillow.change_transform(orig)
self.assertEqual(orig['form']['case'], for_indexing['form']['case'])
self.assertEqual(orig['form']['subcase_0']['case'],
for_indexing['form']['subcase_0']['case'])
self.assertEqual(orig['form']['really']['nested']['case'],
for_indexing['form']['really']['nested']['case'])
