def test_attachments_block_not_dict(self):
block = case_update_from_block({
'@xmlns': V2_NAMESPACE,
'@case_id': '123',
'attachment': " "
})
self.assertEqual(block.get_attachment_action().attachments, {})
def is_case_updated(self, submission, method="couch"):
# use the same case processing utilities the case code does
def _case_ids_in_couch(submission):
case_view = CommCareCase.get_db().view('case/by_xform_id',
key=submission['_id'],
reduce=False).all()
return [row['id'] for row in case_view]
def _case_ids_in_es(submission):
query = {
"filter": {
"and": [
{"term": {"xform_ids": submission['_id']}}
]
},
"from": 0,
"size":1
}
es_results = self.es['hqcases'].post('_search', data=query)
return [row['_source']['_id'] for row in es_results['hits']['hits']] \
if es_results['hits']['hits'] else []
case_blocks = extract_case_blocks(submission)
case_updates = [case_update_from_block(case_block) for case_block in case_blocks]
case_ids_in_form = set(cu.id for cu in case_updates)
case_ids_in_db = set({
"couch": _case_ids_in_couch,
"es": _case_ids_in_es,
}[method](submission))
missing = case_ids_in_form - case_ids_in_db
return list(case_ids_in_form), list(missing), bool(missing)
def test_index_block_not_dict(self):
block = case_update_from_block({
'@xmlns': V2_NAMESPACE,
'@case_id': '123',
'index': " "
})
self.assertEqual(block.get_index_action().indices, [])
def test_attachments_block_not_dict(self):
block = case_update_from_block({
'@xmlns': V2_NAMESPACE,
'@case_id': '123',
'attachment': " "
})
self.assertEqual(block.get_attachment_action().attachments, {})
def test_index_block_not_dict(self):
block = case_update_from_block({
'@xmlns': V2_NAMESPACE,
'@case_id': '123',
'index': " "
})
self.assertEqual(block.get_index_action().indices, [])
def get_case_updates(xform):
updates = sorted(
[case_update_from_block(cb) for cb in extract_case_blocks(xform)],
key=lambda update: update.id)
by_case_id = groupby(updates, lambda update: update.id)
return list(
itertools.chain(
*[order_updates(updates) for case_id, updates in by_case_id]))
def get_case_updates(xform):
if not xform:
return []
updates = sorted(
[case_update_from_block(cb) for cb in extract_case_blocks(xform)],
key=lambda update: update.id
)
by_case_id = groupby(updates, lambda update: update.id)
return list(itertools.chain(
*[order_updates(updates) for case_id, updates in by_case_id]
))
def _get_or_update_model(case_block, xformdoc, case_dbcache):
"""
Gets or updates an existing case, based on a block of data in a
submitted form. Doesn't save anything.
"""
case_update = case_update_from_block(case_block)
case_doc = case_dbcache.get(case_update.id)
if case_doc == None:
case_doc = CommCareCase.from_case_update(case_update, xformdoc)
return case_doc
else:
case_doc.update_from_case_update(case_update, xformdoc)
return case_doc
def get_case_updates(xform):
return [case_update_from_block(cb) for cb in extract_case_blocks(xform)]
def get_case_updates(xform):
return [case_update_from_block(cb) for cb in extract_case_blocks(xform)]
def transform_xform_for_elasticsearch(doc_dict):
"""
Given an XFormInstance, return a copy that is ready to be sent to elasticsearch,
or None, if the form should not be saved to elasticsearch
"""
doc_ret = copy.deepcopy(doc_dict)
if 'meta' in doc_ret['form']:
if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)):
doc_ret['form']['meta']['timeEnd'] = None
if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)):
doc_ret['form']['meta']['timeStart'] = None
# Some docs have their @xmlns and #text here
if isinstance(doc_ret['form']['meta'].get('appVersion'), dict):
doc_ret['form']['meta'] = format_form_meta_for_es(doc_ret['form']['meta'])
app_version_info = get_app_version_info(
doc_ret['domain'],
doc_ret.get('build_id'),
doc_ret.get('version'),
doc_ret['form']['meta'],
)
doc_ret['form']['meta']['commcare_version'] = app_version_info.commcare_version
doc_ret['form']['meta']['app_build_version'] = app_version_info.build_version
try:
geo_point = GeoPointProperty().wrap(doc_ret['form']['meta']['location'])
doc_ret['form']['meta']['geo_point'] = geo_point.lat_lon
except (KeyError, BadValueError):
doc_ret['form']['meta']['geo_point'] = None
pass
try:
user_id = doc_ret['form']['meta']['userID']
except KeyError:
user_id = None
doc_ret['user_type'] = get_user_type(user_id)
doc_ret['inserted_at'] = datetime.datetime.utcnow().isoformat()
try:
case_blocks = extract_case_blocks(doc_ret)
except PhoneDateValueError:
pass
else:
for case_dict in case_blocks:
for date_modified_key in ['date_modified', '@date_modified']:
if not is_valid_date(case_dict.get(date_modified_key, None)):
if case_dict.get(date_modified_key) == '':
case_dict[date_modified_key] = None
else:
case_dict.pop(date_modified_key, None)
# convert all mapped dict properties to nulls if they are empty strings
for object_key in ['index', 'attachment', 'create', 'update']:
if object_key in case_dict and not isinstance(case_dict[object_key], dict):
case_dict[object_key] = None
try:
doc_ret["__retrieved_case_ids"] = list(set(case_update_from_block(cb).id for cb in case_blocks))
except CaseGenerationException:
doc_ret["__retrieved_case_ids"] = []
if 'backend_id' not in doc_ret:
doc_ret['backend_id'] = 'couch'
return doc_ret
def transform_xform_for_elasticsearch(doc_dict):
"""
Given an XFormInstance, return a copy that is ready to be sent to elasticsearch,
or None, if the form should not be saved to elasticsearch
"""
doc_ret = copy.deepcopy(doc_dict)
if 'meta' in doc_ret['form']:
if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)):
doc_ret['form']['meta']['timeEnd'] = None
if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)):
doc_ret['form']['meta']['timeStart'] = None
# Some docs have their @xmlns and #text here
if isinstance(doc_ret['form']['meta'].get('appVersion'), dict):
doc_ret['form']['meta'] = format_form_meta_for_es(
doc_ret['form']['meta'])
app_version_info = get_app_version_info(
doc_ret['domain'],
doc_ret.get('build_id'),
doc_ret.get('version'),
doc_ret['form']['meta'],
)
doc_ret['form']['meta'][
'commcare_version'] = app_version_info.commcare_version
doc_ret['form']['meta'][
'app_build_version'] = app_version_info.build_version
try:
geo_point = GeoPointProperty().wrap(
doc_ret['form']['meta']['location'])
doc_ret['form']['meta']['geo_point'] = geo_point.lat_lon
except (KeyError, BadValueError):
doc_ret['form']['meta']['geo_point'] = None
pass
try:
user_id = doc_ret['form']['meta']['userID']
except KeyError:
user_id = None
doc_ret['user_type'] = get_user_type(user_id)
doc_ret['inserted_at'] = datetime.datetime.utcnow().isoformat()
try:
case_blocks = extract_case_blocks(doc_ret)
except PhoneDateValueError:
pass
else:
for case_dict in case_blocks:
for date_modified_key in ['date_modified', '@date_modified']:
if not is_valid_date(case_dict.get(date_modified_key, None)):
if case_dict.get(date_modified_key) == '':
case_dict[date_modified_key] = None
else:
case_dict.pop(date_modified_key, None)
# convert all mapped dict properties to nulls if they are empty strings
for object_key in ['index', 'attachment', 'create', 'update']:
if object_key in case_dict and not isinstance(
case_dict[object_key], dict):
case_dict[object_key] = None
try:
doc_ret["__retrieved_case_ids"] = list(
set(case_update_from_block(cb).id for cb in case_blocks))
except CaseGenerationException:
doc_ret["__retrieved_case_ids"] = []
if 'backend_id' not in doc_ret:
doc_ret['backend_id'] = 'couch'
return doc_ret
def test_attachments_block_not_dict(self):
block = case_update_from_block({"@xmlns": V2_NAMESPACE, "@case_id": "123", "attachment": " "})
self.assertEqual(block.get_attachment_action().attachments, {})
def test_index_block_not_dict(self):
block = case_update_from_block({"@xmlns": V2_NAMESPACE, "@case_id": "123", "index": " "})
self.assertEqual(block.get_index_action().indices, [])
