def testXFormPillowListCaseProcess(self):
"""
Test that xform pillow can process and cleanup a single xform with a list of cases in it
"""
xform = XFORM_MULTI_CASES
changed = transform_xform_for_elasticsearch(xform)
changed_cases = extract_case_blocks(changed)
orig_cases = extract_case_blocks(xform)
[self.assertIsNotNone(x['@date_modified']) for x in orig_cases]
[self.assertIsNone(x.get('@date_modified')) for x in changed_cases]
def testXFormPillowListCaseProcess(self):
"""
Test that xform pillow can process and cleanup a single xform with a list of cases in it
"""
xform = XFORM_MULTI_CASES
pillow = XFormPillow(create_index=False, online=False)
changed = pillow.change_transform(xform)
changed_cases = extract_case_blocks(changed)
orig_cases = extract_case_blocks(xform)
[self.assertIsNotNone(x["@date_modified"]) for x in orig_cases]
[self.assertIsNone(x["@date_modified"]) for x in changed_cases]
def test_parsing_date_modified(self):
"""
To ensure that extract_case_blocks processes date_modified when passed as datetime.datetime object using
FormAccessors and form_data when at validate_phone_datetime
"""
xml_data = self.get_xml('create')
final_xml = self._formatXForm(CREATE_XFORM_ID, xml_data, {})
response, form, cases = submit_form_locally(
final_xml,
TEST_DOMAIN_NAME,
attachments={},
last_sync_token=None,
received_on=None
)
xform = FormAccessors(TEST_DOMAIN_NAME).get_form(form.get_id)
extract_case_blocks(xform.form_data)
def change_transform(self, doc_dict):
domain = self.get_domain(doc_dict)
if domain is None:
#If the domain is still None (especially when doing updates via the _changes feed)
#skip and do nothing
#the reason being is that changes on the xform instance do not necessarily add
#domain to it, so we need to wait until the domain is at least populated before
#going through with indexing this xform
return None
if domain not in getattr(settings, 'ES_XFORM_FULL_INDEX_DOMAINS', []):
#full indexing is only enabled for select domains on an opt-in basis
return None
doc_ret = copy.deepcopy(doc_dict)
if 'meta' in doc_ret['form']:
if doc_ret['form']['meta'].get('timeEnd', None) == "":
doc_ret['form']['meta']['timeEnd'] = None
if doc_ret['form']['meta'].get('timeStart', None) == "":
doc_ret['form']['meta']['timeStart'] = None
case_blocks = extract_case_blocks(doc_ret)
for case_dict in case_blocks:
for date_modified_key in ['date_modified', '@date_modified']:
if case_dict.get(date_modified_key, None) == "":
case_dict[date_modified_key] = None
#after basic transforms for stupid type mistakes are done, walk all properties.
convert_properties(doc_ret['form'], self.default_mapping['properties']['form'], override_root_keys=['case'])
return doc_ret
def change_transform(self, doc_dict):
if self.get_domain(doc_dict) is None:
#If the domain is still None (especially when doing updates via the _changes feed)
#skip and do nothing
#the reason being is that changes on the xform instance do not necessarily add
#domain to it, so we need to wait until the domain is at least populated before
#going through with indexing this xform
return None
else:
doc_ret = copy.deepcopy(doc_dict)
if 'meta' in doc_ret['form']:
if doc_ret['form']['meta'].get('timeEnd', None) == "":
doc_ret['form']['meta']['timeEnd'] = None
if doc_ret['form']['meta'].get('timeStart', None) == "":
doc_ret['form']['meta']['timeStart'] = None
# Some docs have their @xmlns and #text here
if isinstance(doc_ret['form']['meta'].get('appVersion'), dict):
doc_ret['form']['meta']['appVersion'] = doc_ret['form']['meta']['appVersion'].get('#text')
#see: extract_case_blocks(doc_dict)
case_blocks = extract_case_blocks(doc_ret)
for case_dict in case_blocks:
for date_modified_key in ['date_modified', '@date_modified']:
if case_dict.get(date_modified_key, None) == "":
case_dict[date_modified_key] = None
return doc_ret
def change_transform(self, doc_dict):
if self.get_domain(doc_dict) is None:
# If the domain is still None (especially when doing updates via the _changes feed)
# skip and do nothing
# the reason being is that changes on the xform instance do not necessarily add
# domain to it, so we need to wait until the domain is at least populated before
# going through with indexing this xform
return None
else:
doc_ret = copy.deepcopy(doc_dict)
if "meta" in doc_ret["form"]:
if doc_ret["form"]["meta"].get("timeEnd", None) == "":
doc_ret["form"]["meta"]["timeEnd"] = None
if doc_ret["form"]["meta"].get("timeStart", None) == "":
doc_ret["form"]["meta"]["timeStart"] = None
# Some docs have their @xmlns and #text here
if isinstance(doc_ret["form"]["meta"].get("appVersion"), dict):
doc_ret["form"]["meta"]["appVersion"] = doc_ret["form"]["meta"]["appVersion"].get("#text")
case_blocks = extract_case_blocks(doc_ret)
for case_dict in case_blocks:
for date_modified_key in ["date_modified", "@date_modified"]:
if case_dict.get(date_modified_key, None) == "":
case_dict[date_modified_key] = None
# convert all mapped dict properties to nulls if they are empty strings
for object_key in ["index", "attachment", "create", "update"]:
if object_key in case_dict and not isinstance(case_dict[object_key], dict):
case_dict[object_key] = None
return doc_ret
def allowed_to_forward(self, payload):
"""
Forward the payload if ...
* it did not come from OpenMRS, and
* CaseRepeater says it's OK for the case types and users of any
of the payload's cases, and
* this repeater forwards to the right OpenMRS server for any of
the payload's cases.
:param payload: An XFormInstance (not a case)
"""
if payload.xmlns == XMLNS_OPENMRS:
# payload came from OpenMRS. Don't send it back.
return False
case_blocks = extract_case_blocks(payload)
case_ids = [case_block['@case_id'] for case_block in case_blocks]
cases = CaseAccessors(payload.domain).get_cases(case_ids, ordered=True)
if not any(CaseRepeater.allowed_to_forward(self, case) for case in cases):
# If none of the case updates in the payload are allowed to
# be forwarded, drop it.
return False
repeaters = [repeater for case in cases for repeater in get_case_location_ancestor_repeaters(case)]
if repeaters and self not in repeaters:
# This repeater points to the wrong OpenMRS server for this
# payload. Let the right repeater handle it.
return False
return True
def is_case_updated(self, submission, method="couch"):
# use the same case processing utilities the case code does
def _case_ids_in_couch(submission):
case_view = CommCareCase.get_db().view('case/by_xform_id',
key=submission['_id'],
reduce=False).all()
return [row['id'] for row in case_view]
def _case_ids_in_es(submission):
query = {
"filter": {
"and": [
{"term": {"xform_ids": submission['_id']}}
]
},
"from": 0,
"size":1
}
es_results = self.es['hqcases'].post('_search', data=query)
return [row['_source']['_id'] for row in es_results['hits']['hits']] \
if es_results['hits']['hits'] else []
case_blocks = extract_case_blocks(submission)
case_updates = [case_update_from_block(case_block) for case_block in case_blocks]
case_ids_in_form = set(cu.id for cu in case_updates)
case_ids_in_db = set({
"couch": _case_ids_in_couch,
"es": _case_ids_in_es,
}[method](submission))
missing = case_ids_in_form - case_ids_in_db
return list(case_ids_in_form), list(missing), bool(missing)
def test_simple_path(self):
block = {CASE_ATTR_ID: uuid.uuid4().hex}
block_with_path = extract_case_blocks({
'data': {
'some': 'stuff'
},
'case': block
}, include_path=True)[0]
self.assertEqual(block, block_with_path.caseblock)
self.assertEqual([], block_with_path.path)
def test_basic(self):
block = {CASE_ATTR_ID: uuid.uuid4().hex}
blocks = extract_case_blocks({
'data': {
'some': 'stuff'
},
'case': block
})
self.assertEqual(1, len(blocks))
self.assertEqual(block, blocks[0])
def test_blocks_in_repeat(self):
blocks = [{CASE_ATTR_ID: uuid.uuid4().hex} for i in range(3)]
repeats = [{'group': {'case': block}} for block in blocks]
blocks_back = extract_case_blocks({
'data': {
'parent': {
'repeats': repeats
}
}
}, include_path=True)
self.assertEqual(3, len(blocks))
for i in range(len(blocks)):
self.assertEqual(blocks[i], blocks_back[i].caseblock)
self.assertEqual(['data', 'parent', 'repeats', 'group'], blocks_back[i].path)
def test_nested_path(self):
block = {CASE_ATTR_ID: uuid.uuid4().hex}
blocks = extract_case_blocks({
'data': {
'parents': {
'parent': {
'case': block
}
}
}
}, include_path=True)
self.assertEqual(1, len(blocks))
self.assertEqual(block, blocks[0].caseblock)
self.assertEqual(['data', 'parents', 'parent'], blocks[0].path)
def test_blocks_in_list(self):
blocks = [{CASE_ATTR_ID: uuid.uuid4().hex} for i in range(3)]
blocks_back = extract_case_blocks({
'data': {
'parent': {
'parent': {
'case': blocks
}
}
}
}, include_path=True)
self.assertEqual(3, len(blocks))
for i in range(len(blocks)):
self.assertEqual(blocks[i], blocks_back[i].caseblock)
self.assertEqual(['data', 'parent', 'parent'], blocks_back[i].path)
def get_case_history(case):
from casexml.apps.case.xform import extract_case_blocks
from corehq.apps.reports.display import xmlns_to_name
changes = defaultdict(dict)
for form in FormAccessors(case.domain).get_forms(case.xform_ids):
case_blocks = extract_case_blocks(form)
for block in case_blocks:
if block.get('@case_id') == case.case_id:
property_changes = {
'Form ID': form.form_id,
'Form Name': xmlns_to_name(case.domain, form.xmlns, form.app_id),
'Form Received On': form.received_on,
'Form Submitted By': form.metadata.username,
}
property_changes.update(block.get('create', {}))
property_changes.update(block.get('update', {}))
changes[form.form_id].update(property_changes)
return sorted(six.itervalues(changes), key=lambda f: f['Form Received On'])
def __call__(self, item, context=None):
case_id = self._case_id_expression(item, context)
if not case_id:
return []
forms = self._case_forms_expression(item, context)
cache_key = (self.__class__.__name__, case_id)
if context.get_cache_value(cache_key) is not None:
return context.get_cache_value(cache_key)
case_history = []
for f in forms:
case_blocks = extract_case_blocks(f)
case_history.append(
next(case_block for case_block in case_blocks
if case_block['@case_id'] == case_id))
context.set_cache_value(cache_key, case_history)
return case_history
def transform_xform_for_elasticsearch(doc_dict, include_props=True):
"""
Given an XFormInstance, return a copy that is ready to be sent to elasticsearch,
or None, if the form should not be saved to elasticsearch
"""
if doc_dict.get('domain', None) is None:
# if there is no domain don't bother processing it
return None
else:
doc_ret = copy.deepcopy(doc_dict)
if 'meta' in doc_ret['form']:
if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)):
doc_ret['form']['meta']['timeEnd'] = None
if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)):
doc_ret['form']['meta']['timeStart'] = None
# Some docs have their @xmlns and #text here
if isinstance(doc_ret['form']['meta'].get('appVersion'), dict):
doc_ret['form']['meta']['appVersion'] = doc_ret['form']['meta']['appVersion'].get('#text')
case_blocks = extract_case_blocks(doc_ret)
for case_dict in case_blocks:
for date_modified_key in ['date_modified', '@date_modified']:
if not is_valid_date(case_dict.get(date_modified_key, None)):
if case_dict.get(date_modified_key) == '':
case_dict[date_modified_key] = None
else:
case_dict.pop(date_modified_key, None)
# convert all mapped dict properties to nulls if they are empty strings
for object_key in ['index', 'attachment', 'create', 'update']:
if object_key in case_dict and not isinstance(case_dict[object_key], dict):
case_dict[object_key] = None
doc_ret["__retrieved_case_ids"] = list(get_case_ids_from_form(doc_dict))
if include_props:
form_props = ["%s:%s" % (k, v) for k, v in flatten(doc_ret['form']).iteritems()]
doc_ret["__props_for_querying"] = form_props
return doc_ret
def change_transform(self, doc_dict, include_props=True):
if self.get_domain(doc_dict) is None:
#If the domain is still None (especially when doing updates via the _changes feed)
#skip and do nothing
#the reason being is that changes on the xform instance do not necessarily add
#domain to it, so we need to wait until the domain is at least populated before
#going through with indexing this xform
return None
else:
doc_ret = copy.deepcopy(doc_dict)
if 'meta' in doc_ret['form']:
if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)):
doc_ret['form']['meta']['timeEnd'] = None
if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)):
doc_ret['form']['meta']['timeStart'] = None
# Some docs have their @xmlns and #text here
if isinstance(doc_ret['form']['meta'].get('appVersion'), dict):
doc_ret['form']['meta']['appVersion'] = doc_ret['form']['meta']['appVersion'].get('#text')
case_blocks = extract_case_blocks(doc_ret)
for case_dict in case_blocks:
for date_modified_key in ['date_modified', '@date_modified']:
if not is_valid_date(case_dict.get(date_modified_key, None)):
if case_dict.get(date_modified_key) == '':
case_dict[date_modified_key] = None
else:
case_dict.pop(date_modified_key, None)
# convert all mapped dict properties to nulls if they are empty strings
for object_key in ['index', 'attachment', 'create', 'update']:
if object_key in case_dict and not isinstance(case_dict[object_key], dict):
case_dict[object_key] = None
doc_ret["__retrieved_case_ids"] = list(get_case_ids_from_form(doc_dict))
if include_props:
form_props = ["%s:%s" % (k, v) for k, v in flatten(doc_ret['form']).iteritems()]
doc_ret["__props_for_querying"] = form_props
return doc_ret
def extract_form_info(form, properties=None, case_ids=case_ids):
unknown_number = 0
meta = form["form"].get("meta", dict())
# get case ids
case_blocks = extract_case_blocks(form)
cases = {c["@case_id"] for c in case_blocks}
case_ids |= cases
form_info = {
"form": form,
"attachments": list(),
"name": form["form"].get("@name", "unknown form"),
"user": meta.get("username", "unknown_user"),
"cases": cases,
"id": form["_id"],
}
for k, v in form["_attachments"].iteritems():
if v["content_type"] == "text/xml":
continue
try:
question_id = unicode(u"-".join(find_question_id(form["form"], k)))
except TypeError:
question_id = unicode(u"unknown" + unicode(unknown_number))
unknown_number += 1
if not properties or question_id in properties:
extension = unicode(os.path.splitext(k)[1])
form_info["attachments"].append(
{
"size": v["length"],
"name": k,
"question_id": question_id,
"extension": extension,
"timestamp": parse(form["received_on"]).timetuple(),
}
)
return form_info
def extract_form_info(form, properties=None, case_ids=case_ids):
unknown_number = 0
meta = form['form'].get('meta', dict())
# get case ids
case_blocks = extract_case_blocks(form)
cases = {c['@case_id'] for c in case_blocks}
case_ids |= cases
form_info = {
'form': form,
'attachments': list(),
'name': form['form'].get('@name', 'unknown form'),
'user': meta.get('username', 'unknown_user'),
'cases': cases,
'id': form['_id']
}
for k, v in form['_attachments'].iteritems():
if v['content_type'] == 'text/xml':
continue
try:
question_id = unicode(u'-'.join(find_question_id(form['form'], k)))
except TypeError:
question_id = unicode(u'unknown' + unicode(unknown_number))
unknown_number += 1
if not properties or question_id in properties:
extension = unicode(os.path.splitext(k)[1])
form_info['attachments'].append({
'size': v['length'],
'name': k,
'question_id': question_id,
'extension': extension,
'timestamp': parse(form['received_on']).timetuple(),
})
return form_info
def transform_xform_for_elasticsearch(doc_dict):
"""
Given an XFormInstance, return a copy that is ready to be sent to elasticsearch,
or None, if the form should not be saved to elasticsearch
"""
doc_ret = copy.deepcopy(doc_dict)
if 'meta' in doc_ret['form']:
if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)):
doc_ret['form']['meta']['timeEnd'] = None
if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)):
doc_ret['form']['meta']['timeStart'] = None
# Some docs have their @xmlns and #text here
if isinstance(doc_ret['form']['meta'].get('appVersion'), dict):
doc_ret['form']['meta'] = format_form_meta_for_es(doc_ret['form']['meta'])
app_version_info = get_app_version_info(
doc_ret['domain'],
doc_ret.get('build_id'),
doc_ret.get('version'),
doc_ret['form']['meta'],
)
doc_ret['form']['meta']['commcare_version'] = app_version_info.commcare_version
doc_ret['form']['meta']['app_build_version'] = app_version_info.build_version
try:
geo_point = GeoPointProperty().wrap(doc_ret['form']['meta']['location'])
doc_ret['form']['meta']['geo_point'] = geo_point.lat_lon
except (KeyError, BadValueError):
doc_ret['form']['meta']['geo_point'] = None
pass
try:
user_id = doc_ret['form']['meta']['userID']
except KeyError:
user_id = None
doc_ret['user_type'] = get_user_type(user_id)
doc_ret['inserted_at'] = datetime.datetime.utcnow().isoformat()
try:
case_blocks = extract_case_blocks(doc_ret)
except PhoneDateValueError:
pass
else:
for case_dict in case_blocks:
for date_modified_key in ['date_modified', '@date_modified']:
if not is_valid_date(case_dict.get(date_modified_key, None)):
if case_dict.get(date_modified_key) == '':
case_dict[date_modified_key] = None
else:
case_dict.pop(date_modified_key, None)
# convert all mapped dict properties to nulls if they are empty strings
for object_key in ['index', 'attachment', 'create', 'update']:
if object_key in case_dict and not isinstance(case_dict[object_key], dict):
case_dict[object_key] = None
try:
doc_ret["__retrieved_case_ids"] = list(set(case_update_from_block(cb).id for cb in case_blocks))
except CaseGenerationException:
doc_ret["__retrieved_case_ids"] = []
if 'backend_id' not in doc_ret:
doc_ret['backend_id'] = 'couch'
return doc_ret
def get_case_ids_for_case_upload(case_upload):
for form_record in case_upload.form_records.order_by('pk').all():
form = FormAccessors(case_upload.domain).get_form(form_record.form_id)
for case_block in extract_case_blocks(form):
yield case_block['@case_id']
def render_form(form, domain, options):
"""
Uses options since Django 1.3 doesn't seem to support templatetag kwargs.
Change to kwargs when we're on a version of Django that does.
"""
# don't actually use the passed in timezone since we assume form submissions already come
# in in local time.
# todo: we should revisit this when we properly handle timezones in form processing.
timezone = pytz.utc
case_id = options.get('case_id')
readable_form_data = READABLE_FORM_DATA.enabled(domain)
use_old_reason = ''
case_id_attr = "@%s" % const.CASE_TAG_ID
_get_tables_as_columns = partial(get_tables_as_columns, timezone=timezone)
form_dict = form.top_level_tags()
form_dict.pop('change', None) # this data already in Case Changes tab
# Form Data tab
if readable_form_data:
try:
form_data = get_readable_form_data(form)
except QuestionListNotFound as e:
readable_form_data = False
use_old_reason = e.message
if not readable_form_data:
form_keys = [k for k in form_dict.keys() if form_key_filter(k)]
form_data = _get_tables_as_columns(form_dict, get_definition(form_keys))
# Case Changes tab
case_blocks = extract_case_blocks(form)
for i, block in enumerate(list(case_blocks)):
if case_id and block.get(case_id_attr) == case_id:
case_blocks.pop(i)
case_blocks.insert(0, block)
cases = []
for b in case_blocks:
this_case_id = b.get(case_id_attr)
try:
this_case = CommCareCase.get(this_case_id) if this_case_id else None
valid_case = True
except ResourceNotFound:
this_case = None
valid_case = False
if this_case and this_case._id:
url = reverse('case_details', args=[domain, this_case._id])
else:
url = "#"
definition = get_definition(sorted_case_update_keys(b.keys()))
cases.append({
"is_current_case": case_id and this_case_id == case_id,
"name": case_inline_display(this_case),
"table": _get_tables_as_columns(b, definition),
"url": url,
"valid_case": valid_case
})
# Form Metadata tab
meta = form_dict.pop('meta', {})
definition = get_definition(sorted_form_metadata_keys(meta.keys()))
form_meta_data = _get_tables_as_columns(meta, definition)
if 'auth_context' in form:
auth_context = AuthContext(form.auth_context)
auth_context_user_id = auth_context.user_id
auth_user_info = get_doc_info_by_id(domain, auth_context_user_id)
else:
auth_user_info = get_doc_info_by_id(domain, None)
auth_context = AuthContext(
user_id=None,
authenticated=False,
domain=domain,
)
meta_userID = meta.get('userID')
meta_username = meta.get('username')
if meta_userID == 'demo_user':
user_info = DocInfo(
domain=domain,
display='demo_user',
)
elif meta_username == 'admin':
user_info = DocInfo(
domain=domain,
display='admin',
)
else:
user_info = get_doc_info_by_id(domain, meta_userID)
return render_to_string("reports/form/partials/single_form.html", {
"context_case_id": case_id,
"instance": form,
"is_archived": form.doc_type == "XFormArchived",
"domain": domain,
'readable_form_data': readable_form_data,
'use_old_reason': use_old_reason,
"form_data": form_data,
"cases": cases,
"form_table_options": {
# todo: wells if display config has more than one column
"put_loners_in_wells": False
},
"form_meta_data": form_meta_data,
"auth_context": auth_context,
"auth_user_info": auth_user_info,
"user_info": user_info,
})
def render_form(form, domain, options):
"""
Uses options since Django 1.3 doesn't seem to support templatetag kwargs.
Change to kwargs when we're on a version of Django that does.
"""
timezone = options.get('timezone', pytz.utc)
#display = options.get('display')
case_id = options.get('case_id')
case_id_attr = "@%s" % const.CASE_TAG_ID
_get_tables_as_columns = partial(get_tables_as_columns, timezone=timezone)
# Form Data tab
form_dict = form.top_level_tags()
form_dict.pop('change', None) # this data already in Case Changes tab
form_keys = [k for k in form_dict.keys() if form_key_filter(k)]
form_data = _get_tables_as_columns(form_dict, get_definition(form_keys))
# Case Changes tab
case_blocks = extract_case_blocks(form)
for i, block in enumerate(list(case_blocks)):
if case_id and block.get(case_id_attr) == case_id:
case_blocks.pop(i)
case_blocks.insert(0, block)
cases = []
for b in case_blocks:
this_case_id = b.get(case_id_attr)
try:
this_case = CommCareCase.get(this_case_id) if this_case_id else None
valid_case = True
except ResourceNotFound:
this_case = None
valid_case = False
if this_case and this_case._id:
url = reverse('case_details', args=[domain, this_case._id])
else:
url = "#"
definition = get_definition(sorted_case_update_keys(b.keys()))
cases.append({
"is_current_case": case_id and this_case_id == case_id,
"name": case_inline_display(this_case),
"table": _get_tables_as_columns(b, definition),
"url": url,
"valid_case": valid_case
})
# Form Metadata tab
meta = form_dict.pop('meta', {})
definition = get_definition(sorted_form_metadata_keys(meta.keys()))
form_meta_data = _get_tables_as_columns(meta, definition)
return render_to_string("form/partials/single_form.html", {
"context_case_id": case_id,
"instance": form,
"is_archived": form.doc_type == "XFormArchived",
"domain": domain,
"form_data": form_data,
"cases": cases,
"form_table_options": {
# todo: wells if display config has more than one column
"put_loners_in_wells": False
},
"form_meta_data": form_meta_data,
})
def _extract_form_attachment_info(form, properties):
"""
This is a helper function for build_form_multimedia_zip.
Return a dict containing information about the given form and its relevant
attachments
"""
def find_question_id(form, value):
for k, v in form.iteritems():
if isinstance(v, dict):
ret = find_question_id(v, value)
if ret:
return [k] + ret
else:
if v == value:
return [k]
return None
unknown_number = 0
case_blocks = extract_case_blocks(form.form_data)
form_info = {
'form': form,
'attachments': [],
'case_ids': {c['@case_id'] for c in case_blocks},
'username': form.get_data('form/meta/username')
}
# TODO make form.attachments always return objects that conform to a
# uniform interface. XFormInstance attachment values are dicts, and
# XFormInstanceSQL attachment values are XFormAttachmentSQL objects.
for attachment_name, attachment in form.attachments.iteritems():
if hasattr(attachment, 'content_type'):
content_type = attachment.content_type
else:
content_type = attachment['content_type']
if content_type == 'text/xml':
continue
try:
question_id = unicode(
u'-'.join(find_question_id(form.form_data, attachment_name)))
except TypeError:
question_id = u'unknown' + unicode(unknown_number)
unknown_number += 1
if not properties or question_id in properties:
extension = unicode(os.path.splitext(attachment_name)[1])
if hasattr(attachment, 'content_length'):
# FormAttachmentSQL or BlobMeta
size = attachment.content_length
elif 'content_length' in attachment:
# dict from BlobMeta.to_json() or possibly FormAttachmentSQL
size = attachment['content_length']
else:
# couch attachment dict
size = attachment['length']
form_info['attachments'].append({
'size': size,
'name': attachment_name,
'question_id': question_id,
'extension': extension,
'timestamp': form.received_on.timetuple(),
})
return form_info
def _extract_form_attachment_info(form, properties):
"""
This is a helper function for build_form_multimedia_zip.
Return a dict containing information about the given form and its relevant
attachments
"""
def find_question_id(form, value):
for k, v in six.iteritems(form):
if isinstance(v, dict):
ret = find_question_id(v, value)
if ret:
return [k] + ret
elif isinstance(v, list):
for repeat in v:
ret = find_question_id(repeat, value)
if ret:
return [k] + ret
else:
if v == value:
return [k]
return None
unknown_number = 0
case_blocks = extract_case_blocks(form.form_data)
form_info = {
'form': form,
'attachments': [],
'case_ids': {c['@case_id']
for c in case_blocks},
'username': form.get_data('form/meta/username')
}
# TODO make form.attachments always return objects that conform to a
# uniform interface. XFormInstance attachment values are dicts, and
# XFormInstanceSQL attachment values are XFormAttachmentSQL objects.
for attachment_name, attachment in six.iteritems(form.attachments):
if hasattr(attachment, 'content_type'):
content_type = attachment.content_type
else:
content_type = attachment['content_type']
if content_type == 'text/xml':
continue
try:
question_id = six.text_type('-'.join(
find_question_id(form.form_data, attachment_name)))
except TypeError:
question_id = 'unknown' + six.text_type(unknown_number)
unknown_number += 1
if not properties or question_id in properties:
extension = six.text_type(os.path.splitext(attachment_name)[1])
if hasattr(attachment, 'content_length'):
# FormAttachmentSQL or BlobMeta
size = attachment.content_length
elif 'content_length' in attachment:
# dict from BlobMeta.to_json() or possibly FormAttachmentSQL
size = attachment['content_length']
else:
# couch attachment dict
size = attachment['length']
form_info['attachments'].append({
'size':
size,
'name':
attachment_name,
'question_id':
question_id,
'extension':
extension,
'timestamp':
form.received_on.timetuple(),
})
return form_info
def render_form(form, domain, options):
"""
Uses options since Django 1.3 doesn't seem to support templatetag kwargs.
Change to kwargs when we're on a version of Django that does.
"""
# don't actually use the passed in timezone since we assume form submissions already come
# in in local time.
# todo: we should revisit this when we properly handle timezones in form processing.
timezone = pytz.utc
case_id = options.get('case_id')
side_pane = options.get('side_pane', False)
user = options.get('user', None)
_get_tables_as_columns = partial(get_tables_as_columns, timezone=timezone)
# Form Data tab
form_data, question_list_not_found = get_readable_data_for_submission(form)
# Case Changes tab
case_blocks = extract_case_blocks(form)
for i, block in enumerate(list(case_blocks)):
if case_id and block.get(const.CASE_ATTR_ID) == case_id:
case_blocks.pop(i)
case_blocks.insert(0, block)
cases = []
for b in case_blocks:
this_case_id = b.get(const.CASE_ATTR_ID)
try:
this_case = CommCareCase.get(this_case_id) if this_case_id else None
valid_case = True
except ResourceNotFound:
this_case = None
valid_case = False
if this_case and this_case._id:
url = reverse('case_details', args=[domain, this_case._id])
else:
url = "#"
definition = get_definition(sorted_case_update_keys(b.keys()))
cases.append({
"is_current_case": case_id and this_case_id == case_id,
"name": case_inline_display(this_case),
"table": _get_tables_as_columns(b, definition),
"url": url,
"valid_case": valid_case
})
# Form Metadata tab
meta = form.top_level_tags().get('meta', None) or {}
definition = get_definition(sorted_form_metadata_keys(meta.keys()))
form_meta_data = _get_tables_as_columns(meta, definition)
if 'auth_context' in form:
auth_context = AuthContext(form.auth_context)
auth_context_user_id = auth_context.user_id
auth_user_info = get_doc_info_by_id(domain, auth_context_user_id)
else:
auth_user_info = get_doc_info_by_id(domain, None)
auth_context = AuthContext(
user_id=None,
authenticated=False,
domain=domain,
)
meta_userID = meta.get('userID')
meta_username = meta.get('username')
if meta_userID == 'demo_user':
user_info = DocInfo(
domain=domain,
display='demo_user',
)
elif meta_username == 'admin':
user_info = DocInfo(
domain=domain,
display='admin',
)
else:
user_info = get_doc_info_by_id(domain, meta_userID)
request = options.get('request', None)
user_can_edit = (
request and user and request.domain
and (user.can_edit_data() or user.is_commcare_user())
)
show_edit_submission = (
user_can_edit
and has_privilege(request, privileges.CLOUDCARE)
and toggle_enabled(request, toggles.EDIT_SUBMISSIONS)
)
# stuffing this in the same flag as case rebuild
show_resave = (
user_can_edit and toggle_enabled(request, toggles.CASE_REBUILD)
)
return render_to_string("reports/form/partials/single_form.html", {
"context_case_id": case_id,
"instance": form,
"is_archived": form.doc_type == "XFormArchived",
"domain": domain,
'question_list_not_found': question_list_not_found,
"form_data": form_data,
"cases": cases,
"form_table_options": {
# todo: wells if display config has more than one column
"put_loners_in_wells": False
},
"form_meta_data": form_meta_data,
"auth_context": auth_context,
"auth_user_info": auth_user_info,
"user_info": user_info,
"side_pane": side_pane,
"user": user,
"show_edit_submission": show_edit_submission,
"show_resave": show_resave,
})
def transform_xform_for_elasticsearch(doc_dict):
"""
Given an XFormInstance, return a copy that is ready to be sent to elasticsearch,
or None, if the form should not be saved to elasticsearch
"""
doc_ret = copy.deepcopy(doc_dict)
if 'meta' in doc_ret['form']:
if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)):
doc_ret['form']['meta']['timeEnd'] = None
if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)):
doc_ret['form']['meta']['timeStart'] = None
# Some docs have their @xmlns and #text here
if isinstance(doc_ret['form']['meta'].get('appVersion'), dict):
doc_ret['form']['meta'] = format_form_meta_for_es(
doc_ret['form']['meta'])
app_version_info = get_app_version_info(
doc_ret['domain'],
doc_ret.get('build_id'),
doc_ret.get('version'),
doc_ret['form']['meta'],
)
doc_ret['form']['meta'][
'commcare_version'] = app_version_info.commcare_version
doc_ret['form']['meta'][
'app_build_version'] = app_version_info.build_version
try:
geo_point = GeoPointProperty().wrap(
doc_ret['form']['meta']['location'])
doc_ret['form']['meta']['geo_point'] = geo_point.lat_lon
except (KeyError, BadValueError):
doc_ret['form']['meta']['geo_point'] = None
pass
try:
user_id = doc_ret['form']['meta']['userID']
except KeyError:
user_id = None
doc_ret['user_type'] = get_user_type(user_id)
doc_ret['inserted_at'] = datetime.datetime.utcnow().isoformat()
try:
case_blocks = extract_case_blocks(doc_ret)
except PhoneDateValueError:
pass
else:
for case_dict in case_blocks:
for date_modified_key in ['date_modified', '@date_modified']:
if not is_valid_date(case_dict.get(date_modified_key, None)):
if case_dict.get(date_modified_key) == '':
case_dict[date_modified_key] = None
else:
case_dict.pop(date_modified_key, None)
# convert all mapped dict properties to nulls if they are empty strings
for object_key in ['index', 'attachment', 'create', 'update']:
if object_key in case_dict and not isinstance(
case_dict[object_key], dict):
case_dict[object_key] = None
try:
doc_ret["__retrieved_case_ids"] = list(
set(case_update_from_block(cb).id for cb in case_blocks))
except CaseGenerationException:
doc_ret["__retrieved_case_ids"] = []
if 'backend_id' not in doc_ret:
doc_ret['backend_id'] = 'couch'
return doc_ret
def render_form(form, domain, options):
"""
Uses options since Django 1.3 doesn't seem to support templatetag kwargs.
Change to kwargs when we're on a version of Django that does.
"""
timezone = get_timezone_for_request()
case_id = options.get('case_id')
side_pane = options.get('side_pane', False)
user = options.get('user', None)
request = options.get('request', None)
support_enabled = toggle_enabled(request, toggles.SUPPORT)
_get_tables_as_columns = partial(get_tables_as_columns, timezone=timezone)
# Form Data tab
form_data, question_list_not_found = get_readable_data_for_submission(form)
# Case Changes tab
case_blocks = extract_case_blocks(form)
for i, block in enumerate(list(case_blocks)):
if case_id and block.get(const.CASE_ATTR_ID) == case_id:
case_blocks.pop(i)
case_blocks.insert(0, block)
cases = []
for b in case_blocks:
this_case_id = b.get(const.CASE_ATTR_ID)
try:
this_case = CaseAccessors(domain).get_case(this_case_id) if this_case_id else None
valid_case = True
except ResourceNotFound:
this_case = None
valid_case = False
if this_case and this_case.case_id:
url = reverse('case_details', args=[domain, this_case.case_id])
else:
url = "#"
definition = get_default_definition(
sorted_case_update_keys(b.keys()),
assume_phonetimes=(not form.metadata or
(form.metadata.deviceID != CLOUDCARE_DEVICE_ID)),
)
cases.append({
"is_current_case": case_id and this_case_id == case_id,
"name": case_inline_display(this_case),
"table": _get_tables_as_columns(b, definition),
"url": url,
"valid_case": valid_case
})
# Form Metadata tab
meta = _top_level_tags(form).get('meta', None) or {}
meta['received_on'] = json_format_datetime(form.received_on)
if support_enabled:
meta['last_sync_token'] = form.last_sync_token
definition = get_default_definition(sorted_form_metadata_keys(meta.keys()))
form_meta_data = _get_tables_as_columns(meta, definition)
if getattr(form, 'auth_context', None):
auth_context = AuthContext(form.auth_context)
auth_context_user_id = auth_context.user_id
auth_user_info = get_doc_info_by_id(domain, auth_context_user_id)
else:
auth_user_info = get_doc_info_by_id(domain, None)
auth_context = AuthContext(
user_id=None,
authenticated=False,
domain=domain,
)
meta_userID = meta.get('userID')
meta_username = meta.get('username')
if meta_userID == 'demo_user':
user_info = DocInfo(
domain=domain,
display='demo_user',
)
elif meta_username == 'admin':
user_info = DocInfo(
domain=domain,
display='admin',
)
else:
user_info = get_doc_info_by_id(domain, meta_userID)
user_can_edit = (
request and user and request.domain
and (user.can_edit_data() or user.is_commcare_user())
)
show_edit_options = (
user_can_edit
and can_edit_form_location(domain, user, form)
)
show_edit_submission = (
user_can_edit
and has_privilege(request, privileges.DATA_CLEANUP)
and not form.is_deprecated
)
show_resave = (
user_can_edit and support_enabled
)
def _get_edit_info(instance):
info = {
'was_edited': False,
'is_edit': False,
}
if instance.is_deprecated:
info.update({
'was_edited': True,
'latest_version': instance.orig_id,
})
if getattr(instance, 'edited_on', None) and getattr(instance, 'deprecated_form_id', None):
info.update({
'is_edit': True,
'edited_on': instance.edited_on,
'previous_version': instance.deprecated_form_id
})
return info
return render_to_string("reports/form/partials/single_form.html", {
"context_case_id": case_id,
"instance": form,
"is_archived": form.is_archived,
"edit_info": _get_edit_info(form),
"domain": domain,
'question_list_not_found': question_list_not_found,
"form_data": form_data,
"cases": cases,
"form_table_options": {
# todo: wells if display config has more than one column
"put_loners_in_wells": False
},
"form_meta_data": form_meta_data,
"auth_context": auth_context,
"auth_user_info": auth_user_info,
"user_info": user_info,
"side_pane": side_pane,
"show_edit_options": show_edit_options,
"show_edit_submission": show_edit_submission,
"show_resave": show_resave,
}, RequestContext(request))
def render_form(form, domain, options):
"""
Uses options since Django 1.3 doesn't seem to support templatetag kwargs.
Change to kwargs when we're on a version of Django that does.
"""
# don't actually use the passed in timezone since we assume form submissions already come
# in in local time.
# todo: we should revisit this when we properly handle timezones in form processing.
timezone = pytz.utc
case_id = options.get('case_id')
side_pane = options.get('side_pane', False)
user = options.get('user', None)
case_id_attr = "@%s" % const.CASE_TAG_ID
_get_tables_as_columns = partial(get_tables_as_columns, timezone=timezone)
# Form Data tab
form_data, question_list_not_found = get_readable_data_for_submission(form)
# Case Changes tab
case_blocks = extract_case_blocks(form)
for i, block in enumerate(list(case_blocks)):
if case_id and block.get(case_id_attr) == case_id:
case_blocks.pop(i)
case_blocks.insert(0, block)
cases = []
for b in case_blocks:
this_case_id = b.get(case_id_attr)
try:
this_case = CommCareCase.get(this_case_id) if this_case_id else None
valid_case = True
except ResourceNotFound:
this_case = None
valid_case = False
if this_case and this_case._id:
url = reverse('case_details', args=[domain, this_case._id])
else:
url = "#"
definition = get_definition(sorted_case_update_keys(b.keys()))
cases.append({
"is_current_case": case_id and this_case_id == case_id,
"name": case_inline_display(this_case),
"table": _get_tables_as_columns(b, definition),
"url": url,
"valid_case": valid_case
})
# Form Metadata tab
meta = form.top_level_tags().get('meta', None) or {}
definition = get_definition(sorted_form_metadata_keys(meta.keys()))
form_meta_data = _get_tables_as_columns(meta, definition)
if 'auth_context' in form:
auth_context = AuthContext(form.auth_context)
auth_context_user_id = auth_context.user_id
auth_user_info = get_doc_info_by_id(domain, auth_context_user_id)
else:
auth_user_info = get_doc_info_by_id(domain, None)
auth_context = AuthContext(
user_id=None,
authenticated=False,
domain=domain,
)
meta_userID = meta.get('userID')
meta_username = meta.get('username')
if meta_userID == 'demo_user':
user_info = DocInfo(
domain=domain,
display='demo_user',
)
elif meta_username == 'admin':
user_info = DocInfo(
domain=domain,
display='admin',
)
else:
user_info = get_doc_info_by_id(domain, meta_userID)
edit_session_data = {'user_id': meta_userID}
if len(case_blocks) == 1 and case_blocks[0].get(case_id_attr):
edit_session_data["case_id"] = case_blocks[0].get(case_id_attr)
return render_to_string("reports/form/partials/single_form.html", {
"context_case_id": case_id,
"instance": form,
"form_meta": options.get('form_meta', {}),
"maps_api_key": settings.GMAPS_API_KEY,
"is_archived": form.doc_type == "XFormArchived",
"domain": domain,
'question_list_not_found': question_list_not_found,
"form_data": form_data,
"cases": cases,
"form_table_options": {
# todo: wells if display config has more than one column
"put_loners_in_wells": False
},
"form_meta_data": form_meta_data,
"auth_context": auth_context,
"auth_user_info": auth_user_info,
"user_info": user_info,
"side_pane": side_pane,
"user": user,
"edit_session_data": edit_session_data,
"request": options.get('request', None), # needed for toggles
})