def wrap(cls, case, audit_log_id_ref):
subject = cls(getattr(case, CC_SUBJECT_KEY), getattr(case, CC_STUDY_SUBJECT_ID), case.domain)
subject.enrollment_date = getattr(case, CC_ENROLLMENT_DATE, None)
subject.sex = getattr(case, CC_SEX, None)
subject.dob = getattr(case, CC_DOB, None)
for form in originals_first(case.get_forms()):
# Pass audit log ID by reference to increment it for each audit log
subject.add_data(form.form, form, audit_log_id_ref)
return subject
def subject_rows(self):
audit_log_id_ref = {'id': 0} # To exclude audit logs, set `custom.openclinica.const.AUDIT_LOGS = False`
for case in self.get_study_subject_cases():
subject = Subject(getattr(case, CC_SUBJECT_KEY), getattr(case, CC_STUDY_SUBJECT_ID), self.domain)
subject.enrollment_date = getattr(case, CC_ENROLLMENT_DATE, None)
subject.sex = getattr(case, CC_SEX, None)
subject.dob = getattr(case, CC_DOB, None)
for form in originals_first(case.get_forms()):
# Pass audit log ID by reference to increment it for each audit log
subject.add_data(form.form, form, audit_log_id_ref)
yield subject
def subject_rows(self):
audit_log_id = 0 # To exclude audit logs, set `custom.openclinica.const.AUDIT_LOGS = False`
for case in self.get_subject_cases():
subject = Subject(getattr(case, CC_SUBJECT_KEY), getattr(case, CC_STUDY_SUBJECT_ID), self.domain)
for form in originals_first(case.get_forms()):
updates = form.form['case'].get('update', {})
if updates:
for question, answer in updates.iteritems():
item = get_question_item(self.domain, form.xmlns, question)
if item is None:
# This is a CommCare-only question or form
continue
audit_log_id += 1
subject.add_item(item, form, question, oc_format_date(answer), audit_log_id)
subject.close_form(form)
yield subject
class Subject(object):
"""
Manages data for a subject case
"""
def __init__(self, subject_key, study_subject_id, domain):
self.subject_key = subject_key
self.study_subject_id = study_subject_id
self.enrollment_date = None
self.sex = None
self.dob = None
# We need the domain to get study metadata for study events and item groups
self._domain = Domain.get_by_name(domain)
self._domain_name = domain
# This subject's data. Stored as subject[study_event_oid][i][form_oid][item_group_oid][j][item_oid]
# (Study events and item groups are lists because they can repeat.)
self.data = defaultdict(list)
# Tracks items in self.data by reference using form ID and question. We need this so that we can
# update an item if its form has been edited on HQ, by looking it up with new_form.orig_id.
self.question_items = defaultdict(dict)
# The mobile workers who entered this subject's data. Used for audit logs. The number of mobile workers
# entering data for any single subject should be small, even in large studies with thousands of users.
# Because we are fetching the user for every question, use a dictionary for speed.
self.mobile_workers = {}
def get_study_event(self, item, form, case_id):
"""
Return the current study event. Opens a new study event if necessary.
"""
if len(self.data[item.study_event_oid]):
study_event = self.data[item.study_event_oid][-1]
if study_event.is_repeating and study_event.case_id != case_id:
study_event = StudyEvent(self._domain_name,
item.study_event_oid, case_id)
self.data[item.study_event_oid].append(study_event)
else:
study_event = StudyEvent(self._domain_name, item.study_event_oid,
case_id)
self.data[item.study_event_oid].append(study_event)
return study_event
def get_item_group(self, item, form, case_id):
"""
Return the current item group and study event. Opens a new item group if necessary.
Item groups are analogous to CommCare question groups. Like question groups, they can repeat.
"""
study_event = self.get_study_event(item, form, case_id)
oc_form = study_event.forms[item.form_oid]
if not oc_form[item.item_group_oid]:
oc_form[item.item_group_oid].append(
ItemGroup(self._domain_name, item.item_group_oid))
item_group = oc_form[item.item_group_oid][-1]
return item_group, study_event
def get_item_dict(self, item, form, case_id, question):
"""
Return a dict for storing item data, and current study event.
Return both because both the item dict and study event may be updated by a form or question.
"""
item_group, study_event = self.get_item_group(item, form, case_id)
item_dict = item_group.items[item.item_oid]
self.question_items[form.get_id][question] = (item_dict, study_event)
return item_dict, study_event
@staticmethod
def edit_item(item_dict, form, question, answer, audit_log_id_ref,
oc_user):
if AUDIT_LOGS:
audit_log_id_ref['id'] += 1
item_dict['audit_logs'].append({
'id':
'AL_{}'.format(audit_log_id_ref['id']),
'user_id':
oc_user.user_id,
'username':
oc_user.username,
'full_name':
oc_user.full_name,
'timestamp':
form.received_on,
'audit_type':
'Item data value updated',
'old_value':
item_dict['value'],
'new_value':
answer,
'value_type':
question,
})
item_dict['value'] = answer
@staticmethod
@quickcache(['domain', 'user_id'])
def _get_cc_user(domain, user_id):
return CouchUser.get_by_user_id(user_id, domain)
def _get_oc_user(self, user_id):
if user_id not in self.mobile_workers:
cc_user = self._get_cc_user(self._domain_name, user_id)
oc_user = get_oc_user(self._domain_name, cc_user)
if oc_user is None:
raise OpenClinicaIntegrationError(
'OpenClinica user not found for CommCare user "{}"'.format(
cc_user.username))
self.mobile_workers[user_id] = oc_user
return self.mobile_workers[user_id]
def add_item(self, item, form, case_id, question, answer,
audit_log_id_ref):
answer = oc_format_date(answer)
answer = oc_format_time(answer, self._domain.get_default_timezone())
oc_user = self._get_oc_user(form.auth_context['user_id'])
if getattr(form, 'deprecated_form_id',
None) and question in self.question_items[
form.deprecated_form_id]:
# This form has been edited on HQ. Fetch original item
item_dict, study_event = self.question_items[
form.deprecated_form_id][question]
if item_dict['value'] != answer:
self.edit_item(item_dict, form, question, answer,
audit_log_id_ref, oc_user)
else:
item_dict, study_event = self.get_item_dict(
item, form, case_id, question)
if item_dict and item_dict['value'] != answer:
# This form has been submitted more than once for a non-repeating item group. This is an edit.
self.edit_item(item_dict, form, question, answer,
audit_log_id_ref, oc_user)
else:
item_dict['value'] = answer
if AUDIT_LOGS:
audit_log_id_ref['id'] += 1
item_dict['audit_logs'] = [{
'id':
'AL_{}'.format(audit_log_id_ref['id']),
'user_id':
oc_user.user_id,
'username':
oc_user.username,
'full_name':
oc_user.full_name,
'timestamp':
form.received_on,
'audit_type':
'Item data value updated',
'reason':
'initial value',
'new_value':
answer,
'value_type':
question,
}]
mu_oid = get_item_measurement_unit(self._domain_name, item)
if mu_oid:
item_dict['measurement_unit_oid'] = mu_oid
if study_event.start_datetime is None or form.form['meta'][
'timeStart'] < study_event.start_datetime:
study_event.start_datetime = form.form['meta']['timeStart']
if study_event.end_datetime is None or form.form['meta'][
'timeEnd'] > study_event.end_datetime:
study_event.end_datetime = form.form['meta']['timeEnd']
def add_item_group(self, item, form):
study_event = self.get_study_event(item, form)
oc_form = study_event.forms[item.form_oid]
item_group = ItemGroup(self._domain_name, item.item_group_oid)
oc_form[item.item_group_oid].append(item_group)
def add_data(self, data, form, event_case, audit_log_id_ref):
def get_next_item(event_id, question_list):
for question_ in question_list:
item_ = get_question_item(self._domain_name, event_id,
question_)
if item_:
return item_
return None
event_id = getattr(event_case, 'event_type')
# If a CommCare form is an OpenClinica repeating item group, then we would need to add a new item
# group.
for key, value in six.iteritems(data):
if key in _reserved_keys:
continue
if isinstance(value, list):
# Repeat group
# NOTE: We need to assume that repeat groups can't be edited in later form submissions
item = get_next_item(event_id, value)
if item is None:
# None of the questions in this group are OpenClinica items
continue
self.add_item_group(item, form)
for v in value:
if not isinstance(v, dict):
raise OpenClinicaIntegrationError(
'CommCare question value is an unexpected data type. Form XMLNS: "{}"'
.format(form.xmlns))
self.add_data(v, form, event_case, audit_log_id_ref)
elif isinstance(value, dict):
# Group
self.add_data(value, form, event_case, audit_log_id_ref)
else:
# key is a question and value is its answer
item = get_question_item(self._domain_name, event_id, key)
if item is None:
# This is a CommCare-only question or form
continue
case_id = event_case.get_id
self.add_item(item, form, case_id, key, value,
audit_log_id_ref)
def get_report_events(self):
"""
The events as they appear in the report.
These are useful for scheduling events in OpenClinica, which cannot be imported from ODM until they have
been scheduled.
"""
events = []
for study_events in six.itervalues(self.data):
for study_event in study_events:
events.append('"{name}" ({start} - {end})'.format(
name=study_event.name,
start=study_event.start_short,
end=study_event.end_short))
return ', '.join(events)
def get_export_data(self):
"""
Transform Subject.data into the structure that CdiscOdmExportWriter expects
"""
mkitemlist = lambda d: [
dict(v, item_oid=k) for k, v in six.iteritems(d)
] # `dict()` updates v with item_oid
def mkitemgrouplist(itemgroupdict):
itemgrouplist = []
for oid, item_groups in six.iteritems(itemgroupdict):
for i, item_group in enumerate(item_groups):
itemgrouplist.append({
'item_group_oid': oid,
'repeat_key': i + 1,
'items': mkitemlist(item_group.items)
})
return itemgrouplist
mkformslist = lambda d: [{
'form_oid': k,
'item_groups': mkitemgrouplist(v)
} for k, v in six.iteritems(d)]
def mkeventslist(eventsdict):
eventslist = []
for oid, study_events in six.iteritems(eventsdict):
for i, study_event in enumerate(study_events):
eventslist.append({
'study_event_oid': oid,
'repeat_key': i + 1,
'start_short': study_event.start_short,
'start_long': study_event.start_long,
'end_short': study_event.end_short,
'end_long': study_event.end_long,
'forms': mkformslist(study_event.forms)
})
return eventslist
return mkeventslist(self.data)
@classmethod
def wrap(cls, case, audit_log_id_ref):
subject = cls(getattr(case, CC_SUBJECT_KEY),
getattr(case, CC_STUDY_SUBJECT_ID), case.domain)
subject.enrollment_date = getattr(case, CC_ENROLLMENT_DATE, None)
subject.sex = getattr(case, CC_SEX, None)
subject.dob = getattr(case, CC_DOB, None)
for event in case.get_subcases():
for form in originals_first(event.get_forms()):
# Pass audit log ID by reference to increment it for each audit log
subject.add_data(form.form, form, event, audit_log_id_ref)