def wrap(cls, case, audit_log_id_ref): subject = cls(getattr(case, CC_SUBJECT_KEY), getattr(case, CC_STUDY_SUBJECT_ID), case.domain) subject.enrollment_date = getattr(case, CC_ENROLLMENT_DATE, None) subject.sex = getattr(case, CC_SEX, None) subject.dob = getattr(case, CC_DOB, None) for form in originals_first(case.get_forms()): # Pass audit log ID by reference to increment it for each audit log subject.add_data(form.form, form, audit_log_id_ref) return subject
def subject_rows(self): audit_log_id_ref = {'id': 0} # To exclude audit logs, set `custom.openclinica.const.AUDIT_LOGS = False` for case in self.get_study_subject_cases(): subject = Subject(getattr(case, CC_SUBJECT_KEY), getattr(case, CC_STUDY_SUBJECT_ID), self.domain) subject.enrollment_date = getattr(case, CC_ENROLLMENT_DATE, None) subject.sex = getattr(case, CC_SEX, None) subject.dob = getattr(case, CC_DOB, None) for form in originals_first(case.get_forms()): # Pass audit log ID by reference to increment it for each audit log subject.add_data(form.form, form, audit_log_id_ref) yield subject
def subject_rows(self): audit_log_id = 0 # To exclude audit logs, set `custom.openclinica.const.AUDIT_LOGS = False` for case in self.get_subject_cases(): subject = Subject(getattr(case, CC_SUBJECT_KEY), getattr(case, CC_STUDY_SUBJECT_ID), self.domain) for form in originals_first(case.get_forms()): updates = form.form['case'].get('update', {}) if updates: for question, answer in updates.iteritems(): item = get_question_item(self.domain, form.xmlns, question) if item is None: # This is a CommCare-only question or form continue audit_log_id += 1 subject.add_item(item, form, question, oc_format_date(answer), audit_log_id) subject.close_form(form) yield subject
class Subject(object): """ Manages data for a subject case """ def __init__(self, subject_key, study_subject_id, domain): self.subject_key = subject_key self.study_subject_id = study_subject_id self.enrollment_date = None self.sex = None self.dob = None # We need the domain to get study metadata for study events and item groups self._domain = Domain.get_by_name(domain) self._domain_name = domain # This subject's data. Stored as subject[study_event_oid][i][form_oid][item_group_oid][j][item_oid] # (Study events and item groups are lists because they can repeat.) self.data = defaultdict(list) # Tracks items in self.data by reference using form ID and question. We need this so that we can # update an item if its form has been edited on HQ, by looking it up with new_form.orig_id. self.question_items = defaultdict(dict) # The mobile workers who entered this subject's data. Used for audit logs. The number of mobile workers # entering data for any single subject should be small, even in large studies with thousands of users. # Because we are fetching the user for every question, use a dictionary for speed. self.mobile_workers = {} def get_study_event(self, item, form, case_id): """ Return the current study event. Opens a new study event if necessary. """ if len(self.data[item.study_event_oid]): study_event = self.data[item.study_event_oid][-1] if study_event.is_repeating and study_event.case_id != case_id: study_event = StudyEvent(self._domain_name, item.study_event_oid, case_id) self.data[item.study_event_oid].append(study_event) else: study_event = StudyEvent(self._domain_name, item.study_event_oid, case_id) self.data[item.study_event_oid].append(study_event) return study_event def get_item_group(self, item, form, case_id): """ Return the current item group and study event. Opens a new item group if necessary. Item groups are analogous to CommCare question groups. Like question groups, they can repeat. """ study_event = self.get_study_event(item, form, case_id) oc_form = study_event.forms[item.form_oid] if not oc_form[item.item_group_oid]: oc_form[item.item_group_oid].append( ItemGroup(self._domain_name, item.item_group_oid)) item_group = oc_form[item.item_group_oid][-1] return item_group, study_event def get_item_dict(self, item, form, case_id, question): """ Return a dict for storing item data, and current study event. Return both because both the item dict and study event may be updated by a form or question. """ item_group, study_event = self.get_item_group(item, form, case_id) item_dict = item_group.items[item.item_oid] self.question_items[form.get_id][question] = (item_dict, study_event) return item_dict, study_event @staticmethod def edit_item(item_dict, form, question, answer, audit_log_id_ref, oc_user): if AUDIT_LOGS: audit_log_id_ref['id'] += 1 item_dict['audit_logs'].append({ 'id': 'AL_{}'.format(audit_log_id_ref['id']), 'user_id': oc_user.user_id, 'username': oc_user.username, 'full_name': oc_user.full_name, 'timestamp': form.received_on, 'audit_type': 'Item data value updated', 'old_value': item_dict['value'], 'new_value': answer, 'value_type': question, }) item_dict['value'] = answer @staticmethod @quickcache(['domain', 'user_id']) def _get_cc_user(domain, user_id): return CouchUser.get_by_user_id(user_id, domain) def _get_oc_user(self, user_id): if user_id not in self.mobile_workers: cc_user = self._get_cc_user(self._domain_name, user_id) oc_user = get_oc_user(self._domain_name, cc_user) if oc_user is None: raise OpenClinicaIntegrationError( 'OpenClinica user not found for CommCare user "{}"'.format( cc_user.username)) self.mobile_workers[user_id] = oc_user return self.mobile_workers[user_id] def add_item(self, item, form, case_id, question, answer, audit_log_id_ref): answer = oc_format_date(answer) answer = oc_format_time(answer, self._domain.get_default_timezone()) oc_user = self._get_oc_user(form.auth_context['user_id']) if getattr(form, 'deprecated_form_id', None) and question in self.question_items[ form.deprecated_form_id]: # This form has been edited on HQ. Fetch original item item_dict, study_event = self.question_items[ form.deprecated_form_id][question] if item_dict['value'] != answer: self.edit_item(item_dict, form, question, answer, audit_log_id_ref, oc_user) else: item_dict, study_event = self.get_item_dict( item, form, case_id, question) if item_dict and item_dict['value'] != answer: # This form has been submitted more than once for a non-repeating item group. This is an edit. self.edit_item(item_dict, form, question, answer, audit_log_id_ref, oc_user) else: item_dict['value'] = answer if AUDIT_LOGS: audit_log_id_ref['id'] += 1 item_dict['audit_logs'] = [{ 'id': 'AL_{}'.format(audit_log_id_ref['id']), 'user_id': oc_user.user_id, 'username': oc_user.username, 'full_name': oc_user.full_name, 'timestamp': form.received_on, 'audit_type': 'Item data value updated', 'reason': 'initial value', 'new_value': answer, 'value_type': question, }] mu_oid = get_item_measurement_unit(self._domain_name, item) if mu_oid: item_dict['measurement_unit_oid'] = mu_oid if study_event.start_datetime is None or form.form['meta'][ 'timeStart'] < study_event.start_datetime: study_event.start_datetime = form.form['meta']['timeStart'] if study_event.end_datetime is None or form.form['meta'][ 'timeEnd'] > study_event.end_datetime: study_event.end_datetime = form.form['meta']['timeEnd'] def add_item_group(self, item, form): study_event = self.get_study_event(item, form) oc_form = study_event.forms[item.form_oid] item_group = ItemGroup(self._domain_name, item.item_group_oid) oc_form[item.item_group_oid].append(item_group) def add_data(self, data, form, event_case, audit_log_id_ref): def get_next_item(event_id, question_list): for question_ in question_list: item_ = get_question_item(self._domain_name, event_id, question_) if item_: return item_ return None event_id = getattr(event_case, 'event_type') # If a CommCare form is an OpenClinica repeating item group, then we would need to add a new item # group. for key, value in six.iteritems(data): if key in _reserved_keys: continue if isinstance(value, list): # Repeat group # NOTE: We need to assume that repeat groups can't be edited in later form submissions item = get_next_item(event_id, value) if item is None: # None of the questions in this group are OpenClinica items continue self.add_item_group(item, form) for v in value: if not isinstance(v, dict): raise OpenClinicaIntegrationError( 'CommCare question value is an unexpected data type. Form XMLNS: "{}"' .format(form.xmlns)) self.add_data(v, form, event_case, audit_log_id_ref) elif isinstance(value, dict): # Group self.add_data(value, form, event_case, audit_log_id_ref) else: # key is a question and value is its answer item = get_question_item(self._domain_name, event_id, key) if item is None: # This is a CommCare-only question or form continue case_id = event_case.get_id self.add_item(item, form, case_id, key, value, audit_log_id_ref) def get_report_events(self): """ The events as they appear in the report. These are useful for scheduling events in OpenClinica, which cannot be imported from ODM until they have been scheduled. """ events = [] for study_events in six.itervalues(self.data): for study_event in study_events: events.append('"{name}" ({start} - {end})'.format( name=study_event.name, start=study_event.start_short, end=study_event.end_short)) return ', '.join(events) def get_export_data(self): """ Transform Subject.data into the structure that CdiscOdmExportWriter expects """ mkitemlist = lambda d: [ dict(v, item_oid=k) for k, v in six.iteritems(d) ] # `dict()` updates v with item_oid def mkitemgrouplist(itemgroupdict): itemgrouplist = [] for oid, item_groups in six.iteritems(itemgroupdict): for i, item_group in enumerate(item_groups): itemgrouplist.append({ 'item_group_oid': oid, 'repeat_key': i + 1, 'items': mkitemlist(item_group.items) }) return itemgrouplist mkformslist = lambda d: [{ 'form_oid': k, 'item_groups': mkitemgrouplist(v) } for k, v in six.iteritems(d)] def mkeventslist(eventsdict): eventslist = [] for oid, study_events in six.iteritems(eventsdict): for i, study_event in enumerate(study_events): eventslist.append({ 'study_event_oid': oid, 'repeat_key': i + 1, 'start_short': study_event.start_short, 'start_long': study_event.start_long, 'end_short': study_event.end_short, 'end_long': study_event.end_long, 'forms': mkformslist(study_event.forms) }) return eventslist return mkeventslist(self.data) @classmethod def wrap(cls, case, audit_log_id_ref): subject = cls(getattr(case, CC_SUBJECT_KEY), getattr(case, CC_STUDY_SUBJECT_ID), case.domain) subject.enrollment_date = getattr(case, CC_ENROLLMENT_DATE, None) subject.sex = getattr(case, CC_SEX, None) subject.dob = getattr(case, CC_DOB, None) for event in case.get_subcases(): for form in originals_first(event.get_forms()): # Pass audit log ID by reference to increment it for each audit log subject.add_data(form.form, form, event, audit_log_id_ref)