def get_first_survey_response(self, case, dt): timestamp_start = datetime.combine(dt, time(20, 45)) timestamp_start = tz_utils.adjust_datetime_to_timezone( timestamp_start, self.domain_obj.default_timezone, pytz.utc.zone) timestamp_start = timestamp_start.replace(tzinfo=None) timestamp_start = json_format_datetime(timestamp_start) timestamp_end = datetime.combine(dt + timedelta(days=1), time(11, 0)) timestamp_end = tz_utils.adjust_datetime_to_timezone( timestamp_end, self.domain_obj.default_timezone, pytz.utc.zone) timestamp_end = timestamp_end.replace(tzinfo=None) if timestamp_end > datetime.utcnow(): return RESPONSE_NOT_APPLICABLE timestamp_end = json_format_datetime(timestamp_end) all_inbound = FRISMSLog.view( "sms/by_recipient", startkey=["CommCareCase", case._id, "SMSLog", INCOMING, timestamp_start], endkey=["CommCareCase", case._id, "SMSLog", INCOMING, timestamp_end], reduce=False, include_docs=True ).all() survey_responses = filter(lambda s: s.xforms_session_couch_id is not None, all_inbound) if len(survey_responses) > 0: return survey_responses[0] else: return NO_RESPONSE
def rows(self): participants = self.get_participants() result = [] for case in participants: pid = case.get_case_property("pid") study_arm = case.get_case_property("study_arm") registration_date = get_date(case, "start_date") if registration_date is None: continue first_survey_date = self.get_first_tuesday(registration_date) row = [ self._fmt(pid), self._fmt(case.name), self._fmt(study_arm), ] for i in range(8): next_survey_date = first_survey_date + timedelta(days=7*i) response = self.get_first_survey_response(case, next_survey_date) if response == RESPONSE_NOT_APPLICABLE: row.append(self._fmt("-")) elif response == NO_RESPONSE: row.append(self._fmt(_("No Response"))) else: response_timestamp = tz_utils.adjust_datetime_to_timezone( response.date, pytz.utc.zone, self.domain_obj.default_timezone) row.append(self._fmt_timestamp(response_timestamp)) result.append(row) return result
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = FRISMSLog.view("sms/by_domain", startkey=[self.domain, "SMSLog", startdate], endkey=[self.domain, "SMSLog", enddate], include_docs=True, reduce=False).all() result = [] direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } message_bank_messages = get_message_bank(self.domain, for_comparing=True) case_cache = CaseDbCache(domain=self.domain, strip_history=False, deleted_ok=True) user_cache = UserCache() show_only_survey_traffic = self.show_only_survey_traffic() for message in data: if message.direction == OUTGOING and not message.processed: continue if show_only_survey_traffic and message.xforms_session_couch_id is None: continue # Add metadata from the message bank if it has not been added already if (message.direction == OUTGOING) and ( not message.fri_message_bank_lookup_completed): add_metadata(message, message_bank_messages) if message.couch_recipient_doc_type == "CommCareCase": recipient = case_cache.get(message.couch_recipient) else: recipient = user_cache.get(message.couch_recipient) if message.chat_user_id: sender = user_cache.get(message.chat_user_id) else: sender = None study_arm = None if message.couch_recipient_doc_type == "CommCareCase": study_arm = case_cache.get( message.couch_recipient).get_case_property("study_arm") timestamp = tz_utils.adjust_datetime_to_timezone( message.date, pytz.utc.zone, self.domain_obj.default_timezone) result.append([ self._fmt(self._participant_id(recipient)), self._fmt(study_arm or "-"), self._fmt(self._originator(message, recipient, sender)), self._fmt_timestamp(timestamp), self._fmt(message.text), self._fmt(message.fri_id or "-"), self._fmt(direction_map.get(message.direction, "-")), ]) return result
def handle_domain_specific_delays(msg, domain_object, utcnow): """ Checks whether or not we need to hold off on sending an outbound message due to any restrictions set on the domain, and delays processing of the message if necessary. Returns True if a delay was made, False if not. """ domain_now = tz_utils.adjust_datetime_to_timezone(utcnow, pytz.utc.zone, domain_object.default_timezone) if len(domain_object.restricted_sms_times) > 0: if not time_within_windows(domain_now, domain_object.restricted_sms_times): delay_processing(msg, settings.SMS_QUEUE_DOMAIN_RESTRICTED_RETRY_INTERVAL) return True if msg.chat_user_id is None and len(domain_object.sms_conversation_times) > 0: if time_within_windows(domain_now, domain_object.sms_conversation_times): sms_conversation_length = domain_object.sms_conversation_length conversation_start_timestamp = utcnow - timedelta(minutes=sms_conversation_length) if SMSLog.inbound_entry_exists(msg.couch_recipient_doc_type, msg.couch_recipient, conversation_start_timestamp, utcnow): delay_processing(msg, 1) return True return False
def handle_domain_specific_delays(msg, domain_object, utcnow): """ Checks whether or not we need to hold off on sending an outbound message due to any restrictions set on the domain, and delays processing of the message if necessary. Returns True if a delay was made, False if not. """ domain_now = tz_utils.adjust_datetime_to_timezone( utcnow, pytz.utc.zone, domain_object.default_timezone) if len(domain_object.restricted_sms_times) > 0: if not time_within_windows(domain_now, domain_object.restricted_sms_times): delay_processing( msg, settings.SMS_QUEUE_DOMAIN_RESTRICTED_RETRY_INTERVAL) return True if msg.chat_user_id is None and len( domain_object.sms_conversation_times) > 0: if time_within_windows(domain_now, domain_object.sms_conversation_times): sms_conversation_length = domain_object.sms_conversation_length conversation_start_timestamp = utcnow - timedelta( minutes=sms_conversation_length) if SMSLog.inbound_entry_exists(msg.couch_recipient_doc_type, msg.couch_recipient, conversation_start_timestamp, utcnow): delay_processing(msg, 1) return True return False
def activity_times(self): all_times = [] for user in self.users: for form, info in self.all_relevant_forms.items(): key = make_form_couch_key( self.domain, user_id=user.get('user_id'), xmlns=info['xmlns'], app_id=info['app_id'], by_submission_time=self.by_submission_time) data = get_db().view( "reports_forms/all_forms", reduce=False, startkey=key + [self.datespan.startdate_param_utc], endkey=key + [self.datespan.enddate_param_utc], ).all() all_times.extend( [dateutil.parser.parse(d['key'][-1]) for d in data]) if self.by_submission_time: # completion time is assumed to be in the phone's timezone until we can send proper timezone info all_times = [ tz_utils.adjust_datetime_to_timezone(t, pytz.utc.zone, self.timezone.zone) for t in all_times ] return [(t.weekday(), t.hour) for t in all_times]
def form_inline_display(form_id, timezone=pytz.utc): if form_id: form = XFormInstance.get(form_id) if form: return "%s: %s" % (tz_utils.adjust_datetime_to_timezone(form.received_on, pytz.utc.zone, timezone.zone).date(), form.xmlns) return "missing form: %s" % form_id return "empty form id found"
def get_couch_view_data(self, key, datespan=None): default_value = None try: now = tz_utils.adjust_datetime_to_timezone( getattr(datespan, self.start_or_end or "enddate"), from_tz=datespan.timezone, to_tz=pytz.utc) except Exception: now = datetime.datetime.now(tz=pytz.utc) data = self.view_results( endkey=key, startkey=key + [getattr(datespan, "%s_param_utc" % self.start_or_end)] if datespan else [{}], descending=True, include_docs=True) if not data: return default_value data = data.first() if not data: return default_value doc = data.get('doc', {}) date_property = self._get_property_from_doc( doc, self.property_name.split('.')) try: date_property = dateutil.parser.parse(date_property) if not isinstance(date_property, datetime.datetime): return default_value td = now - date_property days = td.days except Exception: return default_value return days if days > 0 else 0
def get_num_missed_windows(case): """ Get the number of reminder events that were missed on registration day. """ domain_obj = Domain.get_by_name(case.domain, strict=True) opened_timestamp = tz_utils.adjust_datetime_to_timezone( case.opened_on, pytz.utc.zone, domain_obj.default_timezone ) day_of_week = opened_timestamp.weekday() time_of_day = opened_timestamp.time() # In order to use timedelta, we need a datetime current_time = datetime.combine(date(2000, 1, 1), time_of_day) window_time = datetime.combine(date(2000, 1, 1), WINDOWS[day_of_week][0]) if current_time < window_time: return 0 window_interval = (WINDOWS[day_of_week][1] - 60) / 5 for i in range(1, 5): window_time += timedelta(minutes=window_interval) if current_time < window_time: return i return 5
def chat(request, domain, contact_id): domain_obj = Domain.get_by_name(domain, strict=True) timezone = report_utils.get_timezone(None, domain) # floored_utc_timestamp is the datetime in UTC representing # midnight today in local time. This is used to calculate # all message history choices' timestamps, so that choosing # "Yesterday", for example, gives you data from yesterday at # midnight local time. local_date = datetime.now(timezone).date() floored_utc_timestamp = tz_utils.adjust_datetime_to_timezone( datetime.combine(local_date, time(0,0)), timezone.zone, pytz.utc.zone ).replace(tzinfo=None) def _fmt(d): return json_format_datetime(floored_utc_timestamp - timedelta(days=d)) history_choices = [(_(x), _fmt(y)) for (x, y) in SMS_CHAT_HISTORY_CHOICES] history_choices.append( (_("All Time"), json_format_datetime(datetime(1970, 1, 1))) ) context = { "domain" : domain, "contact_id" : contact_id, "contact" : get_contact(contact_id), "message_count_threshold" : domain_obj.chat_message_count_threshold or DEFAULT_MESSAGE_COUNT_THRESHOLD, "custom_case_username" : domain_obj.custom_case_username, "history_choices" : history_choices, } template = settings.CUSTOM_CHAT_TEMPLATES.get(domain_obj.custom_chat_template) or "sms/chat.html" return render(request, template, context)
def get_couch_view_data(self, key, datespan=None): default_value = None try: now = tz_utils.adjust_datetime_to_timezone(getattr(datespan, self.start_or_end or "enddate"), from_tz=datespan.timezone, to_tz=pytz.utc) except Exception: now = datetime.datetime.now(tz=pytz.utc) data = self.view_results( endkey=key, startkey=key+[getattr(datespan, "%s_param_utc" % self.start_or_end)] if datespan else [{}], descending=True, include_docs=True ) if not data: return default_value data = data.first() if not data: return default_value doc = data.get('doc', {}) date_property = self._get_property_from_doc(doc, self.property_name.split('.')) try: date_property = dateutil.parser.parse(date_property) if not isinstance(date_property, datetime.datetime): return default_value td = now - date_property days = td.days except Exception: return default_value return days if days > 0 else 0
def activity_times(self): all_times = [] for user in self.users: for form, info in self.all_relevant_forms.items(): key = make_form_couch_key( self.domain, user_id=user.get("user_id"), xmlns=info["xmlns"], app_id=info["app_id"], by_submission_time=self.by_submission_time, ) data = ( get_db() .view( "reports_forms/all_forms", reduce=False, startkey=key + [self.datespan.startdate_param_utc], endkey=key + [self.datespan.enddate_param_utc], ) .all() ) all_times.extend([dateutil.parser.parse(d["key"][-1]) for d in data]) if self.by_submission_time: # completion time is assumed to be in the phone's timezone until we can send proper timezone info all_times = [tz_utils.adjust_datetime_to_timezone(t, pytz.utc.zone, self.timezone.zone) for t in all_times] return [(t.weekday(), t.hour) for t in all_times]
def rows(self): participants = self.get_participants() result = [] for case in participants: pid = case.get_case_property("pid") study_arm = case.get_case_property("study_arm") registration_date = get_date(case, "start_date") first_name = case.get_case_property("first_name") or "" if registration_date is None: continue first_survey_date = self.get_first_tuesday(registration_date) row = [ self._fmt(pid), self._fmt(first_name), self._fmt(study_arm), ] for i in range(8): next_survey_date = first_survey_date + timedelta(days=7 * i) response = self.get_first_survey_response( case, next_survey_date) if response == RESPONSE_NOT_APPLICABLE: row.append(self._fmt("-")) elif response == NO_RESPONSE: row.append(self._fmt(_("No Response"))) else: response_timestamp = tz_utils.adjust_datetime_to_timezone( response.date, pytz.utc.zone, self.domain_obj.default_timezone) row.append(self._fmt_timestamp(response_timestamp)) result.append(row) return result
def rows(self): rows = list() prefix = ["user"] selected_form = self.request_params.get("form") if selected_form: prefix.append("form_type") total = 0 total_seconds = 0 for user in self.users: key = [" ".join(prefix), self.domain, user.get("user_id")] if selected_form: key.append(selected_form) data = ( get_db() .view( "reports/completion_vs_submission", startkey=key + [self.datespan.startdate_param_utc], endkey=key + [self.datespan.enddate_param_utc], reduce=False, ) .all() ) for item in data: vals = item.get("value") completion_time = dateutil.parser.parse(vals.get("completion_time")).replace(tzinfo=None) completion_dst = ( False if self.timezone == pytz.utc else tz_utils.is_timezone_in_dst(self.timezone, completion_time) ) completion_time = self.timezone.localize(completion_time, is_dst=completion_dst) submission_time = dateutil.parser.parse(vals.get("submission_time")) submission_time = submission_time.replace(tzinfo=pytz.utc) submission_time = tz_utils.adjust_datetime_to_timezone( submission_time, pytz.utc.zone, self.timezone.zone ) td = submission_time - completion_time td_total = td.seconds + td.days * 24 * 3600 rows.append( [ self.get_user_link(user), self._format_date(completion_time), self._format_date(submission_time), self._view_form_link(item.get("id", "")), self.table_cell(td_total, self._format_td_status(td)), ] ) if td_total >= 0: total_seconds += td_total total += 1 self.total_row = [ "Average", "-", "-", "-", self._format_td_status(int(total_seconds / total), False) if total > 0 else "--", ] return rows
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = SMSLog.by_domain_date(self.domain, startdate, enddate) result = [] username_map = { } # Store the results of username lookups for faster loading direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } # Retrieve message log options message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {}) abbreviated_phone_number_domains = message_log_options.get( "abbreviated_phone_number_domains", []) abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains) for message in data: if message.direction == OUTGOING and not message.processed: continue recipient_id = message.couch_recipient if recipient_id in [None, ""]: username = "******" elif recipient_id in username_map: username = username_map.get(recipient_id) else: username = "******" try: if message.couch_recipient_doc_type == "CommCareCase": username = CommCareCase.get(recipient_id).name else: username = CouchUser.get_by_user_id( recipient_id).username except Exception: pass username_map[recipient_id] = username phone_number = message.phone_number if abbreviate_phone_number and phone_number is not None: phone_number = phone_number[0:7] if phone_number[ 0:1] == "+" else phone_number[0:6] timestamp = tz_utils.adjust_datetime_to_timezone( message.date, pytz.utc.zone, self.timezone.zone) result.append([ self._fmt_timestamp(timestamp), self._fmt(username), self._fmt(phone_number), self._fmt(direction_map.get(message.direction, "-")), self._fmt(message.text), ]) return result
def reminders_in_error(request, domain): handler_map = {} if request.method == "POST": form = RemindersInErrorForm(request.POST) if form.is_valid(): kwargs = {} if is_bigcouch(): # Force a write to all nodes before returning kwargs["w"] = bigcouch_quorum_count() current_timestamp = datetime.utcnow() for reminder_id in form.cleaned_data.get("selected_reminders"): reminder = CaseReminder.get(reminder_id) if reminder.domain != domain: continue if reminder.handler_id in handler_map: handler = handler_map[reminder.handler_id] else: handler = reminder.handler handler_map[reminder.handler_id] = handler reminder.error = False reminder.error_msg = None handler.set_next_fire(reminder, current_timestamp) reminder.save(**kwargs) timezone = report_utils.get_timezone(request.couch_user.user_id, domain) reminders = [] for reminder in CaseReminder.view("reminders/reminders_in_error", startkey=[domain], endkey=[domain, {}], include_docs=True).all(): if reminder.handler_id in handler_map: handler = handler_map[reminder.handler_id] else: handler = reminder.handler handler_map[reminder.handler_id] = handler recipient = reminder.recipient case = reminder.case reminders.append({ "reminder_id": reminder._id, "handler_id": reminder.handler_id, "handler_name": handler.nickname, "case_id": case.get_id if case is not None else None, "case_name": case.name if case is not None else None, "next_fire": tz_utils.adjust_datetime_to_timezone( reminder.next_fire, pytz.utc.zone, timezone.zone).strftime("%Y-%m-%d %H:%M:%S"), "error_msg": reminder.error_msg, "recipient_name": get_recipient_name(recipient), })
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = SMSLog.by_domain_date(self.domain, startdate, enddate) result = [] direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } # Retrieve message log options message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {}) abbreviated_phone_number_domains = message_log_options.get( "abbreviated_phone_number_domains", []) abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains) contact_cache = {} for message in data: if message.direction == OUTGOING and not message.processed: continue recipient_id = message.couch_recipient doc = None if recipient_id not in [None, ""]: try: if message.couch_recipient_doc_type == "CommCareCase": doc = CommCareCase.get(recipient_id) else: doc = CouchUser.get_by_user_id(recipient_id) except Exception: pass if doc: doc_info = get_doc_info(doc.to_json(), self.domain, contact_cache) else: doc_info = None phone_number = message.phone_number if abbreviate_phone_number and phone_number is not None: phone_number = phone_number[0:7] if phone_number[ 0:1] == "+" else phone_number[0:6] timestamp = tz_utils.adjust_datetime_to_timezone( message.date, pytz.utc.zone, self.timezone.zone) result.append([ self._fmt_timestamp(timestamp), self._fmt_contact_link(message, doc_info), self._fmt(phone_number), self._fmt(direction_map.get(message.direction, "-")), self._fmt(message.text), ]) return result
def utc_to_timezone(date, timezone, dest_fmt="%b %d, %Y %H:%M %Z"): if not timezone: timezone = pytz.utc if not date: return "---" if not isinstance(date, datetime.datetime): try: date = datetime.datetime.replace(dateutil.parser.parse(date), tzinfo=pytz.utc) except Exception as e: return date return tz_utils.adjust_datetime_to_timezone(date, pytz.utc, timezone.zone).strftime(dest_fmt)
def rows(self): rows = [] total = 0 total_seconds = 0 if self.all_relevant_forms: for user in self.users: if not user.get("user_id"): # calling get_form_data with no user_id will return ALL form data which is not what we want continue for form in self.all_relevant_forms.values(): data = self.get_form_data(user.get("user_id"), form["xmlns"], form["app_id"]) for item in data: vals = item.get("value") completion_time = dateutil.parser.parse(vals.get("completion_time")).replace(tzinfo=None) completion_dst = ( False if self.timezone == pytz.utc else tz_utils.is_timezone_in_dst(self.timezone, completion_time) ) completion_time = self.timezone.localize(completion_time, is_dst=completion_dst) submission_time = dateutil.parser.parse(vals.get("submission_time")) submission_time = submission_time.replace(tzinfo=pytz.utc) submission_time = tz_utils.adjust_datetime_to_timezone( submission_time, pytz.utc.zone, self.timezone.zone ) td = submission_time - completion_time td_total = td.seconds + td.days * 24 * 3600 rows.append( [ self.get_user_link(user), self._format_date(completion_time), self._format_date(submission_time), form["name"], self._view_form_link(item.get("id", "")), self.table_cell(td_total, self._format_td_status(td)), ] ) if td_total >= 0: total_seconds += td_total total += 1 else: rows.append(["No Submissions Available for this Date Range"] + ["--"] * 5) self.total_row = [ _("Average"), "-", "-", "-", "-", self._format_td_status(int(total_seconds / total), False) if total > 0 else "--", ] return rows
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = FRISMSLog.view("sms/by_domain", startkey=[self.domain, "SMSLog", startdate], endkey=[self.domain, "SMSLog", enddate], include_docs=True, reduce=False).all() result = [] direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } message_bank_messages = get_message_bank(self.domain, for_comparing=True) case_cache = CaseDbCache(domain=self.domain, strip_history=False, deleted_ok=True) user_cache = UserCache() show_only_survey_traffic = self.show_only_survey_traffic() for message in data: if message.direction == OUTGOING and not message.processed: continue if show_only_survey_traffic and message.xforms_session_couch_id is None: continue # Add metadata from the message bank if it has not been added already if (message.direction == OUTGOING) and (not message.fri_message_bank_lookup_completed): add_metadata(message, message_bank_messages) if message.couch_recipient_doc_type == "CommCareCase": recipient = case_cache.get(message.couch_recipient) else: recipient = user_cache.get(message.couch_recipient) if message.chat_user_id: sender = user_cache.get(message.chat_user_id) else: sender = None study_arm = None if message.couch_recipient_doc_type == "CommCareCase": study_arm = case_cache.get(message.couch_recipient).get_case_property("study_arm") timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.domain_obj.default_timezone) result.append([ self._fmt(self._participant_id(recipient)), self._fmt(study_arm or "-"), self._fmt(self._originator(message, recipient, sender)), self._fmt_timestamp(timestamp), self._fmt(message.text), self._fmt(message.fri_id or "-"), self._fmt(direction_map.get(message.direction,"-")), ]) return result
def get_display_data(data, prop_def, processors=None, timezone=pytz.utc): # when prop_def came from a couchdbkit document, it will be a LazyDict with # a broken pop method. This conversion also has the effect of a shallow # copy, which we want. prop_def = dict(prop_def) default_processors = { 'yesno': yesno, 'doc_info': lambda value: pretty_doc_info(get_doc_info_by_id( data['domain'], value)) } processors = processors or {} processors.update(default_processors) def format_key(key): key = key.replace('_', ' ') return key.replace('-', ' ') expr = prop_def.pop('expr') name = prop_def.pop('name', format_key(expr)) format = prop_def.pop('format', None) process = prop_def.pop('process', None) # todo: nested attributes, jsonpath, indexing into related documents val = data.get(expr, None) if prop_def.pop('parse_date', None): val = parse_date_or_datetime(val) is_utc = prop_def.pop('is_utc', True) if isinstance(val, datetime.datetime): if is_utc: if val.tzinfo is None: val = val.replace(tzinfo=pytz.utc) val = adjust_datetime_to_timezone(val, val.tzinfo, timezone.zone) else: val = val.replace(tzinfo=timezone) try: val = conditional_escape(processors[process](val)) except KeyError: val = mark_safe( to_html(None, val, timezone=timezone, key_format=format_key, collapse_lists=True, **prop_def)) if format: val = mark_safe(format.format(val)) return {"expr": expr, "name": name, "value": val}
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = SMSLog.by_domain_date(self.domain, startdate, enddate) result = [] username_map = {} # Store the results of username lookups for faster loading direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } # Retrieve message log options message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {}) abbreviated_phone_number_domains = message_log_options.get("abbreviated_phone_number_domains", []) abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains) for message in data: if message.direction == OUTGOING and not message.processed: continue recipient_id = message.couch_recipient if recipient_id in [None, ""]: username = "******" elif recipient_id in username_map: username = username_map.get(recipient_id) else: username = "******" try: if message.couch_recipient_doc_type == "CommCareCase": username = CommCareCase.get(recipient_id).name else: username = CouchUser.get_by_user_id(recipient_id).username except Exception: pass username_map[recipient_id] = username phone_number = message.phone_number if abbreviate_phone_number and phone_number is not None: phone_number = phone_number[0:7] if phone_number[0:1] == "+" else phone_number[0:6] timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.timezone.zone) result.append([ self._fmt_timestamp(timestamp), self._fmt(username), self._fmt(phone_number), self._fmt(direction_map.get(message.direction,"-")), self._fmt(message.text), ]) return result
def get_display_data(data, prop_def, processors=None, timezone=pytz.utc): # when prop_def came from a couchdbkit document, it will be a LazyDict with # a broken pop method. This conversion also has the effect of a shallow # copy, which we want. prop_def = dict(prop_def) default_processors = { 'yesno': yesno, 'doc_info': lambda value: pretty_doc_info( get_doc_info_by_id(data['domain'], value) ) } processors = processors or {} processors.update(default_processors) def format_key(key): key = key.replace('_', ' ') return key.replace('-', ' ') expr = prop_def.pop('expr') name = prop_def.pop('name', format_key(expr)) format = prop_def.pop('format', None) process = prop_def.pop('process', None) # todo: nested attributes, jsonpath, indexing into related documents val = data.get(expr, None) if prop_def.pop('parse_date', None): val = parse_date_or_datetime(val) is_utc = prop_def.pop('is_utc', True) if isinstance(val, datetime.datetime): if is_utc: if val.tzinfo is None: val = val.replace(tzinfo=pytz.utc) val = adjust_datetime_to_timezone(val, val.tzinfo, timezone.zone) else: val = val.replace(tzinfo=timezone) try: val = conditional_escape(processors[process](val)) except KeyError: val = mark_safe(to_html(None, val, timezone=timezone, key_format=format_key, collapse_lists=True, **prop_def)) if format: val = mark_safe(format.format(val)) return { "expr": expr, "name": name, "value": val }
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = SMSLog.by_domain_date(self.domain, startdate, enddate) result = [] direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } # Retrieve message log options message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {}) abbreviated_phone_number_domains = message_log_options.get("abbreviated_phone_number_domains", []) abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains) contact_cache = {} for message in data: if message.direction == OUTGOING and not message.processed: continue recipient_id = message.couch_recipient doc = None if recipient_id not in [None, ""]: try: if message.couch_recipient_doc_type == "CommCareCase": doc = CommCareCase.get(recipient_id) else: doc = CouchUser.get_by_user_id(recipient_id) except Exception: pass if doc: doc_info = get_doc_info(doc.to_json(), self.domain, contact_cache) else: doc_info = None phone_number = message.phone_number if abbreviate_phone_number and phone_number is not None: phone_number = phone_number[0:7] if phone_number[0:1] == "+" else phone_number[0:6] timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.timezone.zone) result.append([ self._fmt_timestamp(timestamp), self._fmt_contact_link(message, doc_info), self._fmt(phone_number), self._fmt(direction_map.get(message.direction,"-")), self._fmt(message.text), ]) return result
def rows(self): if self.dates_in_utc: _dates = [ tz_utils.adjust_datetime_to_timezone(date, self.timezone.zone, pytz.utc.zone) for date in self.dates ] else: _dates = self.dates date_map = dict([(date.strftime(DATE_FORMAT), i + 1) for (i, date) in enumerate(_dates)]) key = [self.domain] results = ( get_db() .view( self.couch_view, reduce=False, startkey=key + [self.datespan.startdate_param_utc if self.dates_in_utc else self.datespan.startdate_param], endkey=key + [self.datespan.enddate_param_utc if self.dates_in_utc else self.datespan.enddate_param], ) .all() ) user_map = dict([(user.get("user_id"), i) for (i, user) in enumerate(self.users)]) rows = [[0] * (2 + len(date_map)) for _ in range(len(self.users))] total_row = [0] * (2 + len(date_map)) for result in results: _, date = result["key"] date = dateutil.parser.parse(date) tz_offset = self.timezone.localize(self.datespan.enddate).strftime("%z") date = date + datetime.timedelta(hours=int(tz_offset[0:3]), minutes=int(tz_offset[0] + tz_offset[3:5])) date = date.isoformat() val = result["value"] user_id = val.get("user_id") if user_id in self.user_ids: date_key = date_map.get(date[0:10], None) if date_key: rows[user_map[user_id]][date_key] += 1 for i, user in enumerate(self.users): rows[i][0] = self.get_user_link(user) total = sum(rows[i][1:-1]) rows[i][-1] = total total_row[1:-1] = [total_row[ind + 1] + val for ind, val in enumerate(rows[i][1:-1])] total_row[-1] += total total_row[0] = "All Users" self.total_row = total_row for row in rows: row[1:] = [self.table_cell(val) for val in row[1:]] return rows
def rows(self): rows = [] all_hist = HQFormData.objects.filter(userID__in=self.user_ids, domain=self.domain) history = all_hist.extra(order_by=['-received_on'])[self.pagination.start:self.pagination.start+self.pagination.count] for data in history: if data.userID in self.user_ids: time = tz_utils.adjust_datetime_to_timezone(data.received_on, pytz.utc.zone, self.timezone.zone) time = time.strftime("%Y-%m-%d %H:%M:%S") xmlns = data.xmlns app_id = data.app_id xmlns = xmlns_to_name(self.domain, xmlns, app_id=app_id) rows.append([self._form_data_link(data.instanceID), self.usernames[data.userID], time, xmlns]) return rows
def utc_to_timezone(date, timezone, dest_fmt="%b %d, %Y %H:%M %Z"): if not timezone: timezone = pytz.utc if not date: return "---" if not isinstance(date, datetime.datetime): try: date = datetime.datetime.replace(dateutil.parser.parse(date), tzinfo=pytz.utc) except Exception as e: return date return tz_utils.adjust_datetime_to_timezone( date, pytz.utc, timezone.zone).strftime(dest_fmt)
def raw_value(self, **kwargs): user_id = kwargs.get('user_id') datespan_keys = None if self.inactivity_milestone > 0: # inactivity milestone takes precedence over any ignore_datespan configurations milestone_days_ago = tz_utils.adjust_datetime_to_timezone(self.report_datespan.enddate, from_tz=self.report_datespan.timezone, to_tz=pytz.utc) - datetime.timedelta(days=self.inactivity_milestone) datespan_keys = [[], [milestone_days_ago.isoformat()]] elif not self.ignore_datespan: datespan_keys = [[self.report_datespan.startdate_param_utc], [self.report_datespan.enddate_param_utc]] status = self.case_status if self.case_status else None cases = self.get_filtered_cases(self.report_domain, user_id, status=status, datespan_keys=datespan_keys) return len(cases) if isinstance(cases, list) else None
def rows(self): rows = [] total = 0 total_seconds = 0 if self.all_relevant_forms: for user in self.users: if not user.get('user_id'): # calling get_form_data with no user_id will return ALL form data which is not what we want continue for form in self.all_relevant_forms.values(): data = self.get_form_data(user.get('user_id'), form['xmlns'], form['app_id']) for item in data: vals = item.get('value') completion_time = dateutil.parser.parse( vals.get('completion_time')).replace(tzinfo=None) completion_dst = False if self.timezone == pytz.utc else\ tz_utils.is_timezone_in_dst(self.timezone, completion_time) completion_time = self.timezone.localize( completion_time, is_dst=completion_dst) submission_time = dateutil.parser.parse( vals.get('submission_time')) submission_time = submission_time.replace( tzinfo=pytz.utc) submission_time = tz_utils.adjust_datetime_to_timezone( submission_time, pytz.utc.zone, self.timezone.zone) td = submission_time - completion_time td_total = (td.seconds + td.days * 24 * 3600) rows.append([ self.get_user_link(user), self._format_date(completion_time), self._format_date(submission_time), form['name'], self._view_form_link(item.get('id', '')), self.table_cell(td_total, self._format_td_status(td)) ]) if td_total >= 0: total_seconds += td_total total += 1 else: rows.append(['No Submissions Available for this Date Range'] + ['--'] * 5) self.total_row = [ _("Average"), "-", "-", "-", "-", self._format_td_status(int(total_seconds / total), False) if total > 0 else "--" ] return rows
def reminders_in_error(request, domain): handler_map = {} if request.method == "POST": form = RemindersInErrorForm(request.POST) if form.is_valid(): kwargs = {} if is_bigcouch(): # Force a write to all nodes before returning kwargs["w"] = bigcouch_quorum_count() current_timestamp = datetime.utcnow() for reminder_id in form.cleaned_data.get("selected_reminders"): reminder = CaseReminder.get(reminder_id) if reminder.domain != domain: continue if reminder.handler_id in handler_map: handler = handler_map[reminder.handler_id] else: handler = reminder.handler handler_map[reminder.handler_id] = handler reminder.error = False reminder.error_msg = None handler.set_next_fire(reminder, current_timestamp) reminder.save(**kwargs) timezone = report_utils.get_timezone(request.couch_user.user_id, domain) reminders = [] for reminder in CaseReminder.view("reminders/reminders_in_error", startkey=[domain], endkey=[domain, {}], include_docs=True).all(): if reminder.handler_id in handler_map: handler = handler_map[reminder.handler_id] else: handler = reminder.handler handler_map[reminder.handler_id] = handler recipient = reminder.recipient case = reminder.case reminders.append({ "reminder_id" : reminder._id, "handler_id" : reminder.handler_id, "handler_name" : handler.nickname, "case_id" : case.get_id if case is not None else None, "case_name" : case.name if case is not None else None, "next_fire" : tz_utils.adjust_datetime_to_timezone(reminder.next_fire, pytz.utc.zone, timezone.zone).strftime("%Y-%m-%d %H:%M:%S"), "error_msg" : reminder.error_msg, "recipient_name" : get_recipient_name(recipient), }) context = { "domain" : domain, "reminders" : reminders, "timezone" : timezone, "timezone_now" : datetime.now(tz=timezone), } return render(request, "reminders/partial/reminders_in_error.html", context)
def rendered_report_title(self): new_title = self.name if self.errors_only: new_title = ( "Errors & Warnings Log <small>for %s</small>" % ( ", ".join(self.device_log_users) ) if self.device_log_users else "Errors & Warnings Log" ) elif self.goto_key: log = self.goto_log date = adjust_datetime_to_timezone(log.date, from_tz=pytz.utc, to_tz=self.timezone) new_title = "Last %s Logs <small>before %s</small>" % (self.limit, date.strftime("%b %d, %Y %H:%M")) return mark_safe(new_title)
def rendered_report_title(self): new_title = self.name if self.errors_only: new_title = ("Errors & Warnings Log <small>for %s</small>" % (", ".join(self.device_log_users)) if self.device_log_users else "Errors & Warnings Log") elif self.goto_key: log = self.goto_log date = adjust_datetime_to_timezone(log.date, from_tz=pytz.utc, to_tz=self.timezone) new_title = "Last %s Logs <small>before %s</small>" % ( self.limit, date.strftime("%b %d, %Y %H:%M")) return mark_safe(new_title)
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = ExpectedCallbackEventLog.by_domain(self.domain, startdate, enddate) result = [] status_descriptions = { CALLBACK_PENDING: _("Pending"), CALLBACK_RECEIVED: _("Received"), CALLBACK_MISSED: _("Missed"), } # Store the results of lookups for faster loading username_map = {} for event in data: recipient_id = event.couch_recipient if recipient_id in [None, ""]: username = "******" elif recipient_id in username_map: username = username_map.get(recipient_id) else: username = "******" try: if event.couch_recipient_doc_type == "CommCareCase": username = CommCareCase.get(recipient_id).name else: username = CouchUser.get_by_user_id( recipient_id).username except Exception: pass username_map[recipient_id] = username timestamp = tz_utils.adjust_datetime_to_timezone( event.date, pytz.utc.zone, self.timezone.zone) row = [ self._fmt_timestamp(timestamp), self._fmt(username), self._fmt(status_descriptions.get(event.status, "-")), ] result.append(row) return result
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = SMSLog.by_domain_date(self.domain, startdate, enddate) result = [] direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } # Retrieve message log options message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {}) abbreviated_phone_number_domains = message_log_options.get("abbreviated_phone_number_domains", []) abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains) contact_cache = {} message_type_filter = self.get_message_type_filter() for message in data: if message.direction == OUTGOING and not message.processed: continue message_types = self._get_message_types(message) if not message_type_filter(message_types): continue doc_info = self.get_recipient_info(message, contact_cache) phone_number = message.phone_number if abbreviate_phone_number and phone_number is not None: phone_number = phone_number[0:7] if phone_number[0:1] == "+" else phone_number[0:6] timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.timezone.zone) result.append([ self._fmt_timestamp(timestamp), self._fmt_contact_link(message, doc_info), self._fmt(phone_number), self._fmt(direction_map.get(message.direction,"-")), self._fmt(message.text), self._fmt(", ".join(message_types)), ]) return result
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = ExpectedCallbackEventLog.by_domain(self.domain, startdate, enddate) result = [] status_descriptions = { CALLBACK_PENDING : _("Pending"), CALLBACK_RECEIVED : _("Received"), CALLBACK_MISSED : _("Missed"), } # Store the results of lookups for faster loading username_map = {} for event in data: recipient_id = event.couch_recipient if recipient_id in [None, ""]: username = "******" elif recipient_id in username_map: username = username_map.get(recipient_id) else: username = "******" try: if event.couch_recipient_doc_type == "CommCareCase": username = CommCareCase.get(recipient_id).name else: username = CouchUser.get_by_user_id(recipient_id).username except Exception: pass username_map[recipient_id] = username timestamp = tz_utils.adjust_datetime_to_timezone(event.date, pytz.utc.zone, self.timezone.zone) row = [ self._fmt_timestamp(timestamp), self._fmt(username), self._fmt(status_descriptions.get(event.status, "-")), ] result.append(row) return result
def get_participants(self): result = CommCareCase.view("hqcase/types_by_domain", key=[self.domain, "participant"], include_docs=True, reduce=False).all() local_now = tz_utils.adjust_datetime_to_timezone( datetime.utcnow(), pytz.utc.zone, self.domain_obj.default_timezone) local_date = local_now.date() def filter_function(case): registration_date = get_date(case, "start_date") if registration_date is None: return False first_tuesday = self.get_first_tuesday(registration_date) end_date = first_tuesday + timedelta(days=56) return end_date >= local_date result = filter(filter_function, result) return result
def get_participants(self): result = CommCareCase.view("hqcase/types_by_domain", key=[self.domain, "participant"], include_docs=True, reduce=False).all() local_now = tz_utils.adjust_datetime_to_timezone( datetime.utcnow(), pytz.utc.zone, self.domain_obj.default_timezone) local_date = local_now.date() def filter_function(case): registration_date = get_date(case, "start_date") if registration_date is None: return False first_tuesday = self.get_first_tuesday(registration_date) end_date = first_tuesday + timedelta(days=56) return (not case.closed) and (end_date >= local_date) result = filter(filter_function, result) return result
def report_context(self): data = defaultdict(lambda: 0) for user in self.users: startkey = [self.domain, user.get("user_id")] endkey = [self.domain, user.get("user_id"), {}] view = get_db().view("formtrends/form_time_by_user", startkey=startkey, endkey=endkey, group=True) for row in view: domain, _user, day, hour = row["key"] if hour and day: # adjust to timezone now = datetime.datetime.utcnow() hour = int(hour) day = int(day) report_time = datetime.datetime(now.year, now.month, now.day, hour, tzinfo=pytz.utc) report_time = tz_utils.adjust_datetime_to_timezone(report_time, pytz.utc.zone, self.timezone.zone) hour = report_time.hour data["%d %02d" % (day, hour)] = data["%d %02d" % (day, hour)] + row["value"] return dict(chart_url=self.generate_chart(data))
def raw_value(self, **kwargs): user_id = kwargs.get('user_id') datespan_keys = None if self.inactivity_milestone > 0: # inactivity milestone takes precedence over any ignore_datespan configurations milestone_days_ago = tz_utils.adjust_datetime_to_timezone( self.report_datespan.enddate, from_tz=self.report_datespan.timezone, to_tz=pytz.utc) - datetime.timedelta( days=self.inactivity_milestone) datespan_keys = [[], [milestone_days_ago.isoformat()]] elif not self.ignore_datespan: datespan_keys = [[self.report_datespan.startdate_param_utc], [self.report_datespan.enddate_param_utc]] status = self.case_status if self.case_status else None cases = self.get_filtered_cases(self.report_domain, user_id, status=status, datespan_keys=datespan_keys) return len(cases) if isinstance(cases, list) else None
def get_display_data(prop): expr = prop['expr'] name = prop.get('name', format_key(expr)) format = prop.get('format') process = prop.get('process') parse_date = prop.get('parse_date') val = get_value(data, expr) if parse_date and not isinstance(val, datetime.datetime): try: val = dateutil.parser.parse(val) except: val = val if val else '---' if isinstance(val, datetime.datetime): if val.tzinfo is None: val = val.replace(tzinfo=pytz.utc) val = adjust_datetime_to_timezone(val, val.tzinfo, timezone.zone) if process: val = escape(processors[process](val)) else: if val is None: val = '---' val = mark_safe(to_html(None, val, timezone=timezone, key_format=format_key, collapse_lists=True)) if format: val = mark_safe(format.format(val)) return { "expr": expr, "name": name, "value": val }
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = CallLog.by_domain_date(self.domain, startdate, enddate) result = [] # Store the results of lookups for faster loading contact_cache = {} form_map = {} xforms_sessions = {} direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } # Retrieve message log options message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {}) abbreviated_phone_number_domains = message_log_options.get( "abbreviated_phone_number_domains", []) abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains) for call in data: doc_info = self.get_recipient_info(call, contact_cache) form_unique_id = call.form_unique_id if form_unique_id in [None, ""]: form_name = "-" elif form_unique_id in form_map: form_name = form_map.get(form_unique_id) else: form_name = get_form_name(form_unique_id) form_map[form_unique_id] = form_name phone_number = call.phone_number if abbreviate_phone_number and phone_number is not None: phone_number = phone_number[0:7] if phone_number[ 0:1] == "+" else phone_number[0:6] timestamp = tz_utils.adjust_datetime_to_timezone( call.date, pytz.utc.zone, self.timezone.zone) if call.direction == INCOMING: answered = "-" else: answered = _("Yes") if call.answered else _("No") if call.xforms_session_id: xforms_sessions[call.xforms_session_id] = None row = [ call.xforms_session_id, self._fmt_timestamp(timestamp), self._fmt_contact_link(call, doc_info), self._fmt(phone_number), self._fmt(direction_map.get(call.direction, "-")), self._fmt(form_name), self._fmt("-"), self._fmt(answered), self._fmt(call.duration), self._fmt(_("Yes") if call.error else _("No")), self._fmt(call.error_message), ] if self.request.couch_user.is_previewer(): row.append(self._fmt(call.gateway_session_id)) result.append(row) # Look up the XFormsSession documents 500 at a time. # Had to do this because looking up one document at a time slows things # down a lot. all_session_ids = xforms_sessions.keys() limit = 500 range_max = int(ceil(len(all_session_ids) * 1.0 / limit)) for i in range(range_max): lower_bound = i * limit upper_bound = (i + 1) * limit sessions = XFormsSession.view( "smsforms/sessions_by_touchforms_id", keys=all_session_ids[lower_bound:upper_bound], include_docs=True).all() for session in sessions: xforms_sessions[session.session_id] = session.submission_id # Add into the final result the link to the submission based on the # outcome of the above lookups. final_result = [] for row in result: final_row = row[1:] session_id = row[0] if session_id: submission_id = xforms_sessions[session_id] if submission_id: final_row[5] = self._fmt_submission_link(submission_id) final_result.append(final_row) return final_result
def api_history(request, domain): result = [] contact_id = request.GET.get("contact_id", None) start_date = request.GET.get("start_date", None) timezone = report_utils.get_timezone(None, domain) domain_obj = Domain.get_by_name(domain, strict=True) try: assert contact_id is not None doc = get_contact(contact_id) assert doc is not None assert doc.domain == domain except Exception: return HttpResponse("[]") query_start_date_str = None if start_date is not None: try: query_start_date = parse(start_date) query_start_date += timedelta(seconds=1) query_start_date_str = json_format_datetime(query_start_date) except Exception: pass if query_start_date_str is not None: data = SMSLog.view("sms/by_recipient", startkey=[doc.doc_type, contact_id, "SMSLog", INCOMING, query_start_date_str], endkey=[doc.doc_type, contact_id, "SMSLog", INCOMING, {}], include_docs=True, reduce=False).all() data += SMSLog.view("sms/by_recipient", startkey=[doc.doc_type, contact_id, "SMSLog", OUTGOING, query_start_date_str], endkey=[doc.doc_type, contact_id, "SMSLog", OUTGOING, {}], include_docs=True, reduce=False).all() else: data = SMSLog.view("sms/by_recipient", startkey=[doc.doc_type, contact_id, "SMSLog"], endkey=[doc.doc_type, contact_id, "SMSLog", {}], include_docs=True, reduce=False).all() data.sort(key=lambda x : x.date) username_map = {} last_sms = None for sms in data: # Don't show outgoing SMS that haven't been processed yet if sms.direction == OUTGOING and not sms.processed: continue # Filter SMS that are tied to surveys if necessary if ((domain_obj.filter_surveys_from_chat and sms.xforms_session_couch_id) and not (domain_obj.show_invalid_survey_responses_in_chat and sms.direction == INCOMING and sms.invalid_survey_response)): continue if sms.direction == INCOMING: if doc.doc_type == "CommCareCase" and domain_obj.custom_case_username: sender = doc.get_case_property(domain_obj.custom_case_username) elif doc.doc_type == "CommCareCase": sender = doc.name else: sender = doc.first_name or doc.raw_username elif sms.chat_user_id is not None: if sms.chat_user_id in username_map: sender = username_map[sms.chat_user_id] else: try: user = CouchUser.get_by_user_id(sms.chat_user_id) sender = user.first_name or user.raw_username except Exception: sender = _("Unknown") username_map[sms.chat_user_id] = sender else: sender = _("System") last_sms = sms result.append({ "sender" : sender, "text" : sms.text, "timestamp" : tz_utils.adjust_datetime_to_timezone(sms.date, pytz.utc.zone, timezone.zone).strftime("%I:%M%p %m/%d/%y").lower(), "utc_timestamp" : json_format_datetime(sms.date), }) if last_sms: try: entry, lock = LastReadMessage.get_locked_obj( sms.domain, request.couch_user._id, sms.couch_recipient, create=True ) if (not entry.message_timestamp or entry.message_timestamp < last_sms.date): entry.message_id = last_sms._id entry.message_timestamp = last_sms.date entry.save() lock.release() except: logging.exception("Could not create/save LastReadMessage for message %s" % last_sms._id) # Don't let this block returning of the data pass return HttpResponse(json.dumps(result))
def get_past_two_weeks(self): now = datetime.utcnow() local_datetime = tz_utils.adjust_datetime_to_timezone(now, pytz.utc.zone, self.timezone.zone) return [(local_datetime + timedelta(days = x)).date() for x in range(-14, 0)]
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = CallLog.by_domain_date(self.domain, startdate, enddate) result = [] # Store the results of lookups for faster loading username_map = {} form_map = {} direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } # Retrieve message log options message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {}) abbreviated_phone_number_domains = message_log_options.get("abbreviated_phone_number_domains", []) abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains) for call in data: recipient_id = call.couch_recipient if recipient_id in [None, ""]: username = "******" elif recipient_id in username_map: username = username_map.get(recipient_id) else: username = "******" try: if call.couch_recipient_doc_type == "CommCareCase": username = CommCareCase.get(recipient_id).name else: username = CouchUser.get_by_user_id(recipient_id).username except Exception: pass username_map[recipient_id] = username form_unique_id = call.form_unique_id if form_unique_id in [None, ""]: form_name = "-" elif form_unique_id in form_map: form_name = form_map.get(form_unique_id) else: form_name = get_form_name(form_unique_id) form_map[form_unique_id] = form_name phone_number = call.phone_number if abbreviate_phone_number and phone_number is not None: phone_number = phone_number[0:7] if phone_number[0:1] == "+" else phone_number[0:6] timestamp = tz_utils.adjust_datetime_to_timezone(call.date, pytz.utc.zone, self.timezone.zone) if call.direction == INCOMING: answered = "-" else: answered = _("Yes") if call.answered else _("No") if call.xforms_session_id is None: submission_id = None else: session = XFormsSession.latest_by_session_id(call.xforms_session_id) submission_id = session.submission_id row = [ self._fmt_timestamp(timestamp), self._fmt(username), self._fmt(phone_number), self._fmt(direction_map.get(call.direction,"-")), self._fmt(form_name), self._fmt("-") if submission_id is None else self._fmt_submission_link(submission_id), self._fmt(answered), self._fmt(call.duration), self._fmt(_("Yes") if call.error else _("No")), self._fmt(call.error_message), ] if self.request.couch_user.is_previewer(): row.append(self._fmt(call.gateway_session_id)) result.append(row) return result
def rows(self): group_id = None if self.request.couch_user.is_commcare_user(): group_ids = self.request.couch_user.get_group_ids() if len(group_ids) > 0: group_id = group_ids[0] cases = CommCareCase.view("hqcase/types_by_domain", key=[self.domain, "participant"], reduce=False, include_docs=True).all() data = {} for case in cases: if case.closed: continue # If a site coordinator is viewing the report, only show participants from that site (group) if group_id is None or group_id == case.owner_id: data[case._id] = { "name" : case.name, "time_zone" : case.get_case_property("time_zone"), "dates" : [None for x in range(14)], } dates = self.get_past_two_weeks() date_strings = [date.strftime("%Y-%m-%d") for date in dates] start_date = dates[0] - timedelta(days=1) end_date = dates[-1] + timedelta(days=2) start_utc_timestamp = json_format_datetime(start_date) end_utc_timestamp = json_format_datetime(end_date) expected_callback_events = ExpectedCallbackEventLog.view("sms/expected_callback_event", startkey=[self.domain, start_utc_timestamp], endkey=[self.domain, end_utc_timestamp], include_docs=True).all() for event in expected_callback_events: if event.couch_recipient in data: event_date = tz_utils.adjust_datetime_to_timezone(event.date, pytz.utc.zone, data[event.couch_recipient]["time_zone"]).date() event_date = event_date.strftime("%Y-%m-%d") if event_date in date_strings: data[event.couch_recipient]["dates"][date_strings.index(event_date)] = event.status result = [] for case_id, data_dict in data.items(): row = [ self._fmt(data_dict["name"]), None, None, None, ] total_no_response = 0 total_indicated = 0 total_pending = 0 for date_status in data_dict["dates"]: if date_status == CALLBACK_PENDING: total_indicated += 1 total_pending += 1 row.append(self._fmt(_("pending"))) elif date_status == CALLBACK_RECEIVED: total_indicated += 1 row.append(self._fmt(_("OK"))) elif date_status == CALLBACK_MISSED: total_indicated += 1 total_no_response += 1 row.append(self._fmt_highlight(_("No Response"))) else: row.append(self._fmt(_("not indicated"))) if total_no_response > 0: row[1] = self._fmt_highlight(total_no_response) else: row[1] = self._fmt(total_no_response) row[2] = self._fmt(total_indicated) row[3] = self._fmt(total_pending) result.append(row) return result
def _fmt_date(somedate): time = tz_utils.adjust_datetime_to_timezone( somedate, pytz.utc.zone, self.timezone.zone) return time.strftime("%Y-%m-%d %H:%M:%S")
class SurveyResponsesReport(FRIReport): name = ugettext_noop("Survey Responses") slug = "fri_survey_responses" description = ugettext_noop( "Shows information pertaining to survey responses.") emailable = False @property def headers(self): cols = [ DataTablesColumn(_("PID")), DataTablesColumn(_("Name")), DataTablesColumn(_("Arm")), DataTablesColumn(_("Week 1")), DataTablesColumn(_("Week 2")), DataTablesColumn(_("Week 3")), DataTablesColumn(_("Week 4")), DataTablesColumn(_("Week 5")), DataTablesColumn(_("Week 6")), DataTablesColumn(_("Week 7")), DataTablesColumn(_("Week 8")), ] header = DataTablesHeader(*cols) header.custom_sort = [[0, "asc"]] return header @property def rows(self): participants = self.get_participants() result = [] for case in participants: pid = case.get_case_property("pid") study_arm = case.get_case_property("study_arm") registration_date = get_date(case, "start_date") first_name = case.get_case_property("first_name") or "" if registration_date is None: continue first_survey_date = self.get_first_tuesday(registration_date) row = [ self._fmt(pid), self._fmt(first_name), self._fmt(study_arm), ] for i in range(8): next_survey_date = first_survey_date + timedelta(days=7 * i) response = self.get_first_survey_response( case, next_survey_date) if response == RESPONSE_NOT_APPLICABLE: row.append(self._fmt("-")) elif response == NO_RESPONSE: row.append(self._fmt(_("No Response"))) else: response_timestamp = tz_utils.adjust_datetime_to_timezone( response.date, pytz.utc.zone, self.domain_obj.default_timezone) row.append(self._fmt_timestamp(response_timestamp)) result.append(row) return result def get_first_tuesday(self, dt): while dt.weekday() != 1: dt = dt + timedelta(days=1) return dt def get_participants(self): result = CommCareCase.view("hqcase/types_by_domain", key=[self.domain, "participant"], include_docs=True, reduce=False).all() local_now = tz_utils.adjust_datetime_to_timezone( datetime.utcnow(), pytz.utc.zone, self.domain_obj.default_timezone) local_date = local_now.date() def filter_function(case): registration_date = get_date(case, "start_date") if registration_date is None: return False first_tuesday = self.get_first_tuesday(registration_date) end_date = first_tuesday + timedelta(days=56) return (not case.closed) and (end_date >= local_date) result = filter(filter_function, result) return result def get_first_survey_response(self, case, dt): timestamp_start = datetime.combine(dt, time(20, 45)) timestamp_start = tz_utils.adjust_datetime_to_timezone( timestamp_start, self.domain_obj.default_timezone, pytz.utc.zone) timestamp_start = timestamp_start.replace(tzinfo=None) timestamp_start = json_format_datetime(timestamp_start) timestamp_end = datetime.combine(dt + timedelta(days=1), time(11, 45)) timestamp_end = tz_utils.adjust_datetime_to_timezone( timestamp_end, self.domain_obj.default_timezone, pytz.utc.zone) timestamp_end = timestamp_end.replace(tzinfo=None) if timestamp_end > datetime.utcnow(): return RESPONSE_NOT_APPLICABLE timestamp_end = json_format_datetime(timestamp_end) all_inbound = FRISMSLog.view("sms/by_recipient", startkey=[ "CommCareCase", case._id, "SMSLog", INCOMING, timestamp_start ], endkey=[ "CommCareCase", case._id, "SMSLog", INCOMING, timestamp_end ], reduce=False, include_docs=True).all() survey_responses = filter( lambda s: s.xforms_session_couch_id is not None, all_inbound) if len(survey_responses) > 0: return survey_responses[0] else: return NO_RESPONSE
def utc_now(self): return tz_utils.adjust_datetime_to_timezone(datetime.datetime.utcnow(), self.timezone.zone, pytz.utc.zone)
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = CallLog.by_domain_date(self.domain, startdate, enddate) result = [] # Store the results of lookups for faster loading username_map = {} form_map = {} direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } # Retrieve message log options message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {}) abbreviated_phone_number_domains = message_log_options.get( "abbreviated_phone_number_domains", []) abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains) for call in data: recipient_id = call.couch_recipient if recipient_id in [None, ""]: username = "******" elif recipient_id in username_map: username = username_map.get(recipient_id) else: username = "******" try: if call.couch_recipient_doc_type == "CommCareCase": username = CommCareCase.get(recipient_id).name else: username = CouchUser.get_by_user_id( recipient_id).username except Exception: pass username_map[recipient_id] = username form_unique_id = call.form_unique_id if form_unique_id in [None, ""]: form_name = "-" elif form_unique_id in form_map: form_name = form_map.get(form_unique_id) else: form_name = get_form_name(form_unique_id) form_map[form_unique_id] = form_name phone_number = call.phone_number if abbreviate_phone_number and phone_number is not None: phone_number = phone_number[0:7] if phone_number[ 0:1] == "+" else phone_number[0:6] timestamp = tz_utils.adjust_datetime_to_timezone( call.date, pytz.utc.zone, self.timezone.zone) if call.direction == INCOMING: answered = "-" else: answered = _("Yes") if call.answered else _("No") if call.xforms_session_id is None: submission_id = None else: session = XFormsSession.latest_by_session_id( call.xforms_session_id) submission_id = session.submission_id row = [ self._fmt_timestamp(timestamp), self._fmt(username), self._fmt(phone_number), self._fmt(direction_map.get(call.direction, "-")), self._fmt(form_name), self._fmt("-") if submission_id is None else self._fmt_submission_link(submission_id), self._fmt(answered), self._fmt(call.duration), self._fmt(_("Yes") if call.error else _("No")), self._fmt(call.error_message), ] if self.request.couch_user.is_previewer(): row.append(self._fmt(call.gateway_session_id)) result.append(row) return result
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = CallLog.by_domain_date(self.domain, startdate, enddate) result = [] # Store the results of lookups for faster loading contact_cache = {} form_map = {} xforms_sessions = {} direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } # Retrieve message log options message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {}) abbreviated_phone_number_domains = message_log_options.get("abbreviated_phone_number_domains", []) abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains) for call in data: doc_info = self.get_recipient_info(call, contact_cache) form_unique_id = call.form_unique_id if form_unique_id in [None, ""]: form_name = "-" elif form_unique_id in form_map: form_name = form_map.get(form_unique_id) else: form_name = get_form_name(form_unique_id) form_map[form_unique_id] = form_name phone_number = call.phone_number if abbreviate_phone_number and phone_number is not None: phone_number = phone_number[0:7] if phone_number[0:1] == "+" else phone_number[0:6] timestamp = tz_utils.adjust_datetime_to_timezone(call.date, pytz.utc.zone, self.timezone.zone) if call.direction == INCOMING: answered = "-" else: answered = _("Yes") if call.answered else _("No") if call.xforms_session_id: xforms_sessions[call.xforms_session_id] = None row = [ call.xforms_session_id, self._fmt_timestamp(timestamp), self._fmt_contact_link(call, doc_info), self._fmt(phone_number), self._fmt(direction_map.get(call.direction,"-")), self._fmt(form_name), self._fmt("-"), self._fmt(answered), self._fmt(call.duration), self._fmt(_("Yes") if call.error else _("No")), self._fmt(call.error_message), ] if self.request.couch_user.is_previewer(): row.append(self._fmt(call.gateway_session_id)) result.append(row) # Look up the XFormsSession documents 500 at a time. # Had to do this because looking up one document at a time slows things # down a lot. all_session_ids = xforms_sessions.keys() limit = 500 range_max = int(ceil(len(all_session_ids) * 1.0 / limit)) for i in range(range_max): lower_bound = i * limit upper_bound = (i + 1) * limit sessions = XFormsSession.view("smsforms/sessions_by_touchforms_id", keys=all_session_ids[lower_bound:upper_bound], include_docs=True).all() for session in sessions: xforms_sessions[session.session_id] = session.submission_id # Add into the final result the link to the submission based on the # outcome of the above lookups. final_result = [] for row in result: final_row = row[1:] session_id = row[0] if session_id: submission_id = xforms_sessions[session_id] if submission_id: final_row[5] = self._fmt_submission_link(submission_id) final_result.append(final_row) return final_result