def get_first_survey_response(self, case, dt):
timestamp_start = datetime.combine(dt, time(20, 45))
timestamp_start = tz_utils.adjust_datetime_to_timezone(
timestamp_start, self.domain_obj.default_timezone, pytz.utc.zone)
timestamp_start = timestamp_start.replace(tzinfo=None)
timestamp_start = json_format_datetime(timestamp_start)
timestamp_end = datetime.combine(dt + timedelta(days=1), time(11, 0))
timestamp_end = tz_utils.adjust_datetime_to_timezone(
timestamp_end, self.domain_obj.default_timezone, pytz.utc.zone)
timestamp_end = timestamp_end.replace(tzinfo=None)
if timestamp_end > datetime.utcnow():
return RESPONSE_NOT_APPLICABLE
timestamp_end = json_format_datetime(timestamp_end)
all_inbound = FRISMSLog.view(
"sms/by_recipient",
startkey=["CommCareCase", case._id, "SMSLog", INCOMING, timestamp_start],
endkey=["CommCareCase", case._id, "SMSLog", INCOMING, timestamp_end],
reduce=False,
include_docs=True
).all()
survey_responses = filter(lambda s: s.xforms_session_couch_id is not None, all_inbound)
if len(survey_responses) > 0:
return survey_responses[0]
else:
return NO_RESPONSE
def rows(self):
participants = self.get_participants()
result = []
for case in participants:
pid = case.get_case_property("pid")
study_arm = case.get_case_property("study_arm")
registration_date = get_date(case, "start_date")
if registration_date is None:
continue
first_survey_date = self.get_first_tuesday(registration_date)
row = [
self._fmt(pid),
self._fmt(case.name),
self._fmt(study_arm),
]
for i in range(8):
next_survey_date = first_survey_date + timedelta(days=7*i)
response = self.get_first_survey_response(case, next_survey_date)
if response == RESPONSE_NOT_APPLICABLE:
row.append(self._fmt("-"))
elif response == NO_RESPONSE:
row.append(self._fmt(_("No Response")))
else:
response_timestamp = tz_utils.adjust_datetime_to_timezone(
response.date, pytz.utc.zone, self.domain_obj.default_timezone)
row.append(self._fmt_timestamp(response_timestamp))
result.append(row)
return result
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = FRISMSLog.view("sms/by_domain",
startkey=[self.domain, "SMSLog", startdate],
endkey=[self.domain, "SMSLog", enddate],
include_docs=True,
reduce=False).all()
result = []
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
message_bank_messages = get_message_bank(self.domain,
for_comparing=True)
case_cache = CaseDbCache(domain=self.domain,
strip_history=False,
deleted_ok=True)
user_cache = UserCache()
show_only_survey_traffic = self.show_only_survey_traffic()
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
if show_only_survey_traffic and message.xforms_session_couch_id is None:
continue
# Add metadata from the message bank if it has not been added already
if (message.direction == OUTGOING) and (
not message.fri_message_bank_lookup_completed):
add_metadata(message, message_bank_messages)
if message.couch_recipient_doc_type == "CommCareCase":
recipient = case_cache.get(message.couch_recipient)
else:
recipient = user_cache.get(message.couch_recipient)
if message.chat_user_id:
sender = user_cache.get(message.chat_user_id)
else:
sender = None
study_arm = None
if message.couch_recipient_doc_type == "CommCareCase":
study_arm = case_cache.get(
message.couch_recipient).get_case_property("study_arm")
timestamp = tz_utils.adjust_datetime_to_timezone(
message.date, pytz.utc.zone, self.domain_obj.default_timezone)
result.append([
self._fmt(self._participant_id(recipient)),
self._fmt(study_arm or "-"),
self._fmt(self._originator(message, recipient, sender)),
self._fmt_timestamp(timestamp),
self._fmt(message.text),
self._fmt(message.fri_id or "-"),
self._fmt(direction_map.get(message.direction, "-")),
])
return result
def handle_domain_specific_delays(msg, domain_object, utcnow):
"""
Checks whether or not we need to hold off on sending an outbound message
due to any restrictions set on the domain, and delays processing of the
message if necessary.
Returns True if a delay was made, False if not.
"""
domain_now = tz_utils.adjust_datetime_to_timezone(utcnow, pytz.utc.zone,
domain_object.default_timezone)
if len(domain_object.restricted_sms_times) > 0:
if not time_within_windows(domain_now, domain_object.restricted_sms_times):
delay_processing(msg, settings.SMS_QUEUE_DOMAIN_RESTRICTED_RETRY_INTERVAL)
return True
if msg.chat_user_id is None and len(domain_object.sms_conversation_times) > 0:
if time_within_windows(domain_now, domain_object.sms_conversation_times):
sms_conversation_length = domain_object.sms_conversation_length
conversation_start_timestamp = utcnow - timedelta(minutes=sms_conversation_length)
if SMSLog.inbound_entry_exists(msg.couch_recipient_doc_type,
msg.couch_recipient,
conversation_start_timestamp,
utcnow):
delay_processing(msg, 1)
return True
return False
def handle_domain_specific_delays(msg, domain_object, utcnow):
"""
Checks whether or not we need to hold off on sending an outbound message
due to any restrictions set on the domain, and delays processing of the
message if necessary.
Returns True if a delay was made, False if not.
"""
domain_now = tz_utils.adjust_datetime_to_timezone(
utcnow, pytz.utc.zone, domain_object.default_timezone)
if len(domain_object.restricted_sms_times) > 0:
if not time_within_windows(domain_now,
domain_object.restricted_sms_times):
delay_processing(
msg, settings.SMS_QUEUE_DOMAIN_RESTRICTED_RETRY_INTERVAL)
return True
if msg.chat_user_id is None and len(
domain_object.sms_conversation_times) > 0:
if time_within_windows(domain_now,
domain_object.sms_conversation_times):
sms_conversation_length = domain_object.sms_conversation_length
conversation_start_timestamp = utcnow - timedelta(
minutes=sms_conversation_length)
if SMSLog.inbound_entry_exists(msg.couch_recipient_doc_type,
msg.couch_recipient,
conversation_start_timestamp,
utcnow):
delay_processing(msg, 1)
return True
return False
def activity_times(self):
all_times = []
for user in self.users:
for form, info in self.all_relevant_forms.items():
key = make_form_couch_key(
self.domain,
user_id=user.get('user_id'),
xmlns=info['xmlns'],
app_id=info['app_id'],
by_submission_time=self.by_submission_time)
data = get_db().view(
"reports_forms/all_forms",
reduce=False,
startkey=key + [self.datespan.startdate_param_utc],
endkey=key + [self.datespan.enddate_param_utc],
).all()
all_times.extend(
[dateutil.parser.parse(d['key'][-1]) for d in data])
if self.by_submission_time:
# completion time is assumed to be in the phone's timezone until we can send proper timezone info
all_times = [
tz_utils.adjust_datetime_to_timezone(t, pytz.utc.zone,
self.timezone.zone)
for t in all_times
]
return [(t.weekday(), t.hour) for t in all_times]
def form_inline_display(form_id, timezone=pytz.utc):
if form_id:
form = XFormInstance.get(form_id)
if form:
return "%s: %s" % (tz_utils.adjust_datetime_to_timezone(form.received_on, pytz.utc.zone, timezone.zone).date(), form.xmlns)
return "missing form: %s" % form_id
return "empty form id found"
def get_couch_view_data(self, key, datespan=None):
default_value = None
try:
now = tz_utils.adjust_datetime_to_timezone(
getattr(datespan, self.start_or_end or "enddate"),
from_tz=datespan.timezone,
to_tz=pytz.utc)
except Exception:
now = datetime.datetime.now(tz=pytz.utc)
data = self.view_results(
endkey=key,
startkey=key +
[getattr(datespan, "%s_param_utc" % self.start_or_end)]
if datespan else [{}],
descending=True,
include_docs=True)
if not data:
return default_value
data = data.first()
if not data:
return default_value
doc = data.get('doc', {})
date_property = self._get_property_from_doc(
doc, self.property_name.split('.'))
try:
date_property = dateutil.parser.parse(date_property)
if not isinstance(date_property, datetime.datetime):
return default_value
td = now - date_property
days = td.days
except Exception:
return default_value
return days if days > 0 else 0
def get_num_missed_windows(case):
"""
Get the number of reminder events that were missed on registration day.
"""
domain_obj = Domain.get_by_name(case.domain, strict=True)
opened_timestamp = tz_utils.adjust_datetime_to_timezone(
case.opened_on,
pytz.utc.zone,
domain_obj.default_timezone
)
day_of_week = opened_timestamp.weekday()
time_of_day = opened_timestamp.time()
# In order to use timedelta, we need a datetime
current_time = datetime.combine(date(2000, 1, 1), time_of_day)
window_time = datetime.combine(date(2000, 1, 1), WINDOWS[day_of_week][0])
if current_time < window_time:
return 0
window_interval = (WINDOWS[day_of_week][1] - 60) / 5
for i in range(1, 5):
window_time += timedelta(minutes=window_interval)
if current_time < window_time:
return i
return 5
def chat(request, domain, contact_id):
domain_obj = Domain.get_by_name(domain, strict=True)
timezone = report_utils.get_timezone(None, domain)
# floored_utc_timestamp is the datetime in UTC representing
# midnight today in local time. This is used to calculate
# all message history choices' timestamps, so that choosing
# "Yesterday", for example, gives you data from yesterday at
# midnight local time.
local_date = datetime.now(timezone).date()
floored_utc_timestamp = tz_utils.adjust_datetime_to_timezone(
datetime.combine(local_date, time(0,0)),
timezone.zone,
pytz.utc.zone
).replace(tzinfo=None)
def _fmt(d):
return json_format_datetime(floored_utc_timestamp - timedelta(days=d))
history_choices = [(_(x), _fmt(y)) for (x, y) in SMS_CHAT_HISTORY_CHOICES]
history_choices.append(
(_("All Time"), json_format_datetime(datetime(1970, 1, 1)))
)
context = {
"domain" : domain,
"contact_id" : contact_id,
"contact" : get_contact(contact_id),
"message_count_threshold" : domain_obj.chat_message_count_threshold or DEFAULT_MESSAGE_COUNT_THRESHOLD,
"custom_case_username" : domain_obj.custom_case_username,
"history_choices" : history_choices,
}
template = settings.CUSTOM_CHAT_TEMPLATES.get(domain_obj.custom_chat_template) or "sms/chat.html"
return render(request, template, context)
def get_couch_view_data(self, key, datespan=None):
default_value = None
try:
now = tz_utils.adjust_datetime_to_timezone(getattr(datespan, self.start_or_end or "enddate"),
from_tz=datespan.timezone, to_tz=pytz.utc)
except Exception:
now = datetime.datetime.now(tz=pytz.utc)
data = self.view_results(
endkey=key,
startkey=key+[getattr(datespan, "%s_param_utc" % self.start_or_end)] if datespan else [{}],
descending=True,
include_docs=True
)
if not data:
return default_value
data = data.first()
if not data:
return default_value
doc = data.get('doc', {})
date_property = self._get_property_from_doc(doc, self.property_name.split('.'))
try:
date_property = dateutil.parser.parse(date_property)
if not isinstance(date_property, datetime.datetime):
return default_value
td = now - date_property
days = td.days
except Exception:
return default_value
return days if days > 0 else 0
def activity_times(self):
all_times = []
for user in self.users:
for form, info in self.all_relevant_forms.items():
key = make_form_couch_key(
self.domain,
user_id=user.get("user_id"),
xmlns=info["xmlns"],
app_id=info["app_id"],
by_submission_time=self.by_submission_time,
)
data = (
get_db()
.view(
"reports_forms/all_forms",
reduce=False,
startkey=key + [self.datespan.startdate_param_utc],
endkey=key + [self.datespan.enddate_param_utc],
)
.all()
)
all_times.extend([dateutil.parser.parse(d["key"][-1]) for d in data])
if self.by_submission_time:
# completion time is assumed to be in the phone's timezone until we can send proper timezone info
all_times = [tz_utils.adjust_datetime_to_timezone(t, pytz.utc.zone, self.timezone.zone) for t in all_times]
return [(t.weekday(), t.hour) for t in all_times]
def rows(self):
participants = self.get_participants()
result = []
for case in participants:
pid = case.get_case_property("pid")
study_arm = case.get_case_property("study_arm")
registration_date = get_date(case, "start_date")
first_name = case.get_case_property("first_name") or ""
if registration_date is None:
continue
first_survey_date = self.get_first_tuesday(registration_date)
row = [
self._fmt(pid),
self._fmt(first_name),
self._fmt(study_arm),
]
for i in range(8):
next_survey_date = first_survey_date + timedelta(days=7 * i)
response = self.get_first_survey_response(
case, next_survey_date)
if response == RESPONSE_NOT_APPLICABLE:
row.append(self._fmt("-"))
elif response == NO_RESPONSE:
row.append(self._fmt(_("No Response")))
else:
response_timestamp = tz_utils.adjust_datetime_to_timezone(
response.date, pytz.utc.zone,
self.domain_obj.default_timezone)
row.append(self._fmt_timestamp(response_timestamp))
result.append(row)
return result
def rows(self):
rows = list()
prefix = ["user"]
selected_form = self.request_params.get("form")
if selected_form:
prefix.append("form_type")
total = 0
total_seconds = 0
for user in self.users:
key = [" ".join(prefix), self.domain, user.get("user_id")]
if selected_form:
key.append(selected_form)
data = (
get_db()
.view(
"reports/completion_vs_submission",
startkey=key + [self.datespan.startdate_param_utc],
endkey=key + [self.datespan.enddate_param_utc],
reduce=False,
)
.all()
)
for item in data:
vals = item.get("value")
completion_time = dateutil.parser.parse(vals.get("completion_time")).replace(tzinfo=None)
completion_dst = (
False if self.timezone == pytz.utc else tz_utils.is_timezone_in_dst(self.timezone, completion_time)
)
completion_time = self.timezone.localize(completion_time, is_dst=completion_dst)
submission_time = dateutil.parser.parse(vals.get("submission_time"))
submission_time = submission_time.replace(tzinfo=pytz.utc)
submission_time = tz_utils.adjust_datetime_to_timezone(
submission_time, pytz.utc.zone, self.timezone.zone
)
td = submission_time - completion_time
td_total = td.seconds + td.days * 24 * 3600
rows.append(
[
self.get_user_link(user),
self._format_date(completion_time),
self._format_date(submission_time),
self._view_form_link(item.get("id", "")),
self.table_cell(td_total, self._format_td_status(td)),
]
)
if td_total >= 0:
total_seconds += td_total
total += 1
self.total_row = [
"Average",
"-",
"-",
"-",
self._format_td_status(int(total_seconds / total), False) if total > 0 else "--",
]
return rows
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = SMSLog.by_domain_date(self.domain, startdate, enddate)
result = []
username_map = {
} # Store the results of username lookups for faster loading
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
# Retrieve message log options
message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {})
abbreviated_phone_number_domains = message_log_options.get(
"abbreviated_phone_number_domains", [])
abbreviate_phone_number = (self.domain
in abbreviated_phone_number_domains)
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
recipient_id = message.couch_recipient
if recipient_id in [None, ""]:
username = "******"
elif recipient_id in username_map:
username = username_map.get(recipient_id)
else:
username = "******"
try:
if message.couch_recipient_doc_type == "CommCareCase":
username = CommCareCase.get(recipient_id).name
else:
username = CouchUser.get_by_user_id(
recipient_id).username
except Exception:
pass
username_map[recipient_id] = username
phone_number = message.phone_number
if abbreviate_phone_number and phone_number is not None:
phone_number = phone_number[0:7] if phone_number[
0:1] == "+" else phone_number[0:6]
timestamp = tz_utils.adjust_datetime_to_timezone(
message.date, pytz.utc.zone, self.timezone.zone)
result.append([
self._fmt_timestamp(timestamp),
self._fmt(username),
self._fmt(phone_number),
self._fmt(direction_map.get(message.direction, "-")),
self._fmt(message.text),
])
return result
def reminders_in_error(request, domain):
handler_map = {}
if request.method == "POST":
form = RemindersInErrorForm(request.POST)
if form.is_valid():
kwargs = {}
if is_bigcouch():
# Force a write to all nodes before returning
kwargs["w"] = bigcouch_quorum_count()
current_timestamp = datetime.utcnow()
for reminder_id in form.cleaned_data.get("selected_reminders"):
reminder = CaseReminder.get(reminder_id)
if reminder.domain != domain:
continue
if reminder.handler_id in handler_map:
handler = handler_map[reminder.handler_id]
else:
handler = reminder.handler
handler_map[reminder.handler_id] = handler
reminder.error = False
reminder.error_msg = None
handler.set_next_fire(reminder, current_timestamp)
reminder.save(**kwargs)
timezone = report_utils.get_timezone(request.couch_user.user_id, domain)
reminders = []
for reminder in CaseReminder.view("reminders/reminders_in_error",
startkey=[domain],
endkey=[domain, {}],
include_docs=True).all():
if reminder.handler_id in handler_map:
handler = handler_map[reminder.handler_id]
else:
handler = reminder.handler
handler_map[reminder.handler_id] = handler
recipient = reminder.recipient
case = reminder.case
reminders.append({
"reminder_id":
reminder._id,
"handler_id":
reminder.handler_id,
"handler_name":
handler.nickname,
"case_id":
case.get_id if case is not None else None,
"case_name":
case.name if case is not None else None,
"next_fire":
tz_utils.adjust_datetime_to_timezone(
reminder.next_fire, pytz.utc.zone,
timezone.zone).strftime("%Y-%m-%d %H:%M:%S"),
"error_msg":
reminder.error_msg,
"recipient_name":
get_recipient_name(recipient),
})
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = SMSLog.by_domain_date(self.domain, startdate, enddate)
result = []
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
# Retrieve message log options
message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {})
abbreviated_phone_number_domains = message_log_options.get(
"abbreviated_phone_number_domains", [])
abbreviate_phone_number = (self.domain
in abbreviated_phone_number_domains)
contact_cache = {}
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
recipient_id = message.couch_recipient
doc = None
if recipient_id not in [None, ""]:
try:
if message.couch_recipient_doc_type == "CommCareCase":
doc = CommCareCase.get(recipient_id)
else:
doc = CouchUser.get_by_user_id(recipient_id)
except Exception:
pass
if doc:
doc_info = get_doc_info(doc.to_json(), self.domain,
contact_cache)
else:
doc_info = None
phone_number = message.phone_number
if abbreviate_phone_number and phone_number is not None:
phone_number = phone_number[0:7] if phone_number[
0:1] == "+" else phone_number[0:6]
timestamp = tz_utils.adjust_datetime_to_timezone(
message.date, pytz.utc.zone, self.timezone.zone)
result.append([
self._fmt_timestamp(timestamp),
self._fmt_contact_link(message, doc_info),
self._fmt(phone_number),
self._fmt(direction_map.get(message.direction, "-")),
self._fmt(message.text),
])
return result
def utc_to_timezone(date, timezone, dest_fmt="%b %d, %Y %H:%M %Z"):
if not timezone:
timezone = pytz.utc
if not date:
return "---"
if not isinstance(date, datetime.datetime):
try:
date = datetime.datetime.replace(dateutil.parser.parse(date), tzinfo=pytz.utc)
except Exception as e:
return date
return tz_utils.adjust_datetime_to_timezone(date, pytz.utc, timezone.zone).strftime(dest_fmt)
def rows(self):
rows = []
total = 0
total_seconds = 0
if self.all_relevant_forms:
for user in self.users:
if not user.get("user_id"):
# calling get_form_data with no user_id will return ALL form data which is not what we want
continue
for form in self.all_relevant_forms.values():
data = self.get_form_data(user.get("user_id"), form["xmlns"], form["app_id"])
for item in data:
vals = item.get("value")
completion_time = dateutil.parser.parse(vals.get("completion_time")).replace(tzinfo=None)
completion_dst = (
False
if self.timezone == pytz.utc
else tz_utils.is_timezone_in_dst(self.timezone, completion_time)
)
completion_time = self.timezone.localize(completion_time, is_dst=completion_dst)
submission_time = dateutil.parser.parse(vals.get("submission_time"))
submission_time = submission_time.replace(tzinfo=pytz.utc)
submission_time = tz_utils.adjust_datetime_to_timezone(
submission_time, pytz.utc.zone, self.timezone.zone
)
td = submission_time - completion_time
td_total = td.seconds + td.days * 24 * 3600
rows.append(
[
self.get_user_link(user),
self._format_date(completion_time),
self._format_date(submission_time),
form["name"],
self._view_form_link(item.get("id", "")),
self.table_cell(td_total, self._format_td_status(td)),
]
)
if td_total >= 0:
total_seconds += td_total
total += 1
else:
rows.append(["No Submissions Available for this Date Range"] + ["--"] * 5)
self.total_row = [
_("Average"),
"-",
"-",
"-",
"-",
self._format_td_status(int(total_seconds / total), False) if total > 0 else "--",
]
return rows
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = FRISMSLog.view("sms/by_domain",
startkey=[self.domain, "SMSLog", startdate],
endkey=[self.domain, "SMSLog", enddate],
include_docs=True,
reduce=False).all()
result = []
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
message_bank_messages = get_message_bank(self.domain, for_comparing=True)
case_cache = CaseDbCache(domain=self.domain, strip_history=False, deleted_ok=True)
user_cache = UserCache()
show_only_survey_traffic = self.show_only_survey_traffic()
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
if show_only_survey_traffic and message.xforms_session_couch_id is None:
continue
# Add metadata from the message bank if it has not been added already
if (message.direction == OUTGOING) and (not message.fri_message_bank_lookup_completed):
add_metadata(message, message_bank_messages)
if message.couch_recipient_doc_type == "CommCareCase":
recipient = case_cache.get(message.couch_recipient)
else:
recipient = user_cache.get(message.couch_recipient)
if message.chat_user_id:
sender = user_cache.get(message.chat_user_id)
else:
sender = None
study_arm = None
if message.couch_recipient_doc_type == "CommCareCase":
study_arm = case_cache.get(message.couch_recipient).get_case_property("study_arm")
timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.domain_obj.default_timezone)
result.append([
self._fmt(self._participant_id(recipient)),
self._fmt(study_arm or "-"),
self._fmt(self._originator(message, recipient, sender)),
self._fmt_timestamp(timestamp),
self._fmt(message.text),
self._fmt(message.fri_id or "-"),
self._fmt(direction_map.get(message.direction,"-")),
])
return result
def get_display_data(data, prop_def, processors=None, timezone=pytz.utc):
# when prop_def came from a couchdbkit document, it will be a LazyDict with
# a broken pop method. This conversion also has the effect of a shallow
# copy, which we want.
prop_def = dict(prop_def)
default_processors = {
'yesno':
yesno,
'doc_info':
lambda value: pretty_doc_info(get_doc_info_by_id(
data['domain'], value))
}
processors = processors or {}
processors.update(default_processors)
def format_key(key):
key = key.replace('_', ' ')
return key.replace('-', ' ')
expr = prop_def.pop('expr')
name = prop_def.pop('name', format_key(expr))
format = prop_def.pop('format', None)
process = prop_def.pop('process', None)
# todo: nested attributes, jsonpath, indexing into related documents
val = data.get(expr, None)
if prop_def.pop('parse_date', None):
val = parse_date_or_datetime(val)
is_utc = prop_def.pop('is_utc', True)
if isinstance(val, datetime.datetime):
if is_utc:
if val.tzinfo is None:
val = val.replace(tzinfo=pytz.utc)
val = adjust_datetime_to_timezone(val, val.tzinfo, timezone.zone)
else:
val = val.replace(tzinfo=timezone)
try:
val = conditional_escape(processors[process](val))
except KeyError:
val = mark_safe(
to_html(None,
val,
timezone=timezone,
key_format=format_key,
collapse_lists=True,
**prop_def))
if format:
val = mark_safe(format.format(val))
return {"expr": expr, "name": name, "value": val}
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = SMSLog.by_domain_date(self.domain, startdate, enddate)
result = []
username_map = {} # Store the results of username lookups for faster loading
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
# Retrieve message log options
message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {})
abbreviated_phone_number_domains = message_log_options.get("abbreviated_phone_number_domains", [])
abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains)
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
recipient_id = message.couch_recipient
if recipient_id in [None, ""]:
username = "******"
elif recipient_id in username_map:
username = username_map.get(recipient_id)
else:
username = "******"
try:
if message.couch_recipient_doc_type == "CommCareCase":
username = CommCareCase.get(recipient_id).name
else:
username = CouchUser.get_by_user_id(recipient_id).username
except Exception:
pass
username_map[recipient_id] = username
phone_number = message.phone_number
if abbreviate_phone_number and phone_number is not None:
phone_number = phone_number[0:7] if phone_number[0:1] == "+" else phone_number[0:6]
timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.timezone.zone)
result.append([
self._fmt_timestamp(timestamp),
self._fmt(username),
self._fmt(phone_number),
self._fmt(direction_map.get(message.direction,"-")),
self._fmt(message.text),
])
return result
def get_display_data(data, prop_def, processors=None, timezone=pytz.utc):
# when prop_def came from a couchdbkit document, it will be a LazyDict with
# a broken pop method. This conversion also has the effect of a shallow
# copy, which we want.
prop_def = dict(prop_def)
default_processors = {
'yesno': yesno,
'doc_info': lambda value: pretty_doc_info(
get_doc_info_by_id(data['domain'], value)
)
}
processors = processors or {}
processors.update(default_processors)
def format_key(key):
key = key.replace('_', ' ')
return key.replace('-', ' ')
expr = prop_def.pop('expr')
name = prop_def.pop('name', format_key(expr))
format = prop_def.pop('format', None)
process = prop_def.pop('process', None)
# todo: nested attributes, jsonpath, indexing into related documents
val = data.get(expr, None)
if prop_def.pop('parse_date', None):
val = parse_date_or_datetime(val)
is_utc = prop_def.pop('is_utc', True)
if isinstance(val, datetime.datetime):
if is_utc:
if val.tzinfo is None:
val = val.replace(tzinfo=pytz.utc)
val = adjust_datetime_to_timezone(val, val.tzinfo, timezone.zone)
else:
val = val.replace(tzinfo=timezone)
try:
val = conditional_escape(processors[process](val))
except KeyError:
val = mark_safe(to_html(None, val,
timezone=timezone, key_format=format_key, collapse_lists=True,
**prop_def))
if format:
val = mark_safe(format.format(val))
return {
"expr": expr,
"name": name,
"value": val
}
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = SMSLog.by_domain_date(self.domain, startdate, enddate)
result = []
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
# Retrieve message log options
message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {})
abbreviated_phone_number_domains = message_log_options.get("abbreviated_phone_number_domains", [])
abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains)
contact_cache = {}
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
recipient_id = message.couch_recipient
doc = None
if recipient_id not in [None, ""]:
try:
if message.couch_recipient_doc_type == "CommCareCase":
doc = CommCareCase.get(recipient_id)
else:
doc = CouchUser.get_by_user_id(recipient_id)
except Exception:
pass
if doc:
doc_info = get_doc_info(doc.to_json(), self.domain,
contact_cache)
else:
doc_info = None
phone_number = message.phone_number
if abbreviate_phone_number and phone_number is not None:
phone_number = phone_number[0:7] if phone_number[0:1] == "+" else phone_number[0:6]
timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.timezone.zone)
result.append([
self._fmt_timestamp(timestamp),
self._fmt_contact_link(message, doc_info),
self._fmt(phone_number),
self._fmt(direction_map.get(message.direction,"-")),
self._fmt(message.text),
])
return result
def rows(self):
if self.dates_in_utc:
_dates = [
tz_utils.adjust_datetime_to_timezone(date, self.timezone.zone, pytz.utc.zone) for date in self.dates
]
else:
_dates = self.dates
date_map = dict([(date.strftime(DATE_FORMAT), i + 1) for (i, date) in enumerate(_dates)])
key = [self.domain]
results = (
get_db()
.view(
self.couch_view,
reduce=False,
startkey=key
+ [self.datespan.startdate_param_utc if self.dates_in_utc else self.datespan.startdate_param],
endkey=key + [self.datespan.enddate_param_utc if self.dates_in_utc else self.datespan.enddate_param],
)
.all()
)
user_map = dict([(user.get("user_id"), i) for (i, user) in enumerate(self.users)])
rows = [[0] * (2 + len(date_map)) for _ in range(len(self.users))]
total_row = [0] * (2 + len(date_map))
for result in results:
_, date = result["key"]
date = dateutil.parser.parse(date)
tz_offset = self.timezone.localize(self.datespan.enddate).strftime("%z")
date = date + datetime.timedelta(hours=int(tz_offset[0:3]), minutes=int(tz_offset[0] + tz_offset[3:5]))
date = date.isoformat()
val = result["value"]
user_id = val.get("user_id")
if user_id in self.user_ids:
date_key = date_map.get(date[0:10], None)
if date_key:
rows[user_map[user_id]][date_key] += 1
for i, user in enumerate(self.users):
rows[i][0] = self.get_user_link(user)
total = sum(rows[i][1:-1])
rows[i][-1] = total
total_row[1:-1] = [total_row[ind + 1] + val for ind, val in enumerate(rows[i][1:-1])]
total_row[-1] += total
total_row[0] = "All Users"
self.total_row = total_row
for row in rows:
row[1:] = [self.table_cell(val) for val in row[1:]]
return rows
def rows(self):
rows = []
all_hist = HQFormData.objects.filter(userID__in=self.user_ids, domain=self.domain)
history = all_hist.extra(order_by=['-received_on'])[self.pagination.start:self.pagination.start+self.pagination.count]
for data in history:
if data.userID in self.user_ids:
time = tz_utils.adjust_datetime_to_timezone(data.received_on, pytz.utc.zone, self.timezone.zone)
time = time.strftime("%Y-%m-%d %H:%M:%S")
xmlns = data.xmlns
app_id = data.app_id
xmlns = xmlns_to_name(self.domain, xmlns, app_id=app_id)
rows.append([self._form_data_link(data.instanceID), self.usernames[data.userID], time, xmlns])
return rows
def rows(self):
rows = []
all_hist = HQFormData.objects.filter(userID__in=self.user_ids, domain=self.domain)
history = all_hist.extra(order_by=['-received_on'])[self.pagination.start:self.pagination.start+self.pagination.count]
for data in history:
if data.userID in self.user_ids:
time = tz_utils.adjust_datetime_to_timezone(data.received_on, pytz.utc.zone, self.timezone.zone)
time = time.strftime("%Y-%m-%d %H:%M:%S")
xmlns = data.xmlns
app_id = data.app_id
xmlns = xmlns_to_name(self.domain, xmlns, app_id=app_id)
rows.append([self._form_data_link(data.instanceID), self.usernames[data.userID], time, xmlns])
return rows
def utc_to_timezone(date, timezone, dest_fmt="%b %d, %Y %H:%M %Z"):
if not timezone:
timezone = pytz.utc
if not date:
return "---"
if not isinstance(date, datetime.datetime):
try:
date = datetime.datetime.replace(dateutil.parser.parse(date),
tzinfo=pytz.utc)
except Exception as e:
return date
return tz_utils.adjust_datetime_to_timezone(
date, pytz.utc, timezone.zone).strftime(dest_fmt)
def raw_value(self, **kwargs):
user_id = kwargs.get('user_id')
datespan_keys = None
if self.inactivity_milestone > 0:
# inactivity milestone takes precedence over any ignore_datespan configurations
milestone_days_ago = tz_utils.adjust_datetime_to_timezone(self.report_datespan.enddate,
from_tz=self.report_datespan.timezone, to_tz=pytz.utc) - datetime.timedelta(days=self.inactivity_milestone)
datespan_keys = [[], [milestone_days_ago.isoformat()]]
elif not self.ignore_datespan:
datespan_keys = [[self.report_datespan.startdate_param_utc], [self.report_datespan.enddate_param_utc]]
status = self.case_status if self.case_status else None
cases = self.get_filtered_cases(self.report_domain, user_id, status=status, datespan_keys=datespan_keys)
return len(cases) if isinstance(cases, list) else None
def rows(self):
rows = []
total = 0
total_seconds = 0
if self.all_relevant_forms:
for user in self.users:
if not user.get('user_id'):
# calling get_form_data with no user_id will return ALL form data which is not what we want
continue
for form in self.all_relevant_forms.values():
data = self.get_form_data(user.get('user_id'),
form['xmlns'], form['app_id'])
for item in data:
vals = item.get('value')
completion_time = dateutil.parser.parse(
vals.get('completion_time')).replace(tzinfo=None)
completion_dst = False if self.timezone == pytz.utc else\
tz_utils.is_timezone_in_dst(self.timezone, completion_time)
completion_time = self.timezone.localize(
completion_time, is_dst=completion_dst)
submission_time = dateutil.parser.parse(
vals.get('submission_time'))
submission_time = submission_time.replace(
tzinfo=pytz.utc)
submission_time = tz_utils.adjust_datetime_to_timezone(
submission_time, pytz.utc.zone, self.timezone.zone)
td = submission_time - completion_time
td_total = (td.seconds + td.days * 24 * 3600)
rows.append([
self.get_user_link(user),
self._format_date(completion_time),
self._format_date(submission_time), form['name'],
self._view_form_link(item.get('id', '')),
self.table_cell(td_total,
self._format_td_status(td))
])
if td_total >= 0:
total_seconds += td_total
total += 1
else:
rows.append(['No Submissions Available for this Date Range'] +
['--'] * 5)
self.total_row = [
_("Average"), "-", "-", "-", "-",
self._format_td_status(int(total_seconds /
total), False) if total > 0 else "--"
]
return rows
def reminders_in_error(request, domain):
handler_map = {}
if request.method == "POST":
form = RemindersInErrorForm(request.POST)
if form.is_valid():
kwargs = {}
if is_bigcouch():
# Force a write to all nodes before returning
kwargs["w"] = bigcouch_quorum_count()
current_timestamp = datetime.utcnow()
for reminder_id in form.cleaned_data.get("selected_reminders"):
reminder = CaseReminder.get(reminder_id)
if reminder.domain != domain:
continue
if reminder.handler_id in handler_map:
handler = handler_map[reminder.handler_id]
else:
handler = reminder.handler
handler_map[reminder.handler_id] = handler
reminder.error = False
reminder.error_msg = None
handler.set_next_fire(reminder, current_timestamp)
reminder.save(**kwargs)
timezone = report_utils.get_timezone(request.couch_user.user_id, domain)
reminders = []
for reminder in CaseReminder.view("reminders/reminders_in_error", startkey=[domain], endkey=[domain, {}], include_docs=True).all():
if reminder.handler_id in handler_map:
handler = handler_map[reminder.handler_id]
else:
handler = reminder.handler
handler_map[reminder.handler_id] = handler
recipient = reminder.recipient
case = reminder.case
reminders.append({
"reminder_id" : reminder._id,
"handler_id" : reminder.handler_id,
"handler_name" : handler.nickname,
"case_id" : case.get_id if case is not None else None,
"case_name" : case.name if case is not None else None,
"next_fire" : tz_utils.adjust_datetime_to_timezone(reminder.next_fire, pytz.utc.zone, timezone.zone).strftime("%Y-%m-%d %H:%M:%S"),
"error_msg" : reminder.error_msg,
"recipient_name" : get_recipient_name(recipient),
})
context = {
"domain" : domain,
"reminders" : reminders,
"timezone" : timezone,
"timezone_now" : datetime.now(tz=timezone),
}
return render(request, "reminders/partial/reminders_in_error.html", context)
def rendered_report_title(self):
new_title = self.name
if self.errors_only:
new_title = (
"Errors & Warnings Log <small>for %s</small>" % (
", ".join(self.device_log_users)
)
if self.device_log_users
else "Errors & Warnings Log"
)
elif self.goto_key:
log = self.goto_log
date = adjust_datetime_to_timezone(log.date, from_tz=pytz.utc, to_tz=self.timezone)
new_title = "Last %s Logs <small>before %s</small>" % (self.limit, date.strftime("%b %d, %Y %H:%M"))
return mark_safe(new_title)
def rendered_report_title(self):
new_title = self.name
if self.errors_only:
new_title = ("Errors & Warnings Log <small>for %s</small>" %
(", ".join(self.device_log_users))
if self.device_log_users else
"Errors & Warnings Log")
elif self.goto_key:
log = self.goto_log
date = adjust_datetime_to_timezone(log.date,
from_tz=pytz.utc,
to_tz=self.timezone)
new_title = "Last %s Logs <small>before %s</small>" % (
self.limit, date.strftime("%b %d, %Y %H:%M"))
return mark_safe(new_title)
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = ExpectedCallbackEventLog.by_domain(self.domain, startdate,
enddate)
result = []
status_descriptions = {
CALLBACK_PENDING: _("Pending"),
CALLBACK_RECEIVED: _("Received"),
CALLBACK_MISSED: _("Missed"),
}
# Store the results of lookups for faster loading
username_map = {}
for event in data:
recipient_id = event.couch_recipient
if recipient_id in [None, ""]:
username = "******"
elif recipient_id in username_map:
username = username_map.get(recipient_id)
else:
username = "******"
try:
if event.couch_recipient_doc_type == "CommCareCase":
username = CommCareCase.get(recipient_id).name
else:
username = CouchUser.get_by_user_id(
recipient_id).username
except Exception:
pass
username_map[recipient_id] = username
timestamp = tz_utils.adjust_datetime_to_timezone(
event.date, pytz.utc.zone, self.timezone.zone)
row = [
self._fmt_timestamp(timestamp),
self._fmt(username),
self._fmt(status_descriptions.get(event.status, "-")),
]
result.append(row)
return result
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = SMSLog.by_domain_date(self.domain, startdate, enddate)
result = []
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
# Retrieve message log options
message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {})
abbreviated_phone_number_domains = message_log_options.get("abbreviated_phone_number_domains", [])
abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains)
contact_cache = {}
message_type_filter = self.get_message_type_filter()
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
message_types = self._get_message_types(message)
if not message_type_filter(message_types):
continue
doc_info = self.get_recipient_info(message, contact_cache)
phone_number = message.phone_number
if abbreviate_phone_number and phone_number is not None:
phone_number = phone_number[0:7] if phone_number[0:1] == "+" else phone_number[0:6]
timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.timezone.zone)
result.append([
self._fmt_timestamp(timestamp),
self._fmt_contact_link(message, doc_info),
self._fmt(phone_number),
self._fmt(direction_map.get(message.direction,"-")),
self._fmt(message.text),
self._fmt(", ".join(message_types)),
])
return result
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = ExpectedCallbackEventLog.by_domain(self.domain, startdate, enddate)
result = []
status_descriptions = {
CALLBACK_PENDING : _("Pending"),
CALLBACK_RECEIVED : _("Received"),
CALLBACK_MISSED : _("Missed"),
}
# Store the results of lookups for faster loading
username_map = {}
for event in data:
recipient_id = event.couch_recipient
if recipient_id in [None, ""]:
username = "******"
elif recipient_id in username_map:
username = username_map.get(recipient_id)
else:
username = "******"
try:
if event.couch_recipient_doc_type == "CommCareCase":
username = CommCareCase.get(recipient_id).name
else:
username = CouchUser.get_by_user_id(recipient_id).username
except Exception:
pass
username_map[recipient_id] = username
timestamp = tz_utils.adjust_datetime_to_timezone(event.date, pytz.utc.zone, self.timezone.zone)
row = [
self._fmt_timestamp(timestamp),
self._fmt(username),
self._fmt(status_descriptions.get(event.status, "-")),
]
result.append(row)
return result
def get_participants(self):
result = CommCareCase.view("hqcase/types_by_domain",
key=[self.domain, "participant"],
include_docs=True,
reduce=False).all()
local_now = tz_utils.adjust_datetime_to_timezone(
datetime.utcnow(), pytz.utc.zone, self.domain_obj.default_timezone)
local_date = local_now.date()
def filter_function(case):
registration_date = get_date(case, "start_date")
if registration_date is None:
return False
first_tuesday = self.get_first_tuesday(registration_date)
end_date = first_tuesday + timedelta(days=56)
return end_date >= local_date
result = filter(filter_function, result)
return result
def get_participants(self):
result = CommCareCase.view("hqcase/types_by_domain",
key=[self.domain, "participant"],
include_docs=True,
reduce=False).all()
local_now = tz_utils.adjust_datetime_to_timezone(
datetime.utcnow(), pytz.utc.zone, self.domain_obj.default_timezone)
local_date = local_now.date()
def filter_function(case):
registration_date = get_date(case, "start_date")
if registration_date is None:
return False
first_tuesday = self.get_first_tuesday(registration_date)
end_date = first_tuesday + timedelta(days=56)
return (not case.closed) and (end_date >= local_date)
result = filter(filter_function, result)
return result
def report_context(self):
data = defaultdict(lambda: 0)
for user in self.users:
startkey = [self.domain, user.get("user_id")]
endkey = [self.domain, user.get("user_id"), {}]
view = get_db().view("formtrends/form_time_by_user", startkey=startkey, endkey=endkey, group=True)
for row in view:
domain, _user, day, hour = row["key"]
if hour and day:
# adjust to timezone
now = datetime.datetime.utcnow()
hour = int(hour)
day = int(day)
report_time = datetime.datetime(now.year, now.month, now.day, hour, tzinfo=pytz.utc)
report_time = tz_utils.adjust_datetime_to_timezone(report_time, pytz.utc.zone, self.timezone.zone)
hour = report_time.hour
data["%d %02d" % (day, hour)] = data["%d %02d" % (day, hour)] + row["value"]
return dict(chart_url=self.generate_chart(data))
def raw_value(self, **kwargs):
user_id = kwargs.get('user_id')
datespan_keys = None
if self.inactivity_milestone > 0:
# inactivity milestone takes precedence over any ignore_datespan configurations
milestone_days_ago = tz_utils.adjust_datetime_to_timezone(
self.report_datespan.enddate,
from_tz=self.report_datespan.timezone,
to_tz=pytz.utc) - datetime.timedelta(
days=self.inactivity_milestone)
datespan_keys = [[], [milestone_days_ago.isoformat()]]
elif not self.ignore_datespan:
datespan_keys = [[self.report_datespan.startdate_param_utc],
[self.report_datespan.enddate_param_utc]]
status = self.case_status if self.case_status else None
cases = self.get_filtered_cases(self.report_domain,
user_id,
status=status,
datespan_keys=datespan_keys)
return len(cases) if isinstance(cases, list) else None
def get_display_data(prop):
expr = prop['expr']
name = prop.get('name', format_key(expr))
format = prop.get('format')
process = prop.get('process')
parse_date = prop.get('parse_date')
val = get_value(data, expr)
if parse_date and not isinstance(val, datetime.datetime):
try:
val = dateutil.parser.parse(val)
except:
val = val if val else '---'
if isinstance(val, datetime.datetime):
if val.tzinfo is None:
val = val.replace(tzinfo=pytz.utc)
val = adjust_datetime_to_timezone(val, val.tzinfo, timezone.zone)
if process:
val = escape(processors[process](val))
else:
if val is None:
val = '---'
val = mark_safe(to_html(None,
val, timezone=timezone, key_format=format_key,
collapse_lists=True))
if format:
val = mark_safe(format.format(val))
return {
"expr": expr,
"name": name,
"value": val
}
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = CallLog.by_domain_date(self.domain, startdate, enddate)
result = []
# Store the results of lookups for faster loading
contact_cache = {}
form_map = {}
xforms_sessions = {}
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
# Retrieve message log options
message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {})
abbreviated_phone_number_domains = message_log_options.get(
"abbreviated_phone_number_domains", [])
abbreviate_phone_number = (self.domain
in abbreviated_phone_number_domains)
for call in data:
doc_info = self.get_recipient_info(call, contact_cache)
form_unique_id = call.form_unique_id
if form_unique_id in [None, ""]:
form_name = "-"
elif form_unique_id in form_map:
form_name = form_map.get(form_unique_id)
else:
form_name = get_form_name(form_unique_id)
form_map[form_unique_id] = form_name
phone_number = call.phone_number
if abbreviate_phone_number and phone_number is not None:
phone_number = phone_number[0:7] if phone_number[
0:1] == "+" else phone_number[0:6]
timestamp = tz_utils.adjust_datetime_to_timezone(
call.date, pytz.utc.zone, self.timezone.zone)
if call.direction == INCOMING:
answered = "-"
else:
answered = _("Yes") if call.answered else _("No")
if call.xforms_session_id:
xforms_sessions[call.xforms_session_id] = None
row = [
call.xforms_session_id,
self._fmt_timestamp(timestamp),
self._fmt_contact_link(call, doc_info),
self._fmt(phone_number),
self._fmt(direction_map.get(call.direction, "-")),
self._fmt(form_name),
self._fmt("-"),
self._fmt(answered),
self._fmt(call.duration),
self._fmt(_("Yes") if call.error else _("No")),
self._fmt(call.error_message),
]
if self.request.couch_user.is_previewer():
row.append(self._fmt(call.gateway_session_id))
result.append(row)
# Look up the XFormsSession documents 500 at a time.
# Had to do this because looking up one document at a time slows things
# down a lot.
all_session_ids = xforms_sessions.keys()
limit = 500
range_max = int(ceil(len(all_session_ids) * 1.0 / limit))
for i in range(range_max):
lower_bound = i * limit
upper_bound = (i + 1) * limit
sessions = XFormsSession.view(
"smsforms/sessions_by_touchforms_id",
keys=all_session_ids[lower_bound:upper_bound],
include_docs=True).all()
for session in sessions:
xforms_sessions[session.session_id] = session.submission_id
# Add into the final result the link to the submission based on the
# outcome of the above lookups.
final_result = []
for row in result:
final_row = row[1:]
session_id = row[0]
if session_id:
submission_id = xforms_sessions[session_id]
if submission_id:
final_row[5] = self._fmt_submission_link(submission_id)
final_result.append(final_row)
return final_result
def api_history(request, domain):
result = []
contact_id = request.GET.get("contact_id", None)
start_date = request.GET.get("start_date", None)
timezone = report_utils.get_timezone(None, domain)
domain_obj = Domain.get_by_name(domain, strict=True)
try:
assert contact_id is not None
doc = get_contact(contact_id)
assert doc is not None
assert doc.domain == domain
except Exception:
return HttpResponse("[]")
query_start_date_str = None
if start_date is not None:
try:
query_start_date = parse(start_date)
query_start_date += timedelta(seconds=1)
query_start_date_str = json_format_datetime(query_start_date)
except Exception:
pass
if query_start_date_str is not None:
data = SMSLog.view("sms/by_recipient",
startkey=[doc.doc_type, contact_id, "SMSLog", INCOMING, query_start_date_str],
endkey=[doc.doc_type, contact_id, "SMSLog", INCOMING, {}],
include_docs=True,
reduce=False).all()
data += SMSLog.view("sms/by_recipient",
startkey=[doc.doc_type, contact_id, "SMSLog", OUTGOING, query_start_date_str],
endkey=[doc.doc_type, contact_id, "SMSLog", OUTGOING, {}],
include_docs=True,
reduce=False).all()
else:
data = SMSLog.view("sms/by_recipient",
startkey=[doc.doc_type, contact_id, "SMSLog"],
endkey=[doc.doc_type, contact_id, "SMSLog", {}],
include_docs=True,
reduce=False).all()
data.sort(key=lambda x : x.date)
username_map = {}
last_sms = None
for sms in data:
# Don't show outgoing SMS that haven't been processed yet
if sms.direction == OUTGOING and not sms.processed:
continue
# Filter SMS that are tied to surveys if necessary
if ((domain_obj.filter_surveys_from_chat and
sms.xforms_session_couch_id)
and not
(domain_obj.show_invalid_survey_responses_in_chat and
sms.direction == INCOMING and
sms.invalid_survey_response)):
continue
if sms.direction == INCOMING:
if doc.doc_type == "CommCareCase" and domain_obj.custom_case_username:
sender = doc.get_case_property(domain_obj.custom_case_username)
elif doc.doc_type == "CommCareCase":
sender = doc.name
else:
sender = doc.first_name or doc.raw_username
elif sms.chat_user_id is not None:
if sms.chat_user_id in username_map:
sender = username_map[sms.chat_user_id]
else:
try:
user = CouchUser.get_by_user_id(sms.chat_user_id)
sender = user.first_name or user.raw_username
except Exception:
sender = _("Unknown")
username_map[sms.chat_user_id] = sender
else:
sender = _("System")
last_sms = sms
result.append({
"sender" : sender,
"text" : sms.text,
"timestamp" : tz_utils.adjust_datetime_to_timezone(sms.date, pytz.utc.zone, timezone.zone).strftime("%I:%M%p %m/%d/%y").lower(),
"utc_timestamp" : json_format_datetime(sms.date),
})
if last_sms:
try:
entry, lock = LastReadMessage.get_locked_obj(
sms.domain,
request.couch_user._id,
sms.couch_recipient,
create=True
)
if (not entry.message_timestamp or
entry.message_timestamp < last_sms.date):
entry.message_id = last_sms._id
entry.message_timestamp = last_sms.date
entry.save()
lock.release()
except:
logging.exception("Could not create/save LastReadMessage for message %s" % last_sms._id)
# Don't let this block returning of the data
pass
return HttpResponse(json.dumps(result))
def get_past_two_weeks(self):
now = datetime.utcnow()
local_datetime = tz_utils.adjust_datetime_to_timezone(now, pytz.utc.zone, self.timezone.zone)
return [(local_datetime + timedelta(days = x)).date() for x in range(-14, 0)]
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = CallLog.by_domain_date(self.domain, startdate, enddate)
result = []
# Store the results of lookups for faster loading
username_map = {}
form_map = {}
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
# Retrieve message log options
message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {})
abbreviated_phone_number_domains = message_log_options.get("abbreviated_phone_number_domains", [])
abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains)
for call in data:
recipient_id = call.couch_recipient
if recipient_id in [None, ""]:
username = "******"
elif recipient_id in username_map:
username = username_map.get(recipient_id)
else:
username = "******"
try:
if call.couch_recipient_doc_type == "CommCareCase":
username = CommCareCase.get(recipient_id).name
else:
username = CouchUser.get_by_user_id(recipient_id).username
except Exception:
pass
username_map[recipient_id] = username
form_unique_id = call.form_unique_id
if form_unique_id in [None, ""]:
form_name = "-"
elif form_unique_id in form_map:
form_name = form_map.get(form_unique_id)
else:
form_name = get_form_name(form_unique_id)
form_map[form_unique_id] = form_name
phone_number = call.phone_number
if abbreviate_phone_number and phone_number is not None:
phone_number = phone_number[0:7] if phone_number[0:1] == "+" else phone_number[0:6]
timestamp = tz_utils.adjust_datetime_to_timezone(call.date, pytz.utc.zone, self.timezone.zone)
if call.direction == INCOMING:
answered = "-"
else:
answered = _("Yes") if call.answered else _("No")
if call.xforms_session_id is None:
submission_id = None
else:
session = XFormsSession.latest_by_session_id(call.xforms_session_id)
submission_id = session.submission_id
row = [
self._fmt_timestamp(timestamp),
self._fmt(username),
self._fmt(phone_number),
self._fmt(direction_map.get(call.direction,"-")),
self._fmt(form_name),
self._fmt("-") if submission_id is None else self._fmt_submission_link(submission_id),
self._fmt(answered),
self._fmt(call.duration),
self._fmt(_("Yes") if call.error else _("No")),
self._fmt(call.error_message),
]
if self.request.couch_user.is_previewer():
row.append(self._fmt(call.gateway_session_id))
result.append(row)
return result
def rows(self):
group_id = None
if self.request.couch_user.is_commcare_user():
group_ids = self.request.couch_user.get_group_ids()
if len(group_ids) > 0:
group_id = group_ids[0]
cases = CommCareCase.view("hqcase/types_by_domain",
key=[self.domain, "participant"],
reduce=False,
include_docs=True).all()
data = {}
for case in cases:
if case.closed:
continue
# If a site coordinator is viewing the report, only show participants from that site (group)
if group_id is None or group_id == case.owner_id:
data[case._id] = {
"name" : case.name,
"time_zone" : case.get_case_property("time_zone"),
"dates" : [None for x in range(14)],
}
dates = self.get_past_two_weeks()
date_strings = [date.strftime("%Y-%m-%d") for date in dates]
start_date = dates[0] - timedelta(days=1)
end_date = dates[-1] + timedelta(days=2)
start_utc_timestamp = json_format_datetime(start_date)
end_utc_timestamp = json_format_datetime(end_date)
expected_callback_events = ExpectedCallbackEventLog.view("sms/expected_callback_event",
startkey=[self.domain, start_utc_timestamp],
endkey=[self.domain, end_utc_timestamp],
include_docs=True).all()
for event in expected_callback_events:
if event.couch_recipient in data:
event_date = tz_utils.adjust_datetime_to_timezone(event.date, pytz.utc.zone, data[event.couch_recipient]["time_zone"]).date()
event_date = event_date.strftime("%Y-%m-%d")
if event_date in date_strings:
data[event.couch_recipient]["dates"][date_strings.index(event_date)] = event.status
result = []
for case_id, data_dict in data.items():
row = [
self._fmt(data_dict["name"]),
None,
None,
None,
]
total_no_response = 0
total_indicated = 0
total_pending = 0
for date_status in data_dict["dates"]:
if date_status == CALLBACK_PENDING:
total_indicated += 1
total_pending += 1
row.append(self._fmt(_("pending")))
elif date_status == CALLBACK_RECEIVED:
total_indicated += 1
row.append(self._fmt(_("OK")))
elif date_status == CALLBACK_MISSED:
total_indicated += 1
total_no_response += 1
row.append(self._fmt_highlight(_("No Response")))
else:
row.append(self._fmt(_("not indicated")))
if total_no_response > 0:
row[1] = self._fmt_highlight(total_no_response)
else:
row[1] = self._fmt(total_no_response)
row[2] = self._fmt(total_indicated)
row[3] = self._fmt(total_pending)
result.append(row)
return result
def _fmt_date(somedate):
time = tz_utils.adjust_datetime_to_timezone(
somedate, pytz.utc.zone, self.timezone.zone)
return time.strftime("%Y-%m-%d %H:%M:%S")
class SurveyResponsesReport(FRIReport):
name = ugettext_noop("Survey Responses")
slug = "fri_survey_responses"
description = ugettext_noop(
"Shows information pertaining to survey responses.")
emailable = False
@property
def headers(self):
cols = [
DataTablesColumn(_("PID")),
DataTablesColumn(_("Name")),
DataTablesColumn(_("Arm")),
DataTablesColumn(_("Week 1")),
DataTablesColumn(_("Week 2")),
DataTablesColumn(_("Week 3")),
DataTablesColumn(_("Week 4")),
DataTablesColumn(_("Week 5")),
DataTablesColumn(_("Week 6")),
DataTablesColumn(_("Week 7")),
DataTablesColumn(_("Week 8")),
]
header = DataTablesHeader(*cols)
header.custom_sort = [[0, "asc"]]
return header
@property
def rows(self):
participants = self.get_participants()
result = []
for case in participants:
pid = case.get_case_property("pid")
study_arm = case.get_case_property("study_arm")
registration_date = get_date(case, "start_date")
first_name = case.get_case_property("first_name") or ""
if registration_date is None:
continue
first_survey_date = self.get_first_tuesday(registration_date)
row = [
self._fmt(pid),
self._fmt(first_name),
self._fmt(study_arm),
]
for i in range(8):
next_survey_date = first_survey_date + timedelta(days=7 * i)
response = self.get_first_survey_response(
case, next_survey_date)
if response == RESPONSE_NOT_APPLICABLE:
row.append(self._fmt("-"))
elif response == NO_RESPONSE:
row.append(self._fmt(_("No Response")))
else:
response_timestamp = tz_utils.adjust_datetime_to_timezone(
response.date, pytz.utc.zone,
self.domain_obj.default_timezone)
row.append(self._fmt_timestamp(response_timestamp))
result.append(row)
return result
def get_first_tuesday(self, dt):
while dt.weekday() != 1:
dt = dt + timedelta(days=1)
return dt
def get_participants(self):
result = CommCareCase.view("hqcase/types_by_domain",
key=[self.domain, "participant"],
include_docs=True,
reduce=False).all()
local_now = tz_utils.adjust_datetime_to_timezone(
datetime.utcnow(), pytz.utc.zone, self.domain_obj.default_timezone)
local_date = local_now.date()
def filter_function(case):
registration_date = get_date(case, "start_date")
if registration_date is None:
return False
first_tuesday = self.get_first_tuesday(registration_date)
end_date = first_tuesday + timedelta(days=56)
return (not case.closed) and (end_date >= local_date)
result = filter(filter_function, result)
return result
def get_first_survey_response(self, case, dt):
timestamp_start = datetime.combine(dt, time(20, 45))
timestamp_start = tz_utils.adjust_datetime_to_timezone(
timestamp_start, self.domain_obj.default_timezone, pytz.utc.zone)
timestamp_start = timestamp_start.replace(tzinfo=None)
timestamp_start = json_format_datetime(timestamp_start)
timestamp_end = datetime.combine(dt + timedelta(days=1), time(11, 45))
timestamp_end = tz_utils.adjust_datetime_to_timezone(
timestamp_end, self.domain_obj.default_timezone, pytz.utc.zone)
timestamp_end = timestamp_end.replace(tzinfo=None)
if timestamp_end > datetime.utcnow():
return RESPONSE_NOT_APPLICABLE
timestamp_end = json_format_datetime(timestamp_end)
all_inbound = FRISMSLog.view("sms/by_recipient",
startkey=[
"CommCareCase", case._id, "SMSLog",
INCOMING, timestamp_start
],
endkey=[
"CommCareCase", case._id, "SMSLog",
INCOMING, timestamp_end
],
reduce=False,
include_docs=True).all()
survey_responses = filter(
lambda s: s.xforms_session_couch_id is not None, all_inbound)
if len(survey_responses) > 0:
return survey_responses[0]
else:
return NO_RESPONSE
def utc_now(self):
return tz_utils.adjust_datetime_to_timezone(datetime.datetime.utcnow(), self.timezone.zone, pytz.utc.zone)
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = CallLog.by_domain_date(self.domain, startdate, enddate)
result = []
# Store the results of lookups for faster loading
username_map = {}
form_map = {}
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
# Retrieve message log options
message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {})
abbreviated_phone_number_domains = message_log_options.get(
"abbreviated_phone_number_domains", [])
abbreviate_phone_number = (self.domain
in abbreviated_phone_number_domains)
for call in data:
recipient_id = call.couch_recipient
if recipient_id in [None, ""]:
username = "******"
elif recipient_id in username_map:
username = username_map.get(recipient_id)
else:
username = "******"
try:
if call.couch_recipient_doc_type == "CommCareCase":
username = CommCareCase.get(recipient_id).name
else:
username = CouchUser.get_by_user_id(
recipient_id).username
except Exception:
pass
username_map[recipient_id] = username
form_unique_id = call.form_unique_id
if form_unique_id in [None, ""]:
form_name = "-"
elif form_unique_id in form_map:
form_name = form_map.get(form_unique_id)
else:
form_name = get_form_name(form_unique_id)
form_map[form_unique_id] = form_name
phone_number = call.phone_number
if abbreviate_phone_number and phone_number is not None:
phone_number = phone_number[0:7] if phone_number[
0:1] == "+" else phone_number[0:6]
timestamp = tz_utils.adjust_datetime_to_timezone(
call.date, pytz.utc.zone, self.timezone.zone)
if call.direction == INCOMING:
answered = "-"
else:
answered = _("Yes") if call.answered else _("No")
if call.xforms_session_id is None:
submission_id = None
else:
session = XFormsSession.latest_by_session_id(
call.xforms_session_id)
submission_id = session.submission_id
row = [
self._fmt_timestamp(timestamp),
self._fmt(username),
self._fmt(phone_number),
self._fmt(direction_map.get(call.direction, "-")),
self._fmt(form_name),
self._fmt("-") if submission_id is None else
self._fmt_submission_link(submission_id),
self._fmt(answered),
self._fmt(call.duration),
self._fmt(_("Yes") if call.error else _("No")),
self._fmt(call.error_message),
]
if self.request.couch_user.is_previewer():
row.append(self._fmt(call.gateway_session_id))
result.append(row)
return result
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = CallLog.by_domain_date(self.domain, startdate, enddate)
result = []
# Store the results of lookups for faster loading
contact_cache = {}
form_map = {}
xforms_sessions = {}
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
# Retrieve message log options
message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {})
abbreviated_phone_number_domains = message_log_options.get("abbreviated_phone_number_domains", [])
abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains)
for call in data:
doc_info = self.get_recipient_info(call, contact_cache)
form_unique_id = call.form_unique_id
if form_unique_id in [None, ""]:
form_name = "-"
elif form_unique_id in form_map:
form_name = form_map.get(form_unique_id)
else:
form_name = get_form_name(form_unique_id)
form_map[form_unique_id] = form_name
phone_number = call.phone_number
if abbreviate_phone_number and phone_number is not None:
phone_number = phone_number[0:7] if phone_number[0:1] == "+" else phone_number[0:6]
timestamp = tz_utils.adjust_datetime_to_timezone(call.date, pytz.utc.zone, self.timezone.zone)
if call.direction == INCOMING:
answered = "-"
else:
answered = _("Yes") if call.answered else _("No")
if call.xforms_session_id:
xforms_sessions[call.xforms_session_id] = None
row = [
call.xforms_session_id,
self._fmt_timestamp(timestamp),
self._fmt_contact_link(call, doc_info),
self._fmt(phone_number),
self._fmt(direction_map.get(call.direction,"-")),
self._fmt(form_name),
self._fmt("-"),
self._fmt(answered),
self._fmt(call.duration),
self._fmt(_("Yes") if call.error else _("No")),
self._fmt(call.error_message),
]
if self.request.couch_user.is_previewer():
row.append(self._fmt(call.gateway_session_id))
result.append(row)
# Look up the XFormsSession documents 500 at a time.
# Had to do this because looking up one document at a time slows things
# down a lot.
all_session_ids = xforms_sessions.keys()
limit = 500
range_max = int(ceil(len(all_session_ids) * 1.0 / limit))
for i in range(range_max):
lower_bound = i * limit
upper_bound = (i + 1) * limit
sessions = XFormsSession.view("smsforms/sessions_by_touchforms_id",
keys=all_session_ids[lower_bound:upper_bound],
include_docs=True).all()
for session in sessions:
xforms_sessions[session.session_id] = session.submission_id
# Add into the final result the link to the submission based on the
# outcome of the above lookups.
final_result = []
for row in result:
final_row = row[1:]
session_id = row[0]
if session_id:
submission_id = xforms_sessions[session_id]
if submission_id:
final_row[5] = self._fmt_submission_link(submission_id)
final_result.append(final_row)
return final_result
def utc_now(self):
return tz_utils.adjust_datetime_to_timezone(datetime.datetime.utcnow(),
self.timezone.zone,
pytz.utc.zone)
