def testSMSSync(self):
self.deleteAllLogs()
self.assertEqual(self.getSMSLogCount(), 0)
self.assertEqual(self.getSMSCount(), 0)
# Test Create
sms = SMS()
self.setRandomSMSValues(sms)
sms.save()
sleep(1)
self.assertEqual(self.getSMSLogCount(), 1)
self.assertEqual(self.getSMSCount(), 1)
smslog = FRISMSLog.get(sms.couch_id)
self.checkFieldValues(smslog, sms, SMS._migration_get_fields())
self.assertTrue(FRISMSLog.get_db().get_rev(smslog._id).startswith('2-'))
# Test Update
self.setRandomSMSValues(sms)
sms.save()
sleep(1)
self.assertEqual(self.getSMSLogCount(), 1)
self.assertEqual(self.getSMSCount(), 1)
smslog = FRISMSLog.get(sms.couch_id)
self.checkFieldValues(smslog, sms, SMS._migration_get_fields())
self.assertTrue(FRISMSLog.get_db().get_rev(smslog._id).startswith('3-'))
def testFRISMSLogSync(self):
prev_couch_count = self.getSMSLogCount()
prev_sql_count = self.getSMSCount()
# Test Create
smslog = SMSLog()
self.setRandomSMSLogValues(smslog)
smslog.save()
sleep(1)
smslog = FRISMSLog.get(smslog._id)
self.setRandomFRISMSLogValues(smslog)
smslog.save()
sleep(1)
self.assertEqual(self.getSMSLogCount(), prev_couch_count + 1)
self.assertEqual(self.getSMSCount(), prev_sql_count + 1)
sms = SMS.objects.get(couch_id=smslog._id)
self.checkFieldValues(smslog, sms, FRISMSLog._migration_get_fields())
self.assertTrue(FRISMSLog.get_db().get_rev(smslog._id).startswith('2-'))
# Test Update
self.setRandomSMSLogValues(smslog)
self.setRandomFRISMSLogValues(smslog)
smslog.save()
sleep(1)
self.assertEqual(self.getSMSLogCount(), prev_couch_count + 1)
self.assertEqual(self.getSMSCount(), prev_sql_count + 1)
sms = SMS.objects.get(couch_id=smslog._id)
self.checkFieldValues(smslog, sms, FRISMSLog._migration_get_fields())
self.assertTrue(SMSLog.get_db().get_rev(smslog._id).startswith('3-'))
def testSMSSync(self):
prev_couch_count = self.getSMSLogCount()
prev_sql_count = self.getSMSCount()
# Test Create
sms = SMS()
self.setRandomSMSValues(sms)
sms.save()
sleep(1)
self.assertEqual(self.getSMSLogCount(), prev_couch_count + 1)
self.assertEqual(self.getSMSCount(), prev_sql_count + 1)
smslog = FRISMSLog.get(sms.couch_id)
self.checkFieldValues(smslog, sms, SMS._migration_get_fields())
self.assertTrue(FRISMSLog.get_db().get_rev(
smslog._id).startswith('2-'))
# Test Update
self.setRandomSMSValues(sms)
sms.save()
sleep(1)
self.assertEqual(self.getSMSLogCount(), prev_couch_count + 1)
self.assertEqual(self.getSMSCount(), prev_sql_count + 1)
smslog = FRISMSLog.get(sms.couch_id)
self.checkFieldValues(smslog, sms, SMS._migration_get_fields())
self.assertTrue(FRISMSLog.get_db().get_rev(
smslog._id).startswith('3-'))
def run_migration(self):
count = 0
ids = self.get_sms_couch_ids()
total_count = len(ids)
for doc in iter_docs(FRISMSLog.get_db(), ids):
try:
self.clean_doc(doc)
couch_sms = FRISMSLog.wrap(doc)
couch_sms._migration_do_sync()
except Exception as e:
print 'Could not sync SMSLog %s: %s' % (doc['_id'], e)
count += 1
if (count % 10000) == 0:
print 'Processed %s / %s documents' % (count, total_count)
def run_migration(self):
count = 0
ids = self.get_sms_couch_ids()
total_count = len(ids)
for doc in iter_docs(FRISMSLog.get_db(), ids):
try:
self.clean_doc(doc)
couch_sms = FRISMSLog.wrap(doc)
couch_sms._migration_do_sync()
except Exception as e:
print "Could not sync SMSLog %s: %s" % (doc["_id"], e)
count += 1
if (count % 10000) == 0:
print "Processed %s / %s documents" % (count, total_count)
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = FRISMSLog.view("sms/by_domain",
startkey=[self.domain, "SMSLog", startdate],
endkey=[self.domain, "SMSLog", enddate],
include_docs=True,
reduce=False).all()
result = []
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
message_bank_messages = get_message_bank(self.domain,
for_comparing=True)
case_cache = CaseDbCache(domain=self.domain,
strip_history=False,
deleted_ok=True)
user_cache = UserCache()
show_only_survey_traffic = self.show_only_survey_traffic()
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
if show_only_survey_traffic and message.xforms_session_couch_id is None:
continue
# Add metadata from the message bank if it has not been added already
if (message.direction == OUTGOING) and (
not message.fri_message_bank_lookup_completed):
add_metadata(message, message_bank_messages)
if message.couch_recipient_doc_type == "CommCareCase":
recipient = case_cache.get(message.couch_recipient)
else:
recipient = user_cache.get(message.couch_recipient)
if message.chat_user_id:
sender = user_cache.get(message.chat_user_id)
else:
sender = None
study_arm = None
if message.couch_recipient_doc_type == "CommCareCase":
study_arm = case_cache.get(
message.couch_recipient).get_case_property("study_arm")
timestamp = ServerTime(message.date).user_time(
self.timezone).done()
result.append([
self._fmt(self._participant_id(recipient)),
self._fmt(study_arm or "-"),
self._fmt(self._originator(message, recipient, sender)),
self._fmt_timestamp(timestamp),
self._fmt(message.text),
self._fmt(message.fri_id or "-"),
self._fmt(direction_map.get(message.direction, "-")),
])
return result
def get_participant_messages(self, case):
result = FRISMSLog.view("sms/by_recipient",
startkey=["CommCareCase", case._id, "SMSLog", OUTGOING],
endkey=["CommCareCase", case._id, "SMSLog", OUTGOING, {}],
reduce=False,
include_docs=True).all()
return result
def get_first_survey_response(self, case, dt):
timestamp_start = datetime.combine(dt, time(20, 45))
timestamp_start = tz_utils.adjust_datetime_to_timezone(
timestamp_start, self.domain_obj.default_timezone, pytz.utc.zone)
timestamp_start = timestamp_start.replace(tzinfo=None)
timestamp_start = json_format_datetime(timestamp_start)
timestamp_end = datetime.combine(dt + timedelta(days=1), time(11, 0))
timestamp_end = tz_utils.adjust_datetime_to_timezone(
timestamp_end, self.domain_obj.default_timezone, pytz.utc.zone)
timestamp_end = timestamp_end.replace(tzinfo=None)
if timestamp_end > datetime.utcnow():
return RESPONSE_NOT_APPLICABLE
timestamp_end = json_format_datetime(timestamp_end)
all_inbound = FRISMSLog.view(
"sms/by_recipient",
startkey=["CommCareCase", case._id, "SMSLog", INCOMING, timestamp_start],
endkey=["CommCareCase", case._id, "SMSLog", INCOMING, timestamp_end],
reduce=False,
include_docs=True
).all()
survey_responses = filter(lambda s: s.xforms_session_couch_id is not None, all_inbound)
if len(survey_responses) > 0:
return survey_responses[0]
else:
return NO_RESPONSE
def get_participant_messages(self, case):
result = FRISMSLog.view(
"sms/by_recipient",
startkey=["CommCareCase", case._id, "SMSLog", OUTGOING],
endkey=["CommCareCase", case._id, "SMSLog", OUTGOING, {}],
reduce=False,
include_docs=True).all()
return result
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = FRISMSLog.view("sms/by_domain",
startkey=[self.domain, "SMSLog", startdate],
endkey=[self.domain, "SMSLog", enddate],
include_docs=True,
reduce=False).all()
result = []
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
message_bank_messages = get_message_bank(self.domain, for_comparing=True)
FormProcessorInterface(self.domain).casedb_cache(
domain=self.domain, strip_history=False, deleted_ok=True
)
user_cache = UserCache()
show_only_survey_traffic = self.show_only_survey_traffic()
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
if show_only_survey_traffic and message.xforms_session_couch_id is None:
continue
# Add metadata from the message bank if it has not been added already
if (message.direction == OUTGOING) and (not message.fri_message_bank_lookup_completed):
add_metadata(message, message_bank_messages)
if message.couch_recipient_doc_type == "CommCareCase":
recipient = case_cache.get(message.couch_recipient)
else:
recipient = user_cache.get(message.couch_recipient)
if message.chat_user_id:
sender = user_cache.get(message.chat_user_id)
else:
sender = None
study_arm = None
if message.couch_recipient_doc_type == "CommCareCase":
study_arm = case_cache.get(message.couch_recipient).get_case_property("study_arm")
timestamp = ServerTime(message.date).user_time(self.timezone).done()
result.append([
self._fmt(self._participant_id(recipient)),
self._fmt(study_arm or "-"),
self._fmt(self._originator(message, recipient, sender)),
self._fmt_timestamp(timestamp),
self._fmt(message.text),
self._fmt(message.fri_id or "-"),
self._fmt(direction_map.get(message.direction,"-")),
])
return result
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = FRISMSLog.view("sms/by_domain",
startkey=[self.domain, "SMSLog", startdate],
endkey=[self.domain, "SMSLog", enddate],
include_docs=True,
reduce=False).all()
result = []
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
message_bank_messages = get_message_bank(self.domain, for_comparing=True)
case_cache = CaseDbCache(domain=self.domain, strip_history=False, deleted_ok=True)
user_cache = UserCache()
for message in data:
# Add metadata from the message bank if it has not been added already
if (message.direction == OUTGOING) and (not message.fri_message_bank_lookup_completed):
add_metadata(message, message_bank_messages)
if message.couch_recipient_doc_type == "CommCareCase":
recipient = case_cache.get(message.couch_recipient)
else:
recipient = user_cache.get(message.couch_recipient)
if message.chat_user_id:
sender = user_cache.get(message.chat_user_id)
else:
sender = None
study_arm = None
if message.couch_recipient_doc_type == "CommCareCase":
study_arm = case_cache.get(message.couch_recipient).get_case_property("study_arm")
timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.timezone.zone)
result.append([
self._fmt(self._participant_id(recipient)),
self._fmt(study_arm or "-"),
self._fmt(self._originator(message, recipient, sender)),
self._fmt_timestamp(timestamp),
self._fmt(message.text),
self._fmt(message.fri_id or "-"),
self._fmt(direction_map.get(message.direction,"-")),
])
return result
class SurveyResponsesReport(FRIReport):
name = ugettext_noop("Survey Responses")
slug = "fri_survey_responses"
description = ugettext_noop(
"Shows information pertaining to survey responses.")
emailable = False
fields = [
"custom.fri.reports.filters.SurveyDateSelector",
]
@property
def survey_report_date(self):
return SurveyDateSelector.get_value(self.request, self.domain)
@property
def headers(self):
cols = [
DataTablesColumn(_("PID")),
DataTablesColumn(_("Name")),
DataTablesColumn(_("Arm")),
DataTablesColumn(_("Week 1")),
DataTablesColumn(_("Week 2")),
DataTablesColumn(_("Week 3")),
DataTablesColumn(_("Week 4")),
DataTablesColumn(_("Week 5")),
DataTablesColumn(_("Week 6")),
DataTablesColumn(_("Week 7")),
DataTablesColumn(_("Week 8")),
]
header = DataTablesHeader(*cols)
header.custom_sort = [[0, "asc"]]
return header
@property
def rows(self):
participants = self.get_participants()
result = []
for case in participants:
pid = case.get_case_property("pid")
study_arm = case.get_case_property("study_arm")
registration_date = get_date(case, "start_date")
first_name = case.get_case_property("first_name") or ""
if registration_date is None:
continue
first_survey_date = self.get_first_tuesday(registration_date)
row = [
self._fmt(pid),
self._fmt(first_name),
self._fmt(study_arm),
]
for i in range(8):
next_survey_date = first_survey_date + timedelta(days=7 * i)
response = self.get_first_survey_response(
case, next_survey_date)
if response == RESPONSE_NOT_APPLICABLE:
row.append(self._fmt("-"))
elif response == NO_RESPONSE:
row.append(self._fmt(_("No Response")))
else:
response_timestamp = ServerTime(response.date).user_time(
self.timezone).done()
row.append(self._fmt_timestamp(response_timestamp))
result.append(row)
return result
def get_first_tuesday(self, dt):
while dt.weekday() != 1:
dt = dt + timedelta(days=1)
return dt
def get_participants(self):
result = get_cases_in_domain(self.domain, 'participant')
survey_report_date = parse(self.survey_report_date).date()
def filter_function(case):
registration_date = get_date(case, "start_date")
if registration_date is None:
return False
first_tuesday = self.get_first_tuesday(registration_date)
last_tuesday = first_tuesday + timedelta(days=49)
return first_tuesday <= survey_report_date <= last_tuesday
result = filter(filter_function, result)
return result
def get_first_survey_response(self, case, dt):
timestamp_start = datetime.combine(dt, time(20, 45))
timestamp_start = UserTime(timestamp_start,
self.timezone).server_time().done()
timestamp_start = json_format_datetime(timestamp_start)
timestamp_end = datetime.combine(dt + timedelta(days=1), time(11, 45))
timestamp_end = UserTime(timestamp_end,
self.timezone).server_time().done()
if timestamp_end > datetime.utcnow():
return RESPONSE_NOT_APPLICABLE
timestamp_end = json_format_datetime(timestamp_end)
all_inbound = FRISMSLog.view("sms/by_recipient",
startkey=[
"CommCareCase", case._id, "SMSLog",
INCOMING, timestamp_start
],
endkey=[
"CommCareCase", case._id, "SMSLog",
INCOMING, timestamp_end
],
reduce=False,
include_docs=True).all()
survey_responses = filter(
lambda s: s.xforms_session_couch_id is not None, all_inbound)
if len(survey_responses) > 0:
return survey_responses[0]
else:
return NO_RESPONSE
class SurveyResponsesReport(FRIReport):
name = ugettext_noop("Survey Responses")
slug = "fri_survey_responses"
description = ugettext_noop(
"Shows information pertaining to survey responses.")
emailable = False
@property
def headers(self):
cols = [
DataTablesColumn(_("PID")),
DataTablesColumn(_("Name")),
DataTablesColumn(_("Arm")),
DataTablesColumn(_("Week 1")),
DataTablesColumn(_("Week 2")),
DataTablesColumn(_("Week 3")),
DataTablesColumn(_("Week 4")),
DataTablesColumn(_("Week 5")),
DataTablesColumn(_("Week 6")),
DataTablesColumn(_("Week 7")),
DataTablesColumn(_("Week 8")),
]
header = DataTablesHeader(*cols)
header.custom_sort = [[0, "asc"]]
return header
@property
def rows(self):
participants = self.get_participants()
result = []
for case in participants:
pid = case.get_case_property("pid")
study_arm = case.get_case_property("study_arm")
registration_date = get_date(case, "start_date")
first_name = case.get_case_property("first_name") or ""
if registration_date is None:
continue
first_survey_date = self.get_first_tuesday(registration_date)
row = [
self._fmt(pid),
self._fmt(first_name),
self._fmt(study_arm),
]
for i in range(8):
next_survey_date = first_survey_date + timedelta(days=7 * i)
response = self.get_first_survey_response(
case, next_survey_date)
if response == RESPONSE_NOT_APPLICABLE:
row.append(self._fmt("-"))
elif response == NO_RESPONSE:
row.append(self._fmt(_("No Response")))
else:
response_timestamp = tz_utils.adjust_datetime_to_timezone(
response.date, pytz.utc.zone,
self.domain_obj.default_timezone)
row.append(self._fmt_timestamp(response_timestamp))
result.append(row)
return result
def get_first_tuesday(self, dt):
while dt.weekday() != 1:
dt = dt + timedelta(days=1)
return dt
def get_participants(self):
result = CommCareCase.view("hqcase/types_by_domain",
key=[self.domain, "participant"],
include_docs=True,
reduce=False).all()
local_now = tz_utils.adjust_datetime_to_timezone(
datetime.utcnow(), pytz.utc.zone, self.domain_obj.default_timezone)
local_date = local_now.date()
def filter_function(case):
registration_date = get_date(case, "start_date")
if registration_date is None:
return False
first_tuesday = self.get_first_tuesday(registration_date)
end_date = first_tuesday + timedelta(days=56)
return (not case.closed) and (end_date >= local_date)
result = filter(filter_function, result)
return result
def get_first_survey_response(self, case, dt):
timestamp_start = datetime.combine(dt, time(20, 45))
timestamp_start = tz_utils.adjust_datetime_to_timezone(
timestamp_start, self.domain_obj.default_timezone, pytz.utc.zone)
timestamp_start = timestamp_start.replace(tzinfo=None)
timestamp_start = json_format_datetime(timestamp_start)
timestamp_end = datetime.combine(dt + timedelta(days=1), time(11, 45))
timestamp_end = tz_utils.adjust_datetime_to_timezone(
timestamp_end, self.domain_obj.default_timezone, pytz.utc.zone)
timestamp_end = timestamp_end.replace(tzinfo=None)
if timestamp_end > datetime.utcnow():
return RESPONSE_NOT_APPLICABLE
timestamp_end = json_format_datetime(timestamp_end)
all_inbound = FRISMSLog.view("sms/by_recipient",
startkey=[
"CommCareCase", case._id, "SMSLog",
INCOMING, timestamp_start
],
endkey=[
"CommCareCase", case._id, "SMSLog",
INCOMING, timestamp_end
],
reduce=False,
include_docs=True).all()
survey_responses = filter(
lambda s: s.xforms_session_couch_id is not None, all_inbound)
if len(survey_responses) > 0:
return survey_responses[0]
else:
return NO_RESPONSE
