def create_session_object(cls, domain, contact, phone_number, app, form, expire_after=MAX_SESSION_LENGTH,
reminder_intervals=None, submit_partially_completed_forms=False,
include_case_updates_in_partial_submissions=False):
now = utcnow()
session = cls(
couch_id=uuid.uuid4().hex,
connection_id=contact.get_id,
form_xmlns=form.xmlns,
start_time=now,
modified_time=now,
completed=False,
domain=domain,
user_id=contact.get_id,
app_id=app.get_id,
session_type=XFORMS_SESSION_SMS,
phone_number=strip_plus(phone_number),
expire_after=expire_after,
session_is_open=True,
reminder_intervals=reminder_intervals or [],
current_reminder_num=0,
submit_partially_completed_forms=submit_partially_completed_forms,
include_case_updates_in_partial_submissions=include_case_updates_in_partial_submissions,
)
session.set_current_action_due_timestamp()
return session
def _sync_case_for_messaging_rule(domain, case_id, rule_id):
case_load_counter("messaging_rule_sync", domain)()
case = CaseAccessors(domain).get_case(case_id)
rule = _get_cached_rule(domain, rule_id)
if rule:
rule.run_rule(case, utcnow())
MessagingRuleProgressHelper(rule_id).increment_current_case_count()
def _clean_xml_for_partial_submission(xml, should_remove_case_actions):
"""
Helper method to cleanup partially completed xml for submission
:param xml: partially completed xml
:param should_remove_case_actions: if True, remove case actions (create, update, close) from xml
:return: byte str of cleaned xml
"""
root = XML(xml)
case_tag_regex = re.compile(
r"^(\{.*\}){0,1}case$"
) # Use regex in order to search regardless of namespace
meta_tag_regex = re.compile(r"^(\{.*\}){0,1}meta$")
timeEnd_tag_regex = re.compile(r"^(\{.*\}){0,1}timeEnd$")
current_timestamp = json_format_datetime(utcnow())
for child in root:
if case_tag_regex.match(child.tag) is not None:
# Found the case tag
case_element = child
case_element.set("date_modified", current_timestamp)
if should_remove_case_actions:
child_elements = [case_action for case_action in case_element]
for case_action in child_elements:
case_element.remove(case_action)
elif meta_tag_regex.match(child.tag) is not None:
# Found the meta tag, now set the value for timeEnd
for meta_child in child:
if timeEnd_tag_regex.match(meta_child.tag):
meta_child.text = current_timestamp
return tostring(root)
def create_session_object(cls, domain, contact, phone_number, app, form, expire_after=MAX_SESSION_LENGTH,
reminder_intervals=None, submit_partially_completed_forms=False,
include_case_updates_in_partial_submissions=False):
now = utcnow()
session = cls(
couch_id=uuid.uuid4().hex,
connection_id=contact.get_id,
form_xmlns=form.xmlns,
start_time=now,
modified_time=now,
completed=False,
domain=domain,
user_id=contact.get_id,
app_id=app.get_id,
session_type=XFORMS_SESSION_SMS,
phone_number=strip_plus(phone_number),
expire_after=expire_after,
session_is_open=True,
reminder_intervals=reminder_intervals or [],
current_reminder_num=0,
submit_partially_completed_forms=submit_partially_completed_forms,
include_case_updates_in_partial_submissions=include_case_updates_in_partial_submissions,
)
session.set_current_action_due_timestamp()
return session
def handle_due_survey_action(domain, contact_id, session_id):
with critical_section_for_smsforms_sessions(contact_id):
session = SQLXFormsSession.by_session_id(session_id)
if (not session or not session.session_is_open
or session.current_action_due > utcnow()):
return
if session.current_action_is_a_reminder:
# Resend the current question in the open survey to the contact
p = PhoneNumber.get_phone_number_for_owner(session.connection_id,
session.phone_number)
if p:
metadata = MessageMetadata(
workflow=session.workflow,
xforms_session_couch_id=session._id,
)
resp = current_question(session.session_id, domain)
send_sms_to_verified_number(
p,
resp.event.text_prompt,
metadata,
logged_subevent=session.related_subevent)
session.move_to_next_action()
session.save()
else:
# Close the session
session.close()
session.save()
def set_first_event_due_timestamp(self, instance, start_date=None):
"""
If start_date is None, we set it automatically ensuring that
self.next_event_due does not get set in the past for the first
event.
"""
if start_date:
instance.start_date = start_date
else:
instance.start_date = ServerTime(util.utcnow()).user_time(
instance.timezone).done().date()
self.set_next_event_due_timestamp(instance)
if (not self.schedule_length == self.MONTHLY and not start_date
and instance.next_event_due < util.utcnow()):
instance.start_date += timedelta(days=1)
instance.next_event_due += timedelta(days=1)
def handle(self, **options):
rule = self.get_rule(options['domain'], options['rule_id'])
print("Fetching case ids...")
case_ids = CaseAccessors(rule.domain).get_case_ids_in_domain(rule.case_type)
case_id_chunks = list(chunked(case_ids, 10))
for case_id_chunk in with_progress_bar(case_id_chunks):
case_id_chunk = list(case_id_chunk)
with CriticalSection([get_sync_key(case_id) for case_id in case_id_chunk], timeout=5 * 60):
for case in CaseAccessors(rule.domain).get_cases(case_id_chunk):
rule.run_rule(case, utcnow())
def submit_unfinished_form(session):
"""
Gets the raw instance of the session's form and submits it. This is used with
sms and ivr surveys to save all questions answered so far in a session that
needs to close.
If session.include_case_updates_in_partial_submissions is False, no case
create / update / close actions will be performed, but the form will still be submitted.
The form is only submitted if the smsforms session has not yet completed.
"""
# Get and clean the raw xml
try:
response = FormplayerInterface(session.session_id,
session.domain).get_raw_instance()
# Formplayer's ExceptionResponseBean includes the exception message,
# stautus ("error"), url, and type ("text")
if response.get('status') == 'error':
raise TouchformsError(response.get('exception'))
xml = response['output']
except InvalidSessionIdException:
return
root = XML(xml)
case_tag_regex = re.compile(
r"^(\{.*\}){0,1}case$"
) # Use regex in order to search regardless of namespace
meta_tag_regex = re.compile(r"^(\{.*\}){0,1}meta$")
timeEnd_tag_regex = re.compile(r"^(\{.*\}){0,1}timeEnd$")
current_timstamp = json_format_datetime(utcnow())
for child in root:
if case_tag_regex.match(child.tag) is not None:
# Found the case tag
case_element = child
case_element.set("date_modified", current_timstamp)
if not session.include_case_updates_in_partial_submissions:
# Remove case actions (create, update, close)
child_elements = [case_action for case_action in case_element]
for case_action in child_elements:
case_element.remove(case_action)
elif meta_tag_regex.match(child.tag) is not None:
# Found the meta tag, now set the value for timeEnd
for meta_child in child:
if timeEnd_tag_regex.match(meta_child.tag):
meta_child.text = current_timstamp
cleaned_xml = tostring(root)
# Submit the xml
result = submit_form_locally(cleaned_xml,
session.domain,
app_id=session.app_id,
partial_submission=True)
session.submission_id = result.xform.form_id
def handle_due_survey_action(domain, contact_id, session_id):
with critical_section_for_smsforms_sessions(contact_id):
session = SQLXFormsSession.by_session_id(session_id)
if (not session or not session.session_is_open
or session.current_action_due > utcnow()):
return
if toggles.ONE_PHONE_NUMBER_MULTIPLE_CONTACTS.enabled(domain):
if not XFormsSessionSynchronization.claim_channel_for_session(
session):
from .management.commands import handle_survey_actions
# Unless we release this lock, handle_survey_actions will be unable to requeue this task
# for the default duration of 1h, which we don't want
handle_survey_actions.Command.get_enqueue_lock(
session_id, session.current_action_due).release()
return
if session_is_stale(session):
# If a session is having some unrecoverable errors that aren't benefitting from
# being retried, those errors should show up in sentry log and the fix should
# be dealt with. In terms of the current session itself, we just close it out
# to allow new sessions to start.
session.mark_completed(False)
session.save()
return
if session.current_action_is_a_reminder:
# Resend the current question in the open survey to the contact
p = PhoneNumber.get_phone_number_for_owner(session.connection_id,
session.phone_number)
if p:
metadata = MessageMetadata(
workflow=session.workflow,
xforms_session_couch_id=session._id,
)
resp = FormplayerInterface(session.session_id,
domain).current_question()
send_sms_to_verified_number(
p,
resp.event.text_prompt,
metadata,
logged_subevent=session.related_subevent)
session.move_to_next_action()
session.save()
else:
# Close the session
session.close()
session.save()
def handle(self, **options):
rule = self.get_rule(options['domain'], options['rule_id'])
print("Fetching case ids...")
case_ids = CaseAccessors(rule.domain).get_case_ids_in_domain(
rule.case_type)
case_id_chunks = list(chunked(case_ids, 10))
for case_id_chunk in with_progress_bar(case_id_chunks):
case_id_chunk = list(case_id_chunk)
with CriticalSection(
[get_sync_key(case_id) for case_id in case_id_chunk],
timeout=5 * 60):
for case in CaseAccessors(
rule.domain).get_cases(case_id_chunk):
rule.run_rule(case, utcnow())
def mark_completed(self, completed):
self.session_is_open = False
self.completed = completed
self.modified_time = self.end_time = utcnow()
if toggles.ONE_PHONE_NUMBER_MULTIPLE_CONTACTS.enabled(self.domain):
XFormsSessionSynchronization.release_channel_for_session(self)
metrics_counter('commcare.smsforms.session_ended', 1, tags={
'domain': self.domain,
'workflow': self.workflow,
'status': (
'success' if self.completed and self.submission_id else
'terminated_partial_submission' if not self.completed and self.submission_id else
'terminated_without_submission' if not self.completed and not self.submission_id else
# Not sure if/how this could ever happen, but worth tracking if it does
'completed_without_submission'
)
})
def test_timeEnd_value_is_set(self):
xml_with_case_action = '''
<data>
<question>answer</question>
<meta>
<timeEnd/>
</meta>
</data>
'''.strip()
now = utcnow()
expected_time_end = json_format_datetime(now)
with patch('corehq.apps.smsforms.app.utcnow', return_value=now):
cleaned_xml = _clean_xml_for_partial_submission(
xml_with_case_action, should_remove_case_actions=True)
xml = XML(cleaned_xml)
self.assertEqual(
xml.find('meta').find('timeEnd').text, expected_time_end)
def _sync_case_for_messaging(domain, case_id):
try:
case = CaseAccessors(domain).get_case(case_id)
sms_tasks.clear_case_caches(case)
except CaseNotFound:
case = None
if case is None or case.is_deleted:
sms_tasks.delete_phone_numbers_for_owners([case_id])
delete_schedule_instances_for_cases(domain, [case_id])
return
if use_phone_entries():
sms_tasks._sync_case_phone_number(case)
rules = AutomaticUpdateRule.by_domain_cached(case.domain, AutomaticUpdateRule.WORKFLOW_SCHEDULING)
rules_by_case_type = AutomaticUpdateRule.organize_rules_by_case_type(rules)
for rule in rules_by_case_type.get(case.type, []):
rule.run_rule(case, utcnow())
def submit_unfinished_form(session):
"""
Gets the raw instance of the session's form and submits it. This is used with
sms and ivr surveys to save all questions answered so far in a session that
needs to close.
If session.include_case_updates_in_partial_submissions is False, no case
create / update / close actions will be performed, but the form will still be submitted.
The form is only submitted if the smsforms session has not yet completed.
"""
# Get and clean the raw xml
try:
xml = get_raw_instance(session.session_id, session.domain)['output']
except InvalidSessionIdException:
return
root = XML(xml)
case_tag_regex = re.compile(r"^(\{.*\}){0,1}case$") # Use regex in order to search regardless of namespace
meta_tag_regex = re.compile(r"^(\{.*\}){0,1}meta$")
timeEnd_tag_regex = re.compile(r"^(\{.*\}){0,1}timeEnd$")
current_timstamp = json_format_datetime(utcnow())
for child in root:
if case_tag_regex.match(child.tag) is not None:
# Found the case tag
case_element = child
case_element.set("date_modified", current_timstamp)
if not session.include_case_updates_in_partial_submissions:
# Remove case actions (create, update, close)
child_elements = [case_action for case_action in case_element]
for case_action in child_elements:
case_element.remove(case_action)
elif meta_tag_regex.match(child.tag) is not None:
# Found the meta tag, now set the value for timeEnd
for meta_child in child:
if timeEnd_tag_regex.match(meta_child.tag):
meta_child.text = current_timstamp
cleaned_xml = tostring(root)
# Submit the xml
result = submit_form_locally(cleaned_xml, session.domain, app_id=session.app_id, partial_submission=True)
session.submission_id = result.xform.form_id
def handle_due_survey_action(domain, contact_id, session_id):
with critical_section_for_smsforms_sessions(contact_id):
session = SQLXFormsSession.by_session_id(session_id)
if (
not session
or not session.session_is_open
or session.current_action_due > utcnow()
):
return
if session_is_stale(session):
# If a session is having some unrecoverable errors that aren't benefitting from
# being retried, those errors should show up in sentry log and the fix should
# be dealt with. In terms of the current session itself, we just close it out
# to allow new sessions to start.
session.mark_completed(False)
session.save()
return
if session.current_action_is_a_reminder:
# Resend the current question in the open survey to the contact
p = PhoneNumber.get_phone_number_for_owner(session.connection_id, session.phone_number)
if p:
metadata = MessageMetadata(
workflow=session.workflow,
xforms_session_couch_id=session._id,
)
resp = current_question(session.session_id, domain)
send_sms_to_verified_number(
p,
resp.event.text_prompt,
metadata,
logged_subevent=session.related_subevent
)
session.move_to_next_action()
session.save()
else:
# Close the session
session.close()
session.save()
def handle_due_survey_action(domain, contact_id, session_id):
with critical_section_for_smsforms_sessions(contact_id):
session = SQLXFormsSession.by_session_id(session_id)
if (
not session
or not session.session_is_open
or session.current_action_due > utcnow()
):
return
if session_is_stale(session):
# If a session is having some unrecoverable errors that aren't benefitting from
# being retried, those errors should show up in sentry log and the fix should
# be dealt with. In terms of the current session itself, we just close it out
# to allow new sessions to start.
session.mark_completed(False)
session.save()
return
if session.current_action_is_a_reminder:
# Resend the current question in the open survey to the contact
p = PhoneNumber.get_phone_number_for_owner(session.connection_id, session.phone_number)
if p:
metadata = MessageMetadata(
workflow=session.workflow,
xforms_session_couch_id=session._id,
)
resp = current_question(session.session_id, domain)
send_sms_to_verified_number(
p,
resp.event.text_prompt,
metadata,
logged_subevent=session.related_subevent
)
session.move_to_next_action()
session.save()
else:
# Close the session
session.close()
session.save()
def set_first_event_due_timestamp(self, instance, start_date=None):
"""
If start_date is None, we set it automatically ensuring that
self.next_event_due does not get set in the past for the first
event.
"""
if start_date:
instance.start_date = start_date
else:
instance.start_date = instance.today_for_recipient
self.set_next_event_due_timestamp(instance)
if (self.schedule_length != self.MONTHLY and not start_date
and instance.next_event_due < util.utcnow()):
if self.start_day_of_week == self.ANY_DAY:
instance.start_date += timedelta(days=1)
instance.next_event_due += timedelta(days=1)
else:
instance.start_date += timedelta(days=7)
instance.next_event_due += timedelta(days=7)
def set_first_event_due_timestamp(self, instance, start_date=None):
"""
If start_date is None, we set it automatically ensuring that
self.next_event_due does not get set in the past for the first
event.
"""
if start_date:
instance.start_date = start_date
else:
instance.start_date = instance.get_today_for_recipient(self)
self.set_next_event_due_timestamp(instance)
# If there was no specific start date for the schedule, we
# start it today. But that can cause us to put the first event
# in the past if it has already passed for the day. So if that
# happens, push the schedule out by 1 day for daily schedules,
# 1 week for weekly schedules, or 1 month for monthly schedules.
if (
not start_date and
instance.next_event_due < util.utcnow()
):
if self.is_monthly:
# Monthly
new_start_date = instance.start_date + relativedelta(months=1)
instance.start_date = date(new_start_date.year, new_start_date.month, 1)
# Current event and schedule iteration might be updated
# in the call to set_next_event_due_timestamp, so reset them
instance.current_event_num = 0
instance.schedule_iteration_num = 1
elif self.start_day_of_week == self.ANY_DAY:
# Daily
instance.start_date += timedelta(days=1)
else:
# Weekly
instance.start_date += timedelta(days=7)
self.set_next_event_due_timestamp(instance)
def set_first_event_due_timestamp(self, instance, start_date=None):
"""
If start_date is None, we set it automatically ensuring that
self.next_event_due does not get set in the past for the first
event.
"""
if start_date:
instance.start_date = start_date
else:
instance.start_date = instance.get_today_for_recipient(self)
self.set_next_event_due_timestamp(instance)
# If there was no specific start date for the schedule, we
# start it today. But that can cause us to put the first event
# in the past if it has already passed for the day. So if that
# happens, push the schedule out by 1 day for daily schedules,
# 1 week for weekly schedules, or 1 month for monthly schedules.
if (
not start_date and
instance.next_event_due < util.utcnow()
):
if self.is_monthly:
# Monthly
new_start_date = instance.start_date + relativedelta(months=1)
instance.start_date = date(new_start_date.year, new_start_date.month, 1)
# Current event and schedule iteration might be updated
# in the call to set_next_event_due_timestamp, so reset them
instance.current_event_num = 0
instance.schedule_iteration_num = 1
elif self.start_day_of_week == self.ANY_DAY:
# Daily
instance.start_date += timedelta(days=1)
else:
# Weekly
instance.start_date += timedelta(days=7)
self.set_next_event_due_timestamp(instance)
def get_today_for_recipient(self, schedule):
return ServerTime(util.utcnow()).user_time(self.get_timezone(schedule)).done().date()
def is_stale(self):
return (util.utcnow() - self.next_event_due) > timedelta(minutes=STALE_SCHEDULE_INSTANCE_INTERVAL)
def mark_completed(self, completed):
self.session_is_open = False
self.completed = completed
self.modified_time = self.end_time = utcnow()
def run_auto_update_rules_for_case(case):
rules = AutomaticUpdateRule.by_domain_cached(
case.domain, AutomaticUpdateRule.WORKFLOW_SCHEDULING)
rules_by_case_type = AutomaticUpdateRule.organize_rules_by_case_type(rules)
for rule in rules_by_case_type.get(case.type, []):
rule.run_rule(case, utcnow())
rules = AutomaticUpdateRule.by_domain_cached(
domain, AutomaticUpdateRule.WORKFLOW_SCHEDULING)
rules = [rule for rule in rules if rule.pk == rule_id]
return rules[0] if len(rules) == 1 else None
def _sync_case_for_messaging_rule(domain, case_id, rule_id):
case_load_counter("messaging_rule_sync", domain)()
try:
case = CaseAccessors(domain).get_case(case_id)
except CaseNotFound:
clear_messaging_for_case(domain, case_id)
return
rule = _get_cached_rule(domain, rule_id)
if rule:
rule.run_rule(case, utcnow())
MessagingRuleProgressHelper(rule_id).increment_current_case_count()
def initiate_messaging_rule_run(rule):
if not rule.active:
return
AutomaticUpdateRule.objects.filter(pk=rule.pk).update(
locked_for_editing=True)
transaction.on_commit(
lambda: run_messaging_rule.delay(rule.domain, rule.pk))
def paginated_case_ids(domain, case_type):
row_generator = paginate_query_across_partitioned_databases(
CommCareCaseSQL,
def move_to_next_action(self):
while self.current_action_is_a_reminder and self.current_action_due < utcnow():
self.current_reminder_num += 1
self.set_current_action_due_timestamp()
def set_first_event_due_timestamp(self, instance, start_date=None):
instance.next_event_due = util.utcnow()
self.set_next_event_due_timestamp(instance)
def mark_completed(self, completed):
self.session_is_open = False
self.completed = completed
self.modified_time = self.end_time = utcnow()
def session_is_stale(session):
return utcnow() > (session.start_time + timedelta(
minutes=SQLXFormsSession.MAX_SESSION_LENGTH * 2))
def today_for_recipient(self):
return ServerTime(util.utcnow()).user_time(self.timezone).done().date()
def set_first_event_due_timestamp(self, instance, start_date=None):
instance.next_event_due = util.utcnow()
self.set_next_event_due_timestamp(instance)
def move_to_next_event_not_in_the_past(self, instance):
while instance.active and instance.next_event_due < util.utcnow():
self.move_to_next_event(instance)
def move_to_next_action(self):
while self.current_action_is_a_reminder and self.current_action_due < utcnow():
self.current_reminder_num += 1
self.set_current_action_due_timestamp()
def session_is_stale(session):
return utcnow() > (session.start_time + timedelta(minutes=SQLXFormsSession.MAX_SESSION_LENGTH * 2))