def _sms_count(user, startdate, enddate, message_type="SMSLog"):
"""
Returns a dictionary of messages seen for a given type, user, and date
range of the format:
{
I: inbound_count,
O: outbound_count
}
"""
# utilizable if we want to stick it somewhere else
start_timestamp = json_format_datetime(startdate)
end_timestamp = json_format_datetime(enddate)
ret = {}
for direction in [INCOMING, OUTGOING]:
results = (
SMSLog.get_db()
.view(
"sms/by_recipient",
startkey=[user.doc_type, user._id, message_type, direction, start_timestamp],
endkey=[user.doc_type, user._id, message_type, direction, end_timestamp],
reduce=True,
)
.all()
)
ret[direction] = results[0]["value"] if results else 0
return ret
def get_first_survey_response(self, case, dt):
timestamp_start = datetime.combine(dt, time(20, 45))
timestamp_start = tz_utils.adjust_datetime_to_timezone(
timestamp_start, self.domain_obj.default_timezone, pytz.utc.zone)
timestamp_start = timestamp_start.replace(tzinfo=None)
timestamp_start = json_format_datetime(timestamp_start)
timestamp_end = datetime.combine(dt + timedelta(days=1), time(11, 0))
timestamp_end = tz_utils.adjust_datetime_to_timezone(
timestamp_end, self.domain_obj.default_timezone, pytz.utc.zone)
timestamp_end = timestamp_end.replace(tzinfo=None)
if timestamp_end > datetime.utcnow():
return RESPONSE_NOT_APPLICABLE
timestamp_end = json_format_datetime(timestamp_end)
all_inbound = FRISMSLog.view(
"sms/by_recipient",
startkey=["CommCareCase", case._id, "SMSLog", INCOMING, timestamp_start],
endkey=["CommCareCase", case._id, "SMSLog", INCOMING, timestamp_end],
reduce=False,
include_docs=True
).all()
survey_responses = filter(lambda s: s.xforms_session_couch_id is not None, all_inbound)
if len(survey_responses) > 0:
return survey_responses[0]
else:
return NO_RESPONSE
def get_number_cases(self, user_id, modified_after=None, modified_before=None, closed=None):
key = [self.domain, {} if closed is None else closed, self.case_type or {}, user_id]
if modified_after is None:
start = ""
else:
start = json_format_datetime(modified_after)
if modified_before is None:
end = {}
else:
end = json_format_datetime(modified_before)
return (
get_db()
.view(
"case/by_date_modified",
startkey=key + [start],
endkey=key + [end],
group=True,
group_level=0,
wrapper=lambda row: row["value"],
)
.one()
or 0
)
def test_ota_consumption(self):
self.ct_settings.consumption_config = ConsumptionConfig(min_transactions=0, min_window=0, optimal_window=60)
self.ct_settings.ota_restore_config = StockRestoreConfig(section_to_consumption_types={"stock": "consumption"})
set_default_monthly_consumption_for_domain(self.domain.name, 5 * DAYS_IN_MONTH)
self._save_settings_and_clear_cache()
amounts = [(p._id, i * 10) for i, p in enumerate(self.products)]
report = _report_soh(amounts, self.sp.case_id, "stock")
balance_blocks = _get_ota_balance_blocks(self.domain, self.user)
self.assertEqual(2, len(balance_blocks))
stock_block, consumption_block = balance_blocks
check_xml_line_by_line(
self,
balance_ota_block(self.sp, "stock", amounts, datestring=json_format_datetime(report.date)),
stock_block,
)
check_xml_line_by_line(
self,
balance_ota_block(
self.sp,
"consumption",
[(p._id, 150) for p in self.products],
datestring=json_format_datetime(report.date),
),
consumption_block,
)
def _cacheable_domain_activity_report(request):
landmarks = json.loads(request.GET.get('landmarks') or "[7, 30, 90]")
landmarks.sort()
now = datetime.utcnow()
dates = []
for landmark in landmarks:
dates.append(now - timedelta(days=landmark))
domains = [{'name': domain.name, 'display_name': domain.display_name()} for domain in Domain.get_all()]
for domain in domains:
domain['users'] = dict([(user.user_id, {'raw_username': user.raw_username}) for user in CommCareUser.by_domain(domain['name'])])
if not domain['users']:
continue
key = make_form_couch_key(domain['name'])
forms = [r['value'] for r in get_db().view('reports_forms/all_forms',
reduce=False,
startkey=key+[json_format_datetime(dates[-1])],
endkey=key+[json_format_datetime(now)],
).all()]
domain['user_sets'] = [dict() for landmark in landmarks]
for form in forms:
user_id = form.get('user_id')
try:
time = string_to_datetime(form['submission_time']).replace(tzinfo = None)
except ValueError:
continue
if user_id in domain['users']:
for i, date in enumerate(dates):
if time > date:
domain['user_sets'][i][user_id] = domain['users'][user_id]
return HttpResponse(json.dumps({'domains': domains, 'landmarks': landmarks}))
def get_pre_migration_copy(app):
from corehq.apps.app_manager.util import get_correct_app_class
def date_key(doc):
return doc.get("built_on") or mindate
mindate = json_format_datetime(datetime(1980, 1, 1))
migrate_date = json_format_datetime(ORIGINAL_MIGRATION_DATE)
skip = 0
docs = None
while docs is None or date_key(docs[-1]) > migrate_date:
docs = saved_apps = [row['doc'] for row in Application.get_db().view(
'app_manager/saved_app',
startkey=[app.domain, app._id, {}],
endkey=[app.domain, app._id],
descending=True,
skip=skip,
limit=5,
include_docs=True,
)]
if not docs:
break
skip += len(docs)
docs = sorted(saved_apps, key=date_key, reverse=True)
for doc in docs:
if date_key(doc) < migrate_date:
copy = get_correct_app_class(doc).wrap(doc)
if copy.version < app.version:
return copy
return None
def process_view(self, request, view_func, view_args, view_kwargs):
if not request.user.is_authenticated():
return
secure_session = request.session.get('secure_session')
domain = getattr(request, "domain", None)
now = datetime.datetime.utcnow()
if not secure_session and (
(domain and Domain.is_secure_session_required(domain)) or
self._user_requires_secure_session(request.couch_user)):
if self._session_expired(settings.SECURE_TIMEOUT, request.user.last_login, now):
django_logout(request, template_name=settings.BASE_TEMPLATE)
# this must be after logout so it is attached to the new session
request.session['secure_session'] = True
return HttpResponseRedirect(reverse('login') + '?next=' + request.path)
else:
request.session['secure_session'] = True
request.session['last_request'] = json_format_datetime(now)
return
else:
last_request = request.session.get('last_request')
timeout = settings.SECURE_TIMEOUT if secure_session else settings.INACTIVITY_TIMEOUT
if self._session_expired(timeout, last_request, now):
django_logout(request, template_name=settings.BASE_TEMPLATE)
return HttpResponseRedirect(reverse('login') + '?next=' + request.path)
request.session['last_request'] = json_format_datetime(now)
def adjust_datetimes(data, parent=None, key=None):
"""
find all datetime-like strings within data (deserialized json)
and format them uniformly, in place.
"""
# this strips the timezone like we've always done
# todo: in the future this will convert to UTC
if isinstance(data, basestring):
if re_loose_datetime.match(data):
if phone_timezones_should_be_processed():
parent[key] = json_format_datetime(
iso8601.parse_date(data).astimezone(pytz.utc)
.replace(tzinfo=None)
)
else:
parent[key] = json_format_datetime(
iso8601.parse_date(data).replace(tzinfo=None))
elif isinstance(data, dict):
for key, value in data.items():
adjust_datetimes(value, parent=data, key=key)
elif isinstance(data, list):
for i, value in enumerate(data):
adjust_datetimes(value, parent=data, key=i)
# return data, just for convenience in testing
# this is the original input, modified, not a new data structure
return data
def is_app_active(app_id, domain):
now = datetime.utcnow()
then = json_format_datetime(now - timedelta(days=30))
now = json_format_datetime(now)
key = ['submission app', domain, app_id]
row = get_db().view("reports_forms/all_forms", startkey=key+[then], endkey=key+[now]).all()
return True if row else False
def get_list(cls, environment, startdate, enddate, limit=50):
return HqDeploy.view(
'hqadmin/deploy_history',
startkey=[environment, json_format_datetime(startdate)],
endkey=[environment, json_format_datetime(enddate)],
reduce=False,
limit=limit,
include_docs=False
).all()
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = FRISMSLog.view("sms/by_domain",
startkey=[self.domain, "SMSLog", startdate],
endkey=[self.domain, "SMSLog", enddate],
include_docs=True,
reduce=False).all()
result = []
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
message_bank_messages = get_message_bank(self.domain, for_comparing=True)
FormProcessorInterface(self.domain).casedb_cache(
domain=self.domain, strip_history=False, deleted_ok=True
)
user_cache = UserCache()
show_only_survey_traffic = self.show_only_survey_traffic()
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
if show_only_survey_traffic and message.xforms_session_couch_id is None:
continue
# Add metadata from the message bank if it has not been added already
if (message.direction == OUTGOING) and (not message.fri_message_bank_lookup_completed):
add_metadata(message, message_bank_messages)
if message.couch_recipient_doc_type == "CommCareCase":
recipient = case_cache.get(message.couch_recipient)
else:
recipient = user_cache.get(message.couch_recipient)
if message.chat_user_id:
sender = user_cache.get(message.chat_user_id)
else:
sender = None
study_arm = None
if message.couch_recipient_doc_type == "CommCareCase":
study_arm = case_cache.get(message.couch_recipient).get_case_property("study_arm")
timestamp = ServerTime(message.date).user_time(self.timezone).done()
result.append([
self._fmt(self._participant_id(recipient)),
self._fmt(study_arm or "-"),
self._fmt(self._originator(message, recipient, sender)),
self._fmt_timestamp(timestamp),
self._fmt(message.text),
self._fmt(message.fri_id or "-"),
self._fmt(direction_map.get(message.direction,"-")),
])
return result
def _formatXForm(self, doc_id, raw_xml, attachment_block):
final_xml = raw_xml % ({
"attachments": attachment_block,
"time_start": json_format_datetime(datetime.utcnow()
- timedelta(minutes=4)),
"time_end": json_format_datetime(datetime.utcnow()),
"date_modified": json_format_datetime(datetime.utcnow()),
"doc_id": doc_id
})
return final_xml
def _formatXForm(self, doc_id, raw_xml, attachment_block, date=None):
if date is None:
date = datetime.utcnow()
final_xml = Template(raw_xml).render(Context({
"attachments": attachment_block,
"time_start": json_format_datetime(date - timedelta(minutes=4)),
"time_end": json_format_datetime(date),
"date_modified": json_format_datetime(date),
"doc_id": doc_id
}))
return final_xml
def update_task(domain, subcase_guid, submitting_user_id, task_owner_id, form_unique_id, task_activation_datetime, task_deactivation_datetime, incentive):
context = {
"subcase_guid" : subcase_guid,
"user_id" : submitting_user_id,
"date_modified" : json_format_datetime(datetime.datetime.utcnow()),
"task_owner_id" : task_owner_id,
"form_unique_id" : form_unique_id,
"task_activation_date" : json_format_datetime(task_activation_datetime),
"task_deactivation_date" : json_format_datetime(task_deactivation_datetime),
"incentive" : incentive,
}
submit_xml(domain, "sms/xml/update_task.xml", context)
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = SMSLog.by_domain_date(self.domain, startdate, enddate)
result = []
username_map = {} # Store the results of username lookups for faster loading
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
# Retrieve message log options
message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {})
abbreviated_phone_number_domains = message_log_options.get("abbreviated_phone_number_domains", [])
abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains)
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
recipient_id = message.couch_recipient
if recipient_id in [None, ""]:
username = "******"
elif recipient_id in username_map:
username = username_map.get(recipient_id)
else:
username = "******"
try:
if message.couch_recipient_doc_type == "CommCareCase":
username = CommCareCase.get(recipient_id).name
else:
username = CouchUser.get_by_user_id(recipient_id).username
except Exception:
pass
username_map[recipient_id] = username
phone_number = message.phone_number
if abbreviate_phone_number and phone_number is not None:
phone_number = phone_number[0:7] if phone_number[0:1] == "+" else phone_number[0:6]
timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.timezone.zone)
result.append([
self._fmt_timestamp(timestamp),
self._fmt(username),
self._fmt(phone_number),
self._fmt(direction_map.get(message.direction,"-")),
self._fmt(message.text),
])
return result
def _household_verification_json(
domain="dodoma",
last_hvid_path=["household_verification"],
next_hvid_path=["followup_id"],
xmlns='http://openrosa.org/formdesigner/9DAACA82-A414-499A-9C40-BC43775CEE79',
range=None
):
if range:
start, end = map(string_to_datetime, range)
else:
now = datetime.utcnow()
start, end = now - timedelta(days=7), now
key = make_form_couch_key(domain, xmlns=xmlns)
submissions = XFormInstance.view('reports_forms/all_forms',
reduce=False,
startkey=key+[json_format_datetime(start)],
endkey=key+[json_format_datetime(end)],
include_docs=True,
)
stats = get_household_verification_data(
submissions=submissions,
next_hvid_path=next_hvid_path,
last_hvid_path=last_hvid_path,
)
stats_by_userID = {}
for s in stats:
stats_by_userID[s['userID']] = s
s['username'] = "******" % s['userID']
users = CommCareUser.by_domain(domain)
for user in users:
userID = user.user_id
username = user_id_to_username(userID)
if userID in stats_by_userID:
stats_by_userID[userID]['username'] = username
else:
stats.append({'userID': userID, 'username': username, 'total': 0, 'correct': 0})
stats.sort(key=lambda s: s['username'])
return {
"headers": ["Username", "Correct", "Total", "Percent Correct"],
"rows": [[
s['username'],
s['correct'],
s['total'],
("%s%%" % int(s['correct']*100/s['total']) if s['total'] else "---")
] for s in stats],
}
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = SMSLog.by_domain_date(self.domain, startdate, enddate)
result = []
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
# Retrieve message log options
message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {})
abbreviated_phone_number_domains = message_log_options.get("abbreviated_phone_number_domains", [])
abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains)
contact_cache = {}
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
recipient_id = message.couch_recipient
doc = None
if recipient_id not in [None, ""]:
try:
if message.couch_recipient_doc_type == "CommCareCase":
doc = CommCareCase.get(recipient_id)
else:
doc = CouchUser.get_by_user_id(recipient_id)
except Exception:
pass
if doc:
doc_info = get_doc_info(doc.to_json(), self.domain,
contact_cache)
else:
doc_info = None
phone_number = message.phone_number
if abbreviate_phone_number and phone_number is not None:
phone_number = phone_number[0:7] if phone_number[0:1] == "+" else phone_number[0:6]
timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.timezone.zone)
result.append([
self._fmt_timestamp(timestamp),
self._fmt_contact_link(message, doc_info),
self._fmt(phone_number),
self._fmt(direction_map.get(message.direction,"-")),
self._fmt(message.text),
])
return result
def active(domain, *args):
now = datetime.utcnow()
then = json_format_datetime(now - timedelta(days=30))
now = json_format_datetime(now)
key = ['submission', domain]
row = get_db().view(
"reports_forms/all_forms",
startkey=key+[then],
endkey=key+[now],
limit=1
).all()
return True if row else False
def app_has_been_submitted_to_in_last_30_days(domain, app_id):
now = datetime.datetime.utcnow()
_30_days = datetime.timedelta(days=30)
then = json_format_datetime(now - _30_days)
now = json_format_datetime(now)
key = ["submission app", domain, app_id]
row = (
XFormInstance.get_db()
.view("reports_forms/all_forms", startkey=key + [then], endkey=key + [now], limit=1, stale=stale_ok())
.all()
)
return True if row else False
def mk_date_range(start=None, end=None, ago=timedelta(days=7), iso=False):
if isinstance(end, basestring):
end = parse_date(end)
if isinstance(start, basestring):
start = parse_date(start)
if not end:
end = datetime.utcnow()
if not start:
start = end - ago
if iso:
return json_format_datetime(start), json_format_datetime(end)
else:
return start, end
def get_reports(cls, domain, location=None, datespan=None):
start = datespan.startdate if datespan else datetime(1900, 1, 1)
end = datespan.end_of_end_day if datespan else datetime.max
timestamp_start = dateparse.json_format_datetime(start)
timestamp_end = dateparse.json_format_datetime(end)
loc_id = location._id if location else None
startkey = [domain, loc_id, timestamp_start]
endkey = [domain, loc_id, timestamp_end]
return [StockReport(f) for f in \
XFormInstance.view('commtrack/stock_reports',
startkey=startkey,
endkey=endkey,
include_docs=True)]
def test_xml(self):
now = datetime.utcnow()
domain = "test"
now_plus_30 = now + timedelta(days=30)
now_minus_30 = now - timedelta(days=30)
record = new_key_record(None, None, now=now)
xml = get_mobile_auth_payload([record], domain, now=now)
check_xml_line_by_line(
self,
xml,
"""
<OpenRosaResponse xmlns="http://openrosa.org/http/response">
<message nature="submit_success">Here are your keys!</message>
<auth_keys domain="{domain}" issued="{now}">
<key_record valid="{now}" expires="{now_plus_30}">
<uuid>{record.uuid}</uuid>
<key type="{record.type}">{record.key}</key>
</key_record>
</auth_keys>
</OpenRosaResponse>
""".format(
now=json_format_datetime(now),
now_plus_30=json_format_datetime(now_plus_30),
record=record,
domain=domain,
),
)
record = new_key_record(None, None, now=now, valid=now_minus_30)
xml = get_mobile_auth_payload([record], domain, now=now)
check_xml_line_by_line(
self,
xml,
"""
<OpenRosaResponse xmlns="http://openrosa.org/http/response">
<message nature="submit_success">Here are your keys!</message>
<auth_keys domain="{domain}" issued="{now}">
<key_record valid="{now_minus_30}" expires="{now_plus_30}">
<uuid>{record.uuid}</uuid>
<key type="{record.type}">{record.key}</key>
</key_record>
</auth_keys>
</OpenRosaResponse>
""".format(
now=json_format_datetime(now),
now_plus_30=json_format_datetime(now_plus_30),
now_minus_30=json_format_datetime(now_minus_30),
record=record,
domain=domain,
),
)
def get_number_of_submissions(domain, user_id, xmlns, app_id, start, end,
by_submission_time=True):
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(domain, user_id=user_id, xmlns=xmlns,
by_submission_time=by_submission_time,
app_id=app_id)
data = XFormInstance.get_db().view(
'reports_forms/all_forms',
reduce=True,
startkey=key + [json_format_datetime(start)],
endkey=key + [json_format_datetime(end)],
stale=stale_ok(),
).first()
return data['value'] if data else 0
def get_case_ids_modified_with_owner_since(domain, owner_id, reference_date, until_date=None):
"""
Gets all cases with a specified owner ID that have been modified
since a particular reference_date (using the server's timestamp)
"""
return [
row['id'] for row in CommCareCase.get_db().view(
'cases_by_server_date/by_owner_server_modified_on',
startkey=[domain, owner_id, json_format_datetime(reference_date)],
endkey=[domain, owner_id, {} if not until_date else json_format_datetime(until_date)],
include_docs=False,
reduce=False
)
]
def _report_soh(soh_reports, case_id, domain):
report_date = json_format_datetime(datetime.utcnow())
balance_blocks = [
get_single_balance_block(
case_id,
report.product_id,
report.amount,
report_date,
section_id=report.section_id
)
for report in soh_reports
]
form = submit_case_blocks(balance_blocks, domain)
return json_format_datetime(FormAccessors(domain).get_form(form.form_id).received_on)
def cases_in_last(domain, days):
"""
Returns the number of open cases that have been modified in the last <days> days
"""
now = datetime.utcnow()
then = json_format_datetime(now - timedelta(days=int(days)))
now = json_format_datetime(now)
q = {"query": {
"range": {
"modified_on": {
"from": then,
"to": now}}}}
data = es_query(params={"domain.exact": domain, 'closed': False}, q=q, es_url=CASE_INDEX + '/case/_search', size=1)
return data['hits']['total'] if data.get('hits') else 0
def notify_sentry_deploy(duration_mins):
from settingshelper import get_release_name
headers = {'Authorization': 'Bearer {}'.format(settings.SENTRY_API_KEY), }
payload = {
'environment': settings.SERVER_ENVIRONMENT,
}
if duration_mins:
utcnow = datetime.utcnow()
payload.update({
'dateStarted': json_format_datetime(utcnow - timedelta(minutes=duration_mins)),
'dateFinished': json_format_datetime(utcnow),
})
version = get_release_name(settings.BASE_DIR, settings.SERVER_ENVIRONMENT)
releases_url = 'https://sentry.io/api/0/organizations/dimagi/releases/{}/deploys/'.format(version)
requests.post(releases_url, headers=headers, json=payload)
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = FRISMSLog.view("sms/by_domain",
startkey=[self.domain, "SMSLog", startdate],
endkey=[self.domain, "SMSLog", enddate],
include_docs=True,
reduce=False).all()
result = []
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
message_bank_messages = get_message_bank(self.domain, for_comparing=True)
case_cache = CaseDbCache(domain=self.domain, strip_history=False, deleted_ok=True)
user_cache = UserCache()
for message in data:
# Add metadata from the message bank if it has not been added already
if (message.direction == OUTGOING) and (not message.fri_message_bank_lookup_completed):
add_metadata(message, message_bank_messages)
if message.couch_recipient_doc_type == "CommCareCase":
recipient = case_cache.get(message.couch_recipient)
else:
recipient = user_cache.get(message.couch_recipient)
if message.chat_user_id:
sender = user_cache.get(message.chat_user_id)
else:
sender = None
study_arm = None
if message.couch_recipient_doc_type == "CommCareCase":
study_arm = case_cache.get(message.couch_recipient).get_case_property("study_arm")
timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.timezone.zone)
result.append([
self._fmt(self._participant_id(recipient)),
self._fmt(study_arm or "-"),
self._fmt(self._originator(message, recipient, sender)),
self._fmt_timestamp(timestamp),
self._fmt(message.text),
self._fmt(message.fri_id or "-"),
self._fmt(direction_map.get(message.direction,"-")),
])
return result
def get_sample_doc_and_indicators(fake_time_now=None, owner_id='some-user-id'):
if fake_time_now is None:
fake_time_now = datetime.utcnow()
date_opened = datetime(2014, 6, 21)
sample_doc = dict(
_id=uuid.uuid4().hex,
opened_on=json_format_datetime(date_opened),
owner_id=owner_id,
doc_type="CommCareCase",
domain='user-reports',
name='sample name',
type='ticket',
category='bug',
tags='easy-win public',
is_starred='yes',
estimate=2.3,
priority=4,
)
expected_indicators = {
'doc_id': sample_doc['_id'],
'repeat_iteration': 0,
'date': date_opened,
'owner': owner_id,
'count': 1,
'category_bug': 1, 'category_feature': 0, 'category_app': 0, 'category_schedule': 0,
'tags_easy-win': 1, 'tags_potential-dupe': 0, 'tags_roadmap': 0, 'tags_public': 1,
'is_starred': 1,
'estimate': Decimal(2.3),
'priority': 4,
'inserted_at': fake_time_now,
}
return sample_doc, expected_indicators
def to_instance(data):
"""convert the parsed sms stock report into an instance like what would be
submitted from a commcare phone"""
E = XML()
M = XML(const.META_XMLNS, 'jrm')
deviceID = get_device_id(data)
timestamp = json_format_datetime(data['timestamp'])
transactions = data['transactions']
category = set(tx.category for tx in transactions).pop()
stock_blocks = convert_transactions_to_blocks(E, transactions)
if category == 'stock':
root = E.stock_report(
M.meta(
M.userID(data['user']._id),
M.deviceID(deviceID),
M.timeStart(timestamp),
M.timeEnd(timestamp)
),
E.location(data['location']._id),
*stock_blocks
)
return etree.tostring(root, encoding='utf-8', pretty_print=True)
else:
return requisition_case_xml(data, stock_blocks)
def get_single_transfer_block(src_id, dest_id, product_id, quantity, date_string=None, section_id='stock'):
date_string = date_string or json_format_datetime(datetime.utcnow())
return """
<transfer xmlns="http://commcarehq.org/ledger/v1" {src} {dest} date="{date}" section-id="{section_id}">
<entry id="{product_id}" quantity="{quantity}" />
</transfer >""".format(
src='src="{}"'.format(src_id) if src_id is not None else '',
dest='dest="{}"'.format(dest_id) if dest_id is not None else '',
product_id=product_id, quantity=quantity,
date=date_string, section_id=section_id,
).strip()
def transform_case_for_elasticsearch(doc_dict):
system_properties = ['case_properties', '_indexed_on']
doc = {
desired_property: doc_dict.get(desired_property)
for desired_property in CASE_SEARCH_MAPPING['properties'].keys()
if desired_property not in system_properties
}
doc['_id'] = doc_dict.get('_id')
doc['_indexed_on'] = json_format_datetime(datetime.utcnow())
doc['case_properties'] = _get_case_properties(doc_dict)
return doc
def get_doc_count_in_domain_by_type(domain,
doc_type,
db,
start_date=None,
end_date=None):
start_key = [domain, doc_type]
end_key = [domain, doc_type]
if start_date is not None:
start_key.append(json_format_datetime(start_date))
if end_date is not None:
end_key.append(json_format_datetime(end_date))
end_key.append({})
row = db.view(
"by_domain_doc_type_date/view",
startkey=start_key,
endkey=end_key,
reduce=True,
).one()
return row["value"] if row else 0
def get_num_forms_since(self, time):
if not hasattr(self, 'domain'):
self.domain = self.couch_user.domain if self.couch_user else None
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(self.domain, user_id=self.userID)
r = get_db().view('reports_forms/all_forms',
startkey=key + [json_format_datetime(time)],
endkey=key + [{}],
group=False).one()
return r['value'] if r else 0
def test_payload(self):
form = self.post_xml(self.xform_xml, self.domain_name).xform
repeat_records = self.repeat_records(self.domain_name).all()
payload = repeat_records[0].get_payload()
cases = cases_referenced_by_xform(form)
self.assertEqual(
json.loads(payload), {
'received_on': json_format_datetime(form.received_on),
'form_id': form.form_id,
'case_ids': [case.case_id for case in cases]
})
def _soft_delete(db, doc_ids, deletion_date=None, deletion_id=None):
from dimagi.utils.couch.undo import DELETED_SUFFIX
deletion_date = json_format_datetime(deletion_date or datetime.utcnow())
def delete(doc):
doc['doc_type'] += DELETED_SUFFIX
doc['-deletion_id'] = deletion_id
doc['-deletion_date'] = deletion_date
return doc
return _operate_on_docs(db, doc_ids, delete)
def fmt(value):
if value is None:
return ''
if isinstance(value, datetime):
return six.text_type(json_format_datetime(value))
elif isinstance(value, six.string_types + (numbers.Number, date)):
return six.text_type(value)
else:
raise CaseBlockError(
"Can't transform to XML: {}; unexpected type {}.".format(
value, type(value)))
def key_for_time(cls, domain, user_id, now):
now_json = json_format_datetime(now)
key_record = cls.view(
'mobile_auth/key_records',
startkey=[domain, user_id, now_json],
endkey=[domain, user_id, ""],
descending=True,
limit=1,
include_docs=True,
).first()
return key_record
def notify_sentry_deploy(duration_mins):
from settingshelper import get_release_name
headers = {
'Authorization': 'Bearer {}'.format(settings.SENTRY_API_KEY),
}
payload = {
'environment': settings.SERVER_ENVIRONMENT,
}
if duration_mins:
utcnow = datetime.utcnow()
payload.update({
'dateStarted':
json_format_datetime(utcnow - timedelta(minutes=duration_mins)),
'dateFinished':
json_format_datetime(utcnow),
})
version = get_release_name(settings.BASE_DIR, settings.SERVER_ENVIRONMENT)
releases_url = 'https://sentry.io/api/0/organizations/dimagi/releases/{}/deploys/'.format(
version)
requests.post(releases_url, headers=headers, json=payload)
def notify_event(domain, couch_user, app_id, form_unique_id, message):
message_obj = RedisMessage(json.dumps({
'domain': domain,
'user_id': couch_user._id,
'username': couch_user.username,
'text': message,
'timestamp': json_format_datetime(datetime.datetime.utcnow()),
}))
RedisPublisher(
facility=get_facility_for_form(domain, app_id, form_unique_id), broadcast=True
).publish_message(message_obj)
def get_synclog_ids_by_date(start_datetime, end_datetime):
'''
Returns all synclog ids that have been modified within a time range. The start date is
exclusive while the end date is inclusive (start_datetime, end_datetime].
'''
from casexml.apps.phone.models import SyncLog
json_start_datetime = json_format_datetime(start_datetime)
results = SyncLog.view("sync_logs_by_date/view",
startkey=[json_start_datetime],
endkey=[json_format_datetime(end_datetime)],
reduce=False,
include_docs=False)
for result in results:
result_modified_datetime = result['key'][0]
# Skip the record if the datetime is equal to the start because this should return
# records with an exclusive start date.
if result_modified_datetime == json_start_datetime:
continue
yield result['id']
def all(cls, domain=None, due_before=None, limit=None):
json_now = json_format_datetime(due_before or datetime.utcnow())
repeat_records = RepeatRecord.view(
"repeaters/repeat_records_by_next_check",
startkey=[domain],
endkey=[domain, json_now, {}],
include_docs=True,
reduce=False,
limit=limit,
)
return repeat_records
def get_overdue_repeat_record_count(
overdue_threshold=datetime.timedelta(minutes=10)):
from .models import RepeatRecord
overdue_datetime = datetime.datetime.utcnow() - overdue_threshold
results = RepeatRecord.view(
"receiverwrapper/repeat_records_by_next_check",
startkey=[None],
endkey=[None, json_format_datetime(overdue_datetime)],
reduce=True,
).one()
return results['value'] if results else 0
def get_case_ids_modified_with_owner_since(domain,
owner_id,
reference_date,
until_date=None):
"""
Gets all cases with a specified owner ID that have been modified
since a particular reference_date (using the server's timestamp)
"""
return [
row['id'] for row in CommCareCase.get_db().view(
'cases_by_server_date/by_owner_server_modified_on',
startkey=[domain, owner_id,
json_format_datetime(reference_date)],
endkey=[
domain, owner_id,
{} if not until_date else json_format_datetime(until_date)
],
include_docs=False,
reduce=False)
]
def _get_form_metadata_context(domain, form, timezone, support_enabled=False):
meta = _top_level_tags(form).get('meta', None) or {}
meta['received_on'] = json_format_datetime(form.received_on)
meta['server_modified_on'] = json_format_datetime(form.server_modified_on) if form.server_modified_on else ''
if support_enabled:
meta['last_sync_token'] = form.last_sync_token
definition = get_default_definition(sorted_form_metadata_keys(list(meta)))
form_meta_data = get_tables_as_columns(meta, definition, timezone=timezone)
if getattr(form, 'auth_context', None):
auth_context = AuthContext(form.auth_context)
auth_context_user_id = auth_context.user_id
auth_user_info = get_doc_info_by_id(domain, auth_context_user_id)
else:
auth_user_info = get_doc_info_by_id(domain, None)
auth_context = AuthContext(
user_id=None,
authenticated=False,
domain=domain,
)
meta_userID = meta.get('userID')
meta_username = meta.get('username')
if meta_userID == 'demo_user':
user_info = DocInfo(
domain=domain,
display='demo_user',
)
elif meta_username == 'admin':
user_info = DocInfo(
domain=domain,
display='admin',
)
else:
user_info = get_doc_info_by_id(domain, meta_userID)
return {
"form_meta_data": form_meta_data,
"auth_context": auth_context,
"auth_user_info": auth_user_info,
"user_info": user_info,
}
def enqueue_directly(self, sms):
"""
This method is used to try to send a QueuedSMS entry directly to the
celery queue, without waiting for it to be enqueued by the handle()
thread.
"""
try:
self.enqueue(sms.pk, json_format_datetime(sms.datetime_to_process))
except:
# If anything goes wrong here, no problem, the handle() thread will
# pick it up later and enqueue.
pass
def enqueue_directly(self, reminder):
"""
This method is used to try to send a reminder directly to the
celery queue, without waiting for it to be enqueued by the handle()
thread.
"""
try:
self.enqueue(reminder._id, json_format_datetime(reminder.next_fire))
except:
# If anything goes wrong here, no problem, the handle() thread will
# pick it up later and enqueue.
pass
def update_contact(domain, case_id, user_id, contact_phone_number=None, contact_phone_number_is_verified=None, contact_backend_id=None, language_code=None, time_zone=None):
context = {
"case_id" : case_id,
"date_modified" : json_format_datetime(datetime.datetime.utcnow()),
"user_id" : user_id,
"contact_phone_number" : contact_phone_number,
"contact_phone_number_is_verified" : contact_phone_number_is_verified,
"contact_backend_id" : contact_backend_id,
"language_code" : language_code,
"time_zone" : time_zone
}
submit_xml(domain, "sms/xml/update_contact.xml", context)
def adjust_datetimes(data, parent=None, key=None):
"""
find all datetime-like strings within data (deserialized json)
and format them uniformly, in place.
this only processes timezones correctly if the call comes from a request with domain information
otherwise it will default to not processing timezones.
to force timezone processing, it can be called as follows
>>> from corehq.apps.tzmigration.api import force_phone_timezones_should_be_processed
>>> with force_phone_timezones_should_be_processed():
>>> adjust_datetimes(form_json)
"""
# this strips the timezone like we've always done
# todo: in the future this will convert to UTC
if isinstance(data, basestring) and jsonobject.re_loose_datetime.match(data):
try:
matching_datetime = iso8601.parse_date(data)
except iso8601.ParseError:
pass
else:
if phone_timezones_should_be_processed():
parent[key] = unicode(json_format_datetime(
matching_datetime.astimezone(pytz.utc).replace(tzinfo=None)
))
else:
parent[key] = unicode(json_format_datetime(
matching_datetime.replace(tzinfo=None)))
elif isinstance(data, dict):
for key, value in data.items():
adjust_datetimes(value, parent=data, key=key)
elif isinstance(data, list):
for i, value in enumerate(data):
adjust_datetimes(value, parent=data, key=i)
# return data, just for convenience in testing
# this is the original input, modified, not a new data structure
return data
def submit_unfinished_form(session):
"""
Gets the raw instance of the session's form and submits it. This is used with
sms and ivr surveys to save all questions answered so far in a session that
needs to close.
If session.include_case_updates_in_partial_submissions is False, no case
create / update / close actions will be performed, but the form will still be submitted.
The form is only submitted if the smsforms session has not yet completed.
"""
# Get and clean the raw xml
try:
response = FormplayerInterface(session.session_id,
session.domain).get_raw_instance()
# Formplayer's ExceptionResponseBean includes the exception message,
# stautus ("error"), url, and type ("text")
if response.get('status') == 'error':
raise TouchformsError(response.get('exception'))
xml = response['output']
except InvalidSessionIdException:
return
root = XML(xml)
case_tag_regex = re.compile(
r"^(\{.*\}){0,1}case$"
) # Use regex in order to search regardless of namespace
meta_tag_regex = re.compile(r"^(\{.*\}){0,1}meta$")
timeEnd_tag_regex = re.compile(r"^(\{.*\}){0,1}timeEnd$")
current_timstamp = json_format_datetime(utcnow())
for child in root:
if case_tag_regex.match(child.tag) is not None:
# Found the case tag
case_element = child
case_element.set("date_modified", current_timstamp)
if not session.include_case_updates_in_partial_submissions:
# Remove case actions (create, update, close)
child_elements = [case_action for case_action in case_element]
for case_action in child_elements:
case_element.remove(case_action)
elif meta_tag_regex.match(child.tag) is not None:
# Found the meta tag, now set the value for timeEnd
for meta_child in child:
if timeEnd_tag_regex.match(meta_child.tag):
meta_child.text = current_timstamp
cleaned_xml = tostring(root)
# Submit the xml
result = submit_form_locally(cleaned_xml,
session.domain,
app_id=session.app_id,
partial_submission=True)
session.submission_id = result.xform.form_id
def test_get_list(self):
"""
Any form in the appropriate domain should be in the list from the API.
"""
# The actual infrastructure involves saving to CouchDB, having PillowTop
# read the changes and write it to ElasticSearch.
# In order to test just the API code, we set up a fake XFormES (this should
# really be a parameter to the XFormInstanceResource constructor)
# and write the translated form directly; we are not trying to test
# the ptop infrastructure.
# the pillow is set to offline mode - elasticsearch not needed to validate
fake_xform_es = FakeXFormES()
v0_4.MOCK_XFORM_ES = fake_xform_es
backend_form = XFormInstance(
xmlns='fake-xmlns',
domain=self.domain.name,
received_on=datetime.utcnow(),
edited_on=datetime.utcnow(),
form={
'#type': 'fake-type',
'@xmlns': 'fake-xmlns',
'meta': {
'userID': 'metadata-user-id'
},
},
auth_context={
'user_id': 'auth-user-id',
'domain': self.domain.name,
'authenticated': True,
},
)
backend_form.save()
self.addCleanup(backend_form.delete)
translated_doc = transform_xform_for_elasticsearch(
backend_form.to_json())
fake_xform_es.add_doc(translated_doc['_id'], translated_doc)
response = self._assert_auth_get_resource(self.list_endpoint)
self.assertEqual(response.status_code, 200)
api_forms = json.loads(response.content)['objects']
self.assertEqual(len(api_forms), 1)
api_form = api_forms[0]
self.assertEqual(api_form['form']['@xmlns'], backend_form.xmlns)
self.assertEqual(api_form['received_on'],
json_format_datetime(backend_form.received_on))
self.assertEqual(api_form['metadata']['userID'], 'metadata-user-id')
self.assertEqual(api_form['edited_by_user_id'], 'auth-user-id')
def test_get_list(self):
"""
Any case in the appropriate domain should be in the list from the API.
"""
backend_case = self._setup_fake_es()
response = self._assert_auth_get_resource(self.list_endpoint)
self.assertEqual(response.status_code, 200)
api_cases = json.loads(response.content)['objects']
self.assertEqual(len(api_cases), 1)
api_case = api_cases[0]
self.assertEqual(api_case['server_date_modified'], json_format_datetime(backend_case.server_modified_on))
def finish(self, save_stats=True, raise_exc=False):
try:
if (save_stats and self.stats_key and self.stats
and self.redis_client):
dumpable = {}
for k, v in self.stats.items():
dumpable[k] = [json_format_datetime(t) for t in v]
self.redis_client.set(self.stats_key, json.dumps(dumpable))
if self.lock:
self.lock.release()
except:
if raise_exc:
raise
def fake_form_submission(userID=userID, username=self.username, xmlns=self.xmlns, time=None):
submission = submission_template % {
"userID": userID,
"username": username,
"xmlns": xmlns
}
f = StringIO(submission.encode('utf-8'))
f.name = "tempfile.xml"
kwargs = dict(HTTP_X_SUBMIT_TIME=json_format_datetime(time)) if time else {}
response = c.post("/a/{self.domain}/receiver/".format(self=self), {
'xml_submission_file': f,
}, **kwargs)
return response
def get_xml(self):
updates = self._case._updates
case_block = get_case_xml(self._case, updates, version='2.0')
diff_block = get_diff_block(self._case)
return render_to_string('hqcase/xml/case_block.xml', {
'xmlns': self.xmlns,
'case_block': case_block.decode('utf-8') + diff_block,
'time': json_format_datetime(self.received_on),
'uid': self.form_id,
'username': "",
'user_id': self.user_id or "",
'device_id': self.device_id,
})
def json_handler(obj):
if callable(getattr(obj, 'to_complete_json', None)):
return obj.to_complete_json()
elif callable(getattr(obj, 'to_json', None)):
return obj.to_json()
elif isinstance(obj, datetime):
return json_format_datetime(obj)
elif isinstance(obj, date):
return obj.isoformat()
elif isinstance(obj, Decimal):
return float(obj) # warning, potential loss of precision
else:
return json.JSONEncoder().default(obj)
def submit_case_blocks(case_blocks,
domain,
username="******",
user_id=None,
xmlns=None,
attachments=None,
form_id=None,
form_extras=None,
case_db=None,
device_id=None):
"""
Submits casexml in a manner similar to how they would be submitted from a phone.
:param xmlns: Form XMLNS. Format: IRI or URN. Historically this was
used in some places to uniquely identify the subsystem that posted
the cases; `device_id` is now recommended for that purpose. Going
forward, it is recommended to use the default value along with
`device_id`, which indicates that the cases were submitted by an
internal system process.
:param device_id: Identifier for the source of posted cases. Ideally
this should uniquely identify the subsystem that is posting cases to
make it easier to trace the source. All new code should use this
argument. A human recognizable value is recommended outside of test
code. Example: "auto-close-rule-<GUID>"
returns the UID of the resulting form.
"""
attachments = attachments or {}
now = json_format_datetime(datetime.datetime.utcnow())
if not isinstance(case_blocks, six.string_types):
case_blocks = ''.join(case_blocks)
soft_assert_type_text(case_blocks)
form_id = form_id or uuid.uuid4().hex
form_xml = render_to_string(
'hqcase/xml/case_block.xml', {
'xmlns': xmlns or SYSTEM_FORM_XMLNS,
'case_block': case_blocks,
'time': now,
'uid': form_id,
'username': username,
'user_id': user_id or "",
'device_id': device_id or "",
})
form_extras = form_extras or {}
result = submit_form_locally(instance=form_xml,
domain=domain,
attachments=attachments,
case_db=case_db,
**form_extras)
return result.xform, result.cases
def iteration_parameters(db, doc_type, domain, view_range, group, chunk_size=1000):
if "view" in group:
view_name = group["view"]
start = end = "-"
assert doc_type is None, doc_type
if domain is not None:
startkey = [domain]
endkey = [domain]
else:
startkey = []
endkey = []
elif domain is not None:
view_name = 'by_domain_doc_type_date/view'
if doc_type is not None:
startkey = [domain, doc_type]
endkey = [domain, doc_type]
else:
startkey = [domain]
endkey = [domain]
elif doc_type is not None:
view_name = 'all_docs/by_doc_type'
startkey = [doc_type]
endkey = [doc_type]
else:
view_name = 'all_docs/by_doc_type'
startkey = []
endkey = []
if view_range is not None:
assert domain or doc_type, (domain, doc_type)
if group.get("date_range"):
assert domain and doc_type and view_name == 'by_domain_doc_type_date/view', \
(domain, doc_type, view_name, view_range)
view_range = [json_format_datetime(parse_date(x)) for x in view_range]
start, end = view_range
startkey.append(start)
endkey.append(end)
else:
start = end = "-"
if startkey == endkey:
endkey.append({})
view_params = {
'startkey': startkey,
'endkey': endkey,
'limit': chunk_size,
'include_docs': False,
'reduce': False,
}
resume_key = f"{db.dbname}.{domain}.{doc_type}.{start}-{end}"
return resume_key, view_name, view_params
def iterate_repeat_records(due_before, chunk_size=10000, database=None):
from .models import RepeatRecord
json_now = json_format_datetime(due_before)
view_kwargs = {
'reduce': False,
'startkey': [None],
'endkey': [None, json_now, {}],
'include_docs': True
}
for doc in paginate_view(RepeatRecord.get_db(),
'receiverwrapper/repeat_records_by_next_check',
chunk_size, **view_kwargs):
yield RepeatRecord.wrap(doc['doc'])
def serialize(value):
"""
Serializes a value so it can properly be parsed into XML
"""
if isinstance(value, datetime.datetime):
return json_format_datetime(value)
elif isinstance(value, datetime.date):
return value.isoformat()
elif isinstance(value, datetime.time):
return value.strftime('%H:%M:%S')
elif isinstance(value, (int, Decimal, float)):
return str(value)
else:
return value if value is not None else ""
