def _get_start_stop_datetime(self, form_config):
"""
Returns a start datetime for the Visit and the Encounter, and a
stop_datetime for the Visit
"""
if form_config.openmrs_start_datetime:
cc_start_datetime_str = form_config.openmrs_start_datetime._get_commcare_value(self.info)
if cc_start_datetime_str is None:
raise ConfigurationError(
'A form config for form XMLNS "{}" uses "openmrs_start_datetime" to get the start of '
'the visit but no value was found in the form.'.format(form_config.xmlns)
)
try:
cc_start_datetime = string_to_utc_datetime(cc_start_datetime_str)
except ValueError:
raise ConfigurationError(
'A form config for form XMLNS "{}" uses "openmrs_start_datetime" to get the start of '
'the visit but an invalid value was found in the form.'.format(form_config.xmlns)
)
cc_stop_datetime = cc_start_datetime + timedelta(days=1) - timedelta(seconds=1)
# We need to use openmrs_start_datetime.serialize()
# for both values because they could be either
# OpenMRS datetimes or OpenMRS dates, and their data
# types must match.
start_datetime = form_config.openmrs_start_datetime.serialize(cc_start_datetime)
stop_datetime = form_config.openmrs_start_datetime.serialize(cc_stop_datetime)
else:
cc_start_datetime = string_to_utc_datetime(self.form_json['form']['meta']['timeEnd'])
cc_stop_datetime = cc_start_datetime + timedelta(days=1) - timedelta(seconds=1)
start_datetime = to_omrs_datetime(cc_start_datetime)
stop_datetime = to_omrs_datetime(cc_stop_datetime)
return start_datetime, stop_datetime
def _to_row(xform_dict):
def _fmt_url(doc_id):
if xform_dict['doc_type'] in [
"XFormInstance",
"XFormArchived",
"XFormError",
"XFormDeprecated"]:
view_name = 'render_form_data'
else:
view_name = 'download_form'
try:
return "<a class='ajax_dialog' href='%(url)s'>%(text)s</a>" % {
"url": reverse(view_name, args=[self.domain, doc_id]),
"text": _("View Form")
}
except NoReverseMatch:
return 'unable to view form'
def _fmt_date(somedate):
time = ServerTime(somedate).user_time(self.timezone).done()
return time.strftime(SERVER_DATETIME_FORMAT)
if xform_dict['form'].get('meta'):
form_name = xmlns_to_name(
self.domain,
xform_dict.get('xmlns'),
app_id=xform_dict.get('app_id'),
)
form_username = xform_dict['form']['meta'].get('username', EMPTY_USER)
else:
form_name = EMPTY_FORM
form_username = EMPTY_USER
error_type = SubmissionErrorType.display_name_by_doc_type(xform_dict['doc_type'])
if xform_dict['doc_type'] == "XFormArchived":
archive_operations = [operation for operation in xform_dict.get('history')
if operation.get('operation') == 'archive']
if archive_operations:
error_type = _("{username} {archived_form} on {date}").format(
username=cached_user_id_to_username(archive_operations[-1].get('user')) or "",
archived_form=SubmissionErrorType.display_name_by_doc_type(xform_dict['doc_type']),
date=_fmt_date(string_to_utc_datetime(archive_operations[-1].get('date'))),
)
return [
_fmt_url(xform_dict['_id']),
form_username,
_fmt_date(string_to_utc_datetime(xform_dict['received_on'])),
form_name,
error_type,
xform_dict.get('problem', EMPTY_ERROR),
self._make_reproces_button(xform_dict) if self.support_toggle_enabled else '',
]
def _is_relevant(case_or_case_state_dict):
if case_or_case_state_dict:
# only case-like things have this.
if 'server_modified_on' in case_or_case_state_dict:
return string_to_utc_datetime(case['server_modified_on']) >= last_sync_token.date
# for case states default to always checking for recent updates
return True
def get_pillow_json(pillow_config):
assert isinstance(pillow_config, PillowConfig)
from pillowtop.listener import AliasedElasticPillow
pillow_class = pillow_config.get_class()
pillow = (pillow_class(online=False) if issubclass(pillow_class, AliasedElasticPillow)
else pillow_config.get_instance())
checkpoint = pillow.get_checkpoint()
timestamp = checkpoint.get('timestamp')
if timestamp:
time_since_last = datetime.utcnow() - string_to_utc_datetime(timestamp)
hours_since_last = time_since_last.total_seconds() // 3600
try:
# remove microsecond portion
time_since_last = str(time_since_last)
time_since_last = time_since_last[0:time_since_last.index('.')]
except ValueError:
pass
else:
time_since_last = ''
hours_since_last = None
return {
'name': pillow_config.name,
'seq': force_seq_int(checkpoint.get('seq')),
'old_seq': force_seq_int(checkpoint.get('old_seq')) or 0,
'db_seq': force_seq_int(pillow.get_change_feed().get_latest_change_id()),
'time_since_last': time_since_last,
'hours_since_last': hours_since_last
}
def get_last_run_meta(export_instance, export_archive_path):
main_table = export_instance.get_table(MAIN_TABLE)
received_on_column_index, received_on_column = main_table.get_column([PathNode(name="received_on")],
'ExportItem', None)
if not received_on_column:
raise CommandError("Export does not contain a field appropriate for finding the last exported date.")
form_id_column_index, form_id_column = main_table.get_column(
[
PathNode(name='form'),
PathNode(name='meta'),
PathNode(name='instanceID')
], 'ExportItem', None
)
if form_id_column_index is None:
print("WARNING: unable to get last form ID. Export may contain a duplicate form")
last_page_path = _get_last_page(export_archive_path, main_table.label)
if last_page_path:
folder, filename = last_page_path.rsplit('/', 1)
matcher = re.match(r'(\d+)_.*', filename)
last_page_number = int(matcher.group(1))
else:
last_page_number = 0
last_form_id, date_col_string = _get_column_value_from_last_line(
last_page_path, form_id_column_index, received_on_column_index
)
last_form_received_on = string_to_utc_datetime(date_col_string)
return (
last_form_id,
last_form_received_on,
last_page_number
)
def process_change(self, change):
synclog = change.get_document()
if not synclog:
return
version = None
app_id = None
try:
sync_date = string_to_utc_datetime(synclog.get('date'))
except (ValueError, AttributeError):
return
build_id = synclog.get('build_id')
if build_id:
version, app_id = get_version_and_app_from_build_id(synclog.get('domain'), build_id)
user_id = synclog.get('user_id')
if user_id:
user = CouchUser.get_by_user_id(user_id)
save = update_last_sync(user, app_id, sync_date, version)
if version:
save |= update_latest_builds(user, app_id, sync_date, version)
app_meta = None
device_id = synclog.get('device_id')
if device_id:
if app_id:
app_meta = DeviceAppMeta(app_id=app_id, build_id=build_id, last_sync=sync_date)
save |= update_device_meta(user, device_id, device_app_meta=app_meta, save=False)
if save:
user.save(fire_signals=False)
def _session_expired(timeout, activity, time):
if activity is None:
return False
if time - string_to_utc_datetime(activity) > datetime.timedelta(minutes=timeout):
return True
else:
return False
def iso_string_to_datetime(iso_string, strict=False):
"""
parse datetime string in iso format with or without microseconds,
always with both date and time
and always with the 'Z' UTC timezone suffix
return an offset-naive datetime representing UTC
>>> iso_string_to_datetime('2015-04-07T19:07:55Z')
datetime.datetime(2015, 4, 7, 19, 7, 55)
>>> iso_string_to_datetime('2015-04-07T19:07:55.437086Z')
datetime.datetime(2015, 4, 7, 19, 7, 55, 437086)
"""
for fmt in ['%Y-%m-%dT%H:%M:%SZ', ISO_DATETIME_FORMAT]:
try:
return datetime.datetime.strptime(iso_string, fmt)
except ValueError:
pass
if strict:
raise ValueError('iso_string_to_datetime input not in expected format: {}'.format(iso_string))
else:
_assert(False, 'iso_string_to_datetime input not in expected format',
iso_string)
from dimagi.utils.parsing import string_to_utc_datetime
return string_to_utc_datetime(iso_string)
def mark_latest_submission(domain, user_id, app_id, build_id, version, metadata, received_on):
user = CouchUser.get_by_user_id(user_id, domain)
if not user or user.is_deleted():
return
try:
received_on_datetime = string_to_utc_datetime(received_on)
except ValueError:
return
last_submission = filter_by_app(user.reporting_metadata.last_submissions, app_id)
if metadata and metadata.get('appVersion') and not isinstance(metadata['appVersion'], six.string_types):
metadata = format_form_meta_for_es(metadata)
app_version_info = get_app_version_info(
domain,
build_id,
version,
metadata
)
if _last_submission_needs_update(last_submission,
received_on_datetime,
app_version_info.build_version,
app_version_info.commcare_version):
if last_submission is None:
last_submission = LastSubmission()
user.reporting_metadata.last_submissions.append(last_submission)
last_submission.submission_date = received_on_datetime
device_id = metadata.get('deviceID')
last_submission.device_id = device_id
last_submission.app_id = app_id
last_submission.build_id = build_id
last_submission.build_version = app_version_info.build_version
last_submission.commcare_version = app_version_info.commcare_version
if app_version_info.build_version:
update_latest_builds(user, app_id, received_on_datetime, app_version_info.build_version)
if _last_submission_needs_update(user.reporting_metadata.last_submission_for_user,
received_on_datetime,
app_version_info.build_version,
app_version_info.commcare_version,
False):
user.reporting_metadata.last_submission_for_user = last_submission
app_meta = DeviceAppMeta(
app_id=app_id,
build_id=build_id,
last_submission=received_on_datetime,
)
update_device_meta(user, device_id, app_version_info.commcare_version, app_meta, save=False)
user.save()
def process_rows(self, users, fmt_for_export=False):
rows = []
for user in users:
last_build = last_seen = last_sub = last_sync = last_sync_date = app_name = commcare_version = None
build_version = _("Unknown")
reporting_metadata = user.get('reporting_metadata', {})
if self.selected_app_id:
last_submissions = reporting_metadata.get('last_submissions')
if last_submissions:
last_sub = self.get_data_for_app(last_submissions, self.selected_app_id)
last_syncs = reporting_metadata.get('last_syncs')
if last_syncs:
last_sync = self.get_data_for_app(last_syncs, self.selected_app_id)
if last_sync is None:
last_sync = self.get_data_for_app(last_syncs, None)
last_builds = reporting_metadata.get('last_builds')
if last_builds:
last_build = self.get_data_for_app(last_builds, self.selected_app_id)
else:
last_sub = reporting_metadata.get('last_submission_for_user', {})
last_sync = reporting_metadata.get('last_sync_for_user', {})
last_build = reporting_metadata.get('last_build_for_user', {})
if last_sub and last_sub.get('commcare_version'):
commcare_version = _get_commcare_version(last_sub.get('commcare_version'))
else:
devices = user.get('devices', None)
if devices:
device = max(devices, key=lambda dev: dev['last_used'])
if device.get('commcare_version', None):
commcare_version = _get_commcare_version(device['commcare_version'])
if last_sub and last_sub.get('submission_date'):
last_seen = string_to_utc_datetime(last_sub['submission_date'])
if last_sync and last_sync.get('sync_date'):
last_sync_date = string_to_utc_datetime(last_sync['sync_date'])
if last_build:
build_version = last_build.get('build_version') or build_version
if last_build.get('app_id'):
app_name = self.get_app_name(last_build['app_id'])
rows.append([
user_display_string(user.get('username', ''),
user.get('first_name', ''),
user.get('last_name', '')),
_fmt_date(last_seen, fmt_for_export), _fmt_date(last_sync_date, fmt_for_export),
app_name or "---", build_version, commcare_version or '---'
])
return rows
def rows(self):
rows = []
user_ids = map(lambda user: user.user_id, self.users)
user_xform_dicts_map = get_last_form_submissions_by_user(self.domain, user_ids, self.selected_app_id)
for user in self.users:
xform_dict = last_seen = last_sync = app_name = None
app_version_info_from_form = app_version_info_from_sync = None
if user_xform_dicts_map.get(user.user_id):
xform_dict = user_xform_dicts_map[user.user_id][0]
if xform_dict:
last_seen = string_to_utc_datetime(xform_dict.get('received_on'))
if xform_dict.get('app_id'):
try:
app = get_app(self.domain, xform_dict.get('app_id'))
except ResourceNotFound:
pass
else:
app_name = app.name
else:
app_name = get_meta_appversion_text(xform_dict['form']['meta'])
app_version_info_from_form = get_app_version_info(
self.domain,
xform_dict.get('build_id'),
xform_dict.get('version'),
xform_dict['form']['meta'],
)
if app_name is None and self.selected_app_id:
continue
last_sync_log = SyncLog.last_for_user(user.user_id)
if last_sync_log:
last_sync = last_sync_log.date
if last_sync_log.build_id:
build_version = get_version_from_build_id(self.domain, last_sync_log.build_id)
app_version_info_from_sync = AppVersionInfo(
build_version,
app_version_info_from_form.commcare_version if app_version_info_from_form else None,
BuildVersionSource.BUILD_ID
)
app_version_info_to_use = _choose_latest_version(
app_version_info_from_sync, app_version_info_from_form,
)
commcare_version = _get_commcare_version(app_version_info_to_use)
build_version = _get_build_version(app_version_info_to_use)
rows.append([
user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync),
app_name or "---", build_version, commcare_version
])
return rows
def process_sql(self, doc_dict, delete=False):
if delete:
return
xform_id = doc_dict.get("_id")
form = doc_dict.get("form", {})
userlogs = get_logs(form, "user_subreport", "user")
UserEntry.objects.filter(xform_id=xform_id).delete()
DeviceReportEntry.objects.filter(xform_id=xform_id).delete()
to_save = []
for i, log in enumerate(force_list(userlogs)):
to_save.append(
UserEntry(
xform_id=xform_id,
i=i,
user_id=log["user_id"],
username=log["username"],
sync_token=log["sync_token"],
)
)
UserEntry.objects.bulk_create(to_save)
domain = doc_dict.get("domain")
logs = get_logs(form, "log_subreport", "log")
logged_in_username = None
logged_in_user_id = None
to_save = []
for i, log in enumerate(force_list(logs)):
if not log:
continue
if log["type"] == "login":
# j2me log = user_id_prefix-username
logged_in_username = log["msg"].split("-")[1]
logged_in_user_id = self.get_user_id(logged_in_username, domain)
elif log["type"] == "user" and log["msg"][:5] == "login":
# android log = login|username|user_id
msg_split = log["msg"].split("|")
logged_in_username = msg_split[1]
logged_in_user_id = msg_split[2]
to_save.append(
DeviceReportEntry(
xform_id=xform_id,
i=i,
domain=domain,
type=log["type"],
msg=log["msg"],
# must accept either date or datetime string
date=dateutil.parser.parse(log["@date"]).replace(tzinfo=None),
server_date=string_to_utc_datetime(doc_dict["received_on"]),
app_version=form.get("app_version"),
device_id=form.get("device_id"),
username=logged_in_username,
user_id=logged_in_user_id,
)
)
DeviceReportEntry.objects.bulk_create(to_save)
def rows(self):
rows = []
selected_app = self.request_params.get(SelectApplicationFilter.slug, None)
user_ids = map(lambda user: user.user_id, self.users)
user_xform_dicts_map = get_last_form_submissions_by_user(self.domain, user_ids, selected_app)
for user in self.users:
xform_dict = last_seen = last_sync = app_name = None
if user_xform_dicts_map.get(user.user_id):
xform_dict = user_xform_dicts_map[user.user_id][0]
if xform_dict:
last_seen = string_to_utc_datetime(xform_dict.get('received_on'))
if xform_dict.get('app_id'):
try:
app = get_app(self.domain, xform_dict.get('app_id'))
except ResourceNotFound:
pass
else:
app_name = app.name
else:
app_name = get_meta_appversion_text(xform_dict['form']['meta'])
app_version_info = get_app_version_info(
self.domain,
xform_dict.get('build_id'),
xform_dict.get('version'),
xform_dict['form']['meta'],
)
build_html = _build_html(app_version_info)
commcare_version = (
'CommCare {}'.format(app_version_info.commcare_version)
if app_version_info.commcare_version
else _("Unknown CommCare Version")
)
commcare_version_html = mark_safe('<span class="label label-info">{}</span>'.format(
commcare_version)
)
app_name = app_name or _("Unknown App")
app_name = format_html(
u'{} {} {}', app_name, mark_safe(build_html), commcare_version_html
)
if app_name is None and selected_app:
continue
last_sync_log = SyncLog.last_for_user(user.user_id)
if last_sync_log:
last_sync = last_sync_log.date
rows.append(
[user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync), app_name or "---"]
)
return rows
def update_user_reporting_data(app_build_id, app_id, build_profile_id,
couch_user, request):
def _safe_int(val):
try:
return int(val)
except:
pass
app_version = _safe_int(request.GET.get('app_version', ''))
device_id = request.GET.get('device_id', '')
last_sync_time = request.GET.get('last_sync_time', '')
num_unsent_forms = _safe_int(request.GET.get('num_unsent_forms', ''))
num_quarantined_forms = _safe_int(
request.GET.get('num_quarantined_forms', ''))
commcare_version = request.GET.get('cc_version', '')
# if mobile cannot determine app version it sends -1
if app_version == -1:
app_version = None
try:
last_sync = adjust_text_to_datetime(last_sync_time)
except iso8601.ParseError:
try:
last_sync = string_to_utc_datetime(last_sync_time)
except (ValueError, OverflowError):
last_sync = None
if settings.USER_REPORTING_METADATA_BATCH_ENABLED:
UserReportingMetadataStaging.add_heartbeat(
request.domain, couch_user._id, app_id, app_build_id, last_sync,
device_id, app_version, num_unsent_forms, num_quarantined_forms,
commcare_version, build_profile_id)
else:
record = UserReportingMetadataStaging(
domain=request.domain,
user_id=couch_user._id,
app_id=app_id,
build_id=app_build_id,
sync_date=last_sync,
device_id=device_id,
app_version=app_version,
num_unsent_forms=num_unsent_forms,
num_quarantined_forms=num_quarantined_forms,
commcare_version=commcare_version,
build_profile_id=build_profile_id,
last_heartbeat=datetime.utcnow(),
modified_on=datetime.utcnow())
try:
record.process_record(couch_user)
except ResourceConflict:
# https://sentry.io/dimagi/commcarehq/issues/521967014/
couch_user = CouchUser.get(couch_user.user_id)
record.process_record(couch_user)
def datespan_export_filter(doc, datespan):
if isinstance(datespan, dict):
datespan = DateSpan(**datespan)
try:
received_on = string_to_utc_datetime(doc['received_on']).replace(tzinfo=pytz.utc)
except Exception:
if settings.DEBUG:
raise
return False
if datespan.startdate <= received_on < (datespan.enddate + timedelta(days=1)):
return True
return False
def _get_start_stop_datetime(self, form_config):
"""
Returns a start datetime for the Visit and the Encounter, and a
stop_datetime for the Visit
"""
if form_config.openmrs_start_datetime:
value_source = as_value_source(form_config.openmrs_start_datetime)
if value_source.can_export:
cc_start_datetime_str = value_source.get_commcare_value(
self.info)
if cc_start_datetime_str is None:
raise ConfigurationError(
'A form config for form XMLNS "{}" uses "openmrs_start_datetime" to get the start of '
'the visit but no value was found in the form.'.format(
form_config.xmlns))
try:
cc_start_datetime = string_to_utc_datetime(
cc_start_datetime_str)
except ValueError:
raise ConfigurationError(
'A form config for form XMLNS "{}" uses "openmrs_start_datetime" to get the start of '
'the visit but an invalid value was found in the form.'
.format(form_config.xmlns))
cc_stop_datetime = cc_start_datetime + timedelta(
days=1) - timedelta(seconds=1)
# We need to serialize both values with the data type of
# openmrs_start_datetime because they could be either
# OpenMRS datetimes or OpenMRS dates, and their data
# types must match.
start_datetime = value_source.serialize(cc_start_datetime)
stop_datetime = value_source.serialize(cc_stop_datetime)
return start_datetime, stop_datetime
cc_start_datetime = string_to_utc_datetime(
self.form_json['form']['meta']['timeEnd'])
cc_stop_datetime = cc_start_datetime + timedelta(days=1) - timedelta(
seconds=1)
start_datetime = to_omrs_datetime(cc_start_datetime)
stop_datetime = to_omrs_datetime(cc_stop_datetime)
return start_datetime, stop_datetime
def process_rows(self, users, fmt_for_export=False):
rows = []
for user in users:
last_build = last_seen = last_sub = last_sync = last_sync_date = app_name = None
build_version = _("Unknown")
reporting_metadata = user.get('reporting_metadata', {})
if self.selected_app_id:
last_submissions = reporting_metadata.get('last_submissions')
if last_submissions:
last_sub = self.get_data_for_app(last_submissions, self.selected_app_id)
last_syncs = reporting_metadata.get('last_syncs')
if last_syncs:
last_sync = self.get_data_for_app(last_syncs, self.selected_app_id)
if last_sync is None:
last_sync = self.get_data_for_app(last_syncs, None)
last_builds = reporting_metadata.get('last_builds')
if last_builds:
last_build = self.get_data_for_app(last_builds, self.selected_app_id)
else:
last_sub = reporting_metadata.get('last_submission_for_user', {})
last_sync = reporting_metadata.get('last_sync_for_user', {})
last_build = reporting_metadata.get('last_build_for_user', {})
commcare_version = _get_commcare_version(last_sub.get('commcare_version'))
if last_sub and last_sub.get('submission_date'):
last_seen = string_to_utc_datetime(last_sub['submission_date'])
if last_sync and last_sync.get('sync_date'):
last_sync_date = string_to_utc_datetime(last_sync['sync_date'])
if last_build:
build_version = last_build.get('build_version') or build_version
if last_build.get('app_id'):
app_name = self.get_app_name(last_build['app_id'])
rows.append([
user_display_string(user.get('username', ''),
user.get('first_name', ''),
user.get('last_name', '')),
_fmt_date(last_seen, fmt_for_export), _fmt_date(last_sync_date, fmt_for_export),
app_name or "---", build_version, commcare_version
])
return rows
def from_case_update(cls, case_update, xformdoc):
"""
Create a case object from a case update object.
"""
assert not is_deprecation(xformdoc) # you should never be able to create a case from a deleted update
case = cls()
case._id = case_update.id
case.modified_on = parsing.string_to_utc_datetime(case_update.modified_on_str) \
if case_update.modified_on_str else datetime.utcnow()
# apply initial updates, if present
case.update_from_case_update(case_update, xformdoc)
return case
def datespan_export_filter(doc, datespan):
if isinstance(datespan, dict):
datespan = DateSpan(**datespan)
try:
received_on = string_to_utc_datetime(doc['received_on']).replace(tzinfo=pytz.utc)
except Exception:
if settings.DEBUG:
raise
return False
if datespan.startdate <= received_on < (datespan.enddate + timedelta(days=1)):
return True
return False
def mark_latest_submission(domain, user_id, app_id, build_id, version,
metadata, received_on):
user = CouchUser.get_by_user_id(user_id, domain)
if not user or user.is_deleted():
return
try:
received_on_datetime = string_to_utc_datetime(received_on)
except ValueError:
return
last_submission = filter_by_app(user.reporting_metadata.last_submissions,
app_id)
if metadata and metadata.get('appVersion') and not isinstance(
metadata['appVersion'], basestring):
metadata = format_form_meta_for_es(metadata)
app_version_info = get_app_version_info(domain, build_id, version,
metadata)
if _last_submission_needs_update(last_submission, received_on_datetime,
app_version_info.build_version,
app_version_info.commcare_version):
if last_submission is None:
last_submission = LastSubmission()
user.reporting_metadata.last_submissions.append(last_submission)
last_submission.submission_date = received_on_datetime
last_submission.device_id = metadata.get('deviceID')
last_submission.app_id = app_id
last_submission.build_id = build_id
last_submission.build_version = app_version_info.build_version
last_submission.commcare_version = app_version_info.commcare_version
if app_version_info.build_version:
update_latest_builds(user, app_id, received_on_datetime,
app_version_info.build_version)
if _last_submission_needs_update(
user.reporting_metadata.last_submission_for_user,
received_on_datetime, app_version_info.build_version,
app_version_info.commcare_version, False):
user.reporting_metadata.last_submission_for_user = last_submission
user.save()
def from_case_update(cls, case_update, xformdoc):
"""
Create a case object from a case update object.
"""
assert not is_deprecation(
xformdoc
) # you should never be able to create a case from a deleted update
case = cls()
case._id = case_update.id
case.modified_on = parsing.string_to_utc_datetime(case_update.modified_on_str) \
if case_update.modified_on_str else datetime.utcnow()
# apply initial updates, if present
case.update_from_case_update(case_update, xformdoc)
return case
def get_consumption_for_ledger_json(ledger_json):
from corehq.apps.domain.models import Domain
from casexml.apps.stock.consumption import compute_daily_consumption
from dimagi.utils.parsing import string_to_utc_datetime
domain_name = ledger_json['domain']
domain = Domain.get_by_name(domain_name)
if domain and domain.commtrack_settings:
consumption_calc = domain.commtrack_settings.get_consumption_config()
else:
consumption_calc = None
daily_consumption = compute_daily_consumption(
domain_name, ledger_json['case_id'], ledger_json['entry_id'],
string_to_utc_datetime(ledger_json['last_modified']), 'stock',
consumption_calc)
return daily_consumption
def process_change(self, change):
if change.deleted or change.metadata is None:
return
doc = change.get_document()
if not doc:
return
build_id = doc.get('build_id')
domain = change.metadata.domain
if build_id and domain:
# Marks if a build has a submission. The function is cached based on domain
# and build_id so that there is no need to fetch the app again after this
# is called. Any subsequent calls with the same arguments will result in
# the same effect, an app having has_submissions set to True.
mark_has_submission(domain, build_id)
user_id = doc.get('form', {}).get('meta', {}).get('userID')
if user_id in WEIRD_USER_IDS:
return
try:
received_on = string_to_utc_datetime(doc.get('received_on'))
except ValueError:
return
app_id = doc.get('app_id')
version = doc.get('version')
try:
metadata = doc['form']['meta']
except KeyError:
metadata = None
if user_id and domain and received_on:
if settings.USER_REPORTING_METADATA_BATCH_ENABLED:
UserReportingMetadataStaging.add_submission(
domain, user_id, app_id, build_id, version, metadata,
received_on)
else:
user = CouchUser.get_by_user_id(user_id, domain)
if not user or user.is_deleted():
return
mark_latest_submission(domain, user, app_id, build_id, version,
metadata, received_on)
def update_user_reporting_data(app_build_id, app_id, couch_user, request):
def _safe_int(val):
try:
return int(val)
except:
pass
app_version = _safe_int(request.GET.get('app_version', ''))
device_id = request.GET.get('device_id', '')
last_sync_time = request.GET.get('last_sync_time', '')
num_unsent_forms = _safe_int(request.GET.get('num_unsent_forms', ''))
num_quarantined_forms = _safe_int(request.GET.get('num_quarantined_forms', ''))
commcare_version = request.GET.get('cc_version', '')
save_user = False
# if mobile cannot determine app version it sends -1
if app_version and app_version > 0:
save_user = update_latest_builds(couch_user, app_id, datetime.utcnow(), app_version)
try:
last_sync = adjust_text_to_datetime(last_sync_time)
except iso8601.ParseError:
try:
last_sync = string_to_utc_datetime(last_sync_time)
except (ValueError, OverflowError):
last_sync = None
else:
save_user |= update_last_sync(couch_user, app_id, last_sync, app_version)
app_meta = DeviceAppMeta(
app_id=app_id,
build_id=app_build_id,
build_version=app_version,
last_heartbeat=datetime.utcnow(),
last_sync=last_sync,
num_unsent_forms=num_unsent_forms,
num_quarantined_forms=num_quarantined_forms
)
save_user |= update_device_meta(
couch_user,
device_id,
commcare_version=commcare_version,
device_app_meta=app_meta,
save=False
)
if save_user:
couch_user.save(fire_signals=False)
def update_user_reporting_data(app_build_id, app_id, couch_user, request):
def _safe_int(val):
try:
return int(val)
except:
pass
app_version = _safe_int(request.GET.get('app_version', ''))
device_id = request.GET.get('device_id', '')
last_sync_time = request.GET.get('last_sync_time', '')
num_unsent_forms = _safe_int(request.GET.get('num_unsent_forms', ''))
num_quarantined_forms = _safe_int(
request.GET.get('num_quarantined_forms', ''))
commcare_version = request.GET.get('cc_version', '')
save_user = False
# if mobile cannot determine app version it sends -1
if app_version and app_version > 0:
save_user = update_latest_builds(couch_user, app_id, datetime.utcnow(),
app_version)
try:
last_sync = adjust_text_to_datetime(last_sync_time)
except iso8601.ParseError:
try:
last_sync = string_to_utc_datetime(last_sync_time)
except (ValueError, OverflowError):
last_sync = None
else:
save_user |= update_last_sync(couch_user, app_id, last_sync,
app_version)
app_meta = DeviceAppMeta(app_id=app_id,
build_id=app_build_id,
build_version=app_version,
last_heartbeat=datetime.utcnow(),
last_sync=last_sync,
num_unsent_forms=num_unsent_forms,
num_quarantined_forms=num_quarantined_forms)
save_user |= update_device_meta(couch_user,
device_id,
commcare_version=commcare_version,
device_app_meta=app_meta,
save=False)
if save_user:
couch_user.save(fire_signals=False)
def _to_row(xform_dict):
def _fmt_url(doc_id):
if xform_dict['doc_type'] in [
"XFormInstance", "XFormArchived", "XFormError",
"XFormDeprecated"
]:
view_name = 'render_form_data'
else:
view_name = 'download_form'
try:
return "<a class='ajax_dialog' href='%(url)s'>%(text)s</a>" % {
"url": reverse(view_name, args=[self.domain, doc_id]),
"text": _("View Form")
}
except NoReverseMatch:
return 'unable to view form'
def _fmt_date(somedate):
time = ServerTime(somedate).user_time(self.timezone).done()
return time.strftime(SERVER_DATETIME_FORMAT)
if xform_dict['form'].get('meta'):
form_name = xmlns_to_name(
self.domain,
xform_dict.get('xmlns'),
app_id=xform_dict.get('app_id'),
)
form_username = xform_dict['form']['meta'].get(
'username', EMPTY_USER)
else:
form_name = EMPTY_FORM
form_username = EMPTY_USER
return [
_fmt_url(xform_dict['_id']),
form_username,
_fmt_date(string_to_utc_datetime(xform_dict['received_on'])),
form_name,
SubmissionErrorType.display_name_by_doc_type(
xform_dict['doc_type']),
xform_dict.get('problem', EMPTY_ERROR),
self._make_reproces_button(xform_dict)
if self.support_toggle_enabled else '',
]
def get_consumption_for_ledger_json(ledger_json):
from corehq.apps.domain.models import Domain
from casexml.apps.stock.consumption import compute_daily_consumption
from dimagi.utils.parsing import string_to_utc_datetime
domain_name = ledger_json['domain']
domain_obj = Domain.get_by_name(domain_name)
if domain_obj and domain_obj.commtrack_settings:
consumption_calc = domain_obj.commtrack_settings.get_consumption_config()
else:
consumption_calc = None
daily_consumption = compute_daily_consumption(
domain_name,
ledger_json['case_id'],
ledger_json['entry_id'],
string_to_utc_datetime(ledger_json['last_modified']),
'stock',
consumption_calc
)
return daily_consumption
def create_visits(requests, info, form_json, form_question_values, openmrs_config, person_uuid):
provider_uuid = getattr(openmrs_config, 'openmrs_provider', None)
info.form_question_values.update(form_question_values)
for form_config in openmrs_config.form_configs:
logger.debug('Send visit for form?', form_config, form_json)
if form_config.xmlns == form_json['form']['@xmlns']:
logger.debug('Yes')
create_visit(
requests,
person_uuid=person_uuid,
provider_uuid=provider_uuid,
visit_datetime=string_to_utc_datetime(form_json['form']['meta']['timeEnd']),
values_for_concept={obs.concept: [obs.value.get_value(info)]
for obs in form_config.openmrs_observations
if obs.value.get_value(info)},
encounter_type=form_config.openmrs_encounter_type,
openmrs_form=form_config.openmrs_form,
visit_type=form_config.openmrs_visit_type,
# location_uuid=, # location of case owner (CHW) > location[meta][openmrs_uuid]
)
def test_get_last_form_submission_by_xmlns(self):
xmlns = 'http://a.b.org'
kwargs = {
'user_id': 'u1',
'app_id': '1234',
'domain': self.domain,
}
first = datetime(2013, 7, 15, 0, 0, 0)
second = datetime(2013, 7, 16, 0, 0, 0)
third = datetime(2013, 7, 17, 0, 0, 0)
self._send_form_to_es(received_on=second, xmlns=xmlns, **kwargs)
self._send_form_to_es(received_on=third, xmlns=xmlns, **kwargs)
self._send_form_to_es(received_on=first, xmlns=xmlns, **kwargs)
form = get_last_form_submission_for_xmlns(self.domain, xmlns)
self.assertEqual(string_to_utc_datetime(form['received_on']), third)
form = get_last_form_submission_for_xmlns(self.domain, 'missing')
self.assertIsNone(form)
def test_get_last_form_submission_by_xmlns(self):
xmlns = 'http://a.b.org'
kwargs = {
'user_id': 'u1',
'app_id': '1234',
'domain': self.domain,
}
first = datetime(2013, 7, 15, 0, 0, 0)
second = datetime(2013, 7, 16, 0, 0, 0)
third = datetime(2013, 7, 17, 0, 0, 0)
self._send_form_to_es(received_on=second, xmlns=xmlns, **kwargs)
self._send_form_to_es(received_on=third, xmlns=xmlns, **kwargs)
self._send_form_to_es(received_on=first, xmlns=xmlns, **kwargs)
form = get_last_form_submission_for_xmlns(self.domain, xmlns)
self.assertEqual(string_to_utc_datetime(form['received_on']), third)
form = get_last_form_submission_for_xmlns(self.domain, 'missing')
self.assertIsNone(form)
def test_app_metadata_tracker_synclog_processed(self):
UserReportingMetadataStaging.add_sync(
self.domain, self.user._id, self.metadata.app_id,
'123', datetime.utcnow(), self.metadata.device_id
)
form, metadata = self._create_form_and_sync_to_es()
self.assertEqual(UserReportingMetadataStaging.objects.count(), 1)
self.assertEqual(UserReportingMetadataStaging.objects.first().user_id, self.user._id)
process_reporting_metadata_staging()
self.assertEqual(UserReportingMetadataStaging.objects.count(), 0)
user = CommCareUser.get_by_user_id(self.user._id, self.domain)
self.assertEqual(len(user.reporting_metadata.last_submissions), 1)
last_submission = user.reporting_metadata.last_submissions[0]
self.assertEqual(
last_submission.submission_date,
string_to_utc_datetime(self.metadata.received_on),
)
self.assertEqual(last_submission.app_id, self.metadata.app_id)
def process_change(self, pillow_instance, change):
synclog = change.get_document()
if not synclog:
return
version = None
app_id = None
try:
sync_date = string_to_utc_datetime(synclog.get('date'))
except (ValueError, AttributeError):
return
build_id = synclog.get('build_id')
if build_id:
version, app_id = get_version_and_app_from_build_id(
synclog.get('domain'), build_id)
user_id = synclog.get('user_id')
if user_id:
user = CouchUser.get_by_user_id(user_id)
last_sync = filter_by_app(user.reporting_metadata.last_syncs,
app_id)
if _last_sync_needs_update(last_sync, sync_date):
if last_sync is None:
last_sync = LastSync()
user.reporting_metadata.last_syncs.append(last_sync)
last_sync.sync_date = sync_date
last_sync.build_version = version
last_sync.app_id = app_id
if _last_sync_needs_update(
user.reporting_metadata.last_sync_for_user, sync_date):
user.reporting_metadata.last_sync_for_user = last_sync
if version:
update_latest_builds(user, app_id, sync_date, version)
user.save()
def mark_latest_submission(domain, user_id, app_id, build_id, version,
metadata, received_on):
user = CouchUser.get_by_user_id(user_id, domain)
if not user or user.is_deleted():
return
try:
received_on_datetime = string_to_utc_datetime(received_on)
except ValueError:
return
last_submissions = filter(
lambda submission: submission.app_id == app_id,
user.reporting_metadata.last_submissions,
)
if last_submissions:
assert len(last_submissions
) == 1, 'Must only have one last submission per app'
last_submission = last_submissions[0]
else:
last_submission = None
app_version_info = get_app_version_info(domain, build_id, version,
metadata)
if last_submission is None or last_submission.submission_date < received_on_datetime:
if last_submission is None:
last_submission = LastSubmission()
user.reporting_metadata.last_submissions.append(last_submission)
last_submission.submission_date = received_on_datetime
last_submission.device_id = metadata.get('deviceID')
last_submission.app_id = app_id
last_submission.build_id = build_id
last_submission.build_version = app_version_info.build_version
last_submission.commcare_version = app_version_info.commcare_version
user.save()
def _to_row(xform_dict):
def _fmt_url(doc_id):
if xform_dict['doc_type'] in [
"XFormInstance",
"XFormArchived",
"XFormError",
"XFormDeprecated"]:
view_name = 'render_form_data'
else:
view_name = 'download_form'
try:
return "<a class='ajax_dialog' href='%(url)s'>%(text)s</a>" % {
"url": reverse(view_name, args=[self.domain, doc_id]),
"text": _("View Form")
}
except NoReverseMatch:
return 'unable to view form'
def _fmt_date(somedate):
time = ServerTime(somedate).user_time(self.timezone).done()
return time.strftime(SERVER_DATETIME_FORMAT)
if xform_dict['form'].get('meta'):
form_name = xmlns_to_name(
self.domain,
xform_dict.get('xmlns'),
app_id=xform_dict.get('app_id'),
)
form_username = xform_dict['form']['meta']['username']
else:
form_name = EMPTY_FORM
form_username = EMPTY_USER
return [
_fmt_url(xform_dict['_id']),
form_username,
_fmt_date(string_to_utc_datetime(xform_dict['received_on'])),
form_name,
SubmissionErrorType.display_name_by_doc_type(xform_dict['doc_type']),
xform_dict.get('problem', EMPTY_ERROR),
]
def test_app_metadata_tracker(self):
form, metadata = self._create_form_and_sync_to_es()
self.assertEqual(UserReportingMetadataStaging.objects.count(), 1)
self.assertEqual(UserReportingMetadataStaging.objects.first().user_id, self.user._id)
# Test two forms before updating
form, metadata = self._create_form_and_sync_to_es()
self.assertEqual(UserReportingMetadataStaging.objects.count(), 1)
self.assertEqual(UserReportingMetadataStaging.objects.first().user_id, self.user._id)
self.assertEqual(0, PillowError.objects.filter(pillow=self.pillow_id).count())
process_reporting_metadata_staging()
self.assertEqual(UserReportingMetadataStaging.objects.count(), 0)
user = CommCareUser.get_by_user_id(self.user._id, self.domain)
self.assertEqual(len(user.reporting_metadata.last_submissions), 1)
last_submission = user.reporting_metadata.last_submissions[0]
self.assertEqual(
last_submission.submission_date,
string_to_utc_datetime(self.metadata.received_on),
)
self.assertEqual(last_submission.app_id, self.metadata.app_id)
def process_change(self, change):
synclog = change.get_document()
if not synclog:
return
user_id = synclog.get('user_id')
domain = synclog.get('domain')
if not user_id or not domain:
return
try:
sync_date = string_to_utc_datetime(synclog.get('date'))
except (ValueError, AttributeError):
return
build_id = synclog.get('build_id')
device_id = synclog.get('device_id')
app_id = synclog.get('app_id')
# WebApps syncs do not provide the app_id.
# For those syncs we go ahead and mark the last synclog synchronously.
if app_id and settings.USER_REPORTING_METADATA_BATCH_ENABLED:
UserReportingMetadataStaging.add_sync(domain, user_id, app_id,
build_id, sync_date,
device_id)
else:
user = CouchUser.get_by_user_id(user_id)
if not user:
return
device_app_meta = None
if device_id and app_id:
device_app_meta = DeviceAppMeta(app_id=app_id,
build_id=build_id,
last_sync=sync_date)
mark_last_synclog(domain, user, app_id, build_id, sync_date,
sync_date, device_id, device_app_meta)
def process_change(self, pillow_instance, change):
synclog = change.get_document()
if not synclog:
return
version = None
app_id = None
try:
sync_date = string_to_utc_datetime(synclog.get('date'))
except (ValueError, AttributeError):
return
build_id = synclog.get('build_id')
if build_id:
version, app_id = get_version_and_app_from_build_id(
synclog.get('domain'), build_id)
user_id = synclog.get('user_id')
if user_id:
user = CouchUser.get_by_user_id(user_id)
save = update_last_sync(user, app_id, sync_date, version)
if version:
save |= update_latest_builds(user, app_id, sync_date, version)
app_meta = None
device_id = synclog.get('device_id')
if device_id:
if app_id:
app_meta = DeviceAppMeta(app_id=app_id,
build_id=build_id,
last_sync=sync_date)
save |= update_device_meta(user,
device_id,
device_app_meta=app_meta,
save=False)
if save:
user.save(fire_signals=False)
def iso_string_to_datetime(iso_string):
"""
parse datetime string in iso format with or without microseconds,
always with both date and time
and always with the 'Z' UTC timezone suffix
return an offset-naive datetime representing UTC
>>> iso_string_to_datetime('2015-04-07T19:07:55Z')
datetime.datetime(2015, 4, 7, 19, 7, 55)
>>> iso_string_to_datetime('2015-04-07T19:07:55.437086Z')
datetime.datetime(2015, 4, 7, 19, 7, 55, 437086)
"""
for fmt in ['%Y-%m-%dT%H:%M:%SZ', ISO_DATETIME_FORMAT]:
try:
return datetime.datetime.strptime(iso_string, fmt)
except ValueError:
pass
_assert(False, 'iso_string_to_datetime input not in expected format',
iso_string)
from dimagi.utils.parsing import string_to_utc_datetime
return string_to_utc_datetime(iso_string)
def process_change(self, pillow_instance, change):
synclog = change.get_document()
if not synclog:
return
version = None
app_id = None
try:
sync_date = string_to_utc_datetime(synclog.get('date'))
except (ValueError, AttributeError):
return
build_id = synclog.get('build_id')
if build_id:
version, app_id = get_version_and_app_from_build_id(
synclog.get('domain'), build_id)
user_id = synclog.get('user_id')
if user_id:
user = CouchUser.get_by_user_id(user_id)
save = update_last_sync(user, app_id, sync_date, version)
if version:
save |= update_latest_builds(user, app_id, sync_date, version)
if save:
user.save()
def _session_expired(timeout, activity, time):
if activity is None:
return False
time_since_activity = time - string_to_utc_datetime(activity)
return time_since_activity > datetime.timedelta(minutes=timeout)
def parse_iso8601(datetime_string):
return string_to_utc_datetime(datetime_string)
def transform_xform_for_elasticsearch(doc_dict):
"""
Given an XFormInstance, return a copy that is ready to be sent to elasticsearch,
or None, if the form should not be saved to elasticsearch
"""
doc_ret = copy.deepcopy(doc_dict)
if 'meta' in doc_ret['form']:
if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)):
doc_ret['form']['meta']['timeEnd'] = None
if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)):
doc_ret['form']['meta']['timeStart'] = None
# Some docs have their @xmlns and #text here
if isinstance(doc_ret['form']['meta'].get('appVersion'), dict):
doc_ret['form']['meta'] = format_form_meta_for_es(
doc_ret['form']['meta'])
app_version_info = get_app_version_info(
doc_ret['domain'],
doc_ret.get('build_id'),
doc_ret.get('version'),
doc_ret['form']['meta'],
)
doc_ret['form']['meta'][
'commcare_version'] = app_version_info.commcare_version
doc_ret['form']['meta'][
'app_build_version'] = app_version_info.build_version
try:
geo_point = GeoPointProperty().wrap(
doc_ret['form']['meta']['location'])
doc_ret['form']['meta']['geo_point'] = geo_point.lat_lon
except (KeyError, BadValueError):
doc_ret['form']['meta']['geo_point'] = None
pass
try:
user_id = doc_ret['form']['meta']['userID']
except KeyError:
user_id = None
doc_ret['user_type'] = get_user_type(user_id)
doc_ret['inserted_at'] = datetime.datetime.utcnow().isoformat()
try:
case_blocks = extract_case_blocks(doc_ret)
except PhoneDateValueError:
pass
else:
for case_dict in case_blocks:
for date_modified_key in ['date_modified', '@date_modified']:
if not is_valid_date(case_dict.get(date_modified_key, None)):
if case_dict.get(date_modified_key) == '':
case_dict[date_modified_key] = None
else:
case_dict.pop(date_modified_key, None)
# convert all mapped dict properties to nulls if they are empty strings
for object_key in ['index', 'attachment', 'create', 'update']:
if object_key in case_dict and not isinstance(
case_dict[object_key], dict):
case_dict[object_key] = None
try:
doc_ret["__retrieved_case_ids"] = list(
set(case_update_from_block(cb).id for cb in case_blocks))
except CaseGenerationException:
doc_ret["__retrieved_case_ids"] = []
if 'backend_id' not in doc_ret:
doc_ret['backend_id'] = 'couch'
server_modified_on = doc_ret['received_on']
if doc_ret.get('edited_on', None):
# doesn't take archiving and unarchiving into account
received_on = string_to_utc_datetime(doc_ret['received_on'])
edited_on = string_to_utc_datetime(doc_ret['edited_on'])
server_modified_on = max(received_on, edited_on).isoformat()
doc_ret['server_modified_on'] = server_modified_on
return doc_ret
def _valid_date(date_str):
try:
return string_to_utc_datetime(date_str)
except ValueError:
raise CommandError('Not a valid date string: {}'.format(date_str))
def get_transaction_date(transaction):
return string_to_utc_datetime(transaction.server_date).date()
def rows(self):
rows = []
user_ids = map(lambda user: user.user_id, self.users)
user_xform_dicts_map = get_last_form_submissions_by_user(
self.domain, user_ids, self.selected_app_id)
for user in self.users:
xform_dict = last_seen = last_sync = app_name = None
app_version_info_from_form = app_version_info_from_sync = None
if user_xform_dicts_map.get(user.user_id):
xform_dict = user_xform_dicts_map[user.user_id][0]
if xform_dict:
last_seen = string_to_utc_datetime(
xform_dict.get('received_on'))
if xform_dict.get('app_id'):
try:
app = get_app(self.domain, xform_dict.get('app_id'))
except ResourceNotFound:
pass
else:
app_name = app.name
else:
app_name = get_meta_appversion_text(
xform_dict['form']['meta'])
app_version_info_from_form = get_app_version_info(
self.domain,
xform_dict.get('build_id'),
xform_dict.get('version'),
xform_dict['form']['meta'],
)
if app_name is None and self.selected_app_id:
continue
last_sync_log = SyncLog.last_for_user(user.user_id)
if last_sync_log:
last_sync = last_sync_log.date
if last_sync_log.build_id:
build_version = get_version_from_build_id(
self.domain, last_sync_log.build_id)
app_version_info_from_sync = AppVersionInfo(
build_version,
app_version_info_from_form.commcare_version
if app_version_info_from_form else None,
BuildVersionSource.BUILD_ID)
app_version_info_to_use = _choose_latest_version(
app_version_info_from_sync,
app_version_info_from_form,
)
commcare_version = _get_commcare_version(app_version_info_to_use)
build_version = _get_build_version(app_version_info_to_use)
rows.append([
user.username_in_report,
_fmt_date(last_seen),
_fmt_date(last_sync), app_name or "---", build_version,
commcare_version
])
return rows
def get_received_on(request):
received_on = request.META.get('HTTP_X_SUBMIT_TIME')
if received_on:
return string_to_utc_datetime(received_on)
else:
return None
def _to_row(xform_dict):
def _fmt_url(doc_id):
if xform_dict['doc_type'] in [
"XFormInstance", "XFormArchived", "XFormError",
"XFormDeprecated"
]:
view_name = 'render_form_data'
else:
view_name = 'download_form'
try:
return format_html(
"<a class='ajax_dialog' href='{url}'>{text}</a>",
url=reverse(view_name, args=[self.domain, doc_id]),
text=_("View Form"))
except NoReverseMatch:
return 'unable to view form'
def _fmt_date(somedate):
time = ServerTime(somedate).user_time(self.timezone).done()
return time.strftime(SERVER_DATETIME_FORMAT)
if xform_dict['form'].get('meta'):
form_name = xmlns_to_name(
self.domain,
xform_dict.get('xmlns'),
app_id=xform_dict.get('app_id'),
)
form_username = xform_dict['form']['meta'].get(
'username', EMPTY_USER)
else:
form_name = EMPTY_FORM
form_username = EMPTY_USER
error_type = SubmissionTypeFilter.display_name_by_doc_type(
xform_dict['doc_type'])
if xform_dict['doc_type'] == "XFormArchived":
archive_operations = [
operation for operation in xform_dict.get('history')
if operation.get('operation') == 'archive'
]
if archive_operations:
error_type = _(
"{username} {archived_form} on {date}").format(
username=cached_user_id_to_username(
archive_operations[-1].get('user')) or "",
archived_form=SubmissionTypeFilter.
display_name_by_doc_type(xform_dict['doc_type']),
date=_fmt_date(
string_to_utc_datetime(
archive_operations[-1].get('date'))),
)
return [
_fmt_url(xform_dict['_id']),
form_username,
_fmt_date(string_to_utc_datetime(xform_dict['received_on'])),
form_name,
error_type,
xform_dict.get('problem', EMPTY_ERROR),
self._make_reproces_button(xform_dict)
if self.support_toggle_enabled else '',
]
def process_rows(self, users, fmt_for_export=False):
rows = []
users = list(users)
if self.include_location_data():
location_ids = {
user['location_id']
for user in users if user['location_id']
}
grouped_ancestor_locs = self.get_bulk_ancestors(location_ids)
self.required_loc_columns = self.get_location_columns(
grouped_ancestor_locs)
for user in users:
last_build = last_seen = last_sub = last_sync = last_sync_date = app_name = commcare_version = None
last_build_profile_name = None
build_version = _("Unknown")
reporting_metadata = user.get('reporting_metadata', {})
if self.selected_app_id:
last_submissions = reporting_metadata.get('last_submissions')
if last_submissions:
last_sub = self.get_data_for_app(last_submissions,
self.selected_app_id)
last_syncs = reporting_metadata.get('last_syncs')
if last_syncs:
last_sync = self.get_data_for_app(last_syncs,
self.selected_app_id)
if last_sync is None:
last_sync = self.get_data_for_app(last_syncs, None)
last_builds = reporting_metadata.get('last_builds')
if last_builds:
last_build = self.get_data_for_app(last_builds,
self.selected_app_id)
else:
last_sub = reporting_metadata.get('last_submission_for_user',
{})
last_sync = reporting_metadata.get('last_sync_for_user', {})
last_build = reporting_metadata.get('last_build_for_user', {})
if last_sub and last_sub.get('commcare_version'):
commcare_version = _get_commcare_version(
last_sub.get('commcare_version'))
else:
devices = user.get('devices', None)
if devices:
device = max(devices, key=lambda dev: dev['last_used'])
if device.get('commcare_version', None):
commcare_version = _get_commcare_version(
device['commcare_version'])
if last_sub and last_sub.get('submission_date'):
last_seen = string_to_utc_datetime(last_sub['submission_date'])
if last_sync and last_sync.get('sync_date'):
last_sync_date = string_to_utc_datetime(last_sync['sync_date'])
if last_build:
build_version = last_build.get(
'build_version') or build_version
if last_build.get('app_id'):
app_name = self.get_app_name(last_build['app_id'])
if self.show_build_profile:
last_build_profile_id = last_build.get('build_profile_id')
if last_build_profile_id:
last_build_profile_name = _("Unknown")
build_profiles = self._get_app_details(
last_build['app_id']).get('build_profiles', {})
if last_build_profile_id in build_profiles:
last_build_profile_name = build_profiles[
last_build_profile_id]
row_data = [
user_display_string(user.get('username', ''),
user.get('first_name', ''),
user.get('last_name', '')),
_fmt_date(last_seen, fmt_for_export),
_fmt_date(last_sync_date, fmt_for_export), app_name or "---",
build_version, commcare_version or '---'
]
if self.show_build_profile:
row_data.append(last_build_profile_name)
if self.include_location_data():
location_data = self.user_locations(
grouped_ancestor_locs.get(user['location_id'], []),
self.required_loc_columns)
row_data = location_data + row_data
rows.append(row_data)
return rows
def _session_expired(timeout, activity, time):
if activity is None:
return False
time_since_activity = time - string_to_utc_datetime(activity)
return time_since_activity > datetime.timedelta(minutes=timeout)
def get_received_on(request):
received_on = request.META.get('HTTP_X_SUBMIT_TIME')
if received_on:
return string_to_utc_datetime(received_on)
else:
return None
def guess_modified_on(self):
"""
Guess the modified date, defaulting to the current time in UTC.
"""
return string_to_utc_datetime(self.modified_on_str) if self.modified_on_str else datetime.datetime.utcnow()
def from_json(value):
return string_to_utc_datetime(value) if value is not None else None