def _get_start_stop_datetime(self, form_config): """ Returns a start datetime for the Visit and the Encounter, and a stop_datetime for the Visit """ if form_config.openmrs_start_datetime: cc_start_datetime_str = form_config.openmrs_start_datetime._get_commcare_value(self.info) if cc_start_datetime_str is None: raise ConfigurationError( 'A form config for form XMLNS "{}" uses "openmrs_start_datetime" to get the start of ' 'the visit but no value was found in the form.'.format(form_config.xmlns) ) try: cc_start_datetime = string_to_utc_datetime(cc_start_datetime_str) except ValueError: raise ConfigurationError( 'A form config for form XMLNS "{}" uses "openmrs_start_datetime" to get the start of ' 'the visit but an invalid value was found in the form.'.format(form_config.xmlns) ) cc_stop_datetime = cc_start_datetime + timedelta(days=1) - timedelta(seconds=1) # We need to use openmrs_start_datetime.serialize() # for both values because they could be either # OpenMRS datetimes or OpenMRS dates, and their data # types must match. start_datetime = form_config.openmrs_start_datetime.serialize(cc_start_datetime) stop_datetime = form_config.openmrs_start_datetime.serialize(cc_stop_datetime) else: cc_start_datetime = string_to_utc_datetime(self.form_json['form']['meta']['timeEnd']) cc_stop_datetime = cc_start_datetime + timedelta(days=1) - timedelta(seconds=1) start_datetime = to_omrs_datetime(cc_start_datetime) stop_datetime = to_omrs_datetime(cc_stop_datetime) return start_datetime, stop_datetime
def _to_row(xform_dict): def _fmt_url(doc_id): if xform_dict['doc_type'] in [ "XFormInstance", "XFormArchived", "XFormError", "XFormDeprecated"]: view_name = 'render_form_data' else: view_name = 'download_form' try: return "<a class='ajax_dialog' href='%(url)s'>%(text)s</a>" % { "url": reverse(view_name, args=[self.domain, doc_id]), "text": _("View Form") } except NoReverseMatch: return 'unable to view form' def _fmt_date(somedate): time = ServerTime(somedate).user_time(self.timezone).done() return time.strftime(SERVER_DATETIME_FORMAT) if xform_dict['form'].get('meta'): form_name = xmlns_to_name( self.domain, xform_dict.get('xmlns'), app_id=xform_dict.get('app_id'), ) form_username = xform_dict['form']['meta'].get('username', EMPTY_USER) else: form_name = EMPTY_FORM form_username = EMPTY_USER error_type = SubmissionErrorType.display_name_by_doc_type(xform_dict['doc_type']) if xform_dict['doc_type'] == "XFormArchived": archive_operations = [operation for operation in xform_dict.get('history') if operation.get('operation') == 'archive'] if archive_operations: error_type = _("{username} {archived_form} on {date}").format( username=cached_user_id_to_username(archive_operations[-1].get('user')) or "", archived_form=SubmissionErrorType.display_name_by_doc_type(xform_dict['doc_type']), date=_fmt_date(string_to_utc_datetime(archive_operations[-1].get('date'))), ) return [ _fmt_url(xform_dict['_id']), form_username, _fmt_date(string_to_utc_datetime(xform_dict['received_on'])), form_name, error_type, xform_dict.get('problem', EMPTY_ERROR), self._make_reproces_button(xform_dict) if self.support_toggle_enabled else '', ]
def _is_relevant(case_or_case_state_dict): if case_or_case_state_dict: # only case-like things have this. if 'server_modified_on' in case_or_case_state_dict: return string_to_utc_datetime(case['server_modified_on']) >= last_sync_token.date # for case states default to always checking for recent updates return True
def get_pillow_json(pillow_config): assert isinstance(pillow_config, PillowConfig) from pillowtop.listener import AliasedElasticPillow pillow_class = pillow_config.get_class() pillow = (pillow_class(online=False) if issubclass(pillow_class, AliasedElasticPillow) else pillow_config.get_instance()) checkpoint = pillow.get_checkpoint() timestamp = checkpoint.get('timestamp') if timestamp: time_since_last = datetime.utcnow() - string_to_utc_datetime(timestamp) hours_since_last = time_since_last.total_seconds() // 3600 try: # remove microsecond portion time_since_last = str(time_since_last) time_since_last = time_since_last[0:time_since_last.index('.')] except ValueError: pass else: time_since_last = '' hours_since_last = None return { 'name': pillow_config.name, 'seq': force_seq_int(checkpoint.get('seq')), 'old_seq': force_seq_int(checkpoint.get('old_seq')) or 0, 'db_seq': force_seq_int(pillow.get_change_feed().get_latest_change_id()), 'time_since_last': time_since_last, 'hours_since_last': hours_since_last }
def get_last_run_meta(export_instance, export_archive_path): main_table = export_instance.get_table(MAIN_TABLE) received_on_column_index, received_on_column = main_table.get_column([PathNode(name="received_on")], 'ExportItem', None) if not received_on_column: raise CommandError("Export does not contain a field appropriate for finding the last exported date.") form_id_column_index, form_id_column = main_table.get_column( [ PathNode(name='form'), PathNode(name='meta'), PathNode(name='instanceID') ], 'ExportItem', None ) if form_id_column_index is None: print("WARNING: unable to get last form ID. Export may contain a duplicate form") last_page_path = _get_last_page(export_archive_path, main_table.label) if last_page_path: folder, filename = last_page_path.rsplit('/', 1) matcher = re.match(r'(\d+)_.*', filename) last_page_number = int(matcher.group(1)) else: last_page_number = 0 last_form_id, date_col_string = _get_column_value_from_last_line( last_page_path, form_id_column_index, received_on_column_index ) last_form_received_on = string_to_utc_datetime(date_col_string) return ( last_form_id, last_form_received_on, last_page_number )
def process_change(self, change): synclog = change.get_document() if not synclog: return version = None app_id = None try: sync_date = string_to_utc_datetime(synclog.get('date')) except (ValueError, AttributeError): return build_id = synclog.get('build_id') if build_id: version, app_id = get_version_and_app_from_build_id(synclog.get('domain'), build_id) user_id = synclog.get('user_id') if user_id: user = CouchUser.get_by_user_id(user_id) save = update_last_sync(user, app_id, sync_date, version) if version: save |= update_latest_builds(user, app_id, sync_date, version) app_meta = None device_id = synclog.get('device_id') if device_id: if app_id: app_meta = DeviceAppMeta(app_id=app_id, build_id=build_id, last_sync=sync_date) save |= update_device_meta(user, device_id, device_app_meta=app_meta, save=False) if save: user.save(fire_signals=False)
def _session_expired(timeout, activity, time): if activity is None: return False if time - string_to_utc_datetime(activity) > datetime.timedelta(minutes=timeout): return True else: return False
def iso_string_to_datetime(iso_string, strict=False): """ parse datetime string in iso format with or without microseconds, always with both date and time and always with the 'Z' UTC timezone suffix return an offset-naive datetime representing UTC >>> iso_string_to_datetime('2015-04-07T19:07:55Z') datetime.datetime(2015, 4, 7, 19, 7, 55) >>> iso_string_to_datetime('2015-04-07T19:07:55.437086Z') datetime.datetime(2015, 4, 7, 19, 7, 55, 437086) """ for fmt in ['%Y-%m-%dT%H:%M:%SZ', ISO_DATETIME_FORMAT]: try: return datetime.datetime.strptime(iso_string, fmt) except ValueError: pass if strict: raise ValueError('iso_string_to_datetime input not in expected format: {}'.format(iso_string)) else: _assert(False, 'iso_string_to_datetime input not in expected format', iso_string) from dimagi.utils.parsing import string_to_utc_datetime return string_to_utc_datetime(iso_string)
def mark_latest_submission(domain, user_id, app_id, build_id, version, metadata, received_on): user = CouchUser.get_by_user_id(user_id, domain) if not user or user.is_deleted(): return try: received_on_datetime = string_to_utc_datetime(received_on) except ValueError: return last_submission = filter_by_app(user.reporting_metadata.last_submissions, app_id) if metadata and metadata.get('appVersion') and not isinstance(metadata['appVersion'], six.string_types): metadata = format_form_meta_for_es(metadata) app_version_info = get_app_version_info( domain, build_id, version, metadata ) if _last_submission_needs_update(last_submission, received_on_datetime, app_version_info.build_version, app_version_info.commcare_version): if last_submission is None: last_submission = LastSubmission() user.reporting_metadata.last_submissions.append(last_submission) last_submission.submission_date = received_on_datetime device_id = metadata.get('deviceID') last_submission.device_id = device_id last_submission.app_id = app_id last_submission.build_id = build_id last_submission.build_version = app_version_info.build_version last_submission.commcare_version = app_version_info.commcare_version if app_version_info.build_version: update_latest_builds(user, app_id, received_on_datetime, app_version_info.build_version) if _last_submission_needs_update(user.reporting_metadata.last_submission_for_user, received_on_datetime, app_version_info.build_version, app_version_info.commcare_version, False): user.reporting_metadata.last_submission_for_user = last_submission app_meta = DeviceAppMeta( app_id=app_id, build_id=build_id, last_submission=received_on_datetime, ) update_device_meta(user, device_id, app_version_info.commcare_version, app_meta, save=False) user.save()
def process_rows(self, users, fmt_for_export=False): rows = [] for user in users: last_build = last_seen = last_sub = last_sync = last_sync_date = app_name = commcare_version = None build_version = _("Unknown") reporting_metadata = user.get('reporting_metadata', {}) if self.selected_app_id: last_submissions = reporting_metadata.get('last_submissions') if last_submissions: last_sub = self.get_data_for_app(last_submissions, self.selected_app_id) last_syncs = reporting_metadata.get('last_syncs') if last_syncs: last_sync = self.get_data_for_app(last_syncs, self.selected_app_id) if last_sync is None: last_sync = self.get_data_for_app(last_syncs, None) last_builds = reporting_metadata.get('last_builds') if last_builds: last_build = self.get_data_for_app(last_builds, self.selected_app_id) else: last_sub = reporting_metadata.get('last_submission_for_user', {}) last_sync = reporting_metadata.get('last_sync_for_user', {}) last_build = reporting_metadata.get('last_build_for_user', {}) if last_sub and last_sub.get('commcare_version'): commcare_version = _get_commcare_version(last_sub.get('commcare_version')) else: devices = user.get('devices', None) if devices: device = max(devices, key=lambda dev: dev['last_used']) if device.get('commcare_version', None): commcare_version = _get_commcare_version(device['commcare_version']) if last_sub and last_sub.get('submission_date'): last_seen = string_to_utc_datetime(last_sub['submission_date']) if last_sync and last_sync.get('sync_date'): last_sync_date = string_to_utc_datetime(last_sync['sync_date']) if last_build: build_version = last_build.get('build_version') or build_version if last_build.get('app_id'): app_name = self.get_app_name(last_build['app_id']) rows.append([ user_display_string(user.get('username', ''), user.get('first_name', ''), user.get('last_name', '')), _fmt_date(last_seen, fmt_for_export), _fmt_date(last_sync_date, fmt_for_export), app_name or "---", build_version, commcare_version or '---' ]) return rows
def rows(self): rows = [] user_ids = map(lambda user: user.user_id, self.users) user_xform_dicts_map = get_last_form_submissions_by_user(self.domain, user_ids, self.selected_app_id) for user in self.users: xform_dict = last_seen = last_sync = app_name = None app_version_info_from_form = app_version_info_from_sync = None if user_xform_dicts_map.get(user.user_id): xform_dict = user_xform_dicts_map[user.user_id][0] if xform_dict: last_seen = string_to_utc_datetime(xform_dict.get('received_on')) if xform_dict.get('app_id'): try: app = get_app(self.domain, xform_dict.get('app_id')) except ResourceNotFound: pass else: app_name = app.name else: app_name = get_meta_appversion_text(xform_dict['form']['meta']) app_version_info_from_form = get_app_version_info( self.domain, xform_dict.get('build_id'), xform_dict.get('version'), xform_dict['form']['meta'], ) if app_name is None and self.selected_app_id: continue last_sync_log = SyncLog.last_for_user(user.user_id) if last_sync_log: last_sync = last_sync_log.date if last_sync_log.build_id: build_version = get_version_from_build_id(self.domain, last_sync_log.build_id) app_version_info_from_sync = AppVersionInfo( build_version, app_version_info_from_form.commcare_version if app_version_info_from_form else None, BuildVersionSource.BUILD_ID ) app_version_info_to_use = _choose_latest_version( app_version_info_from_sync, app_version_info_from_form, ) commcare_version = _get_commcare_version(app_version_info_to_use) build_version = _get_build_version(app_version_info_to_use) rows.append([ user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync), app_name or "---", build_version, commcare_version ]) return rows
def process_sql(self, doc_dict, delete=False): if delete: return xform_id = doc_dict.get("_id") form = doc_dict.get("form", {}) userlogs = get_logs(form, "user_subreport", "user") UserEntry.objects.filter(xform_id=xform_id).delete() DeviceReportEntry.objects.filter(xform_id=xform_id).delete() to_save = [] for i, log in enumerate(force_list(userlogs)): to_save.append( UserEntry( xform_id=xform_id, i=i, user_id=log["user_id"], username=log["username"], sync_token=log["sync_token"], ) ) UserEntry.objects.bulk_create(to_save) domain = doc_dict.get("domain") logs = get_logs(form, "log_subreport", "log") logged_in_username = None logged_in_user_id = None to_save = [] for i, log in enumerate(force_list(logs)): if not log: continue if log["type"] == "login": # j2me log = user_id_prefix-username logged_in_username = log["msg"].split("-")[1] logged_in_user_id = self.get_user_id(logged_in_username, domain) elif log["type"] == "user" and log["msg"][:5] == "login": # android log = login|username|user_id msg_split = log["msg"].split("|") logged_in_username = msg_split[1] logged_in_user_id = msg_split[2] to_save.append( DeviceReportEntry( xform_id=xform_id, i=i, domain=domain, type=log["type"], msg=log["msg"], # must accept either date or datetime string date=dateutil.parser.parse(log["@date"]).replace(tzinfo=None), server_date=string_to_utc_datetime(doc_dict["received_on"]), app_version=form.get("app_version"), device_id=form.get("device_id"), username=logged_in_username, user_id=logged_in_user_id, ) ) DeviceReportEntry.objects.bulk_create(to_save)
def rows(self): rows = [] selected_app = self.request_params.get(SelectApplicationFilter.slug, None) user_ids = map(lambda user: user.user_id, self.users) user_xform_dicts_map = get_last_form_submissions_by_user(self.domain, user_ids, selected_app) for user in self.users: xform_dict = last_seen = last_sync = app_name = None if user_xform_dicts_map.get(user.user_id): xform_dict = user_xform_dicts_map[user.user_id][0] if xform_dict: last_seen = string_to_utc_datetime(xform_dict.get('received_on')) if xform_dict.get('app_id'): try: app = get_app(self.domain, xform_dict.get('app_id')) except ResourceNotFound: pass else: app_name = app.name else: app_name = get_meta_appversion_text(xform_dict['form']['meta']) app_version_info = get_app_version_info( self.domain, xform_dict.get('build_id'), xform_dict.get('version'), xform_dict['form']['meta'], ) build_html = _build_html(app_version_info) commcare_version = ( 'CommCare {}'.format(app_version_info.commcare_version) if app_version_info.commcare_version else _("Unknown CommCare Version") ) commcare_version_html = mark_safe('<span class="label label-info">{}</span>'.format( commcare_version) ) app_name = app_name or _("Unknown App") app_name = format_html( u'{} {} {}', app_name, mark_safe(build_html), commcare_version_html ) if app_name is None and selected_app: continue last_sync_log = SyncLog.last_for_user(user.user_id) if last_sync_log: last_sync = last_sync_log.date rows.append( [user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync), app_name or "---"] ) return rows
def update_user_reporting_data(app_build_id, app_id, build_profile_id, couch_user, request): def _safe_int(val): try: return int(val) except: pass app_version = _safe_int(request.GET.get('app_version', '')) device_id = request.GET.get('device_id', '') last_sync_time = request.GET.get('last_sync_time', '') num_unsent_forms = _safe_int(request.GET.get('num_unsent_forms', '')) num_quarantined_forms = _safe_int( request.GET.get('num_quarantined_forms', '')) commcare_version = request.GET.get('cc_version', '') # if mobile cannot determine app version it sends -1 if app_version == -1: app_version = None try: last_sync = adjust_text_to_datetime(last_sync_time) except iso8601.ParseError: try: last_sync = string_to_utc_datetime(last_sync_time) except (ValueError, OverflowError): last_sync = None if settings.USER_REPORTING_METADATA_BATCH_ENABLED: UserReportingMetadataStaging.add_heartbeat( request.domain, couch_user._id, app_id, app_build_id, last_sync, device_id, app_version, num_unsent_forms, num_quarantined_forms, commcare_version, build_profile_id) else: record = UserReportingMetadataStaging( domain=request.domain, user_id=couch_user._id, app_id=app_id, build_id=app_build_id, sync_date=last_sync, device_id=device_id, app_version=app_version, num_unsent_forms=num_unsent_forms, num_quarantined_forms=num_quarantined_forms, commcare_version=commcare_version, build_profile_id=build_profile_id, last_heartbeat=datetime.utcnow(), modified_on=datetime.utcnow()) try: record.process_record(couch_user) except ResourceConflict: # https://sentry.io/dimagi/commcarehq/issues/521967014/ couch_user = CouchUser.get(couch_user.user_id) record.process_record(couch_user)
def datespan_export_filter(doc, datespan): if isinstance(datespan, dict): datespan = DateSpan(**datespan) try: received_on = string_to_utc_datetime(doc['received_on']).replace(tzinfo=pytz.utc) except Exception: if settings.DEBUG: raise return False if datespan.startdate <= received_on < (datespan.enddate + timedelta(days=1)): return True return False
def _get_start_stop_datetime(self, form_config): """ Returns a start datetime for the Visit and the Encounter, and a stop_datetime for the Visit """ if form_config.openmrs_start_datetime: value_source = as_value_source(form_config.openmrs_start_datetime) if value_source.can_export: cc_start_datetime_str = value_source.get_commcare_value( self.info) if cc_start_datetime_str is None: raise ConfigurationError( 'A form config for form XMLNS "{}" uses "openmrs_start_datetime" to get the start of ' 'the visit but no value was found in the form.'.format( form_config.xmlns)) try: cc_start_datetime = string_to_utc_datetime( cc_start_datetime_str) except ValueError: raise ConfigurationError( 'A form config for form XMLNS "{}" uses "openmrs_start_datetime" to get the start of ' 'the visit but an invalid value was found in the form.' .format(form_config.xmlns)) cc_stop_datetime = cc_start_datetime + timedelta( days=1) - timedelta(seconds=1) # We need to serialize both values with the data type of # openmrs_start_datetime because they could be either # OpenMRS datetimes or OpenMRS dates, and their data # types must match. start_datetime = value_source.serialize(cc_start_datetime) stop_datetime = value_source.serialize(cc_stop_datetime) return start_datetime, stop_datetime cc_start_datetime = string_to_utc_datetime( self.form_json['form']['meta']['timeEnd']) cc_stop_datetime = cc_start_datetime + timedelta(days=1) - timedelta( seconds=1) start_datetime = to_omrs_datetime(cc_start_datetime) stop_datetime = to_omrs_datetime(cc_stop_datetime) return start_datetime, stop_datetime
def process_rows(self, users, fmt_for_export=False): rows = [] for user in users: last_build = last_seen = last_sub = last_sync = last_sync_date = app_name = None build_version = _("Unknown") reporting_metadata = user.get('reporting_metadata', {}) if self.selected_app_id: last_submissions = reporting_metadata.get('last_submissions') if last_submissions: last_sub = self.get_data_for_app(last_submissions, self.selected_app_id) last_syncs = reporting_metadata.get('last_syncs') if last_syncs: last_sync = self.get_data_for_app(last_syncs, self.selected_app_id) if last_sync is None: last_sync = self.get_data_for_app(last_syncs, None) last_builds = reporting_metadata.get('last_builds') if last_builds: last_build = self.get_data_for_app(last_builds, self.selected_app_id) else: last_sub = reporting_metadata.get('last_submission_for_user', {}) last_sync = reporting_metadata.get('last_sync_for_user', {}) last_build = reporting_metadata.get('last_build_for_user', {}) commcare_version = _get_commcare_version(last_sub.get('commcare_version')) if last_sub and last_sub.get('submission_date'): last_seen = string_to_utc_datetime(last_sub['submission_date']) if last_sync and last_sync.get('sync_date'): last_sync_date = string_to_utc_datetime(last_sync['sync_date']) if last_build: build_version = last_build.get('build_version') or build_version if last_build.get('app_id'): app_name = self.get_app_name(last_build['app_id']) rows.append([ user_display_string(user.get('username', ''), user.get('first_name', ''), user.get('last_name', '')), _fmt_date(last_seen, fmt_for_export), _fmt_date(last_sync_date, fmt_for_export), app_name or "---", build_version, commcare_version ]) return rows
def from_case_update(cls, case_update, xformdoc): """ Create a case object from a case update object. """ assert not is_deprecation(xformdoc) # you should never be able to create a case from a deleted update case = cls() case._id = case_update.id case.modified_on = parsing.string_to_utc_datetime(case_update.modified_on_str) \ if case_update.modified_on_str else datetime.utcnow() # apply initial updates, if present case.update_from_case_update(case_update, xformdoc) return case
def mark_latest_submission(domain, user_id, app_id, build_id, version, metadata, received_on): user = CouchUser.get_by_user_id(user_id, domain) if not user or user.is_deleted(): return try: received_on_datetime = string_to_utc_datetime(received_on) except ValueError: return last_submission = filter_by_app(user.reporting_metadata.last_submissions, app_id) if metadata and metadata.get('appVersion') and not isinstance( metadata['appVersion'], basestring): metadata = format_form_meta_for_es(metadata) app_version_info = get_app_version_info(domain, build_id, version, metadata) if _last_submission_needs_update(last_submission, received_on_datetime, app_version_info.build_version, app_version_info.commcare_version): if last_submission is None: last_submission = LastSubmission() user.reporting_metadata.last_submissions.append(last_submission) last_submission.submission_date = received_on_datetime last_submission.device_id = metadata.get('deviceID') last_submission.app_id = app_id last_submission.build_id = build_id last_submission.build_version = app_version_info.build_version last_submission.commcare_version = app_version_info.commcare_version if app_version_info.build_version: update_latest_builds(user, app_id, received_on_datetime, app_version_info.build_version) if _last_submission_needs_update( user.reporting_metadata.last_submission_for_user, received_on_datetime, app_version_info.build_version, app_version_info.commcare_version, False): user.reporting_metadata.last_submission_for_user = last_submission user.save()
def from_case_update(cls, case_update, xformdoc): """ Create a case object from a case update object. """ assert not is_deprecation( xformdoc ) # you should never be able to create a case from a deleted update case = cls() case._id = case_update.id case.modified_on = parsing.string_to_utc_datetime(case_update.modified_on_str) \ if case_update.modified_on_str else datetime.utcnow() # apply initial updates, if present case.update_from_case_update(case_update, xformdoc) return case
def get_consumption_for_ledger_json(ledger_json): from corehq.apps.domain.models import Domain from casexml.apps.stock.consumption import compute_daily_consumption from dimagi.utils.parsing import string_to_utc_datetime domain_name = ledger_json['domain'] domain = Domain.get_by_name(domain_name) if domain and domain.commtrack_settings: consumption_calc = domain.commtrack_settings.get_consumption_config() else: consumption_calc = None daily_consumption = compute_daily_consumption( domain_name, ledger_json['case_id'], ledger_json['entry_id'], string_to_utc_datetime(ledger_json['last_modified']), 'stock', consumption_calc) return daily_consumption
def process_change(self, change): if change.deleted or change.metadata is None: return doc = change.get_document() if not doc: return build_id = doc.get('build_id') domain = change.metadata.domain if build_id and domain: # Marks if a build has a submission. The function is cached based on domain # and build_id so that there is no need to fetch the app again after this # is called. Any subsequent calls with the same arguments will result in # the same effect, an app having has_submissions set to True. mark_has_submission(domain, build_id) user_id = doc.get('form', {}).get('meta', {}).get('userID') if user_id in WEIRD_USER_IDS: return try: received_on = string_to_utc_datetime(doc.get('received_on')) except ValueError: return app_id = doc.get('app_id') version = doc.get('version') try: metadata = doc['form']['meta'] except KeyError: metadata = None if user_id and domain and received_on: if settings.USER_REPORTING_METADATA_BATCH_ENABLED: UserReportingMetadataStaging.add_submission( domain, user_id, app_id, build_id, version, metadata, received_on) else: user = CouchUser.get_by_user_id(user_id, domain) if not user or user.is_deleted(): return mark_latest_submission(domain, user, app_id, build_id, version, metadata, received_on)
def update_user_reporting_data(app_build_id, app_id, couch_user, request): def _safe_int(val): try: return int(val) except: pass app_version = _safe_int(request.GET.get('app_version', '')) device_id = request.GET.get('device_id', '') last_sync_time = request.GET.get('last_sync_time', '') num_unsent_forms = _safe_int(request.GET.get('num_unsent_forms', '')) num_quarantined_forms = _safe_int(request.GET.get('num_quarantined_forms', '')) commcare_version = request.GET.get('cc_version', '') save_user = False # if mobile cannot determine app version it sends -1 if app_version and app_version > 0: save_user = update_latest_builds(couch_user, app_id, datetime.utcnow(), app_version) try: last_sync = adjust_text_to_datetime(last_sync_time) except iso8601.ParseError: try: last_sync = string_to_utc_datetime(last_sync_time) except (ValueError, OverflowError): last_sync = None else: save_user |= update_last_sync(couch_user, app_id, last_sync, app_version) app_meta = DeviceAppMeta( app_id=app_id, build_id=app_build_id, build_version=app_version, last_heartbeat=datetime.utcnow(), last_sync=last_sync, num_unsent_forms=num_unsent_forms, num_quarantined_forms=num_quarantined_forms ) save_user |= update_device_meta( couch_user, device_id, commcare_version=commcare_version, device_app_meta=app_meta, save=False ) if save_user: couch_user.save(fire_signals=False)
def update_user_reporting_data(app_build_id, app_id, couch_user, request): def _safe_int(val): try: return int(val) except: pass app_version = _safe_int(request.GET.get('app_version', '')) device_id = request.GET.get('device_id', '') last_sync_time = request.GET.get('last_sync_time', '') num_unsent_forms = _safe_int(request.GET.get('num_unsent_forms', '')) num_quarantined_forms = _safe_int( request.GET.get('num_quarantined_forms', '')) commcare_version = request.GET.get('cc_version', '') save_user = False # if mobile cannot determine app version it sends -1 if app_version and app_version > 0: save_user = update_latest_builds(couch_user, app_id, datetime.utcnow(), app_version) try: last_sync = adjust_text_to_datetime(last_sync_time) except iso8601.ParseError: try: last_sync = string_to_utc_datetime(last_sync_time) except (ValueError, OverflowError): last_sync = None else: save_user |= update_last_sync(couch_user, app_id, last_sync, app_version) app_meta = DeviceAppMeta(app_id=app_id, build_id=app_build_id, build_version=app_version, last_heartbeat=datetime.utcnow(), last_sync=last_sync, num_unsent_forms=num_unsent_forms, num_quarantined_forms=num_quarantined_forms) save_user |= update_device_meta(couch_user, device_id, commcare_version=commcare_version, device_app_meta=app_meta, save=False) if save_user: couch_user.save(fire_signals=False)
def _to_row(xform_dict): def _fmt_url(doc_id): if xform_dict['doc_type'] in [ "XFormInstance", "XFormArchived", "XFormError", "XFormDeprecated" ]: view_name = 'render_form_data' else: view_name = 'download_form' try: return "<a class='ajax_dialog' href='%(url)s'>%(text)s</a>" % { "url": reverse(view_name, args=[self.domain, doc_id]), "text": _("View Form") } except NoReverseMatch: return 'unable to view form' def _fmt_date(somedate): time = ServerTime(somedate).user_time(self.timezone).done() return time.strftime(SERVER_DATETIME_FORMAT) if xform_dict['form'].get('meta'): form_name = xmlns_to_name( self.domain, xform_dict.get('xmlns'), app_id=xform_dict.get('app_id'), ) form_username = xform_dict['form']['meta'].get( 'username', EMPTY_USER) else: form_name = EMPTY_FORM form_username = EMPTY_USER return [ _fmt_url(xform_dict['_id']), form_username, _fmt_date(string_to_utc_datetime(xform_dict['received_on'])), form_name, SubmissionErrorType.display_name_by_doc_type( xform_dict['doc_type']), xform_dict.get('problem', EMPTY_ERROR), self._make_reproces_button(xform_dict) if self.support_toggle_enabled else '', ]
def get_consumption_for_ledger_json(ledger_json): from corehq.apps.domain.models import Domain from casexml.apps.stock.consumption import compute_daily_consumption from dimagi.utils.parsing import string_to_utc_datetime domain_name = ledger_json['domain'] domain_obj = Domain.get_by_name(domain_name) if domain_obj and domain_obj.commtrack_settings: consumption_calc = domain_obj.commtrack_settings.get_consumption_config() else: consumption_calc = None daily_consumption = compute_daily_consumption( domain_name, ledger_json['case_id'], ledger_json['entry_id'], string_to_utc_datetime(ledger_json['last_modified']), 'stock', consumption_calc ) return daily_consumption
def create_visits(requests, info, form_json, form_question_values, openmrs_config, person_uuid): provider_uuid = getattr(openmrs_config, 'openmrs_provider', None) info.form_question_values.update(form_question_values) for form_config in openmrs_config.form_configs: logger.debug('Send visit for form?', form_config, form_json) if form_config.xmlns == form_json['form']['@xmlns']: logger.debug('Yes') create_visit( requests, person_uuid=person_uuid, provider_uuid=provider_uuid, visit_datetime=string_to_utc_datetime(form_json['form']['meta']['timeEnd']), values_for_concept={obs.concept: [obs.value.get_value(info)] for obs in form_config.openmrs_observations if obs.value.get_value(info)}, encounter_type=form_config.openmrs_encounter_type, openmrs_form=form_config.openmrs_form, visit_type=form_config.openmrs_visit_type, # location_uuid=, # location of case owner (CHW) > location[meta][openmrs_uuid] )
def test_get_last_form_submission_by_xmlns(self): xmlns = 'http://a.b.org' kwargs = { 'user_id': 'u1', 'app_id': '1234', 'domain': self.domain, } first = datetime(2013, 7, 15, 0, 0, 0) second = datetime(2013, 7, 16, 0, 0, 0) third = datetime(2013, 7, 17, 0, 0, 0) self._send_form_to_es(received_on=second, xmlns=xmlns, **kwargs) self._send_form_to_es(received_on=third, xmlns=xmlns, **kwargs) self._send_form_to_es(received_on=first, xmlns=xmlns, **kwargs) form = get_last_form_submission_for_xmlns(self.domain, xmlns) self.assertEqual(string_to_utc_datetime(form['received_on']), third) form = get_last_form_submission_for_xmlns(self.domain, 'missing') self.assertIsNone(form)
def test_app_metadata_tracker_synclog_processed(self): UserReportingMetadataStaging.add_sync( self.domain, self.user._id, self.metadata.app_id, '123', datetime.utcnow(), self.metadata.device_id ) form, metadata = self._create_form_and_sync_to_es() self.assertEqual(UserReportingMetadataStaging.objects.count(), 1) self.assertEqual(UserReportingMetadataStaging.objects.first().user_id, self.user._id) process_reporting_metadata_staging() self.assertEqual(UserReportingMetadataStaging.objects.count(), 0) user = CommCareUser.get_by_user_id(self.user._id, self.domain) self.assertEqual(len(user.reporting_metadata.last_submissions), 1) last_submission = user.reporting_metadata.last_submissions[0] self.assertEqual( last_submission.submission_date, string_to_utc_datetime(self.metadata.received_on), ) self.assertEqual(last_submission.app_id, self.metadata.app_id)
def process_change(self, pillow_instance, change): synclog = change.get_document() if not synclog: return version = None app_id = None try: sync_date = string_to_utc_datetime(synclog.get('date')) except (ValueError, AttributeError): return build_id = synclog.get('build_id') if build_id: version, app_id = get_version_and_app_from_build_id( synclog.get('domain'), build_id) user_id = synclog.get('user_id') if user_id: user = CouchUser.get_by_user_id(user_id) last_sync = filter_by_app(user.reporting_metadata.last_syncs, app_id) if _last_sync_needs_update(last_sync, sync_date): if last_sync is None: last_sync = LastSync() user.reporting_metadata.last_syncs.append(last_sync) last_sync.sync_date = sync_date last_sync.build_version = version last_sync.app_id = app_id if _last_sync_needs_update( user.reporting_metadata.last_sync_for_user, sync_date): user.reporting_metadata.last_sync_for_user = last_sync if version: update_latest_builds(user, app_id, sync_date, version) user.save()
def mark_latest_submission(domain, user_id, app_id, build_id, version, metadata, received_on): user = CouchUser.get_by_user_id(user_id, domain) if not user or user.is_deleted(): return try: received_on_datetime = string_to_utc_datetime(received_on) except ValueError: return last_submissions = filter( lambda submission: submission.app_id == app_id, user.reporting_metadata.last_submissions, ) if last_submissions: assert len(last_submissions ) == 1, 'Must only have one last submission per app' last_submission = last_submissions[0] else: last_submission = None app_version_info = get_app_version_info(domain, build_id, version, metadata) if last_submission is None or last_submission.submission_date < received_on_datetime: if last_submission is None: last_submission = LastSubmission() user.reporting_metadata.last_submissions.append(last_submission) last_submission.submission_date = received_on_datetime last_submission.device_id = metadata.get('deviceID') last_submission.app_id = app_id last_submission.build_id = build_id last_submission.build_version = app_version_info.build_version last_submission.commcare_version = app_version_info.commcare_version user.save()
def _to_row(xform_dict): def _fmt_url(doc_id): if xform_dict['doc_type'] in [ "XFormInstance", "XFormArchived", "XFormError", "XFormDeprecated"]: view_name = 'render_form_data' else: view_name = 'download_form' try: return "<a class='ajax_dialog' href='%(url)s'>%(text)s</a>" % { "url": reverse(view_name, args=[self.domain, doc_id]), "text": _("View Form") } except NoReverseMatch: return 'unable to view form' def _fmt_date(somedate): time = ServerTime(somedate).user_time(self.timezone).done() return time.strftime(SERVER_DATETIME_FORMAT) if xform_dict['form'].get('meta'): form_name = xmlns_to_name( self.domain, xform_dict.get('xmlns'), app_id=xform_dict.get('app_id'), ) form_username = xform_dict['form']['meta']['username'] else: form_name = EMPTY_FORM form_username = EMPTY_USER return [ _fmt_url(xform_dict['_id']), form_username, _fmt_date(string_to_utc_datetime(xform_dict['received_on'])), form_name, SubmissionErrorType.display_name_by_doc_type(xform_dict['doc_type']), xform_dict.get('problem', EMPTY_ERROR), ]
def test_app_metadata_tracker(self): form, metadata = self._create_form_and_sync_to_es() self.assertEqual(UserReportingMetadataStaging.objects.count(), 1) self.assertEqual(UserReportingMetadataStaging.objects.first().user_id, self.user._id) # Test two forms before updating form, metadata = self._create_form_and_sync_to_es() self.assertEqual(UserReportingMetadataStaging.objects.count(), 1) self.assertEqual(UserReportingMetadataStaging.objects.first().user_id, self.user._id) self.assertEqual(0, PillowError.objects.filter(pillow=self.pillow_id).count()) process_reporting_metadata_staging() self.assertEqual(UserReportingMetadataStaging.objects.count(), 0) user = CommCareUser.get_by_user_id(self.user._id, self.domain) self.assertEqual(len(user.reporting_metadata.last_submissions), 1) last_submission = user.reporting_metadata.last_submissions[0] self.assertEqual( last_submission.submission_date, string_to_utc_datetime(self.metadata.received_on), ) self.assertEqual(last_submission.app_id, self.metadata.app_id)
def process_change(self, change): synclog = change.get_document() if not synclog: return user_id = synclog.get('user_id') domain = synclog.get('domain') if not user_id or not domain: return try: sync_date = string_to_utc_datetime(synclog.get('date')) except (ValueError, AttributeError): return build_id = synclog.get('build_id') device_id = synclog.get('device_id') app_id = synclog.get('app_id') # WebApps syncs do not provide the app_id. # For those syncs we go ahead and mark the last synclog synchronously. if app_id and settings.USER_REPORTING_METADATA_BATCH_ENABLED: UserReportingMetadataStaging.add_sync(domain, user_id, app_id, build_id, sync_date, device_id) else: user = CouchUser.get_by_user_id(user_id) if not user: return device_app_meta = None if device_id and app_id: device_app_meta = DeviceAppMeta(app_id=app_id, build_id=build_id, last_sync=sync_date) mark_last_synclog(domain, user, app_id, build_id, sync_date, sync_date, device_id, device_app_meta)
def process_change(self, pillow_instance, change): synclog = change.get_document() if not synclog: return version = None app_id = None try: sync_date = string_to_utc_datetime(synclog.get('date')) except (ValueError, AttributeError): return build_id = synclog.get('build_id') if build_id: version, app_id = get_version_and_app_from_build_id( synclog.get('domain'), build_id) user_id = synclog.get('user_id') if user_id: user = CouchUser.get_by_user_id(user_id) save = update_last_sync(user, app_id, sync_date, version) if version: save |= update_latest_builds(user, app_id, sync_date, version) app_meta = None device_id = synclog.get('device_id') if device_id: if app_id: app_meta = DeviceAppMeta(app_id=app_id, build_id=build_id, last_sync=sync_date) save |= update_device_meta(user, device_id, device_app_meta=app_meta, save=False) if save: user.save(fire_signals=False)
def iso_string_to_datetime(iso_string): """ parse datetime string in iso format with or without microseconds, always with both date and time and always with the 'Z' UTC timezone suffix return an offset-naive datetime representing UTC >>> iso_string_to_datetime('2015-04-07T19:07:55Z') datetime.datetime(2015, 4, 7, 19, 7, 55) >>> iso_string_to_datetime('2015-04-07T19:07:55.437086Z') datetime.datetime(2015, 4, 7, 19, 7, 55, 437086) """ for fmt in ['%Y-%m-%dT%H:%M:%SZ', ISO_DATETIME_FORMAT]: try: return datetime.datetime.strptime(iso_string, fmt) except ValueError: pass _assert(False, 'iso_string_to_datetime input not in expected format', iso_string) from dimagi.utils.parsing import string_to_utc_datetime return string_to_utc_datetime(iso_string)
def process_change(self, pillow_instance, change): synclog = change.get_document() if not synclog: return version = None app_id = None try: sync_date = string_to_utc_datetime(synclog.get('date')) except (ValueError, AttributeError): return build_id = synclog.get('build_id') if build_id: version, app_id = get_version_and_app_from_build_id( synclog.get('domain'), build_id) user_id = synclog.get('user_id') if user_id: user = CouchUser.get_by_user_id(user_id) save = update_last_sync(user, app_id, sync_date, version) if version: save |= update_latest_builds(user, app_id, sync_date, version) if save: user.save()
def _session_expired(timeout, activity, time): if activity is None: return False time_since_activity = time - string_to_utc_datetime(activity) return time_since_activity > datetime.timedelta(minutes=timeout)
def parse_iso8601(datetime_string): return string_to_utc_datetime(datetime_string)
def transform_xform_for_elasticsearch(doc_dict): """ Given an XFormInstance, return a copy that is ready to be sent to elasticsearch, or None, if the form should not be saved to elasticsearch """ doc_ret = copy.deepcopy(doc_dict) if 'meta' in doc_ret['form']: if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)): doc_ret['form']['meta']['timeEnd'] = None if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)): doc_ret['form']['meta']['timeStart'] = None # Some docs have their @xmlns and #text here if isinstance(doc_ret['form']['meta'].get('appVersion'), dict): doc_ret['form']['meta'] = format_form_meta_for_es( doc_ret['form']['meta']) app_version_info = get_app_version_info( doc_ret['domain'], doc_ret.get('build_id'), doc_ret.get('version'), doc_ret['form']['meta'], ) doc_ret['form']['meta'][ 'commcare_version'] = app_version_info.commcare_version doc_ret['form']['meta'][ 'app_build_version'] = app_version_info.build_version try: geo_point = GeoPointProperty().wrap( doc_ret['form']['meta']['location']) doc_ret['form']['meta']['geo_point'] = geo_point.lat_lon except (KeyError, BadValueError): doc_ret['form']['meta']['geo_point'] = None pass try: user_id = doc_ret['form']['meta']['userID'] except KeyError: user_id = None doc_ret['user_type'] = get_user_type(user_id) doc_ret['inserted_at'] = datetime.datetime.utcnow().isoformat() try: case_blocks = extract_case_blocks(doc_ret) except PhoneDateValueError: pass else: for case_dict in case_blocks: for date_modified_key in ['date_modified', '@date_modified']: if not is_valid_date(case_dict.get(date_modified_key, None)): if case_dict.get(date_modified_key) == '': case_dict[date_modified_key] = None else: case_dict.pop(date_modified_key, None) # convert all mapped dict properties to nulls if they are empty strings for object_key in ['index', 'attachment', 'create', 'update']: if object_key in case_dict and not isinstance( case_dict[object_key], dict): case_dict[object_key] = None try: doc_ret["__retrieved_case_ids"] = list( set(case_update_from_block(cb).id for cb in case_blocks)) except CaseGenerationException: doc_ret["__retrieved_case_ids"] = [] if 'backend_id' not in doc_ret: doc_ret['backend_id'] = 'couch' server_modified_on = doc_ret['received_on'] if doc_ret.get('edited_on', None): # doesn't take archiving and unarchiving into account received_on = string_to_utc_datetime(doc_ret['received_on']) edited_on = string_to_utc_datetime(doc_ret['edited_on']) server_modified_on = max(received_on, edited_on).isoformat() doc_ret['server_modified_on'] = server_modified_on return doc_ret
def _valid_date(date_str): try: return string_to_utc_datetime(date_str) except ValueError: raise CommandError('Not a valid date string: {}'.format(date_str))
def get_transaction_date(transaction): return string_to_utc_datetime(transaction.server_date).date()
def rows(self): rows = [] user_ids = map(lambda user: user.user_id, self.users) user_xform_dicts_map = get_last_form_submissions_by_user( self.domain, user_ids, self.selected_app_id) for user in self.users: xform_dict = last_seen = last_sync = app_name = None app_version_info_from_form = app_version_info_from_sync = None if user_xform_dicts_map.get(user.user_id): xform_dict = user_xform_dicts_map[user.user_id][0] if xform_dict: last_seen = string_to_utc_datetime( xform_dict.get('received_on')) if xform_dict.get('app_id'): try: app = get_app(self.domain, xform_dict.get('app_id')) except ResourceNotFound: pass else: app_name = app.name else: app_name = get_meta_appversion_text( xform_dict['form']['meta']) app_version_info_from_form = get_app_version_info( self.domain, xform_dict.get('build_id'), xform_dict.get('version'), xform_dict['form']['meta'], ) if app_name is None and self.selected_app_id: continue last_sync_log = SyncLog.last_for_user(user.user_id) if last_sync_log: last_sync = last_sync_log.date if last_sync_log.build_id: build_version = get_version_from_build_id( self.domain, last_sync_log.build_id) app_version_info_from_sync = AppVersionInfo( build_version, app_version_info_from_form.commcare_version if app_version_info_from_form else None, BuildVersionSource.BUILD_ID) app_version_info_to_use = _choose_latest_version( app_version_info_from_sync, app_version_info_from_form, ) commcare_version = _get_commcare_version(app_version_info_to_use) build_version = _get_build_version(app_version_info_to_use) rows.append([ user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync), app_name or "---", build_version, commcare_version ]) return rows
def get_received_on(request): received_on = request.META.get('HTTP_X_SUBMIT_TIME') if received_on: return string_to_utc_datetime(received_on) else: return None
def _to_row(xform_dict): def _fmt_url(doc_id): if xform_dict['doc_type'] in [ "XFormInstance", "XFormArchived", "XFormError", "XFormDeprecated" ]: view_name = 'render_form_data' else: view_name = 'download_form' try: return format_html( "<a class='ajax_dialog' href='{url}'>{text}</a>", url=reverse(view_name, args=[self.domain, doc_id]), text=_("View Form")) except NoReverseMatch: return 'unable to view form' def _fmt_date(somedate): time = ServerTime(somedate).user_time(self.timezone).done() return time.strftime(SERVER_DATETIME_FORMAT) if xform_dict['form'].get('meta'): form_name = xmlns_to_name( self.domain, xform_dict.get('xmlns'), app_id=xform_dict.get('app_id'), ) form_username = xform_dict['form']['meta'].get( 'username', EMPTY_USER) else: form_name = EMPTY_FORM form_username = EMPTY_USER error_type = SubmissionTypeFilter.display_name_by_doc_type( xform_dict['doc_type']) if xform_dict['doc_type'] == "XFormArchived": archive_operations = [ operation for operation in xform_dict.get('history') if operation.get('operation') == 'archive' ] if archive_operations: error_type = _( "{username} {archived_form} on {date}").format( username=cached_user_id_to_username( archive_operations[-1].get('user')) or "", archived_form=SubmissionTypeFilter. display_name_by_doc_type(xform_dict['doc_type']), date=_fmt_date( string_to_utc_datetime( archive_operations[-1].get('date'))), ) return [ _fmt_url(xform_dict['_id']), form_username, _fmt_date(string_to_utc_datetime(xform_dict['received_on'])), form_name, error_type, xform_dict.get('problem', EMPTY_ERROR), self._make_reproces_button(xform_dict) if self.support_toggle_enabled else '', ]
def process_rows(self, users, fmt_for_export=False): rows = [] users = list(users) if self.include_location_data(): location_ids = { user['location_id'] for user in users if user['location_id'] } grouped_ancestor_locs = self.get_bulk_ancestors(location_ids) self.required_loc_columns = self.get_location_columns( grouped_ancestor_locs) for user in users: last_build = last_seen = last_sub = last_sync = last_sync_date = app_name = commcare_version = None last_build_profile_name = None build_version = _("Unknown") reporting_metadata = user.get('reporting_metadata', {}) if self.selected_app_id: last_submissions = reporting_metadata.get('last_submissions') if last_submissions: last_sub = self.get_data_for_app(last_submissions, self.selected_app_id) last_syncs = reporting_metadata.get('last_syncs') if last_syncs: last_sync = self.get_data_for_app(last_syncs, self.selected_app_id) if last_sync is None: last_sync = self.get_data_for_app(last_syncs, None) last_builds = reporting_metadata.get('last_builds') if last_builds: last_build = self.get_data_for_app(last_builds, self.selected_app_id) else: last_sub = reporting_metadata.get('last_submission_for_user', {}) last_sync = reporting_metadata.get('last_sync_for_user', {}) last_build = reporting_metadata.get('last_build_for_user', {}) if last_sub and last_sub.get('commcare_version'): commcare_version = _get_commcare_version( last_sub.get('commcare_version')) else: devices = user.get('devices', None) if devices: device = max(devices, key=lambda dev: dev['last_used']) if device.get('commcare_version', None): commcare_version = _get_commcare_version( device['commcare_version']) if last_sub and last_sub.get('submission_date'): last_seen = string_to_utc_datetime(last_sub['submission_date']) if last_sync and last_sync.get('sync_date'): last_sync_date = string_to_utc_datetime(last_sync['sync_date']) if last_build: build_version = last_build.get( 'build_version') or build_version if last_build.get('app_id'): app_name = self.get_app_name(last_build['app_id']) if self.show_build_profile: last_build_profile_id = last_build.get('build_profile_id') if last_build_profile_id: last_build_profile_name = _("Unknown") build_profiles = self._get_app_details( last_build['app_id']).get('build_profiles', {}) if last_build_profile_id in build_profiles: last_build_profile_name = build_profiles[ last_build_profile_id] row_data = [ user_display_string(user.get('username', ''), user.get('first_name', ''), user.get('last_name', '')), _fmt_date(last_seen, fmt_for_export), _fmt_date(last_sync_date, fmt_for_export), app_name or "---", build_version, commcare_version or '---' ] if self.show_build_profile: row_data.append(last_build_profile_name) if self.include_location_data(): location_data = self.user_locations( grouped_ancestor_locs.get(user['location_id'], []), self.required_loc_columns) row_data = location_data + row_data rows.append(row_data) return rows
def guess_modified_on(self): """ Guess the modified date, defaulting to the current time in UTC. """ return string_to_utc_datetime(self.modified_on_str) if self.modified_on_str else datetime.datetime.utcnow()
def from_json(value): return string_to_utc_datetime(value) if value is not None else None