def rows(self): rows = [] user_ids = map(lambda user: user.user_id, self.users) user_xform_dicts_map = get_last_form_submissions_by_user( self.domain, user_ids, self.selected_app_id) for user in self.users: xform_dict = last_seen = last_sync = app_name = None if user_xform_dicts_map.get(user.user_id): xform_dict = user_xform_dicts_map[user.user_id][0] if xform_dict: last_seen = string_to_utc_datetime( xform_dict.get('received_on')) if xform_dict.get('app_id'): try: app = get_app(self.domain, xform_dict.get('app_id')) except ResourceNotFound: pass else: app_name = app.name else: app_name = get_meta_appversion_text( xform_dict['form']['meta']) app_version_info = get_app_version_info( self.domain, xform_dict.get('build_id'), xform_dict.get('version'), xform_dict['form']['meta'], ) build_html = _build_html(app_version_info) commcare_version = ('CommCare {}'.format( app_version_info.commcare_version) if app_version_info.commcare_version else _("Unknown CommCare Version")) commcare_version_html = mark_safe( '<span class="label label-info">{}</span>'.format( commcare_version)) app_name = app_name or _("Unknown App") app_name = format_html(u'{} {} {}', app_name, mark_safe(build_html), commcare_version_html) if app_name is None and self.selected_app_id: continue last_sync_log = SyncLog.last_for_user(user.user_id) if last_sync_log: last_sync = last_sync_log.date rows.append([ user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync), app_name or "---" ]) return rows
def mark_latest_submission(domain, user_id, app_id, build_id, version, metadata, received_on): user = CouchUser.get_by_user_id(user_id, domain) if not user or user.is_deleted(): return try: received_on_datetime = string_to_utc_datetime(received_on) except ValueError: return last_submission = filter_by_app(user.reporting_metadata.last_submissions, app_id) if metadata and metadata.get('appVersion'): if not isinstance(metadata['appVersion'], str): metadata = format_form_meta_for_es(metadata) app_version_info = get_app_version_info( domain, build_id, version, metadata ) if _last_submission_needs_update(last_submission, received_on_datetime, app_version_info.build_version, app_version_info.commcare_version): if last_submission is None: last_submission = LastSubmission() user.reporting_metadata.last_submissions.append(last_submission) last_submission.submission_date = received_on_datetime device_id = metadata.get('deviceID') last_submission.device_id = device_id last_submission.app_id = app_id last_submission.build_id = build_id last_submission.build_version = app_version_info.build_version last_submission.commcare_version = app_version_info.commcare_version if app_version_info.build_version: update_latest_builds(user, app_id, received_on_datetime, app_version_info.build_version) if _last_submission_needs_update(user.reporting_metadata.last_submission_for_user, received_on_datetime, app_version_info.build_version, app_version_info.commcare_version, False): user.reporting_metadata.last_submission_for_user = last_submission app_meta = DeviceAppMeta( app_id=app_id, build_id=build_id, last_submission=received_on_datetime, ) update_device_meta(user, device_id, app_version_info.commcare_version, app_meta, save=False) user.save()
def mark_latest_submission(domain, user_id, app_id, build_id, version, metadata, received_on): user = CouchUser.get_by_user_id(user_id, domain) if not user or user.is_deleted(): return try: received_on_datetime = string_to_utc_datetime(received_on) except ValueError: return last_submission = filter_by_app(user.reporting_metadata.last_submissions, app_id) if metadata and metadata.get('appVersion') and not isinstance(metadata['appVersion'], six.string_types): metadata = format_form_meta_for_es(metadata) app_version_info = get_app_version_info( domain, build_id, version, metadata ) if _last_submission_needs_update(last_submission, received_on_datetime, app_version_info.build_version, app_version_info.commcare_version): if last_submission is None: last_submission = LastSubmission() user.reporting_metadata.last_submissions.append(last_submission) last_submission.submission_date = received_on_datetime device_id = metadata.get('deviceID') last_submission.device_id = device_id last_submission.app_id = app_id last_submission.build_id = build_id last_submission.build_version = app_version_info.build_version last_submission.commcare_version = app_version_info.commcare_version if app_version_info.build_version: update_latest_builds(user, app_id, received_on_datetime, app_version_info.build_version) if _last_submission_needs_update(user.reporting_metadata.last_submission_for_user, received_on_datetime, app_version_info.build_version, app_version_info.commcare_version, False): user.reporting_metadata.last_submission_for_user = last_submission app_meta = DeviceAppMeta( app_id=app_id, build_id=build_id, last_submission=received_on_datetime, ) update_device_meta(user, device_id, app_version_info.commcare_version, app_meta, save=False) user.save()
def rows(self): rows = [] user_ids = map(lambda user: user.user_id, self.users) user_xform_dicts_map = get_last_form_submissions_by_user(self.domain, user_ids, self.selected_app_id) for user in self.users: xform_dict = last_seen = last_sync = app_name = None app_version_info_from_form = app_version_info_from_sync = None if user_xform_dicts_map.get(user.user_id): xform_dict = user_xform_dicts_map[user.user_id][0] if xform_dict: last_seen = string_to_utc_datetime(xform_dict.get('received_on')) if xform_dict.get('app_id'): try: app = get_app(self.domain, xform_dict.get('app_id')) except ResourceNotFound: pass else: app_name = app.name else: app_name = get_meta_appversion_text(xform_dict['form']['meta']) app_version_info_from_form = get_app_version_info( self.domain, xform_dict.get('build_id'), xform_dict.get('version'), xform_dict['form']['meta'], ) if app_name is None and self.selected_app_id: continue last_sync_log = SyncLog.last_for_user(user.user_id) if last_sync_log: last_sync = last_sync_log.date if last_sync_log.build_id: build_version = get_version_from_build_id(self.domain, last_sync_log.build_id) app_version_info_from_sync = AppVersionInfo( build_version, app_version_info_from_form.commcare_version if app_version_info_from_form else None, BuildVersionSource.BUILD_ID ) app_version_info_to_use = _choose_latest_version( app_version_info_from_sync, app_version_info_from_form, ) commcare_version = _get_commcare_version(app_version_info_to_use) build_version = _get_build_version(app_version_info_to_use) rows.append([ user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync), app_name or "---", build_version, commcare_version ]) return rows
def rows(self): rows = [] selected_app = self.request_params.get(SelectApplicationFilter.slug, None) user_ids = map(lambda user: user.user_id, self.users) user_xform_dicts_map = get_last_form_submissions_by_user(self.domain, user_ids, selected_app) for user in self.users: xform_dict = last_seen = last_sync = app_name = None if user_xform_dicts_map.get(user.user_id): xform_dict = user_xform_dicts_map[user.user_id][0] if xform_dict: last_seen = string_to_utc_datetime(xform_dict.get('received_on')) if xform_dict.get('app_id'): try: app = get_app(self.domain, xform_dict.get('app_id')) except ResourceNotFound: pass else: app_name = app.name else: app_name = get_meta_appversion_text(xform_dict['form']['meta']) app_version_info = get_app_version_info( self.domain, xform_dict.get('build_id'), xform_dict.get('version'), xform_dict['form']['meta'], ) build_html = _build_html(app_version_info) commcare_version = ( 'CommCare {}'.format(app_version_info.commcare_version) if app_version_info.commcare_version else _("Unknown CommCare Version") ) commcare_version_html = mark_safe('<span class="label label-info">{}</span>'.format( commcare_version) ) app_name = app_name or _("Unknown App") app_name = format_html( u'{} {} {}', app_name, mark_safe(build_html), commcare_version_html ) if app_name is None and selected_app: continue last_sync_log = SyncLog.last_for_user(user.user_id) if last_sync_log: last_sync = last_sync_log.date rows.append( [user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync), app_name or "---"] ) return rows
def rows(self): rows = [] selected_app = self.request_params.get(SelectApplicationFilter.slug, None) for user in self.users: last_seen = last_sync = app_name = None xform = get_last_form_submission_for_user_for_app( self.domain, user.user_id, selected_app) if xform: last_seen = xform.received_on if xform.app_id: try: app = get_app(self.domain, xform.app_id) except ResourceNotFound: pass else: app_name = app.name else: app_name = get_meta_appversion_text(xform) app_version_info = get_app_version_info(xform) build_html = _build_html(app_version_info) commcare_version = ( 'CommCare {}'.format(app_version_info.commcare_version) if app_version_info.commcare_version else _("Unknown CommCare Version") ) commcare_version_html = mark_safe('<span class="label label-info">{}</span>'.format( commcare_version) ) app_name = app_name or _("Unknown App") app_name = format_html( u'{} {} {}', app_name, mark_safe(build_html), commcare_version_html ) if app_name is None and selected_app: continue last_sync_log = SyncLog.last_for_user(user.user_id) if last_sync_log: last_sync = last_sync_log.date rows.append( [user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync), app_name or "---"] ) return rows
def handle(self, *args, **options): domain = options["domain"] case_id = options["case_id"] case_accessor = CaseAccessors(domain=domain) case = case_accessor.get_case(case_id) if ( not case.is_deleted and raw_input( "\n".join(["Case {} is not already deleted. Are you sure you want to delete it? (y/N)".format(case_id)]) ).lower() != "y" ): sys.exit(0) dependent_case_ids = get_entire_case_network(domain, [case_id]) cases_to_delete = filter(lambda case: not case.is_deleted, case_accessor.get_cases(dependent_case_ids)) if cases_to_delete: with open(options["filename"], "w") as csvfile: writer = csv.writer(csvfile) headers = ["case id", "case type", "owner", "opened by", "app version"] writer.writerow(headers) print headers for case in cases_to_delete: form = FormAccessors(domain=domain).get_form(case.xform_ids[0]) app_version_info = get_app_version_info( domain, form.build_id, form.form_data["@version"], form.metadata ) row = [ case.case_id, case.type, cached_owner_id_to_display(case.owner_id) or case.owner_id, cached_owner_id_to_display(case.opened_by), app_version_info.build_version, ] writer.writerow(row) print row if ( cases_to_delete and raw_input("\n".join(["Delete these {} cases? (y/N)".format(len(cases_to_delete))])).lower() == "y" ): case_accessor.soft_delete_cases([c.case_id for c in cases_to_delete]) print "deleted {} cases".format(len(cases_to_delete)) if cases_to_delete: print "details here: {}".format(options["filename"]) else: print "didn't find any cases to delete"
def mark_latest_submission(domain, user_id, app_id, build_id, version, metadata, received_on): user = CouchUser.get_by_user_id(user_id, domain) if not user or user.is_deleted(): return try: received_on_datetime = string_to_utc_datetime(received_on) except ValueError: return last_submissions = filter( lambda submission: submission.app_id == app_id, user.reporting_metadata.last_submissions, ) if last_submissions: assert len(last_submissions ) == 1, 'Must only have one last submission per app' last_submission = last_submissions[0] else: last_submission = None app_version_info = get_app_version_info(domain, build_id, version, metadata) if last_submission is None or last_submission.submission_date < received_on_datetime: if last_submission is None: last_submission = LastSubmission() user.reporting_metadata.last_submissions.append(last_submission) last_submission.submission_date = received_on_datetime last_submission.device_id = metadata.get('deviceID') last_submission.app_id = app_id last_submission.build_id = build_id last_submission.build_version = app_version_info.build_version last_submission.commcare_version = app_version_info.commcare_version user.save()
def handle(self, **options): domain = options['domain'] debug = options['debug'] cleanup = options['cleanup'] domain_query = CaseES().domain(domain) valid_case_ids = set(domain_query.get_ids()) referenced_case_ids = { index['referenced_id'] for hit in domain_query.source('indices.referenced_id').run().hits for index in hit['indices'] } invalid_referenced_ids = referenced_case_ids - valid_case_ids if len(invalid_referenced_ids) > ES_MAX_CLAUSE_COUNT: print( "there's a lot of invalid ids here. ES queries may not handle this well" ) cases_with_invalid_references = (domain_query.term( 'indices.referenced_id', invalid_referenced_ids).source([ '_id', 'type', 'indices', 'owner_id', 'opened_by', 'xform_ids' ]).run().hits) with open(options['filename'], 'w', encoding='utf-8') as csvfile: writer = csv.writer(csvfile) headers = [ 'case id', 'case type', 'creating form id', 'referenced id', 'referenced_type', 'index relationship', 'index identifier', 'owner id', 'owner name', 'opened by id', 'opened by name', ] if debug: headers.append('app version') writer.writerow(headers) for case in cases_with_invalid_references: for index in case['indices']: if index['referenced_id'] in invalid_referenced_ids: form_id = case['xform_ids'][0] row = [ case['_id'], case['type'], form_id, index['referenced_id'], index['referenced_type'], index['relationship'], index['identifier'], case['owner_id'], cached_owner_id_to_display(case['owner_id']), case['opened_by'], cached_owner_id_to_display(case['opened_by']), ] if debug: form = FormAccessors( domain=domain).get_form(form_id) app_version_info = get_app_version_info( domain, form.build_id, form.form_data['@version'], form.metadata, ) row.append(app_version_info.build_version) writer.writerow(row) if cleanup: missing = set() deleted = set() exists = set() for invalid_id in invalid_referenced_ids: try: case = CaseAccessors(domain).get_case(invalid_id) except CaseNotFound: missing.add(invalid_id) else: if case.is_deleted: deleted.add(case) else: exists.add(case)
def transform_xform_for_elasticsearch(doc_dict): """ Given an XFormInstance, return a copy that is ready to be sent to elasticsearch, or None, if the form should not be saved to elasticsearch """ doc_ret = copy.deepcopy(doc_dict) if 'meta' in doc_ret['form']: if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)): doc_ret['form']['meta']['timeEnd'] = None if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)): doc_ret['form']['meta']['timeStart'] = None # Some docs have their @xmlns and #text here if isinstance(doc_ret['form']['meta'].get('appVersion'), dict): doc_ret['form']['meta'] = format_form_meta_for_es(doc_ret['form']['meta']) app_version_info = get_app_version_info( doc_ret['domain'], doc_ret.get('build_id'), doc_ret.get('version'), doc_ret['form']['meta'], ) doc_ret['form']['meta']['commcare_version'] = app_version_info.commcare_version doc_ret['form']['meta']['app_build_version'] = app_version_info.build_version try: geo_point = GeoPointProperty().wrap(doc_ret['form']['meta']['location']) doc_ret['form']['meta']['geo_point'] = geo_point.lat_lon except (KeyError, BadValueError): doc_ret['form']['meta']['geo_point'] = None pass try: user_id = doc_ret['form']['meta']['userID'] except KeyError: user_id = None doc_ret['user_type'] = get_user_type(user_id) doc_ret['inserted_at'] = datetime.datetime.utcnow().isoformat() try: case_blocks = extract_case_blocks(doc_ret) except PhoneDateValueError: pass else: for case_dict in case_blocks: for date_modified_key in ['date_modified', '@date_modified']: if not is_valid_date(case_dict.get(date_modified_key, None)): if case_dict.get(date_modified_key) == '': case_dict[date_modified_key] = None else: case_dict.pop(date_modified_key, None) # convert all mapped dict properties to nulls if they are empty strings for object_key in ['index', 'attachment', 'create', 'update']: if object_key in case_dict and not isinstance(case_dict[object_key], dict): case_dict[object_key] = None try: doc_ret["__retrieved_case_ids"] = list(set(case_update_from_block(cb).id for cb in case_blocks)) except CaseGenerationException: doc_ret["__retrieved_case_ids"] = [] if 'backend_id' not in doc_ret: doc_ret['backend_id'] = 'couch' return doc_ret
def transform_xform_for_elasticsearch(doc_dict): """ Given an XFormInstance, return a copy that is ready to be sent to elasticsearch, or None, if the form should not be saved to elasticsearch """ doc_ret = copy.deepcopy(doc_dict) if 'meta' in doc_ret['form']: if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)): doc_ret['form']['meta']['timeEnd'] = None if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)): doc_ret['form']['meta']['timeStart'] = None # Some docs have their @xmlns and #text here if isinstance(doc_ret['form']['meta'].get('appVersion'), dict): doc_ret['form']['meta'] = format_form_meta_for_es( doc_ret['form']['meta']) app_version_info = get_app_version_info( doc_ret['domain'], doc_ret.get('build_id'), doc_ret.get('version'), doc_ret['form']['meta'], ) doc_ret['form']['meta'][ 'commcare_version'] = app_version_info.commcare_version doc_ret['form']['meta'][ 'app_build_version'] = app_version_info.build_version try: geo_point = GeoPointProperty().wrap( doc_ret['form']['meta']['location']) doc_ret['form']['meta']['geo_point'] = geo_point.lat_lon except (KeyError, BadValueError): doc_ret['form']['meta']['geo_point'] = None pass try: user_id = doc_ret['form']['meta']['userID'] except KeyError: user_id = None doc_ret['user_type'] = get_user_type(user_id) doc_ret['inserted_at'] = datetime.datetime.utcnow().isoformat() try: case_blocks = extract_case_blocks(doc_ret) except PhoneDateValueError: pass else: for case_dict in case_blocks: for date_modified_key in ['date_modified', '@date_modified']: if not is_valid_date(case_dict.get(date_modified_key, None)): if case_dict.get(date_modified_key) == '': case_dict[date_modified_key] = None else: case_dict.pop(date_modified_key, None) # convert all mapped dict properties to nulls if they are empty strings for object_key in ['index', 'attachment', 'create', 'update']: if object_key in case_dict and not isinstance( case_dict[object_key], dict): case_dict[object_key] = None try: doc_ret["__retrieved_case_ids"] = list( set(case_update_from_block(cb).id for cb in case_blocks)) except CaseGenerationException: doc_ret["__retrieved_case_ids"] = [] if 'backend_id' not in doc_ret: doc_ret['backend_id'] = 'couch' return doc_ret
class Command(BaseCommand): help = "Delete all cases that are in a specific case's network/footprint" def add_arguments(self, parser): parser.add_argument('domain', type=unicode) parser.add_argument('case_id', type=unicode) parser.add_argument('--filename', dest='filename', default='case-delete-info.csv') def handle(self, domain, case_id, **options): case_accessor = CaseAccessors(domain=domain) case = case_accessor.get_case(case_id) if not case.is_deleted and raw_input('\n'.join([ 'Case {} is not already deleted. Are you sure you want to delete it? (y/N)' .format(case_id) ])).lower() != 'y': sys.exit(0) dependent_case_ids = get_entire_case_network(domain, [case_id]) cases_to_delete = filter(lambda case: not case.is_deleted, case_accessor.get_cases(dependent_case_ids)) if cases_to_delete: with open(options['filename'], 'w') as csvfile: writer = csv.writer(csvfile) headers = [ 'case id', 'case type', 'owner', 'opened by', 'app version', ] writer.writerow(headers) print(headers) for case in cases_to_delete: form = FormAccessors(domain=domain).get_form( case.xform_ids[0]) app_version_info = get_app_version_info( domain, form.build_id, form.form_data['@version'], form.metadata, ) row = [ case.case_id, case.type, cached_owner_id_to_display(case.owner_id) or case.owner_id, cached_owner_id_to_display(case.opened_by), app_version_info.build_version, ] writer.writerow(row) print(row) if cases_to_delete and raw_input('\n'.join([ 'Delete these {} cases? (y/N)'.format(len(cases_to_delete)), ])).lower() == 'y': case_accessor.soft_delete_cases( [c.case_id for c in cases_to_delete]) print('deleted {} cases'.format(len(cases_to_delete))) if cases_to_delete: print('details here: {}'.format(options['filename'])) else: print("didn't find any cases to delete")
headers = [ 'case id', 'case type', 'owner', 'opened by', 'app version', ] writer.writerow(headers) print(headers) for case in cases_to_delete: form = FormAccessors(domain=domain).get_form( case.xform_ids[0]) app_version_info = get_app_version_info( domain, form.build_id, form.form_data['@version'], form.metadata, ) row = [ case.case_id, case.type, cached_owner_id_to_display(case.owner_id) or case.owner_id, cached_owner_id_to_display(case.opened_by), app_version_info.build_version, ] writer.writerow(row) print(row) if cases_to_delete and input('\n'.join([ 'Delete these {} cases? (y/N)'.format(len(cases_to_delete)),
def handle(self, **options): domain = options['domain'] debug = options['debug'] cleanup = options['cleanup'] domain_query = CaseES().domain(domain) valid_case_ids = set(domain_query.get_ids()) referenced_case_ids = { index['referenced_id'] for hit in domain_query.source('indices.referenced_id').run().hits for index in hit['indices'] } invalid_referenced_ids = referenced_case_ids - valid_case_ids if len(invalid_referenced_ids) > ES_MAX_CLAUSE_COUNT: print("there's a lot of invalid ids here. ES queries may not handle this well") cases_with_invalid_references = ( domain_query .term('indices.referenced_id', invalid_referenced_ids) .source(['_id', 'type', 'indices', 'owner_id', 'opened_by', 'xform_ids']) .run().hits ) with open(options['filename'], 'w', encoding='utf-8') as csvfile: writer = csv.writer(csvfile) headers = [ 'case id', 'case type', 'creating form id', 'referenced id', 'referenced_type', 'index relationship', 'index identifier', 'owner id', 'owner name', 'opened by id', 'opened by name', ] if debug: headers.append('app version') writer.writerow(headers) for case in cases_with_invalid_references: for index in case['indices']: if index['referenced_id'] in invalid_referenced_ids: form_id = case['xform_ids'][0] row = [ case['_id'], case['type'], form_id, index['referenced_id'], index['referenced_type'], index['relationship'], index['identifier'], case['owner_id'], cached_owner_id_to_display(case['owner_id']), case['opened_by'], cached_owner_id_to_display(case['opened_by']), ] if debug: form = FormAccessors(domain=domain).get_form(form_id) app_version_info = get_app_version_info( domain, form.build_id, form.form_data['@version'], form.metadata, ) row.append(app_version_info.build_version) writer.writerow(row) if cleanup: missing = set() deleted = set() exists = set() for invalid_id in invalid_referenced_ids: try: case = CaseAccessors(domain).get_case(invalid_id) except CaseNotFound: missing.add(invalid_id) else: if case.is_deleted: deleted.add(case) else: exists.add(case) for case_to_resync in exists: # if the case actually exists resync it to fix the es search resave_case(domain, case_to_resync, send_post_save_signal=False) if exists: print('resynced {} cases that were actually not deleted'.format(len(exists))) for case in deleted: # delete the deleted case's entire network in one go call_command('delete_related_cases', domain, case.case_id) for case in cases_with_invalid_references: for index in case['indices']: if index['referenced_id'] in missing: # this is just an invalid reference. no recourse but to delete the case itself call_command('delete_related_cases', domain, case['_id'])