def rows(self):
rows = []
user_ids = map(lambda user: user.user_id, self.users)
user_xform_dicts_map = get_last_form_submissions_by_user(
self.domain, user_ids, self.selected_app_id)
for user in self.users:
xform_dict = last_seen = last_sync = app_name = None
if user_xform_dicts_map.get(user.user_id):
xform_dict = user_xform_dicts_map[user.user_id][0]
if xform_dict:
last_seen = string_to_utc_datetime(
xform_dict.get('received_on'))
if xform_dict.get('app_id'):
try:
app = get_app(self.domain, xform_dict.get('app_id'))
except ResourceNotFound:
pass
else:
app_name = app.name
else:
app_name = get_meta_appversion_text(
xform_dict['form']['meta'])
app_version_info = get_app_version_info(
self.domain,
xform_dict.get('build_id'),
xform_dict.get('version'),
xform_dict['form']['meta'],
)
build_html = _build_html(app_version_info)
commcare_version = ('CommCare {}'.format(
app_version_info.commcare_version)
if app_version_info.commcare_version else
_("Unknown CommCare Version"))
commcare_version_html = mark_safe(
'<span class="label label-info">{}</span>'.format(
commcare_version))
app_name = app_name or _("Unknown App")
app_name = format_html(u'{} {} {}', app_name,
mark_safe(build_html),
commcare_version_html)
if app_name is None and self.selected_app_id:
continue
last_sync_log = SyncLog.last_for_user(user.user_id)
if last_sync_log:
last_sync = last_sync_log.date
rows.append([
user.username_in_report,
_fmt_date(last_seen),
_fmt_date(last_sync), app_name or "---"
])
return rows
def mark_latest_submission(domain, user_id, app_id, build_id, version, metadata, received_on):
user = CouchUser.get_by_user_id(user_id, domain)
if not user or user.is_deleted():
return
try:
received_on_datetime = string_to_utc_datetime(received_on)
except ValueError:
return
last_submission = filter_by_app(user.reporting_metadata.last_submissions, app_id)
if metadata and metadata.get('appVersion'):
if not isinstance(metadata['appVersion'], str):
metadata = format_form_meta_for_es(metadata)
app_version_info = get_app_version_info(
domain,
build_id,
version,
metadata
)
if _last_submission_needs_update(last_submission,
received_on_datetime,
app_version_info.build_version,
app_version_info.commcare_version):
if last_submission is None:
last_submission = LastSubmission()
user.reporting_metadata.last_submissions.append(last_submission)
last_submission.submission_date = received_on_datetime
device_id = metadata.get('deviceID')
last_submission.device_id = device_id
last_submission.app_id = app_id
last_submission.build_id = build_id
last_submission.build_version = app_version_info.build_version
last_submission.commcare_version = app_version_info.commcare_version
if app_version_info.build_version:
update_latest_builds(user, app_id, received_on_datetime, app_version_info.build_version)
if _last_submission_needs_update(user.reporting_metadata.last_submission_for_user,
received_on_datetime,
app_version_info.build_version,
app_version_info.commcare_version,
False):
user.reporting_metadata.last_submission_for_user = last_submission
app_meta = DeviceAppMeta(
app_id=app_id,
build_id=build_id,
last_submission=received_on_datetime,
)
update_device_meta(user, device_id, app_version_info.commcare_version, app_meta, save=False)
user.save()
def mark_latest_submission(domain, user_id, app_id, build_id, version, metadata, received_on):
user = CouchUser.get_by_user_id(user_id, domain)
if not user or user.is_deleted():
return
try:
received_on_datetime = string_to_utc_datetime(received_on)
except ValueError:
return
last_submission = filter_by_app(user.reporting_metadata.last_submissions, app_id)
if metadata and metadata.get('appVersion') and not isinstance(metadata['appVersion'], six.string_types):
metadata = format_form_meta_for_es(metadata)
app_version_info = get_app_version_info(
domain,
build_id,
version,
metadata
)
if _last_submission_needs_update(last_submission,
received_on_datetime,
app_version_info.build_version,
app_version_info.commcare_version):
if last_submission is None:
last_submission = LastSubmission()
user.reporting_metadata.last_submissions.append(last_submission)
last_submission.submission_date = received_on_datetime
device_id = metadata.get('deviceID')
last_submission.device_id = device_id
last_submission.app_id = app_id
last_submission.build_id = build_id
last_submission.build_version = app_version_info.build_version
last_submission.commcare_version = app_version_info.commcare_version
if app_version_info.build_version:
update_latest_builds(user, app_id, received_on_datetime, app_version_info.build_version)
if _last_submission_needs_update(user.reporting_metadata.last_submission_for_user,
received_on_datetime,
app_version_info.build_version,
app_version_info.commcare_version,
False):
user.reporting_metadata.last_submission_for_user = last_submission
app_meta = DeviceAppMeta(
app_id=app_id,
build_id=build_id,
last_submission=received_on_datetime,
)
update_device_meta(user, device_id, app_version_info.commcare_version, app_meta, save=False)
user.save()
def rows(self):
rows = []
user_ids = map(lambda user: user.user_id, self.users)
user_xform_dicts_map = get_last_form_submissions_by_user(self.domain, user_ids, self.selected_app_id)
for user in self.users:
xform_dict = last_seen = last_sync = app_name = None
app_version_info_from_form = app_version_info_from_sync = None
if user_xform_dicts_map.get(user.user_id):
xform_dict = user_xform_dicts_map[user.user_id][0]
if xform_dict:
last_seen = string_to_utc_datetime(xform_dict.get('received_on'))
if xform_dict.get('app_id'):
try:
app = get_app(self.domain, xform_dict.get('app_id'))
except ResourceNotFound:
pass
else:
app_name = app.name
else:
app_name = get_meta_appversion_text(xform_dict['form']['meta'])
app_version_info_from_form = get_app_version_info(
self.domain,
xform_dict.get('build_id'),
xform_dict.get('version'),
xform_dict['form']['meta'],
)
if app_name is None and self.selected_app_id:
continue
last_sync_log = SyncLog.last_for_user(user.user_id)
if last_sync_log:
last_sync = last_sync_log.date
if last_sync_log.build_id:
build_version = get_version_from_build_id(self.domain, last_sync_log.build_id)
app_version_info_from_sync = AppVersionInfo(
build_version,
app_version_info_from_form.commcare_version if app_version_info_from_form else None,
BuildVersionSource.BUILD_ID
)
app_version_info_to_use = _choose_latest_version(
app_version_info_from_sync, app_version_info_from_form,
)
commcare_version = _get_commcare_version(app_version_info_to_use)
build_version = _get_build_version(app_version_info_to_use)
rows.append([
user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync),
app_name or "---", build_version, commcare_version
])
return rows
def rows(self):
rows = []
selected_app = self.request_params.get(SelectApplicationFilter.slug, None)
user_ids = map(lambda user: user.user_id, self.users)
user_xform_dicts_map = get_last_form_submissions_by_user(self.domain, user_ids, selected_app)
for user in self.users:
xform_dict = last_seen = last_sync = app_name = None
if user_xform_dicts_map.get(user.user_id):
xform_dict = user_xform_dicts_map[user.user_id][0]
if xform_dict:
last_seen = string_to_utc_datetime(xform_dict.get('received_on'))
if xform_dict.get('app_id'):
try:
app = get_app(self.domain, xform_dict.get('app_id'))
except ResourceNotFound:
pass
else:
app_name = app.name
else:
app_name = get_meta_appversion_text(xform_dict['form']['meta'])
app_version_info = get_app_version_info(
self.domain,
xform_dict.get('build_id'),
xform_dict.get('version'),
xform_dict['form']['meta'],
)
build_html = _build_html(app_version_info)
commcare_version = (
'CommCare {}'.format(app_version_info.commcare_version)
if app_version_info.commcare_version
else _("Unknown CommCare Version")
)
commcare_version_html = mark_safe('<span class="label label-info">{}</span>'.format(
commcare_version)
)
app_name = app_name or _("Unknown App")
app_name = format_html(
u'{} {} {}', app_name, mark_safe(build_html), commcare_version_html
)
if app_name is None and selected_app:
continue
last_sync_log = SyncLog.last_for_user(user.user_id)
if last_sync_log:
last_sync = last_sync_log.date
rows.append(
[user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync), app_name or "---"]
)
return rows
def rows(self):
rows = []
selected_app = self.request_params.get(SelectApplicationFilter.slug, None)
for user in self.users:
last_seen = last_sync = app_name = None
xform = get_last_form_submission_for_user_for_app(
self.domain, user.user_id, selected_app)
if xform:
last_seen = xform.received_on
if xform.app_id:
try:
app = get_app(self.domain, xform.app_id)
except ResourceNotFound:
pass
else:
app_name = app.name
else:
app_name = get_meta_appversion_text(xform)
app_version_info = get_app_version_info(xform)
build_html = _build_html(app_version_info)
commcare_version = (
'CommCare {}'.format(app_version_info.commcare_version)
if app_version_info.commcare_version
else _("Unknown CommCare Version")
)
commcare_version_html = mark_safe('<span class="label label-info">{}</span>'.format(
commcare_version)
)
app_name = app_name or _("Unknown App")
app_name = format_html(
u'{} {} {}', app_name, mark_safe(build_html), commcare_version_html
)
if app_name is None and selected_app:
continue
last_sync_log = SyncLog.last_for_user(user.user_id)
if last_sync_log:
last_sync = last_sync_log.date
rows.append(
[user.username_in_report, _fmt_date(last_seen), _fmt_date(last_sync), app_name or "---"]
)
return rows
def handle(self, *args, **options):
domain = options["domain"]
case_id = options["case_id"]
case_accessor = CaseAccessors(domain=domain)
case = case_accessor.get_case(case_id)
if (
not case.is_deleted
and raw_input(
"\n".join(["Case {} is not already deleted. Are you sure you want to delete it? (y/N)".format(case_id)])
).lower()
!= "y"
):
sys.exit(0)
dependent_case_ids = get_entire_case_network(domain, [case_id])
cases_to_delete = filter(lambda case: not case.is_deleted, case_accessor.get_cases(dependent_case_ids))
if cases_to_delete:
with open(options["filename"], "w") as csvfile:
writer = csv.writer(csvfile)
headers = ["case id", "case type", "owner", "opened by", "app version"]
writer.writerow(headers)
print headers
for case in cases_to_delete:
form = FormAccessors(domain=domain).get_form(case.xform_ids[0])
app_version_info = get_app_version_info(
domain, form.build_id, form.form_data["@version"], form.metadata
)
row = [
case.case_id,
case.type,
cached_owner_id_to_display(case.owner_id) or case.owner_id,
cached_owner_id_to_display(case.opened_by),
app_version_info.build_version,
]
writer.writerow(row)
print row
if (
cases_to_delete
and raw_input("\n".join(["Delete these {} cases? (y/N)".format(len(cases_to_delete))])).lower() == "y"
):
case_accessor.soft_delete_cases([c.case_id for c in cases_to_delete])
print "deleted {} cases".format(len(cases_to_delete))
if cases_to_delete:
print "details here: {}".format(options["filename"])
else:
print "didn't find any cases to delete"
def mark_latest_submission(domain, user_id, app_id, build_id, version,
metadata, received_on):
user = CouchUser.get_by_user_id(user_id, domain)
if not user or user.is_deleted():
return
try:
received_on_datetime = string_to_utc_datetime(received_on)
except ValueError:
return
last_submissions = filter(
lambda submission: submission.app_id == app_id,
user.reporting_metadata.last_submissions,
)
if last_submissions:
assert len(last_submissions
) == 1, 'Must only have one last submission per app'
last_submission = last_submissions[0]
else:
last_submission = None
app_version_info = get_app_version_info(domain, build_id, version,
metadata)
if last_submission is None or last_submission.submission_date < received_on_datetime:
if last_submission is None:
last_submission = LastSubmission()
user.reporting_metadata.last_submissions.append(last_submission)
last_submission.submission_date = received_on_datetime
last_submission.device_id = metadata.get('deviceID')
last_submission.app_id = app_id
last_submission.build_id = build_id
last_submission.build_version = app_version_info.build_version
last_submission.commcare_version = app_version_info.commcare_version
user.save()
def handle(self, **options):
domain = options['domain']
debug = options['debug']
cleanup = options['cleanup']
domain_query = CaseES().domain(domain)
valid_case_ids = set(domain_query.get_ids())
referenced_case_ids = {
index['referenced_id']
for hit in domain_query.source('indices.referenced_id').run().hits
for index in hit['indices']
}
invalid_referenced_ids = referenced_case_ids - valid_case_ids
if len(invalid_referenced_ids) > ES_MAX_CLAUSE_COUNT:
print(
"there's a lot of invalid ids here. ES queries may not handle this well"
)
cases_with_invalid_references = (domain_query.term(
'indices.referenced_id', invalid_referenced_ids).source([
'_id', 'type', 'indices', 'owner_id', 'opened_by', 'xform_ids'
]).run().hits)
with open(options['filename'], 'w', encoding='utf-8') as csvfile:
writer = csv.writer(csvfile)
headers = [
'case id',
'case type',
'creating form id',
'referenced id',
'referenced_type',
'index relationship',
'index identifier',
'owner id',
'owner name',
'opened by id',
'opened by name',
]
if debug:
headers.append('app version')
writer.writerow(headers)
for case in cases_with_invalid_references:
for index in case['indices']:
if index['referenced_id'] in invalid_referenced_ids:
form_id = case['xform_ids'][0]
row = [
case['_id'],
case['type'],
form_id,
index['referenced_id'],
index['referenced_type'],
index['relationship'],
index['identifier'],
case['owner_id'],
cached_owner_id_to_display(case['owner_id']),
case['opened_by'],
cached_owner_id_to_display(case['opened_by']),
]
if debug:
form = FormAccessors(
domain=domain).get_form(form_id)
app_version_info = get_app_version_info(
domain,
form.build_id,
form.form_data['@version'],
form.metadata,
)
row.append(app_version_info.build_version)
writer.writerow(row)
if cleanup:
missing = set()
deleted = set()
exists = set()
for invalid_id in invalid_referenced_ids:
try:
case = CaseAccessors(domain).get_case(invalid_id)
except CaseNotFound:
missing.add(invalid_id)
else:
if case.is_deleted:
deleted.add(case)
else:
exists.add(case)
def transform_xform_for_elasticsearch(doc_dict):
"""
Given an XFormInstance, return a copy that is ready to be sent to elasticsearch,
or None, if the form should not be saved to elasticsearch
"""
doc_ret = copy.deepcopy(doc_dict)
if 'meta' in doc_ret['form']:
if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)):
doc_ret['form']['meta']['timeEnd'] = None
if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)):
doc_ret['form']['meta']['timeStart'] = None
# Some docs have their @xmlns and #text here
if isinstance(doc_ret['form']['meta'].get('appVersion'), dict):
doc_ret['form']['meta'] = format_form_meta_for_es(doc_ret['form']['meta'])
app_version_info = get_app_version_info(
doc_ret['domain'],
doc_ret.get('build_id'),
doc_ret.get('version'),
doc_ret['form']['meta'],
)
doc_ret['form']['meta']['commcare_version'] = app_version_info.commcare_version
doc_ret['form']['meta']['app_build_version'] = app_version_info.build_version
try:
geo_point = GeoPointProperty().wrap(doc_ret['form']['meta']['location'])
doc_ret['form']['meta']['geo_point'] = geo_point.lat_lon
except (KeyError, BadValueError):
doc_ret['form']['meta']['geo_point'] = None
pass
try:
user_id = doc_ret['form']['meta']['userID']
except KeyError:
user_id = None
doc_ret['user_type'] = get_user_type(user_id)
doc_ret['inserted_at'] = datetime.datetime.utcnow().isoformat()
try:
case_blocks = extract_case_blocks(doc_ret)
except PhoneDateValueError:
pass
else:
for case_dict in case_blocks:
for date_modified_key in ['date_modified', '@date_modified']:
if not is_valid_date(case_dict.get(date_modified_key, None)):
if case_dict.get(date_modified_key) == '':
case_dict[date_modified_key] = None
else:
case_dict.pop(date_modified_key, None)
# convert all mapped dict properties to nulls if they are empty strings
for object_key in ['index', 'attachment', 'create', 'update']:
if object_key in case_dict and not isinstance(case_dict[object_key], dict):
case_dict[object_key] = None
try:
doc_ret["__retrieved_case_ids"] = list(set(case_update_from_block(cb).id for cb in case_blocks))
except CaseGenerationException:
doc_ret["__retrieved_case_ids"] = []
if 'backend_id' not in doc_ret:
doc_ret['backend_id'] = 'couch'
return doc_ret
def transform_xform_for_elasticsearch(doc_dict):
"""
Given an XFormInstance, return a copy that is ready to be sent to elasticsearch,
or None, if the form should not be saved to elasticsearch
"""
doc_ret = copy.deepcopy(doc_dict)
if 'meta' in doc_ret['form']:
if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)):
doc_ret['form']['meta']['timeEnd'] = None
if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)):
doc_ret['form']['meta']['timeStart'] = None
# Some docs have their @xmlns and #text here
if isinstance(doc_ret['form']['meta'].get('appVersion'), dict):
doc_ret['form']['meta'] = format_form_meta_for_es(
doc_ret['form']['meta'])
app_version_info = get_app_version_info(
doc_ret['domain'],
doc_ret.get('build_id'),
doc_ret.get('version'),
doc_ret['form']['meta'],
)
doc_ret['form']['meta'][
'commcare_version'] = app_version_info.commcare_version
doc_ret['form']['meta'][
'app_build_version'] = app_version_info.build_version
try:
geo_point = GeoPointProperty().wrap(
doc_ret['form']['meta']['location'])
doc_ret['form']['meta']['geo_point'] = geo_point.lat_lon
except (KeyError, BadValueError):
doc_ret['form']['meta']['geo_point'] = None
pass
try:
user_id = doc_ret['form']['meta']['userID']
except KeyError:
user_id = None
doc_ret['user_type'] = get_user_type(user_id)
doc_ret['inserted_at'] = datetime.datetime.utcnow().isoformat()
try:
case_blocks = extract_case_blocks(doc_ret)
except PhoneDateValueError:
pass
else:
for case_dict in case_blocks:
for date_modified_key in ['date_modified', '@date_modified']:
if not is_valid_date(case_dict.get(date_modified_key, None)):
if case_dict.get(date_modified_key) == '':
case_dict[date_modified_key] = None
else:
case_dict.pop(date_modified_key, None)
# convert all mapped dict properties to nulls if they are empty strings
for object_key in ['index', 'attachment', 'create', 'update']:
if object_key in case_dict and not isinstance(
case_dict[object_key], dict):
case_dict[object_key] = None
try:
doc_ret["__retrieved_case_ids"] = list(
set(case_update_from_block(cb).id for cb in case_blocks))
except CaseGenerationException:
doc_ret["__retrieved_case_ids"] = []
if 'backend_id' not in doc_ret:
doc_ret['backend_id'] = 'couch'
return doc_ret
class Command(BaseCommand):
help = "Delete all cases that are in a specific case's network/footprint"
def add_arguments(self, parser):
parser.add_argument('domain', type=unicode)
parser.add_argument('case_id', type=unicode)
parser.add_argument('--filename',
dest='filename',
default='case-delete-info.csv')
def handle(self, domain, case_id, **options):
case_accessor = CaseAccessors(domain=domain)
case = case_accessor.get_case(case_id)
if not case.is_deleted and raw_input('\n'.join([
'Case {} is not already deleted. Are you sure you want to delete it? (y/N)'
.format(case_id)
])).lower() != 'y':
sys.exit(0)
dependent_case_ids = get_entire_case_network(domain, [case_id])
cases_to_delete = filter(lambda case: not case.is_deleted,
case_accessor.get_cases(dependent_case_ids))
if cases_to_delete:
with open(options['filename'], 'w') as csvfile:
writer = csv.writer(csvfile)
headers = [
'case id',
'case type',
'owner',
'opened by',
'app version',
]
writer.writerow(headers)
print(headers)
for case in cases_to_delete:
form = FormAccessors(domain=domain).get_form(
case.xform_ids[0])
app_version_info = get_app_version_info(
domain,
form.build_id,
form.form_data['@version'],
form.metadata,
)
row = [
case.case_id,
case.type,
cached_owner_id_to_display(case.owner_id)
or case.owner_id,
cached_owner_id_to_display(case.opened_by),
app_version_info.build_version,
]
writer.writerow(row)
print(row)
if cases_to_delete and raw_input('\n'.join([
'Delete these {} cases? (y/N)'.format(len(cases_to_delete)),
])).lower() == 'y':
case_accessor.soft_delete_cases(
[c.case_id for c in cases_to_delete])
print('deleted {} cases'.format(len(cases_to_delete)))
if cases_to_delete:
print('details here: {}'.format(options['filename']))
else:
print("didn't find any cases to delete")
headers = [
'case id',
'case type',
'owner',
'opened by',
'app version',
]
writer.writerow(headers)
print(headers)
for case in cases_to_delete:
form = FormAccessors(domain=domain).get_form(
case.xform_ids[0])
app_version_info = get_app_version_info(
domain,
form.build_id,
form.form_data['@version'],
form.metadata,
)
row = [
case.case_id,
case.type,
cached_owner_id_to_display(case.owner_id)
or case.owner_id,
cached_owner_id_to_display(case.opened_by),
app_version_info.build_version,
]
writer.writerow(row)
print(row)
if cases_to_delete and input('\n'.join([
'Delete these {} cases? (y/N)'.format(len(cases_to_delete)),
def handle(self, **options):
domain = options['domain']
debug = options['debug']
cleanup = options['cleanup']
domain_query = CaseES().domain(domain)
valid_case_ids = set(domain_query.get_ids())
referenced_case_ids = {
index['referenced_id']
for hit in domain_query.source('indices.referenced_id').run().hits
for index in hit['indices']
}
invalid_referenced_ids = referenced_case_ids - valid_case_ids
if len(invalid_referenced_ids) > ES_MAX_CLAUSE_COUNT:
print("there's a lot of invalid ids here. ES queries may not handle this well")
cases_with_invalid_references = (
domain_query
.term('indices.referenced_id', invalid_referenced_ids)
.source(['_id', 'type', 'indices', 'owner_id', 'opened_by', 'xform_ids'])
.run().hits
)
with open(options['filename'], 'w', encoding='utf-8') as csvfile:
writer = csv.writer(csvfile)
headers = [
'case id',
'case type',
'creating form id',
'referenced id',
'referenced_type',
'index relationship',
'index identifier',
'owner id',
'owner name',
'opened by id',
'opened by name',
]
if debug:
headers.append('app version')
writer.writerow(headers)
for case in cases_with_invalid_references:
for index in case['indices']:
if index['referenced_id'] in invalid_referenced_ids:
form_id = case['xform_ids'][0]
row = [
case['_id'],
case['type'],
form_id,
index['referenced_id'],
index['referenced_type'],
index['relationship'],
index['identifier'],
case['owner_id'],
cached_owner_id_to_display(case['owner_id']),
case['opened_by'],
cached_owner_id_to_display(case['opened_by']),
]
if debug:
form = FormAccessors(domain=domain).get_form(form_id)
app_version_info = get_app_version_info(
domain,
form.build_id,
form.form_data['@version'],
form.metadata,
)
row.append(app_version_info.build_version)
writer.writerow(row)
if cleanup:
missing = set()
deleted = set()
exists = set()
for invalid_id in invalid_referenced_ids:
try:
case = CaseAccessors(domain).get_case(invalid_id)
except CaseNotFound:
missing.add(invalid_id)
else:
if case.is_deleted:
deleted.add(case)
else:
exists.add(case)
for case_to_resync in exists:
# if the case actually exists resync it to fix the es search
resave_case(domain, case_to_resync, send_post_save_signal=False)
if exists:
print('resynced {} cases that were actually not deleted'.format(len(exists)))
for case in deleted:
# delete the deleted case's entire network in one go
call_command('delete_related_cases', domain, case.case_id)
for case in cases_with_invalid_references:
for index in case['indices']:
if index['referenced_id'] in missing:
# this is just an invalid reference. no recourse but to delete the case itself
call_command('delete_related_cases', domain, case['_id'])
