def _delete_all_forms(domain_name):
logger.info('Deleting forms...')
form_accessor = FormAccessors(domain_name)
form_ids = form_accessor.get_all_form_ids_in_domain()
for form_id_chunk in chunked(
with_progress_bar(form_ids, stream=silence_during_tests()), 500):
form_accessor.soft_delete_forms(list(form_id_chunk))
logger.info('Deleting forms complete.')
def _delete_all_forms(domain_name):
logger.info('Deleting forms...')
form_accessor = FormAccessors(domain_name)
form_ids = list(itertools.chain(*[
form_accessor.get_all_form_ids_in_domain(doc_type=doc_type)
for doc_type in doc_type_to_state
]))
for form_id_chunk in chunked(with_progress_bar(form_ids, stream=silence_during_tests()), 500):
form_accessor.soft_delete_forms(list(form_id_chunk))
logger.info('Deleting forms complete.')
def _delete_all_forms(domain_name):
logger.info('Deleting forms...')
form_accessor = FormAccessors(domain_name)
form_ids = list(itertools.chain(*[
form_accessor.get_all_form_ids_in_domain(doc_type=doc_type)
for doc_type in doc_type_to_state
]))
for form_id_chunk in chunked(with_progress_bar(form_ids, stream=silence_during_tests()), 500):
form_accessor.soft_delete_forms(list(form_id_chunk))
logger.info('Deleting forms complete.')
def test_soft_delete(self):
meta = TestFormMetadata(domain=DOMAIN)
get_simple_wrapped_form('f1', metadata=meta)
f2 = get_simple_wrapped_form('f2', metadata=meta)
f2.archive()
get_simple_wrapped_form('f3', metadata=meta)
accessors = FormAccessors(DOMAIN)
# delete
num = accessors.soft_delete_forms(['f1', 'f2'], deletion_id='123')
self.assertEqual(num, 2)
for form_id in ['f1', 'f2']:
form = accessors.get_form(form_id)
self.assertTrue(form.is_deleted)
self.assertEqual(form.deletion_id, '123')
form = accessors.get_form('f3')
self.assertFalse(form.is_deleted)
for form_id in ['f1', 'f2']:
form = FormAccessors(DOMAIN).get_form(form_id)
if isinstance(form, PartitionedModel):
form.delete(using=get_db_alias_for_partitioned_doc(form.form_id))
else:
form.delete()
def test_soft_delete(self):
meta = TestFormMetadata(domain=DOMAIN)
get_simple_wrapped_form('f1', metadata=meta)
f2 = get_simple_wrapped_form('f2', metadata=meta)
f2.archive()
get_simple_wrapped_form('f3', metadata=meta)
accessors = FormAccessors(DOMAIN)
# delete
num = accessors.soft_delete_forms(['f1', 'f2'], deletion_id='123')
self.assertEqual(num, 2)
for form_id in ['f1', 'f2']:
form = accessors.get_form(form_id)
self.assertTrue(form.is_deleted)
self.assertEqual(form.deletion_id, '123')
form = accessors.get_form('f3')
self.assertFalse(form.is_deleted)
for form_id in ['f1', 'f2']:
form = FormAccessors(DOMAIN).get_form(form_id)
if isinstance(form, DisabledDbMixin):
super(DisabledDbMixin, form).delete()
else:
form.delete()
def delete_exploded_cases(domain, explosion_id, task=None):
if task:
DownloadBase.set_progress(delete_exploded_case_task, 0, 0)
query = (CaseSearchES().domain(domain).case_property_query(
"cc_explosion_id", explosion_id))
case_ids = query.values_list('_id', flat=True)
if task:
DownloadBase.set_progress(delete_exploded_case_task, 0, len(case_ids))
case_accessor = CaseAccessors(domain)
form_accessor = FormAccessors(domain)
ledger_accessor = LedgerAccessorSQL
deleted_form_ids = set()
num_deleted_ledger_entries = 0
for id in case_ids:
ledger_form_ids = {
tx.form_id
for tx in ledger_accessor.get_ledger_transactions_for_case(id)
}
for form_id in ledger_form_ids:
ledger_accessor.delete_ledger_transactions_for_form([id], form_id)
num_deleted_ledger_entries += ledger_accessor.delete_ledger_values(id)
new_form_ids = set(
case_accessor.get_case_xform_ids(id)) - deleted_form_ids
form_accessor.soft_delete_forms(list(new_form_ids))
deleted_form_ids |= new_form_ids
completed = 0
for ids in chunked(case_ids, 100):
case_accessor.soft_delete_cases(list(ids))
if task:
completed += len(ids)
DownloadBase.set_progress(delete_exploded_case_task, completed,
len(case_ids))
return {
'messages': [
"Successfully deleted {} cases".format(len(case_ids)),
"Successfully deleted {} forms".format(len(deleted_form_ids)),
"Successfully deleted {} ledgers".format(
num_deleted_ledger_entries),
]
}
class Command(BaseCommand):
help = """Expects 4 arguments in order : domain app_id version_number test_run
ex: ./manage.py purge_forms_and_cases testapp c531daeece0633738c9a3676a13e3d4f 88 yes
domain is included with app_id to ensure the user knows what app to delete
version_number to delete data accumulated by versions BEFORE this : integer
test_run should be yes(case-sensitive) for a test_run and anything otherwise
though deletion would be re-confirmed so dont panic
"""
def add_arguments(self, parser):
parser.add_argument('domain')
parser.add_argument('app_id')
parser.add_argument('version_number', type=int)
parser.add_argument('test_run')
def __init__(self):
super(Command, self).__init__()
self.case_ids = set()
self.filtered_xform_ids, self.xform_ids = [], []
self.xform_writer, self.case_writer = None, None
self.forms_accessor, self.case_accessors = None, None
self.domain, self.app_id, self.version_number, self.test_run = None, None, None, None
self.version_mapping = dict()
def setup(self):
self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b'))
self.xform_writer.writerow(XFORM_HEADER)
self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b'))
self.case_writer.writerow(CASE_HEADER)
self.forms_accessor = FormAccessors(self.domain)
self.case_accessors = CaseAccessors(self.domain)
def ensure_prerequisites(self, domain, app_id, version_number, test_run):
self.domain = domain
self.app_id = app_id
self.version_number = version_number
self.test_run = test_run == 'yes'
_notify_parsed_args(domain, app_id, version_number, test_run)
app = Application.get(self.app_id)
if app.domain != self.domain:
raise CommandError('Domain not same as from app id')
self.setup()
def handle(self, domain, app_id, version_number, test_run, **options):
self.ensure_prerequisites(domain, app_id, version_number, test_run)
self.xform_ids = self.forms_accessor.get_all_form_ids_in_domain()
self.iterate_forms_and_collect_case_ids()
_print_final_debug_info(self.xform_ids, self.filtered_xform_ids, self.case_ids)
if self.data_to_delete() and self.delete_permitted():
self.delete_forms_and_cases()
print("Process Completed!! Keep copy of files %s, %s" % (XFORM_FILENAME, CASE_FILE_NAME))
else:
print('Process Finished w/o Changes..')
def iterate_forms_and_collect_case_ids(self):
print("Iterating Through %s XForms and Collecting Case Ids" % len(self.xform_ids))
for xform in self.forms_accessor.iter_forms(self.xform_ids):
# Get app version by fetching app corresponding to xform build_id since xform.form
# does not have updated app version unless form was updated for that version
app_version_built_with = self.get_xform_build_version(xform)
if app_version_built_with and app_version_built_with < self.version_number:
_print_form_details(xform, self.xform_writer, app_version_built_with)
self.ensure_valid_xform(xform)
self.filtered_xform_ids.append(xform.form_id)
self.case_ids = self.case_ids.union(get_case_ids_from_form(xform))
else:
print('skipping xform id: %s' % xform.form_id)
if self.case_ids:
self.print_case_details()
def get_xform_build_version(self, xform):
version_from_mapping = None
if xform.build_id:
version_from_mapping = self.version_mapping.get(xform.build_id, None)
if not version_from_mapping:
try:
get_app_version = get_app(self.domain, xform.build_id).version
except Http404:
get_app_version = None
if get_app_version:
version_from_mapping = int(get_app_version)
self.version_mapping[xform.build_id] = version_from_mapping
return version_from_mapping
def ensure_valid_xform(self, xform):
if xform.app_id != self.app_id and xform.domain != self.domain:
_raise_xform_domain_mismatch(xform)
def print_case_details(self):
for case in self.case_accessors.iter_cases(self.case_ids):
_print_case_details(case, self.case_writer)
def delete_permitted(self):
return not self.test_run and are_you_sure()
def data_to_delete(self):
return len(self.filtered_xform_ids) != 0 or len(self.case_ids) != 0
def delete_forms_and_cases(self):
print('Proceeding with deleting forms and cases')
self.forms_accessor.soft_delete_forms(self.filtered_xform_ids)
self.case_accessors.soft_delete_cases(list(self.case_ids))
class Command(BaseCommand):
help = """Expects 4 arguments in order : domain app_id version_number test_run
ex: ./manage.py purge_forms_and_cases testapp c531daeece0633738c9a3676a13e3d4f 88 yes
domain is included with app_id to ensure the user knows what app to delete
version_number to delete data accumulated by versions BEFORE this : integer
test_run should be yes(case-sensitive) for a test_run and anything otherwise
though deletion would be re-confirmed so dont panic
"""
def add_arguments(self, parser):
parser.add_argument('domain')
parser.add_argument('app_id')
parser.add_argument('version_number', type=int)
parser.add_argument('test_run')
def __init__(self):
super(Command, self).__init__()
self.case_ids = set()
self.filtered_xform_ids, self.xform_ids = [], []
self.xform_writer, self.case_writer = None, None
self.forms_accessor, self.case_accessors = None, None
self.domain, self.app_id, self.version_number, self.test_run = None, None, None, None
self.version_mapping = dict()
def setup(self):
self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b'))
self.xform_writer.writerow(XFORM_HEADER)
self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b'))
self.case_writer.writerow(CASE_HEADER)
self.forms_accessor = FormAccessors(self.domain)
self.case_accessors = CaseAccessors(self.domain)
def ensure_prerequisites(self, domain, app_id, version_number, test_run):
NotAllowed.check(domain)
self.domain = domain
self.app_id = app_id
self.version_number = version_number
self.test_run = test_run == 'yes'
_notify_parsed_args(domain, app_id, version_number, test_run)
app = Application.get(self.app_id)
if app.domain != self.domain:
raise CommandError('Domain not same as from app id')
self.setup()
def handle(self, domain, app_id, version_number, test_run, **options):
self.ensure_prerequisites(domain, app_id, version_number, test_run)
self.xform_ids = self.forms_accessor.get_all_form_ids_in_domain()
self.iterate_forms_and_collect_case_ids()
_print_final_debug_info(self.xform_ids, self.filtered_xform_ids,
self.case_ids)
if self.data_to_delete() and self.delete_permitted():
self.delete_forms_and_cases()
print("Process Completed!! Keep copy of files %s, %s" %
(XFORM_FILENAME, CASE_FILE_NAME))
else:
print('Process Finished w/o Changes..')
def iterate_forms_and_collect_case_ids(self):
print("Iterating Through %s XForms and Collecting Case Ids" %
len(self.xform_ids))
for xform in self.forms_accessor.iter_forms(self.xform_ids):
# Get app version by fetching app corresponding to xform build_id since xform.form
# does not have updated app version unless form was updated for that version
app_version_built_with = self.get_xform_build_version(xform)
if app_version_built_with and app_version_built_with < self.version_number:
_print_form_details(xform, self.xform_writer,
app_version_built_with)
self.ensure_valid_xform(xform)
self.filtered_xform_ids.append(xform.form_id)
self.case_ids = self.case_ids.union(
get_case_ids_from_form(xform))
else:
print('skipping xform id: %s' % xform.form_id)
if self.case_ids:
self.print_case_details()
def get_xform_build_version(self, xform):
version_from_mapping = None
if xform.build_id:
version_from_mapping = self.version_mapping.get(
xform.build_id, None)
if not version_from_mapping:
try:
get_app_version = get_app(self.domain,
xform.build_id).version
except Http404:
get_app_version = None
if get_app_version:
version_from_mapping = int(get_app_version)
self.version_mapping[xform.build_id] = version_from_mapping
return version_from_mapping
def ensure_valid_xform(self, xform):
if xform.app_id != self.app_id and xform.domain != self.domain:
_raise_xform_domain_mismatch(xform)
def print_case_details(self):
for case in self.case_accessors.iter_cases(self.case_ids):
_print_case_details(case, self.case_writer)
def delete_permitted(self):
return not self.test_run and are_you_sure()
def data_to_delete(self):
return len(self.filtered_xform_ids) != 0 or len(self.case_ids) != 0
def delete_forms_and_cases(self):
print('Proceeding with deleting forms and cases')
self.forms_accessor.soft_delete_forms(self.filtered_xform_ids)
self.case_accessors.soft_delete_cases(list(self.case_ids))
