def _delete_all_forms(domain_name): logger.info('Deleting forms...') form_accessor = FormAccessors(domain_name) form_ids = form_accessor.get_all_form_ids_in_domain() for form_id_chunk in chunked( with_progress_bar(form_ids, stream=silence_during_tests()), 500): form_accessor.soft_delete_forms(list(form_id_chunk)) logger.info('Deleting forms complete.')
def _delete_all_forms(domain_name): logger.info('Deleting forms...') form_accessor = FormAccessors(domain_name) form_ids = list(itertools.chain(*[ form_accessor.get_all_form_ids_in_domain(doc_type=doc_type) for doc_type in doc_type_to_state ])) for form_id_chunk in chunked(with_progress_bar(form_ids, stream=silence_during_tests()), 500): form_accessor.soft_delete_forms(list(form_id_chunk)) logger.info('Deleting forms complete.')
def _delete_all_forms(domain_name): logger.info('Deleting forms...') form_accessor = FormAccessors(domain_name) form_ids = list(itertools.chain(*[ form_accessor.get_all_form_ids_in_domain(doc_type=doc_type) for doc_type in doc_type_to_state ])) for form_id_chunk in chunked(with_progress_bar(form_ids, stream=silence_during_tests()), 500): form_accessor.soft_delete_forms(list(form_id_chunk)) logger.info('Deleting forms complete.')
def test_soft_delete(self): meta = TestFormMetadata(domain=DOMAIN) get_simple_wrapped_form('f1', metadata=meta) f2 = get_simple_wrapped_form('f2', metadata=meta) f2.archive() get_simple_wrapped_form('f3', metadata=meta) accessors = FormAccessors(DOMAIN) # delete num = accessors.soft_delete_forms(['f1', 'f2'], deletion_id='123') self.assertEqual(num, 2) for form_id in ['f1', 'f2']: form = accessors.get_form(form_id) self.assertTrue(form.is_deleted) self.assertEqual(form.deletion_id, '123') form = accessors.get_form('f3') self.assertFalse(form.is_deleted) for form_id in ['f1', 'f2']: form = FormAccessors(DOMAIN).get_form(form_id) if isinstance(form, PartitionedModel): form.delete(using=get_db_alias_for_partitioned_doc(form.form_id)) else: form.delete()
def test_soft_delete(self): meta = TestFormMetadata(domain=DOMAIN) get_simple_wrapped_form('f1', metadata=meta) f2 = get_simple_wrapped_form('f2', metadata=meta) f2.archive() get_simple_wrapped_form('f3', metadata=meta) accessors = FormAccessors(DOMAIN) # delete num = accessors.soft_delete_forms(['f1', 'f2'], deletion_id='123') self.assertEqual(num, 2) for form_id in ['f1', 'f2']: form = accessors.get_form(form_id) self.assertTrue(form.is_deleted) self.assertEqual(form.deletion_id, '123') form = accessors.get_form('f3') self.assertFalse(form.is_deleted) for form_id in ['f1', 'f2']: form = FormAccessors(DOMAIN).get_form(form_id) if isinstance(form, DisabledDbMixin): super(DisabledDbMixin, form).delete() else: form.delete()
def delete_exploded_cases(domain, explosion_id, task=None): if task: DownloadBase.set_progress(delete_exploded_case_task, 0, 0) query = (CaseSearchES().domain(domain).case_property_query( "cc_explosion_id", explosion_id)) case_ids = query.values_list('_id', flat=True) if task: DownloadBase.set_progress(delete_exploded_case_task, 0, len(case_ids)) case_accessor = CaseAccessors(domain) form_accessor = FormAccessors(domain) ledger_accessor = LedgerAccessorSQL deleted_form_ids = set() num_deleted_ledger_entries = 0 for id in case_ids: ledger_form_ids = { tx.form_id for tx in ledger_accessor.get_ledger_transactions_for_case(id) } for form_id in ledger_form_ids: ledger_accessor.delete_ledger_transactions_for_form([id], form_id) num_deleted_ledger_entries += ledger_accessor.delete_ledger_values(id) new_form_ids = set( case_accessor.get_case_xform_ids(id)) - deleted_form_ids form_accessor.soft_delete_forms(list(new_form_ids)) deleted_form_ids |= new_form_ids completed = 0 for ids in chunked(case_ids, 100): case_accessor.soft_delete_cases(list(ids)) if task: completed += len(ids) DownloadBase.set_progress(delete_exploded_case_task, completed, len(case_ids)) return { 'messages': [ "Successfully deleted {} cases".format(len(case_ids)), "Successfully deleted {} forms".format(len(deleted_form_ids)), "Successfully deleted {} ledgers".format( num_deleted_ledger_entries), ] }
class Command(BaseCommand): help = """Expects 4 arguments in order : domain app_id version_number test_run ex: ./manage.py purge_forms_and_cases testapp c531daeece0633738c9a3676a13e3d4f 88 yes domain is included with app_id to ensure the user knows what app to delete version_number to delete data accumulated by versions BEFORE this : integer test_run should be yes(case-sensitive) for a test_run and anything otherwise though deletion would be re-confirmed so dont panic """ def add_arguments(self, parser): parser.add_argument('domain') parser.add_argument('app_id') parser.add_argument('version_number', type=int) parser.add_argument('test_run') def __init__(self): super(Command, self).__init__() self.case_ids = set() self.filtered_xform_ids, self.xform_ids = [], [] self.xform_writer, self.case_writer = None, None self.forms_accessor, self.case_accessors = None, None self.domain, self.app_id, self.version_number, self.test_run = None, None, None, None self.version_mapping = dict() def setup(self): self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b')) self.xform_writer.writerow(XFORM_HEADER) self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b')) self.case_writer.writerow(CASE_HEADER) self.forms_accessor = FormAccessors(self.domain) self.case_accessors = CaseAccessors(self.domain) def ensure_prerequisites(self, domain, app_id, version_number, test_run): self.domain = domain self.app_id = app_id self.version_number = version_number self.test_run = test_run == 'yes' _notify_parsed_args(domain, app_id, version_number, test_run) app = Application.get(self.app_id) if app.domain != self.domain: raise CommandError('Domain not same as from app id') self.setup() def handle(self, domain, app_id, version_number, test_run, **options): self.ensure_prerequisites(domain, app_id, version_number, test_run) self.xform_ids = self.forms_accessor.get_all_form_ids_in_domain() self.iterate_forms_and_collect_case_ids() _print_final_debug_info(self.xform_ids, self.filtered_xform_ids, self.case_ids) if self.data_to_delete() and self.delete_permitted(): self.delete_forms_and_cases() print("Process Completed!! Keep copy of files %s, %s" % (XFORM_FILENAME, CASE_FILE_NAME)) else: print('Process Finished w/o Changes..') def iterate_forms_and_collect_case_ids(self): print("Iterating Through %s XForms and Collecting Case Ids" % len(self.xform_ids)) for xform in self.forms_accessor.iter_forms(self.xform_ids): # Get app version by fetching app corresponding to xform build_id since xform.form # does not have updated app version unless form was updated for that version app_version_built_with = self.get_xform_build_version(xform) if app_version_built_with and app_version_built_with < self.version_number: _print_form_details(xform, self.xform_writer, app_version_built_with) self.ensure_valid_xform(xform) self.filtered_xform_ids.append(xform.form_id) self.case_ids = self.case_ids.union(get_case_ids_from_form(xform)) else: print('skipping xform id: %s' % xform.form_id) if self.case_ids: self.print_case_details() def get_xform_build_version(self, xform): version_from_mapping = None if xform.build_id: version_from_mapping = self.version_mapping.get(xform.build_id, None) if not version_from_mapping: try: get_app_version = get_app(self.domain, xform.build_id).version except Http404: get_app_version = None if get_app_version: version_from_mapping = int(get_app_version) self.version_mapping[xform.build_id] = version_from_mapping return version_from_mapping def ensure_valid_xform(self, xform): if xform.app_id != self.app_id and xform.domain != self.domain: _raise_xform_domain_mismatch(xform) def print_case_details(self): for case in self.case_accessors.iter_cases(self.case_ids): _print_case_details(case, self.case_writer) def delete_permitted(self): return not self.test_run and are_you_sure() def data_to_delete(self): return len(self.filtered_xform_ids) != 0 or len(self.case_ids) != 0 def delete_forms_and_cases(self): print('Proceeding with deleting forms and cases') self.forms_accessor.soft_delete_forms(self.filtered_xform_ids) self.case_accessors.soft_delete_cases(list(self.case_ids))
class Command(BaseCommand): help = """Expects 4 arguments in order : domain app_id version_number test_run ex: ./manage.py purge_forms_and_cases testapp c531daeece0633738c9a3676a13e3d4f 88 yes domain is included with app_id to ensure the user knows what app to delete version_number to delete data accumulated by versions BEFORE this : integer test_run should be yes(case-sensitive) for a test_run and anything otherwise though deletion would be re-confirmed so dont panic """ def add_arguments(self, parser): parser.add_argument('domain') parser.add_argument('app_id') parser.add_argument('version_number', type=int) parser.add_argument('test_run') def __init__(self): super(Command, self).__init__() self.case_ids = set() self.filtered_xform_ids, self.xform_ids = [], [] self.xform_writer, self.case_writer = None, None self.forms_accessor, self.case_accessors = None, None self.domain, self.app_id, self.version_number, self.test_run = None, None, None, None self.version_mapping = dict() def setup(self): self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b')) self.xform_writer.writerow(XFORM_HEADER) self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b')) self.case_writer.writerow(CASE_HEADER) self.forms_accessor = FormAccessors(self.domain) self.case_accessors = CaseAccessors(self.domain) def ensure_prerequisites(self, domain, app_id, version_number, test_run): NotAllowed.check(domain) self.domain = domain self.app_id = app_id self.version_number = version_number self.test_run = test_run == 'yes' _notify_parsed_args(domain, app_id, version_number, test_run) app = Application.get(self.app_id) if app.domain != self.domain: raise CommandError('Domain not same as from app id') self.setup() def handle(self, domain, app_id, version_number, test_run, **options): self.ensure_prerequisites(domain, app_id, version_number, test_run) self.xform_ids = self.forms_accessor.get_all_form_ids_in_domain() self.iterate_forms_and_collect_case_ids() _print_final_debug_info(self.xform_ids, self.filtered_xform_ids, self.case_ids) if self.data_to_delete() and self.delete_permitted(): self.delete_forms_and_cases() print("Process Completed!! Keep copy of files %s, %s" % (XFORM_FILENAME, CASE_FILE_NAME)) else: print('Process Finished w/o Changes..') def iterate_forms_and_collect_case_ids(self): print("Iterating Through %s XForms and Collecting Case Ids" % len(self.xform_ids)) for xform in self.forms_accessor.iter_forms(self.xform_ids): # Get app version by fetching app corresponding to xform build_id since xform.form # does not have updated app version unless form was updated for that version app_version_built_with = self.get_xform_build_version(xform) if app_version_built_with and app_version_built_with < self.version_number: _print_form_details(xform, self.xform_writer, app_version_built_with) self.ensure_valid_xform(xform) self.filtered_xform_ids.append(xform.form_id) self.case_ids = self.case_ids.union( get_case_ids_from_form(xform)) else: print('skipping xform id: %s' % xform.form_id) if self.case_ids: self.print_case_details() def get_xform_build_version(self, xform): version_from_mapping = None if xform.build_id: version_from_mapping = self.version_mapping.get( xform.build_id, None) if not version_from_mapping: try: get_app_version = get_app(self.domain, xform.build_id).version except Http404: get_app_version = None if get_app_version: version_from_mapping = int(get_app_version) self.version_mapping[xform.build_id] = version_from_mapping return version_from_mapping def ensure_valid_xform(self, xform): if xform.app_id != self.app_id and xform.domain != self.domain: _raise_xform_domain_mismatch(xform) def print_case_details(self): for case in self.case_accessors.iter_cases(self.case_ids): _print_case_details(case, self.case_writer) def delete_permitted(self): return not self.test_run and are_you_sure() def data_to_delete(self): return len(self.filtered_xform_ids) != 0 or len(self.case_ids) != 0 def delete_forms_and_cases(self): print('Proceeding with deleting forms and cases') self.forms_accessor.soft_delete_forms(self.filtered_xform_ids) self.case_accessors.soft_delete_cases(list(self.case_ids))